u280 timing update.
This commit is contained in:
parent
334b4ce9d7
commit
ec4c4f5b0b
|
@ -1,4 +1,5 @@
|
|||
obj-m := fpga_drv.o
|
||||
obj-m := coyote_drv.o
|
||||
coyote_drv-objs:= fpga_drv.o fpga_isr.o fpga_fops.o fpga_dev.o fpga_mmu.o pci/pci_dev.o eci/eci_dev.o
|
||||
|
||||
KERNELDIR ?= /lib/modules/$(shell uname -r)/build
|
||||
PWD := $(shell pwd)
|
||||
|
@ -10,4 +11,4 @@ all:
|
|||
$(MAKE) -C $(KERNELDIR) M=$(PWD) modules
|
||||
|
||||
clean:
|
||||
rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.c .tmp_versions *.order *.symvers *.dwo *.mod *.mod.dwo
|
||||
rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.c .tmp_versions *.order *.symvers *.mod *mod.dwo *.dwo .??* pci/.??* pci/*.o eci/.??* eci/*.o
|
||||
|
|
|
@ -0,0 +1,715 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __COYOTE_DEV_H__
|
||||
#define __COYOTE_DEV_H__
|
||||
|
||||
#include <asm/io.h>
|
||||
#include <linux/clk.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/hrtimer.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/irq.h>
|
||||
#include <linux/irqchip/chained_irq.h>
|
||||
#include <linux/irqdomain.h>
|
||||
#include <linux/ioport.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/miscdevice.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/cdev.h>
|
||||
#include <linux/pagemap.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/version.h>
|
||||
#include <linux/ioctl.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/msi.h>
|
||||
#include <linux/poll.h>
|
||||
#include <linux/sched.h>
|
||||
#include <asm/delay.h>
|
||||
#include <asm/set_memory.h>
|
||||
#include <linux/hashtable.h>
|
||||
#include <linux/moduleparam.h>
|
||||
#include <linux/stat.h>
|
||||
|
||||
/**
|
||||
* @brief Args
|
||||
*
|
||||
*/
|
||||
|
||||
#define CYT_ARCH_PCI 0
|
||||
#define CYT_ARCH_ECI 1
|
||||
|
||||
static int cyt_arch = CYT_ARCH_PCI;
|
||||
|
||||
module_param(cyt_arch, int, S_IRUSR);
|
||||
MODULE_PARM_DESC(cyt_arch, "target architecture");
|
||||
|
||||
/**
|
||||
* @brief Info
|
||||
*
|
||||
*/
|
||||
|
||||
/* Driver info */
|
||||
#define DRV_NAME "coyote_driver"
|
||||
#define DEV_NAME "fpga"
|
||||
|
||||
/**
|
||||
* @brief Util
|
||||
*
|
||||
*/
|
||||
|
||||
/* Debug print */
|
||||
#define COYOTE_DEBUG 1
|
||||
|
||||
#if (COYOTE_DEBUG == 0)
|
||||
#define dbg_info(...)
|
||||
#else
|
||||
#define dbg_info(fmt, ...) pr_info("%s():" fmt, \
|
||||
__func__, ##__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
|
||||
#define HIGH_32(addr) ((addr >> 16) >> 16)
|
||||
/* Obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
|
||||
#define LOW_32(addr) (addr & 0xffffffffUL)
|
||||
|
||||
/**
|
||||
* @brief Bus
|
||||
*
|
||||
*/
|
||||
|
||||
/* XDMA info */
|
||||
#define MAX_NUM_BARS 3
|
||||
#define MAX_NUM_CHANNELS 4
|
||||
#define MAX_NUM_ENGINES (MAX_NUM_CHANNELS * 2)
|
||||
#define MAX_USER_IRQS 16
|
||||
#define C2H_CHAN_OFFS 0x1000
|
||||
#define H2C_CHAN_OFFS 0x0000
|
||||
#define CHAN_RANGE 0x100
|
||||
#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
|
||||
|
||||
/* Engine IDs */
|
||||
#define XDMA_ID_H2C 0x1fc0U
|
||||
#define XDMA_ID_C2H 0x1fc1U
|
||||
|
||||
/* Engine regs */
|
||||
#define XDMA_ENG_IRQ_NUM (1)
|
||||
#define XDMA_OFS_INT_CTRL (0x2000UL)
|
||||
#define XDMA_OFS_CONFIG (0x3000UL)
|
||||
|
||||
/* Bits of the SG DMA control register */
|
||||
#define XDMA_CTRL_RUN_STOP (1UL << 0)
|
||||
#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
|
||||
#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
|
||||
#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
|
||||
#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
|
||||
#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
|
||||
#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
|
||||
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
|
||||
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
|
||||
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
|
||||
|
||||
/* Bits of the SG DMA status register */
|
||||
#define XDMA_STAT_BUSY (1UL << 0)
|
||||
#define XDMA_STAT_DESC_STOPPED (1UL << 1)
|
||||
#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
|
||||
#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
|
||||
#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
|
||||
#define XDMA_STAT_FETCH_STOPPED (1UL << 5)
|
||||
#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
|
||||
#define XDMA_STAT_READ_ERROR (0x1FUL << 9)
|
||||
#define XDMA_STAT_DESC_ERROR (0x1FUL << 19)
|
||||
|
||||
/* Bits of the performance control register */
|
||||
#define XDMA_PERF_RUN (1UL << 0)
|
||||
#define XDMA_PERF_CLEAR (1UL << 1)
|
||||
#define XDMA_PERF_AUTO (1UL << 2)
|
||||
|
||||
/* Polling */
|
||||
#define WB_COUNT_MASK 0x00ffffffUL
|
||||
#define WB_ERR_MASK (1UL << 31)
|
||||
#define POLL_TIMEOUT_SECONDS 10
|
||||
#define NUM_POLLS_PER_SCHED 100
|
||||
|
||||
/* Network */
|
||||
#define BASE_IP_ADDR_0 0x0A01D497
|
||||
#define BASE_IP_ADDR_1 0x0A01D497
|
||||
#define NODE_ID 0
|
||||
#define N_TOTAL_NODES 2
|
||||
|
||||
/* Physical address (ECI) */
|
||||
#define IO_PHYS_ADDR 0x900000000000UL
|
||||
|
||||
/**
|
||||
* @brief Static layer
|
||||
*
|
||||
*/
|
||||
|
||||
#define BAR_XDMA_CONFIG 0
|
||||
#define BAR_FPGA_CONFIG 1
|
||||
|
||||
/* FPGA static config */
|
||||
#define FPGA_STAT_CNFG_OFFS 0x0
|
||||
#define FPGA_STAT_CNFG_SIZE 32 * 1024
|
||||
|
||||
#define EN_AVX_MASK 0x1
|
||||
#define EN_AVX_SHFT 0x0
|
||||
#define EN_BPSS_MASK 0x2
|
||||
#define EN_BPSS_SHFT 0x1
|
||||
#define EN_TLBF_MASK 0x4
|
||||
#define EN_TLBF_SHFT 0x2
|
||||
#define EN_WB_MASK 0x8
|
||||
#define EN_WB_SHFT 0x3
|
||||
#define TLB_S_ORDER_MASK 0xf0
|
||||
#define TLB_S_ORDER_SHFT 0x4
|
||||
#define TLB_S_ASSOC_MASK 0xf00
|
||||
#define TLB_S_ASSOC_SHFT 0x8
|
||||
#define TLB_L_ORDER_MASK 0xf000
|
||||
#define TLB_L_ORDER_SHFT 0xc
|
||||
#define TLB_L_ASSOC_MASK 0xf0000
|
||||
#define TLB_L_ASSOC_SHFT 0x10
|
||||
#define TLB_S_PG_SHFT_MASK 0x3f00000
|
||||
#define TLB_S_PG_SHFT_SHFT 0x14
|
||||
#define TLB_L_PG_SHFT_MASK 0xfc000000
|
||||
#define TLB_L_PG_SHFT_SHFT 0x1a
|
||||
|
||||
#define EN_DDR_MASK 0x1
|
||||
#define EN_DDR_SHFT 0x0
|
||||
#define EN_HBM_MASK 0x2
|
||||
#define EN_HBM_SHFT 0x1
|
||||
#define N_MEM_CHAN_MASK 0x7d
|
||||
#define N_MEM_CHAN_SHFT 0x2
|
||||
#define EN_RDMA_0_MASK 0x1
|
||||
#define EN_RDMA_1_MASK 0x2
|
||||
#define EN_PR_MASK 0x1
|
||||
#define EN_PR_SHFT 0x0
|
||||
#define EN_RDMA_0_MASK 0x1
|
||||
#define EN_RDMA_0_SHFT 0x0
|
||||
#define EN_RDMA_1_MASK 0x2
|
||||
#define EN_RDMA_1_SHFT 0x1
|
||||
#define EN_TCP_0_MASK 0x1
|
||||
#define EN_TCP_0_SHFT 0x0
|
||||
#define EN_TCP_1_MASK 0x2
|
||||
#define EN_TCP_1_SHFT 0x1
|
||||
|
||||
/**
|
||||
* @brief Dynamic layer
|
||||
*
|
||||
*/
|
||||
|
||||
/* FPGA control regions */
|
||||
#define FPGA_CTRL_SIZE 256 * 1024
|
||||
#define FPGA_CTRL_OFFS 0x100000
|
||||
#define FPGA_CTRL_LTLB_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_LTLB_OFFS 0x0
|
||||
#define FPGA_CTRL_STLB_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_STLB_OFFS 0x10000
|
||||
#define FPGA_CTRL_USER_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_USER_OFFS 0x20000
|
||||
#define FPGA_CTRL_CNFG_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_CNFG_OFFS 0x30000
|
||||
|
||||
#define FPGA_CTRL_CNFG_AVX_SIZE 256 * 1024
|
||||
#define FPGA_CTRL_CNFG_AVX_OFFS 0x1000000
|
||||
|
||||
/* FPGA dynamic control config */
|
||||
#define FPGA_CNFG_CTRL_IRQ_RESTART 0x100
|
||||
|
||||
/* Maximum transfer size */
|
||||
#define TRANSFER_MAX_BYTES (8 * 1024 * 1024)
|
||||
|
||||
/* TLB */
|
||||
#define TLB_VADDR_RANGE 48
|
||||
#define TLB_PADDR_RANGE 40
|
||||
#define PID_SIZE 6
|
||||
#define MAX_MAP_AXIL_PAGES 64
|
||||
|
||||
#define LTLB_PAGE_BITS 21
|
||||
#define STLB_PAGE_BITS 12
|
||||
#define LTLB_PADDR_SIZE (TLB_PADDR_RANGE - LTLB_PAGE_BITS)
|
||||
#define STLB_PADDR_SIZE (TLB_PADDR_RANGE - STLB_PAGE_BITS)
|
||||
|
||||
#define TLBF_CTRL_START 0x7
|
||||
#define TLBF_CTRL_ID_MASK 0xff
|
||||
#define TLBF_CTRL_ID_SHFT 0x3
|
||||
#define TLBF_STAT_DONE 0x1
|
||||
|
||||
/* Memory allocation */
|
||||
#define MAX_BUFF_NUM 64 // maximum number of huge pages allowed
|
||||
#define MAX_PR_BUFF_NUM 64 // maximum number of huge pages allowed
|
||||
#define MAX_N_MAP_PAGES 128 // TODO: link to params, max no pfault 1M
|
||||
#define MAX_N_MAP_HUGE_PAGES (16 * 512) // max no pfault 32M
|
||||
|
||||
/* Max card pages */
|
||||
#define N_LARGE_CHUNKS 512
|
||||
#define N_SMALL_CHUNKS (256 * 1024)
|
||||
#define MEM_SEP (N_SMALL_CHUNKS * (4 * 1024))
|
||||
#define MAX_N_REGIONS 16
|
||||
#define SMALL_CHUNK_ALLOC 0
|
||||
#define LARGE_CHUNK_ALLOC 1
|
||||
|
||||
/* PR */
|
||||
#define PR_CTRL_START_MIDDLE 0x3
|
||||
#define PR_CTRL_START_LAST 0x7
|
||||
#define PR_STAT_DONE 0x1
|
||||
|
||||
/**
|
||||
* @brief Cdev
|
||||
*
|
||||
*/
|
||||
|
||||
/* Major number */
|
||||
#define FPGA_MAJOR 0 // dynamic
|
||||
|
||||
/* MMAP */
|
||||
#define MMAP_CTRL 0x0
|
||||
#define MMAP_CNFG 0x1
|
||||
#define MMAP_CNFG_AVX 0x2
|
||||
#define MMAP_WB 0x3
|
||||
#define MMAP_BUFF 0x200
|
||||
#define MMAP_PR 0x400
|
||||
|
||||
/* IOCTL */
|
||||
#define IOCTL_ALLOC_HOST_USER_MEM _IOW('D', 1, unsigned long) // large pages (no hugepage support)
|
||||
#define IOCTL_FREE_HOST_USER_MEM _IOW('D', 2, unsigned long)
|
||||
#define IOCTL_ALLOC_HOST_PR_MEM _IOW('D', 3, unsigned long) // pr pages
|
||||
#define IOCTL_FREE_HOST_PR_MEM _IOW('D', 4, unsigned long)
|
||||
#define IOCTL_MAP_USER _IOW('D', 5, unsigned long) // map
|
||||
#define IOCTL_UNMAP_USER _IOW('D', 6, unsigned long)
|
||||
#define IOCTL_REGISTER_PID _IOW('D', 7, unsigned long) // register pid
|
||||
#define IOCTL_UNREGISTER_PID _IOW('D', 8, unsigned long)
|
||||
#define IOCTL_RECONFIG_LOAD _IOW('D', 9, unsigned long) // reconfiguration
|
||||
|
||||
#define IOCTL_ARP_LOOKUP _IOW('D', 10, unsigned long) // arp lookup
|
||||
#define IOCTL_SET_IP_ADDRESS _IOW('D', 11, unsigned long)
|
||||
#define IOCTL_SET_BOARD_NUM _IOW('D', 12, unsigned long)
|
||||
#define IOCTL_WRITE_CTX _IOW('D', 13, unsigned long) // qp context
|
||||
#define IOCTL_WRITE_CONN _IOW('D', 14, unsigned long) // qp connection
|
||||
#define IOCTL_SET_TCP_OFFS _IOW('D', 15, unsigned long) // tcp mem offsets
|
||||
|
||||
#define IOCTL_READ_CNFG _IOR('D', 32, unsigned long) // status cnfg
|
||||
#define IOCTL_NET_STATS _IOR('D', 33, unsigned long) // status network
|
||||
#define IOCTL_READ_ENG_STATUS _IOR('D', 35, unsigned long) // status engines
|
||||
|
||||
/* Hash */
|
||||
#define PR_HASH_TABLE_ORDER 8
|
||||
#define PR_BATCH_SIZE (2 * 1024 * 1024)
|
||||
|
||||
#define USER_HASH_TABLE_ORDER 8
|
||||
|
||||
/* PID */
|
||||
#define N_CPID_MAX 64
|
||||
#define WB_SIZE (2 * N_CPID_MAX * sizeof(uint32_t))
|
||||
#define N_WB_PAGES ((WB_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
|
||||
|
||||
/* Network */
|
||||
#define EN_LOWSPEED 0x5
|
||||
#define N_NET_STAT_REGS 9
|
||||
|
||||
/* Copy */
|
||||
#define MAX_USER_WORDS 32
|
||||
|
||||
/**
|
||||
* @brief Reg maps
|
||||
*
|
||||
*/
|
||||
|
||||
/* DMA engine reg map */
|
||||
struct engine_regs {
|
||||
uint32_t id;
|
||||
uint32_t ctrl;
|
||||
uint32_t ctrl_w1s;
|
||||
uint32_t ctrl_w1c;
|
||||
uint32_t rsrvd_1[12];
|
||||
|
||||
uint32_t status;
|
||||
uint32_t status_rc;
|
||||
uint32_t completed_desc_count;
|
||||
uint32_t alignments;
|
||||
uint32_t rsrvd_2[14]; // padding
|
||||
|
||||
uint32_t poll_mode_wb_lo;
|
||||
uint32_t poll_mode_wb_hi;
|
||||
uint32_t interrupt_enable_mask;
|
||||
uint32_t interrupt_enable_mask_w1s;
|
||||
uint32_t interrupt_enable_mask_w1c;
|
||||
uint32_t rsrvd_3[9]; // padding
|
||||
|
||||
uint32_t perf_ctrl;
|
||||
uint32_t perf_cyc_lo;
|
||||
uint32_t perf_cyc_hi;
|
||||
uint32_t perf_dat_lo;
|
||||
uint32_t perf_dat_hi;
|
||||
uint32_t perf_pnd_lo;
|
||||
uint32_t perf_pnd_hi;
|
||||
} __packed;
|
||||
|
||||
/* Interrupt reg map */
|
||||
struct interrupt_regs {
|
||||
uint32_t id;
|
||||
uint32_t user_int_enable;
|
||||
uint32_t user_int_enable_w1s;
|
||||
uint32_t user_int_enable_w1c;
|
||||
uint32_t channel_int_enable;
|
||||
uint32_t channel_int_enable_w1s;
|
||||
uint32_t channel_int_enable_w1c;
|
||||
uint32_t reserved_1[9]; // padding
|
||||
|
||||
uint32_t user_int_request;
|
||||
uint32_t channel_int_request;
|
||||
uint32_t user_int_pending;
|
||||
uint32_t channel_int_pending;
|
||||
uint32_t reserved_2[12]; // padding
|
||||
|
||||
uint32_t user_msi_vector[8];
|
||||
uint32_t channel_msi_vector[8];
|
||||
} __packed;
|
||||
|
||||
/* Polled mode descriptors struct */
|
||||
struct xdma_poll_wb {
|
||||
uint32_t completed_desc_count;
|
||||
uint32_t reserved_1[7];
|
||||
} __packed;
|
||||
|
||||
/* FPGA static config reg map */
|
||||
struct fpga_stat_cnfg_regs {
|
||||
uint64_t probe;
|
||||
uint64_t n_chan;
|
||||
uint64_t n_regions;
|
||||
uint64_t ctrl_cnfg;
|
||||
uint64_t mem_cnfg;
|
||||
uint64_t pr_cnfg;
|
||||
uint64_t rdma_cnfg;
|
||||
uint64_t tcp_cnfg;
|
||||
uint64_t lspeed_cnfg;
|
||||
uint64_t reserved_0[1];
|
||||
uint64_t pr_ctrl;
|
||||
uint64_t pr_stat;
|
||||
uint64_t pr_addr;
|
||||
uint64_t pr_len;
|
||||
uint64_t tlb_ctrl;
|
||||
uint64_t tlb_stat;
|
||||
uint64_t tlb_addr;
|
||||
uint64_t tlb_len;
|
||||
uint64_t reserved_1[2];
|
||||
uint64_t net_0_ip;
|
||||
uint64_t net_0_boardnum;
|
||||
uint64_t net_0_arp;
|
||||
uint64_t rdma_0_qp_ctx[3];
|
||||
uint64_t rdma_0_qp_conn[3];
|
||||
uint64_t tcp_0_offs[2];
|
||||
uint64_t net_0_debug[N_NET_STAT_REGS];
|
||||
uint64_t net_1_ip;
|
||||
uint64_t net_1_boardnum;
|
||||
uint64_t net_1_arp;
|
||||
uint64_t rdma_1_qp_ctx[3];
|
||||
uint64_t rdma_1_qp_conn[3];
|
||||
uint64_t tcp_1_offs[2];
|
||||
uint64_t net_1_debug[N_NET_STAT_REGS];
|
||||
} __packed;
|
||||
|
||||
/* FPGA dynamic config reg map */
|
||||
struct fpga_cnfg_regs {
|
||||
uint64_t ctrl;
|
||||
uint64_t vaddr_rd;
|
||||
uint64_t len_rd;
|
||||
uint64_t vaddr_wr;
|
||||
uint64_t len_wr;
|
||||
uint64_t vaddr_miss;
|
||||
uint64_t len_miss;
|
||||
uint64_t datapath_set;
|
||||
uint64_t datapath_clr;
|
||||
uint64_t stat_cmd_used_rd;
|
||||
uint64_t stat_cmd_used_wr;
|
||||
uint64_t stat_sent[6];
|
||||
uint64_t stat_pfaults;
|
||||
uint64_t wback_rd;
|
||||
uint64_t wback_wr;
|
||||
// Rest of regs not used in the driver
|
||||
} __packed;
|
||||
|
||||
/* FPGA dynamic config reg map */
|
||||
struct fpga_cnfg_regs_avx {
|
||||
uint64_t ctrl[4];
|
||||
uint64_t vaddr_miss;
|
||||
uint64_t len_miss;
|
||||
uint64_t pf[2];
|
||||
uint64_t datapath_set[4];
|
||||
uint64_t datapath_clr[4];
|
||||
uint64_t stat[4];
|
||||
uint64_t wback[4];
|
||||
// Rest not used in the driver
|
||||
} __packed;
|
||||
|
||||
/**
|
||||
* @brief Engine structs
|
||||
*
|
||||
*/
|
||||
|
||||
/* Engine descriptors */
|
||||
struct xdma_sgdma_regs {
|
||||
uint32_t identifier;
|
||||
uint32_t reserved_1[31]; /* padding */
|
||||
|
||||
/* bus address to first descriptor in Root Complex Memory */
|
||||
uint32_t first_desc_lo;
|
||||
uint32_t first_desc_hi;
|
||||
/* number of adjacent descriptors at first_desc */
|
||||
uint32_t first_desc_adjacent;
|
||||
uint32_t credits;
|
||||
} __packed;
|
||||
|
||||
/* Engine struct */
|
||||
struct xdma_engine {
|
||||
int channel; // egnine channel
|
||||
char *name; // engine name
|
||||
struct bus_drvdata *pd; // PCI device
|
||||
|
||||
struct engine_regs *regs; // HW regs, control and status
|
||||
struct engine_sgdma_regs *sgdma_regs; // SGDMA reg BAR offset
|
||||
|
||||
// Config
|
||||
int running; // engine state
|
||||
int c2h; // c2h(write) or h2c(read)
|
||||
uint32_t status;
|
||||
|
||||
int addr_align; // source/dest alignment in bytes
|
||||
int len_granularity; // transfer length multiple
|
||||
int addr_bits; // HW datapath address width
|
||||
|
||||
/* Members associated with polled mode support */
|
||||
uint8_t *poll_mode_addr_virt; /* virt addr for descriptor writeback */
|
||||
uint64_t poll_mode_phys_addr; /* bus addr for descriptor writeback */
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Hash maps
|
||||
*
|
||||
*/
|
||||
|
||||
/* Mapped user pages */
|
||||
struct user_pages {
|
||||
struct hlist_node entry;
|
||||
uint64_t vaddr;
|
||||
bool huge;
|
||||
int32_t cpid;
|
||||
uint64_t n_hpages;
|
||||
uint64_t n_pages;
|
||||
struct page **hpages;
|
||||
uint64_t *cpages;
|
||||
};
|
||||
|
||||
/* Mapped large PR pages */
|
||||
struct pr_pages {
|
||||
struct hlist_node entry;
|
||||
int reg_id;
|
||||
uint64_t vaddr;
|
||||
uint64_t n_pages;
|
||||
struct page **pages;
|
||||
};
|
||||
|
||||
/* User tables */
|
||||
extern struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
|
||||
extern struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
|
||||
|
||||
/* PR table */
|
||||
extern struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
|
||||
|
||||
/**
|
||||
* @brief Mem
|
||||
*
|
||||
*/
|
||||
|
||||
/* Pool chunks */
|
||||
struct chunk {
|
||||
uint32_t id;
|
||||
struct chunk *next;
|
||||
};
|
||||
|
||||
/* TLB order */
|
||||
struct tlb_order {
|
||||
bool hugepage;
|
||||
uint64_t page_shift;
|
||||
uint64_t page_size;
|
||||
uint64_t page_mask;
|
||||
int assoc;
|
||||
|
||||
int key_mask;
|
||||
int key_size;
|
||||
int tag_mask;
|
||||
int tag_size;
|
||||
int phy_mask;
|
||||
int phy_size;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Dev structs
|
||||
*
|
||||
*/
|
||||
|
||||
/* PR controller */
|
||||
struct pr_ctrl {
|
||||
struct bus_drvdata *pd; // PCI device
|
||||
spinlock_t lock;
|
||||
|
||||
// Engines
|
||||
struct xdma_engine *engine_h2c; // h2c engine
|
||||
struct xdma_engine *engine_c2h; // c2h engine
|
||||
|
||||
// Allocated buffers
|
||||
struct pr_pages curr_buff;
|
||||
};
|
||||
|
||||
/* Virtual FPGA device */
|
||||
struct fpga_dev {
|
||||
int id; // identifier
|
||||
struct cdev cdev; // char device
|
||||
struct bus_drvdata *pd; // PCI device
|
||||
struct pr_ctrl *prc; // PR controller
|
||||
|
||||
// Current task
|
||||
struct task_struct *curr_task;
|
||||
struct mm_struct *curr_mm;
|
||||
|
||||
// Control region
|
||||
uint64_t fpga_phys_addr_ctrl;
|
||||
uint64_t fpga_phys_addr_ctrl_avx;
|
||||
|
||||
// Writeback
|
||||
uint32_t *wb_addr_virt;
|
||||
uint64_t wb_phys_addr;
|
||||
|
||||
// TLBs
|
||||
uint64_t *fpga_lTlb; // large page TLB
|
||||
uint64_t *fpga_sTlb; // small page TLB
|
||||
struct fpga_cnfg_regs *fpga_cnfg; // config
|
||||
struct fpga_cnfg_regs_avx *fpga_cnfg_avx; // config AVX
|
||||
|
||||
// PIDs
|
||||
spinlock_t card_pid_lock;
|
||||
struct chunk *pid_chunks;
|
||||
pid_t *pid_array;
|
||||
int num_free_pid_chunks;
|
||||
struct chunk *pid_alloc;
|
||||
|
||||
// Engines
|
||||
struct xdma_engine *engine_h2c; // h2c engine
|
||||
struct xdma_engine *engine_c2h; // c2h engine
|
||||
|
||||
// In use
|
||||
atomic_t in_use; // busy flag
|
||||
|
||||
// Lock
|
||||
spinlock_t lock; // protects concurrent accesses
|
||||
|
||||
// Allocated buffers
|
||||
struct user_pages curr_user_buff;
|
||||
};
|
||||
|
||||
/* PCI driver data */
|
||||
struct bus_drvdata {
|
||||
|
||||
// PCI
|
||||
struct pci_dev *pci_dev;
|
||||
|
||||
// BARs
|
||||
int regions_in_use;
|
||||
int got_regions;
|
||||
void *__iomem bar[MAX_NUM_BARS];
|
||||
unsigned long bar_phys_addr[MAX_NUM_BARS];
|
||||
unsigned long bar_len[MAX_NUM_BARS];
|
||||
|
||||
// Engines
|
||||
int engines_num;
|
||||
|
||||
// ECI
|
||||
// I/O
|
||||
unsigned long io_phys_addr;
|
||||
unsigned long io_len;
|
||||
|
||||
|
||||
// FPGA static config
|
||||
uint probe;
|
||||
int n_fpga_chan;
|
||||
int n_fpga_reg;
|
||||
int en_avx;
|
||||
int en_bypass;
|
||||
int en_tlbf;
|
||||
int en_wb;
|
||||
int en_ddr;
|
||||
int en_hbm;
|
||||
int en_mem;
|
||||
int n_mem_chan;
|
||||
int en_pr;
|
||||
int en_rdma_0;
|
||||
int en_rdma_1;
|
||||
int en_tcp_0;
|
||||
int en_tcp_1;
|
||||
int en_net_0;
|
||||
int en_net_1;
|
||||
volatile struct fpga_stat_cnfg_regs *fpga_stat_cnfg;
|
||||
struct fpga_dev *fpga_dev;
|
||||
|
||||
// PR control
|
||||
struct pr_ctrl prc;
|
||||
|
||||
// TLB order
|
||||
struct tlb_order *stlb_order;
|
||||
struct tlb_order *ltlb_order;
|
||||
|
||||
// Locks
|
||||
spinlock_t stat_lock;
|
||||
spinlock_t prc_lock;
|
||||
spinlock_t tlb_lock;
|
||||
|
||||
// IRQ
|
||||
int irq_count;
|
||||
int irq_line;
|
||||
int msix_enabled;
|
||||
struct msix_entry irq_entry[32];
|
||||
|
||||
// Card memory
|
||||
spinlock_t card_l_lock;
|
||||
struct chunk *lchunks;
|
||||
int num_free_lchunks;
|
||||
struct chunk *lalloc;
|
||||
|
||||
spinlock_t card_s_lock;
|
||||
struct chunk *schunks;
|
||||
int num_free_schunks;
|
||||
struct chunk *salloc;
|
||||
};
|
||||
|
||||
|
||||
#endif // Coyote device
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "eci_dev.h"
|
||||
|
||||
/*
|
||||
____ _ _
|
||||
| _ \ ___ __ _(_)___| |_ ___ _ __
|
||||
| |_) / _ \/ _` | / __| __/ _ \ '__|
|
||||
| _ < __/ (_| | \__ \ || __/ |
|
||||
|_| \_\___|\__, |_|___/\__\___|_|
|
||||
|___/
|
||||
*/
|
||||
|
||||
static struct bus_drvdata *pd;
|
||||
|
||||
/**
|
||||
* @brief ECI device init
|
||||
*
|
||||
*/
|
||||
int eci_init(void)
|
||||
{
|
||||
int ret_val = 0;
|
||||
|
||||
// dynamic major
|
||||
dev_t dev = MKDEV(fpga_major, 0);
|
||||
|
||||
// allocate mem. for device instance
|
||||
pd = kzalloc(sizeof(struct bus_drvdata), GFP_KERNEL);
|
||||
if(!pd) {
|
||||
pr_err("device memory region not obtained\n");
|
||||
ret_val = -ENOMEM;
|
||||
goto err_alloc;
|
||||
}
|
||||
|
||||
// get static config
|
||||
pd->io_phys_addr = IO_PHYS_ADDR;
|
||||
pd->fpga_stat_cnfg = ioremap(pd->io_phys_addr + FPGA_STAT_CNFG_OFFS, FPGA_STAT_CNFG_SIZE);
|
||||
read_static_config(pd);
|
||||
|
||||
// allocate card mem resources
|
||||
ret_val = alloc_card_resources(pd);
|
||||
if (ret_val) {
|
||||
pr_err("card resources could not be allocated\n");
|
||||
goto err_card_alloc; // ERR_CARD_ALLOC
|
||||
}
|
||||
|
||||
// initialize spin locks
|
||||
init_spin_locks(pd);
|
||||
|
||||
// create FPGA devices and register major
|
||||
ret_val = init_char_devices(pd, dev);
|
||||
if (ret_val) {
|
||||
goto err_create_fpga_dev; // ERR_CREATE_FPGA_DEV
|
||||
}
|
||||
|
||||
// initialize vFPGAs
|
||||
ret_val = init_fpga_devices(pd);
|
||||
if (ret_val) {
|
||||
goto err_init_fpga_dev;
|
||||
}
|
||||
|
||||
// Init hash
|
||||
hash_init(pr_buff_map);
|
||||
|
||||
if(ret_val == 0)
|
||||
goto end;
|
||||
|
||||
err_init_fpga_dev:
|
||||
kfree(pd->fpga_dev);
|
||||
class_destroy(fpga_class);
|
||||
err_create_fpga_dev:
|
||||
vfree(pd->schunks);
|
||||
vfree(pd->lchunks);
|
||||
err_card_alloc:
|
||||
err_alloc:
|
||||
end:
|
||||
pr_info("probe returning %d\n", ret_val);
|
||||
return ret_val;
|
||||
}
|
||||
|
||||
void eci_exit(void)
|
||||
{
|
||||
// delete vFPGAs
|
||||
free_fpga_devices(pd);
|
||||
|
||||
// delete char devices
|
||||
free_char_devices(pd);
|
||||
|
||||
// deallocate card resources
|
||||
free_card_resources(pd);
|
||||
|
||||
// free device data
|
||||
kfree(pd);
|
||||
pr_info("device memory freed\n");
|
||||
|
||||
pr_info("removal completed\n");
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __ECI_DEV_H__
|
||||
#define __ECI_DEV_H__
|
||||
|
||||
#include "../coyote_dev.h"
|
||||
#include "../fpga_dev.h"
|
||||
|
||||
/* Probe */
|
||||
int eci_init(void);
|
||||
void eci_exit(void);
|
||||
|
||||
#endif // ECI device
|
|
@ -0,0 +1,421 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "fpga_dev.h"
|
||||
|
||||
int fpga_major = FPGA_MAJOR;
|
||||
struct class *fpga_class = NULL;
|
||||
|
||||
/**
|
||||
* @brief Fops
|
||||
*
|
||||
*/
|
||||
struct file_operations fpga_fops = {
|
||||
.owner = THIS_MODULE,
|
||||
.open = fpga_open,
|
||||
.release = fpga_release,
|
||||
.mmap = fpga_mmap,
|
||||
.unlocked_ioctl = fpga_ioctl,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Read static configuration
|
||||
*
|
||||
*/
|
||||
void read_static_config(struct bus_drvdata *d)
|
||||
{
|
||||
// probe
|
||||
d->probe = d->fpga_stat_cnfg->probe;
|
||||
pr_info("deployment id %08x\n", d->probe);
|
||||
|
||||
// channels and regions
|
||||
d->n_fpga_chan = d->fpga_stat_cnfg->n_chan;
|
||||
d->n_fpga_reg = d->fpga_stat_cnfg->n_regions;
|
||||
pr_info("detected %d virtual FPGA regions, %d FPGA channels\n", d->n_fpga_reg, d->n_fpga_chan);
|
||||
|
||||
// flags
|
||||
d->en_avx = (d->fpga_stat_cnfg->ctrl_cnfg & EN_AVX_MASK) >> EN_AVX_SHFT;
|
||||
d->en_bypass = (d->fpga_stat_cnfg->ctrl_cnfg & EN_BPSS_MASK) >> EN_BPSS_SHFT;
|
||||
d->en_tlbf = (d->fpga_stat_cnfg->ctrl_cnfg & EN_TLBF_MASK) >> EN_TLBF_SHFT;
|
||||
d->en_wb = (d->fpga_stat_cnfg->ctrl_cnfg & EN_WB_MASK) >> EN_WB_SHFT;
|
||||
pr_info("enabled AVX %d, enabled bypass %d, enabled tlb fast %d, enabled writeback %d\n", d->en_avx, d->en_bypass, d->en_tlbf, d->en_wb);
|
||||
|
||||
// mmu
|
||||
d->stlb_order = kzalloc(sizeof(struct tlb_order), GFP_KERNEL);
|
||||
BUG_ON(!d->stlb_order);
|
||||
d->stlb_order->hugepage = false;
|
||||
d->stlb_order->key_size = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_ORDER_MASK) >> TLB_S_ORDER_SHFT;
|
||||
d->stlb_order->assoc = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_ASSOC_MASK) >> TLB_S_ASSOC_SHFT;
|
||||
d->stlb_order->page_shift = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_PG_SHFT_MASK) >> TLB_S_PG_SHFT_SHFT;
|
||||
BUG_ON(d->stlb_order->page_shift != PAGE_SHIFT);
|
||||
d->stlb_order->page_size = PAGE_SIZE;
|
||||
d->stlb_order->page_mask = PAGE_MASK;
|
||||
d->stlb_order->key_mask = (1 << d->stlb_order->key_size) - 1;
|
||||
d->stlb_order->tag_size = TLB_VADDR_RANGE - d->stlb_order->page_shift - d->stlb_order->key_size;
|
||||
d->stlb_order->tag_mask = (1 << d->stlb_order->tag_size) - 1;
|
||||
d->stlb_order->phy_size = TLB_PADDR_RANGE - d->stlb_order->page_shift;
|
||||
d->stlb_order->phy_mask = (1 << d->stlb_order->phy_size) - 1;
|
||||
pr_info("sTLB order %d, sTLB assoc %d, sTLB page size %lld\n", d->stlb_order->key_size, d->stlb_order->assoc, d->stlb_order->page_size);
|
||||
|
||||
d->ltlb_order = kzalloc(sizeof(struct tlb_order), GFP_KERNEL);
|
||||
BUG_ON(!d->ltlb_order);
|
||||
d->ltlb_order->hugepage = true;
|
||||
d->ltlb_order->key_size = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_ORDER_MASK) >> TLB_L_ORDER_SHFT;
|
||||
d->ltlb_order->assoc = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_ASSOC_MASK) >> TLB_L_ASSOC_SHFT;
|
||||
d->ltlb_order->page_shift = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_PG_SHFT_MASK) >> TLB_L_PG_SHFT_SHFT;
|
||||
d->ltlb_order->page_size = 1 << d->ltlb_order->page_shift;
|
||||
d->ltlb_order->page_mask = (~(d->ltlb_order->page_size - 1));
|
||||
d->ltlb_order->key_mask = (1 << d->ltlb_order->key_size) - 1;
|
||||
d->ltlb_order->tag_size = TLB_VADDR_RANGE - d->ltlb_order->page_shift - d->ltlb_order->key_size;
|
||||
d->ltlb_order->tag_mask = (1 << d->ltlb_order->tag_size) - 1;
|
||||
d->ltlb_order->phy_size = TLB_PADDR_RANGE - d->ltlb_order->page_shift ;
|
||||
d->ltlb_order->phy_mask = (1 << d->ltlb_order->phy_size) - 1;
|
||||
pr_info("lTLB order %d, lTLB assoc %d, lTLB page size %lld\n", d->ltlb_order->key_size, d->ltlb_order->assoc, d->ltlb_order->page_size);
|
||||
|
||||
// mem
|
||||
d->en_ddr = (d->fpga_stat_cnfg->mem_cnfg & EN_DDR_MASK) >> EN_DDR_SHFT;
|
||||
d->en_hbm = (d->fpga_stat_cnfg->mem_cnfg & EN_HBM_MASK) >> EN_HBM_SHFT;
|
||||
d->en_mem = d->en_ddr || d->en_hbm;
|
||||
d->n_mem_chan = (d->fpga_stat_cnfg->mem_cnfg & N_MEM_CHAN_MASK) >> N_MEM_CHAN_SHFT;
|
||||
pr_info("enabled DDR %d, number of DDR channels %d\n", d->en_ddr, d->n_mem_chan);
|
||||
|
||||
// pr
|
||||
d->en_pr = (d->fpga_stat_cnfg->pr_cnfg & EN_PR_MASK) >> EN_PR_SHFT;
|
||||
pr_info("enabled PR %d\n", d->en_pr);
|
||||
|
||||
// network
|
||||
d->en_rdma_0 = (d->fpga_stat_cnfg->rdma_cnfg & EN_RDMA_0_MASK) >> EN_RDMA_0_SHFT;
|
||||
d->en_rdma_1 = (d->fpga_stat_cnfg->rdma_cnfg & EN_RDMA_1_MASK) >> EN_RDMA_1_SHFT;
|
||||
pr_info("enabled RDMA on QSFP0 %d, enabled RDMA on QSFP1 %d\n", d->en_rdma_0, d->en_rdma_1);
|
||||
|
||||
d->en_tcp_0 = (d->fpga_stat_cnfg->tcp_cnfg & EN_TCP_0_MASK) >> EN_TCP_0_SHFT;
|
||||
d->en_tcp_1 = (d->fpga_stat_cnfg->tcp_cnfg & EN_TCP_1_MASK) >> EN_TCP_1_SHFT;
|
||||
pr_info("enabled TCP/IP on QSFP0 %d, enabled TCP/IP on QSFP1 %d\n", d->en_tcp_0, d->en_tcp_1);
|
||||
|
||||
d->en_net_0 = d->en_rdma_0 | d->en_tcp_0;
|
||||
d->en_net_1 = d->en_rdma_1 | d->en_tcp_1;
|
||||
|
||||
// network board setup (TODO: maybe move to sw fully?)
|
||||
if (d->en_net_0) {
|
||||
d->fpga_stat_cnfg->net_0_ip = BASE_IP_ADDR_0 + NODE_ID;
|
||||
d->fpga_stat_cnfg->net_0_boardnum = NODE_ID;
|
||||
}
|
||||
if (d->en_net_1) {
|
||||
d->fpga_stat_cnfg->net_1_ip = BASE_IP_ADDR_1 + NODE_ID;
|
||||
d->fpga_stat_cnfg->net_1_boardnum = NODE_ID;
|
||||
}
|
||||
|
||||
// lowspeed ctrl
|
||||
d->fpga_stat_cnfg->lspeed_cnfg = EN_LOWSPEED;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Allocate FPGA memory resources
|
||||
*
|
||||
*/
|
||||
int alloc_card_resources(struct bus_drvdata *d)
|
||||
{
|
||||
int ret_val = 0;
|
||||
int i;
|
||||
|
||||
// init chunks card
|
||||
d->num_free_lchunks = N_LARGE_CHUNKS;
|
||||
d->num_free_schunks = N_SMALL_CHUNKS;
|
||||
|
||||
d->lchunks = vzalloc(N_LARGE_CHUNKS * sizeof(struct chunk));
|
||||
if (!d->lchunks) {
|
||||
pr_err("memory region for cmem structs not obtained\n");
|
||||
goto err_alloc_lchunks; // ERR_ALLOC_LCHUNKS
|
||||
}
|
||||
d->schunks = vzalloc(N_SMALL_CHUNKS * sizeof(struct chunk));
|
||||
if (!d->schunks) {
|
||||
pr_err("memory region for cmem structs not obtained\n");
|
||||
goto err_alloc_schunks; // ERR_ALLOC_SCHUNKS
|
||||
}
|
||||
|
||||
for (i = 0; i < N_LARGE_CHUNKS - 1; i++) {
|
||||
d->lchunks[i].id = i;
|
||||
d->lchunks[i].next = &d->lchunks[i + 1];
|
||||
}
|
||||
for (i = 0; i < N_SMALL_CHUNKS - 1; i++) {
|
||||
d->schunks[i].id = i;
|
||||
d->schunks[i].next = &d->schunks[i + 1];
|
||||
}
|
||||
d->lalloc = &d->lchunks[0];
|
||||
d->salloc = &d->schunks[0];
|
||||
|
||||
goto end;
|
||||
|
||||
err_alloc_schunks:
|
||||
vfree(d->lchunks);
|
||||
err_alloc_lchunks:
|
||||
ret_val = -ENOMEM;
|
||||
end:
|
||||
return ret_val;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Free card memory resources
|
||||
*
|
||||
*/
|
||||
void free_card_resources(struct bus_drvdata *d)
|
||||
{
|
||||
// free card memory structs
|
||||
vfree(d->schunks);
|
||||
vfree(d->lchunks);
|
||||
|
||||
pr_info("card resources deallocated\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Initialize spin locks
|
||||
*
|
||||
*/
|
||||
void init_spin_locks(struct bus_drvdata *d)
|
||||
{
|
||||
// initialize spinlocks
|
||||
spin_lock_init(&d->card_l_lock);
|
||||
spin_lock_init(&d->card_s_lock);
|
||||
spin_lock_init(&d->prc.lock);
|
||||
spin_lock_init(&d->stat_lock);
|
||||
spin_lock_init(&d->prc_lock);
|
||||
spin_lock_init(&d->tlb_lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Init char devices
|
||||
*
|
||||
*/
|
||||
int init_char_devices(struct bus_drvdata *d, dev_t dev)
|
||||
{
|
||||
int ret_val = 0;
|
||||
|
||||
ret_val = alloc_chrdev_region(&dev, 0, d->n_fpga_reg, DEV_NAME);
|
||||
fpga_major = MAJOR(dev);
|
||||
if (ret_val) {
|
||||
pr_err("failed to register virtual FPGA devices");
|
||||
goto end;
|
||||
}
|
||||
pr_info("virtual FPGA device regions allocated, major number %d\n", fpga_major);
|
||||
|
||||
// create device class
|
||||
fpga_class = class_create(THIS_MODULE, DEV_NAME);
|
||||
|
||||
// virtual FPGA devices
|
||||
d->fpga_dev = kmalloc(d->n_fpga_reg * sizeof(struct fpga_dev), GFP_KERNEL);
|
||||
if (!d->fpga_dev) {
|
||||
pr_err("could not allocate memory for fpga devices\n");
|
||||
goto err_char_mem; // ERR_CHAR_MEM
|
||||
}
|
||||
memset(d->fpga_dev, 0, d->n_fpga_reg * sizeof(struct fpga_dev));
|
||||
pr_info("allocated memory for fpga devices\n");
|
||||
|
||||
goto end;
|
||||
|
||||
err_char_mem:
|
||||
unregister_chrdev_region(dev, d->n_fpga_reg);
|
||||
ret_val = -ENOMEM;
|
||||
end:
|
||||
return ret_val;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Delete char devices
|
||||
*
|
||||
*/
|
||||
void free_char_devices(struct bus_drvdata *d)
|
||||
{
|
||||
// free virtual FPGA memory
|
||||
kfree(d->fpga_dev);
|
||||
pr_info("virtual FPGA device memory freed\n");
|
||||
|
||||
// remove class
|
||||
class_destroy(fpga_class);
|
||||
pr_info("fpga class deleted\n");
|
||||
|
||||
// remove char devices
|
||||
unregister_chrdev_region(MKDEV(fpga_major, 0), d->n_fpga_reg);
|
||||
pr_info("char devices unregistered\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Initialize vFPGAs
|
||||
*
|
||||
*/
|
||||
int init_fpga_devices(struct bus_drvdata *d)
|
||||
{
|
||||
int ret_val = 0;
|
||||
int i, j;
|
||||
int devno;
|
||||
|
||||
for (i = 0; i < d->n_fpga_reg; i++) {
|
||||
// ID
|
||||
d->fpga_dev[i].id = i;
|
||||
|
||||
// PCI device
|
||||
d->fpga_dev[i].pd = d;
|
||||
d->fpga_dev[i].prc = &d->prc;
|
||||
|
||||
// physical
|
||||
if(cyt_arch == CYT_ARCH_PCI) {
|
||||
d->fpga_dev[i].fpga_phys_addr_ctrl = d->bar_phys_addr[BAR_FPGA_CONFIG] + FPGA_CTRL_OFFS + i * FPGA_CTRL_SIZE;
|
||||
d->fpga_dev[i].fpga_phys_addr_ctrl_avx = d->bar_phys_addr[BAR_FPGA_CONFIG] + FPGA_CTRL_CNFG_AVX_OFFS + i * FPGA_CTRL_CNFG_AVX_SIZE;
|
||||
} else if(cyt_arch == CYT_ARCH_ECI) {
|
||||
d->fpga_dev[i].fpga_phys_addr_ctrl = d->io_phys_addr + FPGA_CTRL_OFFS + i*FPGA_CTRL_SIZE;
|
||||
}
|
||||
|
||||
// MMU control region
|
||||
d->fpga_dev[i].fpga_lTlb = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_LTLB_OFFS, FPGA_CTRL_LTLB_SIZE);
|
||||
d->fpga_dev[i].fpga_sTlb = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_STLB_OFFS, FPGA_CTRL_STLB_SIZE);
|
||||
|
||||
// FPGA engine control
|
||||
d->fpga_dev[i].fpga_cnfg = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS, FPGA_CTRL_CNFG_SIZE);
|
||||
|
||||
// FPGA engine control AVX
|
||||
if(cyt_arch == CYT_ARCH_PCI) {
|
||||
d->fpga_dev[i].fpga_cnfg_avx = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl_avx, FPGA_CTRL_CNFG_AVX_SIZE);
|
||||
}
|
||||
|
||||
// init chunks pid
|
||||
d->fpga_dev[i].num_free_pid_chunks = N_CPID_MAX;
|
||||
|
||||
d->fpga_dev[i].pid_chunks = vzalloc(N_CPID_MAX * sizeof(struct chunk));
|
||||
if (!d->fpga_dev[i].pid_chunks) {
|
||||
pr_err("memory region for pid chunks not obtained\n");
|
||||
goto err_alloc_pid_chunks; // ERR_ALLOC_PID_CHUNKS
|
||||
}
|
||||
d->fpga_dev[i].pid_array = vzalloc(N_CPID_MAX * sizeof(pid_t));
|
||||
if (!d->fpga_dev[i].pid_array) {
|
||||
pr_err("memory region for pid array not obtained\n");
|
||||
goto err_alloc_pid_array; // ERR_ALLOC_PID_ARRAY
|
||||
}
|
||||
|
||||
for (j = 0; j < N_CPID_MAX - 1; j++) {
|
||||
d->fpga_dev[i].pid_chunks[j].id = j;
|
||||
d->fpga_dev[i].pid_chunks[j].next = &d->fpga_dev[i].pid_chunks[j + 1];
|
||||
}
|
||||
d->fpga_dev[i].pid_alloc = &d->fpga_dev[i].pid_chunks[0];
|
||||
|
||||
// initialize device spinlock
|
||||
spin_lock_init(&d->fpga_dev[i].lock);
|
||||
spin_lock_init(&d->fpga_dev[i].card_pid_lock);
|
||||
|
||||
// writeback setup
|
||||
if(d->en_wb) {
|
||||
d->fpga_dev[i].wb_addr_virt = pci_alloc_consistent(d->pci_dev, WB_SIZE, &d->fpga_dev[i].wb_phys_addr);
|
||||
if(!d->fpga_dev[i].wb_addr_virt) {
|
||||
pr_err("failed to allocate writeback memory\n");
|
||||
goto err_wb;
|
||||
}
|
||||
|
||||
if(cyt_arch == CYT_ARCH_PCI && d->en_avx) {
|
||||
d->fpga_dev[i].fpga_cnfg_avx->wback[0] = d->fpga_dev[i].wb_phys_addr;
|
||||
d->fpga_dev[i].fpga_cnfg_avx->wback[1] = d->fpga_dev[i].wb_phys_addr + N_CPID_MAX * sizeof(uint32_t);
|
||||
} else {
|
||||
d->fpga_dev[i].fpga_cnfg->wback_rd = d->fpga_dev[i].wb_phys_addr;
|
||||
d->fpga_dev[i].fpga_cnfg->wback_wr = d->fpga_dev[i].wb_phys_addr + N_CPID_MAX * sizeof(uint32_t);
|
||||
}
|
||||
|
||||
pr_info("allocated memory for descriptor writeback, vaddr %llx, paddr %llx",
|
||||
(uint64_t)d->fpga_dev[i].wb_addr_virt, d->fpga_dev[i].wb_phys_addr);
|
||||
}
|
||||
|
||||
// create device
|
||||
devno = MKDEV(fpga_major, i);
|
||||
device_create(fpga_class, NULL, devno, NULL, DEV_NAME "%d", i);
|
||||
pr_info("virtual FPGA device %d created\n", i);
|
||||
|
||||
// add device
|
||||
cdev_init(&d->fpga_dev[i].cdev, &fpga_fops);
|
||||
d->fpga_dev[i].cdev.owner = THIS_MODULE;
|
||||
d->fpga_dev[i].cdev.ops = &fpga_fops;
|
||||
|
||||
// Init hash
|
||||
hash_init(user_lbuff_map[i]);
|
||||
hash_init(user_sbuff_map[i]);
|
||||
|
||||
ret_val = cdev_add(&d->fpga_dev[i].cdev, devno, 1);
|
||||
if (ret_val) {
|
||||
pr_err("could not create a virtual FPGA device %d\n", i);
|
||||
goto err_char_reg;
|
||||
}
|
||||
}
|
||||
pr_info("all virtual FPGA devices added\n");
|
||||
|
||||
goto end;
|
||||
|
||||
err_char_reg:
|
||||
for (j = 0; j < i; j++) {
|
||||
device_destroy(fpga_class, MKDEV(fpga_major, j));
|
||||
cdev_del(&d->fpga_dev[j].cdev);
|
||||
}
|
||||
err_wb:
|
||||
if(d->en_wb) {
|
||||
for (j = 0; j < i; j++) {
|
||||
set_memory_wb((uint64_t)d->fpga_dev[j].wb_addr_virt, N_WB_PAGES);
|
||||
pci_free_consistent(d->pci_dev, WB_SIZE,
|
||||
d->fpga_dev[j].wb_addr_virt, d->fpga_dev[j].wb_phys_addr);
|
||||
}
|
||||
}
|
||||
vfree(d->fpga_dev[i].pid_array);
|
||||
err_alloc_pid_array:
|
||||
for (j = 0; j < i; j++) {
|
||||
vfree(d->fpga_dev[j].pid_array);
|
||||
}
|
||||
vfree(d->fpga_dev[i].pid_chunks);
|
||||
err_alloc_pid_chunks:
|
||||
for (j = 0; j < i; j++) {
|
||||
vfree(d->fpga_dev[j].pid_chunks);
|
||||
}
|
||||
ret_val = -ENOMEM;
|
||||
end:
|
||||
return ret_val;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Delete vFPGAs
|
||||
*
|
||||
*/
|
||||
void free_fpga_devices(struct bus_drvdata *d) {
|
||||
int i;
|
||||
|
||||
for(i = 0; i < d->n_fpga_reg; i++) {
|
||||
device_destroy(fpga_class, MKDEV(fpga_major, i));
|
||||
cdev_del(&d->fpga_dev[i].cdev);
|
||||
|
||||
if(d->en_wb) {
|
||||
set_memory_wb((uint64_t)d->fpga_dev[i].wb_addr_virt, N_WB_PAGES);
|
||||
pci_free_consistent(d->pci_dev, WB_SIZE,
|
||||
d->fpga_dev[i].wb_addr_virt, d->fpga_dev[i].wb_phys_addr);
|
||||
}
|
||||
|
||||
vfree(d->fpga_dev[i].pid_array);
|
||||
vfree(d->fpga_dev[i].pid_chunks);
|
||||
}
|
||||
|
||||
pr_info("vFPGAs deleted\n");
|
||||
}
|
|
@ -0,0 +1,58 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __FPGA_DEV_H__
|
||||
#define __FPGA_DEV_H__
|
||||
|
||||
#include "coyote_dev.h"
|
||||
#include "fpga_fops.h"
|
||||
|
||||
/**
|
||||
* @brief Global
|
||||
*
|
||||
*/
|
||||
|
||||
extern int fpga_major;// = FPGA_MAJOR;
|
||||
extern struct class *fpga_class;// = NULL;
|
||||
|
||||
/* Read deployment config */
|
||||
void read_static_config(struct bus_drvdata *d);
|
||||
|
||||
/* Allocate initial card resources */
|
||||
int alloc_card_resources(struct bus_drvdata *d);
|
||||
void free_card_resources(struct bus_drvdata *d);
|
||||
|
||||
/* Spinlock init */
|
||||
void init_spin_locks(struct bus_drvdata *d);
|
||||
|
||||
/* Initialize vFPGAs */
|
||||
int init_char_devices(struct bus_drvdata *d, dev_t dev);
|
||||
void free_char_devices(struct bus_drvdata *d);
|
||||
int init_fpga_devices(struct bus_drvdata *d);
|
||||
void free_fpga_devices(struct bus_drvdata *d);
|
||||
|
||||
#endif // FPGA DEV
|
3134
driver/fpga_drv.c
3134
driver/fpga_drv.c
File diff suppressed because it is too large
Load Diff
|
@ -1,604 +1,40 @@
|
|||
#ifndef COYOTE_DRV_H
|
||||
#define COYOTE_DRV_H
|
||||
|
||||
#include <linux/ioctl.h>
|
||||
#include <linux/cdev.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/hashtable.h>
|
||||
|
||||
/* Driver debug */
|
||||
#define COYOTE_DEBUG 1
|
||||
|
||||
/* Network setup */
|
||||
#define BASE_IP_ADDR_0 0x0B01D4D1
|
||||
#define BASE_IP_ADDR_1 0x0B01D4D1
|
||||
#define NODE_ID 0
|
||||
#define N_TOTAL_NODES 2
|
||||
|
||||
/* Debug print */
|
||||
#if (COYOTE_DEBUG == 0)
|
||||
#define dbg_info(...)
|
||||
#else
|
||||
#define dbg_info(fmt, ...) pr_info("%s():" fmt, \
|
||||
__func__, ##__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
|
||||
#define HIGH_32(addr) ((addr >> 16) >> 16)
|
||||
/* Obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
|
||||
#define LOW_32(addr) (addr & 0xffffffffUL)
|
||||
|
||||
/* Driver info */
|
||||
#define DRV_NAME "coyote_driver"
|
||||
#define DEV_NAME "fpga"
|
||||
|
||||
/**
|
||||
* XDMA info
|
||||
*/
|
||||
#define MAX_NUM_BARS 3
|
||||
#define MAX_NUM_CHANNELS 4
|
||||
#define MAX_NUM_ENGINES (MAX_NUM_CHANNELS * 2)
|
||||
#define MAX_USER_IRQS 16
|
||||
#define C2H_CHAN_OFFS 0x1000
|
||||
#define H2C_CHAN_OFFS 0x0000
|
||||
#define CHAN_RANGE 0x100
|
||||
#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
|
||||
|
||||
/* Engine IDs */
|
||||
#define XDMA_ID_H2C 0x1fc0U
|
||||
#define XDMA_ID_C2H 0x1fc1U
|
||||
|
||||
/* Engine regs */
|
||||
#define XDMA_ENG_IRQ_NUM (1)
|
||||
#define XDMA_OFS_INT_CTRL (0x2000UL)
|
||||
#define XDMA_OFS_CONFIG (0x3000UL)
|
||||
|
||||
/* Bits of the SG DMA control register */
|
||||
#define XDMA_CTRL_RUN_STOP (1UL << 0)
|
||||
#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
|
||||
#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
|
||||
#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
|
||||
#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
|
||||
#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
|
||||
#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
|
||||
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
|
||||
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
|
||||
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
|
||||
|
||||
/* Bits of the SG DMA status register */
|
||||
#define XDMA_STAT_BUSY (1UL << 0)
|
||||
#define XDMA_STAT_DESC_STOPPED (1UL << 1)
|
||||
#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
|
||||
#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
|
||||
#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
|
||||
#define XDMA_STAT_FETCH_STOPPED (1UL << 5)
|
||||
#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
|
||||
#define XDMA_STAT_READ_ERROR (0x1FUL << 9)
|
||||
#define XDMA_STAT_DESC_ERROR (0x1FUL << 19)
|
||||
|
||||
/* Bits of the performance control register */
|
||||
#define XDMA_PERF_RUN (1UL << 0)
|
||||
#define XDMA_PERF_CLEAR (1UL << 1)
|
||||
#define XDMA_PERF_AUTO (1UL << 2)
|
||||
|
||||
/* Polling */
|
||||
#define WB_COUNT_MASK 0x00ffffffUL
|
||||
#define WB_ERR_MASK (1UL << 31)
|
||||
#define POLL_TIMEOUT_SECONDS 10
|
||||
#define NUM_POLLS_PER_SCHED 100
|
||||
|
||||
/**
|
||||
* Static layer
|
||||
*/
|
||||
#define BAR_XDMA_CONFIG 0
|
||||
#define BAR_FPGA_CONFIG 1
|
||||
|
||||
/* FPGA static config */
|
||||
#define FPGA_STAT_CNFG_OFFS 0x0
|
||||
#define FPGA_STAT_CNFG_SIZE 32 * 1024
|
||||
|
||||
#define EN_AVX_MASK 0x1
|
||||
#define EN_AVX_SHFT 0x0
|
||||
#define EN_BPSS_MASK 0x2
|
||||
#define EN_BPSS_SHFT 0x1
|
||||
#define EN_TLBF_MASK 0x4
|
||||
#define EN_TLBF_SHFT 0x2
|
||||
#define EN_WB_MASK 0x8
|
||||
#define EN_WB_SHFT 0x3
|
||||
#define TLB_S_ORDER_MASK 0xf0
|
||||
#define TLB_S_ORDER_SHFT 0x4
|
||||
#define TLB_S_ASSOC_MASK 0xf00
|
||||
#define TLB_S_ASSOC_SHFT 0x8
|
||||
#define TLB_L_ORDER_MASK 0xf000
|
||||
#define TLB_L_ORDER_SHFT 0xc
|
||||
#define TLB_L_ASSOC_MASK 0xf0000
|
||||
#define TLB_L_ASSOC_SHFT 0x10
|
||||
#define TLB_S_PG_SHFT_MASK 0x3f00000
|
||||
#define TLB_S_PG_SHFT_SHFT 0x14
|
||||
#define TLB_L_PG_SHFT_MASK 0xfc000000
|
||||
#define TLB_L_PG_SHFT_SHFT 0x1a
|
||||
|
||||
#define EN_DDR_MASK 0x1
|
||||
#define EN_DDR_SHFT 0x0
|
||||
#define EN_HBM_MASK 0x2
|
||||
#define EN_HBM_SHFT 0x1
|
||||
#define N_MEM_CHAN_MASK 0x7d
|
||||
#define N_MEM_CHAN_SHFT 0x2
|
||||
#define EN_RDMA_0_MASK 0x1
|
||||
#define EN_RDMA_1_MASK 0x2
|
||||
#define EN_PR_MASK 0x1
|
||||
#define EN_PR_SHFT 0x0
|
||||
#define EN_RDMA_0_MASK 0x1
|
||||
#define EN_RDMA_0_SHFT 0x0
|
||||
#define EN_RDMA_1_MASK 0x2
|
||||
#define EN_RDMA_1_SHFT 0x1
|
||||
#define EN_TCP_0_MASK 0x1
|
||||
#define EN_TCP_0_SHFT 0x0
|
||||
#define EN_TCP_1_MASK 0x2
|
||||
#define EN_TCP_1_SHFT 0x1
|
||||
|
||||
/**
|
||||
* Dynamic layer
|
||||
*/
|
||||
|
||||
/* FPGA control regions */
|
||||
#define FPGA_CTRL_SIZE 256 * 1024
|
||||
#define FPGA_CTRL_OFFS 0x100000
|
||||
#define FPGA_CTRL_LTLB_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_LTLB_OFFS 0x0
|
||||
#define FPGA_CTRL_STLB_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_STLB_OFFS 0x10000
|
||||
#define FPGA_CTRL_USER_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_USER_OFFS 0x20000
|
||||
#define FPGA_CTRL_CNFG_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_CNFG_OFFS 0x30000
|
||||
|
||||
#define FPGA_CTRL_CNFG_AVX_SIZE 256 * 1024
|
||||
#define FPGA_CTRL_CNFG_AVX_OFFS 0x1000000
|
||||
|
||||
/* FPGA dynamic control config */
|
||||
#define FPGA_CNFG_CTRL_IRQ_RESTART 0x100
|
||||
|
||||
/* TODO: Maximum transfer size */
|
||||
#define XDMA_TRANSFER_MAX_BYTES (8 * 1024 * 1024)
|
||||
|
||||
/* TLB */
|
||||
#define TLB_VADDR_RANGE 48
|
||||
#define TLB_PADDR_RANGE 40
|
||||
#define PID_SIZE 6
|
||||
#define MAX_MAP_AXIL_PAGES 64
|
||||
|
||||
#define LTLB_PAGE_BITS 21
|
||||
#define STLB_PAGE_BITS 12
|
||||
#define LTLB_PADDR_SIZE (TLB_PADDR_RANGE - LTLB_PAGE_BITS)
|
||||
#define STLB_PADDR_SIZE (TLB_PADDR_RANGE - STLB_PAGE_BITS)
|
||||
|
||||
#define TLBF_CTRL_START 0x7
|
||||
#define TLBF_CTRL_ID_MASK 0xff
|
||||
#define TLBF_CTRL_ID_SHFT 0x3
|
||||
#define TLBF_STAT_DONE 0x1
|
||||
|
||||
/* Memory allocation */
|
||||
#define MAX_BUFF_NUM 64 // maximum number of huge pages allowed
|
||||
#define MAX_PR_BUFF_NUM 64 // maximum number of huge pages allowed
|
||||
#define MAX_N_MAP_PAGES 128 // TODO: link to params, max no pfault 1M
|
||||
#define MAX_N_MAP_HUGE_PAGES (16 * 512) // max no pfault 32M
|
||||
|
||||
/* Max card pages */
|
||||
#define N_LARGE_CHUNKS 512
|
||||
#define N_SMALL_CHUNKS (256 * 1024)
|
||||
#define MEM_SEP (N_SMALL_CHUNKS * (4 * 1024))
|
||||
#define MAX_N_REGIONS 16
|
||||
#define SMALL_CHUNK_ALLOC 0
|
||||
#define LARGE_CHUNK_ALLOC 1
|
||||
|
||||
/* PR */
|
||||
#define PR_CTRL_START_MIDDLE 0x3
|
||||
#define PR_CTRL_START_LAST 0x7
|
||||
#define PR_STAT_DONE 0x1
|
||||
|
||||
|
||||
/* FPGA config commands */
|
||||
|
||||
/**
|
||||
* Cdev
|
||||
*/
|
||||
|
||||
/* Major number */
|
||||
#define FPGA_MAJOR 0 // dynamic
|
||||
|
||||
/* MMAP */
|
||||
#define MMAP_CTRL 0x0
|
||||
#define MMAP_CNFG 0x1
|
||||
#define MMAP_CNFG_AVX 0x2
|
||||
#define MMAP_WB 0x3
|
||||
#define MMAP_BUFF 0x200
|
||||
#define MMAP_PR 0x400
|
||||
|
||||
/* IOCTL */
|
||||
#define IOCTL_ALLOC_HOST_USER_MEM _IOW('D', 1, unsigned long) // large pages (no hugepage support)
|
||||
#define IOCTL_FREE_HOST_USER_MEM _IOW('D', 2, unsigned long)
|
||||
#define IOCTL_ALLOC_HOST_PR_MEM _IOW('D', 3, unsigned long) // pr pages
|
||||
#define IOCTL_FREE_HOST_PR_MEM _IOW('D', 4, unsigned long)
|
||||
#define IOCTL_MAP_USER _IOW('D', 5, unsigned long) // map
|
||||
#define IOCTL_UNMAP_USER _IOW('D', 6, unsigned long)
|
||||
#define IOCTL_REGISTER_PID _IOW('D', 7, unsigned long) // register pid
|
||||
#define IOCTL_UNREGISTER_PID _IOW('D', 8, unsigned long)
|
||||
#define IOCTL_RECONFIG_LOAD _IOW('D', 9, unsigned long) // reconfiguration
|
||||
|
||||
#define IOCTL_ARP_LOOKUP _IOW('D', 10, unsigned long) // arp lookup
|
||||
#define IOCTL_SET_IP_ADDRESS _IOW('D', 11, unsigned long)
|
||||
#define IOCTL_SET_BOARD_NUM _IOW('D', 12, unsigned long)
|
||||
#define IOCTL_WRITE_CTX _IOW('D', 13, unsigned long) // qp context
|
||||
#define IOCTL_WRITE_CONN _IOW('D', 14, unsigned long) // qp connection
|
||||
#define IOCTL_SET_TCP_OFFS _IOW('D', 15, unsigned long) // tcp mem offsets
|
||||
|
||||
#define IOCTL_READ_CNFG _IOR('D', 32, unsigned long) // status cnfg
|
||||
#define IOCTL_NET_STATS _IOR('D', 33, unsigned long) // status network
|
||||
#define IOCTL_READ_ENG_STATUS _IOR('D', 35, unsigned long) // status engines
|
||||
|
||||
/* Hash */
|
||||
#define PR_HASH_TABLE_ORDER 8
|
||||
#define PR_BATCH_SIZE (2 * 1024 * 1024)
|
||||
|
||||
#define USER_HASH_TABLE_ORDER 8
|
||||
|
||||
/* PID */
|
||||
#define N_CPID_MAX 64
|
||||
#define WB_SIZE (2 * N_CPID_MAX * sizeof(uint32_t))
|
||||
#define N_WB_PAGES ((WB_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
|
||||
|
||||
/* Network */
|
||||
#define EN_LOWSPEED 0x5
|
||||
#define N_NET_STAT_REGS 9
|
||||
|
||||
/* Copy */
|
||||
#define MAX_USER_WORDS 32
|
||||
|
||||
/**
|
||||
* Reg maps
|
||||
*/
|
||||
|
||||
/* DMA engine reg map */
|
||||
struct engine_regs {
|
||||
uint32_t id;
|
||||
uint32_t ctrl;
|
||||
uint32_t ctrl_w1s;
|
||||
uint32_t ctrl_w1c;
|
||||
uint32_t rsrvd_1[12];
|
||||
|
||||
uint32_t status;
|
||||
uint32_t status_rc;
|
||||
uint32_t completed_desc_count;
|
||||
uint32_t alignments;
|
||||
uint32_t rsrvd_2[14]; // padding
|
||||
|
||||
uint32_t poll_mode_wb_lo;
|
||||
uint32_t poll_mode_wb_hi;
|
||||
uint32_t interrupt_enable_mask;
|
||||
uint32_t interrupt_enable_mask_w1s;
|
||||
uint32_t interrupt_enable_mask_w1c;
|
||||
uint32_t rsrvd_3[9]; // padding
|
||||
|
||||
uint32_t perf_ctrl;
|
||||
uint32_t perf_cyc_lo;
|
||||
uint32_t perf_cyc_hi;
|
||||
uint32_t perf_dat_lo;
|
||||
uint32_t perf_dat_hi;
|
||||
uint32_t perf_pnd_lo;
|
||||
uint32_t perf_pnd_hi;
|
||||
} __packed;
|
||||
|
||||
/* Interrupt reg map */
|
||||
struct interrupt_regs {
|
||||
uint32_t id;
|
||||
uint32_t user_int_enable;
|
||||
uint32_t user_int_enable_w1s;
|
||||
uint32_t user_int_enable_w1c;
|
||||
uint32_t channel_int_enable;
|
||||
uint32_t channel_int_enable_w1s;
|
||||
uint32_t channel_int_enable_w1c;
|
||||
uint32_t reserved_1[9]; // padding
|
||||
|
||||
uint32_t user_int_request;
|
||||
uint32_t channel_int_request;
|
||||
uint32_t user_int_pending;
|
||||
uint32_t channel_int_pending;
|
||||
uint32_t reserved_2[12]; // padding
|
||||
|
||||
uint32_t user_msi_vector[8];
|
||||
uint32_t channel_msi_vector[8];
|
||||
} __packed;
|
||||
|
||||
/* Polled mode descriptors struct */
|
||||
struct xdma_poll_wb {
|
||||
uint32_t completed_desc_count;
|
||||
uint32_t reserved_1[7];
|
||||
} __packed;
|
||||
|
||||
/* FPGA static config reg map */
|
||||
struct fpga_stat_cnfg_regs {
|
||||
uint64_t probe;
|
||||
uint64_t n_chan;
|
||||
uint64_t n_regions;
|
||||
uint64_t ctrl_cnfg;
|
||||
uint64_t mem_cnfg;
|
||||
uint64_t pr_cnfg;
|
||||
uint64_t rdma_cnfg;
|
||||
uint64_t tcp_cnfg;
|
||||
uint64_t lspeed_cnfg;
|
||||
uint64_t reserved_0[1];
|
||||
uint64_t pr_ctrl;
|
||||
uint64_t pr_stat;
|
||||
uint64_t pr_addr;
|
||||
uint64_t pr_len;
|
||||
uint64_t tlb_ctrl;
|
||||
uint64_t tlb_stat;
|
||||
uint64_t tlb_addr;
|
||||
uint64_t tlb_len;
|
||||
uint64_t reserved_1[2];
|
||||
uint64_t net_0_ip;
|
||||
uint64_t net_0_boardnum;
|
||||
uint64_t net_0_arp;
|
||||
uint64_t rdma_0_qp_ctx[3];
|
||||
uint64_t rdma_0_qp_conn[3];
|
||||
uint64_t tcp_0_offs[2];
|
||||
uint64_t net_0_debug[N_NET_STAT_REGS];
|
||||
uint64_t net_1_ip;
|
||||
uint64_t net_1_boardnum;
|
||||
uint64_t net_1_arp;
|
||||
uint64_t rdma_1_qp_ctx[3];
|
||||
uint64_t rdma_1_qp_conn[3];
|
||||
uint64_t tcp_1_offs[2];
|
||||
uint64_t net_1_debug[N_NET_STAT_REGS];
|
||||
} __packed;
|
||||
|
||||
/* FPGA dynamic config reg map */
|
||||
struct fpga_cnfg_regs {
|
||||
uint64_t ctrl;
|
||||
uint64_t vaddr_rd;
|
||||
uint64_t len_rd;
|
||||
uint64_t vaddr_wr;
|
||||
uint64_t len_wr;
|
||||
uint64_t vaddr_miss;
|
||||
uint64_t len_miss;
|
||||
uint64_t datapath_set;
|
||||
uint64_t datapath_clr;
|
||||
uint64_t stat_cmd_used_rd;
|
||||
uint64_t stat_cmd_used_wr;
|
||||
uint64_t stat_sent[6];
|
||||
uint64_t stat_pfaults;
|
||||
uint64_t wback_rd;
|
||||
uint64_t wback_wr;
|
||||
// Rest of regs not used in the driver
|
||||
} __packed;
|
||||
|
||||
/* FPGA dynamic config reg map */
|
||||
struct fpga_cnfg_regs_avx {
|
||||
uint64_t ctrl[4];
|
||||
uint64_t vaddr_miss;
|
||||
uint64_t len_miss;
|
||||
uint64_t pf[2];
|
||||
uint64_t datapath_set[4];
|
||||
uint64_t datapath_clr[4];
|
||||
uint64_t stat[4];
|
||||
uint64_t wback[4];
|
||||
// Rest not used in the driver
|
||||
} __packed;
|
||||
|
||||
/**
|
||||
* Structs
|
||||
*/
|
||||
|
||||
/* Engine descriptors */
|
||||
struct xdma_sgdma_regs {
|
||||
uint32_t identifier;
|
||||
uint32_t reserved_1[31]; /* padding */
|
||||
|
||||
/* bus address to first descriptor in Root Complex Memory */
|
||||
uint32_t first_desc_lo;
|
||||
uint32_t first_desc_hi;
|
||||
/* number of adjacent descriptors at first_desc */
|
||||
uint32_t first_desc_adjacent;
|
||||
uint32_t credits;
|
||||
} __packed;
|
||||
|
||||
/* Engine struct */
|
||||
struct xdma_engine {
|
||||
int channel; // egnine channel
|
||||
char *name; // engine name
|
||||
struct pci_drvdata *pd; // PCI device
|
||||
|
||||
struct engine_regs *regs; // HW regs, control and status
|
||||
struct engine_sgdma_regs *sgdma_regs; // SGDMA reg BAR offset
|
||||
|
||||
// Config
|
||||
int running; // engine state
|
||||
int c2h; // c2h(write) or h2c(read)
|
||||
uint32_t status;
|
||||
|
||||
int addr_align; // source/dest alignment in bytes
|
||||
int len_granularity; // transfer length multiple
|
||||
int addr_bits; // HW datapath address width
|
||||
|
||||
/* Members associated with polled mode support */
|
||||
uint8_t *poll_mode_addr_virt; /* virt addr for descriptor writeback */
|
||||
uint64_t poll_mode_phys_addr; /* bus addr for descriptor writeback */
|
||||
};
|
||||
|
||||
/* Mapped user pages */
|
||||
struct user_pages {
|
||||
struct hlist_node entry;
|
||||
uint64_t vaddr;
|
||||
bool huge;
|
||||
int32_t cpid;
|
||||
uint64_t n_hpages;
|
||||
uint64_t n_pages;
|
||||
struct page **hpages;
|
||||
uint64_t *cpages;
|
||||
};
|
||||
/* User tables */
|
||||
struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
|
||||
struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
|
||||
|
||||
/* Mapped large PR pages */
|
||||
struct pr_pages {
|
||||
struct hlist_node entry;
|
||||
int reg_id;
|
||||
uint64_t vaddr;
|
||||
uint64_t n_pages;
|
||||
struct page **pages;
|
||||
};
|
||||
/* PR table */
|
||||
struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
|
||||
|
||||
/* Pool chunks */
|
||||
struct chunk {
|
||||
uint32_t id;
|
||||
struct chunk *next;
|
||||
};
|
||||
|
||||
/* Virtual FPGA device */
|
||||
struct fpga_dev {
|
||||
int id; // identifier
|
||||
struct cdev cdev; // char device
|
||||
struct pci_drvdata *pd; // PCI device
|
||||
struct pr_ctrl *prc; // PR controller
|
||||
|
||||
// Current task
|
||||
struct task_struct *curr_task;
|
||||
struct mm_struct *curr_mm;
|
||||
|
||||
// Control region
|
||||
uint64_t fpga_phys_addr_ctrl;
|
||||
uint64_t fpga_phys_addr_ctrl_avx;
|
||||
|
||||
// Writeback
|
||||
uint32_t *wb_addr_virt;
|
||||
uint64_t wb_phys_addr;
|
||||
|
||||
// TLBs
|
||||
uint64_t *fpga_lTlb; // large page TLB
|
||||
uint64_t *fpga_sTlb; // small page TLB
|
||||
struct fpga_cnfg_regs *fpga_cnfg; // config
|
||||
struct fpga_cnfg_regs_avx *fpga_cnfg_avx; // config AVX
|
||||
|
||||
// PIDs
|
||||
spinlock_t card_pid_lock;
|
||||
struct chunk *pid_chunks;
|
||||
pid_t *pid_array;
|
||||
int num_free_pid_chunks;
|
||||
struct chunk *pid_alloc;
|
||||
|
||||
// Engines
|
||||
struct xdma_engine *engine_h2c; // h2c engine
|
||||
struct xdma_engine *engine_c2h; // c2h engine
|
||||
|
||||
// In use
|
||||
atomic_t in_use; // busy flag
|
||||
|
||||
// Lock
|
||||
spinlock_t lock; // protects concurrent accesses
|
||||
|
||||
// Allocated buffers
|
||||
struct user_pages curr_user_buff;
|
||||
};
|
||||
|
||||
/* PR controller */
|
||||
struct pr_ctrl {
|
||||
struct pci_drvdata *pd; // PCI device
|
||||
spinlock_t lock;
|
||||
|
||||
// Engines
|
||||
struct xdma_engine *engine_h2c; // h2c engine
|
||||
struct xdma_engine *engine_c2h; // c2h engine
|
||||
|
||||
// Allocated buffers
|
||||
struct pr_pages curr_buff;
|
||||
};
|
||||
|
||||
/* TLB order */
|
||||
struct tlb_order {
|
||||
bool hugepage;
|
||||
uint64_t page_shift;
|
||||
uint64_t page_size;
|
||||
uint64_t page_mask;
|
||||
int assoc;
|
||||
|
||||
int key_mask;
|
||||
int key_size;
|
||||
int tag_mask;
|
||||
int tag_size;
|
||||
int phy_mask;
|
||||
int phy_size;
|
||||
};
|
||||
|
||||
/* PCI driver data */
|
||||
struct pci_drvdata {
|
||||
struct pci_dev *pci_dev;
|
||||
|
||||
// BARs
|
||||
int regions_in_use;
|
||||
int got_regions;
|
||||
void *__iomem bar[MAX_NUM_BARS];
|
||||
unsigned long bar_phys_addr[MAX_NUM_BARS];
|
||||
unsigned long bar_len[MAX_NUM_BARS];
|
||||
|
||||
// Engines
|
||||
int engines_num;
|
||||
|
||||
// FPGA static config
|
||||
uint probe;
|
||||
int n_fpga_chan;
|
||||
int n_fpga_reg;
|
||||
int en_avx;
|
||||
int en_bypass;
|
||||
int en_tlbf;
|
||||
int en_wb;
|
||||
int en_ddr;
|
||||
int en_hbm;
|
||||
int en_mem;
|
||||
int n_ddr_chan;
|
||||
int en_pr;
|
||||
int en_rdma_0;
|
||||
int en_rdma_1;
|
||||
int en_tcp_0;
|
||||
int en_tcp_1;
|
||||
int en_net_0;
|
||||
int en_net_1;
|
||||
struct fpga_stat_cnfg_regs *fpga_stat_cnfg;
|
||||
struct fpga_dev *fpga_dev;
|
||||
|
||||
// PR control
|
||||
struct pr_ctrl prc;
|
||||
|
||||
// TLB order
|
||||
struct tlb_order *stlb_order;
|
||||
struct tlb_order *ltlb_order;
|
||||
|
||||
// Locks
|
||||
spinlock_t stat_lock;
|
||||
spinlock_t prc_lock;
|
||||
spinlock_t tlb_lock;
|
||||
|
||||
// IRQ
|
||||
int irq_count;
|
||||
int irq_line;
|
||||
int msix_enabled;
|
||||
struct msix_entry irq_entry[32];
|
||||
|
||||
// Card memory
|
||||
spinlock_t card_l_lock;
|
||||
struct chunk *lchunks;
|
||||
int num_free_lchunks;
|
||||
struct chunk *lalloc;
|
||||
|
||||
spinlock_t card_s_lock;
|
||||
struct chunk *schunks;
|
||||
int num_free_schunks;
|
||||
struct chunk *salloc;
|
||||
};
|
||||
|
||||
#endif
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __FPGA_DRV_H__
|
||||
#define __FPGA_DRV_H__
|
||||
|
||||
#include "coyote_dev.h"
|
||||
#include "fpga_dev.h"
|
||||
#include "pci/pci_dev.h"
|
||||
#include "eci/eci_dev.h"
|
||||
|
||||
/* Main */
|
||||
static int __init coyote_init(void);
|
||||
static void __exit coyote_exit(void);
|
||||
|
||||
#endif /* Coyote driver */
|
|
@ -0,0 +1,791 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "fpga_fops.h"
|
||||
|
||||
/*
|
||||
_____
|
||||
| ___|__ _ __ ___
|
||||
| |_ / _ \| '_ \/ __|
|
||||
| _| (_) | |_) \__ \
|
||||
|_| \___/| .__/|___/
|
||||
|_|
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Acquire a region
|
||||
*
|
||||
*/
|
||||
int fpga_open(struct inode *inode, struct file *file)
|
||||
{
|
||||
int minor = iminor(inode);
|
||||
|
||||
struct fpga_dev *d = container_of(inode->i_cdev, struct fpga_dev, cdev);
|
||||
BUG_ON(!d);
|
||||
|
||||
dbg_info("fpga device %d acquired\n", minor);
|
||||
|
||||
// set private data
|
||||
file->private_data = (void *)d;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Release a region
|
||||
*
|
||||
*/
|
||||
int fpga_release(struct inode *inode, struct file *file)
|
||||
{
|
||||
int minor = iminor(inode);
|
||||
|
||||
struct fpga_dev *d = container_of(inode->i_cdev, struct fpga_dev, cdev);
|
||||
BUG_ON(!d);
|
||||
|
||||
// unamp all user pages
|
||||
tlb_put_user_pages_all(d, 1);
|
||||
|
||||
dbg_info("fpga device %d released\n", minor);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief XDMA engine status
|
||||
*
|
||||
*/
|
||||
uint32_t engine_status_read(struct xdma_engine *engine)
|
||||
{
|
||||
uint32_t val;
|
||||
|
||||
BUG_ON(!engine);
|
||||
|
||||
dbg_info("engine %s status:\n", engine->name);
|
||||
val = ioread32(&engine->regs->status);
|
||||
dbg_info("status = 0x%08x: %s%s%s%s%s%s%s%s%s\n", (uint32_t)val,
|
||||
(val & XDMA_STAT_BUSY) ? "BUSY " : "IDLE ",
|
||||
(val & XDMA_STAT_DESC_STOPPED) ? "DESC_STOPPED " : "",
|
||||
(val & XDMA_STAT_DESC_COMPLETED) ? "DESC_COMPLETED " : "",
|
||||
(val & XDMA_STAT_ALIGN_MISMATCH) ? "ALIGN_MISMATCH " : "",
|
||||
(val & XDMA_STAT_MAGIC_STOPPED) ? "MAGIC_STOPPED " : "",
|
||||
(val & XDMA_STAT_FETCH_STOPPED) ? "FETCH_STOPPED " : "",
|
||||
(val & XDMA_STAT_READ_ERROR) ? "READ_ERROR " : "",
|
||||
(val & XDMA_STAT_DESC_ERROR) ? "DESC_ERROR " : "",
|
||||
(val & XDMA_STAT_IDLE_STOPPED) ? "IDLE_STOPPED " : "");
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief ioctl, control and status
|
||||
*
|
||||
*/
|
||||
long fpga_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
|
||||
{
|
||||
int ret_val, i;
|
||||
uint64_t tmp[MAX_USER_WORDS];
|
||||
uint64_t cpid;
|
||||
|
||||
struct fpga_dev *d = (struct fpga_dev *)file->private_data;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
switch (cmd) {
|
||||
|
||||
// allocate host memory (2MB pages) - no hugepages support
|
||||
case IOCTL_ALLOC_HOST_USER_MEM:
|
||||
// read n_pages + cpid
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
ret_val = alloc_user_buffers(d, tmp[0], (uint32_t)tmp[1]);
|
||||
dbg_info("buff_num %lld, arg %lx\n", d->curr_user_buff.n_hpages, arg);
|
||||
if (ret_val != 0) {
|
||||
pr_info("user buffers could not be allocated\n");
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
// free host memory (2MB pages) - no hugepages support
|
||||
case IOCTL_FREE_HOST_USER_MEM:
|
||||
// read vaddr + cpid
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
ret_val = free_user_buffers(d, tmp[0], (uint32_t)tmp[1]);
|
||||
dbg_info("user buffers freed\n");
|
||||
}
|
||||
break;
|
||||
|
||||
// allocate host memory (2MB pages) - bitstream memory
|
||||
case IOCTL_ALLOC_HOST_PR_MEM:
|
||||
// read n_pages
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
ret_val = alloc_pr_buffers(d, tmp[0]);
|
||||
dbg_info("buff_num %lld, arg %lx\n", d->prc->curr_buff.n_pages, arg);
|
||||
if (ret_val != 0) {
|
||||
pr_info("PR buffers could not be allocated\n");
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
// free host memory (2MB pages) - bitstream memory
|
||||
case IOCTL_FREE_HOST_PR_MEM:
|
||||
// read vaddr
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
ret_val = free_pr_buffers(d, tmp[0]);
|
||||
dbg_info("PR buffers freed\n");
|
||||
}
|
||||
break;
|
||||
|
||||
// explicit mapping
|
||||
case IOCTL_MAP_USER:
|
||||
// read vaddr + len + cpid
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 3 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
cpid = (uint32_t)tmp[2];
|
||||
tlb_get_user_pages(d, tmp[0], tmp[1], (int32_t)tmp[2], d->pid_array[cpid]);
|
||||
}
|
||||
break;
|
||||
|
||||
// explicit unmapping
|
||||
case IOCTL_UNMAP_USER:
|
||||
// read vaddr + cpid
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("unmapping user pages\n");
|
||||
tlb_put_user_pages(d, tmp[0], (int32_t)tmp[1], 1);
|
||||
}
|
||||
break;
|
||||
|
||||
// register pid
|
||||
case IOCTL_REGISTER_PID:
|
||||
// read pid
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
cpid = (uint64_t)register_pid(d, tmp[0]);
|
||||
if (cpid == -1)
|
||||
{
|
||||
dbg_info("registration failed pid %lld\n", tmp[0]);
|
||||
return -1;
|
||||
}
|
||||
|
||||
dbg_info("registration succeeded pid %lld, cpid %lld\n", tmp[0], cpid);
|
||||
ret_val = copy_to_user((unsigned long *)arg + 1, &cpid, sizeof(unsigned long));
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
}
|
||||
break;
|
||||
|
||||
// unregister pid
|
||||
case IOCTL_UNREGISTER_PID:
|
||||
// read cpid
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
ret_val = unregister_pid(d, tmp[0]);
|
||||
if (ret_val == -1) {
|
||||
dbg_info("unregistration failed cpid %lld\n", tmp[0]);
|
||||
return -1;
|
||||
}
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
dbg_info("unregistration succeeded cpid %lld\n", tmp[0]);
|
||||
}
|
||||
break;
|
||||
|
||||
// reconfiguration
|
||||
case IOCTL_RECONFIG_LOAD:
|
||||
// read vaddr + len
|
||||
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("trying to obtain reconfig lock\n");
|
||||
spin_lock(&d->prc->lock);
|
||||
if (pd->en_avx)
|
||||
d->fpga_cnfg_avx->datapath_set[0] = 0x1;
|
||||
else
|
||||
d->fpga_cnfg->datapath_set = 0x1;
|
||||
|
||||
ret_val = reconfigure(d, tmp[0], tmp[1]);
|
||||
if (ret_val != 0) {
|
||||
pr_info("reconfiguration failed, return %d\n", ret_val);
|
||||
return -1;
|
||||
}
|
||||
else {
|
||||
dbg_info("reconfiguration successfull\n");
|
||||
}
|
||||
|
||||
dbg_info("releasing reconfig lock, coupling the design\n");
|
||||
if (pd->en_avx)
|
||||
d->fpga_cnfg_avx->datapath_clr[0] = 0x1;
|
||||
else
|
||||
d->fpga_cnfg->datapath_clr = 0x1;
|
||||
|
||||
spin_unlock(&d->prc->lock);
|
||||
}
|
||||
break;
|
||||
|
||||
// arp lookup
|
||||
case IOCTL_ARP_LOOKUP:
|
||||
if (pd->en_net_0 || pd->en_net_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("arp lookup ...");
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
for (i = 0; i < N_TOTAL_NODES; i++) {
|
||||
if (i == NODE_ID)
|
||||
continue;
|
||||
tmp[0] ? (pd->fpga_stat_cnfg->net_1_arp = BASE_IP_ADDR_1 + i) :
|
||||
(pd->fpga_stat_cnfg->net_0_arp = BASE_IP_ADDR_0 + i);
|
||||
}
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
}
|
||||
} else {
|
||||
pr_info("network not enabled\n");
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
|
||||
// set ip address
|
||||
case IOCTL_SET_IP_ADDRESS:
|
||||
if (pd->en_net_0 || pd->en_net_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 2 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
}
|
||||
else {
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
// Set ip
|
||||
tmp[0] ? (pd->fpga_stat_cnfg->net_1_ip = tmp[1]) :
|
||||
(pd->fpga_stat_cnfg->net_0_ip = tmp[1]);
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
dbg_info("ip address changed to %llx\n", tmp[1]);
|
||||
}
|
||||
} else {
|
||||
pr_info("network not enabled\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
// set board number
|
||||
case IOCTL_SET_BOARD_NUM:
|
||||
if (pd->en_net_0 || pd->en_net_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 2 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
pr_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
// Set board number
|
||||
tmp[0] ? (pd->fpga_stat_cnfg->net_1_boardnum = tmp[1]) :
|
||||
(pd->fpga_stat_cnfg->net_0_boardnum = tmp[1]);
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
dbg_info("board number changed to %llx\n", tmp[1]);
|
||||
}
|
||||
} else {
|
||||
pr_info("network not enabled\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
// rdma context
|
||||
case IOCTL_WRITE_CTX:
|
||||
if (pd->en_rdma_0 || pd->en_rdma_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 4 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
dbg_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("writing qp context ...");
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
for (i = 0; i < 3; i++) {
|
||||
tmp[0] ? (pd->fpga_stat_cnfg->rdma_1_qp_ctx[i] = tmp[i+1]) :
|
||||
(pd->fpga_stat_cnfg->rdma_0_qp_ctx[i] = tmp[i+1]);
|
||||
}
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
}
|
||||
}
|
||||
else {
|
||||
dbg_info("RDMA not enabled\n");
|
||||
}
|
||||
break;
|
||||
|
||||
// rdma connection
|
||||
case IOCTL_WRITE_CONN:
|
||||
if (pd->en_rdma_0 || pd->en_rdma_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 4 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
dbg_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("writing qp connection ...");
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
for (i = 0; i < 3; i++) {
|
||||
tmp[0] ? (pd->fpga_stat_cnfg->rdma_1_qp_conn[i] = tmp[i+1]) :
|
||||
(pd->fpga_stat_cnfg->rdma_0_qp_conn[i] = tmp[i+1]);
|
||||
}
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
}
|
||||
}
|
||||
else {
|
||||
dbg_info("RDMA not enabled\n");
|
||||
}
|
||||
break;
|
||||
|
||||
// tcp offsets
|
||||
case IOCTL_SET_TCP_OFFS:
|
||||
if (pd->en_tcp_0 || pd->en_tcp_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 3 * sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
dbg_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("writing tcp mem offsets ...");
|
||||
spin_lock(&pd->stat_lock);
|
||||
|
||||
for (i = 0; i < 2; i++) {
|
||||
tmp[0] ? (pd->fpga_stat_cnfg->tcp_1_offs[i] = tmp[i+1]) :
|
||||
(pd->fpga_stat_cnfg->tcp_0_offs[i] = tmp[i+1]);
|
||||
}
|
||||
|
||||
spin_unlock(&pd->stat_lock);
|
||||
}
|
||||
}
|
||||
else {
|
||||
dbg_info("TCP/IP not enabled\n");
|
||||
}
|
||||
break;
|
||||
|
||||
// read config
|
||||
case IOCTL_READ_CNFG:
|
||||
tmp[0] = ((uint64_t)pd->n_fpga_chan << 32) | ((uint64_t)pd->n_fpga_reg << 40) | ((uint64_t)pd->n_mem_chan << 48) |
|
||||
((uint64_t)pd->en_avx) | ((uint64_t)pd->en_bypass << 1) | ((uint64_t)pd->en_tlbf << 2) | ((uint64_t)pd->en_wb << 3) |
|
||||
((uint64_t)pd->en_mem << 4) | ((uint64_t)pd->en_pr << 5) |
|
||||
((uint64_t)pd->en_rdma_0 << 6) | ((uint64_t)pd->en_rdma_1 << 7) | ((uint64_t)pd->en_tcp_0 << 8) | ((uint64_t)pd->en_tcp_1 << 9);
|
||||
dbg_info("reading config %llx\n", tmp[0]);
|
||||
ret_val = copy_to_user((unsigned long *)arg, &tmp, sizeof(unsigned long));
|
||||
break;
|
||||
|
||||
// network status
|
||||
case IOCTL_NET_STATS:
|
||||
if (pd->en_net_0 || pd->en_net_1) {
|
||||
ret_val = copy_from_user(&tmp, (unsigned long*) arg, sizeof(unsigned long));
|
||||
if (ret_val != 0) {
|
||||
dbg_info("user data could not be coppied, return %d\n", ret_val);
|
||||
} else {
|
||||
dbg_info("retreiving network status for port %llx", tmp[0]);
|
||||
if(tmp[0]) {
|
||||
for (i = 0; i < N_NET_STAT_REGS; i++) {
|
||||
tmp[i] = pd->fpga_stat_cnfg->net_1_debug[i];
|
||||
}
|
||||
} else {
|
||||
for (i = 0; i < N_NET_STAT_REGS; i++) {
|
||||
tmp[i] = pd->fpga_stat_cnfg->net_0_debug[i];
|
||||
}
|
||||
}
|
||||
|
||||
ret_val = copy_to_user((unsigned long *)arg, &tmp, N_NET_STAT_REGS * sizeof(unsigned long));
|
||||
}
|
||||
}
|
||||
else {
|
||||
dbg_info("network not enabled\n");
|
||||
}
|
||||
break;
|
||||
|
||||
// engine status
|
||||
case IOCTL_READ_ENG_STATUS:
|
||||
dbg_info("fpga dev %d engine report\n", d->id);
|
||||
engine_status_read(d->engine_c2h);
|
||||
engine_status_read(d->engine_h2c);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Mmap control and mem
|
||||
*
|
||||
*/
|
||||
int fpga_mmap(struct file *file, struct vm_area_struct *vma)
|
||||
{
|
||||
int i;
|
||||
unsigned long vaddr;
|
||||
unsigned long vaddr_tmp;
|
||||
struct fpga_dev *d;
|
||||
struct pr_ctrl *prc;
|
||||
struct user_pages *new_user_buff;
|
||||
struct pr_pages *new_pr_buff;
|
||||
struct bus_drvdata *pd;
|
||||
uint64_t *map_array;
|
||||
|
||||
|
||||
d = (struct fpga_dev *)file->private_data;
|
||||
BUG_ON(!d);
|
||||
prc = d->prc;
|
||||
BUG_ON(!prc);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
vaddr = vma->vm_start;
|
||||
|
||||
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
|
||||
|
||||
// map user ctrl region
|
||||
if (vma->vm_pgoff == MMAP_CTRL) {
|
||||
dbg_info("fpga dev. %d, memory mapping user ctrl region at %llx of size %x\n",
|
||||
d->id, d->fpga_phys_addr_ctrl + FPGA_CTRL_USER_OFFS, FPGA_CTRL_USER_SIZE);
|
||||
if (remap_pfn_range(vma, vma->vm_start, (d->fpga_phys_addr_ctrl + FPGA_CTRL_USER_OFFS) >> PAGE_SHIFT,
|
||||
FPGA_CTRL_USER_SIZE, vma->vm_page_prot)) {
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// map cnfg region
|
||||
if (vma->vm_pgoff == MMAP_CNFG) {
|
||||
dbg_info("fpga dev. %d, memory mapping config region at %llx of size %x\n",
|
||||
d->id, d->fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS, FPGA_CTRL_CNFG_SIZE);
|
||||
if (remap_pfn_range(vma, vma->vm_start, (d->fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS) >> PAGE_SHIFT,
|
||||
FPGA_CTRL_CNFG_SIZE, vma->vm_page_prot)) {
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// map cnfg AVX region
|
||||
if (vma->vm_pgoff == MMAP_CNFG_AVX) {
|
||||
dbg_info("fpga dev. %d, memory mapping config AVX region at %llx of size %x\n",
|
||||
d->id, d->fpga_phys_addr_ctrl_avx, FPGA_CTRL_CNFG_AVX_SIZE);
|
||||
if (remap_pfn_range(vma, vma->vm_start, d->fpga_phys_addr_ctrl_avx >> PAGE_SHIFT,
|
||||
FPGA_CTRL_CNFG_AVX_SIZE, vma->vm_page_prot)) {
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// map writeback
|
||||
if (vma->vm_pgoff == MMAP_WB) {
|
||||
set_memory_uc((uint64_t)d->wb_addr_virt, N_WB_PAGES);
|
||||
dbg_info("fpga dev. %d, memory mapping writeback regions at %llx of size %lx\n",
|
||||
d->id, d->wb_phys_addr, WB_SIZE);
|
||||
if (remap_pfn_range(vma, vma->vm_start, (d->wb_phys_addr) >> PAGE_SHIFT,
|
||||
WB_SIZE, vma->vm_page_prot)) {
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
// map user buffers
|
||||
if (vma->vm_pgoff == MMAP_BUFF) {
|
||||
dbg_info("fpga dev. %d, memory mapping buffer\n", d->id);
|
||||
|
||||
// aligned page virtual address
|
||||
vaddr = ((vma->vm_start + pd->ltlb_order->page_size - 1) >> pd->ltlb_order->page_shift) << pd->ltlb_order->page_shift;
|
||||
vaddr_tmp = vaddr;
|
||||
|
||||
if (d->curr_user_buff.n_hpages != 0) {
|
||||
|
||||
new_user_buff = kzalloc(sizeof(struct user_pages), GFP_KERNEL);
|
||||
BUG_ON(!new_user_buff);
|
||||
|
||||
// Map entry
|
||||
new_user_buff->vaddr = vaddr;
|
||||
new_user_buff->huge = d->curr_user_buff.huge;
|
||||
new_user_buff->cpid = d->curr_user_buff.cpid;
|
||||
new_user_buff->n_hpages = d->curr_user_buff.n_hpages;
|
||||
new_user_buff->n_pages = d->curr_user_buff.n_pages;
|
||||
new_user_buff->hpages = d->curr_user_buff.hpages;
|
||||
new_user_buff->cpages = d->curr_user_buff.cpages;
|
||||
|
||||
hash_add(user_lbuff_map[d->id], &new_user_buff->entry, vaddr);
|
||||
|
||||
for (i = 0; i < d->curr_user_buff.n_hpages; i++) {
|
||||
// map to user space
|
||||
if (remap_pfn_range(vma, vaddr_tmp, page_to_pfn(d->curr_user_buff.hpages[i]),
|
||||
pd->ltlb_order->page_size, vma->vm_page_prot)) {
|
||||
return -EIO;
|
||||
}
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// reset
|
||||
vaddr_tmp = vaddr;
|
||||
|
||||
// map array
|
||||
map_array = (uint64_t *)kzalloc(d->curr_user_buff.n_hpages * 2 * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (map_array == NULL) {
|
||||
dbg_info("map buffers could not be allocated\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
// fill mappings
|
||||
for (i = 0; i < d->curr_user_buff.n_hpages; i++) {
|
||||
tlb_create_map(pd->ltlb_order, vaddr_tmp, page_to_phys(d->curr_user_buff.hpages[i]), (pd->en_mem ? d->curr_user_buff.cpages[i] : 0), d->curr_user_buff.cpid, &map_array[2*i]);
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->ltlb_order, map_array, d->curr_user_buff.n_hpages);
|
||||
|
||||
// free
|
||||
kfree((void *)map_array);
|
||||
|
||||
// Current host buff empty
|
||||
d->curr_user_buff.n_hpages = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// map PR buffers
|
||||
if (vma->vm_pgoff == MMAP_PR)
|
||||
{
|
||||
dbg_info("fpga dev. %d, memory mapping PR buffer\n", d->id);
|
||||
|
||||
// aligned page virtual address
|
||||
vaddr = ((vma->vm_start + pd->ltlb_order->page_size - 1) >> pd->ltlb_order->page_shift) << pd->ltlb_order->page_shift;
|
||||
vaddr_tmp = vaddr;
|
||||
|
||||
if (prc->curr_buff.n_pages != 0) {
|
||||
// obtain PR lock
|
||||
spin_lock(&prc->lock);
|
||||
|
||||
new_pr_buff = kzalloc(sizeof(struct pr_pages), GFP_KERNEL);
|
||||
BUG_ON(!new_pr_buff);
|
||||
|
||||
// Map entry
|
||||
new_pr_buff->vaddr = vaddr;
|
||||
new_pr_buff->reg_id = d->id;
|
||||
new_pr_buff->n_pages = prc->curr_buff.n_pages;
|
||||
new_pr_buff->pages = prc->curr_buff.pages;
|
||||
|
||||
hash_add(pr_buff_map, &new_pr_buff->entry, vaddr);
|
||||
|
||||
for (i = 0; i < prc->curr_buff.n_pages; i++) {
|
||||
// map to user space
|
||||
if (remap_pfn_range(vma, vaddr_tmp, page_to_pfn(prc->curr_buff.pages[i]),
|
||||
pd->ltlb_order->page_size, vma->vm_page_prot)) {
|
||||
return -EIO;
|
||||
}
|
||||
// next page vaddr
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// Current host buff empty
|
||||
prc->curr_buff.n_pages = 0;
|
||||
|
||||
// release PR lock
|
||||
spin_unlock(&prc->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
_ _ _ _ _
|
||||
| | | | |_(_) |
|
||||
| | | | __| | |
|
||||
| |_| | |_| | |
|
||||
\___/ \__|_|_|
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Register PID
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param pid - user PID
|
||||
* @return int32_t - Coyote PID
|
||||
*/
|
||||
int32_t register_pid(struct fpga_dev *d, pid_t pid)
|
||||
{
|
||||
int32_t cpid;
|
||||
|
||||
BUG_ON(!d);
|
||||
|
||||
// lock
|
||||
spin_lock(&d->card_pid_lock);
|
||||
|
||||
if(d->num_free_pid_chunks == 0) {
|
||||
dbg_info("not enough CPID slots\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
cpid = (int32_t)d->pid_alloc->id;
|
||||
d->pid_array[d->pid_alloc->id] = pid;
|
||||
d->pid_alloc = d->pid_alloc->next;
|
||||
|
||||
// unlock
|
||||
spin_unlock(&d->card_pid_lock);
|
||||
|
||||
return cpid;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Unregister Coyote PID
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param cpid - Coyote PID
|
||||
*/
|
||||
int unregister_pid(struct fpga_dev *d, int32_t cpid)
|
||||
{
|
||||
BUG_ON(!d);
|
||||
|
||||
// lock
|
||||
spin_lock(&d->card_pid_lock);
|
||||
|
||||
d->pid_chunks[cpid].next = d->pid_alloc;
|
||||
d->pid_alloc = &d->pid_chunks[cpid];
|
||||
|
||||
// release lock
|
||||
spin_unlock(&d->card_pid_lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
____________________
|
||||
\______ \______ \
|
||||
| ___/| _/
|
||||
| | | | \
|
||||
|____| |____|_ /
|
||||
\/
|
||||
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Reconfigure the vFPGA
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param vaddr - bitstream vaddr
|
||||
* @param len - bitstream length
|
||||
*/
|
||||
int reconfigure(struct fpga_dev *d, uint64_t vaddr, uint64_t len)
|
||||
{
|
||||
struct pr_ctrl *prc;
|
||||
struct pr_pages *tmp_buff;
|
||||
int i;
|
||||
uint64_t fsz_m;
|
||||
uint64_t fsz_r;
|
||||
uint64_t pr_bsize = PR_BATCH_SIZE;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
prc = d->prc;
|
||||
BUG_ON(!prc);
|
||||
|
||||
// lock
|
||||
spin_lock(&pd->prc_lock);
|
||||
|
||||
hash_for_each_possible(pr_buff_map, tmp_buff, entry, vaddr) {
|
||||
if (tmp_buff->vaddr == vaddr && tmp_buff->reg_id == d->id) {
|
||||
// Reconfiguration
|
||||
fsz_m = len / pr_bsize;
|
||||
fsz_r = len % pr_bsize;
|
||||
dbg_info("bitstream full %lld, partial %lld\n", fsz_m, fsz_r);
|
||||
|
||||
// full
|
||||
for (i = 0; i < fsz_m; i++) {
|
||||
dbg_info("page %d, phys %llx, len %llx\n", i, page_to_phys(prc->curr_buff.pages[i]), pr_bsize);
|
||||
pd->fpga_stat_cnfg->pr_addr = page_to_phys(tmp_buff->pages[i]);
|
||||
pd->fpga_stat_cnfg->pr_len = pr_bsize;
|
||||
if (fsz_r == 0 && i == fsz_m - 1)
|
||||
pd->fpga_stat_cnfg->pr_ctrl = PR_CTRL_START_LAST;
|
||||
else
|
||||
pd->fpga_stat_cnfg->pr_ctrl = PR_CTRL_START_MIDDLE;
|
||||
}
|
||||
|
||||
// partial
|
||||
if (fsz_r > 0) {
|
||||
dbg_info("page %lld, phys %llx, len %llx\n", fsz_m, page_to_phys(prc->curr_buff.pages[fsz_m]), fsz_r);
|
||||
pd->fpga_stat_cnfg->pr_addr = page_to_phys(tmp_buff->pages[fsz_m]);
|
||||
pd->fpga_stat_cnfg->pr_len = fsz_r;
|
||||
pd->fpga_stat_cnfg->pr_ctrl = PR_CTRL_START_LAST;
|
||||
while ((pd->fpga_stat_cnfg->pr_stat & PR_STAT_DONE) != 0x1)
|
||||
ndelay(100);
|
||||
} else {
|
||||
while ((pd->fpga_stat_cnfg->pr_stat & PR_STAT_DONE) != 0x1)
|
||||
ndelay(100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unlock
|
||||
spin_unlock(&pd->prc_lock);
|
||||
|
||||
return 0;
|
||||
}
|
|
@ -0,0 +1,53 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __FPGA_FOPS_H__
|
||||
#define __FPGA_FOPS_H__
|
||||
|
||||
#include "coyote_dev.h"
|
||||
#include "fpga_mmu.h"
|
||||
|
||||
/* Pid */
|
||||
int32_t register_pid(struct fpga_dev *d, pid_t pid);
|
||||
int unregister_pid(struct fpga_dev *d, int32_t cpid);
|
||||
|
||||
/* Engine status read */
|
||||
uint32_t engine_status_read(struct xdma_engine *engine);
|
||||
|
||||
/* Reconfiguration */
|
||||
int reconfigure(struct fpga_dev *d, uint64_t vaddr, uint64_t len);
|
||||
int alloc_pr_buffers(struct fpga_dev *d, unsigned long n_pages);
|
||||
int free_pr_buffers(struct fpga_dev *d, uint64_t vaddr);
|
||||
|
||||
|
||||
/* Fops */
|
||||
int fpga_open(struct inode *inode, struct file *file);
|
||||
int fpga_release(struct inode *inode, struct file *file);
|
||||
int fpga_mmap(struct file *file, struct vm_area_struct *vma);
|
||||
long fpga_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
|
||||
|
||||
#endif // FPGA FOPS
|
|
@ -0,0 +1,101 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "fpga_isr.h"
|
||||
|
||||
/*
|
||||
___ ____ ____
|
||||
|_ _/ ___|| _ \
|
||||
| |\___ \| |_) |
|
||||
| | ___) | _ <
|
||||
|___|____/|_| \_\
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief TLB page fault handling
|
||||
*
|
||||
*/
|
||||
irqreturn_t fpga_tlb_miss_isr(int irq, void *dev_id)
|
||||
{
|
||||
unsigned long flags;
|
||||
uint64_t vaddr;
|
||||
uint32_t len;
|
||||
int32_t cpid;
|
||||
struct fpga_dev *d;
|
||||
struct bus_drvdata *pd;
|
||||
int ret_val = 0;
|
||||
pid_t pid;
|
||||
uint64_t tmp;
|
||||
|
||||
dbg_info("(irq=%d) page fault ISR\n", irq);
|
||||
BUG_ON(!dev_id);
|
||||
|
||||
BUG_ON(cyt_arch == CYT_ARCH_ECI);
|
||||
|
||||
d = (struct fpga_dev *)dev_id;
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
// lock
|
||||
spin_lock_irqsave(&(d->lock), flags);
|
||||
|
||||
// read page fault
|
||||
if (pd->en_avx) {
|
||||
vaddr = d->fpga_cnfg_avx->vaddr_miss;
|
||||
tmp = d->fpga_cnfg_avx->len_miss;
|
||||
len = LOW_32(tmp);
|
||||
cpid = (int32_t)HIGH_32(tmp);
|
||||
}
|
||||
else {
|
||||
vaddr = d->fpga_cnfg->vaddr_miss;
|
||||
tmp = d->fpga_cnfg->len_miss;
|
||||
len = LOW_32(tmp);
|
||||
cpid = (int32_t)HIGH_32(tmp);
|
||||
}
|
||||
dbg_info("page fault, vaddr %llx, length %x, cpid %d\n", vaddr, len, cpid);
|
||||
|
||||
// get user pages
|
||||
pid = d->pid_array[cpid];
|
||||
ret_val = tlb_get_user_pages(d, vaddr, len, cpid, pid);
|
||||
|
||||
if (ret_val > 0) {
|
||||
// restart the engine
|
||||
if (pd->en_avx)
|
||||
d->fpga_cnfg_avx->ctrl[0] = FPGA_CNFG_CTRL_IRQ_RESTART;
|
||||
else
|
||||
d->fpga_cnfg->ctrl = FPGA_CNFG_CTRL_IRQ_RESTART;
|
||||
}
|
||||
else {
|
||||
dbg_info("pages could not be obtained\n");
|
||||
}
|
||||
|
||||
// unlock
|
||||
spin_unlock_irqrestore(&(d->lock), flags);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __FPGA_ISR_H__
|
||||
#define __FPGA_ISR_H__
|
||||
|
||||
#include "coyote_dev.h"
|
||||
#include "fpga_mmu.h"
|
||||
|
||||
/* Interrupt service routine */
|
||||
irqreturn_t fpga_tlb_miss_isr(int irq, void *dev_id);
|
||||
|
||||
#endif // FPGA ISR
|
|
@ -0,0 +1,921 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "fpga_mmu.h"
|
||||
|
||||
/* User tables */
|
||||
struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
|
||||
struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
|
||||
|
||||
/* PR table */
|
||||
struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
|
||||
|
||||
/**
|
||||
* @brief ALlocate user buffers (used in systems without hugepage support)
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param n_pages - number of pages to allocate
|
||||
* @param cpid - Coyote PID
|
||||
*/
|
||||
int alloc_user_buffers(struct fpga_dev *d, unsigned long n_pages, int32_t cpid)
|
||||
{
|
||||
int i, ret_val = 0;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
if (d->curr_user_buff.n_hpages) {
|
||||
dbg_info("allocated user buffers exist and are not mapped\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// check host
|
||||
if (n_pages > MAX_BUFF_NUM)
|
||||
d->curr_user_buff.n_hpages = MAX_BUFF_NUM;
|
||||
else
|
||||
d->curr_user_buff.n_hpages = n_pages;
|
||||
|
||||
// check card
|
||||
if(pd->en_mem)
|
||||
if (d->curr_user_buff.n_hpages > pd->num_free_lchunks)
|
||||
return -ENOMEM;
|
||||
|
||||
d->curr_user_buff.huge = true;
|
||||
d->curr_user_buff.cpid = cpid;
|
||||
|
||||
// alloc host
|
||||
d->curr_user_buff.hpages = kzalloc(d->curr_user_buff.n_hpages * sizeof(*d->curr_user_buff.hpages), GFP_KERNEL);
|
||||
if (d->curr_user_buff.hpages == NULL) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
dbg_info("allocated %llu bytes for page pointer array for %lld user host buffers @0x%p.\n",
|
||||
d->curr_user_buff.n_hpages * sizeof(*d->curr_user_buff.hpages), d->curr_user_buff.n_hpages, d->curr_user_buff.hpages);
|
||||
|
||||
for (i = 0; i < d->curr_user_buff.n_hpages; i++) {
|
||||
d->curr_user_buff.hpages[i] = alloc_pages(GFP_ATOMIC, pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
if (!d->curr_user_buff.hpages[i]) {
|
||||
dbg_info("user host buffer %d could not be allocated\n", i);
|
||||
goto fail_host_alloc;
|
||||
}
|
||||
|
||||
dbg_info("user host buffer allocated @ %llx device %d\n", page_to_phys(d->curr_user_buff.hpages[i]), d->id);
|
||||
}
|
||||
|
||||
// alloc card
|
||||
if(pd->en_mem) {
|
||||
d->curr_user_buff.n_pages = d->curr_user_buff.n_hpages;
|
||||
d->curr_user_buff.cpages = kzalloc(d->curr_user_buff.n_pages * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (d->curr_user_buff.cpages == NULL) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
dbg_info("allocated %llu bytes for page pointer array for %lld user card buffers @0x%p.\n",
|
||||
d->curr_user_buff.n_pages * sizeof(*d->curr_user_buff.cpages), d->curr_user_buff.n_pages, d->curr_user_buff.cpages);
|
||||
|
||||
ret_val = card_alloc(d, d->curr_user_buff.cpages, d->curr_user_buff.n_pages, LARGE_CHUNK_ALLOC);
|
||||
if (ret_val) {
|
||||
dbg_info("user card buffer %d could not be allocated\n", i);
|
||||
goto fail_card_alloc;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
fail_host_alloc:
|
||||
while (i)
|
||||
__free_pages(d->curr_user_buff.hpages[--i], pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
|
||||
d->curr_user_buff.n_hpages = 0;
|
||||
|
||||
kfree(d->curr_user_buff.hpages);
|
||||
|
||||
return -ENOMEM;
|
||||
|
||||
fail_card_alloc:
|
||||
// release host
|
||||
for (i = 0; i < d->curr_user_buff.n_hpages; i++)
|
||||
__free_pages(d->curr_user_buff.hpages[i], pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
|
||||
d->curr_user_buff.n_hpages = 0;
|
||||
d->curr_user_buff.n_pages = 0;
|
||||
|
||||
kfree(d->curr_user_buff.hpages);
|
||||
kfree(d->curr_user_buff.cpages);
|
||||
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Free user buffers
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param vaddr - virtual address
|
||||
* @param cpid - Coyote PID
|
||||
*/
|
||||
int free_user_buffers(struct fpga_dev *d, uint64_t vaddr, int32_t cpid)
|
||||
{
|
||||
int i;
|
||||
uint64_t vaddr_tmp;
|
||||
struct user_pages *tmp_buff;
|
||||
uint64_t *map_array;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
hash_for_each_possible(user_lbuff_map[d->id], tmp_buff, entry, vaddr) {
|
||||
|
||||
if (tmp_buff->vaddr == vaddr && tmp_buff->cpid == cpid) {
|
||||
|
||||
vaddr_tmp = tmp_buff->vaddr;
|
||||
|
||||
// free host pages
|
||||
for (i = 0; i < tmp_buff->n_hpages; i++) {
|
||||
if (tmp_buff->hpages[i])
|
||||
__free_pages(tmp_buff->hpages[i], pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
}
|
||||
kfree(tmp_buff->hpages);
|
||||
|
||||
// free card pages
|
||||
if(pd->en_mem) {
|
||||
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, LARGE_CHUNK_ALLOC);
|
||||
kfree(tmp_buff->cpages);
|
||||
}
|
||||
|
||||
// map array
|
||||
map_array = (uint64_t *)kzalloc(tmp_buff->n_hpages * 2 * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (map_array == NULL) {
|
||||
dbg_info("map buffers could not be allocated\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
// fill mappings
|
||||
for (i = 0; i < tmp_buff->n_hpages; i++) {
|
||||
tlb_create_unmap(pd->ltlb_order, vaddr_tmp, cpid, &map_array[2*i]);
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->ltlb_order, map_array, tmp_buff->n_hpages);
|
||||
|
||||
// free
|
||||
kfree((void *)map_array);
|
||||
|
||||
// Free from hash
|
||||
hash_del(&tmp_buff->entry);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Allocate PR buffers
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param n_pages - number of pages to allocate
|
||||
*/
|
||||
int alloc_pr_buffers(struct fpga_dev *d, unsigned long n_pages)
|
||||
{
|
||||
int i;
|
||||
struct pr_ctrl *prc;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
prc = d->prc;
|
||||
BUG_ON(!prc);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
// obtain PR lock
|
||||
spin_lock(&prc->lock);
|
||||
|
||||
if (prc->curr_buff.n_pages) {
|
||||
dbg_info("allocated PR buffers exist and are not mapped\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (n_pages > MAX_PR_BUFF_NUM)
|
||||
prc->curr_buff.n_pages = MAX_PR_BUFF_NUM;
|
||||
else
|
||||
prc->curr_buff.n_pages = n_pages;
|
||||
|
||||
prc->curr_buff.pages = kzalloc(n_pages * sizeof(*prc->curr_buff.pages), GFP_KERNEL);
|
||||
if (prc->curr_buff.pages == NULL) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
dbg_info("allocated %lu bytes for page pointer array for %ld PR buffers @0x%p.\n",
|
||||
n_pages * sizeof(*prc->curr_buff.pages), n_pages, prc->curr_buff.pages);
|
||||
|
||||
for (i = 0; i < prc->curr_buff.n_pages; i++) {
|
||||
prc->curr_buff.pages[i] = alloc_pages(GFP_ATOMIC, pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
if (!prc->curr_buff.pages[i]) {
|
||||
dbg_info("PR buffer %d could not be allocated\n", i);
|
||||
goto fail_alloc;
|
||||
}
|
||||
|
||||
dbg_info("PR buffer allocated @ %llx \n", page_to_phys(prc->curr_buff.pages[i]));
|
||||
}
|
||||
|
||||
// release PR lock
|
||||
spin_unlock(&prc->lock);
|
||||
|
||||
return 0;
|
||||
fail_alloc:
|
||||
while (i)
|
||||
__free_pages(prc->curr_buff.pages[--i], pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
// release PR lock
|
||||
spin_unlock(&prc->lock);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Free PR pages
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param vaddr - virtual address
|
||||
*/
|
||||
int free_pr_buffers(struct fpga_dev *d, uint64_t vaddr)
|
||||
{
|
||||
int i;
|
||||
struct pr_pages *tmp_buff;
|
||||
struct pr_ctrl *prc;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
prc = d->prc;
|
||||
BUG_ON(!prc);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
// obtain PR lock
|
||||
spin_lock(&prc->lock);
|
||||
|
||||
hash_for_each_possible(pr_buff_map, tmp_buff, entry, vaddr) {
|
||||
if (tmp_buff->vaddr == vaddr && tmp_buff->reg_id == d->id) {
|
||||
|
||||
// free pages
|
||||
for (i = 0; i < tmp_buff->n_pages; i++) {
|
||||
if (tmp_buff->pages[i])
|
||||
__free_pages(tmp_buff->pages[i], pd->ltlb_order->page_shift - PAGE_SHIFT);
|
||||
}
|
||||
|
||||
kfree(tmp_buff->pages);
|
||||
|
||||
// Free from hash
|
||||
hash_del(&tmp_buff->entry);
|
||||
}
|
||||
}
|
||||
|
||||
// obtain PR lock
|
||||
spin_unlock(&prc->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Allocate card memory
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param card_paddr - card physical address
|
||||
* @param n_pages - number of pages to allocate
|
||||
* @param type - page size
|
||||
*/
|
||||
int card_alloc(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type)
|
||||
{
|
||||
int i;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
|
||||
switch (type) {
|
||||
case 0: //
|
||||
// lock
|
||||
spin_lock(&pd->card_s_lock);
|
||||
|
||||
if(pd->num_free_schunks < n_pages) {
|
||||
dbg_info("not enough free small card pages\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
for(i = 0; i < n_pages; i++) {
|
||||
card_paddr[i] = pd->salloc->id << STLB_PAGE_BITS;
|
||||
dbg_info("user card buffer allocated @ %llx device %d\n", card_paddr[i], d->id);
|
||||
pd->salloc = pd->salloc->next;
|
||||
}
|
||||
|
||||
// release lock
|
||||
spin_unlock(&pd->card_s_lock);
|
||||
|
||||
break;
|
||||
case 1:
|
||||
// lock
|
||||
spin_lock(&pd->card_l_lock);
|
||||
|
||||
if(pd->num_free_lchunks < n_pages) {
|
||||
dbg_info("not enough free large card pages\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
for(i = 0; i < n_pages; i++) {
|
||||
card_paddr[i] = (pd->lalloc->id << LTLB_PAGE_BITS) + MEM_SEP;
|
||||
dbg_info("user card buffer allocated @ %llx device %d\n", card_paddr[i], d->id);
|
||||
pd->lalloc = pd->lalloc->next;
|
||||
}
|
||||
|
||||
// release lock
|
||||
spin_unlock(&pd->card_l_lock);
|
||||
|
||||
break;
|
||||
default: // TODO: Shared mem
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Free card memory
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param card_paddr - card physical address
|
||||
* @param n_pages - number of pages to free
|
||||
* @param type - page size
|
||||
*/
|
||||
void card_free(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type)
|
||||
{
|
||||
int i;
|
||||
uint64_t tmp_id;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
switch (type) {
|
||||
case 0: // small pages
|
||||
// lock
|
||||
spin_lock(&pd->card_s_lock);
|
||||
|
||||
for(i = n_pages - 1; i >= 0; i--) {
|
||||
tmp_id = card_paddr[i] >> STLB_PAGE_BITS;
|
||||
pd->schunks[tmp_id].next = pd->salloc;
|
||||
pd->salloc = &pd->schunks[tmp_id];
|
||||
}
|
||||
|
||||
// release lock
|
||||
spin_unlock(&pd->card_s_lock);
|
||||
|
||||
break;
|
||||
case 1: // large pages
|
||||
// lock
|
||||
spin_lock(&pd->card_l_lock);
|
||||
|
||||
for(i = n_pages - 1; i >= 0; i--) {
|
||||
tmp_id = (card_paddr[i] - MEM_SEP) >> LTLB_PAGE_BITS;
|
||||
pd->lchunks[tmp_id].next = pd->lalloc;
|
||||
pd->lalloc = &pd->lchunks[tmp_id];
|
||||
}
|
||||
|
||||
// release lock
|
||||
spin_unlock(&pd->card_l_lock);
|
||||
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Page map list
|
||||
*
|
||||
* @param vaddr - starting vaddr
|
||||
* @param paddr_host - host physical address
|
||||
* @param paddr_card - card physical address
|
||||
* @param cpid - Coyote PID
|
||||
* @param entry - liste entry
|
||||
*/
|
||||
void tlb_create_map(struct tlb_order *tlb_ord, uint64_t vaddr, uint64_t paddr_host, uint64_t paddr_card, int32_t cpid, uint64_t *entry)
|
||||
{
|
||||
uint64_t key;
|
||||
uint64_t tag;
|
||||
uint64_t phost;
|
||||
uint64_t pcard;
|
||||
|
||||
key = (vaddr >> tlb_ord->page_shift) & tlb_ord->key_mask;
|
||||
tag = vaddr >> (tlb_ord->page_shift + tlb_ord->key_size);
|
||||
phost = (paddr_host >> tlb_ord->page_shift) & tlb_ord->phy_mask;
|
||||
pcard = ((paddr_card >> tlb_ord->page_shift) & tlb_ord->phy_mask) << tlb_ord->phy_size;
|
||||
|
||||
// new entry
|
||||
entry[0] |= key |
|
||||
(tag << tlb_ord->key_size) |
|
||||
((uint64_t)cpid << (tlb_ord->key_size + tlb_ord->tag_size)) |
|
||||
(1UL << (tlb_ord->key_size + tlb_ord->tag_size + PID_SIZE));
|
||||
entry[1] |= phost | (pcard << tlb_ord->phy_size);
|
||||
|
||||
dbg_info("creating new TLB entry, vaddr %llx, phost %llx, pcard %llx, cpid %d, hugepage %d\n", vaddr, paddr_host, paddr_card, cpid, tlb_ord->hugepage);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Page unmap lists
|
||||
*
|
||||
* @param vaddr - starting vaddr
|
||||
* @param cpid - Coyote PID
|
||||
* @param entry - list entry
|
||||
*/
|
||||
void tlb_create_unmap(struct tlb_order *tlb_ord, uint64_t vaddr, int32_t cpid, uint64_t *entry)
|
||||
{
|
||||
uint64_t tag;
|
||||
uint64_t key;
|
||||
|
||||
key = (vaddr >> tlb_ord->page_shift) & tlb_ord->key_mask;
|
||||
tag = vaddr >> (tlb_ord->page_shift + tlb_ord->key_size);
|
||||
|
||||
// entry host
|
||||
entry[0] |= key |
|
||||
(tag << tlb_ord->key_size) |
|
||||
((uint64_t)cpid << (tlb_ord->key_size + tlb_ord->tag_size)) |
|
||||
(0UL << (tlb_ord->key_size + tlb_ord->tag_size + PID_SIZE));
|
||||
entry[1] |= 0;
|
||||
|
||||
dbg_info("unmapping TLB entry, vaddr %llx, cpid %d, hugepage %d\n", vaddr, cpid, tlb_ord->hugepage);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Map TLB
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param en_tlbf - TLBF enabled
|
||||
* @param map_array - prepped map array
|
||||
* @param paddr - physical address
|
||||
* @param cpid - Coyote PID
|
||||
* @param card - map card mem as well
|
||||
*/
|
||||
void tlb_service_dev(struct fpga_dev *d, struct tlb_order *tlb_ord, uint64_t* map_array, uint32_t n_pages)
|
||||
{
|
||||
int i = 0;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
// lock
|
||||
spin_lock(&d->lock);
|
||||
|
||||
if(pd->en_tlbf && (n_pages > MAX_MAP_AXIL_PAGES)) {
|
||||
// lock
|
||||
spin_lock(&pd->tlb_lock);
|
||||
|
||||
// start DMA
|
||||
pd->fpga_stat_cnfg->tlb_addr = virt_to_phys((void *)map_array);
|
||||
pd->fpga_stat_cnfg->tlb_len = n_pages * 2 * sizeof(uint64_t);
|
||||
if(tlb_ord->hugepage) {
|
||||
pd->fpga_stat_cnfg->tlb_ctrl = TLBF_CTRL_START | ((d->id && TLBF_CTRL_ID_MASK) << TLBF_CTRL_ID_SHFT);
|
||||
} else {
|
||||
pd->fpga_stat_cnfg->tlb_ctrl = TLBF_CTRL_START | (((pd->n_fpga_reg + d->id) && TLBF_CTRL_ID_MASK) << TLBF_CTRL_ID_SHFT);
|
||||
}
|
||||
|
||||
// poll
|
||||
while ((pd->fpga_stat_cnfg->tlb_stat & TLBF_STAT_DONE) != 0x1)
|
||||
ndelay(100);
|
||||
|
||||
// unlock
|
||||
spin_unlock(&pd->tlb_lock);
|
||||
} else {
|
||||
// map each page through AXIL
|
||||
for (i = 0; i < n_pages; i++) {
|
||||
if(tlb_ord->hugepage) {
|
||||
d->fpga_lTlb[0] = map_array[2*i+0];
|
||||
d->fpga_lTlb[1] = map_array[2*i+1];
|
||||
} else {
|
||||
d->fpga_sTlb[0] = map_array[2*i+0];
|
||||
d->fpga_sTlb[1] = map_array[2*i+1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unlock
|
||||
spin_unlock(&d->lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Release all remaining user pages
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param dirtied - modified
|
||||
*/
|
||||
int tlb_put_user_pages_all(struct fpga_dev *d, int dirtied)
|
||||
{
|
||||
int i, bkt;
|
||||
struct user_pages *tmp_buff;
|
||||
uint64_t vaddr_tmp;
|
||||
int32_t cpid_tmp;
|
||||
uint64_t *map_array;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
hash_for_each(user_sbuff_map[d->id], bkt, tmp_buff, entry) {
|
||||
|
||||
// release host pages
|
||||
if(dirtied)
|
||||
for(i = 0; i < tmp_buff->n_hpages; i++)
|
||||
SetPageDirty(tmp_buff->hpages[i]);
|
||||
|
||||
for(i = 0; i < tmp_buff->n_hpages; i++)
|
||||
put_page(tmp_buff->hpages[i]);
|
||||
|
||||
kfree(tmp_buff->hpages);
|
||||
|
||||
// release card pages
|
||||
if(pd->en_mem) {
|
||||
if(tmp_buff->huge)
|
||||
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, LARGE_CHUNK_ALLOC);
|
||||
else
|
||||
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, SMALL_CHUNK_ALLOC);
|
||||
}
|
||||
|
||||
// unmap from TLB
|
||||
vaddr_tmp = tmp_buff->vaddr;
|
||||
cpid_tmp = tmp_buff->cpid;
|
||||
|
||||
// map array
|
||||
map_array = (uint64_t *)kzalloc(tmp_buff->n_pages * 2 * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (map_array == NULL) {
|
||||
dbg_info("map buffers could not be allocated\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
// huge pages
|
||||
if(tmp_buff->huge) {
|
||||
// fill mappings
|
||||
for (i = 0; i < tmp_buff->n_pages; i++) {
|
||||
tlb_create_unmap(pd->ltlb_order, vaddr_tmp, cpid_tmp, &map_array[2*i]);
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->ltlb_order, map_array, tmp_buff->n_pages);
|
||||
|
||||
// small pages
|
||||
} else {
|
||||
// fill mappings
|
||||
for (i = 0; i < tmp_buff->n_pages; i++) {
|
||||
tlb_create_unmap(pd->stlb_order, vaddr_tmp, cpid_tmp, &map_array[2*i]);
|
||||
vaddr_tmp += PAGE_SIZE;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->stlb_order, map_array, tmp_buff->n_pages);
|
||||
}
|
||||
|
||||
// free
|
||||
kfree((void *)map_array);
|
||||
|
||||
// remove from map
|
||||
hash_del(&tmp_buff->entry);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Release user pages
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param vaddr - starting vaddr
|
||||
* @param cpid - Coyote PID
|
||||
* @param dirtied - modified
|
||||
*/
|
||||
int tlb_put_user_pages(struct fpga_dev *d, uint64_t vaddr, int32_t cpid, int dirtied)
|
||||
{
|
||||
int i;
|
||||
struct user_pages *tmp_buff;
|
||||
uint64_t vaddr_tmp;
|
||||
uint64_t *map_array;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
hash_for_each_possible(user_sbuff_map[d->id], tmp_buff, entry, vaddr) {
|
||||
if(tmp_buff->vaddr == vaddr && tmp_buff->cpid == cpid) {
|
||||
|
||||
// release host pages
|
||||
if(dirtied)
|
||||
for(i = 0; i < tmp_buff->n_hpages; i++)
|
||||
SetPageDirty(tmp_buff->hpages[i]);
|
||||
|
||||
for(i = 0; i < tmp_buff->n_hpages; i++)
|
||||
put_page(tmp_buff->hpages[i]);
|
||||
|
||||
kfree(tmp_buff->hpages);
|
||||
|
||||
// release card pages
|
||||
if(pd->en_mem) {
|
||||
if(tmp_buff->huge)
|
||||
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, LARGE_CHUNK_ALLOC);
|
||||
else
|
||||
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, SMALL_CHUNK_ALLOC);
|
||||
}
|
||||
|
||||
// unmap from TLB
|
||||
vaddr_tmp = vaddr;
|
||||
|
||||
// map array
|
||||
map_array = (uint64_t *)kzalloc(tmp_buff->n_pages * 2 * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (map_array == NULL) {
|
||||
dbg_info("map buffers could not be allocated\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
// huge pages
|
||||
if(tmp_buff->huge) {
|
||||
// fill mappings
|
||||
for (i = 0; i < tmp_buff->n_pages; i++) {
|
||||
tlb_create_unmap(pd->ltlb_order, vaddr_tmp, cpid, &map_array[2*i]);
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->ltlb_order, map_array, tmp_buff->n_pages);
|
||||
|
||||
// small pages
|
||||
} else {
|
||||
// fill mappings
|
||||
for (i = 0; i < tmp_buff->n_pages; i++) {
|
||||
tlb_create_unmap(pd->stlb_order, vaddr_tmp, cpid, &map_array[2*i]);
|
||||
vaddr_tmp += PAGE_SIZE;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->stlb_order, map_array, tmp_buff->n_pages);
|
||||
}
|
||||
|
||||
// free
|
||||
kfree((void *)map_array);
|
||||
|
||||
// remove from map
|
||||
hash_del(&tmp_buff->entry);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Get user pages and fill TLB
|
||||
*
|
||||
* @param d - vFPGA
|
||||
* @param start - starting vaddr
|
||||
* @param count - number of pages to map
|
||||
* @param cpid - Coyote PID
|
||||
* @param pid - user PID
|
||||
*/
|
||||
int tlb_get_user_pages(struct fpga_dev *d, uint64_t start, size_t count, int32_t cpid, pid_t pid)
|
||||
{
|
||||
int ret_val = 0, i, j;
|
||||
int n_pages, n_pages_huge;
|
||||
uint64_t first;
|
||||
uint64_t last;
|
||||
struct user_pages *user_pg;
|
||||
struct vm_area_struct *vma_area_init;
|
||||
int hugepages;
|
||||
uint64_t *hpages_phys;
|
||||
uint64_t curr_vaddr, last_vaddr;
|
||||
struct task_struct *curr_task;
|
||||
struct mm_struct *curr_mm;
|
||||
uint64_t *map_array;
|
||||
uint64_t vaddr_tmp;
|
||||
struct bus_drvdata *pd;
|
||||
|
||||
BUG_ON(!d);
|
||||
pd = d->pd;
|
||||
BUG_ON(!pd);
|
||||
|
||||
// context
|
||||
curr_task = pid_task(find_vpid(pid), PIDTYPE_PID);
|
||||
dbg_info("pid found = %d", pid);
|
||||
curr_mm = curr_task->mm;
|
||||
|
||||
// hugepages?
|
||||
vma_area_init = find_vma(curr_mm, start);
|
||||
hugepages = is_vm_hugetlb_page(vma_area_init);
|
||||
|
||||
// number of pages
|
||||
first = (start & PAGE_MASK) >> PAGE_SHIFT;
|
||||
last = ((start + count - 1) & PAGE_MASK) >> PAGE_SHIFT;
|
||||
n_pages = last - first + 1;
|
||||
|
||||
if(hugepages) {
|
||||
if(n_pages > MAX_N_MAP_HUGE_PAGES)
|
||||
n_pages = MAX_N_MAP_HUGE_PAGES;
|
||||
} else {
|
||||
if(n_pages > MAX_N_MAP_PAGES)
|
||||
n_pages = MAX_N_MAP_PAGES;
|
||||
}
|
||||
|
||||
if (start + count < start)
|
||||
return -EINVAL;
|
||||
if (count == 0)
|
||||
return 0;
|
||||
|
||||
// alloc
|
||||
user_pg = kzalloc(sizeof(struct user_pages), GFP_KERNEL);
|
||||
BUG_ON(!user_pg);
|
||||
|
||||
user_pg->hpages = kcalloc(n_pages, sizeof(*user_pg->hpages), GFP_KERNEL);
|
||||
if (user_pg->hpages == NULL) {
|
||||
return -1;
|
||||
}
|
||||
dbg_info("allocated %lu bytes for page pointer array for %d pages @0x%p, passed size %ld.\n",
|
||||
n_pages * sizeof(*user_pg->hpages), n_pages, user_pg->hpages, count);
|
||||
|
||||
dbg_info("pages=0x%p\n", user_pg->hpages);
|
||||
dbg_info("first = %llx, last = %llx\n", first, last);
|
||||
|
||||
for (i = 0; i < n_pages - 1; i++) {
|
||||
user_pg->hpages[i] = NULL;
|
||||
}
|
||||
|
||||
// pin
|
||||
//ret_val = get_user_pages_remote(curr_task, curr_mm, (unsigned long)start, n_pages, 1, user_pg->hpages, NULL, NULL);
|
||||
ret_val = get_user_pages_remote(curr_mm, (unsigned long)start, n_pages, 1, user_pg->hpages, NULL, NULL);
|
||||
//ret_val = pin_user_pages_remote(curr_mm, (unsigned long)start, n_pages, 1, user_pg->hpages, NULL, NULL);
|
||||
dbg_info("get_user_pages_remote(%llx, n_pages = %d, page start = %lx, hugepages = %d)\n", start, n_pages, page_to_pfn(user_pg->hpages[0]), hugepages);
|
||||
|
||||
if(ret_val < n_pages) {
|
||||
dbg_info("could not get all user pages, %d\n", ret_val);
|
||||
goto fail_host_unmap;
|
||||
}
|
||||
|
||||
// flush cache
|
||||
for(i = 0; i < n_pages; i++)
|
||||
flush_dcache_page(user_pg->hpages[i]);
|
||||
|
||||
// add mapped entry
|
||||
user_pg->vaddr = start;
|
||||
user_pg->n_hpages = n_pages;
|
||||
user_pg->huge = hugepages;
|
||||
|
||||
vaddr_tmp = start;
|
||||
|
||||
// huge pages
|
||||
if (hugepages) {
|
||||
first = (start & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift;
|
||||
last = ((start + count - 1) & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift;
|
||||
n_pages_huge = last - first + 1;
|
||||
user_pg->n_pages = n_pages_huge;
|
||||
|
||||
// prep hpages
|
||||
hpages_phys = kzalloc(n_pages_huge * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (hpages_phys == NULL) {
|
||||
dbg_info("card buffer %d could not be allocated\n", i);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
j = 0;
|
||||
curr_vaddr = start;
|
||||
last_vaddr = -1;
|
||||
for (i = 0; i < n_pages; i++) {
|
||||
if (((curr_vaddr & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift) != ((last_vaddr & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift)) {
|
||||
hpages_phys[j] = page_to_phys(user_pg->hpages[i]) & pd->ltlb_order->page_mask;
|
||||
dbg_info("hugepage %d at %llx\n", j, hpages_phys[j]);
|
||||
last_vaddr = curr_vaddr;
|
||||
j++;
|
||||
}
|
||||
curr_vaddr += PAGE_SIZE;
|
||||
}
|
||||
|
||||
// card alloc
|
||||
if(pd->en_mem) {
|
||||
user_pg->cpages = kzalloc(n_pages_huge * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (user_pg->cpages == NULL) {
|
||||
dbg_info("card buffer %d could not be allocated\n", i);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
ret_val = card_alloc(d, user_pg->cpages, n_pages_huge, LARGE_CHUNK_ALLOC);
|
||||
if (ret_val) {
|
||||
dbg_info("could not get all card pages, %d\n", ret_val);
|
||||
goto fail_card_unmap;
|
||||
}
|
||||
dbg_info("card allocated %d hugepages\n", n_pages_huge);
|
||||
}
|
||||
|
||||
// map array
|
||||
map_array = (uint64_t *)kzalloc(n_pages_huge * 2 * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (map_array == NULL) {
|
||||
dbg_info("map buffers could not be allocated\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
// fill mappings
|
||||
for (i = 0; i < n_pages_huge; i++) {
|
||||
tlb_create_map(pd->ltlb_order, vaddr_tmp, hpages_phys[i], (pd->en_mem ? user_pg->cpages[i] : 0), cpid, &map_array[2*i]);
|
||||
vaddr_tmp += pd->ltlb_order->page_size;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->ltlb_order, map_array, n_pages_huge);
|
||||
|
||||
// free
|
||||
kfree((void *)map_array);
|
||||
|
||||
// small pages
|
||||
} else {
|
||||
user_pg->n_pages = n_pages;
|
||||
|
||||
// card alloc
|
||||
if(pd->en_mem) {
|
||||
user_pg->cpages = kzalloc(n_pages * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (user_pg->cpages == NULL) {
|
||||
dbg_info("card buffer %d could not be allocated\n", i);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
ret_val = card_alloc(d, user_pg->cpages, n_pages, SMALL_CHUNK_ALLOC);
|
||||
if (ret_val) {
|
||||
dbg_info("could not get all card pages, %d\n", ret_val);
|
||||
goto fail_card_unmap;
|
||||
}
|
||||
dbg_info("card allocated %d regular pages\n", n_pages);
|
||||
}
|
||||
|
||||
// map array
|
||||
map_array = (uint64_t *)kzalloc(n_pages * 2 * sizeof(uint64_t), GFP_KERNEL);
|
||||
if (map_array == NULL) {
|
||||
dbg_info("map buffers could not be allocated\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
// fill mappings
|
||||
for (i = 0; i < n_pages; i++) {
|
||||
tlb_create_map(pd->stlb_order, vaddr_tmp, page_to_phys(user_pg->hpages[i]), (pd->en_mem ? user_pg->cpages[i] : 0), cpid, &map_array[2*i]);
|
||||
vaddr_tmp += PAGE_SIZE;
|
||||
}
|
||||
|
||||
// fire
|
||||
tlb_service_dev(d, pd->stlb_order, map_array, n_pages);
|
||||
|
||||
// free
|
||||
kfree((void *)map_array);
|
||||
}
|
||||
|
||||
hash_add(user_sbuff_map[d->id], &user_pg->entry, start);
|
||||
|
||||
return n_pages;
|
||||
|
||||
fail_host_unmap:
|
||||
// release host pages
|
||||
for(i = 0; i < ret_val; i++) {
|
||||
put_page(user_pg->hpages[i]);
|
||||
}
|
||||
|
||||
kfree(user_pg->hpages);
|
||||
|
||||
return -ENOMEM;
|
||||
|
||||
fail_card_unmap:
|
||||
// release host pages
|
||||
for(i = 0; i < user_pg->n_hpages; i++) {
|
||||
put_page(user_pg->hpages[i]);
|
||||
}
|
||||
|
||||
kfree(user_pg->hpages);
|
||||
kfree(user_pg->cpages);
|
||||
|
||||
return -ENOMEM;
|
||||
}
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __FPGA_MMU_H__
|
||||
#define __FPGA_MMU_H__
|
||||
|
||||
#include "coyote_dev.h"
|
||||
|
||||
/* Alloc huge pages (should be used only if no host hugepages enabled!) */
|
||||
int alloc_user_buffers(struct fpga_dev *d, unsigned long n_pages, int32_t cpid);
|
||||
int free_user_buffers(struct fpga_dev *d, uint64_t vaddr, int32_t cpid);
|
||||
|
||||
/* PR memory regions */
|
||||
int alloc_pr_buffers(struct fpga_dev *d, unsigned long n_pages);
|
||||
int free_pr_buffers(struct fpga_dev *d, uint64_t vaddr);
|
||||
|
||||
/* Card memory resources */
|
||||
int card_alloc(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type);
|
||||
void card_free(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type);
|
||||
|
||||
/* TLB mappings */
|
||||
void tlb_create_map(struct tlb_order *tlb_ord, uint64_t vaddr, uint64_t paddr_host, uint64_t paddr_card, int32_t cpid, uint64_t *entry);
|
||||
void tlb_create_unmap(struct tlb_order *tlb_ord, uint64_t vaddr, int32_t cpid, uint64_t *entry);
|
||||
|
||||
/* TLB control */
|
||||
void tlb_service_dev(struct fpga_dev *d, struct tlb_order *tlb_ord, uint64_t *entry, uint32_t n_pages);
|
||||
|
||||
/* Page table walks */
|
||||
int tlb_get_user_pages(struct fpga_dev *d, uint64_t start, size_t count, int32_t cpid, pid_t pid);
|
||||
int tlb_put_user_pages(struct fpga_dev *d, uint64_t vaddr, int32_t cpid, int dirtied);
|
||||
int tlb_put_user_pages_all(struct fpga_dev *d, int dirtied);
|
||||
|
||||
#endif /* FPGA MMU */
|
|
@ -0,0 +1,13 @@
|
|||
obj-m := fpga_drv.o
|
||||
|
||||
KERNELDIR ?= /lib/modules/$(shell uname -r)/build
|
||||
PWD := $(shell pwd)
|
||||
|
||||
ROOT := $(dir $(M))
|
||||
XILINXINCLUDE := -I$(ROOT)../include -I$(ROOT)/include
|
||||
|
||||
all:
|
||||
$(MAKE) -C $(KERNELDIR) M=$(PWD) modules
|
||||
|
||||
clean:
|
||||
rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.c .tmp_versions *.order *.symvers *.dwo *.mod *.mod.dwo
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,604 @@
|
|||
#ifndef COYOTE_DRV_H
|
||||
#define COYOTE_DRV_H
|
||||
|
||||
#include <linux/ioctl.h>
|
||||
#include <linux/cdev.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/hashtable.h>
|
||||
|
||||
/* Driver debug */
|
||||
#define COYOTE_DEBUG 1
|
||||
|
||||
/* Network setup */
|
||||
#define BASE_IP_ADDR_0 0x0B01D4D1
|
||||
#define BASE_IP_ADDR_1 0x0B01D4D1
|
||||
#define NODE_ID 0
|
||||
#define N_TOTAL_NODES 2
|
||||
|
||||
/* Debug print */
|
||||
#if (COYOTE_DEBUG == 0)
|
||||
#define dbg_info(...)
|
||||
#else
|
||||
#define dbg_info(fmt, ...) pr_info("%s():" fmt, \
|
||||
__func__, ##__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
|
||||
#define HIGH_32(addr) ((addr >> 16) >> 16)
|
||||
/* Obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
|
||||
#define LOW_32(addr) (addr & 0xffffffffUL)
|
||||
|
||||
/* Driver info */
|
||||
#define DRV_NAME "coyote_driver"
|
||||
#define DEV_NAME "fpga"
|
||||
|
||||
/**
|
||||
* XDMA info
|
||||
*/
|
||||
#define MAX_NUM_BARS 3
|
||||
#define MAX_NUM_CHANNELS 4
|
||||
#define MAX_NUM_ENGINES (MAX_NUM_CHANNELS * 2)
|
||||
#define MAX_USER_IRQS 16
|
||||
#define C2H_CHAN_OFFS 0x1000
|
||||
#define H2C_CHAN_OFFS 0x0000
|
||||
#define CHAN_RANGE 0x100
|
||||
#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
|
||||
|
||||
/* Engine IDs */
|
||||
#define XDMA_ID_H2C 0x1fc0U
|
||||
#define XDMA_ID_C2H 0x1fc1U
|
||||
|
||||
/* Engine regs */
|
||||
#define XDMA_ENG_IRQ_NUM (1)
|
||||
#define XDMA_OFS_INT_CTRL (0x2000UL)
|
||||
#define XDMA_OFS_CONFIG (0x3000UL)
|
||||
|
||||
/* Bits of the SG DMA control register */
|
||||
#define XDMA_CTRL_RUN_STOP (1UL << 0)
|
||||
#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
|
||||
#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
|
||||
#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
|
||||
#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
|
||||
#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
|
||||
#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
|
||||
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
|
||||
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
|
||||
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
|
||||
|
||||
/* Bits of the SG DMA status register */
|
||||
#define XDMA_STAT_BUSY (1UL << 0)
|
||||
#define XDMA_STAT_DESC_STOPPED (1UL << 1)
|
||||
#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
|
||||
#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
|
||||
#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
|
||||
#define XDMA_STAT_FETCH_STOPPED (1UL << 5)
|
||||
#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
|
||||
#define XDMA_STAT_READ_ERROR (0x1FUL << 9)
|
||||
#define XDMA_STAT_DESC_ERROR (0x1FUL << 19)
|
||||
|
||||
/* Bits of the performance control register */
|
||||
#define XDMA_PERF_RUN (1UL << 0)
|
||||
#define XDMA_PERF_CLEAR (1UL << 1)
|
||||
#define XDMA_PERF_AUTO (1UL << 2)
|
||||
|
||||
/* Polling */
|
||||
#define WB_COUNT_MASK 0x00ffffffUL
|
||||
#define WB_ERR_MASK (1UL << 31)
|
||||
#define POLL_TIMEOUT_SECONDS 10
|
||||
#define NUM_POLLS_PER_SCHED 100
|
||||
|
||||
/**
|
||||
* Static layer
|
||||
*/
|
||||
#define BAR_XDMA_CONFIG 0
|
||||
#define BAR_FPGA_CONFIG 1
|
||||
|
||||
/* FPGA static config */
|
||||
#define FPGA_STAT_CNFG_OFFS 0x0
|
||||
#define FPGA_STAT_CNFG_SIZE 32 * 1024
|
||||
|
||||
#define EN_AVX_MASK 0x1
|
||||
#define EN_AVX_SHFT 0x0
|
||||
#define EN_BPSS_MASK 0x2
|
||||
#define EN_BPSS_SHFT 0x1
|
||||
#define EN_TLBF_MASK 0x4
|
||||
#define EN_TLBF_SHFT 0x2
|
||||
#define EN_WB_MASK 0x8
|
||||
#define EN_WB_SHFT 0x3
|
||||
#define TLB_S_ORDER_MASK 0xf0
|
||||
#define TLB_S_ORDER_SHFT 0x4
|
||||
#define TLB_S_ASSOC_MASK 0xf00
|
||||
#define TLB_S_ASSOC_SHFT 0x8
|
||||
#define TLB_L_ORDER_MASK 0xf000
|
||||
#define TLB_L_ORDER_SHFT 0xc
|
||||
#define TLB_L_ASSOC_MASK 0xf0000
|
||||
#define TLB_L_ASSOC_SHFT 0x10
|
||||
#define TLB_S_PG_SHFT_MASK 0x3f00000
|
||||
#define TLB_S_PG_SHFT_SHFT 0x14
|
||||
#define TLB_L_PG_SHFT_MASK 0xfc000000
|
||||
#define TLB_L_PG_SHFT_SHFT 0x1a
|
||||
|
||||
#define EN_DDR_MASK 0x1
|
||||
#define EN_DDR_SHFT 0x0
|
||||
#define EN_HBM_MASK 0x2
|
||||
#define EN_HBM_SHFT 0x1
|
||||
#define N_MEM_CHAN_MASK 0x7d
|
||||
#define N_MEM_CHAN_SHFT 0x2
|
||||
#define EN_RDMA_0_MASK 0x1
|
||||
#define EN_RDMA_1_MASK 0x2
|
||||
#define EN_PR_MASK 0x1
|
||||
#define EN_PR_SHFT 0x0
|
||||
#define EN_RDMA_0_MASK 0x1
|
||||
#define EN_RDMA_0_SHFT 0x0
|
||||
#define EN_RDMA_1_MASK 0x2
|
||||
#define EN_RDMA_1_SHFT 0x1
|
||||
#define EN_TCP_0_MASK 0x1
|
||||
#define EN_TCP_0_SHFT 0x0
|
||||
#define EN_TCP_1_MASK 0x2
|
||||
#define EN_TCP_1_SHFT 0x1
|
||||
|
||||
/**
|
||||
* Dynamic layer
|
||||
*/
|
||||
|
||||
/* FPGA control regions */
|
||||
#define FPGA_CTRL_SIZE 256 * 1024
|
||||
#define FPGA_CTRL_OFFS 0x100000
|
||||
#define FPGA_CTRL_LTLB_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_LTLB_OFFS 0x0
|
||||
#define FPGA_CTRL_STLB_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_STLB_OFFS 0x10000
|
||||
#define FPGA_CTRL_USER_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_USER_OFFS 0x20000
|
||||
#define FPGA_CTRL_CNFG_SIZE FPGA_CTRL_SIZE / 4
|
||||
#define FPGA_CTRL_CNFG_OFFS 0x30000
|
||||
|
||||
#define FPGA_CTRL_CNFG_AVX_SIZE 256 * 1024
|
||||
#define FPGA_CTRL_CNFG_AVX_OFFS 0x1000000
|
||||
|
||||
/* FPGA dynamic control config */
|
||||
#define FPGA_CNFG_CTRL_IRQ_RESTART 0x100
|
||||
|
||||
/* TODO: Maximum transfer size */
|
||||
#define XDMA_TRANSFER_MAX_BYTES (8 * 1024 * 1024)
|
||||
|
||||
/* TLB */
|
||||
#define TLB_VADDR_RANGE 48
|
||||
#define TLB_PADDR_RANGE 40
|
||||
#define PID_SIZE 6
|
||||
#define MAX_MAP_AXIL_PAGES 64
|
||||
|
||||
#define LTLB_PAGE_BITS 21
|
||||
#define STLB_PAGE_BITS 12
|
||||
#define LTLB_PADDR_SIZE (TLB_PADDR_RANGE - LTLB_PAGE_BITS)
|
||||
#define STLB_PADDR_SIZE (TLB_PADDR_RANGE - STLB_PAGE_BITS)
|
||||
|
||||
#define TLBF_CTRL_START 0x7
|
||||
#define TLBF_CTRL_ID_MASK 0xff
|
||||
#define TLBF_CTRL_ID_SHFT 0x3
|
||||
#define TLBF_STAT_DONE 0x1
|
||||
|
||||
/* Memory allocation */
|
||||
#define MAX_BUFF_NUM 64 // maximum number of huge pages allowed
|
||||
#define MAX_PR_BUFF_NUM 64 // maximum number of huge pages allowed
|
||||
#define MAX_N_MAP_PAGES 128 // TODO: link to params, max no pfault 1M
|
||||
#define MAX_N_MAP_HUGE_PAGES (16 * 512) // max no pfault 32M
|
||||
|
||||
/* Max card pages */
|
||||
#define N_LARGE_CHUNKS 512
|
||||
#define N_SMALL_CHUNKS (256 * 1024)
|
||||
#define MEM_SEP (N_SMALL_CHUNKS * (4 * 1024))
|
||||
#define MAX_N_REGIONS 16
|
||||
#define SMALL_CHUNK_ALLOC 0
|
||||
#define LARGE_CHUNK_ALLOC 1
|
||||
|
||||
/* PR */
|
||||
#define PR_CTRL_START_MIDDLE 0x3
|
||||
#define PR_CTRL_START_LAST 0x7
|
||||
#define PR_STAT_DONE 0x1
|
||||
|
||||
|
||||
/* FPGA config commands */
|
||||
|
||||
/**
|
||||
* Cdev
|
||||
*/
|
||||
|
||||
/* Major number */
|
||||
#define FPGA_MAJOR 0 // dynamic
|
||||
|
||||
/* MMAP */
|
||||
#define MMAP_CTRL 0x0
|
||||
#define MMAP_CNFG 0x1
|
||||
#define MMAP_CNFG_AVX 0x2
|
||||
#define MMAP_WB 0x3
|
||||
#define MMAP_BUFF 0x200
|
||||
#define MMAP_PR 0x400
|
||||
|
||||
/* IOCTL */
|
||||
#define IOCTL_ALLOC_HOST_USER_MEM _IOW('D', 1, unsigned long) // large pages (no hugepage support)
|
||||
#define IOCTL_FREE_HOST_USER_MEM _IOW('D', 2, unsigned long)
|
||||
#define IOCTL_ALLOC_HOST_PR_MEM _IOW('D', 3, unsigned long) // pr pages
|
||||
#define IOCTL_FREE_HOST_PR_MEM _IOW('D', 4, unsigned long)
|
||||
#define IOCTL_MAP_USER _IOW('D', 5, unsigned long) // map
|
||||
#define IOCTL_UNMAP_USER _IOW('D', 6, unsigned long)
|
||||
#define IOCTL_REGISTER_PID _IOW('D', 7, unsigned long) // register pid
|
||||
#define IOCTL_UNREGISTER_PID _IOW('D', 8, unsigned long)
|
||||
#define IOCTL_RECONFIG_LOAD _IOW('D', 9, unsigned long) // reconfiguration
|
||||
|
||||
#define IOCTL_ARP_LOOKUP _IOW('D', 10, unsigned long) // arp lookup
|
||||
#define IOCTL_SET_IP_ADDRESS _IOW('D', 11, unsigned long)
|
||||
#define IOCTL_SET_BOARD_NUM _IOW('D', 12, unsigned long)
|
||||
#define IOCTL_WRITE_CTX _IOW('D', 13, unsigned long) // qp context
|
||||
#define IOCTL_WRITE_CONN _IOW('D', 14, unsigned long) // qp connection
|
||||
#define IOCTL_SET_TCP_OFFS _IOW('D', 15, unsigned long) // tcp mem offsets
|
||||
|
||||
#define IOCTL_READ_CNFG _IOR('D', 32, unsigned long) // status cnfg
|
||||
#define IOCTL_NET_STATS _IOR('D', 33, unsigned long) // status network
|
||||
#define IOCTL_READ_ENG_STATUS _IOR('D', 35, unsigned long) // status engines
|
||||
|
||||
/* Hash */
|
||||
#define PR_HASH_TABLE_ORDER 8
|
||||
#define PR_BATCH_SIZE (2 * 1024 * 1024)
|
||||
|
||||
#define USER_HASH_TABLE_ORDER 8
|
||||
|
||||
/* PID */
|
||||
#define N_CPID_MAX 64
|
||||
#define WB_SIZE (2 * N_CPID_MAX * sizeof(uint32_t))
|
||||
#define N_WB_PAGES ((WB_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
|
||||
|
||||
/* Network */
|
||||
#define EN_LOWSPEED 0x5
|
||||
#define N_NET_STAT_REGS 9
|
||||
|
||||
/* Copy */
|
||||
#define MAX_USER_WORDS 32
|
||||
|
||||
/**
|
||||
* Reg maps
|
||||
*/
|
||||
|
||||
/* DMA engine reg map */
|
||||
struct engine_regs {
|
||||
uint32_t id;
|
||||
uint32_t ctrl;
|
||||
uint32_t ctrl_w1s;
|
||||
uint32_t ctrl_w1c;
|
||||
uint32_t rsrvd_1[12];
|
||||
|
||||
uint32_t status;
|
||||
uint32_t status_rc;
|
||||
uint32_t completed_desc_count;
|
||||
uint32_t alignments;
|
||||
uint32_t rsrvd_2[14]; // padding
|
||||
|
||||
uint32_t poll_mode_wb_lo;
|
||||
uint32_t poll_mode_wb_hi;
|
||||
uint32_t interrupt_enable_mask;
|
||||
uint32_t interrupt_enable_mask_w1s;
|
||||
uint32_t interrupt_enable_mask_w1c;
|
||||
uint32_t rsrvd_3[9]; // padding
|
||||
|
||||
uint32_t perf_ctrl;
|
||||
uint32_t perf_cyc_lo;
|
||||
uint32_t perf_cyc_hi;
|
||||
uint32_t perf_dat_lo;
|
||||
uint32_t perf_dat_hi;
|
||||
uint32_t perf_pnd_lo;
|
||||
uint32_t perf_pnd_hi;
|
||||
} __packed;
|
||||
|
||||
/* Interrupt reg map */
|
||||
struct interrupt_regs {
|
||||
uint32_t id;
|
||||
uint32_t user_int_enable;
|
||||
uint32_t user_int_enable_w1s;
|
||||
uint32_t user_int_enable_w1c;
|
||||
uint32_t channel_int_enable;
|
||||
uint32_t channel_int_enable_w1s;
|
||||
uint32_t channel_int_enable_w1c;
|
||||
uint32_t reserved_1[9]; // padding
|
||||
|
||||
uint32_t user_int_request;
|
||||
uint32_t channel_int_request;
|
||||
uint32_t user_int_pending;
|
||||
uint32_t channel_int_pending;
|
||||
uint32_t reserved_2[12]; // padding
|
||||
|
||||
uint32_t user_msi_vector[8];
|
||||
uint32_t channel_msi_vector[8];
|
||||
} __packed;
|
||||
|
||||
/* Polled mode descriptors struct */
|
||||
struct xdma_poll_wb {
|
||||
uint32_t completed_desc_count;
|
||||
uint32_t reserved_1[7];
|
||||
} __packed;
|
||||
|
||||
/* FPGA static config reg map */
|
||||
struct fpga_stat_cnfg_regs {
|
||||
uint64_t probe;
|
||||
uint64_t n_chan;
|
||||
uint64_t n_regions;
|
||||
uint64_t ctrl_cnfg;
|
||||
uint64_t mem_cnfg;
|
||||
uint64_t pr_cnfg;
|
||||
uint64_t rdma_cnfg;
|
||||
uint64_t tcp_cnfg;
|
||||
uint64_t lspeed_cnfg;
|
||||
uint64_t reserved_0[1];
|
||||
uint64_t pr_ctrl;
|
||||
uint64_t pr_stat;
|
||||
uint64_t pr_addr;
|
||||
uint64_t pr_len;
|
||||
uint64_t tlb_ctrl;
|
||||
uint64_t tlb_stat;
|
||||
uint64_t tlb_addr;
|
||||
uint64_t tlb_len;
|
||||
uint64_t reserved_1[2];
|
||||
uint64_t net_0_ip;
|
||||
uint64_t net_0_boardnum;
|
||||
uint64_t net_0_arp;
|
||||
uint64_t rdma_0_qp_ctx[3];
|
||||
uint64_t rdma_0_qp_conn[3];
|
||||
uint64_t tcp_0_offs[2];
|
||||
uint64_t net_0_debug[N_NET_STAT_REGS];
|
||||
uint64_t net_1_ip;
|
||||
uint64_t net_1_boardnum;
|
||||
uint64_t net_1_arp;
|
||||
uint64_t rdma_1_qp_ctx[3];
|
||||
uint64_t rdma_1_qp_conn[3];
|
||||
uint64_t tcp_1_offs[2];
|
||||
uint64_t net_1_debug[N_NET_STAT_REGS];
|
||||
} __packed;
|
||||
|
||||
/* FPGA dynamic config reg map */
|
||||
struct fpga_cnfg_regs {
|
||||
uint64_t ctrl;
|
||||
uint64_t vaddr_rd;
|
||||
uint64_t len_rd;
|
||||
uint64_t vaddr_wr;
|
||||
uint64_t len_wr;
|
||||
uint64_t vaddr_miss;
|
||||
uint64_t len_miss;
|
||||
uint64_t datapath_set;
|
||||
uint64_t datapath_clr;
|
||||
uint64_t stat_cmd_used_rd;
|
||||
uint64_t stat_cmd_used_wr;
|
||||
uint64_t stat_sent[6];
|
||||
uint64_t stat_pfaults;
|
||||
uint64_t wback_rd;
|
||||
uint64_t wback_wr;
|
||||
// Rest of regs not used in the driver
|
||||
} __packed;
|
||||
|
||||
/* FPGA dynamic config reg map */
|
||||
struct fpga_cnfg_regs_avx {
|
||||
uint64_t ctrl[4];
|
||||
uint64_t vaddr_miss;
|
||||
uint64_t len_miss;
|
||||
uint64_t pf[2];
|
||||
uint64_t datapath_set[4];
|
||||
uint64_t datapath_clr[4];
|
||||
uint64_t stat[4];
|
||||
uint64_t wback[4];
|
||||
// Rest not used in the driver
|
||||
} __packed;
|
||||
|
||||
/**
|
||||
* Structs
|
||||
*/
|
||||
|
||||
/* Engine descriptors */
|
||||
struct xdma_sgdma_regs {
|
||||
uint32_t identifier;
|
||||
uint32_t reserved_1[31]; /* padding */
|
||||
|
||||
/* bus address to first descriptor in Root Complex Memory */
|
||||
uint32_t first_desc_lo;
|
||||
uint32_t first_desc_hi;
|
||||
/* number of adjacent descriptors at first_desc */
|
||||
uint32_t first_desc_adjacent;
|
||||
uint32_t credits;
|
||||
} __packed;
|
||||
|
||||
/* Engine struct */
|
||||
struct xdma_engine {
|
||||
int channel; // egnine channel
|
||||
char *name; // engine name
|
||||
struct pci_drvdata *pd; // PCI device
|
||||
|
||||
struct engine_regs *regs; // HW regs, control and status
|
||||
struct engine_sgdma_regs *sgdma_regs; // SGDMA reg BAR offset
|
||||
|
||||
// Config
|
||||
int running; // engine state
|
||||
int c2h; // c2h(write) or h2c(read)
|
||||
uint32_t status;
|
||||
|
||||
int addr_align; // source/dest alignment in bytes
|
||||
int len_granularity; // transfer length multiple
|
||||
int addr_bits; // HW datapath address width
|
||||
|
||||
/* Members associated with polled mode support */
|
||||
uint8_t *poll_mode_addr_virt; /* virt addr for descriptor writeback */
|
||||
uint64_t poll_mode_phys_addr; /* bus addr for descriptor writeback */
|
||||
};
|
||||
|
||||
/* Mapped user pages */
|
||||
struct user_pages {
|
||||
struct hlist_node entry;
|
||||
uint64_t vaddr;
|
||||
bool huge;
|
||||
int32_t cpid;
|
||||
uint64_t n_hpages;
|
||||
uint64_t n_pages;
|
||||
struct page **hpages;
|
||||
uint64_t *cpages;
|
||||
};
|
||||
/* User tables */
|
||||
struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
|
||||
struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
|
||||
|
||||
/* Mapped large PR pages */
|
||||
struct pr_pages {
|
||||
struct hlist_node entry;
|
||||
int reg_id;
|
||||
uint64_t vaddr;
|
||||
uint64_t n_pages;
|
||||
struct page **pages;
|
||||
};
|
||||
/* PR table */
|
||||
struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
|
||||
|
||||
/* Pool chunks */
|
||||
struct chunk {
|
||||
uint32_t id;
|
||||
struct chunk *next;
|
||||
};
|
||||
|
||||
/* Virtual FPGA device */
|
||||
struct fpga_dev {
|
||||
int id; // identifier
|
||||
struct cdev cdev; // char device
|
||||
struct pci_drvdata *pd; // PCI device
|
||||
struct pr_ctrl *prc; // PR controller
|
||||
|
||||
// Current task
|
||||
struct task_struct *curr_task;
|
||||
struct mm_struct *curr_mm;
|
||||
|
||||
// Control region
|
||||
uint64_t fpga_phys_addr_ctrl;
|
||||
uint64_t fpga_phys_addr_ctrl_avx;
|
||||
|
||||
// Writeback
|
||||
uint32_t *wb_addr_virt;
|
||||
uint64_t wb_phys_addr;
|
||||
|
||||
// TLBs
|
||||
uint64_t *fpga_lTlb; // large page TLB
|
||||
uint64_t *fpga_sTlb; // small page TLB
|
||||
struct fpga_cnfg_regs *fpga_cnfg; // config
|
||||
struct fpga_cnfg_regs_avx *fpga_cnfg_avx; // config AVX
|
||||
|
||||
// PIDs
|
||||
spinlock_t card_pid_lock;
|
||||
struct chunk *pid_chunks;
|
||||
pid_t *pid_array;
|
||||
int num_free_pid_chunks;
|
||||
struct chunk *pid_alloc;
|
||||
|
||||
// Engines
|
||||
struct xdma_engine *engine_h2c; // h2c engine
|
||||
struct xdma_engine *engine_c2h; // c2h engine
|
||||
|
||||
// In use
|
||||
atomic_t in_use; // busy flag
|
||||
|
||||
// Lock
|
||||
spinlock_t lock; // protects concurrent accesses
|
||||
|
||||
// Allocated buffers
|
||||
struct user_pages curr_user_buff;
|
||||
};
|
||||
|
||||
/* PR controller */
|
||||
struct pr_ctrl {
|
||||
struct pci_drvdata *pd; // PCI device
|
||||
spinlock_t lock;
|
||||
|
||||
// Engines
|
||||
struct xdma_engine *engine_h2c; // h2c engine
|
||||
struct xdma_engine *engine_c2h; // c2h engine
|
||||
|
||||
// Allocated buffers
|
||||
struct pr_pages curr_buff;
|
||||
};
|
||||
|
||||
/* TLB order */
|
||||
struct tlb_order {
|
||||
bool hugepage;
|
||||
uint64_t page_shift;
|
||||
uint64_t page_size;
|
||||
uint64_t page_mask;
|
||||
int assoc;
|
||||
|
||||
int key_mask;
|
||||
int key_size;
|
||||
int tag_mask;
|
||||
int tag_size;
|
||||
int phy_mask;
|
||||
int phy_size;
|
||||
};
|
||||
|
||||
/* PCI driver data */
|
||||
struct pci_drvdata {
|
||||
struct pci_dev *pci_dev;
|
||||
|
||||
// BARs
|
||||
int regions_in_use;
|
||||
int got_regions;
|
||||
void *__iomem bar[MAX_NUM_BARS];
|
||||
unsigned long bar_phys_addr[MAX_NUM_BARS];
|
||||
unsigned long bar_len[MAX_NUM_BARS];
|
||||
|
||||
// Engines
|
||||
int engines_num;
|
||||
|
||||
// FPGA static config
|
||||
uint probe;
|
||||
int n_fpga_chan;
|
||||
int n_fpga_reg;
|
||||
int en_avx;
|
||||
int en_bypass;
|
||||
int en_tlbf;
|
||||
int en_wb;
|
||||
int en_ddr;
|
||||
int en_hbm;
|
||||
int en_mem;
|
||||
int n_ddr_chan;
|
||||
int en_pr;
|
||||
int en_rdma_0;
|
||||
int en_rdma_1;
|
||||
int en_tcp_0;
|
||||
int en_tcp_1;
|
||||
int en_net_0;
|
||||
int en_net_1;
|
||||
struct fpga_stat_cnfg_regs *fpga_stat_cnfg;
|
||||
struct fpga_dev *fpga_dev;
|
||||
|
||||
// PR control
|
||||
struct pr_ctrl prc;
|
||||
|
||||
// TLB order
|
||||
struct tlb_order *stlb_order;
|
||||
struct tlb_order *ltlb_order;
|
||||
|
||||
// Locks
|
||||
spinlock_t stat_lock;
|
||||
spinlock_t prc_lock;
|
||||
spinlock_t tlb_lock;
|
||||
|
||||
// IRQ
|
||||
int irq_count;
|
||||
int irq_line;
|
||||
int msix_enabled;
|
||||
struct msix_entry irq_entry[32];
|
||||
|
||||
// Card memory
|
||||
spinlock_t card_l_lock;
|
||||
struct chunk *lchunks;
|
||||
int num_free_lchunks;
|
||||
struct chunk *lalloc;
|
||||
|
||||
spinlock_t card_s_lock;
|
||||
struct chunk *schunks;
|
||||
int num_free_schunks;
|
||||
struct chunk *salloc;
|
||||
};
|
||||
|
||||
#endif
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,80 @@
|
|||
/**
|
||||
* Copyright (c) 2021, Systems Group, ETH Zurich
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification,
|
||||
* are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
||||
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __PCI_DEV_H__
|
||||
#define __PCI_DEV_H__
|
||||
|
||||
#include "../coyote_dev.h"
|
||||
#include "../fpga_dev.h"
|
||||
#include "../fpga_isr.h"
|
||||
|
||||
/* Interrupts */
|
||||
irqreturn_t fpga_tlb_miss_isr(int irq, void *dev_id);
|
||||
void user_interrupts_enable(struct bus_drvdata *d, uint32_t mask);
|
||||
void user_interrupts_disable(struct bus_drvdata *d, uint32_t mask);
|
||||
uint32_t read_interrupts(struct bus_drvdata *d);
|
||||
uint32_t build_vector_reg(uint32_t a, uint32_t b, uint32_t c, uint32_t d);
|
||||
void write_msix_vectors(struct bus_drvdata *d);
|
||||
int msix_irq_setup(struct bus_drvdata *d);
|
||||
int irq_setup(struct bus_drvdata *d, struct pci_dev *pdev);
|
||||
void irq_teardown(struct bus_drvdata *d);
|
||||
int msix_capable(struct pci_dev *pdev, int type);
|
||||
int pci_check_msix(struct bus_drvdata *d, struct pci_dev *pdev);
|
||||
|
||||
/* Engine */
|
||||
uint32_t get_engine_channel_id(struct engine_regs *regs);
|
||||
uint32_t get_engine_id(struct engine_regs *regs);
|
||||
void engine_destroy(struct bus_drvdata *d, struct xdma_engine *engine);
|
||||
void remove_engines(struct bus_drvdata *d);
|
||||
void engine_alignments(struct xdma_engine *engine);
|
||||
struct xdma_engine *engine_create(struct bus_drvdata *d, int offs, int c2h, int channel);
|
||||
int probe_for_engine(struct bus_drvdata *d, int c2h, int channel);
|
||||
int probe_engines(struct bus_drvdata *d);
|
||||
uint32_t engine_status_read(struct xdma_engine *engine);
|
||||
//#define XDMA_WBACK
|
||||
#ifdef XDMA_WBACK
|
||||
void engine_writeback_teardown(struct bus_drvdata *d, struct xdma_engine *engine);
|
||||
int engine_writeback_setup(struct bus_drvdata *d, struct xdma_engine *engine);
|
||||
#endif
|
||||
|
||||
/* BARs */
|
||||
int map_single_bar(struct bus_drvdata *d, struct pci_dev *pdev, int idx, int curr_idx);
|
||||
void unmap_bars(struct bus_drvdata *d, struct pci_dev *pdev);
|
||||
int map_bars(struct bus_drvdata *d, struct pci_dev *pdev);
|
||||
|
||||
/* Regions */
|
||||
int request_regions(struct bus_drvdata *d, struct pci_dev *pdev);
|
||||
|
||||
/* Probe */
|
||||
int pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
|
||||
void pci_remove(struct pci_dev *pdev);
|
||||
|
||||
/* Init */
|
||||
int pci_init(void);
|
||||
void pci_exit(void);
|
||||
|
||||
#endif // PCIe device
|
Loading…
Reference in New Issue