replica
/
Coyote
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

u280 timing update.

This commit is contained in:
d-kor 2022-05-17 16:14:15 +02:00
parent 334b4ce9d7
commit ec4c4f5b0b
19 changed files with 8319 additions and 3695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
driver/Makefile
View File

@ -1,4 +1,5 @@
obj-m := fpga_drv.o
obj-m := coyote_drv.o
coyote_drv-objs:= fpga_drv.o fpga_isr.o fpga_fops.o fpga_dev.o fpga_mmu.o pci/pci_dev.o eci/eci_dev.o
KERNELDIR ?= /lib/modules/$(shell uname -r)/build
PWD := $(shell pwd)
@ -10,4 +11,4 @@ all:
$(MAKE) -C $(KERNELDIR) M=$(PWD) modules
clean:
rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.c .tmp_versions *.order *.symvers *.dwo *.mod *.mod.dwo
rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.c .tmp_versions *.order *.symvers *.mod *mod.dwo *.dwo .??* pci/.??* pci/*.o eci/.??* eci/*.o

715
driver/coyote_dev.h Normal file
View File

@ -0,0 +1,715 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __COYOTE_DEV_H__
#define __COYOTE_DEV_H__
#include <asm/io.h>
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/hrtimer.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqchip/chained_irq.h>
#include <linux/irqdomain.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/types.h>
#include <linux/cdev.h>
#include <linux/pagemap.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/version.h>
#include <linux/ioctl.h>
#include <linux/compiler.h>
#include <linux/msi.h>
#include <linux/poll.h>
#include <linux/sched.h>
#include <asm/delay.h>
#include <asm/set_memory.h>
#include <linux/hashtable.h>
#include <linux/moduleparam.h>
#include <linux/stat.h>
/**
* @brief Args
*
*/
#define CYT_ARCH_PCI 0
#define CYT_ARCH_ECI 1
static int cyt_arch = CYT_ARCH_PCI;
module_param(cyt_arch, int, S_IRUSR);
MODULE_PARM_DESC(cyt_arch, "target architecture");
/**
* @brief Info
*
*/
/* Driver info */
#define DRV_NAME "coyote_driver"
#define DEV_NAME "fpga"
/**
* @brief Util
*
*/
/* Debug print */
#define COYOTE_DEBUG 1
#if (COYOTE_DEBUG == 0)
#define dbg_info(...)
#else
#define dbg_info(fmt, ...) pr_info("%s():" fmt, \
__func__, ##__VA_ARGS__)
#endif
/* Obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
#define HIGH_32(addr) ((addr >> 16) >> 16)
/* Obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
#define LOW_32(addr) (addr & 0xffffffffUL)
/**
* @brief Bus
*
*/
/* XDMA info */
#define MAX_NUM_BARS 3
#define MAX_NUM_CHANNELS 4
#define MAX_NUM_ENGINES (MAX_NUM_CHANNELS * 2)
#define MAX_USER_IRQS 16
#define C2H_CHAN_OFFS 0x1000
#define H2C_CHAN_OFFS 0x0000
#define CHAN_RANGE 0x100
#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
/* Engine IDs */
#define XDMA_ID_H2C 0x1fc0U
#define XDMA_ID_C2H 0x1fc1U
/* Engine regs */
#define XDMA_ENG_IRQ_NUM (1)
#define XDMA_OFS_INT_CTRL (0x2000UL)
#define XDMA_OFS_CONFIG (0x3000UL)
/* Bits of the SG DMA control register */
#define XDMA_CTRL_RUN_STOP (1UL << 0)
#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
/* Bits of the SG DMA status register */
#define XDMA_STAT_BUSY (1UL << 0)
#define XDMA_STAT_DESC_STOPPED (1UL << 1)
#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
#define XDMA_STAT_FETCH_STOPPED (1UL << 5)
#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
#define XDMA_STAT_READ_ERROR (0x1FUL << 9)
#define XDMA_STAT_DESC_ERROR (0x1FUL << 19)
/* Bits of the performance control register */
#define XDMA_PERF_RUN (1UL << 0)
#define XDMA_PERF_CLEAR (1UL << 1)
#define XDMA_PERF_AUTO (1UL << 2)
/* Polling */
#define WB_COUNT_MASK 0x00ffffffUL
#define WB_ERR_MASK (1UL << 31)
#define POLL_TIMEOUT_SECONDS 10
#define NUM_POLLS_PER_SCHED 100
/* Network */
#define BASE_IP_ADDR_0 0x0A01D497
#define BASE_IP_ADDR_1 0x0A01D497
#define NODE_ID 0
#define N_TOTAL_NODES 2
/* Physical address (ECI) */
#define IO_PHYS_ADDR 0x900000000000UL
/**
* @brief Static layer
*
*/
#define BAR_XDMA_CONFIG 0
#define BAR_FPGA_CONFIG 1
/* FPGA static config */
#define FPGA_STAT_CNFG_OFFS 0x0
#define FPGA_STAT_CNFG_SIZE 32 * 1024
#define EN_AVX_MASK 0x1
#define EN_AVX_SHFT 0x0
#define EN_BPSS_MASK 0x2
#define EN_BPSS_SHFT 0x1
#define EN_TLBF_MASK 0x4
#define EN_TLBF_SHFT 0x2
#define EN_WB_MASK 0x8
#define EN_WB_SHFT 0x3
#define TLB_S_ORDER_MASK 0xf0
#define TLB_S_ORDER_SHFT 0x4
#define TLB_S_ASSOC_MASK 0xf00
#define TLB_S_ASSOC_SHFT 0x8
#define TLB_L_ORDER_MASK 0xf000
#define TLB_L_ORDER_SHFT 0xc
#define TLB_L_ASSOC_MASK 0xf0000
#define TLB_L_ASSOC_SHFT 0x10
#define TLB_S_PG_SHFT_MASK 0x3f00000
#define TLB_S_PG_SHFT_SHFT 0x14
#define TLB_L_PG_SHFT_MASK 0xfc000000
#define TLB_L_PG_SHFT_SHFT 0x1a
#define EN_DDR_MASK 0x1
#define EN_DDR_SHFT 0x0
#define EN_HBM_MASK 0x2
#define EN_HBM_SHFT 0x1
#define N_MEM_CHAN_MASK 0x7d
#define N_MEM_CHAN_SHFT 0x2
#define EN_RDMA_0_MASK 0x1
#define EN_RDMA_1_MASK 0x2
#define EN_PR_MASK 0x1
#define EN_PR_SHFT 0x0
#define EN_RDMA_0_MASK 0x1
#define EN_RDMA_0_SHFT 0x0
#define EN_RDMA_1_MASK 0x2
#define EN_RDMA_1_SHFT 0x1
#define EN_TCP_0_MASK 0x1
#define EN_TCP_0_SHFT 0x0
#define EN_TCP_1_MASK 0x2
#define EN_TCP_1_SHFT 0x1
/**
* @brief Dynamic layer
*
*/
/* FPGA control regions */
#define FPGA_CTRL_SIZE 256 * 1024
#define FPGA_CTRL_OFFS 0x100000
#define FPGA_CTRL_LTLB_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_LTLB_OFFS 0x0
#define FPGA_CTRL_STLB_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_STLB_OFFS 0x10000
#define FPGA_CTRL_USER_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_USER_OFFS 0x20000
#define FPGA_CTRL_CNFG_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_CNFG_OFFS 0x30000
#define FPGA_CTRL_CNFG_AVX_SIZE 256 * 1024
#define FPGA_CTRL_CNFG_AVX_OFFS 0x1000000
/* FPGA dynamic control config */
#define FPGA_CNFG_CTRL_IRQ_RESTART 0x100
/* Maximum transfer size */
#define TRANSFER_MAX_BYTES (8 * 1024 * 1024)
/* TLB */
#define TLB_VADDR_RANGE 48
#define TLB_PADDR_RANGE 40
#define PID_SIZE 6
#define MAX_MAP_AXIL_PAGES 64
#define LTLB_PAGE_BITS 21
#define STLB_PAGE_BITS 12
#define LTLB_PADDR_SIZE (TLB_PADDR_RANGE - LTLB_PAGE_BITS)
#define STLB_PADDR_SIZE (TLB_PADDR_RANGE - STLB_PAGE_BITS)
#define TLBF_CTRL_START 0x7
#define TLBF_CTRL_ID_MASK 0xff
#define TLBF_CTRL_ID_SHFT 0x3
#define TLBF_STAT_DONE 0x1
/* Memory allocation */
#define MAX_BUFF_NUM 64 // maximum number of huge pages allowed
#define MAX_PR_BUFF_NUM 64 // maximum number of huge pages allowed
#define MAX_N_MAP_PAGES 128 // TODO: link to params, max no pfault 1M
#define MAX_N_MAP_HUGE_PAGES (16 * 512) // max no pfault 32M
/* Max card pages */
#define N_LARGE_CHUNKS 512
#define N_SMALL_CHUNKS (256 * 1024)
#define MEM_SEP (N_SMALL_CHUNKS * (4 * 1024))
#define MAX_N_REGIONS 16
#define SMALL_CHUNK_ALLOC 0
#define LARGE_CHUNK_ALLOC 1
/* PR */
#define PR_CTRL_START_MIDDLE 0x3
#define PR_CTRL_START_LAST 0x7
#define PR_STAT_DONE 0x1
/**
* @brief Cdev
*
*/
/* Major number */
#define FPGA_MAJOR 0 // dynamic
/* MMAP */
#define MMAP_CTRL 0x0
#define MMAP_CNFG 0x1
#define MMAP_CNFG_AVX 0x2
#define MMAP_WB 0x3
#define MMAP_BUFF 0x200
#define MMAP_PR 0x400
/* IOCTL */
#define IOCTL_ALLOC_HOST_USER_MEM _IOW('D', 1, unsigned long) // large pages (no hugepage support)
#define IOCTL_FREE_HOST_USER_MEM _IOW('D', 2, unsigned long)
#define IOCTL_ALLOC_HOST_PR_MEM _IOW('D', 3, unsigned long) // pr pages
#define IOCTL_FREE_HOST_PR_MEM _IOW('D', 4, unsigned long)
#define IOCTL_MAP_USER _IOW('D', 5, unsigned long) // map
#define IOCTL_UNMAP_USER _IOW('D', 6, unsigned long)
#define IOCTL_REGISTER_PID _IOW('D', 7, unsigned long) // register pid
#define IOCTL_UNREGISTER_PID _IOW('D', 8, unsigned long)
#define IOCTL_RECONFIG_LOAD _IOW('D', 9, unsigned long) // reconfiguration
#define IOCTL_ARP_LOOKUP _IOW('D', 10, unsigned long) // arp lookup
#define IOCTL_SET_IP_ADDRESS _IOW('D', 11, unsigned long)
#define IOCTL_SET_BOARD_NUM _IOW('D', 12, unsigned long)
#define IOCTL_WRITE_CTX _IOW('D', 13, unsigned long) // qp context
#define IOCTL_WRITE_CONN _IOW('D', 14, unsigned long) // qp connection
#define IOCTL_SET_TCP_OFFS _IOW('D', 15, unsigned long) // tcp mem offsets
#define IOCTL_READ_CNFG _IOR('D', 32, unsigned long) // status cnfg
#define IOCTL_NET_STATS _IOR('D', 33, unsigned long) // status network
#define IOCTL_READ_ENG_STATUS _IOR('D', 35, unsigned long) // status engines
/* Hash */
#define PR_HASH_TABLE_ORDER 8
#define PR_BATCH_SIZE (2 * 1024 * 1024)
#define USER_HASH_TABLE_ORDER 8
/* PID */
#define N_CPID_MAX 64
#define WB_SIZE (2 * N_CPID_MAX * sizeof(uint32_t))
#define N_WB_PAGES ((WB_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
/* Network */
#define EN_LOWSPEED 0x5
#define N_NET_STAT_REGS 9
/* Copy */
#define MAX_USER_WORDS 32
/**
* @brief Reg maps
*
*/
/* DMA engine reg map */
struct engine_regs {
uint32_t id;
uint32_t ctrl;
uint32_t ctrl_w1s;
uint32_t ctrl_w1c;
uint32_t rsrvd_1[12];
uint32_t status;
uint32_t status_rc;
uint32_t completed_desc_count;
uint32_t alignments;
uint32_t rsrvd_2[14]; // padding
uint32_t poll_mode_wb_lo;
uint32_t poll_mode_wb_hi;
uint32_t interrupt_enable_mask;
uint32_t interrupt_enable_mask_w1s;
uint32_t interrupt_enable_mask_w1c;
uint32_t rsrvd_3[9]; // padding
uint32_t perf_ctrl;
uint32_t perf_cyc_lo;
uint32_t perf_cyc_hi;
uint32_t perf_dat_lo;
uint32_t perf_dat_hi;
uint32_t perf_pnd_lo;
uint32_t perf_pnd_hi;
} __packed;
/* Interrupt reg map */
struct interrupt_regs {
uint32_t id;
uint32_t user_int_enable;
uint32_t user_int_enable_w1s;
uint32_t user_int_enable_w1c;
uint32_t channel_int_enable;
uint32_t channel_int_enable_w1s;
uint32_t channel_int_enable_w1c;
uint32_t reserved_1[9]; // padding
uint32_t user_int_request;
uint32_t channel_int_request;
uint32_t user_int_pending;
uint32_t channel_int_pending;
uint32_t reserved_2[12]; // padding
uint32_t user_msi_vector[8];
uint32_t channel_msi_vector[8];
} __packed;
/* Polled mode descriptors struct */
struct xdma_poll_wb {
uint32_t completed_desc_count;
uint32_t reserved_1[7];
} __packed;
/* FPGA static config reg map */
struct fpga_stat_cnfg_regs {
uint64_t probe;
uint64_t n_chan;
uint64_t n_regions;
uint64_t ctrl_cnfg;
uint64_t mem_cnfg;
uint64_t pr_cnfg;
uint64_t rdma_cnfg;
uint64_t tcp_cnfg;
uint64_t lspeed_cnfg;
uint64_t reserved_0[1];
uint64_t pr_ctrl;
uint64_t pr_stat;
uint64_t pr_addr;
uint64_t pr_len;
uint64_t tlb_ctrl;
uint64_t tlb_stat;
uint64_t tlb_addr;
uint64_t tlb_len;
uint64_t reserved_1[2];
uint64_t net_0_ip;
uint64_t net_0_boardnum;
uint64_t net_0_arp;
uint64_t rdma_0_qp_ctx[3];
uint64_t rdma_0_qp_conn[3];
uint64_t tcp_0_offs[2];
uint64_t net_0_debug[N_NET_STAT_REGS];
uint64_t net_1_ip;
uint64_t net_1_boardnum;
uint64_t net_1_arp;
uint64_t rdma_1_qp_ctx[3];
uint64_t rdma_1_qp_conn[3];
uint64_t tcp_1_offs[2];
uint64_t net_1_debug[N_NET_STAT_REGS];
} __packed;
/* FPGA dynamic config reg map */
struct fpga_cnfg_regs {
uint64_t ctrl;
uint64_t vaddr_rd;
uint64_t len_rd;
uint64_t vaddr_wr;
uint64_t len_wr;
uint64_t vaddr_miss;
uint64_t len_miss;
uint64_t datapath_set;
uint64_t datapath_clr;
uint64_t stat_cmd_used_rd;
uint64_t stat_cmd_used_wr;
uint64_t stat_sent[6];
uint64_t stat_pfaults;
uint64_t wback_rd;
uint64_t wback_wr;
// Rest of regs not used in the driver
} __packed;
/* FPGA dynamic config reg map */
struct fpga_cnfg_regs_avx {
uint64_t ctrl[4];
uint64_t vaddr_miss;
uint64_t len_miss;
uint64_t pf[2];
uint64_t datapath_set[4];
uint64_t datapath_clr[4];
uint64_t stat[4];
uint64_t wback[4];
// Rest not used in the driver
} __packed;
/**
* @brief Engine structs
*
*/
/* Engine descriptors */
struct xdma_sgdma_regs {
uint32_t identifier;
uint32_t reserved_1[31]; /* padding */
/* bus address to first descriptor in Root Complex Memory */
uint32_t first_desc_lo;
uint32_t first_desc_hi;
/* number of adjacent descriptors at first_desc */
uint32_t first_desc_adjacent;
uint32_t credits;
} __packed;
/* Engine struct */
struct xdma_engine {
int channel; // egnine channel
char *name; // engine name
struct bus_drvdata *pd; // PCI device
struct engine_regs *regs; // HW regs, control and status
struct engine_sgdma_regs *sgdma_regs; // SGDMA reg BAR offset
// Config
int running; // engine state
int c2h; // c2h(write) or h2c(read)
uint32_t status;
int addr_align; // source/dest alignment in bytes
int len_granularity; // transfer length multiple
int addr_bits; // HW datapath address width
/* Members associated with polled mode support */
uint8_t *poll_mode_addr_virt; /* virt addr for descriptor writeback */
uint64_t poll_mode_phys_addr; /* bus addr for descriptor writeback */
};
/**
* @brief Hash maps
*
*/
/* Mapped user pages */
struct user_pages {
struct hlist_node entry;
uint64_t vaddr;
bool huge;
int32_t cpid;
uint64_t n_hpages;
uint64_t n_pages;
struct page **hpages;
uint64_t *cpages;
};
/* Mapped large PR pages */
struct pr_pages {
struct hlist_node entry;
int reg_id;
uint64_t vaddr;
uint64_t n_pages;
struct page **pages;
};
/* User tables */
extern struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
extern struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
/* PR table */
extern struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
/**
* @brief Mem
*
*/
/* Pool chunks */
struct chunk {
uint32_t id;
struct chunk *next;
};
/* TLB order */
struct tlb_order {
bool hugepage;
uint64_t page_shift;
uint64_t page_size;
uint64_t page_mask;
int assoc;
int key_mask;
int key_size;
int tag_mask;
int tag_size;
int phy_mask;
int phy_size;
};
/**
* @brief Dev structs
*
*/
/* PR controller */
struct pr_ctrl {
struct bus_drvdata *pd; // PCI device
spinlock_t lock;
// Engines
struct xdma_engine *engine_h2c; // h2c engine
struct xdma_engine *engine_c2h; // c2h engine
// Allocated buffers
struct pr_pages curr_buff;
};
/* Virtual FPGA device */
struct fpga_dev {
int id; // identifier
struct cdev cdev; // char device
struct bus_drvdata *pd; // PCI device
struct pr_ctrl *prc; // PR controller
// Current task
struct task_struct *curr_task;
struct mm_struct *curr_mm;
// Control region
uint64_t fpga_phys_addr_ctrl;
uint64_t fpga_phys_addr_ctrl_avx;
// Writeback
uint32_t *wb_addr_virt;
uint64_t wb_phys_addr;
// TLBs
uint64_t *fpga_lTlb; // large page TLB
uint64_t *fpga_sTlb; // small page TLB
struct fpga_cnfg_regs *fpga_cnfg; // config
struct fpga_cnfg_regs_avx *fpga_cnfg_avx; // config AVX
// PIDs
spinlock_t card_pid_lock;
struct chunk *pid_chunks;
pid_t *pid_array;
int num_free_pid_chunks;
struct chunk *pid_alloc;
// Engines
struct xdma_engine *engine_h2c; // h2c engine
struct xdma_engine *engine_c2h; // c2h engine
// In use
atomic_t in_use; // busy flag
// Lock
spinlock_t lock; // protects concurrent accesses
// Allocated buffers
struct user_pages curr_user_buff;
};
/* PCI driver data */
struct bus_drvdata {
// PCI
struct pci_dev *pci_dev;
// BARs
int regions_in_use;
int got_regions;
void *__iomem bar[MAX_NUM_BARS];
unsigned long bar_phys_addr[MAX_NUM_BARS];
unsigned long bar_len[MAX_NUM_BARS];
// Engines
int engines_num;
// ECI
// I/O
unsigned long io_phys_addr;
unsigned long io_len;
// FPGA static config
uint probe;
int n_fpga_chan;
int n_fpga_reg;
int en_avx;
int en_bypass;
int en_tlbf;
int en_wb;
int en_ddr;
int en_hbm;
int en_mem;
int n_mem_chan;
int en_pr;
int en_rdma_0;
int en_rdma_1;
int en_tcp_0;
int en_tcp_1;
int en_net_0;
int en_net_1;
volatile struct fpga_stat_cnfg_regs *fpga_stat_cnfg;
struct fpga_dev *fpga_dev;
// PR control
struct pr_ctrl prc;
// TLB order
struct tlb_order *stlb_order;
struct tlb_order *ltlb_order;
// Locks
spinlock_t stat_lock;
spinlock_t prc_lock;
spinlock_t tlb_lock;
// IRQ
int irq_count;
int irq_line;
int msix_enabled;
struct msix_entry irq_entry[32];
// Card memory
spinlock_t card_l_lock;
struct chunk *lchunks;
int num_free_lchunks;
struct chunk *lalloc;
spinlock_t card_s_lock;
struct chunk *schunks;
int num_free_schunks;
struct chunk *salloc;
};
#endif // Coyote device

123
driver/eci/eci_dev.c Normal file
View File

@ -0,0 +1,123 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "eci_dev.h"
/*
____ _ _
| _ \ ___ __ _(_)___| |_ ___ _ __
| |_) / _ \/ _` | / __| __/ _ \ '__|
| _ < __/ (_| | \__ \ || __/ |
|_| \_\___|\__, |_|___/\__\___|_|
|___/
*/
static struct bus_drvdata *pd;
/**
* @brief ECI device init
*
*/
int eci_init(void)
{
int ret_val = 0;
// dynamic major
dev_t dev = MKDEV(fpga_major, 0);
// allocate mem. for device instance
pd = kzalloc(sizeof(struct bus_drvdata), GFP_KERNEL);
if(!pd) {
pr_err("device memory region not obtained\n");
ret_val = -ENOMEM;
goto err_alloc;
}
// get static config
pd->io_phys_addr = IO_PHYS_ADDR;
pd->fpga_stat_cnfg = ioremap(pd->io_phys_addr + FPGA_STAT_CNFG_OFFS, FPGA_STAT_CNFG_SIZE);
read_static_config(pd);
// allocate card mem resources
ret_val = alloc_card_resources(pd);
if (ret_val) {
pr_err("card resources could not be allocated\n");
goto err_card_alloc; // ERR_CARD_ALLOC
}
// initialize spin locks
init_spin_locks(pd);
// create FPGA devices and register major
ret_val = init_char_devices(pd, dev);
if (ret_val) {
goto err_create_fpga_dev; // ERR_CREATE_FPGA_DEV
}
// initialize vFPGAs
ret_val = init_fpga_devices(pd);
if (ret_val) {
goto err_init_fpga_dev;
}
// Init hash
hash_init(pr_buff_map);
if(ret_val == 0)
goto end;
err_init_fpga_dev:
kfree(pd->fpga_dev);
class_destroy(fpga_class);
err_create_fpga_dev:
vfree(pd->schunks);
vfree(pd->lchunks);
err_card_alloc:
err_alloc:
end:
pr_info("probe returning %d\n", ret_val);
return ret_val;
}
void eci_exit(void)
{
// delete vFPGAs
free_fpga_devices(pd);
// delete char devices
free_char_devices(pd);
// deallocate card resources
free_card_resources(pd);
// free device data
kfree(pd);
pr_info("device memory freed\n");
pr_info("removal completed\n");
}

38
driver/eci/eci_dev.h Normal file
View File

@ -0,0 +1,38 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __ECI_DEV_H__
#define __ECI_DEV_H__
#include "../coyote_dev.h"
#include "../fpga_dev.h"
/* Probe */
int eci_init(void);
void eci_exit(void);
#endif // ECI device

421
driver/fpga_dev.c Normal file
View File

@ -0,0 +1,421 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "fpga_dev.h"
int fpga_major = FPGA_MAJOR;
struct class *fpga_class = NULL;
/**
* @brief Fops
*
*/
struct file_operations fpga_fops = {
.owner = THIS_MODULE,
.open = fpga_open,
.release = fpga_release,
.mmap = fpga_mmap,
.unlocked_ioctl = fpga_ioctl,
};
/**
* @brief Read static configuration
*
*/
void read_static_config(struct bus_drvdata *d)
{
// probe
d->probe = d->fpga_stat_cnfg->probe;
pr_info("deployment id %08x\n", d->probe);
// channels and regions
d->n_fpga_chan = d->fpga_stat_cnfg->n_chan;
d->n_fpga_reg = d->fpga_stat_cnfg->n_regions;
pr_info("detected %d virtual FPGA regions, %d FPGA channels\n", d->n_fpga_reg, d->n_fpga_chan);
// flags
d->en_avx = (d->fpga_stat_cnfg->ctrl_cnfg & EN_AVX_MASK) >> EN_AVX_SHFT;
d->en_bypass = (d->fpga_stat_cnfg->ctrl_cnfg & EN_BPSS_MASK) >> EN_BPSS_SHFT;
d->en_tlbf = (d->fpga_stat_cnfg->ctrl_cnfg & EN_TLBF_MASK) >> EN_TLBF_SHFT;
d->en_wb = (d->fpga_stat_cnfg->ctrl_cnfg & EN_WB_MASK) >> EN_WB_SHFT;
pr_info("enabled AVX %d, enabled bypass %d, enabled tlb fast %d, enabled writeback %d\n", d->en_avx, d->en_bypass, d->en_tlbf, d->en_wb);
// mmu
d->stlb_order = kzalloc(sizeof(struct tlb_order), GFP_KERNEL);
BUG_ON(!d->stlb_order);
d->stlb_order->hugepage = false;
d->stlb_order->key_size = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_ORDER_MASK) >> TLB_S_ORDER_SHFT;
d->stlb_order->assoc = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_ASSOC_MASK) >> TLB_S_ASSOC_SHFT;
d->stlb_order->page_shift = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_PG_SHFT_MASK) >> TLB_S_PG_SHFT_SHFT;
BUG_ON(d->stlb_order->page_shift != PAGE_SHIFT);
d->stlb_order->page_size = PAGE_SIZE;
d->stlb_order->page_mask = PAGE_MASK;
d->stlb_order->key_mask = (1 << d->stlb_order->key_size) - 1;
d->stlb_order->tag_size = TLB_VADDR_RANGE - d->stlb_order->page_shift - d->stlb_order->key_size;
d->stlb_order->tag_mask = (1 << d->stlb_order->tag_size) - 1;
d->stlb_order->phy_size = TLB_PADDR_RANGE - d->stlb_order->page_shift;
d->stlb_order->phy_mask = (1 << d->stlb_order->phy_size) - 1;
pr_info("sTLB order %d, sTLB assoc %d, sTLB page size %lld\n", d->stlb_order->key_size, d->stlb_order->assoc, d->stlb_order->page_size);
d->ltlb_order = kzalloc(sizeof(struct tlb_order), GFP_KERNEL);
BUG_ON(!d->ltlb_order);
d->ltlb_order->hugepage = true;
d->ltlb_order->key_size = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_ORDER_MASK) >> TLB_L_ORDER_SHFT;
d->ltlb_order->assoc = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_ASSOC_MASK) >> TLB_L_ASSOC_SHFT;
d->ltlb_order->page_shift = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_PG_SHFT_MASK) >> TLB_L_PG_SHFT_SHFT;
d->ltlb_order->page_size = 1 << d->ltlb_order->page_shift;
d->ltlb_order->page_mask = (~(d->ltlb_order->page_size - 1));
d->ltlb_order->key_mask = (1 << d->ltlb_order->key_size) - 1;
d->ltlb_order->tag_size = TLB_VADDR_RANGE - d->ltlb_order->page_shift - d->ltlb_order->key_size;
d->ltlb_order->tag_mask = (1 << d->ltlb_order->tag_size) - 1;
d->ltlb_order->phy_size = TLB_PADDR_RANGE - d->ltlb_order->page_shift ;
d->ltlb_order->phy_mask = (1 << d->ltlb_order->phy_size) - 1;
pr_info("lTLB order %d, lTLB assoc %d, lTLB page size %lld\n", d->ltlb_order->key_size, d->ltlb_order->assoc, d->ltlb_order->page_size);
// mem
d->en_ddr = (d->fpga_stat_cnfg->mem_cnfg & EN_DDR_MASK) >> EN_DDR_SHFT;
d->en_hbm = (d->fpga_stat_cnfg->mem_cnfg & EN_HBM_MASK) >> EN_HBM_SHFT;
d->en_mem = d->en_ddr || d->en_hbm;
d->n_mem_chan = (d->fpga_stat_cnfg->mem_cnfg & N_MEM_CHAN_MASK) >> N_MEM_CHAN_SHFT;
pr_info("enabled DDR %d, number of DDR channels %d\n", d->en_ddr, d->n_mem_chan);
// pr
d->en_pr = (d->fpga_stat_cnfg->pr_cnfg & EN_PR_MASK) >> EN_PR_SHFT;
pr_info("enabled PR %d\n", d->en_pr);
// network
d->en_rdma_0 = (d->fpga_stat_cnfg->rdma_cnfg & EN_RDMA_0_MASK) >> EN_RDMA_0_SHFT;
d->en_rdma_1 = (d->fpga_stat_cnfg->rdma_cnfg & EN_RDMA_1_MASK) >> EN_RDMA_1_SHFT;
pr_info("enabled RDMA on QSFP0 %d, enabled RDMA on QSFP1 %d\n", d->en_rdma_0, d->en_rdma_1);
d->en_tcp_0 = (d->fpga_stat_cnfg->tcp_cnfg & EN_TCP_0_MASK) >> EN_TCP_0_SHFT;
d->en_tcp_1 = (d->fpga_stat_cnfg->tcp_cnfg & EN_TCP_1_MASK) >> EN_TCP_1_SHFT;
pr_info("enabled TCP/IP on QSFP0 %d, enabled TCP/IP on QSFP1 %d\n", d->en_tcp_0, d->en_tcp_1);
d->en_net_0 = d->en_rdma_0 | d->en_tcp_0;
d->en_net_1 = d->en_rdma_1 | d->en_tcp_1;
// network board setup (TODO: maybe move to sw fully?)
if (d->en_net_0) {
d->fpga_stat_cnfg->net_0_ip = BASE_IP_ADDR_0 + NODE_ID;
d->fpga_stat_cnfg->net_0_boardnum = NODE_ID;
}
if (d->en_net_1) {
d->fpga_stat_cnfg->net_1_ip = BASE_IP_ADDR_1 + NODE_ID;
d->fpga_stat_cnfg->net_1_boardnum = NODE_ID;
}
// lowspeed ctrl
d->fpga_stat_cnfg->lspeed_cnfg = EN_LOWSPEED;
}
/**
* @brief Allocate FPGA memory resources
*
*/
int alloc_card_resources(struct bus_drvdata *d)
{
int ret_val = 0;
int i;
// init chunks card
d->num_free_lchunks = N_LARGE_CHUNKS;
d->num_free_schunks = N_SMALL_CHUNKS;
d->lchunks = vzalloc(N_LARGE_CHUNKS * sizeof(struct chunk));
if (!d->lchunks) {
pr_err("memory region for cmem structs not obtained\n");
goto err_alloc_lchunks; // ERR_ALLOC_LCHUNKS
}
d->schunks = vzalloc(N_SMALL_CHUNKS * sizeof(struct chunk));
if (!d->schunks) {
pr_err("memory region for cmem structs not obtained\n");
goto err_alloc_schunks; // ERR_ALLOC_SCHUNKS
}
for (i = 0; i < N_LARGE_CHUNKS - 1; i++) {
d->lchunks[i].id = i;
d->lchunks[i].next = &d->lchunks[i + 1];
}
for (i = 0; i < N_SMALL_CHUNKS - 1; i++) {
d->schunks[i].id = i;
d->schunks[i].next = &d->schunks[i + 1];
}
d->lalloc = &d->lchunks[0];
d->salloc = &d->schunks[0];
goto end;
err_alloc_schunks:
vfree(d->lchunks);
err_alloc_lchunks:
ret_val = -ENOMEM;
end:
return ret_val;
}
/**
* @brief Free card memory resources
*
*/
void free_card_resources(struct bus_drvdata *d)
{
// free card memory structs
vfree(d->schunks);
vfree(d->lchunks);
pr_info("card resources deallocated\n");
}
/**
* @brief Initialize spin locks
*
*/
void init_spin_locks(struct bus_drvdata *d)
{
// initialize spinlocks
spin_lock_init(&d->card_l_lock);
spin_lock_init(&d->card_s_lock);
spin_lock_init(&d->prc.lock);
spin_lock_init(&d->stat_lock);
spin_lock_init(&d->prc_lock);
spin_lock_init(&d->tlb_lock);
}
/**
* @brief Init char devices
*
*/
int init_char_devices(struct bus_drvdata *d, dev_t dev)
{
int ret_val = 0;
ret_val = alloc_chrdev_region(&dev, 0, d->n_fpga_reg, DEV_NAME);
fpga_major = MAJOR(dev);
if (ret_val) {
pr_err("failed to register virtual FPGA devices");
goto end;
}
pr_info("virtual FPGA device regions allocated, major number %d\n", fpga_major);
// create device class
fpga_class = class_create(THIS_MODULE, DEV_NAME);
// virtual FPGA devices
d->fpga_dev = kmalloc(d->n_fpga_reg * sizeof(struct fpga_dev), GFP_KERNEL);
if (!d->fpga_dev) {
pr_err("could not allocate memory for fpga devices\n");
goto err_char_mem; // ERR_CHAR_MEM
}
memset(d->fpga_dev, 0, d->n_fpga_reg * sizeof(struct fpga_dev));
pr_info("allocated memory for fpga devices\n");
goto end;
err_char_mem:
unregister_chrdev_region(dev, d->n_fpga_reg);
ret_val = -ENOMEM;
end:
return ret_val;
}
/**
* @brief Delete char devices
*
*/
void free_char_devices(struct bus_drvdata *d)
{
// free virtual FPGA memory
kfree(d->fpga_dev);
pr_info("virtual FPGA device memory freed\n");
// remove class
class_destroy(fpga_class);
pr_info("fpga class deleted\n");
// remove char devices
unregister_chrdev_region(MKDEV(fpga_major, 0), d->n_fpga_reg);
pr_info("char devices unregistered\n");
}
/**
* @brief Initialize vFPGAs
*
*/
int init_fpga_devices(struct bus_drvdata *d)
{
int ret_val = 0;
int i, j;
int devno;
for (i = 0; i < d->n_fpga_reg; i++) {
// ID
d->fpga_dev[i].id = i;
// PCI device
d->fpga_dev[i].pd = d;
d->fpga_dev[i].prc = &d->prc;
// physical
if(cyt_arch == CYT_ARCH_PCI) {
d->fpga_dev[i].fpga_phys_addr_ctrl = d->bar_phys_addr[BAR_FPGA_CONFIG] + FPGA_CTRL_OFFS + i * FPGA_CTRL_SIZE;
d->fpga_dev[i].fpga_phys_addr_ctrl_avx = d->bar_phys_addr[BAR_FPGA_CONFIG] + FPGA_CTRL_CNFG_AVX_OFFS + i * FPGA_CTRL_CNFG_AVX_SIZE;
} else if(cyt_arch == CYT_ARCH_ECI) {
d->fpga_dev[i].fpga_phys_addr_ctrl = d->io_phys_addr + FPGA_CTRL_OFFS + i*FPGA_CTRL_SIZE;
}
// MMU control region
d->fpga_dev[i].fpga_lTlb = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_LTLB_OFFS, FPGA_CTRL_LTLB_SIZE);
d->fpga_dev[i].fpga_sTlb = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_STLB_OFFS, FPGA_CTRL_STLB_SIZE);
// FPGA engine control
d->fpga_dev[i].fpga_cnfg = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS, FPGA_CTRL_CNFG_SIZE);
// FPGA engine control AVX
if(cyt_arch == CYT_ARCH_PCI) {
d->fpga_dev[i].fpga_cnfg_avx = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl_avx, FPGA_CTRL_CNFG_AVX_SIZE);
}
// init chunks pid
d->fpga_dev[i].num_free_pid_chunks = N_CPID_MAX;
d->fpga_dev[i].pid_chunks = vzalloc(N_CPID_MAX * sizeof(struct chunk));
if (!d->fpga_dev[i].pid_chunks) {
pr_err("memory region for pid chunks not obtained\n");
goto err_alloc_pid_chunks; // ERR_ALLOC_PID_CHUNKS
}
d->fpga_dev[i].pid_array = vzalloc(N_CPID_MAX * sizeof(pid_t));
if (!d->fpga_dev[i].pid_array) {
pr_err("memory region for pid array not obtained\n");
goto err_alloc_pid_array; // ERR_ALLOC_PID_ARRAY
}
for (j = 0; j < N_CPID_MAX - 1; j++) {
d->fpga_dev[i].pid_chunks[j].id = j;
d->fpga_dev[i].pid_chunks[j].next = &d->fpga_dev[i].pid_chunks[j + 1];
}
d->fpga_dev[i].pid_alloc = &d->fpga_dev[i].pid_chunks[0];
// initialize device spinlock
spin_lock_init(&d->fpga_dev[i].lock);
spin_lock_init(&d->fpga_dev[i].card_pid_lock);
// writeback setup
if(d->en_wb) {
d->fpga_dev[i].wb_addr_virt = pci_alloc_consistent(d->pci_dev, WB_SIZE, &d->fpga_dev[i].wb_phys_addr);
if(!d->fpga_dev[i].wb_addr_virt) {
pr_err("failed to allocate writeback memory\n");
goto err_wb;
}
if(cyt_arch == CYT_ARCH_PCI && d->en_avx) {
d->fpga_dev[i].fpga_cnfg_avx->wback[0] = d->fpga_dev[i].wb_phys_addr;
d->fpga_dev[i].fpga_cnfg_avx->wback[1] = d->fpga_dev[i].wb_phys_addr + N_CPID_MAX * sizeof(uint32_t);
} else {
d->fpga_dev[i].fpga_cnfg->wback_rd = d->fpga_dev[i].wb_phys_addr;
d->fpga_dev[i].fpga_cnfg->wback_wr = d->fpga_dev[i].wb_phys_addr + N_CPID_MAX * sizeof(uint32_t);
}
pr_info("allocated memory for descriptor writeback, vaddr %llx, paddr %llx",
(uint64_t)d->fpga_dev[i].wb_addr_virt, d->fpga_dev[i].wb_phys_addr);
}
// create device
devno = MKDEV(fpga_major, i);
device_create(fpga_class, NULL, devno, NULL, DEV_NAME "%d", i);
pr_info("virtual FPGA device %d created\n", i);
// add device
cdev_init(&d->fpga_dev[i].cdev, &fpga_fops);
d->fpga_dev[i].cdev.owner = THIS_MODULE;
d->fpga_dev[i].cdev.ops = &fpga_fops;
// Init hash
hash_init(user_lbuff_map[i]);
hash_init(user_sbuff_map[i]);
ret_val = cdev_add(&d->fpga_dev[i].cdev, devno, 1);
if (ret_val) {
pr_err("could not create a virtual FPGA device %d\n", i);
goto err_char_reg;
}
}
pr_info("all virtual FPGA devices added\n");
goto end;
err_char_reg:
for (j = 0; j < i; j++) {
device_destroy(fpga_class, MKDEV(fpga_major, j));
cdev_del(&d->fpga_dev[j].cdev);
}
err_wb:
if(d->en_wb) {
for (j = 0; j < i; j++) {
set_memory_wb((uint64_t)d->fpga_dev[j].wb_addr_virt, N_WB_PAGES);
pci_free_consistent(d->pci_dev, WB_SIZE,
d->fpga_dev[j].wb_addr_virt, d->fpga_dev[j].wb_phys_addr);
}
}
vfree(d->fpga_dev[i].pid_array);
err_alloc_pid_array:
for (j = 0; j < i; j++) {
vfree(d->fpga_dev[j].pid_array);
}
vfree(d->fpga_dev[i].pid_chunks);
err_alloc_pid_chunks:
for (j = 0; j < i; j++) {
vfree(d->fpga_dev[j].pid_chunks);
}
ret_val = -ENOMEM;
end:
return ret_val;
}
/**
* @brief Delete vFPGAs
*
*/
void free_fpga_devices(struct bus_drvdata *d) {
int i;
for(i = 0; i < d->n_fpga_reg; i++) {
device_destroy(fpga_class, MKDEV(fpga_major, i));
cdev_del(&d->fpga_dev[i].cdev);
if(d->en_wb) {
set_memory_wb((uint64_t)d->fpga_dev[i].wb_addr_virt, N_WB_PAGES);
pci_free_consistent(d->pci_dev, WB_SIZE,
d->fpga_dev[i].wb_addr_virt, d->fpga_dev[i].wb_phys_addr);
}
vfree(d->fpga_dev[i].pid_array);
vfree(d->fpga_dev[i].pid_chunks);
}
pr_info("vFPGAs deleted\n");
}

58
driver/fpga_dev.h Normal file
View File

@ -0,0 +1,58 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __FPGA_DEV_H__
#define __FPGA_DEV_H__
#include "coyote_dev.h"
#include "fpga_fops.h"
/**
* @brief Global
*
*/
extern int fpga_major;// = FPGA_MAJOR;
extern struct class *fpga_class;// = NULL;
/* Read deployment config */
void read_static_config(struct bus_drvdata *d);
/* Allocate initial card resources */
int alloc_card_resources(struct bus_drvdata *d);
void free_card_resources(struct bus_drvdata *d);
/* Spinlock init */
void init_spin_locks(struct bus_drvdata *d);
/* Initialize vFPGAs */
int init_char_devices(struct bus_drvdata *d, dev_t dev);
void free_char_devices(struct bus_drvdata *d);
int init_fpga_devices(struct bus_drvdata *d);
void free_fpga_devices(struct bus_drvdata *d);
#endif // FPGA DEV

3134
driver/fpga_drv.c
View File

File diff suppressed because it is too large Load Diff

632
driver/fpga_drv.h
View File

@ -1,604 +1,40 @@
#ifndef COYOTE_DRV_H
#define COYOTE_DRV_H
#include <linux/ioctl.h>
#include <linux/cdev.h>
#include <linux/pci.h>
#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/hashtable.h>
/* Driver debug */
#define COYOTE_DEBUG 1
/* Network setup */
#define BASE_IP_ADDR_0 0x0B01D4D1
#define BASE_IP_ADDR_1 0x0B01D4D1
#define NODE_ID 0
#define N_TOTAL_NODES 2
/* Debug print */
#if (COYOTE_DEBUG == 0)
#define dbg_info(...)
#else
#define dbg_info(fmt, ...) pr_info("%s():" fmt, \
__func__, ##__VA_ARGS__)
#endif
/* Obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
#define HIGH_32(addr) ((addr >> 16) >> 16)
/* Obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
#define LOW_32(addr) (addr & 0xffffffffUL)
/* Driver info */
#define DRV_NAME "coyote_driver"
#define DEV_NAME "fpga"
/**
* XDMA info
*/
#define MAX_NUM_BARS 3
#define MAX_NUM_CHANNELS 4
#define MAX_NUM_ENGINES (MAX_NUM_CHANNELS * 2)
#define MAX_USER_IRQS 16
#define C2H_CHAN_OFFS 0x1000
#define H2C_CHAN_OFFS 0x0000
#define CHAN_RANGE 0x100
#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
/* Engine IDs */
#define XDMA_ID_H2C 0x1fc0U
#define XDMA_ID_C2H 0x1fc1U
/* Engine regs */
#define XDMA_ENG_IRQ_NUM (1)
#define XDMA_OFS_INT_CTRL (0x2000UL)
#define XDMA_OFS_CONFIG (0x3000UL)
/* Bits of the SG DMA control register */
#define XDMA_CTRL_RUN_STOP (1UL << 0)
#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
/* Bits of the SG DMA status register */
#define XDMA_STAT_BUSY (1UL << 0)
#define XDMA_STAT_DESC_STOPPED (1UL << 1)
#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
#define XDMA_STAT_FETCH_STOPPED (1UL << 5)
#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
#define XDMA_STAT_READ_ERROR (0x1FUL << 9)
#define XDMA_STAT_DESC_ERROR (0x1FUL << 19)
/* Bits of the performance control register */
#define XDMA_PERF_RUN (1UL << 0)
#define XDMA_PERF_CLEAR (1UL << 1)
#define XDMA_PERF_AUTO (1UL << 2)
/* Polling */
#define WB_COUNT_MASK 0x00ffffffUL
#define WB_ERR_MASK (1UL << 31)
#define POLL_TIMEOUT_SECONDS 10
#define NUM_POLLS_PER_SCHED 100
/**
* Static layer
*/
#define BAR_XDMA_CONFIG 0
#define BAR_FPGA_CONFIG 1
/* FPGA static config */
#define FPGA_STAT_CNFG_OFFS 0x0
#define FPGA_STAT_CNFG_SIZE 32 * 1024
#define EN_AVX_MASK 0x1
#define EN_AVX_SHFT 0x0
#define EN_BPSS_MASK 0x2
#define EN_BPSS_SHFT 0x1
#define EN_TLBF_MASK 0x4
#define EN_TLBF_SHFT 0x2
#define EN_WB_MASK 0x8
#define EN_WB_SHFT 0x3
#define TLB_S_ORDER_MASK 0xf0
#define TLB_S_ORDER_SHFT 0x4
#define TLB_S_ASSOC_MASK 0xf00
#define TLB_S_ASSOC_SHFT 0x8
#define TLB_L_ORDER_MASK 0xf000
#define TLB_L_ORDER_SHFT 0xc
#define TLB_L_ASSOC_MASK 0xf0000
#define TLB_L_ASSOC_SHFT 0x10
#define TLB_S_PG_SHFT_MASK 0x3f00000
#define TLB_S_PG_SHFT_SHFT 0x14
#define TLB_L_PG_SHFT_MASK 0xfc000000
#define TLB_L_PG_SHFT_SHFT 0x1a
#define EN_DDR_MASK 0x1
#define EN_DDR_SHFT 0x0
#define EN_HBM_MASK 0x2
#define EN_HBM_SHFT 0x1
#define N_MEM_CHAN_MASK 0x7d
#define N_MEM_CHAN_SHFT 0x2
#define EN_RDMA_0_MASK 0x1
#define EN_RDMA_1_MASK 0x2
#define EN_PR_MASK 0x1
#define EN_PR_SHFT 0x0
#define EN_RDMA_0_MASK 0x1
#define EN_RDMA_0_SHFT 0x0
#define EN_RDMA_1_MASK 0x2
#define EN_RDMA_1_SHFT 0x1
#define EN_TCP_0_MASK 0x1
#define EN_TCP_0_SHFT 0x0
#define EN_TCP_1_MASK 0x2
#define EN_TCP_1_SHFT 0x1
/**
* Dynamic layer
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/* FPGA control regions */
#define FPGA_CTRL_SIZE 256 * 1024
#define FPGA_CTRL_OFFS 0x100000
#define FPGA_CTRL_LTLB_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_LTLB_OFFS 0x0
#define FPGA_CTRL_STLB_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_STLB_OFFS 0x10000
#define FPGA_CTRL_USER_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_USER_OFFS 0x20000
#define FPGA_CTRL_CNFG_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_CNFG_OFFS 0x30000
#ifndef __FPGA_DRV_H__
#define __FPGA_DRV_H__
#define FPGA_CTRL_CNFG_AVX_SIZE 256 * 1024
#define FPGA_CTRL_CNFG_AVX_OFFS 0x1000000
#include "coyote_dev.h"
#include "fpga_dev.h"
#include "pci/pci_dev.h"
#include "eci/eci_dev.h"
/* FPGA dynamic control config */
#define FPGA_CNFG_CTRL_IRQ_RESTART 0x100
/* Main */
static int __init coyote_init(void);
static void __exit coyote_exit(void);
/* TODO: Maximum transfer size */
#define XDMA_TRANSFER_MAX_BYTES (8 * 1024 * 1024)
/* TLB */
#define TLB_VADDR_RANGE 48
#define TLB_PADDR_RANGE 40
#define PID_SIZE 6
#define MAX_MAP_AXIL_PAGES 64
#define LTLB_PAGE_BITS 21
#define STLB_PAGE_BITS 12
#define LTLB_PADDR_SIZE (TLB_PADDR_RANGE - LTLB_PAGE_BITS)
#define STLB_PADDR_SIZE (TLB_PADDR_RANGE - STLB_PAGE_BITS)
#define TLBF_CTRL_START 0x7
#define TLBF_CTRL_ID_MASK 0xff
#define TLBF_CTRL_ID_SHFT 0x3
#define TLBF_STAT_DONE 0x1
/* Memory allocation */
#define MAX_BUFF_NUM 64 // maximum number of huge pages allowed
#define MAX_PR_BUFF_NUM 64 // maximum number of huge pages allowed
#define MAX_N_MAP_PAGES 128 // TODO: link to params, max no pfault 1M
#define MAX_N_MAP_HUGE_PAGES (16 * 512) // max no pfault 32M
/* Max card pages */
#define N_LARGE_CHUNKS 512
#define N_SMALL_CHUNKS (256 * 1024)
#define MEM_SEP (N_SMALL_CHUNKS * (4 * 1024))
#define MAX_N_REGIONS 16
#define SMALL_CHUNK_ALLOC 0
#define LARGE_CHUNK_ALLOC 1
/* PR */
#define PR_CTRL_START_MIDDLE 0x3
#define PR_CTRL_START_LAST 0x7
#define PR_STAT_DONE 0x1
/* FPGA config commands */
/**
* Cdev
*/
/* Major number */
#define FPGA_MAJOR 0 // dynamic
/* MMAP */
#define MMAP_CTRL 0x0
#define MMAP_CNFG 0x1
#define MMAP_CNFG_AVX 0x2
#define MMAP_WB 0x3
#define MMAP_BUFF 0x200
#define MMAP_PR 0x400
/* IOCTL */
#define IOCTL_ALLOC_HOST_USER_MEM _IOW('D', 1, unsigned long) // large pages (no hugepage support)
#define IOCTL_FREE_HOST_USER_MEM _IOW('D', 2, unsigned long)
#define IOCTL_ALLOC_HOST_PR_MEM _IOW('D', 3, unsigned long) // pr pages
#define IOCTL_FREE_HOST_PR_MEM _IOW('D', 4, unsigned long)
#define IOCTL_MAP_USER _IOW('D', 5, unsigned long) // map
#define IOCTL_UNMAP_USER _IOW('D', 6, unsigned long)
#define IOCTL_REGISTER_PID _IOW('D', 7, unsigned long) // register pid
#define IOCTL_UNREGISTER_PID _IOW('D', 8, unsigned long)
#define IOCTL_RECONFIG_LOAD _IOW('D', 9, unsigned long) // reconfiguration
#define IOCTL_ARP_LOOKUP _IOW('D', 10, unsigned long) // arp lookup
#define IOCTL_SET_IP_ADDRESS _IOW('D', 11, unsigned long)
#define IOCTL_SET_BOARD_NUM _IOW('D', 12, unsigned long)
#define IOCTL_WRITE_CTX _IOW('D', 13, unsigned long) // qp context
#define IOCTL_WRITE_CONN _IOW('D', 14, unsigned long) // qp connection
#define IOCTL_SET_TCP_OFFS _IOW('D', 15, unsigned long) // tcp mem offsets
#define IOCTL_READ_CNFG _IOR('D', 32, unsigned long) // status cnfg
#define IOCTL_NET_STATS _IOR('D', 33, unsigned long) // status network
#define IOCTL_READ_ENG_STATUS _IOR('D', 35, unsigned long) // status engines
/* Hash */
#define PR_HASH_TABLE_ORDER 8
#define PR_BATCH_SIZE (2 * 1024 * 1024)
#define USER_HASH_TABLE_ORDER 8
/* PID */
#define N_CPID_MAX 64
#define WB_SIZE (2 * N_CPID_MAX * sizeof(uint32_t))
#define N_WB_PAGES ((WB_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
/* Network */
#define EN_LOWSPEED 0x5
#define N_NET_STAT_REGS 9
/* Copy */
#define MAX_USER_WORDS 32
/**
* Reg maps
*/
/* DMA engine reg map */
struct engine_regs {
uint32_t id;
uint32_t ctrl;
uint32_t ctrl_w1s;
uint32_t ctrl_w1c;
uint32_t rsrvd_1[12];
uint32_t status;
uint32_t status_rc;
uint32_t completed_desc_count;
uint32_t alignments;
uint32_t rsrvd_2[14]; // padding
uint32_t poll_mode_wb_lo;
uint32_t poll_mode_wb_hi;
uint32_t interrupt_enable_mask;
uint32_t interrupt_enable_mask_w1s;
uint32_t interrupt_enable_mask_w1c;
uint32_t rsrvd_3[9]; // padding
uint32_t perf_ctrl;
uint32_t perf_cyc_lo;
uint32_t perf_cyc_hi;
uint32_t perf_dat_lo;
uint32_t perf_dat_hi;
uint32_t perf_pnd_lo;
uint32_t perf_pnd_hi;
} __packed;
/* Interrupt reg map */
struct interrupt_regs {
uint32_t id;
uint32_t user_int_enable;
uint32_t user_int_enable_w1s;
uint32_t user_int_enable_w1c;
uint32_t channel_int_enable;
uint32_t channel_int_enable_w1s;
uint32_t channel_int_enable_w1c;
uint32_t reserved_1[9]; // padding
uint32_t user_int_request;
uint32_t channel_int_request;
uint32_t user_int_pending;
uint32_t channel_int_pending;
uint32_t reserved_2[12]; // padding
uint32_t user_msi_vector[8];
uint32_t channel_msi_vector[8];
} __packed;
/* Polled mode descriptors struct */
struct xdma_poll_wb {
uint32_t completed_desc_count;
uint32_t reserved_1[7];
} __packed;
/* FPGA static config reg map */
struct fpga_stat_cnfg_regs {
uint64_t probe;
uint64_t n_chan;
uint64_t n_regions;
uint64_t ctrl_cnfg;
uint64_t mem_cnfg;
uint64_t pr_cnfg;
uint64_t rdma_cnfg;
uint64_t tcp_cnfg;
uint64_t lspeed_cnfg;
uint64_t reserved_0[1];
uint64_t pr_ctrl;
uint64_t pr_stat;
uint64_t pr_addr;
uint64_t pr_len;
uint64_t tlb_ctrl;
uint64_t tlb_stat;
uint64_t tlb_addr;
uint64_t tlb_len;
uint64_t reserved_1[2];
uint64_t net_0_ip;
uint64_t net_0_boardnum;
uint64_t net_0_arp;
uint64_t rdma_0_qp_ctx[3];
uint64_t rdma_0_qp_conn[3];
uint64_t tcp_0_offs[2];
uint64_t net_0_debug[N_NET_STAT_REGS];
uint64_t net_1_ip;
uint64_t net_1_boardnum;
uint64_t net_1_arp;
uint64_t rdma_1_qp_ctx[3];
uint64_t rdma_1_qp_conn[3];
uint64_t tcp_1_offs[2];
uint64_t net_1_debug[N_NET_STAT_REGS];
} __packed;
/* FPGA dynamic config reg map */
struct fpga_cnfg_regs {
uint64_t ctrl;
uint64_t vaddr_rd;
uint64_t len_rd;
uint64_t vaddr_wr;
uint64_t len_wr;
uint64_t vaddr_miss;
uint64_t len_miss;
uint64_t datapath_set;
uint64_t datapath_clr;
uint64_t stat_cmd_used_rd;
uint64_t stat_cmd_used_wr;
uint64_t stat_sent[6];
uint64_t stat_pfaults;
uint64_t wback_rd;
uint64_t wback_wr;
// Rest of regs not used in the driver
} __packed;
/* FPGA dynamic config reg map */
struct fpga_cnfg_regs_avx {
uint64_t ctrl[4];
uint64_t vaddr_miss;
uint64_t len_miss;
uint64_t pf[2];
uint64_t datapath_set[4];
uint64_t datapath_clr[4];
uint64_t stat[4];
uint64_t wback[4];
// Rest not used in the driver
} __packed;
/**
* Structs
*/
/* Engine descriptors */
struct xdma_sgdma_regs {
uint32_t identifier;
uint32_t reserved_1[31]; /* padding */
/* bus address to first descriptor in Root Complex Memory */
uint32_t first_desc_lo;
uint32_t first_desc_hi;
/* number of adjacent descriptors at first_desc */
uint32_t first_desc_adjacent;
uint32_t credits;
} __packed;
/* Engine struct */
struct xdma_engine {
int channel; // egnine channel
char *name; // engine name
struct pci_drvdata *pd; // PCI device
struct engine_regs *regs; // HW regs, control and status
struct engine_sgdma_regs *sgdma_regs; // SGDMA reg BAR offset
// Config
int running; // engine state
int c2h; // c2h(write) or h2c(read)
uint32_t status;
int addr_align; // source/dest alignment in bytes
int len_granularity; // transfer length multiple
int addr_bits; // HW datapath address width
/* Members associated with polled mode support */
uint8_t *poll_mode_addr_virt; /* virt addr for descriptor writeback */
uint64_t poll_mode_phys_addr; /* bus addr for descriptor writeback */
};
/* Mapped user pages */
struct user_pages {
struct hlist_node entry;
uint64_t vaddr;
bool huge;
int32_t cpid;
uint64_t n_hpages;
uint64_t n_pages;
struct page **hpages;
uint64_t *cpages;
};
/* User tables */
struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
/* Mapped large PR pages */
struct pr_pages {
struct hlist_node entry;
int reg_id;
uint64_t vaddr;
uint64_t n_pages;
struct page **pages;
};
/* PR table */
struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
/* Pool chunks */
struct chunk {
uint32_t id;
struct chunk *next;
};
/* Virtual FPGA device */
struct fpga_dev {
int id; // identifier
struct cdev cdev; // char device
struct pci_drvdata *pd; // PCI device
struct pr_ctrl *prc; // PR controller
// Current task
struct task_struct *curr_task;
struct mm_struct *curr_mm;
// Control region
uint64_t fpga_phys_addr_ctrl;
uint64_t fpga_phys_addr_ctrl_avx;
// Writeback
uint32_t *wb_addr_virt;
uint64_t wb_phys_addr;
// TLBs
uint64_t *fpga_lTlb; // large page TLB
uint64_t *fpga_sTlb; // small page TLB
struct fpga_cnfg_regs *fpga_cnfg; // config
struct fpga_cnfg_regs_avx *fpga_cnfg_avx; // config AVX
// PIDs
spinlock_t card_pid_lock;
struct chunk *pid_chunks;
pid_t *pid_array;
int num_free_pid_chunks;
struct chunk *pid_alloc;
// Engines
struct xdma_engine *engine_h2c; // h2c engine
struct xdma_engine *engine_c2h; // c2h engine
// In use
atomic_t in_use; // busy flag
// Lock
spinlock_t lock; // protects concurrent accesses
// Allocated buffers
struct user_pages curr_user_buff;
};
/* PR controller */
struct pr_ctrl {
struct pci_drvdata *pd; // PCI device
spinlock_t lock;
// Engines
struct xdma_engine *engine_h2c; // h2c engine
struct xdma_engine *engine_c2h; // c2h engine
// Allocated buffers
struct pr_pages curr_buff;
};
/* TLB order */
struct tlb_order {
bool hugepage;
uint64_t page_shift;
uint64_t page_size;
uint64_t page_mask;
int assoc;
int key_mask;
int key_size;
int tag_mask;
int tag_size;
int phy_mask;
int phy_size;
};
/* PCI driver data */
struct pci_drvdata {
struct pci_dev *pci_dev;
// BARs
int regions_in_use;
int got_regions;
void *__iomem bar[MAX_NUM_BARS];
unsigned long bar_phys_addr[MAX_NUM_BARS];
unsigned long bar_len[MAX_NUM_BARS];
// Engines
int engines_num;
// FPGA static config
uint probe;
int n_fpga_chan;
int n_fpga_reg;
int en_avx;
int en_bypass;
int en_tlbf;
int en_wb;
int en_ddr;
int en_hbm;
int en_mem;
int n_ddr_chan;
int en_pr;
int en_rdma_0;
int en_rdma_1;
int en_tcp_0;
int en_tcp_1;
int en_net_0;
int en_net_1;
struct fpga_stat_cnfg_regs *fpga_stat_cnfg;
struct fpga_dev *fpga_dev;
// PR control
struct pr_ctrl prc;
// TLB order
struct tlb_order *stlb_order;
struct tlb_order *ltlb_order;
// Locks
spinlock_t stat_lock;
spinlock_t prc_lock;
spinlock_t tlb_lock;
// IRQ
int irq_count;
int irq_line;
int msix_enabled;
struct msix_entry irq_entry[32];
// Card memory
spinlock_t card_l_lock;
struct chunk *lchunks;
int num_free_lchunks;
struct chunk *lalloc;
spinlock_t card_s_lock;
struct chunk *schunks;
int num_free_schunks;
struct chunk *salloc;
};
#endif
#endif /* Coyote driver */

791
driver/fpga_fops.c Normal file
View File

@ -0,0 +1,791 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "fpga_fops.h"
/*
_____
| ___|__ _ __ ___
| |_ / _ \| '_ \/ __|
| _| (_) | |_) \__ \
|_| \___/| .__/|___/
|_|
*/
/**
* @brief Acquire a region
*
*/
int fpga_open(struct inode *inode, struct file *file)
{
int minor = iminor(inode);
struct fpga_dev *d = container_of(inode->i_cdev, struct fpga_dev, cdev);
BUG_ON(!d);
dbg_info("fpga device %d acquired\n", minor);
// set private data
file->private_data = (void *)d;
return 0;
}
/**
* @brief Release a region
*
*/
int fpga_release(struct inode *inode, struct file *file)
{
int minor = iminor(inode);
struct fpga_dev *d = container_of(inode->i_cdev, struct fpga_dev, cdev);
BUG_ON(!d);
// unamp all user pages
tlb_put_user_pages_all(d, 1);
dbg_info("fpga device %d released\n", minor);
return 0;
}
/**
* @brief XDMA engine status
*
*/
uint32_t engine_status_read(struct xdma_engine *engine)
{
uint32_t val;
BUG_ON(!engine);
dbg_info("engine %s status:\n", engine->name);
val = ioread32(&engine->regs->status);
dbg_info("status = 0x%08x: %s%s%s%s%s%s%s%s%s\n", (uint32_t)val,
(val & XDMA_STAT_BUSY) ? "BUSY " : "IDLE ",
(val & XDMA_STAT_DESC_STOPPED) ? "DESC_STOPPED " : "",
(val & XDMA_STAT_DESC_COMPLETED) ? "DESC_COMPLETED " : "",
(val & XDMA_STAT_ALIGN_MISMATCH) ? "ALIGN_MISMATCH " : "",
(val & XDMA_STAT_MAGIC_STOPPED) ? "MAGIC_STOPPED " : "",
(val & XDMA_STAT_FETCH_STOPPED) ? "FETCH_STOPPED " : "",
(val & XDMA_STAT_READ_ERROR) ? "READ_ERROR " : "",
(val & XDMA_STAT_DESC_ERROR) ? "DESC_ERROR " : "",
(val & XDMA_STAT_IDLE_STOPPED) ? "IDLE_STOPPED " : "");
return val;
}
/**
* @brief ioctl, control and status
*
*/
long fpga_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
int ret_val, i;
uint64_t tmp[MAX_USER_WORDS];
uint64_t cpid;
struct fpga_dev *d = (struct fpga_dev *)file->private_data;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
switch (cmd) {
// allocate host memory (2MB pages) - no hugepages support
case IOCTL_ALLOC_HOST_USER_MEM:
// read n_pages + cpid
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
ret_val = alloc_user_buffers(d, tmp[0], (uint32_t)tmp[1]);
dbg_info("buff_num %lld, arg %lx\n", d->curr_user_buff.n_hpages, arg);
if (ret_val != 0) {
pr_info("user buffers could not be allocated\n");
}
}
break;
// free host memory (2MB pages) - no hugepages support
case IOCTL_FREE_HOST_USER_MEM:
// read vaddr + cpid
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
ret_val = free_user_buffers(d, tmp[0], (uint32_t)tmp[1]);
dbg_info("user buffers freed\n");
}
break;
// allocate host memory (2MB pages) - bitstream memory
case IOCTL_ALLOC_HOST_PR_MEM:
// read n_pages
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
ret_val = alloc_pr_buffers(d, tmp[0]);
dbg_info("buff_num %lld, arg %lx\n", d->prc->curr_buff.n_pages, arg);
if (ret_val != 0) {
pr_info("PR buffers could not be allocated\n");
}
}
break;
// free host memory (2MB pages) - bitstream memory
case IOCTL_FREE_HOST_PR_MEM:
// read vaddr
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
ret_val = free_pr_buffers(d, tmp[0]);
dbg_info("PR buffers freed\n");
}
break;
// explicit mapping
case IOCTL_MAP_USER:
// read vaddr + len + cpid
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 3 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
} else {
cpid = (uint32_t)tmp[2];
tlb_get_user_pages(d, tmp[0], tmp[1], (int32_t)tmp[2], d->pid_array[cpid]);
}
break;
// explicit unmapping
case IOCTL_UNMAP_USER:
// read vaddr + cpid
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("unmapping user pages\n");
tlb_put_user_pages(d, tmp[0], (int32_t)tmp[1], 1);
}
break;
// register pid
case IOCTL_REGISTER_PID:
// read pid
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
spin_lock(&pd->stat_lock);
cpid = (uint64_t)register_pid(d, tmp[0]);
if (cpid == -1)
{
dbg_info("registration failed pid %lld\n", tmp[0]);
return -1;
}
dbg_info("registration succeeded pid %lld, cpid %lld\n", tmp[0], cpid);
ret_val = copy_to_user((unsigned long *)arg + 1, &cpid, sizeof(unsigned long));
spin_unlock(&pd->stat_lock);
}
break;
// unregister pid
case IOCTL_UNREGISTER_PID:
// read cpid
ret_val = copy_from_user(&tmp, (unsigned long *)arg, sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
spin_lock(&pd->stat_lock);
ret_val = unregister_pid(d, tmp[0]);
if (ret_val == -1) {
dbg_info("unregistration failed cpid %lld\n", tmp[0]);
return -1;
}
spin_unlock(&pd->stat_lock);
dbg_info("unregistration succeeded cpid %lld\n", tmp[0]);
}
break;
// reconfiguration
case IOCTL_RECONFIG_LOAD:
// read vaddr + len
ret_val = copy_from_user(&tmp, (unsigned long *)arg, 2 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("trying to obtain reconfig lock\n");
spin_lock(&d->prc->lock);
if (pd->en_avx)
d->fpga_cnfg_avx->datapath_set[0] = 0x1;
else
d->fpga_cnfg->datapath_set = 0x1;
ret_val = reconfigure(d, tmp[0], tmp[1]);
if (ret_val != 0) {
pr_info("reconfiguration failed, return %d\n", ret_val);
return -1;
}
else {
dbg_info("reconfiguration successfull\n");
}
dbg_info("releasing reconfig lock, coupling the design\n");
if (pd->en_avx)
d->fpga_cnfg_avx->datapath_clr[0] = 0x1;
else
d->fpga_cnfg->datapath_clr = 0x1;
spin_unlock(&d->prc->lock);
}
break;
// arp lookup
case IOCTL_ARP_LOOKUP:
if (pd->en_net_0 || pd->en_net_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("arp lookup ...");
spin_lock(&pd->stat_lock);
for (i = 0; i < N_TOTAL_NODES; i++) {
if (i == NODE_ID)
continue;
tmp[0] ? (pd->fpga_stat_cnfg->net_1_arp = BASE_IP_ADDR_1 + i) :
(pd->fpga_stat_cnfg->net_0_arp = BASE_IP_ADDR_0 + i);
}
spin_unlock(&pd->stat_lock);
}
} else {
pr_info("network not enabled\n");
return -1;
}
break;
// set ip address
case IOCTL_SET_IP_ADDRESS:
if (pd->en_net_0 || pd->en_net_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 2 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
}
else {
spin_lock(&pd->stat_lock);
// Set ip
tmp[0] ? (pd->fpga_stat_cnfg->net_1_ip = tmp[1]) :
(pd->fpga_stat_cnfg->net_0_ip = tmp[1]);
spin_unlock(&pd->stat_lock);
dbg_info("ip address changed to %llx\n", tmp[1]);
}
} else {
pr_info("network not enabled\n");
return -1;
}
break;
// set board number
case IOCTL_SET_BOARD_NUM:
if (pd->en_net_0 || pd->en_net_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 2 * sizeof(unsigned long));
if (ret_val != 0) {
pr_info("user data could not be coppied, return %d\n", ret_val);
} else {
spin_lock(&pd->stat_lock);
// Set board number
tmp[0] ? (pd->fpga_stat_cnfg->net_1_boardnum = tmp[1]) :
(pd->fpga_stat_cnfg->net_0_boardnum = tmp[1]);
spin_unlock(&pd->stat_lock);
dbg_info("board number changed to %llx\n", tmp[1]);
}
} else {
pr_info("network not enabled\n");
return -1;
}
break;
// rdma context
case IOCTL_WRITE_CTX:
if (pd->en_rdma_0 || pd->en_rdma_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 4 * sizeof(unsigned long));
if (ret_val != 0) {
dbg_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("writing qp context ...");
spin_lock(&pd->stat_lock);
for (i = 0; i < 3; i++) {
tmp[0] ? (pd->fpga_stat_cnfg->rdma_1_qp_ctx[i] = tmp[i+1]) :
(pd->fpga_stat_cnfg->rdma_0_qp_ctx[i] = tmp[i+1]);
}
spin_unlock(&pd->stat_lock);
}
}
else {
dbg_info("RDMA not enabled\n");
}
break;
// rdma connection
case IOCTL_WRITE_CONN:
if (pd->en_rdma_0 || pd->en_rdma_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 4 * sizeof(unsigned long));
if (ret_val != 0) {
dbg_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("writing qp connection ...");
spin_lock(&pd->stat_lock);
for (i = 0; i < 3; i++) {
tmp[0] ? (pd->fpga_stat_cnfg->rdma_1_qp_conn[i] = tmp[i+1]) :
(pd->fpga_stat_cnfg->rdma_0_qp_conn[i] = tmp[i+1]);
}
spin_unlock(&pd->stat_lock);
}
}
else {
dbg_info("RDMA not enabled\n");
}
break;
// tcp offsets
case IOCTL_SET_TCP_OFFS:
if (pd->en_tcp_0 || pd->en_tcp_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, 3 * sizeof(unsigned long));
if (ret_val != 0) {
dbg_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("writing tcp mem offsets ...");
spin_lock(&pd->stat_lock);
for (i = 0; i < 2; i++) {
tmp[0] ? (pd->fpga_stat_cnfg->tcp_1_offs[i] = tmp[i+1]) :
(pd->fpga_stat_cnfg->tcp_0_offs[i] = tmp[i+1]);
}
spin_unlock(&pd->stat_lock);
}
}
else {
dbg_info("TCP/IP not enabled\n");
}
break;
// read config
case IOCTL_READ_CNFG:
tmp[0] = ((uint64_t)pd->n_fpga_chan << 32) | ((uint64_t)pd->n_fpga_reg << 40) | ((uint64_t)pd->n_mem_chan << 48) |
((uint64_t)pd->en_avx) | ((uint64_t)pd->en_bypass << 1) | ((uint64_t)pd->en_tlbf << 2) | ((uint64_t)pd->en_wb << 3) |
((uint64_t)pd->en_mem << 4) | ((uint64_t)pd->en_pr << 5) |
((uint64_t)pd->en_rdma_0 << 6) | ((uint64_t)pd->en_rdma_1 << 7) | ((uint64_t)pd->en_tcp_0 << 8) | ((uint64_t)pd->en_tcp_1 << 9);
dbg_info("reading config %llx\n", tmp[0]);
ret_val = copy_to_user((unsigned long *)arg, &tmp, sizeof(unsigned long));
break;
// network status
case IOCTL_NET_STATS:
if (pd->en_net_0 || pd->en_net_1) {
ret_val = copy_from_user(&tmp, (unsigned long*) arg, sizeof(unsigned long));
if (ret_val != 0) {
dbg_info("user data could not be coppied, return %d\n", ret_val);
} else {
dbg_info("retreiving network status for port %llx", tmp[0]);
if(tmp[0]) {
for (i = 0; i < N_NET_STAT_REGS; i++) {
tmp[i] = pd->fpga_stat_cnfg->net_1_debug[i];
}
} else {
for (i = 0; i < N_NET_STAT_REGS; i++) {
tmp[i] = pd->fpga_stat_cnfg->net_0_debug[i];
}
}
ret_val = copy_to_user((unsigned long *)arg, &tmp, N_NET_STAT_REGS * sizeof(unsigned long));
}
}
else {
dbg_info("network not enabled\n");
}
break;
// engine status
case IOCTL_READ_ENG_STATUS:
dbg_info("fpga dev %d engine report\n", d->id);
engine_status_read(d->engine_c2h);
engine_status_read(d->engine_h2c);
break;
default:
break;
}
return 0;
}
/**
* @brief Mmap control and mem
*
*/
int fpga_mmap(struct file *file, struct vm_area_struct *vma)
{
int i;
unsigned long vaddr;
unsigned long vaddr_tmp;
struct fpga_dev *d;
struct pr_ctrl *prc;
struct user_pages *new_user_buff;
struct pr_pages *new_pr_buff;
struct bus_drvdata *pd;
uint64_t *map_array;
d = (struct fpga_dev *)file->private_data;
BUG_ON(!d);
prc = d->prc;
BUG_ON(!prc);
pd = d->pd;
BUG_ON(!pd);
vaddr = vma->vm_start;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
// map user ctrl region
if (vma->vm_pgoff == MMAP_CTRL) {
dbg_info("fpga dev. %d, memory mapping user ctrl region at %llx of size %x\n",
d->id, d->fpga_phys_addr_ctrl + FPGA_CTRL_USER_OFFS, FPGA_CTRL_USER_SIZE);
if (remap_pfn_range(vma, vma->vm_start, (d->fpga_phys_addr_ctrl + FPGA_CTRL_USER_OFFS) >> PAGE_SHIFT,
FPGA_CTRL_USER_SIZE, vma->vm_page_prot)) {
return -EIO;
}
return 0;
}
// map cnfg region
if (vma->vm_pgoff == MMAP_CNFG) {
dbg_info("fpga dev. %d, memory mapping config region at %llx of size %x\n",
d->id, d->fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS, FPGA_CTRL_CNFG_SIZE);
if (remap_pfn_range(vma, vma->vm_start, (d->fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS) >> PAGE_SHIFT,
FPGA_CTRL_CNFG_SIZE, vma->vm_page_prot)) {
return -EIO;
}
return 0;
}
// map cnfg AVX region
if (vma->vm_pgoff == MMAP_CNFG_AVX) {
dbg_info("fpga dev. %d, memory mapping config AVX region at %llx of size %x\n",
d->id, d->fpga_phys_addr_ctrl_avx, FPGA_CTRL_CNFG_AVX_SIZE);
if (remap_pfn_range(vma, vma->vm_start, d->fpga_phys_addr_ctrl_avx >> PAGE_SHIFT,
FPGA_CTRL_CNFG_AVX_SIZE, vma->vm_page_prot)) {
return -EIO;
}
return 0;
}
// map writeback
if (vma->vm_pgoff == MMAP_WB) {
set_memory_uc((uint64_t)d->wb_addr_virt, N_WB_PAGES);
dbg_info("fpga dev. %d, memory mapping writeback regions at %llx of size %lx\n",
d->id, d->wb_phys_addr, WB_SIZE);
if (remap_pfn_range(vma, vma->vm_start, (d->wb_phys_addr) >> PAGE_SHIFT,
WB_SIZE, vma->vm_page_prot)) {
return -EIO;
}
return 0;
}
// map user buffers
if (vma->vm_pgoff == MMAP_BUFF) {
dbg_info("fpga dev. %d, memory mapping buffer\n", d->id);
// aligned page virtual address
vaddr = ((vma->vm_start + pd->ltlb_order->page_size - 1) >> pd->ltlb_order->page_shift) << pd->ltlb_order->page_shift;
vaddr_tmp = vaddr;
if (d->curr_user_buff.n_hpages != 0) {
new_user_buff = kzalloc(sizeof(struct user_pages), GFP_KERNEL);
BUG_ON(!new_user_buff);
// Map entry
new_user_buff->vaddr = vaddr;
new_user_buff->huge = d->curr_user_buff.huge;
new_user_buff->cpid = d->curr_user_buff.cpid;
new_user_buff->n_hpages = d->curr_user_buff.n_hpages;
new_user_buff->n_pages = d->curr_user_buff.n_pages;
new_user_buff->hpages = d->curr_user_buff.hpages;
new_user_buff->cpages = d->curr_user_buff.cpages;
hash_add(user_lbuff_map[d->id], &new_user_buff->entry, vaddr);
for (i = 0; i < d->curr_user_buff.n_hpages; i++) {
// map to user space
if (remap_pfn_range(vma, vaddr_tmp, page_to_pfn(d->curr_user_buff.hpages[i]),
pd->ltlb_order->page_size, vma->vm_page_prot)) {
return -EIO;
}
vaddr_tmp += pd->ltlb_order->page_size;
}
// reset
vaddr_tmp = vaddr;
// map array
map_array = (uint64_t *)kzalloc(d->curr_user_buff.n_hpages * 2 * sizeof(uint64_t), GFP_KERNEL);
if (map_array == NULL) {
dbg_info("map buffers could not be allocated\n");
return -ENOMEM;
}
// fill mappings
for (i = 0; i < d->curr_user_buff.n_hpages; i++) {
tlb_create_map(pd->ltlb_order, vaddr_tmp, page_to_phys(d->curr_user_buff.hpages[i]), (pd->en_mem ? d->curr_user_buff.cpages[i] : 0), d->curr_user_buff.cpid, &map_array[2*i]);
vaddr_tmp += pd->ltlb_order->page_size;
}
// fire
tlb_service_dev(d, pd->ltlb_order, map_array, d->curr_user_buff.n_hpages);
// free
kfree((void *)map_array);
// Current host buff empty
d->curr_user_buff.n_hpages = 0;
return 0;
}
}
// map PR buffers
if (vma->vm_pgoff == MMAP_PR)
{
dbg_info("fpga dev. %d, memory mapping PR buffer\n", d->id);
// aligned page virtual address
vaddr = ((vma->vm_start + pd->ltlb_order->page_size - 1) >> pd->ltlb_order->page_shift) << pd->ltlb_order->page_shift;
vaddr_tmp = vaddr;
if (prc->curr_buff.n_pages != 0) {
// obtain PR lock
spin_lock(&prc->lock);
new_pr_buff = kzalloc(sizeof(struct pr_pages), GFP_KERNEL);
BUG_ON(!new_pr_buff);
// Map entry
new_pr_buff->vaddr = vaddr;
new_pr_buff->reg_id = d->id;
new_pr_buff->n_pages = prc->curr_buff.n_pages;
new_pr_buff->pages = prc->curr_buff.pages;
hash_add(pr_buff_map, &new_pr_buff->entry, vaddr);
for (i = 0; i < prc->curr_buff.n_pages; i++) {
// map to user space
if (remap_pfn_range(vma, vaddr_tmp, page_to_pfn(prc->curr_buff.pages[i]),
pd->ltlb_order->page_size, vma->vm_page_prot)) {
return -EIO;
}
// next page vaddr
vaddr_tmp += pd->ltlb_order->page_size;
}
// Current host buff empty
prc->curr_buff.n_pages = 0;
// release PR lock
spin_unlock(&prc->lock);
return 0;
}
}
return -EINVAL;
}
/*
_ _ _ _ _
| | | | |_(_) |
| | | | __| | |
| |_| | |_| | |
\___/ \__|_|_|
*/
/**
* @brief Register PID
*
* @param d - vFPGA
* @param pid - user PID
* @return int32_t - Coyote PID
*/
int32_t register_pid(struct fpga_dev *d, pid_t pid)
{
int32_t cpid;
BUG_ON(!d);
// lock
spin_lock(&d->card_pid_lock);
if(d->num_free_pid_chunks == 0) {
dbg_info("not enough CPID slots\n");
return -ENOMEM;
}
cpid = (int32_t)d->pid_alloc->id;
d->pid_array[d->pid_alloc->id] = pid;
d->pid_alloc = d->pid_alloc->next;
// unlock
spin_unlock(&d->card_pid_lock);
return cpid;
}
/**
* @brief Unregister Coyote PID
*
* @param d - vFPGA
* @param cpid - Coyote PID
*/
int unregister_pid(struct fpga_dev *d, int32_t cpid)
{
BUG_ON(!d);
// lock
spin_lock(&d->card_pid_lock);
d->pid_chunks[cpid].next = d->pid_alloc;
d->pid_alloc = &d->pid_chunks[cpid];
// release lock
spin_unlock(&d->card_pid_lock);
return 0;
}
/*
____________________
\______ \______ \
| ___/| _/
| | | | \
|____| |____|_ /
\/
*/
/**
* @brief Reconfigure the vFPGA
*
* @param d - vFPGA
* @param vaddr - bitstream vaddr
* @param len - bitstream length
*/
int reconfigure(struct fpga_dev *d, uint64_t vaddr, uint64_t len)
{
struct pr_ctrl *prc;
struct pr_pages *tmp_buff;
int i;
uint64_t fsz_m;
uint64_t fsz_r;
uint64_t pr_bsize = PR_BATCH_SIZE;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
prc = d->prc;
BUG_ON(!prc);
// lock
spin_lock(&pd->prc_lock);
hash_for_each_possible(pr_buff_map, tmp_buff, entry, vaddr) {
if (tmp_buff->vaddr == vaddr && tmp_buff->reg_id == d->id) {
// Reconfiguration
fsz_m = len / pr_bsize;
fsz_r = len % pr_bsize;
dbg_info("bitstream full %lld, partial %lld\n", fsz_m, fsz_r);
// full
for (i = 0; i < fsz_m; i++) {
dbg_info("page %d, phys %llx, len %llx\n", i, page_to_phys(prc->curr_buff.pages[i]), pr_bsize);
pd->fpga_stat_cnfg->pr_addr = page_to_phys(tmp_buff->pages[i]);
pd->fpga_stat_cnfg->pr_len = pr_bsize;
if (fsz_r == 0 && i == fsz_m - 1)
pd->fpga_stat_cnfg->pr_ctrl = PR_CTRL_START_LAST;
else
pd->fpga_stat_cnfg->pr_ctrl = PR_CTRL_START_MIDDLE;
}
// partial
if (fsz_r > 0) {
dbg_info("page %lld, phys %llx, len %llx\n", fsz_m, page_to_phys(prc->curr_buff.pages[fsz_m]), fsz_r);
pd->fpga_stat_cnfg->pr_addr = page_to_phys(tmp_buff->pages[fsz_m]);
pd->fpga_stat_cnfg->pr_len = fsz_r;
pd->fpga_stat_cnfg->pr_ctrl = PR_CTRL_START_LAST;
while ((pd->fpga_stat_cnfg->pr_stat & PR_STAT_DONE) != 0x1)
ndelay(100);
} else {
while ((pd->fpga_stat_cnfg->pr_stat & PR_STAT_DONE) != 0x1)
ndelay(100);
}
}
}
// unlock
spin_unlock(&pd->prc_lock);
return 0;
}

53
driver/fpga_fops.h Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __FPGA_FOPS_H__
#define __FPGA_FOPS_H__
#include "coyote_dev.h"
#include "fpga_mmu.h"
/* Pid */
int32_t register_pid(struct fpga_dev *d, pid_t pid);
int unregister_pid(struct fpga_dev *d, int32_t cpid);
/* Engine status read */
uint32_t engine_status_read(struct xdma_engine *engine);
/* Reconfiguration */
int reconfigure(struct fpga_dev *d, uint64_t vaddr, uint64_t len);
int alloc_pr_buffers(struct fpga_dev *d, unsigned long n_pages);
int free_pr_buffers(struct fpga_dev *d, uint64_t vaddr);
/* Fops */
int fpga_open(struct inode *inode, struct file *file);
int fpga_release(struct inode *inode, struct file *file);
int fpga_mmap(struct file *file, struct vm_area_struct *vma);
long fpga_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
#endif // FPGA FOPS

101
driver/fpga_isr.c Normal file
View File

@ -0,0 +1,101 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "fpga_isr.h"
/*
___ ____ ____
|_ _/ ___|| _ \
| |\___ \| |_) |
| | ___) | _ <
|___|____/|_| \_\
*/
/**
* @brief TLB page fault handling
*
*/
irqreturn_t fpga_tlb_miss_isr(int irq, void *dev_id)
{
unsigned long flags;
uint64_t vaddr;
uint32_t len;
int32_t cpid;
struct fpga_dev *d;
struct bus_drvdata *pd;
int ret_val = 0;
pid_t pid;
uint64_t tmp;
dbg_info("(irq=%d) page fault ISR\n", irq);
BUG_ON(!dev_id);
BUG_ON(cyt_arch == CYT_ARCH_ECI);
d = (struct fpga_dev *)dev_id;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
// lock
spin_lock_irqsave(&(d->lock), flags);
// read page fault
if (pd->en_avx) {
vaddr = d->fpga_cnfg_avx->vaddr_miss;
tmp = d->fpga_cnfg_avx->len_miss;
len = LOW_32(tmp);
cpid = (int32_t)HIGH_32(tmp);
}
else {
vaddr = d->fpga_cnfg->vaddr_miss;
tmp = d->fpga_cnfg->len_miss;
len = LOW_32(tmp);
cpid = (int32_t)HIGH_32(tmp);
}
dbg_info("page fault, vaddr %llx, length %x, cpid %d\n", vaddr, len, cpid);
// get user pages
pid = d->pid_array[cpid];
ret_val = tlb_get_user_pages(d, vaddr, len, cpid, pid);
if (ret_val > 0) {
// restart the engine
if (pd->en_avx)
d->fpga_cnfg_avx->ctrl[0] = FPGA_CNFG_CTRL_IRQ_RESTART;
else
d->fpga_cnfg->ctrl = FPGA_CNFG_CTRL_IRQ_RESTART;
}
else {
dbg_info("pages could not be obtained\n");
}
// unlock
spin_unlock_irqrestore(&(d->lock), flags);
return IRQ_HANDLED;
}

37
driver/fpga_isr.h Normal file
View File

@ -0,0 +1,37 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __FPGA_ISR_H__
#define __FPGA_ISR_H__
#include "coyote_dev.h"
#include "fpga_mmu.h"
/* Interrupt service routine */
irqreturn_t fpga_tlb_miss_isr(int irq, void *dev_id);
#endif // FPGA ISR

921
driver/fpga_mmu.c Normal file
View File

@ -0,0 +1,921 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "fpga_mmu.h"
/* User tables */
struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
/* PR table */
struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
/**
* @brief ALlocate user buffers (used in systems without hugepage support)
*
* @param d - vFPGA
* @param n_pages - number of pages to allocate
* @param cpid - Coyote PID
*/
int alloc_user_buffers(struct fpga_dev *d, unsigned long n_pages, int32_t cpid)
{
int i, ret_val = 0;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
if (d->curr_user_buff.n_hpages) {
dbg_info("allocated user buffers exist and are not mapped\n");
return -1;
}
// check host
if (n_pages > MAX_BUFF_NUM)
d->curr_user_buff.n_hpages = MAX_BUFF_NUM;
else
d->curr_user_buff.n_hpages = n_pages;
// check card
if(pd->en_mem)
if (d->curr_user_buff.n_hpages > pd->num_free_lchunks)
return -ENOMEM;
d->curr_user_buff.huge = true;
d->curr_user_buff.cpid = cpid;
// alloc host
d->curr_user_buff.hpages = kzalloc(d->curr_user_buff.n_hpages * sizeof(*d->curr_user_buff.hpages), GFP_KERNEL);
if (d->curr_user_buff.hpages == NULL) {
return -ENOMEM;
}
dbg_info("allocated %llu bytes for page pointer array for %lld user host buffers @0x%p.\n",
d->curr_user_buff.n_hpages * sizeof(*d->curr_user_buff.hpages), d->curr_user_buff.n_hpages, d->curr_user_buff.hpages);
for (i = 0; i < d->curr_user_buff.n_hpages; i++) {
d->curr_user_buff.hpages[i] = alloc_pages(GFP_ATOMIC, pd->ltlb_order->page_shift - PAGE_SHIFT);
if (!d->curr_user_buff.hpages[i]) {
dbg_info("user host buffer %d could not be allocated\n", i);
goto fail_host_alloc;
}
dbg_info("user host buffer allocated @ %llx device %d\n", page_to_phys(d->curr_user_buff.hpages[i]), d->id);
}
// alloc card
if(pd->en_mem) {
d->curr_user_buff.n_pages = d->curr_user_buff.n_hpages;
d->curr_user_buff.cpages = kzalloc(d->curr_user_buff.n_pages * sizeof(uint64_t), GFP_KERNEL);
if (d->curr_user_buff.cpages == NULL) {
return -ENOMEM;
}
dbg_info("allocated %llu bytes for page pointer array for %lld user card buffers @0x%p.\n",
d->curr_user_buff.n_pages * sizeof(*d->curr_user_buff.cpages), d->curr_user_buff.n_pages, d->curr_user_buff.cpages);
ret_val = card_alloc(d, d->curr_user_buff.cpages, d->curr_user_buff.n_pages, LARGE_CHUNK_ALLOC);
if (ret_val) {
dbg_info("user card buffer %d could not be allocated\n", i);
goto fail_card_alloc;
}
}
return 0;
fail_host_alloc:
while (i)
__free_pages(d->curr_user_buff.hpages[--i], pd->ltlb_order->page_shift - PAGE_SHIFT);
d->curr_user_buff.n_hpages = 0;
kfree(d->curr_user_buff.hpages);
return -ENOMEM;
fail_card_alloc:
// release host
for (i = 0; i < d->curr_user_buff.n_hpages; i++)
__free_pages(d->curr_user_buff.hpages[i], pd->ltlb_order->page_shift - PAGE_SHIFT);
d->curr_user_buff.n_hpages = 0;
d->curr_user_buff.n_pages = 0;
kfree(d->curr_user_buff.hpages);
kfree(d->curr_user_buff.cpages);
return -ENOMEM;
}
/**
* @brief Free user buffers
*
* @param d - vFPGA
* @param vaddr - virtual address
* @param cpid - Coyote PID
*/
int free_user_buffers(struct fpga_dev *d, uint64_t vaddr, int32_t cpid)
{
int i;
uint64_t vaddr_tmp;
struct user_pages *tmp_buff;
uint64_t *map_array;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
hash_for_each_possible(user_lbuff_map[d->id], tmp_buff, entry, vaddr) {
if (tmp_buff->vaddr == vaddr && tmp_buff->cpid == cpid) {
vaddr_tmp = tmp_buff->vaddr;
// free host pages
for (i = 0; i < tmp_buff->n_hpages; i++) {
if (tmp_buff->hpages[i])
__free_pages(tmp_buff->hpages[i], pd->ltlb_order->page_shift - PAGE_SHIFT);
}
kfree(tmp_buff->hpages);
// free card pages
if(pd->en_mem) {
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, LARGE_CHUNK_ALLOC);
kfree(tmp_buff->cpages);
}
// map array
map_array = (uint64_t *)kzalloc(tmp_buff->n_hpages * 2 * sizeof(uint64_t), GFP_KERNEL);
if (map_array == NULL) {
dbg_info("map buffers could not be allocated\n");
return -ENOMEM;
}
// fill mappings
for (i = 0; i < tmp_buff->n_hpages; i++) {
tlb_create_unmap(pd->ltlb_order, vaddr_tmp, cpid, &map_array[2*i]);
vaddr_tmp += pd->ltlb_order->page_size;
}
// fire
tlb_service_dev(d, pd->ltlb_order, map_array, tmp_buff->n_hpages);
// free
kfree((void *)map_array);
// Free from hash
hash_del(&tmp_buff->entry);
}
}
return 0;
}
/**
* @brief Allocate PR buffers
*
* @param d - vFPGA
* @param n_pages - number of pages to allocate
*/
int alloc_pr_buffers(struct fpga_dev *d, unsigned long n_pages)
{
int i;
struct pr_ctrl *prc;
struct bus_drvdata *pd;
BUG_ON(!d);
prc = d->prc;
BUG_ON(!prc);
pd = d->pd;
BUG_ON(!pd);
// obtain PR lock
spin_lock(&prc->lock);
if (prc->curr_buff.n_pages) {
dbg_info("allocated PR buffers exist and are not mapped\n");
return -1;
}
if (n_pages > MAX_PR_BUFF_NUM)
prc->curr_buff.n_pages = MAX_PR_BUFF_NUM;
else
prc->curr_buff.n_pages = n_pages;
prc->curr_buff.pages = kzalloc(n_pages * sizeof(*prc->curr_buff.pages), GFP_KERNEL);
if (prc->curr_buff.pages == NULL) {
return -ENOMEM;
}
dbg_info("allocated %lu bytes for page pointer array for %ld PR buffers @0x%p.\n",
n_pages * sizeof(*prc->curr_buff.pages), n_pages, prc->curr_buff.pages);
for (i = 0; i < prc->curr_buff.n_pages; i++) {
prc->curr_buff.pages[i] = alloc_pages(GFP_ATOMIC, pd->ltlb_order->page_shift - PAGE_SHIFT);
if (!prc->curr_buff.pages[i]) {
dbg_info("PR buffer %d could not be allocated\n", i);
goto fail_alloc;
}
dbg_info("PR buffer allocated @ %llx \n", page_to_phys(prc->curr_buff.pages[i]));
}
// release PR lock
spin_unlock(&prc->lock);
return 0;
fail_alloc:
while (i)
__free_pages(prc->curr_buff.pages[--i], pd->ltlb_order->page_shift - PAGE_SHIFT);
// release PR lock
spin_unlock(&prc->lock);
return -ENOMEM;
}
/**
* @brief Free PR pages
*
* @param d - vFPGA
* @param vaddr - virtual address
*/
int free_pr_buffers(struct fpga_dev *d, uint64_t vaddr)
{
int i;
struct pr_pages *tmp_buff;
struct pr_ctrl *prc;
struct bus_drvdata *pd;
BUG_ON(!d);
prc = d->prc;
BUG_ON(!prc);
pd = d->pd;
BUG_ON(!pd);
// obtain PR lock
spin_lock(&prc->lock);
hash_for_each_possible(pr_buff_map, tmp_buff, entry, vaddr) {
if (tmp_buff->vaddr == vaddr && tmp_buff->reg_id == d->id) {
// free pages
for (i = 0; i < tmp_buff->n_pages; i++) {
if (tmp_buff->pages[i])
__free_pages(tmp_buff->pages[i], pd->ltlb_order->page_shift - PAGE_SHIFT);
}
kfree(tmp_buff->pages);
// Free from hash
hash_del(&tmp_buff->entry);
}
}
// obtain PR lock
spin_unlock(&prc->lock);
return 0;
}
/**
* @brief Allocate card memory
*
* @param d - vFPGA
* @param card_paddr - card physical address
* @param n_pages - number of pages to allocate
* @param type - page size
*/
int card_alloc(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type)
{
int i;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
switch (type) {
case 0: //
// lock
spin_lock(&pd->card_s_lock);
if(pd->num_free_schunks < n_pages) {
dbg_info("not enough free small card pages\n");
return -ENOMEM;
}
for(i = 0; i < n_pages; i++) {
card_paddr[i] = pd->salloc->id << STLB_PAGE_BITS;
dbg_info("user card buffer allocated @ %llx device %d\n", card_paddr[i], d->id);
pd->salloc = pd->salloc->next;
}
// release lock
spin_unlock(&pd->card_s_lock);
break;
case 1:
// lock
spin_lock(&pd->card_l_lock);
if(pd->num_free_lchunks < n_pages) {
dbg_info("not enough free large card pages\n");
return -ENOMEM;
}
for(i = 0; i < n_pages; i++) {
card_paddr[i] = (pd->lalloc->id << LTLB_PAGE_BITS) + MEM_SEP;
dbg_info("user card buffer allocated @ %llx device %d\n", card_paddr[i], d->id);
pd->lalloc = pd->lalloc->next;
}
// release lock
spin_unlock(&pd->card_l_lock);
break;
default: // TODO: Shared mem
break;
}
return 0;
}
/**
* @brief Free card memory
*
* @param d - vFPGA
* @param card_paddr - card physical address
* @param n_pages - number of pages to free
* @param type - page size
*/
void card_free(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type)
{
int i;
uint64_t tmp_id;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
switch (type) {
case 0: // small pages
// lock
spin_lock(&pd->card_s_lock);
for(i = n_pages - 1; i >= 0; i--) {
tmp_id = card_paddr[i] >> STLB_PAGE_BITS;
pd->schunks[tmp_id].next = pd->salloc;
pd->salloc = &pd->schunks[tmp_id];
}
// release lock
spin_unlock(&pd->card_s_lock);
break;
case 1: // large pages
// lock
spin_lock(&pd->card_l_lock);
for(i = n_pages - 1; i >= 0; i--) {
tmp_id = (card_paddr[i] - MEM_SEP) >> LTLB_PAGE_BITS;
pd->lchunks[tmp_id].next = pd->lalloc;
pd->lalloc = &pd->lchunks[tmp_id];
}
// release lock
spin_unlock(&pd->card_l_lock);
break;
default:
break;
}
}
/**
* @brief Page map list
*
* @param vaddr - starting vaddr
* @param paddr_host - host physical address
* @param paddr_card - card physical address
* @param cpid - Coyote PID
* @param entry - liste entry
*/
void tlb_create_map(struct tlb_order *tlb_ord, uint64_t vaddr, uint64_t paddr_host, uint64_t paddr_card, int32_t cpid, uint64_t *entry)
{
uint64_t key;
uint64_t tag;
uint64_t phost;
uint64_t pcard;
key = (vaddr >> tlb_ord->page_shift) & tlb_ord->key_mask;
tag = vaddr >> (tlb_ord->page_shift + tlb_ord->key_size);
phost = (paddr_host >> tlb_ord->page_shift) & tlb_ord->phy_mask;
pcard = ((paddr_card >> tlb_ord->page_shift) & tlb_ord->phy_mask) << tlb_ord->phy_size;
// new entry
entry[0] |= key |
(tag << tlb_ord->key_size) |
((uint64_t)cpid << (tlb_ord->key_size + tlb_ord->tag_size)) |
(1UL << (tlb_ord->key_size + tlb_ord->tag_size + PID_SIZE));
entry[1] |= phost | (pcard << tlb_ord->phy_size);
dbg_info("creating new TLB entry, vaddr %llx, phost %llx, pcard %llx, cpid %d, hugepage %d\n", vaddr, paddr_host, paddr_card, cpid, tlb_ord->hugepage);
}
/**
* @brief Page unmap lists
*
* @param vaddr - starting vaddr
* @param cpid - Coyote PID
* @param entry - list entry
*/
void tlb_create_unmap(struct tlb_order *tlb_ord, uint64_t vaddr, int32_t cpid, uint64_t *entry)
{
uint64_t tag;
uint64_t key;
key = (vaddr >> tlb_ord->page_shift) & tlb_ord->key_mask;
tag = vaddr >> (tlb_ord->page_shift + tlb_ord->key_size);
// entry host
entry[0] |= key |
(tag << tlb_ord->key_size) |
((uint64_t)cpid << (tlb_ord->key_size + tlb_ord->tag_size)) |
(0UL << (tlb_ord->key_size + tlb_ord->tag_size + PID_SIZE));
entry[1] |= 0;
dbg_info("unmapping TLB entry, vaddr %llx, cpid %d, hugepage %d\n", vaddr, cpid, tlb_ord->hugepage);
}
/**
* @brief Map TLB
*
* @param d - vFPGA
* @param en_tlbf - TLBF enabled
* @param map_array - prepped map array
* @param paddr - physical address
* @param cpid - Coyote PID
* @param card - map card mem as well
*/
void tlb_service_dev(struct fpga_dev *d, struct tlb_order *tlb_ord, uint64_t* map_array, uint32_t n_pages)
{
int i = 0;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
// lock
spin_lock(&d->lock);
if(pd->en_tlbf && (n_pages > MAX_MAP_AXIL_PAGES)) {
// lock
spin_lock(&pd->tlb_lock);
// start DMA
pd->fpga_stat_cnfg->tlb_addr = virt_to_phys((void *)map_array);
pd->fpga_stat_cnfg->tlb_len = n_pages * 2 * sizeof(uint64_t);
if(tlb_ord->hugepage) {
pd->fpga_stat_cnfg->tlb_ctrl = TLBF_CTRL_START | ((d->id && TLBF_CTRL_ID_MASK) << TLBF_CTRL_ID_SHFT);
} else {
pd->fpga_stat_cnfg->tlb_ctrl = TLBF_CTRL_START | (((pd->n_fpga_reg + d->id) && TLBF_CTRL_ID_MASK) << TLBF_CTRL_ID_SHFT);
}
// poll
while ((pd->fpga_stat_cnfg->tlb_stat & TLBF_STAT_DONE) != 0x1)
ndelay(100);
// unlock
spin_unlock(&pd->tlb_lock);
} else {
// map each page through AXIL
for (i = 0; i < n_pages; i++) {
if(tlb_ord->hugepage) {
d->fpga_lTlb[0] = map_array[2*i+0];
d->fpga_lTlb[1] = map_array[2*i+1];
} else {
d->fpga_sTlb[0] = map_array[2*i+0];
d->fpga_sTlb[1] = map_array[2*i+1];
}
}
}
// unlock
spin_unlock(&d->lock);
}
/**
* @brief Release all remaining user pages
*
* @param d - vFPGA
* @param dirtied - modified
*/
int tlb_put_user_pages_all(struct fpga_dev *d, int dirtied)
{
int i, bkt;
struct user_pages *tmp_buff;
uint64_t vaddr_tmp;
int32_t cpid_tmp;
uint64_t *map_array;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
hash_for_each(user_sbuff_map[d->id], bkt, tmp_buff, entry) {
// release host pages
if(dirtied)
for(i = 0; i < tmp_buff->n_hpages; i++)
SetPageDirty(tmp_buff->hpages[i]);
for(i = 0; i < tmp_buff->n_hpages; i++)
put_page(tmp_buff->hpages[i]);
kfree(tmp_buff->hpages);
// release card pages
if(pd->en_mem) {
if(tmp_buff->huge)
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, LARGE_CHUNK_ALLOC);
else
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, SMALL_CHUNK_ALLOC);
}
// unmap from TLB
vaddr_tmp = tmp_buff->vaddr;
cpid_tmp = tmp_buff->cpid;
// map array
map_array = (uint64_t *)kzalloc(tmp_buff->n_pages * 2 * sizeof(uint64_t), GFP_KERNEL);
if (map_array == NULL) {
dbg_info("map buffers could not be allocated\n");
return -ENOMEM;
}
// huge pages
if(tmp_buff->huge) {
// fill mappings
for (i = 0; i < tmp_buff->n_pages; i++) {
tlb_create_unmap(pd->ltlb_order, vaddr_tmp, cpid_tmp, &map_array[2*i]);
vaddr_tmp += pd->ltlb_order->page_size;
}
// fire
tlb_service_dev(d, pd->ltlb_order, map_array, tmp_buff->n_pages);
// small pages
} else {
// fill mappings
for (i = 0; i < tmp_buff->n_pages; i++) {
tlb_create_unmap(pd->stlb_order, vaddr_tmp, cpid_tmp, &map_array[2*i]);
vaddr_tmp += PAGE_SIZE;
}
// fire
tlb_service_dev(d, pd->stlb_order, map_array, tmp_buff->n_pages);
}
// free
kfree((void *)map_array);
// remove from map
hash_del(&tmp_buff->entry);
}
return 0;
}
/**
* @brief Release user pages
*
* @param d - vFPGA
* @param vaddr - starting vaddr
* @param cpid - Coyote PID
* @param dirtied - modified
*/
int tlb_put_user_pages(struct fpga_dev *d, uint64_t vaddr, int32_t cpid, int dirtied)
{
int i;
struct user_pages *tmp_buff;
uint64_t vaddr_tmp;
uint64_t *map_array;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
hash_for_each_possible(user_sbuff_map[d->id], tmp_buff, entry, vaddr) {
if(tmp_buff->vaddr == vaddr && tmp_buff->cpid == cpid) {
// release host pages
if(dirtied)
for(i = 0; i < tmp_buff->n_hpages; i++)
SetPageDirty(tmp_buff->hpages[i]);
for(i = 0; i < tmp_buff->n_hpages; i++)
put_page(tmp_buff->hpages[i]);
kfree(tmp_buff->hpages);
// release card pages
if(pd->en_mem) {
if(tmp_buff->huge)
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, LARGE_CHUNK_ALLOC);
else
card_free(d, tmp_buff->cpages, tmp_buff->n_pages, SMALL_CHUNK_ALLOC);
}
// unmap from TLB
vaddr_tmp = vaddr;
// map array
map_array = (uint64_t *)kzalloc(tmp_buff->n_pages * 2 * sizeof(uint64_t), GFP_KERNEL);
if (map_array == NULL) {
dbg_info("map buffers could not be allocated\n");
return -ENOMEM;
}
// huge pages
if(tmp_buff->huge) {
// fill mappings
for (i = 0; i < tmp_buff->n_pages; i++) {
tlb_create_unmap(pd->ltlb_order, vaddr_tmp, cpid, &map_array[2*i]);
vaddr_tmp += pd->ltlb_order->page_size;
}
// fire
tlb_service_dev(d, pd->ltlb_order, map_array, tmp_buff->n_pages);
// small pages
} else {
// fill mappings
for (i = 0; i < tmp_buff->n_pages; i++) {
tlb_create_unmap(pd->stlb_order, vaddr_tmp, cpid, &map_array[2*i]);
vaddr_tmp += PAGE_SIZE;
}
// fire
tlb_service_dev(d, pd->stlb_order, map_array, tmp_buff->n_pages);
}
// free
kfree((void *)map_array);
// remove from map
hash_del(&tmp_buff->entry);
}
}
return 0;
}
/**
* @brief Get user pages and fill TLB
*
* @param d - vFPGA
* @param start - starting vaddr
* @param count - number of pages to map
* @param cpid - Coyote PID
* @param pid - user PID
*/
int tlb_get_user_pages(struct fpga_dev *d, uint64_t start, size_t count, int32_t cpid, pid_t pid)
{
int ret_val = 0, i, j;
int n_pages, n_pages_huge;
uint64_t first;
uint64_t last;
struct user_pages *user_pg;
struct vm_area_struct *vma_area_init;
int hugepages;
uint64_t *hpages_phys;
uint64_t curr_vaddr, last_vaddr;
struct task_struct *curr_task;
struct mm_struct *curr_mm;
uint64_t *map_array;
uint64_t vaddr_tmp;
struct bus_drvdata *pd;
BUG_ON(!d);
pd = d->pd;
BUG_ON(!pd);
// context
curr_task = pid_task(find_vpid(pid), PIDTYPE_PID);
dbg_info("pid found = %d", pid);
curr_mm = curr_task->mm;
// hugepages?
vma_area_init = find_vma(curr_mm, start);
hugepages = is_vm_hugetlb_page(vma_area_init);
// number of pages
first = (start & PAGE_MASK) >> PAGE_SHIFT;
last = ((start + count - 1) & PAGE_MASK) >> PAGE_SHIFT;
n_pages = last - first + 1;
if(hugepages) {
if(n_pages > MAX_N_MAP_HUGE_PAGES)
n_pages = MAX_N_MAP_HUGE_PAGES;
} else {
if(n_pages > MAX_N_MAP_PAGES)
n_pages = MAX_N_MAP_PAGES;
}
if (start + count < start)
return -EINVAL;
if (count == 0)
return 0;
// alloc
user_pg = kzalloc(sizeof(struct user_pages), GFP_KERNEL);
BUG_ON(!user_pg);
user_pg->hpages = kcalloc(n_pages, sizeof(*user_pg->hpages), GFP_KERNEL);
if (user_pg->hpages == NULL) {
return -1;
}
dbg_info("allocated %lu bytes for page pointer array for %d pages @0x%p, passed size %ld.\n",
n_pages * sizeof(*user_pg->hpages), n_pages, user_pg->hpages, count);
dbg_info("pages=0x%p\n", user_pg->hpages);
dbg_info("first = %llx, last = %llx\n", first, last);
for (i = 0; i < n_pages - 1; i++) {
user_pg->hpages[i] = NULL;
}
// pin
//ret_val = get_user_pages_remote(curr_task, curr_mm, (unsigned long)start, n_pages, 1, user_pg->hpages, NULL, NULL);
ret_val = get_user_pages_remote(curr_mm, (unsigned long)start, n_pages, 1, user_pg->hpages, NULL, NULL);
//ret_val = pin_user_pages_remote(curr_mm, (unsigned long)start, n_pages, 1, user_pg->hpages, NULL, NULL);
dbg_info("get_user_pages_remote(%llx, n_pages = %d, page start = %lx, hugepages = %d)\n", start, n_pages, page_to_pfn(user_pg->hpages[0]), hugepages);
if(ret_val < n_pages) {
dbg_info("could not get all user pages, %d\n", ret_val);
goto fail_host_unmap;
}
// flush cache
for(i = 0; i < n_pages; i++)
flush_dcache_page(user_pg->hpages[i]);
// add mapped entry
user_pg->vaddr = start;
user_pg->n_hpages = n_pages;
user_pg->huge = hugepages;
vaddr_tmp = start;
// huge pages
if (hugepages) {
first = (start & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift;
last = ((start + count - 1) & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift;
n_pages_huge = last - first + 1;
user_pg->n_pages = n_pages_huge;
// prep hpages
hpages_phys = kzalloc(n_pages_huge * sizeof(uint64_t), GFP_KERNEL);
if (hpages_phys == NULL) {
dbg_info("card buffer %d could not be allocated\n", i);
return -ENOMEM;
}
j = 0;
curr_vaddr = start;
last_vaddr = -1;
for (i = 0; i < n_pages; i++) {
if (((curr_vaddr & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift) != ((last_vaddr & pd->ltlb_order->page_mask) >> pd->ltlb_order->page_shift)) {
hpages_phys[j] = page_to_phys(user_pg->hpages[i]) & pd->ltlb_order->page_mask;
dbg_info("hugepage %d at %llx\n", j, hpages_phys[j]);
last_vaddr = curr_vaddr;
j++;
}
curr_vaddr += PAGE_SIZE;
}
// card alloc
if(pd->en_mem) {
user_pg->cpages = kzalloc(n_pages_huge * sizeof(uint64_t), GFP_KERNEL);
if (user_pg->cpages == NULL) {
dbg_info("card buffer %d could not be allocated\n", i);
return -ENOMEM;
}
ret_val = card_alloc(d, user_pg->cpages, n_pages_huge, LARGE_CHUNK_ALLOC);
if (ret_val) {
dbg_info("could not get all card pages, %d\n", ret_val);
goto fail_card_unmap;
}
dbg_info("card allocated %d hugepages\n", n_pages_huge);
}
// map array
map_array = (uint64_t *)kzalloc(n_pages_huge * 2 * sizeof(uint64_t), GFP_KERNEL);
if (map_array == NULL) {
dbg_info("map buffers could not be allocated\n");
return -ENOMEM;
}
// fill mappings
for (i = 0; i < n_pages_huge; i++) {
tlb_create_map(pd->ltlb_order, vaddr_tmp, hpages_phys[i], (pd->en_mem ? user_pg->cpages[i] : 0), cpid, &map_array[2*i]);
vaddr_tmp += pd->ltlb_order->page_size;
}
// fire
tlb_service_dev(d, pd->ltlb_order, map_array, n_pages_huge);
// free
kfree((void *)map_array);
// small pages
} else {
user_pg->n_pages = n_pages;
// card alloc
if(pd->en_mem) {
user_pg->cpages = kzalloc(n_pages * sizeof(uint64_t), GFP_KERNEL);
if (user_pg->cpages == NULL) {
dbg_info("card buffer %d could not be allocated\n", i);
return -ENOMEM;
}
ret_val = card_alloc(d, user_pg->cpages, n_pages, SMALL_CHUNK_ALLOC);
if (ret_val) {
dbg_info("could not get all card pages, %d\n", ret_val);
goto fail_card_unmap;
}
dbg_info("card allocated %d regular pages\n", n_pages);
}
// map array
map_array = (uint64_t *)kzalloc(n_pages * 2 * sizeof(uint64_t), GFP_KERNEL);
if (map_array == NULL) {
dbg_info("map buffers could not be allocated\n");
return -ENOMEM;
}
// fill mappings
for (i = 0; i < n_pages; i++) {
tlb_create_map(pd->stlb_order, vaddr_tmp, page_to_phys(user_pg->hpages[i]), (pd->en_mem ? user_pg->cpages[i] : 0), cpid, &map_array[2*i]);
vaddr_tmp += PAGE_SIZE;
}
// fire
tlb_service_dev(d, pd->stlb_order, map_array, n_pages);
// free
kfree((void *)map_array);
}
hash_add(user_sbuff_map[d->id], &user_pg->entry, start);
return n_pages;
fail_host_unmap:
// release host pages
for(i = 0; i < ret_val; i++) {
put_page(user_pg->hpages[i]);
}
kfree(user_pg->hpages);
return -ENOMEM;
fail_card_unmap:
// release host pages
for(i = 0; i < user_pg->n_hpages; i++) {
put_page(user_pg->hpages[i]);
}
kfree(user_pg->hpages);
kfree(user_pg->cpages);
return -ENOMEM;
}

57
driver/fpga_mmu.h Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __FPGA_MMU_H__
#define __FPGA_MMU_H__
#include "coyote_dev.h"
/* Alloc huge pages (should be used only if no host hugepages enabled!) */
int alloc_user_buffers(struct fpga_dev *d, unsigned long n_pages, int32_t cpid);
int free_user_buffers(struct fpga_dev *d, uint64_t vaddr, int32_t cpid);
/* PR memory regions */
int alloc_pr_buffers(struct fpga_dev *d, unsigned long n_pages);
int free_pr_buffers(struct fpga_dev *d, uint64_t vaddr);
/* Card memory resources */
int card_alloc(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type);
void card_free(struct fpga_dev *d, uint64_t *card_paddr, uint64_t n_pages, int type);
/* TLB mappings */
void tlb_create_map(struct tlb_order *tlb_ord, uint64_t vaddr, uint64_t paddr_host, uint64_t paddr_card, int32_t cpid, uint64_t *entry);
void tlb_create_unmap(struct tlb_order *tlb_ord, uint64_t vaddr, int32_t cpid, uint64_t *entry);
/* TLB control */
void tlb_service_dev(struct fpga_dev *d, struct tlb_order *tlb_ord, uint64_t *entry, uint32_t n_pages);
/* Page table walks */
int tlb_get_user_pages(struct fpga_dev *d, uint64_t start, size_t count, int32_t cpid, pid_t pid);
int tlb_put_user_pages(struct fpga_dev *d, uint64_t vaddr, int32_t cpid, int dirtied);
int tlb_put_user_pages_all(struct fpga_dev *d, int dirtied);
#endif /* FPGA MMU */

13
driver/old/Makefile Normal file
View File

@ -0,0 +1,13 @@
obj-m := fpga_drv.o
KERNELDIR ?= /lib/modules/$(shell uname -r)/build
PWD := $(shell pwd)
ROOT := $(dir $(M))
XILINXINCLUDE := -I$(ROOT)../include -I$(ROOT)/include
all:
$(MAKE) -C $(KERNELDIR) M=$(PWD) modules
clean:
rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.c .tmp_versions *.order *.symvers *.dwo *.mod *.mod.dwo

3111
driver/old/fpga_drv.c Normal file
View File

File diff suppressed because it is too large Load Diff

604
driver/old/fpga_drv.h Normal file
View File

@ -0,0 +1,604 @@
#ifndef COYOTE_DRV_H
#define COYOTE_DRV_H
#include <linux/ioctl.h>
#include <linux/cdev.h>
#include <linux/pci.h>
#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/hashtable.h>
/* Driver debug */
#define COYOTE_DEBUG 1
/* Network setup */
#define BASE_IP_ADDR_0 0x0B01D4D1
#define BASE_IP_ADDR_1 0x0B01D4D1
#define NODE_ID 0
#define N_TOTAL_NODES 2
/* Debug print */
#if (COYOTE_DEBUG == 0)
#define dbg_info(...)
#else
#define dbg_info(fmt, ...) pr_info("%s():" fmt, \
__func__, ##__VA_ARGS__)
#endif
/* Obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
#define HIGH_32(addr) ((addr >> 16) >> 16)
/* Obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
#define LOW_32(addr) (addr & 0xffffffffUL)
/* Driver info */
#define DRV_NAME "coyote_driver"
#define DEV_NAME "fpga"
/**
* XDMA info
*/
#define MAX_NUM_BARS 3
#define MAX_NUM_CHANNELS 4
#define MAX_NUM_ENGINES (MAX_NUM_CHANNELS * 2)
#define MAX_USER_IRQS 16
#define C2H_CHAN_OFFS 0x1000
#define H2C_CHAN_OFFS 0x0000
#define CHAN_RANGE 0x100
#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
/* Engine IDs */
#define XDMA_ID_H2C 0x1fc0U
#define XDMA_ID_C2H 0x1fc1U
/* Engine regs */
#define XDMA_ENG_IRQ_NUM (1)
#define XDMA_OFS_INT_CTRL (0x2000UL)
#define XDMA_OFS_CONFIG (0x3000UL)
/* Bits of the SG DMA control register */
#define XDMA_CTRL_RUN_STOP (1UL << 0)
#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
/* Bits of the SG DMA status register */
#define XDMA_STAT_BUSY (1UL << 0)
#define XDMA_STAT_DESC_STOPPED (1UL << 1)
#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
#define XDMA_STAT_FETCH_STOPPED (1UL << 5)
#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
#define XDMA_STAT_READ_ERROR (0x1FUL << 9)
#define XDMA_STAT_DESC_ERROR (0x1FUL << 19)
/* Bits of the performance control register */
#define XDMA_PERF_RUN (1UL << 0)
#define XDMA_PERF_CLEAR (1UL << 1)
#define XDMA_PERF_AUTO (1UL << 2)
/* Polling */
#define WB_COUNT_MASK 0x00ffffffUL
#define WB_ERR_MASK (1UL << 31)
#define POLL_TIMEOUT_SECONDS 10
#define NUM_POLLS_PER_SCHED 100
/**
* Static layer
*/
#define BAR_XDMA_CONFIG 0
#define BAR_FPGA_CONFIG 1
/* FPGA static config */
#define FPGA_STAT_CNFG_OFFS 0x0
#define FPGA_STAT_CNFG_SIZE 32 * 1024
#define EN_AVX_MASK 0x1
#define EN_AVX_SHFT 0x0
#define EN_BPSS_MASK 0x2
#define EN_BPSS_SHFT 0x1
#define EN_TLBF_MASK 0x4
#define EN_TLBF_SHFT 0x2
#define EN_WB_MASK 0x8
#define EN_WB_SHFT 0x3
#define TLB_S_ORDER_MASK 0xf0
#define TLB_S_ORDER_SHFT 0x4
#define TLB_S_ASSOC_MASK 0xf00
#define TLB_S_ASSOC_SHFT 0x8
#define TLB_L_ORDER_MASK 0xf000
#define TLB_L_ORDER_SHFT 0xc
#define TLB_L_ASSOC_MASK 0xf0000
#define TLB_L_ASSOC_SHFT 0x10
#define TLB_S_PG_SHFT_MASK 0x3f00000
#define TLB_S_PG_SHFT_SHFT 0x14
#define TLB_L_PG_SHFT_MASK 0xfc000000
#define TLB_L_PG_SHFT_SHFT 0x1a
#define EN_DDR_MASK 0x1
#define EN_DDR_SHFT 0x0
#define EN_HBM_MASK 0x2
#define EN_HBM_SHFT 0x1
#define N_MEM_CHAN_MASK 0x7d
#define N_MEM_CHAN_SHFT 0x2
#define EN_RDMA_0_MASK 0x1
#define EN_RDMA_1_MASK 0x2
#define EN_PR_MASK 0x1
#define EN_PR_SHFT 0x0
#define EN_RDMA_0_MASK 0x1
#define EN_RDMA_0_SHFT 0x0
#define EN_RDMA_1_MASK 0x2
#define EN_RDMA_1_SHFT 0x1
#define EN_TCP_0_MASK 0x1
#define EN_TCP_0_SHFT 0x0
#define EN_TCP_1_MASK 0x2
#define EN_TCP_1_SHFT 0x1
/**
* Dynamic layer
*/
/* FPGA control regions */
#define FPGA_CTRL_SIZE 256 * 1024
#define FPGA_CTRL_OFFS 0x100000
#define FPGA_CTRL_LTLB_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_LTLB_OFFS 0x0
#define FPGA_CTRL_STLB_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_STLB_OFFS 0x10000
#define FPGA_CTRL_USER_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_USER_OFFS 0x20000
#define FPGA_CTRL_CNFG_SIZE FPGA_CTRL_SIZE / 4
#define FPGA_CTRL_CNFG_OFFS 0x30000
#define FPGA_CTRL_CNFG_AVX_SIZE 256 * 1024
#define FPGA_CTRL_CNFG_AVX_OFFS 0x1000000
/* FPGA dynamic control config */
#define FPGA_CNFG_CTRL_IRQ_RESTART 0x100
/* TODO: Maximum transfer size */
#define XDMA_TRANSFER_MAX_BYTES (8 * 1024 * 1024)
/* TLB */
#define TLB_VADDR_RANGE 48
#define TLB_PADDR_RANGE 40
#define PID_SIZE 6
#define MAX_MAP_AXIL_PAGES 64
#define LTLB_PAGE_BITS 21
#define STLB_PAGE_BITS 12
#define LTLB_PADDR_SIZE (TLB_PADDR_RANGE - LTLB_PAGE_BITS)
#define STLB_PADDR_SIZE (TLB_PADDR_RANGE - STLB_PAGE_BITS)
#define TLBF_CTRL_START 0x7
#define TLBF_CTRL_ID_MASK 0xff
#define TLBF_CTRL_ID_SHFT 0x3
#define TLBF_STAT_DONE 0x1
/* Memory allocation */
#define MAX_BUFF_NUM 64 // maximum number of huge pages allowed
#define MAX_PR_BUFF_NUM 64 // maximum number of huge pages allowed
#define MAX_N_MAP_PAGES 128 // TODO: link to params, max no pfault 1M
#define MAX_N_MAP_HUGE_PAGES (16 * 512) // max no pfault 32M
/* Max card pages */
#define N_LARGE_CHUNKS 512
#define N_SMALL_CHUNKS (256 * 1024)
#define MEM_SEP (N_SMALL_CHUNKS * (4 * 1024))
#define MAX_N_REGIONS 16
#define SMALL_CHUNK_ALLOC 0
#define LARGE_CHUNK_ALLOC 1
/* PR */
#define PR_CTRL_START_MIDDLE 0x3
#define PR_CTRL_START_LAST 0x7
#define PR_STAT_DONE 0x1
/* FPGA config commands */
/**
* Cdev
*/
/* Major number */
#define FPGA_MAJOR 0 // dynamic
/* MMAP */
#define MMAP_CTRL 0x0
#define MMAP_CNFG 0x1
#define MMAP_CNFG_AVX 0x2
#define MMAP_WB 0x3
#define MMAP_BUFF 0x200
#define MMAP_PR 0x400
/* IOCTL */
#define IOCTL_ALLOC_HOST_USER_MEM _IOW('D', 1, unsigned long) // large pages (no hugepage support)
#define IOCTL_FREE_HOST_USER_MEM _IOW('D', 2, unsigned long)
#define IOCTL_ALLOC_HOST_PR_MEM _IOW('D', 3, unsigned long) // pr pages
#define IOCTL_FREE_HOST_PR_MEM _IOW('D', 4, unsigned long)
#define IOCTL_MAP_USER _IOW('D', 5, unsigned long) // map
#define IOCTL_UNMAP_USER _IOW('D', 6, unsigned long)
#define IOCTL_REGISTER_PID _IOW('D', 7, unsigned long) // register pid
#define IOCTL_UNREGISTER_PID _IOW('D', 8, unsigned long)
#define IOCTL_RECONFIG_LOAD _IOW('D', 9, unsigned long) // reconfiguration
#define IOCTL_ARP_LOOKUP _IOW('D', 10, unsigned long) // arp lookup
#define IOCTL_SET_IP_ADDRESS _IOW('D', 11, unsigned long)
#define IOCTL_SET_BOARD_NUM _IOW('D', 12, unsigned long)
#define IOCTL_WRITE_CTX _IOW('D', 13, unsigned long) // qp context
#define IOCTL_WRITE_CONN _IOW('D', 14, unsigned long) // qp connection
#define IOCTL_SET_TCP_OFFS _IOW('D', 15, unsigned long) // tcp mem offsets
#define IOCTL_READ_CNFG _IOR('D', 32, unsigned long) // status cnfg
#define IOCTL_NET_STATS _IOR('D', 33, unsigned long) // status network
#define IOCTL_READ_ENG_STATUS _IOR('D', 35, unsigned long) // status engines
/* Hash */
#define PR_HASH_TABLE_ORDER 8
#define PR_BATCH_SIZE (2 * 1024 * 1024)
#define USER_HASH_TABLE_ORDER 8
/* PID */
#define N_CPID_MAX 64
#define WB_SIZE (2 * N_CPID_MAX * sizeof(uint32_t))
#define N_WB_PAGES ((WB_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
/* Network */
#define EN_LOWSPEED 0x5
#define N_NET_STAT_REGS 9
/* Copy */
#define MAX_USER_WORDS 32
/**
* Reg maps
*/
/* DMA engine reg map */
struct engine_regs {
uint32_t id;
uint32_t ctrl;
uint32_t ctrl_w1s;
uint32_t ctrl_w1c;
uint32_t rsrvd_1[12];
uint32_t status;
uint32_t status_rc;
uint32_t completed_desc_count;
uint32_t alignments;
uint32_t rsrvd_2[14]; // padding
uint32_t poll_mode_wb_lo;
uint32_t poll_mode_wb_hi;
uint32_t interrupt_enable_mask;
uint32_t interrupt_enable_mask_w1s;
uint32_t interrupt_enable_mask_w1c;
uint32_t rsrvd_3[9]; // padding
uint32_t perf_ctrl;
uint32_t perf_cyc_lo;
uint32_t perf_cyc_hi;
uint32_t perf_dat_lo;
uint32_t perf_dat_hi;
uint32_t perf_pnd_lo;
uint32_t perf_pnd_hi;
} __packed;
/* Interrupt reg map */
struct interrupt_regs {
uint32_t id;
uint32_t user_int_enable;
uint32_t user_int_enable_w1s;
uint32_t user_int_enable_w1c;
uint32_t channel_int_enable;
uint32_t channel_int_enable_w1s;
uint32_t channel_int_enable_w1c;
uint32_t reserved_1[9]; // padding
uint32_t user_int_request;
uint32_t channel_int_request;
uint32_t user_int_pending;
uint32_t channel_int_pending;
uint32_t reserved_2[12]; // padding
uint32_t user_msi_vector[8];
uint32_t channel_msi_vector[8];
} __packed;
/* Polled mode descriptors struct */
struct xdma_poll_wb {
uint32_t completed_desc_count;
uint32_t reserved_1[7];
} __packed;
/* FPGA static config reg map */
struct fpga_stat_cnfg_regs {
uint64_t probe;
uint64_t n_chan;
uint64_t n_regions;
uint64_t ctrl_cnfg;
uint64_t mem_cnfg;
uint64_t pr_cnfg;
uint64_t rdma_cnfg;
uint64_t tcp_cnfg;
uint64_t lspeed_cnfg;
uint64_t reserved_0[1];
uint64_t pr_ctrl;
uint64_t pr_stat;
uint64_t pr_addr;
uint64_t pr_len;
uint64_t tlb_ctrl;
uint64_t tlb_stat;
uint64_t tlb_addr;
uint64_t tlb_len;
uint64_t reserved_1[2];
uint64_t net_0_ip;
uint64_t net_0_boardnum;
uint64_t net_0_arp;
uint64_t rdma_0_qp_ctx[3];
uint64_t rdma_0_qp_conn[3];
uint64_t tcp_0_offs[2];
uint64_t net_0_debug[N_NET_STAT_REGS];
uint64_t net_1_ip;
uint64_t net_1_boardnum;
uint64_t net_1_arp;
uint64_t rdma_1_qp_ctx[3];
uint64_t rdma_1_qp_conn[3];
uint64_t tcp_1_offs[2];
uint64_t net_1_debug[N_NET_STAT_REGS];
} __packed;
/* FPGA dynamic config reg map */
struct fpga_cnfg_regs {
uint64_t ctrl;
uint64_t vaddr_rd;
uint64_t len_rd;
uint64_t vaddr_wr;
uint64_t len_wr;
uint64_t vaddr_miss;
uint64_t len_miss;
uint64_t datapath_set;
uint64_t datapath_clr;
uint64_t stat_cmd_used_rd;
uint64_t stat_cmd_used_wr;
uint64_t stat_sent[6];
uint64_t stat_pfaults;
uint64_t wback_rd;
uint64_t wback_wr;
// Rest of regs not used in the driver
} __packed;
/* FPGA dynamic config reg map */
struct fpga_cnfg_regs_avx {
uint64_t ctrl[4];
uint64_t vaddr_miss;
uint64_t len_miss;
uint64_t pf[2];
uint64_t datapath_set[4];
uint64_t datapath_clr[4];
uint64_t stat[4];
uint64_t wback[4];
// Rest not used in the driver
} __packed;
/**
* Structs
*/
/* Engine descriptors */
struct xdma_sgdma_regs {
uint32_t identifier;
uint32_t reserved_1[31]; /* padding */
/* bus address to first descriptor in Root Complex Memory */
uint32_t first_desc_lo;
uint32_t first_desc_hi;
/* number of adjacent descriptors at first_desc */
uint32_t first_desc_adjacent;
uint32_t credits;
} __packed;
/* Engine struct */
struct xdma_engine {
int channel; // egnine channel
char *name; // engine name
struct pci_drvdata *pd; // PCI device
struct engine_regs *regs; // HW regs, control and status
struct engine_sgdma_regs *sgdma_regs; // SGDMA reg BAR offset
// Config
int running; // engine state
int c2h; // c2h(write) or h2c(read)
uint32_t status;
int addr_align; // source/dest alignment in bytes
int len_granularity; // transfer length multiple
int addr_bits; // HW datapath address width
/* Members associated with polled mode support */
uint8_t *poll_mode_addr_virt; /* virt addr for descriptor writeback */
uint64_t poll_mode_phys_addr; /* bus addr for descriptor writeback */
};
/* Mapped user pages */
struct user_pages {
struct hlist_node entry;
uint64_t vaddr;
bool huge;
int32_t cpid;
uint64_t n_hpages;
uint64_t n_pages;
struct page **hpages;
uint64_t *cpages;
};
/* User tables */
struct hlist_head user_lbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // large alloc
struct hlist_head user_sbuff_map[MAX_N_REGIONS][1 << (USER_HASH_TABLE_ORDER)]; // main alloc
/* Mapped large PR pages */
struct pr_pages {
struct hlist_node entry;
int reg_id;
uint64_t vaddr;
uint64_t n_pages;
struct page **pages;
};
/* PR table */
struct hlist_head pr_buff_map[1 << (PR_HASH_TABLE_ORDER)];
/* Pool chunks */
struct chunk {
uint32_t id;
struct chunk *next;
};
/* Virtual FPGA device */
struct fpga_dev {
int id; // identifier
struct cdev cdev; // char device
struct pci_drvdata *pd; // PCI device
struct pr_ctrl *prc; // PR controller
// Current task
struct task_struct *curr_task;
struct mm_struct *curr_mm;
// Control region
uint64_t fpga_phys_addr_ctrl;
uint64_t fpga_phys_addr_ctrl_avx;
// Writeback
uint32_t *wb_addr_virt;
uint64_t wb_phys_addr;
// TLBs
uint64_t *fpga_lTlb; // large page TLB
uint64_t *fpga_sTlb; // small page TLB
struct fpga_cnfg_regs *fpga_cnfg; // config
struct fpga_cnfg_regs_avx *fpga_cnfg_avx; // config AVX
// PIDs
spinlock_t card_pid_lock;
struct chunk *pid_chunks;
pid_t *pid_array;
int num_free_pid_chunks;
struct chunk *pid_alloc;
// Engines
struct xdma_engine *engine_h2c; // h2c engine
struct xdma_engine *engine_c2h; // c2h engine
// In use
atomic_t in_use; // busy flag
// Lock
spinlock_t lock; // protects concurrent accesses
// Allocated buffers
struct user_pages curr_user_buff;
};
/* PR controller */
struct pr_ctrl {
struct pci_drvdata *pd; // PCI device
spinlock_t lock;
// Engines
struct xdma_engine *engine_h2c; // h2c engine
struct xdma_engine *engine_c2h; // c2h engine
// Allocated buffers
struct pr_pages curr_buff;
};
/* TLB order */
struct tlb_order {
bool hugepage;
uint64_t page_shift;
uint64_t page_size;
uint64_t page_mask;
int assoc;
int key_mask;
int key_size;
int tag_mask;
int tag_size;
int phy_mask;
int phy_size;
};
/* PCI driver data */
struct pci_drvdata {
struct pci_dev *pci_dev;
// BARs
int regions_in_use;
int got_regions;
void *__iomem bar[MAX_NUM_BARS];
unsigned long bar_phys_addr[MAX_NUM_BARS];
unsigned long bar_len[MAX_NUM_BARS];
// Engines
int engines_num;
// FPGA static config
uint probe;
int n_fpga_chan;
int n_fpga_reg;
int en_avx;
int en_bypass;
int en_tlbf;
int en_wb;
int en_ddr;
int en_hbm;
int en_mem;
int n_ddr_chan;
int en_pr;
int en_rdma_0;
int en_rdma_1;
int en_tcp_0;
int en_tcp_1;
int en_net_0;
int en_net_1;
struct fpga_stat_cnfg_regs *fpga_stat_cnfg;
struct fpga_dev *fpga_dev;
// PR control
struct pr_ctrl prc;
// TLB order
struct tlb_order *stlb_order;
struct tlb_order *ltlb_order;
// Locks
spinlock_t stat_lock;
spinlock_t prc_lock;
spinlock_t tlb_lock;
// IRQ
int irq_count;
int irq_line;
int msix_enabled;
struct msix_entry irq_entry[32];
// Card memory
spinlock_t card_l_lock;
struct chunk *lchunks;
int num_free_lchunks;
struct chunk *lalloc;
spinlock_t card_s_lock;
struct chunk *schunks;
int num_free_schunks;
struct chunk *salloc;
};
#endif

1110
driver/pci/pci_dev.c Normal file
View File

File diff suppressed because it is too large Load Diff

80
driver/pci/pci_dev.h Normal file
View File

@ -0,0 +1,80 @@
/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __PCI_DEV_H__
#define __PCI_DEV_H__
#include "../coyote_dev.h"
#include "../fpga_dev.h"
#include "../fpga_isr.h"
/* Interrupts */
irqreturn_t fpga_tlb_miss_isr(int irq, void *dev_id);
void user_interrupts_enable(struct bus_drvdata *d, uint32_t mask);
void user_interrupts_disable(struct bus_drvdata *d, uint32_t mask);
uint32_t read_interrupts(struct bus_drvdata *d);
uint32_t build_vector_reg(uint32_t a, uint32_t b, uint32_t c, uint32_t d);
void write_msix_vectors(struct bus_drvdata *d);
int msix_irq_setup(struct bus_drvdata *d);
int irq_setup(struct bus_drvdata *d, struct pci_dev *pdev);
void irq_teardown(struct bus_drvdata *d);
int msix_capable(struct pci_dev *pdev, int type);
int pci_check_msix(struct bus_drvdata *d, struct pci_dev *pdev);
/* Engine */
uint32_t get_engine_channel_id(struct engine_regs *regs);
uint32_t get_engine_id(struct engine_regs *regs);
void engine_destroy(struct bus_drvdata *d, struct xdma_engine *engine);
void remove_engines(struct bus_drvdata *d);
void engine_alignments(struct xdma_engine *engine);
struct xdma_engine *engine_create(struct bus_drvdata *d, int offs, int c2h, int channel);
int probe_for_engine(struct bus_drvdata *d, int c2h, int channel);
int probe_engines(struct bus_drvdata *d);
uint32_t engine_status_read(struct xdma_engine *engine);
//#define XDMA_WBACK
#ifdef XDMA_WBACK
void engine_writeback_teardown(struct bus_drvdata *d, struct xdma_engine *engine);
int engine_writeback_setup(struct bus_drvdata *d, struct xdma_engine *engine);
#endif
/* BARs */
int map_single_bar(struct bus_drvdata *d, struct pci_dev *pdev, int idx, int curr_idx);
void unmap_bars(struct bus_drvdata *d, struct pci_dev *pdev);
int map_bars(struct bus_drvdata *d, struct pci_dev *pdev);
/* Regions */
int request_regions(struct bus_drvdata *d, struct pci_dev *pdev);
/* Probe */
int pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
void pci_remove(struct pci_dev *pdev);
/* Init */
int pci_init(void);
void pci_exit(void);
#endif // PCIe device