replica
/
Coyote
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Coyote/driver/fpga_dev.c

511 lines
18 KiB
C
Raw Blame History

/**
* Copyright (c) 2021, Systems Group, ETH Zurich
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "fpga_dev.h"
#include "fpga_sysfs.h"
int fpga_major = FPGA_MAJOR;
struct class *fpga_class = NULL;
/**
* @brief Fops
*
*/
struct file_operations fpga_fops = {
.owner = THIS_MODULE,
.open = fpga_open,
.release = fpga_release,
.mmap = fpga_mmap,
.unlocked_ioctl = fpga_ioctl,
};
/**
* @brief Sysfs
*
*/
static struct kobj_attribute kobj_attr_ip_q0 = __ATTR(cyt_attr_ip_q0, 0664, cyt_attr_ip_q0_show, cyt_attr_ip_q0_store);
static struct kobj_attribute kobj_attr_ip_q1 = __ATTR(cyt_attr_ip_q1, 0664, cyt_attr_ip_q1_show, cyt_attr_ip_q1_store);
static struct kobj_attribute kobj_attr_mac_q0 = __ATTR(cyt_attr_mac_q0, 0664, cyt_attr_mac_q0_show, cyt_attr_mac_q0_store);
static struct kobj_attribute kobj_attr_mac_q1 = __ATTR(cyt_attr_mac_q1, 0664, cyt_attr_mac_q1_show, cyt_attr_mac_q1_store);
static struct kobj_attribute kobj_attr_nstats_q0 = __ATTR_RO(cyt_attr_nstats_q0);
static struct kobj_attribute kobj_attr_nstats_q1 = __ATTR_RO(cyt_attr_nstats_q1);
static struct kobj_attribute kobj_attr_xstats = __ATTR_RO(cyt_attr_xstats);
static struct kobj_attribute kobj_attr_cnfg = __ATTR_RO(cyt_attr_cnfg);
static struct kobj_attribute kobj_attr_eost = __ATTR(cyt_attr_eost, 0664, cyt_attr_eost_show, cyt_attr_eost_store);
static struct attribute *attrs[] = {
&kobj_attr_ip_q0.attr,
&kobj_attr_ip_q1.attr,
&kobj_attr_mac_q0.attr,
&kobj_attr_mac_q1.attr,
&kobj_attr_nstats_q0.attr,
&kobj_attr_nstats_q1.attr,
&kobj_attr_xstats.attr,
&kobj_attr_cnfg.attr,
&kobj_attr_eost.attr,
NULL,
};
static struct attribute_group attr_group = {
.attrs = attrs,
};
/**
* @brief Read static configuration
*
*/
int read_static_config(struct bus_drvdata *d)
{
long tmp;
int ret_val = 0;
// probe
d->probe = d->fpga_stat_cnfg->probe;
pr_info("deployment id %08x\n", d->probe);
// channels and regions
d->n_fpga_chan = d->fpga_stat_cnfg->n_chan;
d->n_fpga_reg = d->fpga_stat_cnfg->n_regions;
pr_info("detected %d virtual FPGA regions, %d FPGA channels\n", d->n_fpga_reg, d->n_fpga_chan);
// flags
d->en_avx = (d->fpga_stat_cnfg->ctrl_cnfg & EN_AVX_MASK) >> EN_AVX_SHFT;
d->en_bypass = (d->fpga_stat_cnfg->ctrl_cnfg & EN_BPSS_MASK) >> EN_BPSS_SHFT;
d->en_tlbf = (d->fpga_stat_cnfg->ctrl_cnfg & EN_TLBF_MASK) >> EN_TLBF_SHFT;
d->en_wb = (d->fpga_stat_cnfg->ctrl_cnfg & EN_WB_MASK) >> EN_WB_SHFT;
pr_info("enabled AVX %d, enabled bypass %d, enabled tlb fast %d, enabled writeback %d\n", d->en_avx, d->en_bypass, d->en_tlbf, d->en_wb);
// mmu
d->stlb_order = kzalloc(sizeof(struct tlb_order), GFP_KERNEL);
BUG_ON(!d->stlb_order);
d->stlb_order->hugepage = false;
d->stlb_order->key_size = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_ORDER_MASK) >> TLB_S_ORDER_SHFT;
d->stlb_order->assoc = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_ASSOC_MASK) >> TLB_S_ASSOC_SHFT;
d->stlb_order->page_shift = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_S_PG_SHFT_MASK) >> TLB_S_PG_SHFT_SHFT;
BUG_ON(d->stlb_order->page_shift != PAGE_SHIFT);
d->stlb_order->page_size = PAGE_SIZE;
d->stlb_order->page_mask = PAGE_MASK;
d->stlb_order->key_mask = (1 << d->stlb_order->key_size) - 1;
d->stlb_order->tag_size = TLB_VADDR_RANGE - d->stlb_order->page_shift - d->stlb_order->key_size;
d->stlb_order->tag_mask = (1 << d->stlb_order->tag_size) - 1;
d->stlb_order->phy_size = TLB_PADDR_RANGE - d->stlb_order->page_shift;
d->stlb_order->phy_mask = (1 << d->stlb_order->phy_size) - 1;
pr_info("sTLB order %d, sTLB assoc %d, sTLB page size %lld\n", d->stlb_order->key_size, d->stlb_order->assoc, d->stlb_order->page_size);
d->ltlb_order = kzalloc(sizeof(struct tlb_order), GFP_KERNEL);
BUG_ON(!d->ltlb_order);
d->ltlb_order->hugepage = true;
d->ltlb_order->key_size = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_ORDER_MASK) >> TLB_L_ORDER_SHFT;
d->ltlb_order->assoc = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_ASSOC_MASK) >> TLB_L_ASSOC_SHFT;
d->ltlb_order->page_shift = (d->fpga_stat_cnfg->ctrl_cnfg & TLB_L_PG_SHFT_MASK) >> TLB_L_PG_SHFT_SHFT;
d->ltlb_order->page_size = 1 << d->ltlb_order->page_shift;
d->ltlb_order->page_mask = (~(d->ltlb_order->page_size - 1));
d->ltlb_order->key_mask = (1 << d->ltlb_order->key_size) - 1;
d->ltlb_order->tag_size = TLB_VADDR_RANGE - d->ltlb_order->page_shift - d->ltlb_order->key_size;
d->ltlb_order->tag_mask = (1 << d->ltlb_order->tag_size) - 1;
d->ltlb_order->phy_size = TLB_PADDR_RANGE - d->ltlb_order->page_shift ;
d->ltlb_order->phy_mask = (1 << d->ltlb_order->phy_size) - 1;
pr_info("lTLB order %d, lTLB assoc %d, lTLB page size %lld\n", d->ltlb_order->key_size, d->ltlb_order->assoc, d->ltlb_order->page_size);
// mem
d->en_strm = (d->fpga_stat_cnfg->mem_cnfg & EN_STRM_MASK) >> EN_STRM_SHFT;
d->en_mem = (d->fpga_stat_cnfg->mem_cnfg & EN_MEM_MASK) >> EN_MEM_SHFT;
pr_info("enabled host streams %d, enabled card streams (mem) %d\n", d->en_strm, d->en_mem);
// pr
d->en_pr = (d->fpga_stat_cnfg->pr_cnfg & EN_PR_MASK) >> EN_PR_SHFT;
pr_info("enabled PR %d\n", d->en_pr);
// set eost
if(d->en_pr) {
d->eost = eost;
d->fpga_stat_cnfg->pr_eost = eost;
pr_info("set EOST [clks] %lld\n", d->eost);
}
// network
d->en_rdma_0 = (d->fpga_stat_cnfg->rdma_cnfg & EN_RDMA_0_MASK) >> EN_RDMA_0_SHFT;
d->en_rdma_1 = (d->fpga_stat_cnfg->rdma_cnfg & EN_RDMA_1_MASK) >> EN_RDMA_1_SHFT;
pr_info("enabled RDMA on QSFP0 %d, enabled RDMA on QSFP1 %d\n", d->en_rdma_0, d->en_rdma_1);
d->en_tcp_0 = (d->fpga_stat_cnfg->tcp_cnfg & EN_TCP_0_MASK) >> EN_TCP_0_SHFT;
d->en_tcp_1 = (d->fpga_stat_cnfg->tcp_cnfg & EN_TCP_1_MASK) >> EN_TCP_1_SHFT;
pr_info("enabled TCP/IP on QSFP0 %d, enabled TCP/IP on QSFP1 %d\n", d->en_tcp_0, d->en_tcp_1);
// set ip and mac
d->en_net_0 = d->en_rdma_0 | d->en_tcp_0;
if(d->en_net_0) {
ret_val = kstrtol(ip_addr_q0, 16, &tmp);
d->net_0_ip_addr = (uint64_t) tmp;
ret_val = kstrtol(mac_addr_q0, 16, &tmp);
d->net_0_mac_addr = (uint64_t) tmp;
d->fpga_stat_cnfg->net_0_ip = d->net_0_ip_addr;
d->fpga_stat_cnfg->net_0_mac = d->net_0_mac_addr;
pr_info("set QSFP0 ip %08x, mac %012llx\n", d->net_0_ip_addr, d->net_0_mac_addr);
}
d->en_net_1 = d->en_rdma_1 | d->en_tcp_1;
if(d->en_net_1) {
ret_val = kstrtol(ip_addr_q1, 16, &tmp);
d->net_1_ip_addr = (uint64_t) tmp;
ret_val = kstrtol(mac_addr_q1, 16, &tmp);
d->net_1_mac_addr = (uint64_t) tmp;
d->fpga_stat_cnfg->net_1_ip = d->net_1_ip_addr;
d->fpga_stat_cnfg->net_1_mac = d->net_1_mac_addr;
pr_info("set QSFP1 ip %08x, mac %012llx\n", d->net_1_ip_addr, d->net_1_mac_addr);
}
// lowspeed ctrl
d->fpga_stat_cnfg->lspeed_cnfg = EN_LOWSPEED;
return ret_val;
}
/**
* @brief Allocate FPGA memory resources
*
*/
int alloc_card_resources(struct bus_drvdata *d)
{
int ret_val = 0;
int i;
// init chunks card
d->num_free_lchunks = N_LARGE_CHUNKS;
d->num_free_schunks = N_SMALL_CHUNKS;
d->lchunks = vzalloc(N_LARGE_CHUNKS * sizeof(struct chunk));
if (!d->lchunks) {
pr_err("memory region for cmem structs not obtained\n");
goto err_alloc_lchunks; // ERR_ALLOC_LCHUNKS
}
d->schunks = vzalloc(N_SMALL_CHUNKS * sizeof(struct chunk));
if (!d->schunks) {
pr_err("memory region for cmem structs not obtained\n");
goto err_alloc_schunks; // ERR_ALLOC_SCHUNKS
}
for (i = 0; i < N_LARGE_CHUNKS - 1; i++) {
d->lchunks[i].used = false;
d->lchunks[i].id = i;
d->lchunks[i].next = &d->lchunks[i + 1];
}
for (i = 0; i < N_SMALL_CHUNKS - 1; i++) {
d->schunks[i].used = false;
d->schunks[i].id = i;
d->schunks[i].next = &d->schunks[i + 1];
}
d->lalloc = &d->lchunks[0];
d->salloc = &d->schunks[0];
goto end;
err_alloc_schunks:
vfree(d->lchunks);
err_alloc_lchunks:
ret_val = -ENOMEM;
end:
return ret_val;
}
/**
* @brief Free card memory resources
*
*/
void free_card_resources(struct bus_drvdata *d)
{
// free card memory structs
vfree(d->schunks);
vfree(d->lchunks);
pr_info("card resources deallocated\n");
}
/**
* @brief Initialize spin locks
*
*/
void init_spin_locks(struct bus_drvdata *d)
{
// initialize spinlocks
spin_lock_init(&d->card_l_lock);
spin_lock_init(&d->card_s_lock);
spin_lock_init(&d->prc.lock);
spin_lock_init(&d->stat_lock);
spin_lock_init(&d->prc_lock);
spin_lock_init(&d->tlb_lock);
}
static struct kobj_type cyt_kobj_type = {
.sysfs_ops = &kobj_sysfs_ops,
};
/**
* @brief Create sysfs entry
*
*/
int create_sysfs_entry(struct bus_drvdata *d) {
int ret_val = 0;
pr_info("creating sysfs entry - coyote_cnfg\n");
ret_val = kobject_init_and_add(&d->cyt_kobj, &cyt_kobj_type, kernel_kobj, "coyote_cnfg");
if(ret_val) {
return -ENOMEM;
}
ret_val = sysfs_create_group(&d->cyt_kobj, &attr_group);
if(ret_val) {
return -ENODEV;
}
return ret_val;
}
/**
* @brief Remove sysfs entry
*
*/
void remove_sysfs_entry(struct bus_drvdata *d) {
sysfs_remove_group(&d->cyt_kobj, &attr_group);
kobject_put(&d->cyt_kobj);
}
/**
* @brief Init char devices
*
*/
int init_char_devices(struct bus_drvdata *d, dev_t dev)
{
int ret_val = 0;
ret_val = alloc_chrdev_region(&dev, 0, d->n_fpga_reg, DEV_NAME);
fpga_major = MAJOR(dev);
if (ret_val) {
pr_err("failed to register virtual FPGA devices");
goto end;
}
pr_info("virtual FPGA device regions allocated, major number %d\n", fpga_major);
// create device class
fpga_class = class_create(THIS_MODULE, DEV_NAME);
// virtual FPGA devices
d->fpga_dev = kmalloc(d->n_fpga_reg * sizeof(struct fpga_dev), GFP_KERNEL);
if (!d->fpga_dev) {
pr_err("could not allocate memory for fpga devices\n");
goto err_char_mem; // ERR_CHAR_MEM
}
memset(d->fpga_dev, 0, d->n_fpga_reg * sizeof(struct fpga_dev));
pr_info("allocated memory for fpga devices\n");
goto end;
err_char_mem:
unregister_chrdev_region(dev, d->n_fpga_reg);
ret_val = -ENOMEM;
end:
return ret_val;
}
/**
* @brief Delete char devices
*
*/
void free_char_devices(struct bus_drvdata *d)
{
// free virtual FPGA memory
kfree(d->fpga_dev);
pr_info("virtual FPGA device memory freed\n");
// remove class
class_destroy(fpga_class);
pr_info("fpga class deleted\n");
// remove char devices
unregister_chrdev_region(MKDEV(fpga_major, 0), d->n_fpga_reg);
pr_info("char devices unregistered\n");
}
/**
* @brief Initialize vFPGAs
*
*/
int init_fpga_devices(struct bus_drvdata *d)
{
int ret_val = 0;
int i, j;
int devno;
for (i = 0; i < d->n_fpga_reg; i++) {
// ID
d->fpga_dev[i].id = i;
// PCI device
d->fpga_dev[i].pd = d;
d->fpga_dev[i].prc = &d->prc;
// physical
if(cyt_arch == CYT_ARCH_PCI) {
d->fpga_dev[i].fpga_phys_addr_ctrl = d->bar_phys_addr[BAR_FPGA_CONFIG] + FPGA_CTRL_OFFS + i * FPGA_CTRL_SIZE;
d->fpga_dev[i].fpga_phys_addr_ctrl_avx = d->bar_phys_addr[BAR_FPGA_CONFIG] + FPGA_CTRL_CNFG_AVX_OFFS + i * FPGA_CTRL_CNFG_AVX_SIZE;
} else if(cyt_arch == CYT_ARCH_ECI) {
d->fpga_dev[i].fpga_phys_addr_ctrl = d->io_phys_addr + FPGA_CTRL_OFFS + i*FPGA_CTRL_SIZE;
}
// MMU control region
d->fpga_dev[i].fpga_lTlb = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_LTLB_OFFS, FPGA_CTRL_LTLB_SIZE);
d->fpga_dev[i].fpga_sTlb = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_STLB_OFFS, FPGA_CTRL_STLB_SIZE);
// FPGA engine control
d->fpga_dev[i].fpga_cnfg = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl + FPGA_CTRL_CNFG_OFFS, FPGA_CTRL_CNFG_SIZE);
// FPGA engine control AVX
if(cyt_arch == CYT_ARCH_PCI) {
d->fpga_dev[i].fpga_cnfg_avx = ioremap(d->fpga_dev[i].fpga_phys_addr_ctrl_avx, FPGA_CTRL_CNFG_AVX_SIZE);
}
// init chunks pid
d->fpga_dev[i].num_free_pid_chunks = N_CPID_MAX;
d->fpga_dev[i].pid_chunks = vzalloc(N_CPID_MAX * sizeof(struct chunk));
if (!d->fpga_dev[i].pid_chunks) {
pr_err("memory region for pid chunks not obtained\n");
goto err_alloc_pid_chunks; // ERR_ALLOC_PID_CHUNKS
}
d->fpga_dev[i].pid_array = vzalloc(N_CPID_MAX * sizeof(pid_t));
if (!d->fpga_dev[i].pid_array) {
pr_err("memory region for pid array not obtained\n");
goto err_alloc_pid_array; // ERR_ALLOC_PID_ARRAY
}
for (j = 0; j < N_CPID_MAX - 1; j++) {
d->fpga_dev[i].pid_chunks[j].used = false;
d->fpga_dev[i].pid_chunks[j].id = j;
d->fpga_dev[i].pid_chunks[j].next = &d->fpga_dev[i].pid_chunks[j + 1];
}
d->fpga_dev[i].pid_alloc = &d->fpga_dev[i].pid_chunks[0];
// initialize device spinlock
spin_lock_init(&d->fpga_dev[i].lock);
spin_lock_init(&d->fpga_dev[i].card_pid_lock);
// writeback setup
if(d->en_wb) {
d->fpga_dev[i].wb_addr_virt = dma_alloc_coherent(&d->pci_dev->dev, WB_SIZE, &d->fpga_dev[i].wb_phys_addr, GFP_ATOMIC);
if(!d->fpga_dev[i].wb_addr_virt) {
pr_err("failed to allocate writeback memory\n");
goto err_wb;
}
if(cyt_arch == CYT_ARCH_PCI && d->en_avx) {
for(j = 0; j < WB_BLOCKS; j++)
d->fpga_dev[i].fpga_cnfg_avx->wback[j] = d->fpga_dev[i].wb_phys_addr + j*(N_CPID_MAX * sizeof(uint32_t));
} else {
for(j = 0; j < WB_BLOCKS; j++)
d->fpga_dev[i].fpga_cnfg->wback[j] = d->fpga_dev[i].wb_phys_addr + j*(N_CPID_MAX * sizeof(uint32_t));
}
pr_info("allocated memory for descriptor writeback, vaddr %llx, paddr %llx",
(uint64_t)d->fpga_dev[i].wb_addr_virt, d->fpga_dev[i].wb_phys_addr);
}
// create device
devno = MKDEV(fpga_major, i);
device_create(fpga_class, NULL, devno, NULL, DEV_NAME "%d", i);
pr_info("virtual FPGA device %d created\n", i);
// add device
cdev_init(&d->fpga_dev[i].cdev, &fpga_fops);
d->fpga_dev[i].cdev.owner = THIS_MODULE;
d->fpga_dev[i].cdev.ops = &fpga_fops;
// Init hash
hash_init(pid_cpid_map[i]);
hash_init(user_lbuff_map[i]);
hash_init(user_sbuff_map[i]);
ret_val = cdev_add(&d->fpga_dev[i].cdev, devno, 1);
if (ret_val) {
pr_err("could not create a virtual FPGA device %d\n", i);
goto err_char_reg;
}
}
pr_info("all virtual FPGA devices added\n");
goto end;
err_char_reg:
for (j = 0; j < i; j++) {
device_destroy(fpga_class, MKDEV(fpga_major, j));
cdev_del(&d->fpga_dev[j].cdev);
}
err_wb:
if(d->en_wb) {
for (j = 0; j < i; j++) {
set_memory_wb((uint64_t)d->fpga_dev[j].wb_addr_virt, N_WB_PAGES);
dma_free_coherent(&d->pci_dev->dev, WB_SIZE,
d->fpga_dev[j].wb_addr_virt, d->fpga_dev[j].wb_phys_addr);
}
}
vfree(d->fpga_dev[i].pid_array);
err_alloc_pid_array:
for (j = 0; j < i; j++) {
vfree(d->fpga_dev[j].pid_array);
}
vfree(d->fpga_dev[i].pid_chunks);
err_alloc_pid_chunks:
for (j = 0; j < i; j++) {
vfree(d->fpga_dev[j].pid_chunks);
}
ret_val = -ENOMEM;
end:
return ret_val;
}
/**
* @brief Delete vFPGAs
*
*/
void free_fpga_devices(struct bus_drvdata *d) {
int i;
for(i = 0; i < d->n_fpga_reg; i++) {
device_destroy(fpga_class, MKDEV(fpga_major, i));
cdev_del(&d->fpga_dev[i].cdev);
if(d->en_wb) {
set_memory_wb((uint64_t)d->fpga_dev[i].wb_addr_virt, N_WB_PAGES);
dma_free_coherent(&d->pci_dev->dev, WB_SIZE,
d->fpga_dev[i].wb_addr_virt, d->fpga_dev[i].wb_phys_addr);
}
vfree(d->fpga_dev[i].pid_array);
vfree(d->fpga_dev[i].pid_chunks);
}
pr_info("vFPGAs deleted\n");
}
Reference in New Issue View Git Blame Copy Permalink