// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2015-2022 Graphcore Ltd. */ /* * IPU device driver. * * Description: PCI device handling. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) #include #else #include #endif #ifdef CONFIG_PCI_P2PDMA #include #endif #include "pci.h" #include "chrdev.h" #include "ipu_driver.h" #include "ipu_signals.h" #include "ipu_boards.h" #include "sysfs.h" static struct idr idr; static struct class *cl; /* pick memmap_start and memmap_size from kernel command line */ unsigned long memmap_start[MAX_CMEM_NODES] = {0}; unsigned int count_memmap_start; module_param_array(memmap_start, ulong, &count_memmap_start, 0444); unsigned long memmap_size[MAX_CMEM_NODES] = {0}; unsigned int count_memmap_size; module_param_array(memmap_size, ulong, &count_memmap_size, 0444); unsigned int count_cmem_nodes; /* Create a list to correlate primary and secondary interfaces */ struct pci_dev_pair { u64 serial; struct ipu_device *ipu_dev; void __iomem *config_sec; void __iomem *exchange_sec; }; int num_devices_C600; // track number of C600 devices static struct pci_dev_pair device_pairs[MAX_IPUS_PCIE_C600]; /* mutex to num_devices_C600 and device_pairs */ struct mutex C600_dev_pairs_mutex; static struct pci_device_id ids[] = { {PCI_DEVICE(IPU_PCI_VENDOR_ID, IPU_PCI_DEVICE_ID_C2)}, {PCI_DEVICE(IPU_PCI_VENDOR_ID, IPU_PCI_DEVICE_ID_S1)}, #ifdef IAI_IPU2_SUPPORTED {PCI_DEVICE(IPU_PCI_VENDOR_ID, IPU_PCI_DEVICE_ID_C200)}, {PCI_DEVICE(IPU_PCI_VENDOR_ID, IPU_PCI_DEVICE_ID_C600)}, {PCI_DEVICE(IPU_PCI_VENDOR_ID, IPU_PCI_DEVICE_ID_M2000)}, {PCI_DEVICE(IPU_PCI_VENDOR_ID, IPU_PCI_DEVICE_ID_M2000W)}, #endif {0} }; int global_log_enable; /* Disabled by default. */ /* cmem virtual address */ void __iomem *contiguous_mem_virt[MAX_CMEM_NODES] = {NULL}; /* cmem manager (genalloc API) */ struct gen_pool *contiguous_mem_pool[MAX_CMEM_NODES] = {NULL}; static pci_ers_result_t ipu_pci_err_detected(struct pci_dev *dev, pci_channel_state_t error); static const struct pci_error_handlers ipu_err_handler = { .error_detected = ipu_pci_err_detected }; static int ipu_pci_probe(struct pci_dev *dev, const struct pci_device_id *id); static void ipu_pci_remove(struct pci_dev *dev); static void ipu_pci_shutdown(struct pci_dev *dev); static struct pci_driver pci_drv_ipu = { .name = IPU_MODULE_NAME, .id_table = ids, .probe = ipu_pci_probe, .remove = ipu_pci_remove, .shutdown = ipu_pci_shutdown, .err_handler = &ipu_err_handler }; struct ipu_hexopt_constant ipu_hexopt_constants[2] = { { // IPU0/IPU1 .HEXOPT_NB_ENTRIES = 0x8000, .HEXOPT_PAGE_SIZE = 4096, }, { // IPU2 .HEXOPT_NB_ENTRIES = 0x8000 * 2, .HEXOPT_PAGE_SIZE = 4096, // default 4K, change to 2MB for HUGEPAGE }, }; static void record_uncorrectable_error(struct ipu_device *ipu_dev, u32 tile_id) { ipu_err(ipu_dev, "ipu%d: uncorrectable parity error logged on tile %u.", ipu_dev->filename_id, tile_id); ipu_err(ipu_dev, "ipu%d: threshold: number of errors: %u, interval: %us", ipu_dev->filename_id, ipu_dev->ipu_error_threshold_num, ipu_dev->ipu_error_threshold_seconds); ipu_dev->ipu_error_state = UNCORRECTABLE_ERROR; icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_SET_DRIVER_ERROR_STATE, ipu_dev->icu_index, UNCORRECTABLE_ERROR, 0, 0}, NULL, 0); } static void record_parity_error(struct ipu_device *ipu_dev, u32 tile_id) { s64 seconds; u32 threshold; u32 resp[2]; u32 num_errors; u32 encoded_tile_id; int ret_val; ipu_err(ipu_dev, "ipu%d: parity error detected on tile %u", ipu_dev->filename_id, tile_id); seconds = ktime_divns(ktime_get_real(), NSEC_PER_SEC); threshold = seconds - ipu_dev->ipu_error_threshold_seconds; encoded_tile_id = tile_id << ICU_CMD_STORE_IPU_PARITY_ERROR_IPU_SHIFT; ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_STORE_IPU_PARITY_ERROR, seconds, encoded_tile_id | ipu_dev->icu_index, threshold}, resp, sizeof(resp)); if (ret_val == 0) { num_errors = resp[ICU_RESP_STORE_IPU_PARITY_ERROR_NUM_ERRORS_INDEX]; ipu_err(ipu_dev, "ipu%d: %u parity errors detected on tile %u", ipu_dev->filename_id, num_errors, tile_id); if (num_errors >= ipu_dev->ipu_error_threshold_num) record_uncorrectable_error(ipu_dev, tile_id); } /* Stop recording parity errors until after the next IPU reset otherwise * we will keep seeing the same parity error. */ ipu_dev->check_parity_errors = 0; } static void check_for_parity_error(struct ipu_device *ipu_dev) { u32 tile_id; u32 err; bool check_threshold = false; u32 num_tiles = ARCH_MAP(IPU_TILE_INSTANCES_WITH_REPAIR); if (!ipu_dev->icu_supports_parity_info) return; /* Errors are only cleared by a reset so we avoid checking unless there * has been a reset. */ if (!ipu_dev->check_parity_errors) return; /* Check top level status to see if any tile has failed */ err = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_CIUERRVR_OFFSET)); err = err >> ARCH_MAP(PCI_COMPLEX_CIUERRVR_IPUSOFTERR_SHIFT); err &= ARCH_MAP(PCI_COMPLEX_CIUERRVR_IPUSOFTERR_MASK); if (!(err || ipu_dev->ipu_fake_parity_error != -1)) return; /* Work out which tile actually failed. */ for (tile_id = 0; tile_id < num_tiles; tile_id++) { u32 address = (ARCH_MAP(SOC_IPU_MEM_BASE) + (tile_id * ARCH_MAP(SOC_IPU_MEM_TILE_STRIDE)) + (ARCH_MAP(TDI_CTXT_STS_OFFSET) * ARCH_MAP(TDBG_TDI_REGISTER_WIDTH))); u32 ctxt_sts = readl(ipu_dev->config + address); bool mem_err = (ctxt_sts >> ARCH_MAP(TDI_CTXT_STS_MERR_SHIFT)); mem_err &= ARCH_MAP(TDI_CTXT_STS_MERR_MASK); /* Check if we've been asked to fake a parity error. */ if (tile_id == ipu_dev->ipu_fake_parity_error) { ipu_dev->ipu_fake_parity_error = -1; mem_err = 1; } if (mem_err) { record_parity_error(ipu_dev, tile_id); check_threshold = true; } } } /* * This is called by UE's that are marked non-fatal. * Currently, no handling is done and the only action is that the * framework logs the error. */ static pci_ers_result_t ipu_pci_err_detected(struct pci_dev *dev, pci_channel_state_t error) { int pos; u32 aer_data; pci_ers_result_t status; struct ipu_device *ipu_dev = (struct ipu_device *)pci_get_drvdata(dev); if (error == pci_channel_io_normal) { if (is_ipum_board(ipu_dev->board_type)) { /* Find AER */ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (!pos) { ipu_err(ipu_dev, "ipu%d: failed to find AER capability", ipu_dev->filename_id); return PCI_ERS_RESULT_DISCONNECT; } pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &aer_data); if (ipu_dev->user_pid) { ipu_dev->aer_data = aer_data; send_signal(ipu_dev, PCIE_AER_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); } } status = PCI_ERS_RESULT_RECOVERED; } else { ipu_err(ipu_dev, "ipu%d: channel down", ipu_dev->filename_id); status = PCI_ERS_RESULT_DISCONNECT; } check_for_parity_error(ipu_dev); return status; } static ssize_t precise_ver_show(struct device_driver *dri, char *buf) { int len; const char *sha_version_string = STRING(SHA_VERSION); len = sprintf(buf, "%s\n", sha_version_string); if (len <= 0) pr_err(IPU_MODULE_NAME "Invalid precise version string"); return len; } static ssize_t abi_revision_show(struct device_driver *drv, char *buf) { ssize_t len = sprintf(buf, "%u\n", IPU_MODULE_ABI_REVISION); return len; } static int tile_clk_speed_run_thread(void *data) { struct ipu_device *ipu_dev = (struct ipu_device *)data; u32 start, end, cycles = 0; u64 start_time, end_time, delta_time; allow_signal(SIGKILL); while (!kthread_should_stop()) { u32 clk_throttle_level = 0; check_for_parity_error(ipu_dev); if (ipu_dev->tile_clk_speed_poll_speed == 0) break; start = readl(ipu_dev->config + ARCH_MAP(SS_IPUCLKCOUNTLSBR_OFFSET)); start_time = ktime_get_raw_ns(); msleep_interruptible(100); if (signal_pending(current)) { flush_signals(current); break; } end_time = ktime_get_raw_ns(); end = readl(ipu_dev->config + ARCH_MAP(SS_IPUCLKCOUNTLSBR_OFFSET)); /* Measure actual time elapsed in nanoseconds * between start and end measurement */ delta_time = end_time - start_time; /* Output in MHz */ delta_time /= 1000; /* Cycle counter tick is 4 clock cycles */ if (delta_time) cycles = ((end << 2) - (start << 2)) / delta_time; cycles = cycles + 5; cycles -= cycles % 10; atomic_set(&ipu_dev->tile_clk_speed, cycles); if (cycles < atomic_read(&ipu_dev->clk_throttle_threshold3)) clk_throttle_level = 3; else if (cycles < atomic_read(&ipu_dev->clk_throttle_threshold2)) clk_throttle_level = 2; else if (cycles < atomic_read(&ipu_dev->clk_throttle_threshold1)) clk_throttle_level = 1; if (clk_throttle_level > ipu_dev->prev_clk_throttle_level) { if (has_power_save_mode(ipu_dev) && !ipu_dev->power_save_active && !ipu_dev->user_pid) { /* power save mode enabled if not attached */ ipu_info(ipu_dev, "ipu%d: entering power save mode, " "clock reduced to %dMHz", ipu_dev->filename_id, cycles); ipu_dev->power_save_active = true; } else { dev_info(&ipu_dev->pci_dev->dev, "ipu%d: IPU clock reduced to %dMHz", ipu_dev->filename_id, cycles); } } else if (clk_throttle_level < ipu_dev->prev_clk_throttle_level) { if (ipu_dev->power_save_active && ipu_dev->user_pid) { ipu_info(ipu_dev, "ipu%d: leaving power save mode, " "clock increased to %dMHz", ipu_dev->filename_id, cycles); ipu_dev->power_save_active = false; } else { dev_info(&ipu_dev->pci_dev->dev, "ipu%d: IPU clock increased to %dMHz", ipu_dev->filename_id, cycles); } } ipu_dev->prev_clk_throttle_level = clk_throttle_level; if (ipu_dev->tile_clk_speed_poll_speed > 0) msleep_interruptible ((ipu_dev->tile_clk_speed_poll_speed * 1000) - 100); if (signal_pending(current)) { flush_signals(current); break; } } flush_signals(current); return 0; } void start_tile_clk_speed_thread(struct device *dev) { struct ipu_device *ipu_dev = NULL; if (!dev) return; ipu_dev = dev_get_drvdata(dev); if (ipu_dev && !ipu_dev->tile_clk_speed_kthread) { /* start thread and run */ ipu_dev->tile_clk_speed_kthread = kthread_run (tile_clk_speed_run_thread, (void *)ipu_dev, "Tile clk speed thread"); } } void stop_tile_clk_speed_thread(struct device *dev) { struct ipu_device *ipu_dev = NULL; if (!dev) return; ipu_dev = dev_get_drvdata(dev); if (ipu_dev && ipu_dev->tile_clk_speed_kthread) { send_sig_info (SIGKILL, SEND_SIG_PRIV, ipu_dev->tile_clk_speed_kthread); kthread_stop(ipu_dev->tile_clk_speed_kthread); ipu_dev->tile_clk_speed_kthread = NULL; } } u32 read_hsp_marks(struct ipu_device *ipu_dev) { u32 hsp = 0; u32 total = 0; dma_sync_single_for_cpu(&ipu_dev->pci_dev->dev, ipu_dev->hsp_pa[0], sizeof(u32), DMA_FROM_DEVICE); hsp = ipu_dev->hsp[0]; /* Note that dma_sync_single_for_device is not required when CPU * only reads from DMA_FROM_DEVICE mapped area. */ if ((hsp >> ARCH_MAP(PCI_COMPLEX_HSPGS1CR_NOTIFY_SHIFT)) & ARCH_MAP(PCI_COMPLEX_HSPGS1CR_NOTIFY_MASK)) { total = (hsp >> ARCH_MAP(PCI_COMPLEX_HSPGS1CR_MARK_SHIFT) & ARCH_MAP(PCI_COMPLEX_HSPGS1CR_MARK_MASK)); } dma_sync_single_for_cpu(&ipu_dev->pci_dev->dev, ipu_dev->hsp_pa[1], sizeof(u32), DMA_FROM_DEVICE); hsp = ipu_dev->hsp[1]; if ((hsp >> ARCH_MAP(PCI_COMPLEX_HSPGS2CR_NOTIFY_SHIFT)) & ARCH_MAP(PCI_COMPLEX_HSPGS2CR_NOTIFY_MASK)) { total += (hsp >> ARCH_MAP(PCI_COMPLEX_HSPGS2CR_MARK_SHIFT) & ARCH_MAP(PCI_COMPLEX_HSPGS2CR_MARK_MASK)); } return total; } static ssize_t global_log_enable_show(struct device_driver *dri, char *buf) { int len; len = sprintf(buf, "%u\n", global_log_enable); if (len <= 0) pr_err(IPU_MODULE_NAME "Invalid sprintf len: %d", len); return len; } static ssize_t global_log_enable_store(struct device_driver *dri, const char *buf, size_t count) { if (count == 0) return 0; global_log_enable = isdigit(buf[0]) ? (buf[0] - '0') : 0; return count; } static DRIVER_ATTR_RW(global_log_enable); static DRIVER_ATTR_RO(precise_ver); static DRIVER_ATTR_RO(abi_revision); static void log_signal(struct ipu_device *ipu_dev, const char *msg) { if (ipu_dev->is_initialised) { ipu_info_ratelimited(ipu_dev, "ipu%d: %s", ipu_dev->filename_id, msg); } else { ipu_info_ratelimited(ipu_dev, "%s", msg); } } void send_signal(struct ipu_device *dev, int sig_num, struct pid *user_pid, int user_device_id) { #ifdef RHEL_RELEASE_CODE #if RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8, 2) struct kernel_siginfo info; #else struct siginfo info; #endif #else #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) struct kernel_siginfo info; #else struct siginfo info; #endif #endif struct task_struct *task; memset(&info, 0, sizeof(info)); info.si_signo = sig_num; info.si_code = SI_QUEUE; info.si_int = user_device_id; task = get_pid_task(user_pid, PIDTYPE_PID); if (task) { if (send_sig_info(sig_num, &info, task) < 0) dev_err(&dev->pci_dev->dev, "ipu%d: failed to signal process", dev->filename_id); } else { dev_err(&dev->pci_dev->dev, "ipu%d: failed to find attached process", dev->filename_id); } } static void decode_pcie_slot_number(const struct dmi_header *dmi, void *dev_p) { struct ipu_device *dev = dev_p; if (dmi->type == 9) { u8 *bus = (u8 *)dmi + 0x0f; char *slot = (char *)dmi + dmi->length; int ignore = memcmp(slot, "PCIe SSD Slot 0 in Bay 0", sizeof("PCIe SSD Slot 0 in Bay 0")) == 0; if ((*bus == dev->pci_dev->bus->number) && !ignore) memcpy(&dev->slot, slot, strlen(slot) + 1); } } static irqreturn_t ipu_ipuex_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); log_signal(ipu_dev, "IPU Exception interrupt"); if (ipu_dev->user_pid && !(ipu_dev->ints_mask & (1 << IPU_IPUEX_ISR))) send_signal(ipu_dev, PCIE_IPU_EXCEPTION_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); check_for_parity_error(ipu_dev); return IRQ_HANDLED; } static irqreturn_t ipu_icu_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); if (ipu_dev->mcu_ioctl_read_pending) { /* Interrupt triggered by driver ioctl pcie mailbox access * so inform ioctl */ log_signal(ipu_dev, "ICU Message response"); ipu_dev->mcu_ioctl_read_pending = 0; complete(&ipu_dev->mailbox_rsp_available); } else { /* Interrupt triggered by userspace pcie mailbox access * so inform userspace */ log_signal(ipu_dev, "ICU Message (signal to userspace)"); if (ipu_dev->user_pid && !(ipu_dev->ints_mask & (1 << IPU_ICU_ISR))) send_signal(ipu_dev, PCIE_ICU_DR_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); } return IRQ_HANDLED; } static irqreturn_t ipu_hspgs1e0_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); log_signal(ipu_dev, "HSPGS1E0 interrupt"); if (ipu_dev->user_pid && !(ipu_dev->ints_mask & (1 << IPU_HSPGS1E0_ISR))) send_signal(ipu_dev, PCIE_HSPGS1E0_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); return IRQ_HANDLED; } static irqreturn_t ipu_hspgs1e1_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); log_signal(ipu_dev, "HSPGS1E1 interrupt"); if (ipu_dev->user_pid && !(ipu_dev->ints_mask & (1 << IPU_HSPGS1E1_ISR))) send_signal(ipu_dev, PCIE_HSPGS1E1_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); return IRQ_HANDLED; } static irqreturn_t ipu_hspgs2e0_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); log_signal(ipu_dev, "HSPGS2E0 interrupt"); if (ipu_dev->user_pid && !(ipu_dev->ints_mask & (1 << IPU_HSPGS2E0_ISR))) send_signal(ipu_dev, PCIE_HSPGS2E0_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); return IRQ_HANDLED; } static irqreturn_t ipu_hspgs2e1_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); log_signal(ipu_dev, "HSPGS2E1 interrupt"); if (ipu_dev->user_pid && !(ipu_dev->ints_mask & (1 << IPU_HSPGS2E1_ISR))) send_signal(ipu_dev, PCIE_HSPGS2E1_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); return IRQ_HANDLED; } static irqreturn_t ipu_icu_msg_isr(int irq, void *dev_id) { /* * This is the ICU async message interrupt. * Unfortunately its timing can not be relied on as is sometimes * arrives before and sometimes after the mailbox interrupt, so we * ignore it. */ return IRQ_HANDLED; } static irqreturn_t ipu_reserved2_isr(int irq, void *dev_id) { struct pci_dev *pci_dev = (struct pci_dev *)dev_id; struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(pci_dev); log_signal(ipu_dev, "IPU reserved 2 interrupt"); return IRQ_HANDLED; } static irqreturn_t(*isr_routines[IPU_NUM_INTERRUPTS]) (int irq, void *dev_id) = { [IPU_IPUEX_ISR] = ipu_ipuex_isr, [IPU_ICU_ISR] = ipu_icu_isr, [IPU_HSPGS1E0_ISR] = ipu_hspgs1e0_isr, [IPU_HSPGS1E1_ISR] = ipu_hspgs1e1_isr, [IPU_HSPGS2E0_ISR] = ipu_hspgs2e0_isr, [IPU_HSPGS2E1_ISR] = ipu_hspgs2e1_isr, [IPU_ICU_MSG_ISR] = ipu_icu_msg_isr, [IPU_RESERVED2_ISR] = ipu_reserved2_isr }; static int enable_msix(struct pci_dev *dev) { struct ipu_device *ipu_dev = NULL; int i = 0; int err = 0; ipu_dev = (struct ipu_device *)pci_get_drvdata(dev); ipu_dev->ints_mask = 0; ipu_dev->num_ints = IPU_NUM_INTERRUPTS; ipu_dev->interrupts = kmalloc_array(ipu_dev->num_ints, sizeof(struct msix_entry), GFP_KERNEL); if (!ipu_dev->interrupts) return -ENOMEM; for (i = 0; i < ipu_dev->num_ints; i++) ipu_dev->interrupts[i].entry = i; ipu_dev->num_ints = pci_enable_msix_range(dev, ipu_dev->interrupts, 1, ipu_dev->num_ints); if (ipu_dev->num_ints < 0) { dev_err(&dev->dev, "failed to initialize msix"); err = ipu_dev->num_ints; goto enable_msix_error; } for (i = 0; i < ipu_dev->num_ints; i++) { err = request_irq(ipu_dev->interrupts[i].vector, isr_routines[i], IRQF_TRIGGER_NONE, IPU_MODULE_NAME, dev); if (err) { dev_err(&dev->dev, "failed to enable interrupt %d", i); ipu_dev->num_ints = i; goto request_irq_error; } } ipu_info_ratelimited(ipu_dev, "enabled %d IPU interrupts", ipu_dev->num_ints); return 0; request_irq_error: for (i = 0; i < ipu_dev->num_ints; i++) free_irq(ipu_dev->interrupts[i].vector, dev); pci_disable_msix(dev); enable_msix_error: kfree(ipu_dev->interrupts); return err; } static void disable_msix(struct pci_dev *dev) { struct ipu_device *ipu_dev = NULL; int i = 0; ipu_dev = (struct ipu_device *)pci_get_drvdata(dev); for (i = 0; i < ipu_dev->num_ints; i++) free_irq(ipu_dev->interrupts[i].vector, dev); kfree(ipu_dev->interrupts); pci_disable_msix(dev); } void restore_board_info(struct ipu_device *ipu_dev) { writel(ipu_dev->mcu_datr_copy[0], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR0_OFFSET)); writel(ipu_dev->mcu_datr_copy[1], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR1_OFFSET)); writel(ipu_dev->mcu_datr_copy[2], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR2_OFFSET)); writel((ipu_dev->dnc & ARCH_MAP(NLC_CSR_IPUID_MASK)) << ARCH_MAP(NLC_CSR_IPUID_SHIFT), ipu_dev->config + ARCH_MAP(SOC_NLC_W_0B_BASE) + ARCH_MAP(NLC_CSR_OFFSET)); ipu_info_ratelimited(ipu_dev, "ipu%d: Restored board info", ipu_dev->filename_id); } static void enable_ipu_tile_clk_cnt(struct ipu_device *ipu_dev) { u32 reg_val; reg_val = readl(ipu_dev->config + ARCH_MAP(SS_SIPUPLLCR_OFFSET)); reg_val |= ARCH_MAP(SS_SIPUPLLCR_IPUCLKCNTEN_MASK) << ARCH_MAP(SS_SIPUPLLCR_IPUCLKCNTEN_SHIFT); writel(reg_val, ipu_dev->config + ARCH_MAP(SS_SIPUPLLCR_OFFSET)); } static void fallback_icu_info(struct ipu_device *ipu_dev) { u32 resp[4] = {0}; int ret_val; ipu_dev->icu_index = 0; // always 0 on C600 ipu_dev->serial = 0; ipu_dev->fw_ver = 0; ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_BOARD_ID_READ, ipu_dev->icu_index, 0, 0, 0}, resp, sizeof(resp)); if (ret_val == 0) { unsigned int serial = resp[0] & 0xffff; unsigned int assembly = resp[0] >> 16; unsigned int batch = resp[1] & 0xffff; ipu_info_ratelimited(ipu_dev, "Board serial %d.%d.%d", serial, assembly, batch); ipu_dev->serial = (u64)serial << 48; ipu_dev->serial |= (u64)assembly << 32; ipu_dev->serial |= batch; } ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_FIRMWARE_VERSION_READ, ipu_dev->icu_index, 0, 0, 0}, resp, sizeof(resp)); if (ret_val == 0) { unsigned int major = resp[0] & 0xffff; unsigned int minor = resp[0] >> 16; unsigned int point = resp[1] & 0xffff; ipu_info_ratelimited(ipu_dev, "FW version %d.%d.%d", major, minor, point); ipu_dev->fw_ver = (u64)major << ICU_FW_MAJOR_SHIFT; ipu_dev->fw_ver |= (u64)minor << ICU_FW_MINOR_SHIFT; ipu_dev->fw_ver |= (u64)point << ICU_FW_POINT_SHIFT; } } static int populate_board_info(struct ipu_device *ipu_dev) { u64 pci_cmd; u32 fw_version; u8 major, minor, point; pci_cmd = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCICMDR_OFFSET)); /* If pci_cmd is 0 then this is a cold boot, and we should read from * MCUDATRn, else read from PCIDATRn. */ if (pci_cmd) { ipu_info_ratelimited(ipu_dev, "Driver reload, reading ICU info from PCIDATRn"); ipu_dev->mcu_datr_copy[0] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR0_OFFSET)); ipu_dev->mcu_datr_copy[1] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR1_OFFSET)); ipu_dev->mcu_datr_copy[2] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR2_OFFSET)); // Read from the MCUCMDR register to clear out any leftover // messages in the mailbox. This can occur if a driver is // reloaded whilst asynchronous ICU Reporting is on. readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUCMDR_OFFSET)); } else { ipu_info_ratelimited(ipu_dev, "Cold boot, reading ICU info from MCUDATRn"); ipu_dev->mcu_datr_copy[0] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR0_OFFSET)); ipu_dev->mcu_datr_copy[1] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR1_OFFSET)); ipu_dev->mcu_datr_copy[2] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR2_OFFSET)); } if (ipu_dev->mcu_datr_copy[2] == 0) { dev_warn(&ipu_dev->pci_dev->dev, "Missing ICU info. Unclean VM shutdown?"); if (is_single_ipu_board(ipu_dev)) { dev_warn(&ipu_dev->pci_dev->dev, "Using fallback method to get ICU info."); fallback_icu_info(ipu_dev); } } else { ipu_dev->serial = ipu_dev->mcu_datr_copy[0]; ipu_dev->serial <<= 32; ipu_dev->serial |= ipu_dev->mcu_datr_copy[1]; ipu_dev->fw_ver = ipu_dev->mcu_datr_copy[2]; ipu_dev->icu_index = (ipu_dev->fw_ver >> 24) & 0xf; } if (ipu_dev->serial == 0) { dev_err(&ipu_dev->pci_dev->dev, "Failed to get serial number."); return -ENOTSUPP; } if (ipu_dev->fw_ver == 0) { dev_err(&ipu_dev->pci_dev->dev, "Failed to get firmware version."); return -ENOTSUPP; } major = (ipu_dev->fw_ver >> ICU_FW_MAJOR_SHIFT) & ICU_FW_VERSION_MASK; minor = (ipu_dev->fw_ver >> ICU_FW_MINOR_SHIFT) & ICU_FW_VERSION_MASK; point = (ipu_dev->fw_ver >> ICU_FW_POINT_SHIFT) & ICU_FW_VERSION_MASK; fw_version = (major * 1000000) + (minor * 1000) + point; ipu_dev->icu_supports_telemetry = fw_version >= ICU_FIRMWARE_VER_SUPPORTS_TELEMETRY; ipu_dev->icu_supports_parity_info = fw_version >= ICU_FIRMWARE_VER_SUPPORTS_PARITY_CHECK; ipu_dev->icu_use_v2_protocol = fw_version >= ICU_FIRMWARE_VER_USE_V2_PROTOCOL; ipu_dev->icu_has_firmware_ver_str = fw_version >= ICU_FIRMWARE_VER_HAS_FIRMWARE_VER; ipu_dev->icu_has_bootloader_ver_str = fw_version >= ICU_FIRMWARE_VER_HAS_BOOTLOADER_VER; ipu_info_ratelimited(ipu_dev, "icu features: %s%s%s%s%s", ipu_dev->icu_supports_telemetry ? "telemetry " : "", ipu_dev->icu_supports_parity_info ? "parity_info " : "", ipu_dev->icu_use_v2_protocol ? "use_v2_protocol " : "", ipu_dev->icu_has_firmware_ver_str ? "icu_has_firmware_ver_str " : "", ipu_dev->icu_has_bootloader_ver_str ? "icu_has_bootloader_ver_str " : "" ); return 0; } static int ipu_map_p2p_exchange_bar(struct ipu_device *ipu_dev) { #ifdef CONFIG_PCI_P2PDMA int err = 0; u64 size, offset = 0; int exchange_bar = 2; struct pci_dev *pdev = ipu_dev->pci_dev; switch (ipu_dev->pci_dev->subsystem_device) { case IPU0_PCI_SUBSYSTEM_ID: case IPU1_PCI_SUBSYSTEM_ID: /* hardcoded offset and size since both are not define in IAI */ offset = PCI_EXCHANGE_BAR_START_OFFSET_IPU1; size = PCI_EXCHANGE_BAR_SIZE_IPU1 - PCI_EXCHANGE_BAR_START_OFFSET_IPU1; break; default: #if defined(IAI_IPU2_SUPPORTED) || defined(IAI_IPU21_SUPPORTED) case IPU2_PCI_SUBSYSTEM_ID: case IPU2_1_PCI_SUBSYSTEM_ID: #endif // defined(IAI_IPU2_SUPPORTED) || defined(IAI_IPU21_SUPPORTED) /* The size is set to 0 to use the whole BAR size */ offset = 0; size = 0; break; } err = pci_p2pdma_add_resource(pdev, exchange_bar, size, offset); if (err) { dev_warn(&ipu_dev->pci_dev->dev, "failed to add exchange as p2p resource %d", err); return err; } pci_p2pmem_publish(pdev, true); dev_info(&ipu_dev->pci_dev->dev, "allocate exchange size %llu as p2p resource", size); #endif // CONFIG_PCI_P2PDMA return 0; } static void init_tile_parity_handling(struct ipu_device *ipu_dev) { if (ipu_dev->icu_supports_parity_info) { u32 resp[2]; int ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_GET_DRIVER_ERROR_STATE, ipu_dev->icu_index, 0, 0, 0}, resp, sizeof(resp)); if (ret_val) { /* on ICU comms error assume IPU okay */ ipu_dev->ipu_error_state = 0; } else { ipu_dev->ipu_error_state = resp[ICU_RESP_GET_DRIVER_ERROR_STATE_VALUE_INDEX]; } /* Default to 4 or more parity errors within ~3 months. */ ipu_dev->ipu_error_threshold_seconds = 60 * 60 * 24 * 30 * 3; ipu_dev->ipu_error_threshold_num = 4; ipu_dev->ipu_fake_parity_error = -1; ipu_dev->check_parity_errors = 1; if (ipu_dev->ipu_error_state) dev_info(&ipu_dev->pci_dev->dev, "Error: one or more tiles has exceeded parity error threshold"); dev_info(&ipu_dev->pci_dev->dev, "ipu %d: set threshold for detecting tile parity issues: %u occurrences in %u seconds on the same tile.", ipu_dev->filename_id, ipu_dev->ipu_error_threshold_num, ipu_dev->ipu_error_threshold_seconds); } else { dev_info(&ipu_dev->pci_dev->dev, "ipu%d: ICU firmware upgrade required for tile memory parity detection", ipu_dev->filename_id); } } static void set_m2000_variant(struct ipu_device *ipu_dev) { if (ipu_dev->pci_dev->subsystem_device == IPU2_1_PCI_SUBSYSTEM_ID) { ipu_dev->board_variant = IPU_BOARD_VARIANT_M2000WX; ipu_dev->board_variant_name = IPU_BOARD_VARIANT_NAME_M2000WX; } else { ipu_dev->board_variant = IPU_BOARD_VARIANT_M2000W; ipu_dev->board_variant_name = IPU_BOARD_VARIANT_NAME_M2000W; } } static u64 get_dsn(struct pci_dev *dev) { u32 dword, pos; u64 dsn; // pcie device serial number // The Device Serial Number at offset 4 bytes from // the capability position. First dword is the lower half and // the second is the upper half. pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN); if (pos == 0) return 0; pci_read_config_dword(dev, pos + 4, &dword); dsn = dword; pci_read_config_dword(dev, pos + 8, &dword); dsn |= ((u64)dword << 32); return dsn; } static int setup_dev_bus_bars(struct pci_dev *dev, struct ipu_device *ipu_dev, bool *is_secondary) { int err = 0; err = pci_enable_device(dev); if (err != 0) { pr_err(IPU_MODULE_NAME " failed to enable PCI device"); return err; } pci_set_master(dev); err = -1; err = pci_request_regions(dev, IPU_MODULE_NAME); if (err != 0) { pr_err(IPU_MODULE_NAME " failed to request PCI regions"); goto request_region_error; } pci_set_drvdata(dev, (void *)ipu_dev); ipu_dev->pci_dev = dev; ipu_dev->config_start = pci_resource_start(dev, 0); ipu_dev->config_size = pci_resource_end(dev, 0) - ipu_dev->config_start; ipu_dev->exchange_start = pci_resource_start(dev, 2); ipu_dev->exchange_size = pci_resource_end(dev, 2) - ipu_dev->exchange_start; ipu_dev->config = #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ioremap(ipu_dev->config_start, ipu_dev->config_size); #else ioremap_nocache(ipu_dev->config_start, ipu_dev->config_size); #endif if (ipu_dev->config) dev_info(&dev->dev, "IPU config addr = 0x%llx, size = 0x%llx", ipu_dev->config_start, ipu_dev->config_size); ipu_dev->exchange = ioremap_wc(ipu_dev->exchange_start, ipu_dev->exchange_size); if (ipu_dev->exchange) { dev_info(&dev->dev, " Exchange addr = %p", ipu_dev->exchange); dev_info(&dev->dev, "IPU exchange addr = 0x%llx, size = 0x%llx", ipu_dev->exchange_start, ipu_dev->exchange_size); } if (!ipu_dev->config || !ipu_dev->exchange) { dev_err(&dev->dev, "failed to map IPU BARs"); err = -EIO; goto map_bar_error; } // In C600 secondary bar, we access hexopt and hexoatt using pointers // without mmap'ing the bars if (dev->device == IPU_PCI_DEVICE_ID_C600) { u32 i; u64 dsn = get_dsn(dev); // pcie device serial number if (dsn == 0) { /* Ignore secondary ops if serial number not set */ ipu_dev->config_sec = NULL; ipu_dev->exchange_sec = NULL; if (dev->subsystem_device == IPU2_1_PCI_SUBSYSTEM_ID) dev_warn(&dev->dev, "C600 primary dsn missing"); if (dev->subsystem_device == IPU2_1_2_PCI_SUBSYSTEM_ID) { *is_secondary = true; dev_warn(&dev->dev, "C600 secondary dsn missing"); } return err; // err = 0 } // primary bar if (dev->subsystem_device == IPU2_1_PCI_SUBSYSTEM_ID) { dev_info(&dev->dev, "serial number 0x%llx C600 primary bar", dsn); mutex_lock(&C600_dev_pairs_mutex); for (i = 0; i < num_devices_C600; i++) { if (dsn == device_pairs[i].serial) break; // secondary already appeared } if (i == MAX_IPUS_PCIE_C600) { dev_err(&dev->dev, "Ignoring device." "Maximum C600s supported = %d", MAX_IPUS_PCIE_C600); mutex_unlock(&C600_dev_pairs_mutex); err = -1; goto unmap_bars; } device_pairs[i].ipu_dev = ipu_dev; if (i == num_devices_C600) { device_pairs[i].serial = dsn; num_devices_C600++; } else { /* Note secondary into primary ipu_dev */ ipu_dev->config_sec = device_pairs[i].config_sec; ipu_dev->exchange_sec = device_pairs[i].exchange_sec; } mutex_unlock(&C600_dev_pairs_mutex); } else if (dev->subsystem_device == IPU2_1_2_PCI_SUBSYSTEM_ID) { dev_info(&dev->dev, "serial number 0x%llx C600 secondary bar", dsn); mutex_lock(&C600_dev_pairs_mutex); for (i = 0; i < num_devices_C600; i++) { if (dsn == device_pairs[i].serial) break; // primary already appeared } if (i == MAX_IPUS_PCIE_C600) { dev_err(&dev->dev, "Ignoring device." "Maximum C600s supported = %d", MAX_IPUS_PCIE_C600); mutex_unlock(&C600_dev_pairs_mutex); err = -1; goto unmap_bars; } device_pairs[i].config_sec = ipu_dev->config; device_pairs[i].exchange_sec = ipu_dev->exchange; if (i == num_devices_C600) { /* primary hasn't appeared yet */ device_pairs[i].serial = dsn; num_devices_C600++; } else { struct ipu_device *primary = device_pairs[i].ipu_dev; /* Note secondary in primary ipu_dev */ primary->config_sec = ipu_dev->config; primary->exchange_sec = ipu_dev->exchange; } *is_secondary = true; // nothing else on secondary mutex_unlock(&C600_dev_pairs_mutex); } else { dev_err(&dev->dev, "C600 unknown subsystem device"); goto map_bar_error; } } return err; /* err = 0 */ unmap_bars: iounmap(ipu_dev->config); iounmap(ipu_dev->exchange); map_bar_error: pci_release_regions(dev); pci_set_drvdata(dev, NULL); pci_disable_device(dev); request_region_error: kfree(ipu_dev); return err; } static int ipu_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { int err = 0, cmem_node = -1; struct ipu_device *ipu_dev = NULL; bool is_secondary = false; pr_info(IPU_MODULE_NAME " driver: IPU%d device (0x%x) found %s (id %04x:%04x)", ipu_arch_num_from_subsystem(dev->subsystem_device), dev->subsystem_device, pci_name(dev), dev->vendor, dev->device); ipu_dev = kzalloc(sizeof(*ipu_dev), GFP_KERNEL); if (!ipu_dev) { err = -ENOMEM; return err; } is_secondary = false; err = setup_dev_bus_bars(dev, ipu_dev, &is_secondary); if (err != 0) return err; if (is_secondary) return 0; sema_init(&ipu_dev->config_available, 1); sema_init(&ipu_dev->exchange_available, 1); sema_init(&ipu_dev->memory_available, 1); init_completion(&ipu_dev->mailbox_rsp_available); mutex_init(&ipu_dev->mailbox_available); pci_set_drvdata(dev, (void *)ipu_dev); ipu_dev->is_initialised = 0; ipu_dev->supports_dma = 1; memset(&ipu_dev->mcu_sync_msg_response, 0, sizeof(ipu_dev->mcu_sync_msg_response)); hash_init(ipu_dev->coherent_mem_allocs); ipu_info(ipu_dev, "IPU%d on NUMA - %d", ipu_arch_num_from_subsystem(dev->subsystem_device), dev->dev.numa_node); cmem_node = get_cmem_node(ipu_dev); if (cmem_node < 0) { ipu_dev->memory_start = 0; ipu_dev->memory_size = 0; ipu_dev->memory = NULL; } else { ipu_dev->memory_start = memmap_start[cmem_node]; ipu_dev->memory_size = memmap_size[cmem_node]; ipu_dev->memory = NULL; } ipu_dev->idr = &idr; ipu_dev->cl = cl; switch (dev->subsystem_device) { case IPU0_PCI_SUBSYSTEM_ID: case IPU1_PCI_SUBSYSTEM_ID: ipu_dev->arch_map = &ipu_arch_maps[IPU1_ARCH_MAP_INDEX]; ipu_dev->hexopt_constants = &ipu_hexopt_constants[0]; break; #ifdef IAI_IPU2_SUPPORTED case IPU2_PCI_SUBSYSTEM_ID: case IPU2_1_PCI_SUBSYSTEM_ID: ipu_dev->arch_map = &ipu_arch_maps[IPU2_ARCH_MAP_INDEX]; ipu_dev->hexopt_constants = &ipu_hexopt_constants[1]; break; #endif default: err = -ENOTSUPP; dev_err(&dev->dev, "Unsupported IPU architecture"); goto mem_error; } /* ipu_dev->config = pci_iomap(dev, 0, 0); */ if (ipu_map_p2p_exchange_bar(ipu_dev)) dev_err(&dev->dev, "failed to map P2P exchange BAR"); if (cmem_node >= 0 && ipu_dev->memory_size >= GENALLOC_SIZE_MINIMUM) { ipu_dev->memory = contiguous_mem_virt[cmem_node]; if (!ipu_dev->memory) dev_err(&dev->dev, "Failed to map IPU reserved memory"); } else { ipu_dev->memory_start = 0; ipu_dev->memory_size = 0; ipu_dev->memory = NULL; } ipu_dev->mcu_ioctl_read_pending = 0; ipu_info(ipu_dev, "ioctl mailbox messaging available"); /* Ask for 64bit DMA addresses. * If the platform does not support these, we use physical page * addresses in HexOPT. */ if (dma_set_mask(&dev->dev, DMA_BIT_MASK(64))) { dev_warn(&dev->dev, "64-bit DMA addresses unsupported - disabling DMA"); ipu_dev->supports_dma = 0; } err = enable_msix(dev); if (err != 0) { dev_err(&dev->dev, "failed to enable interrupts"); goto enable_msix_error; } ipu_dev->board_type = IPU_BOARD_TYPE_UNKNOWN; ipu_dev->board_name = "unknown"; ipu_dev->board_variant = IPU_BOARD_VARIANT_STANDARD; #ifdef CONFIG_OF { const char *machine_prop = NULL; struct device_node *chosen = of_find_node_by_path("/chosen"); if (of_property_read_string(chosen, "machine", &machine_prop) == 0) { if (strncmp(machine_prop, "ipum-p1", 7) == 0) { ipu_dev->board_type = IPU_BOARD_TYPE_M1; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_M1; } else if (strncmp(machine_prop, "ipum-p2", 7) == 0) { ipu_dev->board_type = IPU_BOARD_TYPE_M2000; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_M2000; } } of_node_put(chosen); } #endif if (ipu_dev->board_type == IPU_BOARD_TYPE_UNKNOWN) { /* nothing in device tree, check PCI Device Id */ switch (ipu_dev->pci_dev->device) { case IPU_PCI_DEVICE_ID_C2: ipu_dev->board_type = IPU_BOARD_TYPE_C2; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_C2; break; case IPU_PCI_DEVICE_ID_S1: ipu_dev->board_type = IPU_BOARD_TYPE_S1; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_S1; break; case IPU_PCI_DEVICE_ID_C200: ipu_dev->board_type = IPU_BOARD_TYPE_C200; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_C200; break; case IPU_PCI_DEVICE_ID_C600: ipu_dev->board_type = IPU_BOARD_TYPE_C600; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_C600; break; case IPU_PCI_DEVICE_ID_M2000: ipu_dev->board_type = IPU_BOARD_TYPE_M2000; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_M2000; break; case IPU_PCI_DEVICE_ID_M2000W: ipu_dev->board_type = IPU_BOARD_TYPE_M2000; ipu_dev->board_name = IPU_BOARD_TYPE_NAME_M2000; set_m2000_variant(ipu_dev); break; default: break; } } else if (ipu_dev->board_type == IPU_BOARD_TYPE_M2000 && ipu_dev->pci_dev->device == IPU_PCI_DEVICE_ID_M2000W) { set_m2000_variant(ipu_dev); } if (ipu_dev->board_variant == IPU_BOARD_VARIANT_STANDARD) ipu_dev->board_variant_name = ipu_dev->board_name; dev_info(&ipu_dev->pci_dev->dev, "IPU board type: %s, variant: %s", ipu_dev->board_name, ipu_dev->board_variant_name); // Enable AER on M2000 only. if (is_ipum_board(ipu_dev->board_type)) { /* Find AER */ int pos; u32 config; pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (!pos) { err = -ENOTTY; dev_err(&dev->dev, "Failed to find AER capability"); goto enable_msix_error; } /* Clear Uncorrectable Error Status */ pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &config); pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, config); /* Convert everything to non-fatal */ /* Surprise Down not supported */ pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, 0); /* Unmask everything */ pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, 0); ipu_info(ipu_dev, "AER UE unmasked"); } else { /* Disable AER otherwise */ unsigned int uncorrectable_mask = 0; unsigned int correctable_mask = 0; pci_read_config_dword(dev, 0x108, (u32 *)&uncorrectable_mask); pci_read_config_dword(dev, 0x114, (u32 *)&correctable_mask); /* Set all mask bits in both registers */ uncorrectable_mask |= 0x7fff030; correctable_mask |= 0xf1c1; ipu_info(ipu_dev, "AER Disabled"); ipu_info(ipu_dev, "AER Correctable Mask [0x%x]", correctable_mask); ipu_info(ipu_dev, "AER Uncorrectable Mask [0x%x]", uncorrectable_mask); pci_write_config_dword(dev, 0x108, uncorrectable_mask); pci_write_config_dword(dev, 0x114, correctable_mask); } ipu_dev->slot[0] = '\0'; ipu_dev->serial = 0; ipu_dev->fw_ver = 0; atomic_set(&ipu_dev->tile_clk_speed, 0); ipu_dev->tile_clk_speed_poll_speed = 1; ipu_dev->tile_clk_speed_kthread = NULL; ipu_dev->filename_id = 0; err = populate_board_info(ipu_dev); if (err != 0) { dev_err(&dev->dev, "Failed to populate board info"); goto enable_msix_error; } dmi_walk(decode_pcie_slot_number, ipu_dev); if (!ipu_dev->slot[0]) { ipu_dev->dnc = (readl(ipu_dev->config + ARCH_MAP(SOC_NLC_W_0B_BASE) + ARCH_MAP(NLC_CSR_OFFSET)) >> ARCH_MAP(NLC_CSR_IPUID_SHIFT)) & ARCH_MAP(NLC_CSR_IPUID_MASK); snprintf(ipu_dev->slot, SLOT_STRING_SIZE, "DNC: %d", ipu_dev->dnc); } ipu_dev->spb_acc = NULL; if (ipu_dev->slot[0]) ipu_info(ipu_dev, "Found PCIe slot info: %s", ipu_dev->slot); err = ipu_chrdev_init(ipu_dev); if (err != 0) { dev_err(&dev->dev, "failed to initialize chrdev"); goto chrdev_init_error; } init_tile_parity_handling(ipu_dev); icu_read_fw_version(ipu_dev); icu_read_bootloader_version(ipu_dev); enable_ipu_tile_clk_cnt(ipu_dev); if (ipu_sysfs_add_attributes(ipu_dev) != 0) goto sysfs_error; start_tile_clk_speed_thread(&dev->dev); ipu_dev->is_initialised = 1; return 0; sysfs_error: ipu_chrdev_exit(ipu_dev); chrdev_init_error: disable_msix(dev); enable_msix_error: pci_set_drvdata(dev, NULL); mem_error: pci_release_regions(dev); //pci_disable_pcie_error_reporting(dev); pci_disable_device(dev); kfree(ipu_dev); return err; } static void ipu_pci_shutdown(struct pci_dev *dev) { struct ipu_device *ipu_dev = NULL; ipu_dev = (struct ipu_device *)pci_get_drvdata(dev); if (!ipu_dev->is_initialised) return; restore_board_info(ipu_dev); } static void ipu_pci_remove(struct pci_dev *dev) { struct ipu_device *ipu_dev = NULL; struct task_struct *task = NULL; bool is_C600_secondary = (dev->device == IPU_PCI_DEVICE_ID_C600) && (dev->subsystem_device == IPU2_1_2_PCI_SUBSYSTEM_ID); bool is_C600_primary = (dev->device == IPU_PCI_DEVICE_ID_C600) && (dev->subsystem_device == IPU2_1_PCI_SUBSYSTEM_ID); ipu_dev = (struct ipu_device *)pci_get_drvdata(dev); if (is_C600_primary) { /* primary and secondary will be removed so remove * item from device_pairs */ u32 i; u64 dsn = get_dsn(dev); // pcie device serial number if (dsn != 0) { mutex_lock(&C600_dev_pairs_mutex); // clear from list for (i = 0; i < num_devices_C600; i++) { if (dsn == device_pairs[i].serial) { num_devices_C600--; break; } } while (i < num_devices_C600) { memcpy(&device_pairs[i], &device_pairs[i + 1], sizeof(struct pci_dev_pair)); } mutex_unlock(&C600_dev_pairs_mutex); } } if (!is_C600_secondary) { /* Kill if thread active */ stop_tile_clk_speed_thread(&dev->dev); if (ipu_dev->user_pid) { task = get_pid_task(ipu_dev->user_pid, PIDTYPE_PID); dev_warn(&ipu_dev->pci_dev->dev, "ipu%d: device removed while PID %d attached.\n" "Sending PCIE_IPU_REMOVED_SIG to %s.", ipu_dev->filename_id, task->pid, task->comm); send_signal(ipu_dev, PCIE_IPU_REMOVED_SIG, ipu_dev->user_pid, ipu_dev->user_device_id); } ipu_sysfs_remove_attributes(ipu_dev); ipu_chrdev_exit(ipu_dev); disable_msix(dev); } iounmap(ipu_dev->config); ipu_dev->config = NULL; iounmap(ipu_dev->exchange); ipu_dev->exchange = NULL; pci_release_regions(dev); pci_disable_device(dev); kfree(ipu_dev); if (ipu_dev) pci_set_drvdata(dev, NULL); } static int contiguous_mem_pool_init(int count_cmem) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) int i, ret = -1; void __iomem *cmem_virt; for (i = 0; i < count_cmem; i++) { /* Create genalloc memory manager */ contiguous_mem_pool[i] = gen_pool_create(GENALLOC_ORDER_MINIMUM, -1); /* No NUMA node, -1 */ if (contiguous_mem_pool[i] && memmap_start[i] && memmap_size[i]) { cmem_virt = contiguous_mem_virt[i]; /* Add reserved contiguous mem to memory manager */ ret = gen_pool_add_virt(contiguous_mem_pool[i], (unsigned long)cmem_virt, memmap_start[i], memmap_size[i], -1); if (ret < 0) { pr_err(IPU_MODULE_NAME " genpool cmem %d failed\n", i); gen_pool_destroy(contiguous_mem_pool[i]); contiguous_mem_pool[i] = NULL; break; } pr_info(IPU_MODULE_NAME " genpool cmem %d success\n", i); } } return ret; #else return -EINVAL; #endif } static void contiguous_mem_init(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) int i; // No cmem support if (count_memmap_start == 0 || count_memmap_size == 0) { count_cmem_nodes = 0; return; } // check equal number of comma separated values in // ipu_driver.memmap_start = 0xA, 0xB.. and // ipu_driver.memmap_size = 0xA, 0xB.. // else only use first parameter if (count_memmap_start == count_memmap_size && count_memmap_start <= MAX_CMEM_NODES) { count_cmem_nodes = count_memmap_start; } else { count_cmem_nodes = 1; pr_info(IPU_MODULE_NAME " Fallback to one memmap node - %s\n", (count_memmap_start != count_memmap_size) ? "unmatched memmap starts and sizes in boot cmdline" : "exceeded maximum memmap nodes in boot cmdline"); } for (i = 0; i < count_cmem_nodes; i++) { contiguous_mem_virt[i] = NULL; contiguous_mem_pool[i] = NULL; if (memmap_size[i] >= GENALLOC_SIZE_MINIMUM) { contiguous_mem_virt[i] = memremap(memmap_start[i], memmap_size[i], MEMREMAP_WB); if (!contiguous_mem_virt[i]) { pr_err(IPU_MODULE_NAME " Failed to map IPU reserved memory"); goto memerr; } pr_info(IPU_MODULE_NAME " Memmapped 0x%lx bytes starting at 0x%lx\n", memmap_size[i], memmap_start[i]); } } if (contiguous_mem_pool_init(count_cmem_nodes) < 0) goto memerr; return; memerr: for (i = 0; i < count_cmem_nodes; i++) contiguous_mem_virt[i] = NULL; #endif } static void contiguous_mem_deinit(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) int i; for (i = 0; i < count_cmem_nodes; i++) { if (contiguous_mem_pool[i]) gen_pool_destroy(contiguous_mem_pool[i]); if (contiguous_mem_virt[i]) memunmap(contiguous_mem_virt[i]); } #endif } int get_cmem_node(struct ipu_device *ipu_dev) { int numa_node = ipu_dev->pci_dev->dev.numa_node; int cmem_node = -1; if (count_cmem_nodes > 0) cmem_node = 0; // default one node if (count_cmem_nodes > 1 && numa_node < count_cmem_nodes) cmem_node = numa_node; // pick memory in NUMA node return cmem_node; } /* allocate a chunk from reserved contiguous mem */ int contiguous_mem_alloc(u64 size, u64 *virt, u64 *phys, int node) { /** * 3.13 kernel added a handy API gen_pool_dma_alloc() which internally calls - * gen_pool_alloc() & gen_pool_virt_to_phys(). * So to support kernel version older than 3.13, use these 2 APIs. */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) struct gen_pool *cmem_pool; if (!virt || !phys) return -EINVAL; cmem_pool = contiguous_mem_pool[node]; if (cmem_pool) { u64 vaddr = gen_pool_alloc(cmem_pool, size); if (vaddr) { phys_addr_t paddr = gen_pool_virt_to_phys(cmem_pool, vaddr); if (paddr > 0) { *phys = paddr; *virt = vaddr; } else { return -ENOMEM; } } else { return -ENOMEM; } } else { return -ENOMEM; } return 0; #else return -EINVAL; #endif } /* Give back a chunk to reserved contiguous mem */ int contiguous_mem_free(u64 size, u64 virt, int node) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) if (contiguous_mem_pool[node]) { /* initialise the memory to MEM_INIT_VAL * before returning to the pool */ memset((void *)virt, MEM_INIT_VAL, size); gen_pool_free(contiguous_mem_pool[node], (unsigned long)virt, size); return 0; } else { return -EINVAL; } #else return -EINVAL; #endif } int ipu_pci_init(void) { int ret = 0; const char *sha_ver = STRING(SHA_VERSION); idr_init(&idr); pr_info(IPU_MODULE_NAME " driver: version %d.%d.%d (ABI %u)\n", IPU_DRIVER_VERSION_MAJOR, IPU_DRIVER_VERSION_MINOR, IPU_DRIVER_VERSION_POINT, IPU_MODULE_ABI_REVISION); cl = class_create(IPU_MODULE_NAME); if (!cl) pr_err(IPU_MODULE_NAME " failed to create device class"); contiguous_mem_init(); mutex_init(&C600_dev_pairs_mutex); ret = pci_register_driver(&pci_drv_ipu); if (ret) { pr_err(IPU_MODULE_NAME " failed to register ipu pcie driver"); return ret; } ret = driver_create_file(&pci_drv_ipu.driver, &driver_attr_global_log_enable); if (ret) pr_err(IPU_MODULE_NAME " failed to create sysfs global log control entry"); ret = driver_create_file(&pci_drv_ipu.driver, &driver_attr_abi_revision); if (ret) pr_err(IPU_MODULE_NAME " failed to create sysfs abi_revision entry"); ret = driver_create_file(&pci_drv_ipu.driver, &driver_attr_precise_ver); if (ret) pr_err(IPU_MODULE_NAME " failed to create sysfs precise_ver entry"); else pr_info(IPU_MODULE_NAME " precise version %s\n", sha_ver); return 0; } void ipu_pci_exit(void) { driver_remove_file(&pci_drv_ipu.driver, &driver_attr_global_log_enable); contiguous_mem_deinit(); pci_unregister_driver(&pci_drv_ipu); class_destroy(cl); } MODULE_DEVICE_TABLE(pci, ids);