// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2015-2023 Graphcore Ltd. */ /* * IPU device driver. * * Description: Character device handling. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef CONFIG_PCI_P2PDMA #include #endif #include "chrdev.h" #include "pci.h" #define USER_COPY_LIMIT (0x40000000 - 1) // 1GB (see T13823) #define USER_COPY_CHUNK_SIZE (0x40000000) // 1GB chunks #define MBOX_TIMEOUT_MS (10000) #define MBOX_TIMEOUT_JIFFIES (msecs_to_jiffies(MBOX_TIMEOUT_MS)) #define IPU_GWLINKS_ENABLE_OFFSET (0x720) #define IPU_GWLINKS_ENABLE_VALUE (2) #define IPU_GWLINKS_DISABLE_VALUE (0) #define SPB_RDWR_SERVICE_TABLE_ADDR_OFFSET (0x44) #define SPB_WR_SERVICE_TABLE_KICK_OFFSET (0x48) #define SPB_WR_SERVICE_TABLE_ENTRY_LOW_OFFSET (0x4C) #define SPB_WR_SERVICE_TABLE_ENTRY_HIGH_OFFSET (0x50) #define SPB_CONFIG_REG_OFFSET (0x94) #define SPB_SERVICE_TABLE_FOR_WRITES_START_INDEX (0) #define SPB_SERVICE_TABLE_FOR_READS_START_INDEX (2016) #define SPB_SERVICE_TABLE_FOR_READS_ENTRIES (32) #define SPB_CONFIG_REG_ENABLE_MULTI_READ_ST_ENTRY BIT(10) /* * https://graphcore.cdox.net/vdocs/GC-000812-SP-A-Gateway%201%20FPGA.pdf * 7.2.2 Internal Registers */ #define SPB_ERROR_PKT_COUNT (0x28) static u32 spb_acc_base_addr[4] = { 0xb0002000, 0xb0003000, 0xb0004000, 0xb0005000 }; static loff_t ipu_seek(struct file *, loff_t, int); static ssize_t ipu_read(struct file *filp, __user char *buff, size_t count, loff_t *ppos); static ssize_t ipu_write(struct file *filp, __user const char *buff, size_t count, loff_t *ppos); static int ipu_mmap(struct file *filp, struct vm_area_struct *vma); static int ipu_open_config(struct inode *inode, struct file *filp); static int ipu_open_exchange(struct inode *inode, struct file *filp); static int ipu_open_memory(struct inode *inode, struct file *filp); static int ipu_release(struct inode *inode, struct file *filp); static long ipu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); static long put_hsp_notify(struct ipu_device *ipu_dev, struct hspgs_notify_addr *addrs); #ifdef CONFIG_PCI_P2PDMA static int ipu_p2p_mmap(struct file *filp, struct vm_area_struct *vma); static int ipu_p2p_open_exchange(struct inode *inode, struct file *filp); static void ipu_p2p_vma_open(struct vm_area_struct *vma); static vm_fault_t ipu_p2p_vma_fault(struct vm_fault *vmf); static void ipu_p2p_vma_close(struct vm_area_struct *vma); #endif struct attach_buffer_data { unsigned int index; unsigned int buffer_size; u64 buffer_addr; }; struct version_info { unsigned int major; unsigned int minor; unsigned int point; }; static const struct file_operations ipu_fops = { .owner = THIS_MODULE, .open = ipu_open_config, .release = ipu_release, .read = ipu_read, .write = ipu_write, .llseek = ipu_seek, .unlocked_ioctl = ipu_ioctl, .mmap = ipu_mmap }; static const struct file_operations ipu_exchange_fops = { .owner = THIS_MODULE, .open = ipu_open_exchange, .release = ipu_release, .read = ipu_read, .write = ipu_write, .llseek = ipu_seek, .unlocked_ioctl = ipu_ioctl, .mmap = ipu_mmap }; static const struct file_operations ipu_memory_fops = { .owner = THIS_MODULE, .open = ipu_open_memory, .release = ipu_release, .read = ipu_read, .write = ipu_write, .llseek = ipu_seek, .unlocked_ioctl = ipu_ioctl, .mmap = ipu_mmap }; #ifdef CONFIG_PCI_P2PDMA static const struct file_operations ipu_p2p_exchange_fops = { .owner = THIS_MODULE, .open = ipu_p2p_open_exchange, .release = ipu_release, .mmap = ipu_p2p_mmap }; #endif static const struct version_info version = { .major = IPU_DRIVER_VERSION_MAJOR, .minor = IPU_DRIVER_VERSION_MINOR, .point = IPU_DRIVER_VERSION_POINT, }; static void ipu_select_bar(struct ipu_device *dev, unsigned int bar_region) { dev_err(&dev->pci_dev->dev, "ipu%d: invalid bar select", dev->filename_id); } static void wait_hexopt_updated(struct ipu_device *ipu_dev, u32 last_idx, u64 last_addr) { #if defined(IAI_IPU2_SUPPORTED) || defined(IAI_IPU21_SUPPORTED) int try_usecs = 25; u32 hir, lor, idx; u64 addr; #endif /* We have no way to make sure the HEXOPT has been updated since writes * are posted writes (meaning the CPU does not wait for the * completion), that we cannot read back the exchange space and that * reading the HEXOPT via the config debug interface while there is PCI * traffic yields an undefined result. Hence the sleep. */ switch (ipu_dev->pci_dev->subsystem_device) { case IPU0_PCI_SUBSYSTEM_ID: case IPU1_PCI_SUBSYSTEM_ID: usleep_range(10, 20); break; #if defined(IAI_IPU2_SUPPORTED) || defined(IAI_IPU21_SUPPORTED) case IPU2_PCI_SUBSYSTEM_ID: case IPU2_1_PCI_SUBSYSTEM_ID: ipu_info_ratelimited(ipu_dev, "waiting for hexopt entry %u-0x%llx", last_idx, last_addr); hir = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HEXOPTUPDDATAHIR_OFFSET)); lor = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HEXOPTUPDDATALOR_OFFSET)); idx = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HEXOPTUPDINDEXR_OFFSET)); addr = (u64)(((u64)hir << 32) + (lor << 12)); ipu_info_ratelimited(ipu_dev, "last addr 0x%llx", addr); while (idx != last_idx || addr != last_addr) { if (try_usecs <= 0) break; try_usecs--; udelay(1); hir = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HEXOPTUPDDATAHIR_OFFSET)); lor = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HEXOPTUPDDATALOR_OFFSET)); idx = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HEXOPTUPDINDEXR_OFFSET)); addr = (u64)(((u64)hir << 32) + (lor << 12)); ipu_info_ratelimited(ipu_dev, "last addr 0x%llx", addr); } if (try_usecs == 0) ipu_info_ratelimited(ipu_dev, "timed out hexopt wait"); break; #endif } } static void ipu_put_user_pages(int nr_pages, struct page **pages) { int i; /* Release the user pages. */ for (i = 0; i < nr_pages; i++) { set_page_dirty_lock(pages[i]); #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) unpin_user_page(pages[i]); #else put_page(pages[i]); #endif } } static int ipu_get_user_pages(unsigned long start, int nr_pages, struct page **pages) { int num_pin_pages = 0, ret; u64 userptr = start; struct page **p = pages; #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) down_read(¤t->mm->mmap_lock); #else down_read(¤t->mm->mmap_sem); #endif do { userptr += (num_pin_pages * PAGE_SIZE); p += num_pin_pages; #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) ret = pin_user_pages(userptr, nr_pages - num_pin_pages, /* Do want to write into it */ 1, p); #elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) ret = get_user_pages(userptr, nr_pages - num_pin_pages, /* Do want to write into it */ 1, p); #elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) ret = get_user_pages(userptr, nr_pages - num_pin_pages, /* Do want to write into it */ 1, /* Doc says "you do not want this" */ 0, p); #else ret = get_user_pages(current, current->mm, userptr, nr_pages - num_pin_pages, /* Do want to write into it */ 1, /* Doc says "you do not want this" */ 0, p); #endif if (ret < 0) break; num_pin_pages += ret; } while (num_pin_pages < nr_pages); #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) up_read(¤t->mm->mmap_lock); #else up_read(¤t->mm->mmap_sem); #endif if (num_pin_pages != nr_pages) { /* We do not tolerate less pages pinned than requested */ if (num_pin_pages > 0) ipu_put_user_pages(num_pin_pages, pages); num_pin_pages = ret; } /* return err or number of paged pinned(all) */ return num_pin_pages; } static int ipu_attach_buffer(struct ipu_device *ipu_dev, struct attach_buffer_data *bd) { struct attached_buffer *curr = NULL, *info = NULL; struct rb_node *parent = NULL; struct rb_node **new = &ipu_dev->buffers_tree.rb_node; u64 *hexopt_ptr = NULL; u64 *hexopt2_ptr = NULL; unsigned int nb_entries = 0, i = 0, sg_size = 0; int error = 0, sg_count = 0; int ret = 0, num_entries = 0; u32 hexopt_page_size = ipu_dev->hexopt_constants->HEXOPT_PAGE_SIZE; u32 hexopt_nb_entries = ipu_dev->hexopt_constants->HEXOPT_NB_ENTRIES; dma_addr_t dma_addr = 0; unsigned long phys_addr = 0; struct scatterlist *sg_itr; /* Check if platform supports 64-bit DMA we need */ if (ipu_dev->supports_dma) ipu_info_ratelimited(ipu_dev, "ipu%d: attach_buffer using DMA", ipu_dev->filename_id); else ipu_info_ratelimited(ipu_dev, "ipu%d: attach_buffer using phys-addr", ipu_dev->filename_id); nb_entries += bd->buffer_size / hexopt_page_size; nb_entries += (bd->buffer_size % hexopt_page_size) > 0 ? 1 : 0; /* Check for invalid arguments. */ if (nb_entries == 0 || nb_entries + bd->index > hexopt_nb_entries || nb_entries + bd->index < nb_entries /*overflow */) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: attaching buffer with invalid info", ipu_dev->filename_id); return -EINVAL; } /* Search for where to insert the new buffer info. */ while (*new) { curr = container_of(*new, struct attached_buffer, node); parent = *new; if (bd->index < curr->index) { if (bd->index + nb_entries > curr->index) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: cannot overwrite attached buffer.", ipu_dev->filename_id); return -EBUSY; } new = &((*new)->rb_left); } else if (bd->index > curr->index) { if (bd->index < curr->index + curr->nb_entries) { return -EBUSY; dev_err(&ipu_dev->pci_dev->dev, "ipu%d: cannot overwrite attached buffer.", ipu_dev->filename_id); } new = &((*new)->rb_right); } else { /* bd->index == curr.index */ dev_err(&ipu_dev->pci_dev->dev, "ipu%d: cannot overwrite attached buffer.", ipu_dev->filename_id); return -EBUSY; } } /* We have found where to insert the info about the new attached */ /* buffer, now insert the info to the tree. */ info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; info->buffer_pages = kzalloc((nb_entries * sizeof(struct page *)), GFP_KERNEL); if (!info->buffer_pages) { error = -ENOMEM; goto _info_fail; } info->index = bd->index; info->nb_entries = nb_entries; rb_link_node(&info->node, parent, new); rb_insert_color(&info->node, &ipu_dev->buffers_tree); /* The buffer info are recorded, do the actual attach work: */ /* First we pin the page to memory. */ error = ipu_get_user_pages(bd->buffer_addr, info->nb_entries, info->buffer_pages); if (error != info->nb_entries) goto _dma_fail; hexopt_ptr = ((u64 *)ipu_dev->exchange) + info->index; ipu_info(ipu_dev, "ipu%d: primary exchange addr at %p", ipu_dev->filename_id, hexopt_ptr); if (ipu_dev->board_type == IPU_BOARD_TYPE_C600) { if (ipu_dev->exchange_sec) { hexopt2_ptr = ((u64 *)ipu_dev->exchange_sec) + info->index; ipu_info(ipu_dev, "ipu%d: secondary exchange addr at %p", ipu_dev->filename_id, hexopt2_ptr); } else { ipu_info(ipu_dev, "ipu%d: Secondary exchange addr not set", ipu_dev->filename_id); } } /* Now we have user's pages, create sg-table for holding DMA map */ if (ipu_dev->supports_dma) { if (sg_alloc_table(&info->sgt_dma, info->nb_entries, GFP_KERNEL)) { dev_err(&ipu_dev->pci_dev->dev, "sg-table alloc error ret = %d", ret); error = -ENOMEM; ipu_put_user_pages(info->nb_entries, info->buffer_pages); goto _dma_fail; } for_each_sg(info->sgt_dma.sgl, sg_itr, info->nb_entries, i) sg_set_page(sg_itr, info->buffer_pages[i], PAGE_SIZE, 0); /* Map the sg-table */ sg_count = dma_map_sg(&ipu_dev->pci_dev->dev, info->sgt_dma.sgl, info->nb_entries, DMA_BIDIRECTIONAL); /* dma_map_sg returns 0 on error */ if (sg_count == 0) { dev_err(&ipu_dev->pci_dev->dev, "dma-map-sg error"); error = -EFAULT; goto _dma_map_fail; } /* Iterate sg-table and program hexOPT * Note: we use sg_count returned by dma_map_sg since that * holds actual entries it mapped after merging adjacent * consecutive entries */ for_each_sg(info->sgt_dma.sgl, sg_itr, sg_count, i) { dma_addr = sg_dma_address(sg_itr); sg_size = sg_dma_len(sg_itr); /* Iterate each entry of sg-table * if its greater than PAGE_SIZE, split its DMA * address into chunks of PAGE_SIZE and * then program into HexOPT entry */ /* so here in C600, in the secondary BAR */ /* ipu_dev->exchange_sec = dma_addr */ while (sg_size >= PAGE_SIZE) { *hexopt_ptr = dma_addr; if (hexopt2_ptr) { *hexopt2_ptr = dma_addr; hexopt2_ptr++; } /* Order the memory writes. See T9238. */ smp_wmb(); ipu_info_ratelimited(ipu_dev, "ipu%d: HEXOPT: page %d -- DMA 0x%llx", ipu_dev->filename_id, info->index + i, dma_addr); dma_addr += PAGE_SIZE; sg_size -= PAGE_SIZE; hexopt_ptr++; num_entries++; } } /* Ensure we programmed all requested HexOPT entries */ if (num_entries != info->nb_entries) { dev_err(&ipu_dev->pci_dev->dev, "hex-opt update error"); error = -EFAULT; goto _hex_opt_err; } } else { for (i = 0; i < nb_entries; i++) { phys_addr = page_to_phys(info->buffer_pages[i]); ipu_info_ratelimited(ipu_dev, "ipu%d: HEXOPT: page %d -- PHYS 0x%lx", ipu_dev->filename_id, info->index + i, phys_addr); *hexopt_ptr = phys_addr; if (hexopt2_ptr) { *hexopt2_ptr = phys_addr; hexopt2_ptr++; } smp_wmb(); /* Order the memory writes. See T9238. */ hexopt_ptr++; } } if (ipu_dev->supports_dma) /* In the while loop above, after the last sg-entry is processed * we program dma_addr into HexOPT and increment it by * PAGE_SIZE. So the last address programmed into HexOPT is * (dma_addr - PAGE_SIZE) */ wait_hexopt_updated(ipu_dev, info->index + info->nb_entries - 1, dma_addr - PAGE_SIZE); else wait_hexopt_updated(ipu_dev, info->index + info->nb_entries - 1, phys_addr); atomic64_add(info->nb_entries * hexopt_page_size, &ipu_dev->hexopt_used_bytes); return 0; _hex_opt_err: if (ipu_dev->supports_dma) dma_unmap_sg(&ipu_dev->pci_dev->dev, info->sgt_dma.sgl, info->nb_entries, DMA_BIDIRECTIONAL); hexopt_ptr = ((u64 *)ipu_dev->exchange); for (i = 0; i < info->nb_entries; i++) { /* Restore hexOPT indexes to init value */ if (ipu_dev->supports_dma) hexopt_ptr[info->index + i] = ipu_dev->default_hexopt_dma; else hexopt_ptr[info->index + i] = ipu_dev->default_hexopt_phys; smp_wmb(); /* Order the memory writes. See T9238. */ } if (hexopt2_ptr) { hexopt2_ptr = ((u64 *)ipu_dev->exchange_sec); for (i = 0; i < info->nb_entries; i++) { /* Restore hexOPT indexes to init value */ if (ipu_dev->supports_dma) hexopt2_ptr[info->index + i] = ipu_dev->default_hexopt_dma; else hexopt2_ptr[info->index + i] = ipu_dev->default_hexopt_phys; smp_wmb(); /* Order the memory writes. See T9238. */ } } wait_hexopt_updated(ipu_dev, info->index + info->nb_entries - 1, ipu_dev->supports_dma ? ipu_dev->default_hexopt_dma : ipu_dev->default_hexopt_phys); _dma_map_fail: sg_free_table(&info->sgt_dma); _dma_fail: rb_erase(&info->node, &ipu_dev->buffers_tree); kfree(info->buffer_pages); _info_fail: kfree(info); return error; } static void setup_hexopt(struct ipu_device *ipu_dev) { /* Make sure you can access hexopt in case userspace failed to setup */ #if defined(IAI_IPU2_SUPPORTED) || defined(IAI_IPU21_SUPPORTED) if (ipu_dev->pci_dev->subsystem_device == IPU2_PCI_SUBSYSTEM_ID || ipu_dev->pci_dev->subsystem_device == IPU2_1_PCI_SUBSYSTEM_ID) { u32 wbaser; wbaser = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_OFFSET)); wbaser |= (ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_HEXOPT_MASK) << ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_HEXOPT_SHIFT)); writel(wbaser, ipu_dev->config + ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_OFFSET)); wbaser = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_OFFSET)); if (wbaser & (ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_HEXOPT_MASK) << ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_HEXOPT_SHIFT))) ipu_info_ratelimited(ipu_dev, "Set EXCHWINDOWBASER to hexopt"); else ipu_err(ipu_dev, "Failed to set EXCHWINDOWBASER to hexopt"); } #endif } static int ipu_detach_buffer(struct ipu_device *ipu_dev, unsigned int index) { struct rb_node *node = ipu_dev->buffers_tree.rb_node; struct attached_buffer *info = NULL; u64 *hexopt_ptr = NULL; u64 *hexopt2_ptr = NULL; unsigned int i = 0; ipu_info_ratelimited(ipu_dev, "ipu%d: IPU detach buffer", ipu_dev->filename_id); while (node) { info = container_of(node, struct attached_buffer, node); if (index < info->index) node = node->rb_left; else if (index > info->index) node = node->rb_right; else break; /* Found. */ } if (!node) { dev_err_ratelimited(&ipu_dev->pci_dev->dev, "ipu%d: cannot detach at index %d - not found", ipu_dev->filename_id, index); return -EINVAL; } hexopt_ptr = ((u64 *)ipu_dev->exchange); if (ipu_dev->board_type == IPU_BOARD_TYPE_C600) { if (ipu_dev->exchange_sec) hexopt2_ptr = ((u64 *)ipu_dev->exchange_sec); else ipu_info(ipu_dev, "ipu%d: Secondary exchange bar not set", ipu_dev->filename_id); } setup_hexopt(ipu_dev); /* Point the HEXOPT to the default page. */ for (i = 0; i < info->nb_entries; i++) { if (ipu_dev->supports_dma) hexopt_ptr[info->index + i] = ipu_dev->default_hexopt_dma; else hexopt_ptr[info->index + i] = ipu_dev->default_hexopt_phys; /* Order the memory writes. See T9238. */ smp_wmb(); } if (hexopt2_ptr) { /* Point secondary HEXOPT to the default page. */ for (i = 0; i < info->nb_entries; i++) { if (ipu_dev->supports_dma) hexopt2_ptr[info->index + i] = ipu_dev->default_hexopt_dma; else hexopt2_ptr[info->index + i] = ipu_dev->default_hexopt_phys; /* Order the memory writes. See T9238. */ smp_wmb(); } } wait_hexopt_updated(ipu_dev, info->index + info->nb_entries - 1, ipu_dev->supports_dma ? ipu_dev->default_hexopt_dma : ipu_dev->default_hexopt_phys); /* Unmap the page in sg-table and free it */ if (ipu_dev->supports_dma) { dma_unmap_sg(&ipu_dev->pci_dev->dev, info->sgt_dma.sgl, info->nb_entries, DMA_BIDIRECTIONAL); sg_free_table(&info->sgt_dma); } /* Release the user pages. */ for (i = 0; i < info->nb_entries; i++) { set_page_dirty_lock(info->buffer_pages[i]); #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) unpin_user_page(info->buffer_pages[i]); #else put_page(info->buffer_pages[i]); #endif } atomic64_sub(info->nb_entries * ipu_dev->hexopt_constants->HEXOPT_PAGE_SIZE, &ipu_dev->hexopt_used_bytes); rb_erase(&info->node, &ipu_dev->buffers_tree); kfree(info->buffer_pages); kfree(info); return 0; } static void ipu_detach_all_buffers(struct ipu_device *ipu) { struct rb_node *node = rb_first(&ipu->buffers_tree); struct attached_buffer *info = NULL; while (node) { info = container_of(node, struct attached_buffer, node); ipu_detach_buffer(ipu, info->index); /* Cannot fail. */ node = rb_first(&ipu->buffers_tree); } } static void free_hsp_buffers(struct ipu_device *ipu_dev) { /* Free HSP buffers. */ if (ipu_dev->config) put_hsp_notify(ipu_dev, NULL); if (ipu_dev->hsp_notify[0]) { set_page_dirty_lock(ipu_dev->hsp_notify[0]); #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) unpin_user_page(ipu_dev->hsp_notify[0]); #else put_page(ipu_dev->hsp_notify[0]); #endif } if (ipu_dev->hsp_notify[1]) { set_page_dirty_lock(ipu_dev->hsp_notify[1]); #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) unpin_user_page(ipu_dev->hsp_notify[1]); #else put_page(ipu_dev->hsp_notify[1]); #endif } ipu_dev->hsp_notify[0] = 0; ipu_dev->hsp_notify[1] = 0; } static int ipu_mask_interrupt(struct ipu_device *dev, unsigned long mask) { unsigned int i = 0; /* We do not support nesting of enable and disable */ for (i = 0; i < dev->num_ints; i++) { if ((mask & (1 << i)) && !(dev->ints_mask & (1 << i))) { disable_irq(dev->interrupts[i].vector); dev->ints_mask |= (1 << i); } } return 0; } static int ipu_unmask_interrupt(struct ipu_device *dev, unsigned long mask) { unsigned int i = 0; /* Disable interrupts only if enabled */ for (i = 0; i < dev->num_ints; i++) { if ((mask & (1 << i)) && (dev->ints_mask & (1 << i))) { enable_irq(dev->interrupts[i].vector); dev->ints_mask &= ~(1 << i); } } return 0; } static void set_sensor_value(struct ipu_device *ipu_dev, unsigned int index, int value) { struct device *dev = &ipu_dev->pci_dev->dev; if (index < NUM_ALL_SENSORS) { atomic_set(&ipu_dev->sensor[index].value, value); ipu_dev->sensor[index].valid = 1; } else { dev_err(dev, "ipu%d: invalid sensor index %d", ipu_dev->filename_id, index); } } static void clear_all_sensors(struct ipu_device *ipu_dev) { int i; for (i = 0; i < NUM_ALL_SENSORS; i++) ipu_dev->sensor[i].valid = 0; } static int ipu_store_sensor_reading(struct ipu_device *ipu_dev, struct ipu_sensor_reading_t *sr) { struct device *dev = &ipu_dev->pci_dev->dev; switch (sr->sensor_type) { case IPU_SENSOR_DIE_TEMP: set_sensor_value(ipu_dev, TEMP_SENSOR_DIE_START + sr->index, sr->value); break; case IPU_SENSOR_BOARD_TEMP: set_sensor_value(ipu_dev, TEMP_SENSOR_BOARD_START + sr->index, sr->value); break; case IPU_SENSOR_POWER: set_sensor_value(ipu_dev, POWER_SENSOR_START + sr->index, sr->value); break; case APP_SIZES: set_sensor_value(ipu_dev, APP_SIZES_START + sr->index, sr->value); break; case IPU_BUSY_DURATION: set_sensor_value(ipu_dev, IPU_BUSY_DURATION_START + sr->index, sr->value); break; default: dev_err(dev, "ipu%d: invalid sensor type %d", ipu_dev->filename_id, sr->sensor_type); } return 0; } /* Free all contiguous mem chunks held by this driver instance */ static void ipu_free_all_contiguous_mem(struct ipu_device *ipu_dev) { struct contiguous_mem_list_entry *tmp, *tmp2; int cmem_node = get_cmem_node(ipu_dev); if (cmem_node < 0) return; list_for_each_entry_safe(tmp, tmp2, &ipu_dev->contiguous_mem_head, list) { ipu_info_ratelimited(ipu_dev, "ipu%d cmem free - 0x%llx at 0x%llx", ipu_dev->filename_id, tmp->size, tmp->phys_addr); contiguous_mem_free(tmp->size, tmp->virt_addr, cmem_node); atomic64_sub(tmp->size, &ipu_dev->hexoatt_used_bytes); list_del(&tmp->list); kfree(tmp); } } static int ipu_get_entry_contiguous_mem(struct ipu_device *ipu_dev, u64 *entry, u64 handle) { struct contiguous_mem_list_entry *tmp, *tmp2; list_for_each_entry_safe(tmp, tmp2, &ipu_dev->contiguous_mem_head, list) { if (handle == tmp->handle) { *entry = tmp->phys_addr; return 0; } } return -EINVAL; } static void use_copy_to_user(struct ipu_device *ipu_dev, struct contiguous_buffer_ops_t *ops, struct contiguous_mem_list_entry *tmp) { u64 bytes; u64 offset, user_buffer; u64 size_chunk, num_chunk; /* If 4GB or more is to be copied, split into chunks */ if (ops->size > USER_COPY_LIMIT) { size_chunk = USER_COPY_CHUNK_SIZE; num_chunk = ops->size / size_chunk; offset = ops->offset; user_buffer = ops->user_buffer; do { bytes = copy_to_user((void *)user_buffer, (void *)(tmp->virt_addr + offset), size_chunk); if (bytes != 0) { ipu_err(ipu_dev, "ipu%d failed copying bytes to user", ipu_dev->filename_id); return; } offset += size_chunk; user_buffer += size_chunk; num_chunk--; } while (num_chunk > 0); if (ops->size % size_chunk) { bytes = copy_to_user((void *)user_buffer, (void *)(tmp->virt_addr + offset), (ops->size % size_chunk)); if (bytes != 0) { ipu_err(ipu_dev, "ipu%d failed copying bytes to user", ipu_dev->filename_id); return; } } } else { bytes = copy_to_user((void *)ops->user_buffer, (void *)(tmp->virt_addr + ops->offset), ops->size); if (bytes != 0) { ipu_err(ipu_dev, "ipu%d failed copying bytes to user", ipu_dev->filename_id); return; } } ipu_info_ratelimited(ipu_dev, "ipu%d copied all bytes to 0x%lx", ipu_dev->filename_id, ops->user_buffer); } static void ipu_read_contiguous_mem(struct ipu_device *ipu_dev, struct contiguous_buffer_ops_t *ops) { struct contiguous_mem_list_entry *tmp, *tmp2; u64 mem_start, mem_size; int cmem_node = get_cmem_node(ipu_dev); if (cmem_node < 0) return; ipu_info_ratelimited(ipu_dev, "ipu%d readops size 0x%lx", ipu_dev->filename_id, ops->size); mem_start = memmap_start[cmem_node]; mem_size = memmap_size[cmem_node]; list_for_each_entry_safe(tmp, tmp2, &ipu_dev->contiguous_mem_head, list) { if (ops->handle == tmp->handle) { if ((u64)(tmp->phys_addr + ops->offset + ops->size) <= (u64) ((u64)mem_start + mem_size)) { use_copy_to_user(ipu_dev, ops, tmp); } else { ipu_err(ipu_dev, "ipu%d error reading at handle %llx", ipu_dev->filename_id, tmp->handle); } break; } } } static void use_copy_from_user(struct ipu_device *ipu_dev, struct contiguous_buffer_ops_t *ops, struct contiguous_mem_list_entry *tmp) { u64 bytes; u64 offset, user_buffer; u64 size_chunk, num_chunk; /* If 4GB or more is to be copied, split into chunks */ if (ops->size > USER_COPY_LIMIT) { size_chunk = USER_COPY_CHUNK_SIZE; num_chunk = ops->size / size_chunk; offset = ops->offset; user_buffer = ops->user_buffer; do { bytes = copy_from_user((void *)(tmp->virt_addr + offset), (void *)user_buffer, size_chunk); if (bytes != 0) { ipu_err(ipu_dev, "ipu%d failed copying bytes from user", ipu_dev->filename_id); return; } offset += size_chunk; user_buffer += size_chunk; num_chunk--; } while (num_chunk > 0); if (ops->size % size_chunk) { bytes = copy_from_user((void *)(tmp->virt_addr + offset), (void *)user_buffer, (ops->size % size_chunk)); if (bytes != 0) { ipu_err(ipu_dev, "ipu%d failed copying bytes from user", ipu_dev->filename_id); return; } } } else { bytes = copy_from_user((void *)(tmp->virt_addr + ops->offset), (void *)ops->user_buffer, ops->size); if (bytes != 0) { ipu_err(ipu_dev, "ipu%d failed copying bytes from user", ipu_dev->filename_id); return; } } ipu_info_ratelimited(ipu_dev, "ipu%d copied all bytes from 0x%lx", ipu_dev->filename_id, ops->user_buffer); } static void ipu_write_contiguous_mem(struct ipu_device *ipu_dev, struct contiguous_buffer_ops_t *ops) { struct contiguous_mem_list_entry *tmp, *tmp2; u64 mem_start, mem_size; int cmem_node = get_cmem_node(ipu_dev); if (cmem_node < 0) return; ipu_info_ratelimited(ipu_dev, "ipu%d writeops size 0x%lx", ipu_dev->filename_id, ops->size); mem_start = memmap_start[cmem_node]; mem_size = memmap_size[cmem_node]; list_for_each_entry_safe(tmp, tmp2, &ipu_dev->contiguous_mem_head, list) { if (ops->handle == tmp->handle) { if ((u64)(tmp->phys_addr + ops->offset + ops->size) <= (u64) ((u64)mem_start + mem_size)) { use_copy_from_user(ipu_dev, ops, tmp); } else { ipu_err(ipu_dev, "ipu%d error writing at handle %llx", ipu_dev->filename_id, tmp->handle); } break; } } } /* Free contiguous mem chunk if held by this driver instance */ static void ipu_free_contiguous_mem(struct ipu_device *ipu_dev, u64 handle) { struct contiguous_mem_list_entry *tmp, *tmp2; int cmem_node = get_cmem_node(ipu_dev); if (cmem_node < 0) return; list_for_each_entry_safe(tmp, tmp2, &ipu_dev->contiguous_mem_head, list) { if (handle == tmp->handle) { ipu_info_ratelimited(ipu_dev, "ipu%d cmem free - 0x%llx at 0x%llx", ipu_dev->filename_id, tmp->size, tmp->phys_addr); contiguous_mem_free(tmp->size, tmp->virt_addr, cmem_node); atomic64_sub(tmp->size, &ipu_dev->hexoatt_used_bytes); list_del(&tmp->list); kfree(tmp); break; } } } /* Convert a user virtual address into a physical address. */ static phys_addr_t user_virt_to_phys(struct ipu_device *dev, u64 user_virt, struct page **page) { phys_addr_t addr; void *page_vir; unsigned int nb_entries = 1; if (nb_entries != ipu_get_user_pages(user_virt, nb_entries, page)) goto fail; if (dev->supports_dma) { page_vir = page_address(page[0]); addr = dma_map_single(&dev->pci_dev->dev, page_vir, sizeof(u64), DMA_FROM_DEVICE); } else { addr = page_to_phys(page[0]); } return addr + (user_virt % PAGE_SIZE); fail: dev_err(&dev->pci_dev->dev, "Failed to decode address %p for HSP NOTIFY", (void *)user_virt); return -EFAULT; } static void teardown_dma_for_hsp_notify(struct ipu_device *ipu_dev) { if (ipu_dev->hsp_pa[0]) { dma_unmap_single(&ipu_dev->pci_dev->dev, ipu_dev->hsp_pa[0], sizeof(u32), DMA_FROM_DEVICE); ipu_dev->hsp_pa[0] = 0; } if (ipu_dev->hsp_pa[1]) { dma_unmap_single(&ipu_dev->pci_dev->dev, ipu_dev->hsp_pa[1], sizeof(u32), DMA_FROM_DEVICE); ipu_dev->hsp_pa[1] = 0; } } static void setup_dma_for_hsp_notify(struct ipu_device *ipu_dev) { if (!ipu_dev->supports_dma) { ipu_err(ipu_dev, "No DMA support - cannot setup HSP notify"); return; } ipu_dev->hsp_pa[0] = dma_map_single(&ipu_dev->pci_dev->dev, &ipu_dev->hsp[0], sizeof(u32), DMA_FROM_DEVICE); if (dma_mapping_error(&ipu_dev->pci_dev->dev, ipu_dev->hsp_pa[0])) { ipu_err(ipu_dev, "Failed to register DMA address for HSP1"); return; } ipu_info(ipu_dev, "%s: HSPGS1 hsp_pa=%pad", __func__, &ipu_dev->hsp_pa[0]); ipu_dev->hsp_pa[1] = dma_map_single(&ipu_dev->pci_dev->dev, &ipu_dev->hsp[1], sizeof(u32), DMA_FROM_DEVICE); if (dma_mapping_error(&ipu_dev->pci_dev->dev, ipu_dev->hsp_pa[1])) { ipu_err(ipu_dev, "Failed to register DMA address for HSP2"); ipu_dev->hsp_pa[1] = 0; teardown_dma_for_hsp_notify(ipu_dev); return; } ipu_info(ipu_dev, "%s: HSPGS2 hsp_pa=%pad", __func__, &ipu_dev->hsp_pa[1]); } static long put_hsp_notify(struct ipu_device *ipu_dev, struct hspgs_notify_addr *addrs) { phys_addr_t phys_addr = 0; u32 reg = 0; void __iomem *reg_addr; if (addrs) { phys_addr = user_virt_to_phys(ipu_dev, addrs->hsp[0], &ipu_dev->hsp_notify[0]); if (phys_addr <= 0) return -EFAULT; } // Populate physical mem address of HSPGS1 and HSPGS2 'marks' // into device registers. Avoid resetting the registers to 0 // if they already read as 0. This is important for non POSTED modes. // In non POSTED modes these registers are not used but resetting // them to 0 can cause the marks to be cleared which is not desirable, // as we might want to investigate marks on exit ipu_info_ratelimited(ipu_dev, "HSPGS1 notify address: %llx", phys_addr); reg_addr = ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HSPGS1LR_OFFSET); reg = readl(reg_addr); if (reg != 0 || phys_addr != 0) writel(((phys_addr & 0xffffffff) >> 2), reg_addr); reg_addr = ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HSPGS1HILR_OFFSET); reg = readl(reg_addr); if (reg != 0 || phys_addr != 0) writel((phys_addr >> 32), reg_addr); if (addrs) { phys_addr = user_virt_to_phys(ipu_dev, addrs->hsp[1], &ipu_dev->hsp_notify[1]); if (phys_addr <= 0) return -EFAULT; } ipu_info_ratelimited(ipu_dev, "HSPGS2 notify address: %llx", phys_addr); reg_addr = ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HSPGS2LR_OFFSET); reg = readl(reg_addr); if (reg != 0 || phys_addr != 0) writel(((phys_addr & 0xffffffff) >> 2), reg_addr); reg_addr = ipu_dev->config + ARCH_MAP(PCI_COMPLEX_HSPGS2HILR_OFFSET); reg = readl(reg_addr); if (reg != 0 || phys_addr != 0) writel((phys_addr >> 32), reg_addr); return 0; } /* * Check if given NLC link is up. */ bool is_nlc_link_up(struct ipu_device *ipu_dev, u32 nlc_base) { u32 nlc_coredn_shift; u32 nlc_coredn_mask; u32 nlc_linkdn_shift; u32 nlc_linkdn_mask; u32 nlc_lcsr; u32 coredn; u32 linkdn; if (ipu_dev && nlc_base) { nlc_coredn_shift = ARCH_MAP(NLC_LCSR_COREDN_SHIFT); nlc_coredn_mask = ARCH_MAP(NLC_LCSR_COREDN_MASK); nlc_linkdn_shift = ARCH_MAP(NLC_LCSR_LINKDN_SHIFT); nlc_linkdn_mask = ARCH_MAP(NLC_LCSR_LINKDN_MASK); nlc_lcsr = readl(ipu_dev->config + nlc_base + ARCH_MAP(NLC_LCSR_OFFSET)); coredn = ((nlc_lcsr >> nlc_coredn_shift) & nlc_coredn_mask); linkdn = ((nlc_lcsr >> nlc_linkdn_shift) & nlc_linkdn_mask); if (coredn == 0 && linkdn == 0) return true; } return false; } /* * Update all the NLC correctable error counters. It requires * ipu_dev->nlc_mutex to be taken beforehand. */ static void ipu_accumulate_nlc_errcnt_unsafe(struct ipu_device *ipu_dev) { int idx; u32 nlc_errcnt_shift = ARCH_MAP(NLC_CSR_CORERRCNT_SHIFT); u32 nlc_errcnt_mask = ARCH_MAP(NLC_CSR_CORERRCNT_MASK); u32 nlc_error_shift = ARCH_MAP(NLC_CSR_CORERROR_SHIFT); u32 nlc_error_mask = ARCH_MAP(NLC_CSR_CORERROR_MASK); u32 nlc_csr; u32 new_val; for (idx = 0; idx < NUM_NLCS; idx++) { if (!is_nlc_link_up(ipu_dev, ipu_dev->nlc_data[idx].nlc_base)) continue; nlc_csr = readl(ipu_dev->config + ipu_dev->nlc_data[idx].nlc_base + ARCH_MAP(NLC_CSR_OFFSET)); new_val = ((nlc_csr >> nlc_errcnt_shift) & nlc_errcnt_mask); if (new_val == 0) continue; /* Accumulate the NLC error count - saturate if needed */ check_and_saturate_u64(&ipu_dev->nlc_data[idx].acc_cor_err_cnt, new_val); check_and_saturate_u64(&ipu_dev->nlc_total_errcnt, new_val); check_and_saturate_u64(&ipu_dev->nlc_total_errcnt_session, new_val); /* Zero the register error count, otherwise the next ioctl call * would accumulate the same value again. */ writel(((1u & nlc_error_mask) << nlc_error_shift), ipu_dev->config + ipu_dev->nlc_data[idx].nlc_base + ARCH_MAP(NLC_CSR_OFFSET)); } } static void mailbox_read(struct ipu_device *ipu_dev, struct mailbox_msg *msg_from_icu) { struct mailbox_msg event; //Read the ICU mailbox event.msg[1] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR0_OFFSET)); event.msg[2] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR1_OFFSET)); event.msg[3] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR2_OFFSET)); event.msg[4] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUDATR3_OFFSET)); if (ipu_dev->icu_use_v2_protocol) { event.msg[0] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR3_OFFSET)); } else { /* This last read is the one that unblocks the mailbox */ event.msg[0] = readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUCMDR_OFFSET)); } memcpy(&ipu_dev->mcu_sync_msg_response, &event, sizeof(event)); memcpy(msg_from_icu, &ipu_dev->mcu_sync_msg_response, sizeof(struct mailbox_msg)); ipu_info_verbose(ipu_dev, "ipu%d ICU sync read: %08x %08x %08x %08x %08x", ipu_dev->filename_id, msg_from_icu->msg[0], msg_from_icu->msg[1], msg_from_icu->msg[2], msg_from_icu->msg[3], msg_from_icu->msg[4]); /* clear to avoid accidental reuse */ memset(&ipu_dev->mcu_sync_msg_response, 0, sizeof(struct mailbox_msg)); } static ssize_t mailbox_write(struct ipu_device *ipu_dev, struct mailbox_msg *msg_for_icu) { ssize_t ret_val; ipu_info_verbose(ipu_dev, "ipu%d ICU sync write: %08x %08x %08x %08x %08x", ipu_dev->filename_id, msg_for_icu->msg[0], msg_for_icu->msg[1], msg_for_icu->msg[2], msg_for_icu->msg[3], msg_for_icu->msg[4]); reinit_completion(&ipu_dev->mailbox_rsp_available); ipu_dev->mcu_ioctl_read_pending = 1; writel(msg_for_icu->msg[1], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR0_OFFSET)); writel(msg_for_icu->msg[2], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR1_OFFSET)); writel(msg_for_icu->msg[3], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR2_OFFSET)); writel(msg_for_icu->msg[4], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCIDATR3_OFFSET)); writel(msg_for_icu->msg[0], ipu_dev->config + ARCH_MAP(PCI_COMPLEX_PCICMDR_OFFSET)); ret_val = wait_for_completion_killable_timeout (&ipu_dev->mailbox_rsp_available, MBOX_TIMEOUT_JIFFIES); // clear mcu msg status, in case interrupt missed (discard) ipu_dev->mcu_ioctl_read_pending = 0; if (ret_val == 0) { /* Failed to get a response */ if (!ipu_dev->icu_use_v2_protocol) { /* * We read the mailbox anyway in case for some reason * there was a previous reply blocking it. In normal * operation this can't happen unless an interrupt was * missed. */ readl(ipu_dev->config + ARCH_MAP(PCI_COMPLEX_MCUCMDR_OFFSET)); } ipu_err(ipu_dev, "ipu%d: mailbox timeout expired", ipu_dev->filename_id); return -ETIME; } else if (ret_val < 0) { return -EINTR; } return 0; } int icu_mailbox_transaction(struct ipu_device *ipu_dev, u32 mailbox[MBOX_MSG_SIZE_WORDS], u32 *result, size_t result_nbytes) { struct mailbox_msg msg_from_icu, msg_for_icu; ssize_t ret_val = -EIO; int i; uint result_words = result_nbytes / sizeof(u32); if (result_words > (MBOX_MSG_SIZE_WORDS - 1)) { ipu_err(ipu_dev, "unexpected result_words = %u", result_words); return -EOPNOTSUPP; } for (i = 0; i < MBOX_MSG_SIZE_WORDS; i++) msg_for_icu.msg[i] = mailbox[i]; ret_val = mutex_lock_killable(&ipu_dev->mailbox_available); if (ret_val < 0) { ipu_err(ipu_dev, "ipu%d: failed to acquire ICU mutex", ipu_dev->filename_id); return -EBUSY; } ret_val = mailbox_write(ipu_dev, &msg_for_icu); if (ret_val == 0) { mailbox_read(ipu_dev, &msg_from_icu); mutex_unlock(&ipu_dev->mailbox_available); if (result) { ipu_info(ipu_dev, "resp header 0x%x", msg_from_icu.msg[0]); u32 *resp_ptr = &msg_from_icu.msg[1]; // skip header for (i = 0; i < result_words; i++) result[i] = resp_ptr[i]; } } else { mutex_unlock(&ipu_dev->mailbox_available); ipu_err(ipu_dev, "ipu%d: failed to communicate with the ICU", ipu_dev->filename_id); } return ret_val; } #define AER_FLAG(data, flag) ((data & flag) ? '+' : '-') static void aer_data_print(struct ipu_device *ipu_dev) { u32 aer_data = ipu_dev->aer_data; // Print UESta flags which lspci would print char *print_aer_ue_status = "ipu%d: AER UESTATUS: DLP%c SDES%c TLP%c FCP%c CmpltTO%c " "CmpltAbrt%c UnxCmplt%c RxOF%c MalfTLP%c ECRC%c UnsupReq%c ACSViol%c "; ipu_info(ipu_dev, print_aer_ue_status, ipu_dev->filename_id, AER_FLAG(aer_data, PCI_ERR_UNC_DLP), AER_FLAG(aer_data, PCI_ERR_UNC_SURPDN), AER_FLAG(aer_data, PCI_ERR_UNC_POISON_TLP), AER_FLAG(aer_data, PCI_ERR_UNC_FCP), AER_FLAG(aer_data, PCI_ERR_UNC_COMP_TIME), AER_FLAG(aer_data, PCI_ERR_UNC_COMP_ABORT), AER_FLAG(aer_data, PCI_ERR_UNC_UNX_COMP), AER_FLAG(aer_data, PCI_ERR_UNC_RX_OVER), AER_FLAG(aer_data, PCI_ERR_UNC_MALF_TLP), AER_FLAG(aer_data, PCI_ERR_UNC_ECRC), AER_FLAG(aer_data, PCI_ERR_UNC_UNSUP), AER_FLAG(aer_data, PCI_ERR_UNC_ACSV) ); } static long enable_gwlinks(struct ipu_device *ipu_dev, unsigned int enable) { ipu_info(ipu_dev, "ipu%d: %s IPU GW-Links", ipu_dev->filename_id, enable ? "Enabling" : "Disabling"); if (!is_ipum_board(ipu_dev->board_type)) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: %s only supported on IPU-M", ipu_dev->filename_id, __func__); return -EFAULT; } pci_write_config_dword(ipu_dev->pci_dev, IPU_GWLINKS_ENABLE_OFFSET, enable ? IPU_GWLINKS_ENABLE_VALUE : IPU_GWLINKS_DISABLE_VALUE); return 0; } // Temperature and power values return by the ICU are in linear11 format static s32 linear11_to_int(u16 linear11_value) { bool negative_exponent; bool negative_mantissa; u16 exponent; u16 mantissa; u64 result; /* * Cannot bit shift negative numbers, or shift by a negative number: * Exponent >= 0, shift left * Exponent < 0, shift right by 2s complement * Mantissa >= 0, just shift * Mantissa < 0, shift the 2s complement, then do the integer 2s * complement (i.e. -) afterwards */ negative_exponent = linear11_value & (1u << 15u); negative_mantissa = linear11_value & (1u << 10u); exponent = (linear11_value >> 11u); mantissa = (linear11_value & 0x7ff); if (negative_exponent) exponent = ~exponent + 1u; // Calculate absolute value exponent &= 0x1Fu; if (negative_mantissa) mantissa = ~mantissa + 1u; // Calculate absolute value mantissa &= 0x7FF; /* * Pre-shift the result by 16 bits to preserve precision when * doing right shifts */ result = mantissa; result <<= 16u; if (negative_exponent) result >>= exponent; // Divide else result <<= exponent; // Multiply /* Multiply by the given scalar */ result *= 1000; /* Shift out the extra 16u bits */ result >>= 16u; if (negative_mantissa) { /* * Shifting was performed on the absolute value, * so make it negative now */ return -(s32)result; } return (s32)result; } void icu_read_board_temps(struct ipu_device *ipu_dev) { u64 now; u32 val; uint i; int ret_val; u16 temps[4] = {0}; if (!ipu_dev->icu_supports_telemetry) return; now = ktime_divns(ktime_get_real(), NSEC_PER_SEC); if (now - ipu_dev->board_temp_last_update_ts == 0) return; ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_GET_TELEMETRY_SENSOR_TMP, ipu_dev->icu_index, 0, 0, 0}, (u32 *)temps, sizeof(temps)); if (ret_val == 0) { for (i = 0; i < 4; i++) { val = linear11_to_int(temps[i]); set_sensor_value(ipu_dev, TEMP_SENSOR_BOARD_START + i, val); ipu_info_ratelimited(ipu_dev, "board temp %d: %d", i, val); } } ipu_dev->board_temp_last_update_ts = now; } void icu_send_board_detach(struct ipu_device *ipu_dev) { int ret_val; ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_BOARD_DETACH, ipu_dev->icu_index, 0, 0, 0}, 0, 0); if (ret_val) dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to send ICU_BOARD_DETACH", ipu_dev->filename_id); } void icu_read_ipu_die_temps(struct ipu_device *ipu_dev) { uint chip_index; u16 pvt_temps[2][2]; // [sensor][chip] u32 west, east; u64 now; int ret_val; if (!ipu_dev->icu_supports_telemetry) return; now = ktime_divns(ktime_get_real(), NSEC_PER_SEC); if (now - ipu_dev->die_temp_last_update_ts == 0) return; chip_index = ipu_dev->icu_index; ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_GET_TELEMETRY_PVT_TMP, ipu_dev->icu_index, 0, 0, 0}, (u32 *)pvt_temps, sizeof(pvt_temps)); if (ret_val == 0) { west = linear11_to_int(pvt_temps[chip_index][0]); east = linear11_to_int(pvt_temps[chip_index][1]); // ICU only supplies 2 sensors, so duplicate into our 4 slots set_sensor_value(ipu_dev, TEMP_SENSOR_DIE_START + 0, west); set_sensor_value(ipu_dev, TEMP_SENSOR_DIE_START + 1, west); set_sensor_value(ipu_dev, TEMP_SENSOR_DIE_START + 2, east); set_sensor_value(ipu_dev, TEMP_SENSOR_DIE_START + 3, east); ipu_info_ratelimited(ipu_dev, "die temp west: %d", west); ipu_info_ratelimited(ipu_dev, "die temp east: %d", east); } ipu_dev->die_temp_last_update_ts = now; } void icu_read_ipu_power(struct ipu_device *ipu_dev) { u16 powers[2]; // [chip] u32 power; u64 now; uint i; int ret_val; if (!ipu_dev->icu_supports_telemetry) return; now = ktime_divns(ktime_get_real(), NSEC_PER_SEC); if (now - ipu_dev->power_last_update_ts == 0) return; ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_GET_TELEMETRY_POWER, ipu_dev->icu_index, 0, 0, 0}, (u32 *)powers, sizeof(powers)); if (ret_val == 0) { /* * For each IPU we store the power readings for chips 0 and 1 * (i.e. both the current device and the other one) */ for (i = 0; i < 2; i++) { power = linear11_to_int(powers[i]); set_sensor_value(ipu_dev, POWER_SENSOR_START + i, power); ipu_info_ratelimited(ipu_dev, "power %d: %d", i, power); } } ipu_dev->power_last_update_ts = now; } static void icu_transfer_version_string(struct ipu_device *ipu_dev, u32 type, char *string_buf, int buf_len) { /* * the FW and bootloader version strings are sent as * multiple result packets, with each packet containing * a 15-byte payload and a 1 byte "bytes remaining" field. */ int ret_val; int req_index = 0; // [0] bytes remaining, [1-15] payload char response[16]; char *bytes_remaining_ptr = &response[0]; // set up an alias memset(string_buf, 0, buf_len); buf_len--; // ensure null terminated do { u32 msg[MBOX_MSG_SIZE_WORDS] = { ICU_CMD_FIRMWARE_VERSION_READ_AS_STRING, type, req_index, 0, 0}; memset(response, 0, sizeof(response)); ret_val = icu_mailbox_transaction(ipu_dev, msg, (u32 *)response, sizeof(response)); if (ret_val != 0) break; unsigned int payload_bytes = min(15, *bytes_remaining_ptr); payload_bytes = min(payload_bytes, buf_len); if (payload_bytes) strncpy(string_buf, &response[1], payload_bytes); string_buf += payload_bytes; req_index += payload_bytes; buf_len -= payload_bytes; } while ((*bytes_remaining_ptr > 15) && (buf_len > 0)); } #define ICU_VER_STRING (2) void icu_read_fw_version(struct ipu_device *ipu_dev) { if (!ipu_dev->icu_has_firmware_ver_str) return; icu_transfer_version_string(ipu_dev, ICU_VER_STRING, ipu_dev->firmware_version_str, sizeof(ipu_dev->firmware_version_str)); } #define ICU_VER_BOOTLOADER (5) void icu_read_bootloader_version(struct ipu_device *ipu_dev) { if (!ipu_dev->icu_has_bootloader_ver_str) return; icu_transfer_version_string(ipu_dev, ICU_VER_BOOTLOADER, ipu_dev->bootloader_version_str, sizeof(ipu_dev->bootloader_version_str)); } static void write_spb_service_table(struct ipu_device *ipu_dev, unsigned int entry, unsigned int low, unsigned int high) { iowrite32(entry, ipu_dev->spb_acc + SPB_RDWR_SERVICE_TABLE_ADDR_OFFSET); iowrite32(low, ipu_dev->spb_acc + SPB_WR_SERVICE_TABLE_ENTRY_LOW_OFFSET); iowrite32(high, ipu_dev->spb_acc + SPB_WR_SERVICE_TABLE_ENTRY_HIGH_OFFSET); iowrite32(1, ipu_dev->spb_acc + SPB_WR_SERVICE_TABLE_KICK_OFFSET); } bool check_spb_supported(struct ipu_device *ipu_dev) { if (ipu_dev->board_type != IPU_BOARD_TYPE_M2000 || !ipu_dev->spb_acc) return false; return true; } /* * Update the accumulated error packet counters in SPB. It requires * ipu_dev->spb_mutex to be taken beforehand. */ void spb_error_pkt_count_update_unsafe(struct ipu_device *ipu_dev) { /* Read current error count; will be zero'ed after reading */ u32 new_val = ioread32(ipu_dev->spb_acc + SPB_ERROR_PKT_COUNT); /* Add it to accumulated error counters */ ipu_dev->spb_error_pkt_count += new_val; ipu_dev->spb_error_pkt_count_session += new_val; } static long clear_all_spb_service_tables(struct ipu_device *ipu_dev) { if (check_spb_supported(ipu_dev)) { unsigned int i; unsigned int start; ipu_info(ipu_dev, "ipu%d: clearing service tables", ipu_dev->filename_id); // clear entries in write service table start = SPB_SERVICE_TABLE_FOR_WRITES_START_INDEX; for (i = 0; i < ARCH_MAP(IPU_TILE_INSTANCES_WITH_REPAIR); i++) write_spb_service_table(ipu_dev, start + i, 0, 0); // clear entries in read service table start = SPB_SERVICE_TABLE_FOR_READS_START_INDEX; for (i = 0; i < SPB_SERVICE_TABLE_FOR_READS_ENTRIES; i++) write_spb_service_table(ipu_dev, start + i, 0, 0); return 0; } return -EFAULT; } static int setup_hexopt_secondary(struct ipu_device *ipu_dev) { icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_SET_HEXOPT, // secondary complex(1) 6th byte // hexopt set (1) 5th byte ((1 << 8) | 0x01), 0, 0, 0}, NULL, 0); u32 exch_reg_msg[2] = {0}; int ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_GET_EXCH_WIN_BASE_REG, // secondary complex(1) 5th byte 1, 0, 0, 0}, exch_reg_msg, sizeof(exch_reg_msg)); if (ret_val == 0 && exch_reg_msg[0] == 0) { u32 hexopt_bit = ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_HEXOPT_MASK) << ARCH_MAP(PCI_COMPLEX_EXCHWINDOWBASER_HEXOPT_SHIFT); if (exch_reg_msg[1] & hexopt_bit) return 0; dev_err(&ipu_dev->pci_dev->dev, "Failed to configure secondary hexopt"); } return -1; } static long set_spb_multi_read_st_enable(struct ipu_device *ipu_dev, unsigned int enable) { if (check_spb_supported(ipu_dev)) { unsigned int value, value2; value = ioread32(ipu_dev->spb_acc + SPB_CONFIG_REG_OFFSET); if (enable) value |= SPB_CONFIG_REG_ENABLE_MULTI_READ_ST_ENTRY; else value &= ~SPB_CONFIG_REG_ENABLE_MULTI_READ_ST_ENTRY; iowrite32(value, ipu_dev->spb_acc + SPB_CONFIG_REG_OFFSET); /* confirm as older gateways do not support this feature */ value2 = ioread32(ipu_dev->spb_acc + SPB_CONFIG_REG_OFFSET); if ((value & SPB_CONFIG_REG_ENABLE_MULTI_READ_ST_ENTRY) == (value2 & SPB_CONFIG_REG_ENABLE_MULTI_READ_ST_ENTRY)) { ipu_info(ipu_dev, "ipu%d: %s service table multi read", ipu_dev->filename_id, (enable ? "enabled" : "disabled")); return 0; } ipu_info(ipu_dev, "ipu%d: unable to set service table multi read", ipu_dev->filename_id); } return -EFAULT; } static int ipu_set_hexopt_default_table(struct ipu_device *ipu_dev) { u64 *hexopt_ptr = NULL; u32 hexopt_nb_entries = ipu_dev->hexopt_constants->HEXOPT_NB_ENTRIES; unsigned int i = 0; ipu_info(ipu_dev, "ipu%d: IPU set hexopt to default page", ipu_dev->filename_id); setup_hexopt(ipu_dev); hexopt_ptr = ((u64 *)ipu_dev->exchange); /* Point the HEXOPT to default page. */ for (i = 0; i < hexopt_nb_entries; i++) { if (ipu_dev->supports_dma) hexopt_ptr[i] = ipu_dev->default_hexopt_dma; else hexopt_ptr[i] = ipu_dev->default_hexopt_phys; /* Order the memory writes. See T9238. */ smp_wmb(); } wait_hexopt_updated(ipu_dev, hexopt_nb_entries - 1, ipu_dev->supports_dma ? ipu_dev->default_hexopt_dma : ipu_dev->default_hexopt_phys); return 0; } static int ipu_set_hexopt_identity_table(struct ipu_device *ipu_dev) { u64 *hexopt_ptr = NULL; u32 hexopt_nb_entries = ipu_dev->hexopt_constants->HEXOPT_NB_ENTRIES; unsigned int i = 0; ipu_info(ipu_dev, "ipu%d: IPU set hexopt to identity table", ipu_dev->filename_id); setup_hexopt(ipu_dev); hexopt_ptr = ((u64 *)ipu_dev->exchange); /* In order for hexopt to not adjust the xaddress, bits 33:12 of each * entry should be equal to the page number (an identity mapping). * This is so that the xaddress can be correctly handled by the * later service table mapping. */ for (i = 0; i < hexopt_nb_entries; i++) { hexopt_ptr[i] = i << 12; /* Order the memory writes. See T9238. */ smp_wmb(); } wait_hexopt_updated(ipu_dev, hexopt_nb_entries - 1, (hexopt_nb_entries - 1) << 12); return 0; } static long ipu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { struct ipu_device *ipu_dev = filp->private_data; resource_size_t large_buff_size = 0; long bytes; if (_IOC_TYPE(cmd) != IPU_IOCTL_MAGIC_VAL) return -ENOTTY; switch (cmd) { case IPU_SET_IPU_ID: ipu_dev->user_device_id = arg; break; case IPU_PUT_HSPGS_NOTIFY_ADDR: { struct hspgs_notify_addr *user = (struct hspgs_notify_addr *)arg; struct hspgs_notify_addr addrs; bytes = copy_from_user(&addrs, user, sizeof(struct hspgs_notify_addr)); if (bytes) return -EFAULT; return put_hsp_notify(ipu_dev, &addrs); } case IPU_SELECT_BAR: ipu_select_bar(ipu_dev, arg); break; case IPU_USER_ATTACH: if (!ipu_dev->icu_supports_telemetry) clear_all_sensors(ipu_dev); atomic_set(&ipu_dev->contiguous_mem_handle, 0); ipu_dev->user_pid = get_task_pid(current->group_leader, PIDTYPE_PID); ipu_info_ratelimited(ipu_dev, "ipu%d: user PID attached", ipu_dev->filename_id); break; case IPU_BUFFER_ATTACH: { struct attach_buffer_data *userbd = (struct attach_buffer_data *)arg; struct attach_buffer_data bd; bytes = copy_from_user(&bd, userbd, sizeof(struct attach_buffer_data)); if (bytes) return -EFAULT; return ipu_attach_buffer(ipu_dev, &bd); } break; case IPU_BUFFER_DETACH: return ipu_detach_buffer(ipu_dev, arg); case IPU_BUFFER_DETACH_ALL: ipu_detach_all_buffers(ipu_dev); break; case IPU_GET_CONTIGUOUS_BUFFER: { struct contiguous_mem_list_entry *entry; u64 virt = 0, phys = 0; int cmem_node = get_cmem_node(ipu_dev); if (cmem_node < 0) return -ENOMEM; /* User requests a size from reserved contiguous * memory and we give a handle(key) to the addr */ bytes = copy_from_user(&large_buff_size, (resource_size_t *)arg, sizeof(resource_size_t)); if (bytes) return -EFAULT; entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return -ENOMEM; /* Round up to minimum allocation size */ if (large_buff_size < GENALLOC_SIZE_MINIMUM) large_buff_size = GENALLOC_SIZE_MINIMUM; /* Allocate from our pool, add to list for cleanup */ if (contiguous_mem_alloc(large_buff_size, &virt, &phys, cmem_node) < 0) { kfree(entry); return -ENOMEM; } entry->phys_addr = phys; entry->virt_addr = virt; entry->size = large_buff_size; entry->handle = (u64)atomic_inc_return(&ipu_dev->contiguous_mem_handle); ipu_info_ratelimited(ipu_dev, "ipu%d cmem 0x%llx at 0x%llx", ipu_dev->filename_id, entry->size, entry->phys_addr); ipu_info_ratelimited(ipu_dev, "ipu%d cmem 0x%llx with 0x%llx", ipu_dev->filename_id, entry->virt_addr, entry->handle); list_add(&entry->list, &ipu_dev->contiguous_mem_head); atomic64_add(entry->size, &ipu_dev->hexoatt_used_bytes); /* Now copy address handle to same arg */ if (copy_to_user((void *)arg, (void *)&entry->handle, sizeof(u64)) != 0) { ipu_err(ipu_dev, "ipu%d: failed to get large buff size", ipu_dev->filename_id); return -EFAULT; } } break; case IPU_GET_CONTIGUOUS_BUFFER_ENTRY: { u64 handle; u64 entry; int ret; bytes = copy_from_user(&handle, (u64 *)arg, sizeof(u64)); if (bytes) return -EFAULT; ret = ipu_get_entry_contiguous_mem(ipu_dev, &entry, handle); if (ret < 0 || (copy_to_user((void *)arg, (void *)&entry, sizeof(u64)) != 0)) { ipu_err(ipu_dev, "ipu%d: failed to get buff entry", ipu_dev->filename_id); return -EFAULT; } } break; case IPU_READ_CONTIGUOUS_BUFFER: { struct contiguous_buffer_ops_t read_ops; if (copy_from_user (&read_ops, (struct contiguous_buffer_ops_t *)arg, sizeof(struct contiguous_buffer_ops_t)) != 0) { ipu_info_ratelimited(ipu_dev, "ipu%d: cmem read failed", ipu_dev->filename_id); return -EFAULT; } ipu_read_contiguous_mem(ipu_dev, &read_ops); } break; case IPU_WRITE_CONTIGUOUS_BUFFER: { struct contiguous_buffer_ops_t write_ops; if (copy_from_user (&write_ops, (struct contiguous_buffer_ops_t *)arg, sizeof(struct contiguous_buffer_ops_t)) != 0) { ipu_info_ratelimited(ipu_dev, "ipu%d: cmem write failed", ipu_dev->filename_id); return -EFAULT; } ipu_write_contiguous_mem(ipu_dev, &write_ops); } break; case IPU_PUT_ALL_CONTIGUOUS_BUFFER: ipu_free_all_contiguous_mem(ipu_dev); break; case IPU_PUT_CONTIGUOUS_BUFFER: { u64 handle; if (copy_from_user(&handle, (u64 *)arg, 8) != 0) { ipu_err(ipu_dev, "ipu%d: failed to copy addr", ipu_dev->filename_id); return -EFAULT; } ipu_free_contiguous_mem(ipu_dev, handle); } break; /* TODO T6047 remove this */ case IPU_GET_DRIVER_VERSION: { void *user_version = (void *)arg; if (copy_to_user(user_version, &version, sizeof(struct version_info)) != 0) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to copy version to user space", ipu_dev->filename_id); return -EFAULT; } } break; case IPU_RESTORE_BOARD_INFO: restore_board_info(ipu_dev); break; case IPU_SET_PARITY_INIT_FLAG: ipu_dev->parity_init_flag = arg; ipu_info_ratelimited(ipu_dev, "ipu%d: set parity init flag : %ld", ipu_dev->filename_id, arg); break; case IPU_STORE_SENSOR_READING: { struct ipu_sensor_reading_t *usersr = (struct ipu_sensor_reading_t *)arg; struct ipu_sensor_reading_t sr; bytes = copy_from_user(&sr, usersr, sizeof(struct ipu_sensor_reading_t)); if (bytes) return -EFAULT; ipu_store_sensor_reading(ipu_dev, &sr); } break; case IPU_MASK_INTERRUPT: return ipu_mask_interrupt(ipu_dev, arg); case IPU_UNMASK_INTERRUPT: return ipu_unmask_interrupt(ipu_dev, arg); case IPU_MAILBOX_WRITE_READ: { struct mailbox_arg *user = (struct mailbox_arg *)arg; struct mailbox_arg mbox_args; ssize_t ret_val; struct mailbox_msg msg_for_icu = { {0} }; /* copy structure holding pointers */ ret_val = copy_from_user(&mbox_args, user, sizeof(struct mailbox_arg)); if (ret_val) return -EFAULT; /* copy data to be sent */ ret_val = copy_from_user((void *)&msg_for_icu, mbox_args.sent_msg, sizeof(struct mailbox_msg)); if (ret_val) return ret_val; ret_val = mutex_lock_killable(&ipu_dev->mailbox_available); if (ret_val < 0) return ret_val; ret_val = mailbox_write(ipu_dev, &msg_for_icu); if (ret_val == 0) { struct mailbox_msg msg_from_icu = { {0} }; mailbox_read(ipu_dev, &msg_from_icu); ret_val = copy_to_user(mbox_args.recv_msg, &msg_from_icu, sizeof(struct mailbox_msg)); } mutex_unlock(&ipu_dev->mailbox_available); return ret_val; } break; case IPU_MCU_GET_EVENT: return -EFAULT; case IPU_RESET: { ipu_info(ipu_dev, "ipu%d: IPU reset (%lu)", ipu_dev->filename_id, arg); ipu_dev->check_parity_errors = true; /* * Accumulate all NLC correctable error counters and * reset the session's total. */ mutex_lock(&ipu_dev->nlc_mutex); ipu_accumulate_nlc_errcnt_unsafe(ipu_dev); ipu_dev->nlc_total_errcnt_session = 0; mutex_unlock(&ipu_dev->nlc_mutex); ipu_info(ipu_dev, "ipu%d: session's NLC correctable error count reset", ipu_dev->filename_id); if (check_spb_supported(ipu_dev)) { /* * Accumulate error packet counters in SPB and * reset the session's total. */ mutex_lock(&ipu_dev->spb_mutex); spb_error_pkt_count_update_unsafe(ipu_dev); ipu_dev->spb_error_pkt_count_session = 0; mutex_unlock(&ipu_dev->spb_mutex); ipu_info(ipu_dev, "ipu%d: session's error packet count in SPB reset", ipu_dev->filename_id); } } break; case IPU_STOP_MONITORING_THREADS: { stop_tile_clk_speed_thread(&ipu_dev->pci_dev->dev); } break; case IPU_START_MONITORING_THREADS: { start_tile_clk_speed_thread(&ipu_dev->pci_dev->dev); } break; case IPU_ACCUMULATE_NLC_ERRCNT: { mutex_lock(&ipu_dev->nlc_mutex); ipu_accumulate_nlc_errcnt_unsafe(ipu_dev); mutex_unlock(&ipu_dev->nlc_mutex); } break; case IPU_READ_AER_DATA: { aer_data_print(ipu_dev); if (copy_to_user((void *)arg, &ipu_dev->aer_data, sizeof(u32)) != 0) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to aer data to user space", ipu_dev->filename_id); return -EFAULT; }; } break; case IPU_ENABLE_GWLINKS: return enable_gwlinks(ipu_dev, arg); case IPU_SET_THROTTLE_LOG_THRESHOLDS: { struct throttle_log_thresholds_t *user_tlt = (struct throttle_log_thresholds_t *)arg; struct throttle_log_thresholds_t tlt; bytes = copy_from_user(&tlt, user_tlt, sizeof(struct throttle_log_thresholds_t)); if (bytes) return -EFAULT; atomic_set(&ipu_dev->clk_throttle_threshold1, tlt.level1); atomic_set(&ipu_dev->clk_throttle_threshold2, tlt.level2); atomic_set(&ipu_dev->clk_throttle_threshold3, tlt.level3); ipu_info_ratelimited(ipu_dev, "ipu%d: set clock throttle " "log thresholds %lu,%lu,%lu", ipu_dev->filename_id, tlt.level1, tlt.level2, tlt.level3); } break; case IPU_CLEAR_SPB_SERVICE_TABLES: return clear_all_spb_service_tables(ipu_dev); case IPU_SECONDARY_CONFIG_OPS: { if (ipu_dev->board_type == IPU_BOARD_TYPE_C600) { struct secondary_config_reg_ops *user = (struct secondary_config_reg_ops *)arg; ssize_t ret_val; ssize_t sz = sizeof(struct secondary_config_reg_ops); struct secondary_config_reg_ops cfg; /* copy structure holding pointers */ ret_val = copy_from_user(&cfg, user, sz); if (ret_val == 0 && ipu_dev->config_sec) { if (cfg.write) writel(cfg.value, ipu_dev->config_sec + cfg.offset); if (cfg.read) { cfg.value = readl(ipu_dev->config_sec + cfg.offset); copy_to_user((void *)arg, (void *)&cfg, sz); } break; } } return -EFAULT; } case IPU_SET_SECONDARY_IPUID: { if (ipu_dev->board_type == IPU_BOARD_TYPE_C600) { u32 ipu_id; bytes = copy_from_user(&ipu_id, (u32 *)arg, sizeof(u32)); if (bytes) return -EFAULT; int ret_val = icu_mailbox_transaction(ipu_dev, (u32[]) {ICU_CMD_SET_IPUID, // secondary complex(1) 6th byte // ipuid 5th byte ((1 << 8) | (ipu_id)), 0, 0, 0}, NULL, 0); if (ret_val == 0 && setup_hexopt_secondary(ipu_dev) == 0) break; } return -EFAULT; } case IPU_SET_USER_DATA: ipu_dev->user_data = arg; ipu_info_ratelimited(ipu_dev, "ipu%d: set user data : %lu", ipu_dev->filename_id, arg); break; case IPU_GET_USER_DATA: if (copy_to_user((void *)arg, &ipu_dev->user_data, sizeof(ipu_dev->user_data)) != 0) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to copy user data to user space", ipu_dev->filename_id); return -EFAULT; } break; case IPU_SET_SPB_MULTI_READ_ST_ENABLE: return set_spb_multi_read_st_enable(ipu_dev, arg); case IPU_SET_HEXOPT_IDENTITY_TABLE: return ipu_set_hexopt_identity_table(ipu_dev); default: dev_err(&ipu_dev->pci_dev->dev, "ipu%d: invalid IOCTL cmd %x", ipu_dev->filename_id, cmd); return -ENOTTY; } return 0; } static int ipu_open_config(struct inode *inode, struct file *filp) { struct ipu_device *dev; dev = container_of(inode->i_cdev, struct ipu_device, cdev[CHRDEV_CONFIG]); ipu_info_ratelimited(dev, "ipu%d: open(config)", dev->filename_id); if (down_trylock(&dev->config_available)) return -EBUSY; filp->private_data = dev; dev->user_pid = NULL; return 0; } #ifdef CONFIG_PCI_P2PDMA static int ipu_p2p_open_exchange(struct inode *inode, struct file *filp) { struct ipu_device *dev; dev = container_of(inode->i_cdev, struct ipu_device, cdev[CHRDEV_P2P_EXCHANGE]); ipu_info_ratelimited(dev, "ipu%d: open(p2p_exchange)", dev->filename_id); if (down_trylock(&dev->exchange_available)) return -EBUSY; filp->private_data = dev; return 0; } #endif static int ipu_open_exchange(struct inode *inode, struct file *filp) { struct ipu_device *dev; dev = container_of(inode->i_cdev, struct ipu_device, cdev[CHRDEV_EXCHANGE]); ipu_info_ratelimited(dev, "ipu%d: open(exchange)", dev->filename_id); if (down_trylock(&dev->exchange_available)) return -EBUSY; filp->private_data = dev; return 0; } static int ipu_open_memory(struct inode *inode, struct file *filp) { struct ipu_device *dev; dev = container_of(inode->i_cdev, struct ipu_device, cdev[CHRDEV_MEMORY]); ipu_info_ratelimited(dev, "ipu%d: open(memory)", dev->filename_id); if (down_trylock(&dev->memory_available)) return -EBUSY; filp->private_data = dev; return 0; } int ipu_release(struct inode *inode, struct file *filp) { struct ipu_device *dev = filp->private_data; if (dev->config) { ipu_detach_all_buffers(dev); // ensure all hexopt entries are set to the default page, // in case set_hexopt_identity_table was called ipu_set_hexopt_default_table(dev); } free_hsp_buffers(dev); if (iminor(inode) == CHRDEV_CONFIG) { ipu_unmask_interrupt(dev, IPU_ALL_INTERRUPTS); /* If config space is unmapped, * no OATT access, so disable OATT */ if (dev->config) { writel(0, dev->config); writel(0, dev->config + 4); } ipu_free_all_contiguous_mem(dev); if (dev->user_pid) { put_pid(dev->user_pid); dev->user_pid = NULL; } up(&dev->config_available); icu_send_board_detach(dev); } else if (iminor(inode) == CHRDEV_EXCHANGE) { up(&dev->exchange_available); } else if (iminor(inode) == CHRDEV_MEMORY) { up(&dev->memory_available); #ifdef CONFIG_PCI_P2PDMA } else if (iminor(inode) == CHRDEV_P2P_EXCHANGE) { up(&dev->exchange_available); #endif } return 0; } #ifdef CONFIG_PCI_P2PDMA const struct vm_operations_struct ipu_p2p_vmops = { .open = ipu_p2p_vma_open, .close = ipu_p2p_vma_close, .fault = ipu_p2p_vma_fault, }; static void ipu_p2p_vma_open(struct vm_area_struct *vma) { struct ipu_p2p_vma *pv = vma->vm_private_data; if (pv) ipu_info_ratelimited(pv->ipu_dev, "ipu%d: vma open(p2p_exchange)", pv->ipu_dev->filename_id); } static vm_fault_t ipu_p2p_vma_fault(struct vm_fault *vmf) { vm_fault_t rc = 0; struct page *pg; struct ipu_p2p_vma *pv = vmf->vma->vm_private_data; unsigned long phys_addr; unsigned long pfn; unsigned int pg_idx = (vmf->address - vmf->vma->vm_start) / PAGE_SIZE; mutex_lock(&pv->mutex); pg = virt_to_page(pci_alloc_p2pmem(pv->ipu_dev->pci_dev, PAGE_SIZE)); if (!pg) { mutex_unlock(&pv->mutex); return VM_FAULT_OOM; } pv->used_pages[pg_idx] = pg; phys_addr = page_to_phys(pg); pfn = PHYS_PFN(phys_addr); vmf->page = pg; rc = vmf_insert_mixed(vmf->vma, vmf->address, pfn); ipu_info_ratelimited(pv->ipu_dev, "ipu%d: va=0x%lx pa=%pad pfn=%lx", pv->ipu_dev->filename_id, vmf->address, &phys_addr, pfn); mutex_unlock(&pv->mutex); return rc; } static void ipu_p2p_vma_free_pages(struct vm_area_struct *vma) { int i; struct ipu_p2p_vma *pv = vma->vm_private_data; mutex_lock(&pv->mutex); for (i = 0; i < pv->nr_pages; i++) { if (pv->used_pages[i]) { pci_free_p2pmem(pv->ipu_dev->pci_dev, page_to_virt(pv->used_pages[i]), PAGE_SIZE); pv->used_pages[i] = NULL; } } mutex_unlock(&pv->mutex); } static void ipu_p2p_vma_close(struct vm_area_struct *vma) { struct ipu_p2p_vma *pv = vma->vm_private_data; if (pv) { ipu_p2p_vma_free_pages(vma); kfree(pv); vma->vm_private_data = NULL; } } static int ipu_p2p_mmap(struct file *filp, struct vm_area_struct *vma) { struct ipu_device *dev = filp->private_data; struct ipu_p2p_vma *pv; size_t nr_pages = (vma->vm_end - vma->vm_start) / PAGE_SIZE; if (!nr_pages) nr_pages = 1; pv = kzalloc((sizeof(*pv) + nr_pages * sizeof(struct page *)), GFP_KERNEL); if (!pv) return -ENOMEM; ipu_info_ratelimited(dev, "ipu%d: va_start=0x%lx va_end=0x%lx nr_pages=%zu", dev->filename_id, vma->vm_start, vma->vm_end, nr_pages); mutex_init(&pv->mutex); pv->nr_pages = nr_pages; pv->ipu_dev = dev; vma->vm_private_data = pv; vma->vm_ops = &ipu_p2p_vmops; return 0; } #endif static int ipu_mmap(struct file *filp, struct vm_area_struct *vma) { unsigned long size; struct ipu_device *dev = filp->private_data; unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; size = vma->vm_end - vma->vm_start; if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_CONFIG) { offset += (unsigned long)dev->config_start; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); } else if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_EXCHANGE) { offset += (unsigned long)dev->exchange_start; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); } else if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_MEMORY) { offset += (unsigned long)dev->memory_start; } ipu_info_ratelimited(dev, "ipu%d: IPU mmap address 0x%lx, size %ld", dev->filename_id, offset, size); if (remap_pfn_range(vma, vma->vm_start, offset >> PAGE_SHIFT, size, vma->vm_page_prot)) { dev_err(&dev->pci_dev->dev, "ipu%d: failed to map", dev->filename_id); return -EAGAIN; } return 0; } static loff_t ipu_seek(struct file *filp, loff_t off, int whence) { loff_t size; loff_t newpos; struct ipu_device *dev = filp->private_data; if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_CONFIG) size = dev->config_size; else if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_EXCHANGE) size = dev->exchange_size; else if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_MEMORY) size = dev->memory_size; else return -EINVAL; switch (whence) { case SEEK_SET: newpos = off; break; case SEEK_CUR: newpos = filp->f_pos + off; break; case SEEK_END: newpos = size + off; break; default: return -EINVAL; } if (newpos > size) newpos = size; if (newpos < 0) return -EINVAL; filp->f_pos = newpos; return newpos; } static ssize_t ipu_read(struct file *filp, __user char *buff, size_t count, loff_t *ppos) { unsigned int bytes_not_copied = 0; struct ipu_device *dev = (struct ipu_device *)filp->private_data; if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_EXCHANGE) { return -EPERM; } else if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_CONFIG) { /* Adjust count if beyond target memory size. */ if ((*ppos) + count > dev->config_size) count = dev->config_size - (*ppos); if (count > 0) { memcpy_fromio(dev->config_buffer + *ppos, dev->config + *ppos, count); bytes_not_copied = copy_to_user(buff, dev->config_buffer + *ppos, count); *ppos += (count - bytes_not_copied); } return count - bytes_not_copied; } else if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_MEMORY) { if ((*ppos) + count > dev->memory_size) count = dev->memory_size - (*ppos); if (!dev->memory) return -EFAULT; if (count > 0) { bytes_not_copied = copy_to_user(buff, dev->memory + *ppos, count); *ppos += (count - bytes_not_copied); } return count - bytes_not_copied; } else { return -EINVAL; } } static ssize_t ipu_write(struct file *filp, __user const char *buff, size_t count, loff_t *ppos) { struct ipu_device *dev = filp->private_data; resource_size_t size; void __iomem *mem; if (MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_CONFIG) { size = dev->config_size; mem = dev->config; } else if ((MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_EXCHANGE)) { size = dev->exchange_size; mem = dev->exchange; } else if ((MINOR(filp->f_path.dentry->d_inode->i_rdev) == CHRDEV_MEMORY)) { size = dev->exchange_size; mem = dev->memory; } else { return -EINVAL; } if (!mem) return -EFAULT; /* Adjust count if beyond target memory size. */ if ((*ppos) + count > size) count = size - (*ppos); if (count > 0) (*ppos) += copy_from_user(mem + (*ppos), buff, count); return count; } int ipu_chrdev_init(struct ipu_device *ipu_dev) { unsigned int driver_major = IPU_CHRDEV_MAJOR; unsigned int driver_minor = IPU_CHRDEV_MINOR; struct page *page = NULL; void *dev = NULL; int err = 0, i = 0; u64 *hexopt_ptr = NULL; ipu_dev->user_pid = NULL; ipu_dev->user_device_id = 0; ipu_dev->buffers_tree = RB_ROOT; ipu_dev->parity_init_flag = 0; ipu_dev->user_data = 0; ipu_dev->spb_acc = NULL; ipu_dev->hsp_notify[0] = NULL; ipu_dev->hsp_notify[1] = NULL; atomic64_set(&ipu_dev->hexopt_used_bytes, 0); atomic64_set(&ipu_dev->hexoatt_used_bytes, 0); ipu_dev->prev_clk_throttle_level = 0; atomic_set(&ipu_dev->clk_throttle_threshold1, 0); atomic_set(&ipu_dev->clk_throttle_threshold2, 0); atomic_set(&ipu_dev->clk_throttle_threshold3, 0); ipu_dev->power_save_active = false; if (PAGE_SIZE != ipu_dev->hexopt_constants->HEXOPT_PAGE_SIZE) { dev_err(&ipu_dev->pci_dev->dev, "unexpected page size"); return -EOPNOTSUPP; } /* Allocate memory to be used for standard file IO. */ ipu_dev->config_buffer = kzalloc(ipu_dev->config_size, GFP_KERNEL); if (!ipu_dev->config_buffer) return -ENOMEM; /* Allocate the default HEXOPT page, pin it and store it's address. */ ipu_dev->default_page_vptr = vmalloc(PAGE_SIZE); if (!ipu_dev->default_page_vptr) return -ENOMEM; page = vmalloc_to_page(ipu_dev->default_page_vptr); get_page(page); if (ipu_dev->supports_dma) { struct device *d = &ipu_dev->pci_dev->dev; ipu_dev->default_hexopt_dma = dma_map_page(d, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); ipu_info_ratelimited(ipu_dev, "The default hexopt DMA address is 0x%llx", ipu_dev->default_hexopt_dma); } else { ipu_dev->default_hexopt_phys = page_to_phys(page); ipu_info_ratelimited(ipu_dev, "The default hexopt PHYS address is 0x%llx", ipu_dev->default_hexopt_phys); } setup_hexopt(ipu_dev); /* Initialise the HEXOPT to the default address. */ hexopt_ptr = ((u64 *)ipu_dev->exchange); /* To speed up qemu boots, avoid this */ if (!dmi_match(DMI_SYS_VENDOR, "QEMU")) { ipu_info_ratelimited(ipu_dev, "Filling Hexopt table to default"); for (i = 0; i < ipu_dev->hexopt_constants->HEXOPT_NB_ENTRIES; i++) { if (ipu_dev->supports_dma) hexopt_ptr[i] = ipu_dev->default_hexopt_dma; else hexopt_ptr[i] = ipu_dev->default_hexopt_phys; smp_wmb(); /* Order the memory writes. See T9238. */ } } else { ipu_info_ratelimited(ipu_dev, "Skipping hexopt default for QEMU"); } ipu_info_ratelimited(ipu_dev, "hexopt init done"); /* Hard-coded values - easier during development. */ if (driver_major != 0) { ipu_dev->device_id = MKDEV(driver_major, driver_minor); err = register_chrdev_region(ipu_dev->device_id, CHRDEV_NUM_FILES, IPU_MODULE_NAME); } else { /* Dynamically allocate device numbers. */ err = alloc_chrdev_region(&ipu_dev->device_id, 0, CHRDEV_NUM_FILES, IPU_MODULE_NAME); driver_major = MAJOR(ipu_dev->device_id); driver_minor = MINOR(ipu_dev->device_id); } if (err) { dev_err(&ipu_dev->pci_dev->dev, "failed to register device number %d:%d", driver_major, driver_minor); goto device_number_error; } ipu_dev->filename_id = idr_alloc(ipu_dev->idr, ipu_dev, 0, 0, GFP_KERNEL); if (ipu_dev->filename_id < 0) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to allocate device id", ipu_dev->filename_id); err = ipu_dev->filename_id; goto id_alloc_error; } for (i = 0; i < CHRDEV_NUM_FILES; i++) { char *suffix[CHRDEV_NUM_FILES] = { "", "_ex", "_mem" #ifdef CONFIG_PCI_P2PDMA , "_p2p" #endif }; const struct file_operations *fops[CHRDEV_NUM_FILES] = { &ipu_fops, &ipu_exchange_fops, &ipu_memory_fops #ifdef CONFIG_PCI_P2PDMA , &ipu_p2p_exchange_fops #endif }; int devno = MKDEV(driver_major, i); cdev_init(&ipu_dev->cdev[i], fops[i]); ipu_dev->cdev[i].owner = THIS_MODULE; ipu_dev->cdev[i].ops = fops[i]; err = cdev_add(&ipu_dev->cdev[i], devno, 1); if (err) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to add device number %d:%d", ipu_dev->filename_id, driver_major, i); goto cdev_add_error; } ipu_info_ratelimited(ipu_dev, "ipu%d: creating file %s%d%s", ipu_dev->filename_id, IPU_MODULE_NAME, ipu_dev->filename_id, suffix[i]); dev = device_create(ipu_dev->cl, &ipu_dev->pci_dev->dev, devno, NULL, IPU_MODULE_NAME "%d%s", ipu_dev->filename_id, suffix[i]); if (!dev) { dev_err(&ipu_dev->pci_dev->dev, "ipu%d: failed to create fs node", ipu_dev->filename_id); err = PTR_ERR(dev); goto device_create_error; } } INIT_LIST_HEAD(&ipu_dev->contiguous_mem_head); setup_dma_for_hsp_notify(ipu_dev); if (ipu_dev->board_type == IPU_BOARD_TYPE_M2000 && ipu_dev->filename_id < 4) { ipu_dev->spb_acc = ioremap(spb_acc_base_addr[ipu_dev->filename_id], 0x200); } return 0; device_create_error: idr_remove(ipu_dev->idr, ipu_dev->filename_id); id_alloc_error: idr_remove(ipu_dev->idr, ipu_dev->filename_id); for (i = 0; i < CHRDEV_NUM_FILES; i++) cdev_del(&ipu_dev->cdev[i]); cdev_add_error: unregister_chrdev_region(ipu_dev->device_id, CHRDEV_NUM_FILES); device_number_error: kfree(ipu_dev->config_buffer); return err; } void ipu_chrdev_exit(struct ipu_device *ipu_dev) { struct page *page = NULL; int i = 0; if (!ipu_dev->is_initialised) return; if (ipu_dev->spb_acc) iounmap(ipu_dev->spb_acc); restore_board_info(ipu_dev); teardown_dma_for_hsp_notify(ipu_dev); if (ipu_dev->supports_dma) dma_unmap_page(&ipu_dev->pci_dev->dev, ipu_dev->default_hexopt_dma, PAGE_SIZE, DMA_BIDIRECTIONAL); page = vmalloc_to_page(ipu_dev->default_page_vptr); set_page_dirty_lock(page); put_page(page); vfree(ipu_dev->default_page_vptr); kfree(ipu_dev->config_buffer); put_pid(ipu_dev->user_pid); ipu_dev->user_pid = NULL; ipu_dev->user_device_id = 0; ipu_detach_all_buffers(ipu_dev); device_destroy(ipu_dev->cl, ipu_dev->device_id); for (i = 1; i < CHRDEV_NUM_FILES; i++) { device_destroy(ipu_dev->cl, MKDEV(MAJOR(ipu_dev->device_id), i)); } idr_remove(ipu_dev->idr, ipu_dev->filename_id); for (i = 0; i < CHRDEV_NUM_FILES; i++) cdev_del(&ipu_dev->cdev[i]); if (ipu_dev->device_id != 0) unregister_chrdev_region(ipu_dev->device_id, CHRDEV_NUM_FILES); }