/* * Copyright (c) 2022 Intel Corporation * SPDX-License-Identifier: Apache-2.0 * * Derived from FreeBSD original driver made by Jim Harris * with contributions from Alexander Motin and Wojciech Macek */ #ifndef ZEPHYR_DRIVERS_DISK_NVME_NHME_H_ #define ZEPHYR_DRIVERS_DISK_NVME_NVME_H_ #include "nvme_helpers.h" #include "nvme_cmd.h" #include "nvme_namespace.h" struct nvme_registers { uint32_t cap_lo; /* controller capabilities */ uint32_t cap_hi; uint32_t vs; /* version */ uint32_t intms; /* interrupt mask set */ uint32_t intmc; /* interrupt mask clear */ uint32_t cc; /* controller configuration */ uint32_t reserved1; uint32_t csts; /* controller status */ uint32_t nssr; /* NVM Subsystem Reset */ uint32_t aqa; /* admin queue attributes */ uint64_t asq; /* admin submission queue base addr */ uint64_t acq; /* admin completion queue base addr */ uint32_t cmbloc; /* Controller Memory Buffer Location */ uint32_t cmbsz; /* Controller Memory Buffer Size */ uint32_t bpinfo; /* Boot Partition Information */ uint32_t bprsel; /* Boot Partition Read Select */ uint64_t bpmbl; /* Boot Partition Memory Buffer Location */ uint64_t cmbmsc; /* Controller Memory Buffer Memory Space Control */ uint32_t cmbsts; /* Controller Memory Buffer Status */ uint8_t reserved3[3492]; /* 5Ch - DFFh */ uint32_t pmrcap; /* Persistent Memory Capabilities */ uint32_t pmrctl; /* Persistent Memory Region Control */ uint32_t pmrsts; /* Persistent Memory Region Status */ uint32_t pmrebs; /* Persistent Memory Region Elasticity Buffer Size */ uint32_t pmrswtp; /* Persistent Memory Region Sustained Write Throughput */ uint32_t pmrmsc_lo; /* Persistent Memory Region Controller Memory Space Control */ uint32_t pmrmsc_hi; uint8_t reserved4[484]; /* E1Ch - FFFh */ struct { uint32_t sq_tdbl; /* submission queue tail doorbell */ uint32_t cq_hdbl; /* completion queue head doorbell */ } doorbell[1]; }; struct nvme_power_state { /** Maximum Power */ uint16_t mp; uint8_t ps_rsvd1; /** Max Power Scale, Non-Operational State */ uint8_t mps_nops; /** Entry Latency */ uint32_t enlat; /** Exit Latency */ uint32_t exlat; /** Relative Read Throughput */ uint8_t rrt; /** Relative Read Latency */ uint8_t rrl; /** Relative Write Throughput */ uint8_t rwt; /** Relative Write Latency */ uint8_t rwl; /** Idle Power */ uint16_t idlp; /** Idle Power Scale */ uint8_t ips; uint8_t ps_rsvd8; /** Active Power */ uint16_t actp; /** Active Power Workload, Active Power Scale */ uint8_t apw_aps; uint8_t ps_rsvd10[9]; } __packed; #define NVME_SERIAL_NUMBER_LENGTH 20 #define NVME_MODEL_NUMBER_LENGTH 40 #define NVME_FIRMWARE_REVISION_LENGTH 8 struct nvme_controller_data { /* bytes 0-255: controller capabilities and features */ /** pci vendor id */ uint16_t vid; /** pci subsystem vendor id */ uint16_t ssvid; /** serial number */ uint8_t sn[NVME_SERIAL_NUMBER_LENGTH]; /** model number */ uint8_t mn[NVME_MODEL_NUMBER_LENGTH]; /** firmware revision */ uint8_t fr[NVME_FIRMWARE_REVISION_LENGTH]; /** recommended arbitration burst */ uint8_t rab; /** ieee oui identifier */ uint8_t ieee[3]; /** multi-interface capabilities */ uint8_t mic; /** maximum data transfer size */ uint8_t mdts; /** Controller ID */ uint16_t ctrlr_id; /** Version */ uint32_t ver; /** RTD3 Resume Latency */ uint32_t rtd3r; /** RTD3 Enter Latency */ uint32_t rtd3e; /** Optional Asynchronous Events Supported */ uint32_t oaes; /* bitfield really */ /** Controller Attributes */ uint32_t ctratt; /* bitfield really */ /** Read Recovery Levels Supported */ uint16_t rrls; uint8_t reserved1[9]; /** Controller Type */ uint8_t cntrltype; /** FRU Globally Unique Identifier */ uint8_t fguid[16]; /** Command Retry Delay Time 1 */ uint16_t crdt1; /** Command Retry Delay Time 2 */ uint16_t crdt2; /** Command Retry Delay Time 3 */ uint16_t crdt3; uint8_t reserved2[122]; /* bytes 256-511: admin command set attributes */ /** optional admin command support */ uint16_t oacs; /** abort command limit */ uint8_t acl; /** asynchronous event request limit */ uint8_t aerl; /** firmware updates */ uint8_t frmw; /** log page attributes */ uint8_t lpa; /** error log page entries */ uint8_t elpe; /** number of power states supported */ uint8_t npss; /** admin vendor specific command configuration */ uint8_t avscc; /** Autonomous Power State Transition Attributes */ uint8_t apsta; /** Warning Composite Temperature Threshold */ uint16_t wctemp; /** Critical Composite Temperature Threshold */ uint16_t cctemp; /** Maximum Time for Firmware Activation */ uint16_t mtfa; /** Host Memory Buffer Preferred Size */ uint32_t hmpre; /** Host Memory Buffer Minimum Size */ uint32_t hmmin; /** Name space capabilities */ struct { /* if nsmgmt, report tnvmcap and unvmcap */ uint8_t tnvmcap[16]; uint8_t unvmcap[16]; } __packed untncap; /** Replay Protected Memory Block Support */ uint32_t rpmbs; /* Really a bitfield */ /** Extended Device Self-test Time */ uint16_t edstt; /** Device Self-test Options */ uint8_t dsto; /* Really a bitfield */ /** Firmware Update Granularity */ uint8_t fwug; /** Keep Alive Support */ uint16_t kas; /** Host Controlled Thermal Management Attributes */ uint16_t hctma; /* Really a bitfield */ /** Minimum Thermal Management Temperature */ uint16_t mntmt; /** Maximum Thermal Management Temperature */ uint16_t mxtmt; /** Sanitize Capabilities */ uint32_t sanicap; /* Really a bitfield */ /** Host Memory Buffer Minimum Descriptor Entry Size */ uint32_t hmminds; /** Host Memory Maximum Descriptors Entries */ uint16_t hmmaxd; /** NVM Set Identifier Maximum */ uint16_t nsetidmax; /** Endurance Group Identifier Maximum */ uint16_t endgidmax; /** ANA Transition Time */ uint8_t anatt; /** Asymmetric Namespace Access Capabilities */ uint8_t anacap; /** ANA Group Identifier Maximum */ uint32_t anagrpmax; /** Number of ANA Group Identifiers */ uint32_t nanagrpid; /** Persistent Event Log Size */ uint32_t pels; uint8_t reserved3[156]; /* bytes 512-703: nvm command set attributes */ /** submission queue entry size */ uint8_t sqes; /** completion queue entry size */ uint8_t cqes; /** Maximum Outstanding Commands */ uint16_t maxcmd; /** number of namespaces */ uint32_t nn; /** optional nvm command support */ uint16_t oncs; /** fused operation support */ uint16_t fuses; /** format nvm attributes */ uint8_t fna; /** volatile write cache */ uint8_t vwc; /** Atomic Write Unit Normal */ uint16_t awun; /** Atomic Write Unit Power Fail */ uint16_t awupf; /** NVM Vendor Specific Command Configuration */ uint8_t nvscc; /** Namespace Write Protection Capabilities */ uint8_t nwpc; /** Atomic Compare & Write Unit */ uint16_t acwu; uint16_t reserved6; /** SGL Support */ uint32_t sgls; /** Maximum Number of Allowed Namespaces */ uint32_t mnan; /* bytes 540-767: Reserved */ uint8_t reserved7[224]; /** NVM Subsystem NVMe Qualified Name */ uint8_t subnqn[256]; /* bytes 1024-1791: Reserved */ uint8_t reserved8[768]; /* bytes 1792-2047: NVMe over Fabrics specification */ uint8_t reserved9[256]; /* bytes 2048-3071: power state descriptors */ struct nvme_power_state power_state[32]; /* bytes 3072-4095: vendor specific */ uint8_t vs[1024]; } __packed __aligned(4); static inline void nvme_controller_data_swapbytes(struct nvme_controller_data *s) { #if _BYTE_ORDER != _LITTLE_ENDIAN s->vid = sys_le16_to_cpu(s->vid); s->ssvid = sys_le16_to_cpu(s->ssvid); s->ctrlr_id = sys_le16_to_cpu(s->ctrlr_id); s->ver = sys_le32_to_cpu(s->ver); s->rtd3r = sys_le32_to_cpu(s->rtd3r); s->rtd3e = sys_le32_to_cpu(s->rtd3e); s->oaes = sys_le32_to_cpu(s->oaes); s->ctratt = sys_le32_to_cpu(s->ctratt); s->rrls = sys_le16_to_cpu(s->rrls); s->crdt1 = sys_le16_to_cpu(s->crdt1); s->crdt2 = sys_le16_to_cpu(s->crdt2); s->crdt3 = sys_le16_to_cpu(s->crdt3); s->oacs = sys_le16_to_cpu(s->oacs); s->wctemp = sys_le16_to_cpu(s->wctemp); s->cctemp = sys_le16_to_cpu(s->cctemp); s->mtfa = sys_le16_to_cpu(s->mtfa); s->hmpre = sys_le32_to_cpu(s->hmpre); s->hmmin = sys_le32_to_cpu(s->hmmin); s->rpmbs = sys_le32_to_cpu(s->rpmbs); s->edstt = sys_le16_to_cpu(s->edstt); s->kas = sys_le16_to_cpu(s->kas); s->hctma = sys_le16_to_cpu(s->hctma); s->mntmt = sys_le16_to_cpu(s->mntmt); s->mxtmt = sys_le16_to_cpu(s->mxtmt); s->sanicap = sys_le32_to_cpu(s->sanicap); s->hmminds = sys_le32_to_cpu(s->hmminds); s->hmmaxd = sys_le16_to_cpu(s->hmmaxd); s->nsetidmax = sys_le16_to_cpu(s->nsetidmax); s->endgidmax = sys_le16_to_cpu(s->endgidmax); s->anagrpmax = sys_le32_to_cpu(s->anagrpmax); s->nanagrpid = sys_le32_to_cpu(s->nanagrpid); s->pels = sys_le32_to_cpu(s->pels); s->maxcmd = sys_le16_to_cpu(s->maxcmd); s->nn = sys_le32_to_cpu(s->nn); s->oncs = sys_le16_to_cpu(s->oncs); s->fuses = sys_le16_to_cpu(s->fuses); s->awun = sys_le16_to_cpu(s->awun); s->awupf = sys_le16_to_cpu(s->awupf); s->acwu = sys_le16_to_cpu(s->acwu); s->sgls = sys_le32_to_cpu(s->sgls); s->mnan = sys_le32_to_cpu(s->mnan); #else ARG_UNUSED(s); #endif } #include #include #include #define NVME_PCIE_BAR_IDX 0 #define NVME_REQUEST_AMOUNT (CONFIG_NVME_ADMIN_ENTRIES + \ CONFIG_NVME_IO_ENTRIES) /* admin queue + io queue(s) */ #define NVME_PCIE_MSIX_VECTORS 1 + CONFIG_NVME_IO_QUEUES #define NVME_QUEUE_ALLOCATE(name, n_entries) \ static struct nvme_command cmd_##name[n_entries] __aligned(0x1000); \ static struct nvme_completion cpl_##name[n_entries] __aligned(0x1000); \ \ static struct nvme_cmd_qpair name = { \ .num_entries = n_entries, \ .cmd = cmd_##name, \ .cpl = cpl_##name, \ } #define NVME_ADMINQ_ALLOCATE(n, n_entries) \ NVME_QUEUE_ALLOCATE(admin_##n, n_entries) #define NVME_IOQ_ALLOCATE(n, n_entries) \ NVME_QUEUE_ALLOCATE(io_##n, n_entries) struct nvme_controller_config { struct pcie_dev *pcie; }; struct nvme_controller { DEVICE_MMIO_RAM; const struct device *dev; struct k_mutex lock; uint32_t id; msi_vector_t vectors[NVME_PCIE_MSIX_VECTORS]; struct nvme_controller_data cdata; uint32_t num_io_queues; struct nvme_cmd_qpair *adminq; struct nvme_cmd_qpair *ioq; uint32_t ready_timeout_in_ms; /** LO and HI capacity mask */ uint32_t cap_lo; uint32_t cap_hi; /** Page size and log2(page_size) - 12 that we're currently using */ uint32_t page_size; uint32_t mps; /** doorbell stride */ uint32_t dstrd; /** maximum i/o size in bytes */ uint32_t max_xfer_size; struct nvme_namespace ns[CONFIG_NVME_MAX_NAMESPACES]; }; static inline bool nvme_controller_has_dataset_mgmt(struct nvme_controller *ctrlr) { /* Assumes cd was byte swapped by nvme_controller_data_swapbytes() */ return ((ctrlr->cdata.oncs >> NVME_CTRLR_DATA_ONCS_DSM_SHIFT) & NVME_CTRLR_DATA_ONCS_DSM_MASK); } static inline void nvme_lock(const struct device *dev) { struct nvme_controller *nvme_ctrlr = dev->data; k_mutex_lock(&nvme_ctrlr->lock, K_FOREVER); } static inline void nvme_unlock(const struct device *dev) { struct nvme_controller *nvme_ctrlr = dev->data; k_mutex_unlock(&nvme_ctrlr->lock); } #endif /* ZEPHYR_DRIVERS_DISK_NVME_NHME_H_ */