From abab90726fc25562a8c80bdc68546d03c5034036 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Sun, 20 Aug 2023 20:31:33 +0200 Subject: [PATCH 01/23] mmio: convert calls of 'addr + offset' to 'base-addr, offset' Changing the mmio read/write functions to have a separate base-address and offset. This enables macOS/DriverKit to override the mmio_X functions elegantly. Signed-off-by: Mads Ynddal --- examples/cmb-p2p.c | 10 ++++---- examples/cmb.c | 12 +++++----- examples/regs.c | 48 +++++++++++++++++++------------------- include/vfn/nvme/queue.h | 4 ++-- include/vfn/support/mmio.h | 43 +++++++++++++++++++--------------- src/nvme/core.c | 26 ++++++++++----------- 6 files changed, 74 insertions(+), 69 deletions(-) diff --git a/examples/cmb-p2p.c b/examples/cmb-p2p.c index bda13f03..b5ebf3e8 100644 --- a/examples/cmb-p2p.c +++ b/examples/cmb-p2p.c @@ -59,13 +59,13 @@ static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *hwaddr, uint64 struct iommu_iova_range *iova_ranges; int num_iova_ranges; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); if (NVME_GET(cap, CAP_CMBS)) - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(0x1)); + mmio_hl_write64(ctrl->regs, NVME_REG_CMBMSC, cpu_to_le64(0x1)); - cmbsz = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBSZ)); - cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC)); + cmbsz = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CMBSZ)); + cmbloc = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CMBLOC)); szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU)); len = szu * NVME_GET(cmbsz, CMBSZ_SZ); @@ -103,7 +103,7 @@ static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *hwaddr, uint64 printf("assigned cmb base address is 0x%lx\n", *cba); /* set the base address and enable the memory space */ - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(*cba | 0x3)); + mmio_hl_write64(ctrl->regs, NVME_REG_CMBMSC, cpu_to_le64(*cba | 0x3)); return cmb; } diff --git a/examples/cmb.c b/examples/cmb.c index 682e990d..b475a9bb 100644 --- a/examples/cmb.c +++ b/examples/cmb.c @@ -68,19 +68,19 @@ int main(int argc, char **argv) if (nvme_init(&ctrl, bdf, NULL)) err(1, "failed to init nvme controller"); - vs = le32_to_cpu(mmio_read32(ctrl.regs + NVME_REG_VS)); + vs = le32_to_cpu(mmio_read32(ctrl.regs, NVME_REG_VS)); if (vs < NVME_VERSION(1, 4, 0)) errx(1, "controller must be compliant with at least nvme v1.4.0"); - cap = le64_to_cpu(mmio_read64(ctrl.regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl.regs, NVME_REG_CAP)); if (!NVME_GET(cap, CAP_CMBS)) errx(1, "controller memory buffer not supported (cap 0x%lx)", cap); /* enable the CMBSZ and CMBLOC registers */ - mmio_hl_write64(ctrl.regs + NVME_REG_CMBMSC, cpu_to_le64(0x1)); + mmio_hl_write64(ctrl.regs, NVME_REG_CMBMSC, cpu_to_le64(0x1)); - cmbsz = le32_to_cpu(mmio_read32(ctrl.regs + NVME_REG_CMBSZ)); - cmbloc = le32_to_cpu(mmio_read32(ctrl.regs + NVME_REG_CMBLOC)); + cmbsz = le32_to_cpu(mmio_read32(ctrl.regs, NVME_REG_CMBSZ)); + cmbloc = le32_to_cpu(mmio_read32(ctrl.regs, NVME_REG_CMBLOC)); szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU)); len = szu * NVME_GET(cmbsz, CMBSZ_SZ); @@ -113,7 +113,7 @@ int main(int argc, char **argv) printf("assigned cmb base address is 0x%lx\n", cba); /* set the base address and enable the memory space */ - mmio_hl_write64(ctrl.regs + NVME_REG_CMBMSC, cpu_to_le64(cba | 0x3)); + mmio_hl_write64(ctrl.regs, NVME_REG_CMBMSC, cpu_to_le64(cba | 0x3)); cmd = (union nvme_cmd) { .opcode = nvme_admin_identify, diff --git a/examples/regs.c b/examples/regs.c index 00244da8..38f87267 100644 --- a/examples/regs.c +++ b/examples/regs.c @@ -66,30 +66,30 @@ int main(int argc, char **argv) regs = ctrl.regs; - cap = le64_to_cpu(mmio_read64(regs + NVME_REG_CAP)); - vs = le32_to_cpu(mmio_read32(regs + NVME_REG_VS)); - intms = le32_to_cpu(mmio_read32(regs + NVME_REG_INTMS)); - intmc = le32_to_cpu(mmio_read32(regs + NVME_REG_INTMC)); - cc = le32_to_cpu(mmio_read32(regs + NVME_REG_CC)); - csts = le32_to_cpu(mmio_read32(regs + NVME_REG_CSTS)); - nssr = le32_to_cpu(mmio_read32(regs + NVME_REG_NSSR)); - aqa = le32_to_cpu(mmio_read32(regs + NVME_REG_AQA)); - asq = le64_to_cpu(mmio_read64(regs + NVME_REG_ASQ)); - acq = le64_to_cpu(mmio_read64(regs + NVME_REG_ACQ)); - cmbloc = le32_to_cpu(mmio_read32(regs + NVME_REG_CMBLOC)); - cmbsz = le32_to_cpu(mmio_read32(regs + NVME_REG_CMBSZ)); - bpinfo = le32_to_cpu(mmio_read32(regs + NVME_REG_BPINFO)); - bprsel = le32_to_cpu(mmio_read32(regs + NVME_REG_BPRSEL)); - bpmbl = le64_to_cpu(mmio_read64(regs + NVME_REG_BPMBL)); - cmbmsc = le64_to_cpu(mmio_read64(regs + NVME_REG_CMBMSC)); - cmbsts = le32_to_cpu(mmio_read32(regs + NVME_REG_CMBSTS)); - pmrcap = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRCAP)); - pmrctl = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRCTL)); - pmrsts = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRSTS)); - pmrebs = le32_to_cpu(mmio_read32(regs + NVME_REG_PMREBS)); - pmrswtp = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRSWTP)); - pmrmsc = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRMSCL)) | - (uint64_t)le32_to_cpu(mmio_read32(regs + NVME_REG_PMRMSCU)) << 32; + cap = le64_to_cpu(mmio_read64(regs, NVME_REG_CAP)); + vs = le32_to_cpu(mmio_read32(regs, NVME_REG_VS)); + intms = le32_to_cpu(mmio_read32(regs, NVME_REG_INTMS)); + intmc = le32_to_cpu(mmio_read32(regs, NVME_REG_INTMC)); + cc = le32_to_cpu(mmio_read32(regs, NVME_REG_CC)); + csts = le32_to_cpu(mmio_read32(regs, NVME_REG_CSTS)); + nssr = le32_to_cpu(mmio_read32(regs, NVME_REG_NSSR)); + aqa = le32_to_cpu(mmio_read32(regs, NVME_REG_AQA)); + asq = le64_to_cpu(mmio_read64(regs, NVME_REG_ASQ)); + acq = le64_to_cpu(mmio_read64(regs, NVME_REG_ACQ)); + cmbloc = le32_to_cpu(mmio_read32(regs, NVME_REG_CMBLOC)); + cmbsz = le32_to_cpu(mmio_read32(regs, NVME_REG_CMBSZ)); + bpinfo = le32_to_cpu(mmio_read32(regs, NVME_REG_BPINFO)); + bprsel = le32_to_cpu(mmio_read32(regs, NVME_REG_BPRSEL)); + bpmbl = le64_to_cpu(mmio_read64(regs, NVME_REG_BPMBL)); + cmbmsc = le64_to_cpu(mmio_read64(regs, NVME_REG_CMBMSC)); + cmbsts = le32_to_cpu(mmio_read32(regs, NVME_REG_CMBSTS)); + pmrcap = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRCAP)); + pmrctl = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRCTL)); + pmrsts = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRSTS)); + pmrebs = le32_to_cpu(mmio_read32(regs, NVME_REG_PMREBS)); + pmrswtp = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRSWTP)); + pmrmsc = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRMSCL)) | + (uint64_t)le32_to_cpu(mmio_read32(regs, NVME_REG_PMRMSCU)) << 32; printf("%-16s %lx\n", "CAP", cap); printf("%-16s %lx\n", "CAP.MQES", NVME_CAP_MQES(cap)); diff --git a/include/vfn/nvme/queue.h b/include/vfn/nvme/queue.h index b2316826..ae5ccc3c 100644 --- a/include/vfn/nvme/queue.h +++ b/include/vfn/nvme/queue.h @@ -147,7 +147,7 @@ static inline void nvme_sq_update_tail(struct nvme_sq *sq) /* do not reorder queue entry store with doorbell store */ wmb(); - mmio_write32(sq->doorbell, cpu_to_le32(sq->tail)); + mmio_write32(sq->doorbell, 0, cpu_to_le32(sq->tail)); } sq->ptail = sq->tail; @@ -190,7 +190,7 @@ static inline void nvme_cq_update_head(struct nvme_cq *cq) } if (nvme_try_dbbuf(cq->head, &cq->dbbuf)) - mmio_write32(cq->doorbell, cpu_to_le32(cq->head)); + mmio_write32(cq->doorbell, 0, cpu_to_le32(cq->head)); } /** diff --git a/include/vfn/support/mmio.h b/include/vfn/support/mmio.h index 5929e7c8..1b52ae23 100644 --- a/include/vfn/support/mmio.h +++ b/include/vfn/support/mmio.h @@ -15,75 +15,80 @@ /** * mmio_read32 - read 4 bytes in memory-mapped register - * @addr: memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address * * Return: read value (native endian) */ -static inline leint32_t mmio_read32(void *addr) +static inline leint32_t mmio_read32(void *base_addr, uint64_t offset) { /* memory-mapped register */ - return *(const volatile leint32_t __force *)addr; + return *(const volatile leint32_t __force *)(base_addr + offset); } /** * mmio_lh_read64 - read 8 bytes in memory-mapped register - * @addr: memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address * * Read the low 4 bytes first, then the high 4 bytes. * * Return: read value (native endian) */ -static inline leint64_t mmio_lh_read64(void *addr) +static inline leint64_t mmio_lh_read64(void *base_addr, uint64_t offset) { uint32_t lo, hi; /* memory-mapped register */ - lo = *(const volatile uint32_t __force *)addr; + lo = *(const volatile uint32_t __force *)(base_addr + offset); /* memory-mapped register */ - hi = *(const volatile uint32_t __force *)(addr + 4); + hi = *(const volatile uint32_t __force *)(base_addr + offset + 4); return (leint64_t __force)(((uint64_t)hi << 32) | lo); } -#define mmio_read64(addr) mmio_lh_read64(addr) +#define mmio_read64(base_addr, offset) mmio_lh_read64(base_addr, offset) /** * mmio_write32 - write 4 bytes to memory-mapped register - * @addr: memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address * @v: value to write (native endian) */ -static inline void mmio_write32(void *addr, leint32_t v) +static inline void mmio_write32(void *base_addr, uint64_t offset, leint32_t v) { /* memory-mapped register */ - *(volatile leint32_t __force *)addr = v; + *(volatile leint32_t __force *)(base_addr + offset) = v; } /** * mmio_lh_write64 - write 8 bytes to memory-mapped register - * @addr: memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address * @v: value to write (native endian) * * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes * first, then high). */ -static inline void mmio_lh_write64(void *addr, leint64_t v) +static inline void mmio_lh_write64(void *base_addr, uint64_t offset, leint64_t v) { - mmio_write32(addr, (leint32_t __force)v); - mmio_write32(addr + 4, (leint32_t __force)(v >> 32)); + mmio_write32(base_addr, offset, (leint32_t __force)v); + mmio_write32(base_addr, offset + 4, (leint32_t __force)(v >> 32)); } /** * mmio_hl_write64 - write 8 bytes to memory-mapped register - * @addr: memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address * @v: value to write (native endian) * * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes * first, then low). */ -static inline void mmio_hl_write64(void *addr, leint64_t v) +static inline void mmio_hl_write64(void *base_addr, uint64_t offset, leint64_t v) { - mmio_write32(addr + 4, (leint32_t __force)((uint64_t __force)v >> 32)); - mmio_write32(addr, (leint32_t __force)v); + mmio_write32(base_addr, offset + 4, (leint32_t __force)((uint64_t __force)v >> 32)); + mmio_write32(base_addr, offset, (leint32_t __force)v); } #endif /* LIBVFN_SUPPORT_MMIO_H */ diff --git a/src/nvme/core.c b/src/nvme/core.c index 96073c3e..09700464 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -60,7 +60,7 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec uint8_t dstrd; size_t len; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); if (qid && qid > ctrl->config.ncqa + 1) { @@ -132,7 +132,7 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, uint8_t dstrd; ssize_t len; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); if (qid && qid > ctrl->config.nsqa + 1) { @@ -268,9 +268,9 @@ static int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags) aqa = NVME_AQ_QSIZE - 1; aqa |= aqa << 16; - mmio_write32(ctrl->regs + NVME_REG_AQA, cpu_to_le32(aqa)); - mmio_hl_write64(ctrl->regs + NVME_REG_ASQ, cpu_to_le64(sq->iova)); - mmio_hl_write64(ctrl->regs + NVME_REG_ACQ, cpu_to_le64(cq->iova)); + mmio_write32(ctrl->regs, NVME_REG_AQA, cpu_to_le32(aqa)); + mmio_hl_write64(ctrl->regs, NVME_REG_ASQ, cpu_to_le64(sq->iova)); + mmio_hl_write64(ctrl->regs, NVME_REG_ACQ, cpu_to_le64(cq->iova)); return 0; @@ -402,7 +402,7 @@ static int nvme_wait_rdy(struct nvme_ctrl *ctrl, unsigned short rdy) unsigned long timeout_ms; struct timeabs deadline; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); timeout_ms = 500 * (NVME_FIELD_GET(cap, CAP_TO) + 1); deadline = timeabs_add(time_now(), time_from_msec(timeout_ms)); @@ -414,7 +414,7 @@ static int nvme_wait_rdy(struct nvme_ctrl *ctrl, unsigned short rdy) return -1; } - csts = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CSTS)); + csts = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CSTS)); } while (NVME_FIELD_GET(csts, CSTS_RDY) != rdy); return 0; @@ -426,7 +426,7 @@ int nvme_enable(struct nvme_ctrl *ctrl) uint32_t cc; uint64_t cap; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); css = NVME_FIELD_GET(cap, CAP_CSS); cc = @@ -444,7 +444,7 @@ int nvme_enable(struct nvme_ctrl *ctrl) else cc |= NVME_FIELD_SET(NVME_CC_CSS_NVM, CC_CSS); - mmio_write32(ctrl->regs + NVME_REG_CC, cpu_to_le32(cc)); + mmio_write32(ctrl->regs, NVME_REG_CC, cpu_to_le32(cc)); return nvme_wait_rdy(ctrl, 1); } @@ -453,8 +453,8 @@ int nvme_reset(struct nvme_ctrl *ctrl) { uint32_t cc; - cc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CC)); - mmio_write32(ctrl->regs + NVME_REG_CC, cpu_to_le32(cc & 0xfe)); + cc = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CC)); + mmio_write32(ctrl->regs, NVME_REG_CC, cpu_to_le32(cc & 0xfe)); return nvme_wait_rdy(ctrl, 0); } @@ -489,7 +489,7 @@ static int nvme_init_dbconfig(struct nvme_ctrl *ctrl) uint64_t cap; uint8_t dstrd; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); ctrl->adminq.cq->dbbuf.doorbell = cqhdbl(ctrl->dbbuf.doorbells, NVME_AQ, dstrd); @@ -545,7 +545,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op return -1; } - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); mpsmin = NVME_FIELD_GET(cap, CAP_MPSMIN); mpsmax = NVME_FIELD_GET(cap, CAP_MPSMAX); From 1982a332295b4bae9c9aaf9e6bc032177436d280 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Tue, 12 Sep 2023 22:29:28 +0200 Subject: [PATCH 02/23] support/linux: move linux mem.c to src/support/platform/linux/ Moved file without any changes to prepare for other platforms. Signed-off-by: Mads Ynddal --- src/support/meson.build | 3 ++- src/support/{ => platform/linux}/mem.c | 0 src/support/platform/linux/meson.build | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) rename src/support/{ => platform/linux}/mem.c (100%) create mode 100644 src/support/platform/linux/meson.build diff --git a/src/support/meson.build b/src/support/meson.build index 30e01276..c435d615 100644 --- a/src/support/meson.build +++ b/src/support/meson.build @@ -1,11 +1,12 @@ support_sources = files( 'io.c', 'log.c', - 'mem.c', 'ticks.c', 'timer.c', ) +subdir('platform/linux') + if host_machine.cpu_family() == 'x86_64' subdir('arch/x86_64') endif diff --git a/src/support/mem.c b/src/support/platform/linux/mem.c similarity index 100% rename from src/support/mem.c rename to src/support/platform/linux/mem.c diff --git a/src/support/platform/linux/meson.build b/src/support/platform/linux/meson.build new file mode 100644 index 00000000..b59fba9c --- /dev/null +++ b/src/support/platform/linux/meson.build @@ -0,0 +1,5 @@ +support_sources_platform_linux = files( + 'mem.c', +) + +support_sources += support_sources_platform_linux From 37b5cbfaaeac343fcdaec1e63abfd07785f022c2 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 08:29:44 +0200 Subject: [PATCH 03/23] support/linux: prepartory work of moving log, mem, mmio, mutex headers into platform/linux Moved file without any changes to prepare for other platforms. Signed-off-by: Mads Ynddal --- include/vfn/support/log.h | 87 +------------- include/vfn/support/mem.h | 116 +------------------ include/vfn/support/mmio.h | 83 +------------- include/vfn/support/mutex.h | 38 +----- include/vfn/support/platform/linux/log.h | 98 ++++++++++++++++ include/vfn/support/platform/linux/mem.h | 127 +++++++++++++++++++++ include/vfn/support/platform/linux/mmio.h | 94 +++++++++++++++ include/vfn/support/platform/linux/mutex.h | 49 ++++++++ 8 files changed, 372 insertions(+), 320 deletions(-) create mode 100644 include/vfn/support/platform/linux/log.h create mode 100644 include/vfn/support/platform/linux/mem.h create mode 100644 include/vfn/support/platform/linux/mmio.h create mode 100644 include/vfn/support/platform/linux/mutex.h diff --git a/include/vfn/support/log.h b/include/vfn/support/log.h index 32d3faf9..806f1542 100644 --- a/include/vfn/support/log.h +++ b/include/vfn/support/log.h @@ -10,89 +10,4 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_LOG_H -#define LIBVFN_SUPPORT_LOG_H - -extern struct log_state { - int v; -} __log_state; - -enum __log_level { - LOG_ERROR, - LOG_INFO, - LOG_DEBUG, -}; - -/** - * logv - Determine if log verbosity is as given - * @v: target verbositry level - * - * Return: ``true`` if verbosity is at least @v, ``false`` otherwise. - */ -static inline bool logv(int v) -{ - if (atomic_load_acquire(&__log_state.v) >= v) - return true; - - return false; -} - -/** - * logv_set - Set log verbosity level - * @v: verbosity level - */ -static inline void logv_set(int v) -{ - atomic_store_release(&__log_state.v, v); -} - -/** - * __log - Log a message at a given verbosity level - * @v: verbosity level - * @fmt: format string - * @...: format string arguments - * - * Log a formatted message to stderr if current verbosity level is at least @v. - */ -static inline void __attribute__((format(printf, 2, 3))) __log(int v, char const *fmt, ...) -{ - va_list va; - - if (!logv(v)) - return; - - va_start(va, fmt); - vfprintf(stderr, fmt, va); - va_end(va); -} - -#ifndef log_fmt -#define log_fmt(fmt) fmt -#endif - -#ifdef DEBUG -# define log_error(fmt, ...) __log(LOG_ERROR, "E %s (%s:%d): " log_fmt(fmt), \ - __func__, __FILE__, __LINE__, ##__VA_ARGS__) -# define log_info(fmt, ...) __log(LOG_INFO, "I %s (%s:%d): " log_fmt(fmt), \ - __func__, __FILE__, __LINE__, ##__VA_ARGS__) -# define log_debug(fmt, ...) __log(LOG_DEBUG, "D %s (%s:%d): " log_fmt(fmt), \ - __func__, __FILE__, __LINE__, ##__VA_ARGS__) -#else -# define log_error(fmt, ...) __log(LOG_ERROR, log_fmt(fmt), ##__VA_ARGS__) -# define log_info(fmt, ...) __log(LOG_INFO, log_fmt(fmt), ##__VA_ARGS__) -# define log_debug(fmt, ...) __log(LOG_DEBUG, log_fmt(fmt), ##__VA_ARGS__) -#endif /* DEBUG */ - -#define log_fatal(fmt, ...) \ - do { \ - log_error(fmt, ##__VA_ARGS__); \ - exit(errno); \ - } while (0) - -#define log_fatal_if(expr, fmt, ...) \ - do { \ - if (expr) \ - log_fatal(fmt, ##__VA_ARGS__); \ - } while (0) - -#endif /* LIBVFN_SUPPORT_LOG_H */ +#include diff --git a/include/vfn/support/mem.h b/include/vfn/support/mem.h index 6773c4b1..fad8f32a 100644 --- a/include/vfn/support/mem.h +++ b/include/vfn/support/mem.h @@ -10,118 +10,4 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MEM_H -#define LIBVFN_SUPPORT_MEM_H - -extern size_t __VFN_PAGESIZE; -extern int __VFN_PAGESHIFT; - -void backtrace_abort(void); - -static inline void __do_autofree(void *mem) -{ - void **memp = (void **)mem; - - free(*memp); -} - -#define __autofree __attribute__((cleanup(__do_autofree))) - -static inline bool would_overflow(unsigned int n, size_t sz) -{ - return n > SIZE_MAX / sz; -} - -static inline size_t __abort_on_overflow(unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) - backtrace_abort(); - - return n * sz; -} - -/** - * xmalloc - version of malloc that cannot fail - * @sz: number of bytes to allocate - * - * Call malloc, but only return NULL when @sz is zero. Otherwise, abort. - * - * Return: pointer to allocated memory - */ -static inline void *xmalloc(size_t sz) -{ - void *mem; - - if (unlikely(!sz)) - return NULL; - - mem = malloc(sz); - if (unlikely(!mem)) - backtrace_abort(); - - return mem; -} - -static inline void *zmalloc(size_t sz) -{ - void *mem; - - if (unlikely(!sz)) - return NULL; - - mem = calloc(1, sz); - if (unlikely(!mem)) - backtrace_abort(); - - return mem; -} - -static inline void *mallocn(unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) { - fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); - - backtrace_abort(); - } - - return xmalloc(n * sz); -} - -static inline void *zmallocn(unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) { - fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); - - backtrace_abort(); - } - - return zmalloc(n * sz); -} - -static inline void *reallocn(void *mem, unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) { - fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); - - backtrace_abort(); - } - - return realloc(mem, n * sz); -} - -#define _new_t(t, n, f) \ - ((t *) f(n, sizeof(t))) - -#define new_t(t, n) _new_t(t, n, mallocn) -#define znew_t(t, n) _new_t(t, n, zmallocn) - -ssize_t pgmap(void **mem, size_t sz); -ssize_t pgmapn(void **mem, unsigned int n, size_t sz); - -static inline void pgunmap(void *mem, size_t len) -{ - if (munmap(mem, len)) - backtrace_abort(); -} - -#endif /* LIBVFN_SUPPORT_MEM_H */ +#include diff --git a/include/vfn/support/mmio.h b/include/vfn/support/mmio.h index 1b52ae23..d0bb0820 100644 --- a/include/vfn/support/mmio.h +++ b/include/vfn/support/mmio.h @@ -10,85 +10,4 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MMIO_H -#define LIBVFN_SUPPORT_MMIO_H - -/** - * mmio_read32 - read 4 bytes in memory-mapped register - * @base_addr: base address of memory-mapped register - * @offset: offset into the base address - * - * Return: read value (native endian) - */ -static inline leint32_t mmio_read32(void *base_addr, uint64_t offset) -{ - /* memory-mapped register */ - return *(const volatile leint32_t __force *)(base_addr + offset); -} - -/** - * mmio_lh_read64 - read 8 bytes in memory-mapped register - * @base_addr: base address of memory-mapped register - * @offset: offset into the base address - * - * Read the low 4 bytes first, then the high 4 bytes. - * - * Return: read value (native endian) - */ -static inline leint64_t mmio_lh_read64(void *base_addr, uint64_t offset) -{ - uint32_t lo, hi; - - /* memory-mapped register */ - lo = *(const volatile uint32_t __force *)(base_addr + offset); - /* memory-mapped register */ - hi = *(const volatile uint32_t __force *)(base_addr + offset + 4); - - return (leint64_t __force)(((uint64_t)hi << 32) | lo); -} - -#define mmio_read64(base_addr, offset) mmio_lh_read64(base_addr, offset) - -/** - * mmio_write32 - write 4 bytes to memory-mapped register - * @base_addr: base address of memory-mapped register - * @offset: offset into the base address - * @v: value to write (native endian) - */ -static inline void mmio_write32(void *base_addr, uint64_t offset, leint32_t v) -{ - /* memory-mapped register */ - *(volatile leint32_t __force *)(base_addr + offset) = v; -} - -/** - * mmio_lh_write64 - write 8 bytes to memory-mapped register - * @base_addr: base address of memory-mapped register - * @offset: offset into the base address - * @v: value to write (native endian) - * - * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes - * first, then high). - */ -static inline void mmio_lh_write64(void *base_addr, uint64_t offset, leint64_t v) -{ - mmio_write32(base_addr, offset, (leint32_t __force)v); - mmio_write32(base_addr, offset + 4, (leint32_t __force)(v >> 32)); -} - -/** - * mmio_hl_write64 - write 8 bytes to memory-mapped register - * @base_addr: base address of memory-mapped register - * @offset: offset into the base address - * @v: value to write (native endian) - * - * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes - * first, then low). - */ -static inline void mmio_hl_write64(void *base_addr, uint64_t offset, leint64_t v) -{ - mmio_write32(base_addr, offset + 4, (leint32_t __force)((uint64_t __force)v >> 32)); - mmio_write32(base_addr, offset, (leint32_t __force)v); -} - -#endif /* LIBVFN_SUPPORT_MMIO_H */ +#include diff --git a/include/vfn/support/mutex.h b/include/vfn/support/mutex.h index 3b84af26..df0a0e7f 100644 --- a/include/vfn/support/mutex.h +++ b/include/vfn/support/mutex.h @@ -10,40 +10,4 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MUTEX_H -#define LIBVFN_SUPPORT_MUTEX_H - -/* - * DOC: Autolockable mutex - * - * Define a __autolock() macro that will lock the given mutex as well as ensure - * that it is unlocked when going out of scope. This is inspired by the - * polymorphic locking functions in QEMU (include/qemu/lockable.h), but this - * version only supports the pthread_mutex_t. - */ - -static inline pthread_mutex_t *__mutex_auto_lock(pthread_mutex_t *mutex) -{ - pthread_mutex_lock(mutex); - return mutex; -} - -static inline void __mutex_auto_unlock(pthread_mutex_t *mutex) -{ - if (mutex) - pthread_mutex_unlock(mutex); -} - -DEFINE_AUTOPTR(pthread_mutex_t, __mutex_auto_unlock) - -/** - * __autolock - autolock mutex - * @x: pointer to pthread mutex - * - * Lock the mutex and unlock it automatically when going out of scope. - */ -#define __autolock(x) \ - __autoptr(pthread_mutex_t) \ - glue(autolock, __COUNTER__) __attribute__((__unused__)) = __mutex_auto_lock(x) - -#endif /* LIBVFN_SUPPORT_MUTEX_H */ +#include diff --git a/include/vfn/support/platform/linux/log.h b/include/vfn/support/platform/linux/log.h new file mode 100644 index 00000000..32d3faf9 --- /dev/null +++ b/include/vfn/support/platform/linux/log.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_LOG_H +#define LIBVFN_SUPPORT_LOG_H + +extern struct log_state { + int v; +} __log_state; + +enum __log_level { + LOG_ERROR, + LOG_INFO, + LOG_DEBUG, +}; + +/** + * logv - Determine if log verbosity is as given + * @v: target verbositry level + * + * Return: ``true`` if verbosity is at least @v, ``false`` otherwise. + */ +static inline bool logv(int v) +{ + if (atomic_load_acquire(&__log_state.v) >= v) + return true; + + return false; +} + +/** + * logv_set - Set log verbosity level + * @v: verbosity level + */ +static inline void logv_set(int v) +{ + atomic_store_release(&__log_state.v, v); +} + +/** + * __log - Log a message at a given verbosity level + * @v: verbosity level + * @fmt: format string + * @...: format string arguments + * + * Log a formatted message to stderr if current verbosity level is at least @v. + */ +static inline void __attribute__((format(printf, 2, 3))) __log(int v, char const *fmt, ...) +{ + va_list va; + + if (!logv(v)) + return; + + va_start(va, fmt); + vfprintf(stderr, fmt, va); + va_end(va); +} + +#ifndef log_fmt +#define log_fmt(fmt) fmt +#endif + +#ifdef DEBUG +# define log_error(fmt, ...) __log(LOG_ERROR, "E %s (%s:%d): " log_fmt(fmt), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__) +# define log_info(fmt, ...) __log(LOG_INFO, "I %s (%s:%d): " log_fmt(fmt), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__) +# define log_debug(fmt, ...) __log(LOG_DEBUG, "D %s (%s:%d): " log_fmt(fmt), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__) +#else +# define log_error(fmt, ...) __log(LOG_ERROR, log_fmt(fmt), ##__VA_ARGS__) +# define log_info(fmt, ...) __log(LOG_INFO, log_fmt(fmt), ##__VA_ARGS__) +# define log_debug(fmt, ...) __log(LOG_DEBUG, log_fmt(fmt), ##__VA_ARGS__) +#endif /* DEBUG */ + +#define log_fatal(fmt, ...) \ + do { \ + log_error(fmt, ##__VA_ARGS__); \ + exit(errno); \ + } while (0) + +#define log_fatal_if(expr, fmt, ...) \ + do { \ + if (expr) \ + log_fatal(fmt, ##__VA_ARGS__); \ + } while (0) + +#endif /* LIBVFN_SUPPORT_LOG_H */ diff --git a/include/vfn/support/platform/linux/mem.h b/include/vfn/support/platform/linux/mem.h new file mode 100644 index 00000000..6773c4b1 --- /dev/null +++ b/include/vfn/support/platform/linux/mem.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MEM_H +#define LIBVFN_SUPPORT_MEM_H + +extern size_t __VFN_PAGESIZE; +extern int __VFN_PAGESHIFT; + +void backtrace_abort(void); + +static inline void __do_autofree(void *mem) +{ + void **memp = (void **)mem; + + free(*memp); +} + +#define __autofree __attribute__((cleanup(__do_autofree))) + +static inline bool would_overflow(unsigned int n, size_t sz) +{ + return n > SIZE_MAX / sz; +} + +static inline size_t __abort_on_overflow(unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) + backtrace_abort(); + + return n * sz; +} + +/** + * xmalloc - version of malloc that cannot fail + * @sz: number of bytes to allocate + * + * Call malloc, but only return NULL when @sz is zero. Otherwise, abort. + * + * Return: pointer to allocated memory + */ +static inline void *xmalloc(size_t sz) +{ + void *mem; + + if (unlikely(!sz)) + return NULL; + + mem = malloc(sz); + if (unlikely(!mem)) + backtrace_abort(); + + return mem; +} + +static inline void *zmalloc(size_t sz) +{ + void *mem; + + if (unlikely(!sz)) + return NULL; + + mem = calloc(1, sz); + if (unlikely(!mem)) + backtrace_abort(); + + return mem; +} + +static inline void *mallocn(unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) { + fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); + + backtrace_abort(); + } + + return xmalloc(n * sz); +} + +static inline void *zmallocn(unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) { + fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); + + backtrace_abort(); + } + + return zmalloc(n * sz); +} + +static inline void *reallocn(void *mem, unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) { + fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); + + backtrace_abort(); + } + + return realloc(mem, n * sz); +} + +#define _new_t(t, n, f) \ + ((t *) f(n, sizeof(t))) + +#define new_t(t, n) _new_t(t, n, mallocn) +#define znew_t(t, n) _new_t(t, n, zmallocn) + +ssize_t pgmap(void **mem, size_t sz); +ssize_t pgmapn(void **mem, unsigned int n, size_t sz); + +static inline void pgunmap(void *mem, size_t len) +{ + if (munmap(mem, len)) + backtrace_abort(); +} + +#endif /* LIBVFN_SUPPORT_MEM_H */ diff --git a/include/vfn/support/platform/linux/mmio.h b/include/vfn/support/platform/linux/mmio.h new file mode 100644 index 00000000..1b52ae23 --- /dev/null +++ b/include/vfn/support/platform/linux/mmio.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MMIO_H +#define LIBVFN_SUPPORT_MMIO_H + +/** + * mmio_read32 - read 4 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Return: read value (native endian) + */ +static inline leint32_t mmio_read32(void *base_addr, uint64_t offset) +{ + /* memory-mapped register */ + return *(const volatile leint32_t __force *)(base_addr + offset); +} + +/** + * mmio_lh_read64 - read 8 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Read the low 4 bytes first, then the high 4 bytes. + * + * Return: read value (native endian) + */ +static inline leint64_t mmio_lh_read64(void *base_addr, uint64_t offset) +{ + uint32_t lo, hi; + + /* memory-mapped register */ + lo = *(const volatile uint32_t __force *)(base_addr + offset); + /* memory-mapped register */ + hi = *(const volatile uint32_t __force *)(base_addr + offset + 4); + + return (leint64_t __force)(((uint64_t)hi << 32) | lo); +} + +#define mmio_read64(base_addr, offset) mmio_lh_read64(base_addr, offset) + +/** + * mmio_write32 - write 4 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + */ +static inline void mmio_write32(void *base_addr, uint64_t offset, leint32_t v) +{ + /* memory-mapped register */ + *(volatile leint32_t __force *)(base_addr + offset) = v; +} + +/** + * mmio_lh_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes + * first, then high). + */ +static inline void mmio_lh_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32(base_addr, offset, (leint32_t __force)v); + mmio_write32(base_addr, offset + 4, (leint32_t __force)(v >> 32)); +} + +/** + * mmio_hl_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes + * first, then low). + */ +static inline void mmio_hl_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32(base_addr, offset + 4, (leint32_t __force)((uint64_t __force)v >> 32)); + mmio_write32(base_addr, offset, (leint32_t __force)v); +} + +#endif /* LIBVFN_SUPPORT_MMIO_H */ diff --git a/include/vfn/support/platform/linux/mutex.h b/include/vfn/support/platform/linux/mutex.h new file mode 100644 index 00000000..3b84af26 --- /dev/null +++ b/include/vfn/support/platform/linux/mutex.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MUTEX_H +#define LIBVFN_SUPPORT_MUTEX_H + +/* + * DOC: Autolockable mutex + * + * Define a __autolock() macro that will lock the given mutex as well as ensure + * that it is unlocked when going out of scope. This is inspired by the + * polymorphic locking functions in QEMU (include/qemu/lockable.h), but this + * version only supports the pthread_mutex_t. + */ + +static inline pthread_mutex_t *__mutex_auto_lock(pthread_mutex_t *mutex) +{ + pthread_mutex_lock(mutex); + return mutex; +} + +static inline void __mutex_auto_unlock(pthread_mutex_t *mutex) +{ + if (mutex) + pthread_mutex_unlock(mutex); +} + +DEFINE_AUTOPTR(pthread_mutex_t, __mutex_auto_unlock) + +/** + * __autolock - autolock mutex + * @x: pointer to pthread mutex + * + * Lock the mutex and unlock it automatically when going out of scope. + */ +#define __autolock(x) \ + __autoptr(pthread_mutex_t) \ + glue(autolock, __COUNTER__) __attribute__((__unused__)) = __mutex_auto_lock(x) + +#endif /* LIBVFN_SUPPORT_MUTEX_H */ From 3f8f21ef0bf9a87722078692a344424791c276b5 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:03:20 +0200 Subject: [PATCH 04/23] support/macos: add macos replacement for byteswap.h Simple replacement using built-in byteswap. Signed-off-by: Mads Ynddal --- include/vfn/support/platform/macos/byteswap.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 include/vfn/support/platform/macos/byteswap.h diff --git a/include/vfn/support/platform/macos/byteswap.h b/include/vfn/support/platform/macos/byteswap.h new file mode 100644 index 00000000..123fc537 --- /dev/null +++ b/include/vfn/support/platform/macos/byteswap.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#define bswap_16 __builtin_bswap16 +#define bswap_32 __builtin_bswap32 +#define bswap_64 __builtin_bswap64 From cedbf4017d92fbbe0951334c5902d06a33e1a762 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:03:37 +0200 Subject: [PATCH 05/23] support/macos: add macos replacement for errno.h Sparsely added definitions needed on macOS+DriverKit. Signed-off-by: Mads Ynddal --- include/vfn/support/platform/macos/errno.h | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 include/vfn/support/platform/macos/errno.h diff --git a/include/vfn/support/platform/macos/errno.h b/include/vfn/support/platform/macos/errno.h new file mode 100644 index 00000000..5d63071e --- /dev/null +++ b/include/vfn/support/platform/macos/errno.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_ERRNO_H +#define LIBVFN_SUPPORT_ERRNO_H + +extern "C" { +extern int errno; + +#define ENOENT 2 +#define EIO 5 +#define EBUSY 16 +#define EEXIST 17 +#define EINVAL 22 +#define EAGAIN 35 +#define ETIMEDOUT 60 +} + +#endif From c6980073e70838765e2b1d1bc94ccae6d2d69cdc Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:06:07 +0200 Subject: [PATCH 06/23] support/macos: added include/.../platform/macos/log.h A port of the Linux log.h using DriverKit os_log. Signed-off-by: Mads Ynddal --- docs/api/support/log.rst | 2 +- include/vfn/support/log.h | 16 ++++++ include/vfn/support/platform/linux/log.h | 12 ----- include/vfn/support/platform/macos/log.h | 63 ++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 include/vfn/support/platform/macos/log.h diff --git a/docs/api/support/log.rst b/docs/api/support/log.rst index 3b0aebd5..e5ef12a0 100644 --- a/docs/api/support/log.rst +++ b/docs/api/support/log.rst @@ -3,4 +3,4 @@ Logging ======= -.. kernel-doc:: include/vfn/support/log.h +.. kernel-doc:: include/vfn/support/platform/linux/log.h diff --git a/include/vfn/support/log.h b/include/vfn/support/log.h index 806f1542..3f15878e 100644 --- a/include/vfn/support/log.h +++ b/include/vfn/support/log.h @@ -10,4 +10,20 @@ * COPYING and LICENSE files for more information. */ +#ifdef __APPLE__ +#include +#else #include +#endif + +#define log_fatal(fmt, ...) \ + do { \ + log_error(fmt, ##__VA_ARGS__); \ + abort(); \ + } while (0) + +#define log_fatal_if(expr, fmt, ...) \ + do { \ + if (expr) \ + log_fatal(fmt, ##__VA_ARGS__); \ + } while (0) diff --git a/include/vfn/support/platform/linux/log.h b/include/vfn/support/platform/linux/log.h index 32d3faf9..74c66e79 100644 --- a/include/vfn/support/platform/linux/log.h +++ b/include/vfn/support/platform/linux/log.h @@ -83,16 +83,4 @@ static inline void __attribute__((format(printf, 2, 3))) __log(int v, char const # define log_debug(fmt, ...) __log(LOG_DEBUG, log_fmt(fmt), ##__VA_ARGS__) #endif /* DEBUG */ -#define log_fatal(fmt, ...) \ - do { \ - log_error(fmt, ##__VA_ARGS__); \ - exit(errno); \ - } while (0) - -#define log_fatal_if(expr, fmt, ...) \ - do { \ - if (expr) \ - log_fatal(fmt, ##__VA_ARGS__); \ - } while (0) - #endif /* LIBVFN_SUPPORT_LOG_H */ diff --git a/include/vfn/support/platform/macos/log.h b/include/vfn/support/platform/macos/log.h new file mode 100644 index 00000000..890037dc --- /dev/null +++ b/include/vfn/support/platform/macos/log.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_LOG_H +#define LIBVFN_SUPPORT_LOG_H + +#include +#include +#include + +extern struct log_state { + unsigned int v; +} __log_state; + +enum __log_level { + LOG_ERROR, + LOG_INFO, + LOG_DEBUG, +}; + +/** + * logv - Determine if log verbosity is as given + * @v: target verbositry level + * + * Return: ``true`` if verbosity is at least @v, ``false`` otherwise. + */ +static inline bool logv(unsigned int v) +{ + if (atomic_load_acquire(&__log_state.v) >= v) + return true; + + return false; +} + +/** + * logv_set - Set log verbosity level + * @v: verbosity level + */ +static inline void logv_set(unsigned int v) +{ + atomic_store_release(&__log_state.v, v); +} + +#ifdef DEBUG +#define log_error(fmt, ...) os_log(OS_LOG_DEFAULT, "MacVFN ERROR: " fmt "\n", ##__VA_ARGS__) +#define log_info(fmt, ...) os_log(OS_LOG_DEFAULT, "MacVFN INFO: " fmt "\n", ##__VA_ARGS__) +#define log_debug(fmt, ...) os_log(OS_LOG_DEFAULT, "MacVFN DEBUG: " fmt "\n", ##__VA_ARGS__) +#else +#define log_error(fmt, ...) +#define log_info(fmt, ...) +#define log_debug(fmt, ...) +#endif /* DEBUG */ + +#endif /* LIBVFN_SUPPORT_LOG_H */ From b09b63ca984b9b3a9fe112e6ca49e787192c15b8 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:06:19 +0200 Subject: [PATCH 07/23] support/macos: added include/.../platform/macos/mem.h Added port of mem.h for macOS using DriverKit IOMallocZero and IOFree. Signed-off-by: Mads Ynddal --- docs/api/support/mem.rst | 2 +- include/vfn/support/mem.h | 4 ++ include/vfn/support/platform/macos/mem.h | 67 ++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 include/vfn/support/platform/macos/mem.h diff --git a/docs/api/support/mem.rst b/docs/api/support/mem.rst index 9a629f06..66f9f662 100644 --- a/docs/api/support/mem.rst +++ b/docs/api/support/mem.rst @@ -3,4 +3,4 @@ Memory Allocation Helpers ========================= -.. kernel-doc:: include/vfn/support/mem.h +.. kernel-doc:: include/vfn/support/platform/linux/mem.h diff --git a/include/vfn/support/mem.h b/include/vfn/support/mem.h index fad8f32a..88cea31b 100644 --- a/include/vfn/support/mem.h +++ b/include/vfn/support/mem.h @@ -10,4 +10,8 @@ * COPYING and LICENSE files for more information. */ +#ifdef __APPLE__ +#include +#else #include +#endif diff --git a/include/vfn/support/platform/macos/mem.h b/include/vfn/support/platform/macos/mem.h new file mode 100644 index 00000000..0fe06310 --- /dev/null +++ b/include/vfn/support/platform/macos/mem.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MEM_H +#define LIBVFN_SUPPORT_MEM_H + +#include +#include + +extern size_t __VFN_PAGESIZE; +extern int __VFN_PAGESHIFT; + +void backtrace_abort(void); + +static inline void __do_autofree(void *mem) +{ + free(mem); +} + +#define __autofree __attribute__((cleanup(__do_autofree))) + +static inline bool would_overflow(unsigned int n, size_t sz) +{ + return n > SIZE_MAX / sz; +} + +static inline void *zmalloc(size_t sz) +{ + void *mem; + + if (unlikely(!sz)) + return NULL; + + sz += sizeof(uint64_t); + mem = IOMallocZero(sz); + if (unlikely(!mem)) { + log_debug("IOMallocZero failed!"); + backtrace_abort(); + } + + *(uint64_t *)mem = sz; // Save length of malloc -- used on free. + return (void *)(((uint8_t *)mem) + sizeof(uint64_t)); +} + +void *mallocn(unsigned int n, size_t sz); + +static inline void *zmallocn(unsigned int n, size_t sz) +{ + return zmalloc(n * sz); +} + +inline void free(void *ptr) +{ + void *base_ptr = (void *)(((uint8_t *)ptr)-sizeof(uint64_t)); + uint64_t length = *(uint64_t *)base_ptr; + + IOFree(base_ptr, length); +} From 0e640ea6a31d1b75f8b16c0bf9c99336a169a539 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:06:26 +0200 Subject: [PATCH 08/23] support/macos: added include/.../platform/macos/mmio.h Added port of mmio.h for macOS using faux BAR mappings. Signed-off-by: Mads Ynddal --- docs/api/support/mmio.rst | 2 +- include/vfn/support/mmio.h | 4 + include/vfn/support/platform/macos/mmio.h | 110 ++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 include/vfn/support/platform/macos/mmio.h diff --git a/docs/api/support/mmio.rst b/docs/api/support/mmio.rst index 55f21eb1..ff1bdbf9 100644 --- a/docs/api/support/mmio.rst +++ b/docs/api/support/mmio.rst @@ -6,4 +6,4 @@ Memory-mapped I/O These helpers expect and return little endian types (i.e. ``leint32_t``) which may help with locating endianness related bugs using the sparse static analyzer. -.. kernel-doc:: include/vfn/support/mmio.h +.. kernel-doc:: include/vfn/support/platform/linux/mmio.h diff --git a/include/vfn/support/mmio.h b/include/vfn/support/mmio.h index d0bb0820..21ba30b5 100644 --- a/include/vfn/support/mmio.h +++ b/include/vfn/support/mmio.h @@ -10,4 +10,8 @@ * COPYING and LICENSE files for more information. */ +#ifdef __APPLE__ +#include +#else #include +#endif diff --git a/include/vfn/support/platform/macos/mmio.h b/include/vfn/support/platform/macos/mmio.h new file mode 100644 index 00000000..48a60310 --- /dev/null +++ b/include/vfn/support/platform/macos/mmio.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MMIO_H +#define LIBVFN_SUPPORT_MMIO_H + +struct macvfn_pci_map_bar { + struct driverkit_pci_device *pci; + unsigned int idx; + size_t len; + uint64_t offset; +}; + +/** + * mmio_read32 - read 4 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Return: read value (native endian) + */ +static inline leint32_t mmio_read32(void *base_addr, uint64_t offset) +{ + /* memory-mapped register */ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) base_addr; + uint32_t val; + + mapping->pci->dev->MemoryRead32(mapping->pci->bar_region_info[mapping->idx].memory_index, + mapping->offset + offset, &val); + + return (leint32_t __force)(val); +} + +/** + * mmio_lh_read64 - read 8 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Read the low 4 bytes first, then the high 4 bytes. + * + * Return: read value (native endian) + */ +static inline leint64_t mmio_lh_read64(void *base_addr, uint64_t offset) +{ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) base_addr; + uint64_t val; + + mapping->pci->dev->MemoryRead64(mapping->pci->bar_region_info[mapping->idx].memory_index, + mapping->offset + offset, &val); + + return (leint64_t __force)(val); +} + +#define mmio_read64(base_addr, offset) mmio_lh_read64(base_addr, offset) + +/** + * mmio_write32 - write 4 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + */ +static inline void mmio_write32(void *base_addr, uint64_t offset, leint32_t v) +{ + /* memory-mapped register */ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) base_addr; + + mapping->pci->dev->MemoryWrite32(mapping->pci->bar_region_info[mapping->idx].memory_index, + mapping->offset + offset, v); +} + +/** + * mmio_lh_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes + * first, then high). + */ +static inline void mmio_lh_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32(base_addr, offset, (leint32_t __force)v); + mmio_write32((uint8_t *)base_addr, offset + 4, (leint32_t __force)(v >> 32)); +} + +/** + * mmio_hl_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes + * first, then low). + */ +static inline void mmio_hl_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32((uint8_t *)base_addr, offset + 4, + (leint32_t __force)((uint64_t __force)v >> 32)); + mmio_write32(base_addr, offset, (leint32_t __force)v); +} + +#endif /* LIBVFN_SUPPORT_MMIO_H */ From 6a75519bf24769b4206452e0c2a880c5e5d28908 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:06:41 +0200 Subject: [PATCH 09/23] support/macos: added include/.../platform/macos/mutex.h Added port of mutex.h for macOS using IOLock instead of pthread. Signed-off-by: Mads Ynddal --- docs/api/support/mutex.rst | 2 +- include/vfn/support/mutex.h | 4 ++ include/vfn/support/platform/macos/mutex.h | 43 ++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 include/vfn/support/platform/macos/mutex.h diff --git a/docs/api/support/mutex.rst b/docs/api/support/mutex.rst index b1bbc337..ef85dd4b 100644 --- a/docs/api/support/mutex.rst +++ b/docs/api/support/mutex.rst @@ -3,4 +3,4 @@ Autolockable Mutex ================== -.. kernel-doc:: include/vfn/support/mutex.h +.. kernel-doc:: include/vfn/support/platform/linux/mutex.h diff --git a/include/vfn/support/mutex.h b/include/vfn/support/mutex.h index df0a0e7f..26aa3546 100644 --- a/include/vfn/support/mutex.h +++ b/include/vfn/support/mutex.h @@ -10,4 +10,8 @@ * COPYING and LICENSE files for more information. */ +#ifdef __APPLE__ +#include +#else #include +#endif diff --git a/include/vfn/support/platform/macos/mutex.h b/include/vfn/support/platform/macos/mutex.h new file mode 100644 index 00000000..a3d2a666 --- /dev/null +++ b/include/vfn/support/platform/macos/mutex.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MUTEX_H +#define LIBVFN_SUPPORT_MUTEX_H + +/* + * Autolockable mutex + * + * Define a __autolock() macro that will lock the given mutex as well as ensure + * that it is unlocked when going out of scope. This is inspired by the + * polymorphic locking functions in QEMU (include/qemu/lockable.h), but this + * version only supports the IOLock. + */ + +static inline IOLock *__mutex_auto_lock(IOLock *mutex) +{ + IOLockLock(mutex); + return mutex; +} + +static inline void __mutex_auto_unlock(IOLock *mutex) +{ + if (mutex) + IOLockUnlock(mutex); +} + +DEFINE_AUTOPTR(IOLock, __mutex_auto_unlock) + +#define __autolock(x) \ + __autoptr(IOLock) \ + glue(autolock, __COUNTER__) __attribute__((__unused__)) = __mutex_auto_lock(*x) + +#endif /* LIBVFN_SUPPORT_MUTEX_H */ From b7fbecc58ee8ff7511d53851f7fdd74564dca012 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 11:20:57 +0200 Subject: [PATCH 10/23] misc/cpp: add explicit cast of void* and change string concat to accomodate C++ For macOS we need to use C++, and it forces us to explicitly do type-casting etc. Signed-off-by: Mads Ynddal --- src/iommu/dma.c | 6 +++--- src/nvme/core.c | 8 +++++--- src/nvme/rq.c | 8 ++++---- src/nvme/util.c | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/iommu/dma.c b/src/iommu/dma.c index 3aa7b289..de4edb66 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -32,7 +32,7 @@ static int iova_cmp(const void *vaddr, const struct skiplist_node *n) if (vaddr < m->vaddr) return -1; - else if (vaddr >= m->vaddr + m->len) + else if ((uintptr_t)vaddr >= ((uintptr_t)m->vaddr) + m->len) return 1; return 0; @@ -106,7 +106,7 @@ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova) struct iova_mapping *m = iova_map_find(&ctx->map, vaddr); if (m) { - *iova = m->iova + (vaddr - m->vaddr); + *iova = m->iova + ((uintptr_t)vaddr - (uintptr_t)m->vaddr); return true; } @@ -173,7 +173,7 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) static void __unmap_mapping(void *opaque, struct skiplist_node *n) { - struct iommu_ctx *ctx = opaque; + struct iommu_ctx *ctx = (struct iommu_ctx *)opaque; struct iova_mapping *m = container_of_var(n, m, list); log_fatal_if(ctx->ops.dma_unmap(ctx, m->len, m->iova), diff --git a/src/nvme/core.c b/src/nvme/core.c index 09700464..a936067e 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -187,8 +187,10 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, rq->sq = sq; rq->cid = (uint16_t)i; - rq->page.vaddr = sq->pages.vaddr + (i << __mps_to_pageshift(ctrl->config.mps)); - rq->page.iova = sq->pages.iova + (i << __mps_to_pageshift(ctrl->config.mps)); + rq->page.vaddr = (void *)((uintptr_t)sq->pages.vaddr + + ((uint64_t)i << __mps_to_pageshift(ctrl->config.mps))); + rq->page.iova = sq->pages.iova + + ((uint64_t)i << __mps_to_pageshift(ctrl->config.mps)); if (i > 0) rq->rq_next = &sq->rqs[i - 1]; @@ -625,7 +627,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op goto out; } - oacs = le16_to_cpu(*(leint16_t *)(vaddr + NVME_IDENTIFY_CTRL_OACS)); + oacs = le16_to_cpu(*(leint16_t *)(((uintptr_t) vaddr) + NVME_IDENTIFY_CTRL_OACS)); if (oacs & NVME_IDENTIFY_CTRL_OACS_DBCONFIG) ret = nvme_init_dbconfig(ctrl); diff --git a/src/nvme/rq.c b/src/nvme/rq.c index 332ba4e0..1ada3327 100644 --- a/src/nvme/rq.c +++ b/src/nvme/rq.c @@ -100,7 +100,7 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * size_t len) { int prpcount; - leint64_t *prplist = rq->page.vaddr; + leint64_t *prplist = (leint64_t *)rq->page.vaddr; prpcount = __map_first(&cmd->dptr.prp1, prplist, iova, len, __mps_to_pageshift(ctrl->config.mps)); @@ -123,7 +123,7 @@ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd struct iovec *iov, int niov) { int prpcount, _prpcount; - leint64_t *prplist = rq->page.vaddr; + leint64_t *prplist = (leint64_t *)rq->page.vaddr; uint64_t iova = (uint64_t)iov->iov_base; size_t len = iov->iov_len; int pageshift = __mps_to_pageshift(ctrl->config.mps); @@ -164,7 +164,7 @@ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd if (!ALIGNED(iova, pagesize)) { - log_error("unaligned iov[%u].iov_base (0x%"PRIx64")\n", i, iova); + log_error("unaligned iov[%u].iov_base (0x%" PRIx64 ")\n", i, iova); goto invalid; } @@ -213,7 +213,7 @@ int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) if (logv(LOG_DEBUG)) { uint16_t status = le16_to_cpu(cqe.sfp) >> 1; - log_debug("cqe status 0x%" PRIx16 "\n", status & 0x7ff); + log_debug("cqe status 0x%" PRIx16 "\n", (uint16_t)(status & 0x7ff)); } return nvme_set_errno_from_cqe(&cqe); diff --git a/src/nvme/util.c b/src/nvme/util.c index a509ceb7..bec90a33 100644 --- a/src/nvme/util.c +++ b/src/nvme/util.c @@ -113,7 +113,7 @@ int nvme_sync(struct nvme_ctrl *ctrl, struct nvme_sq *sq, union nvme_cmd *sqe, v while (nvme_rq_spin(rq, &cqe) < 0) { if (errno == EAGAIN) { log_error("SPURIOUS CQE (cq %" PRIu16 " cid %" PRIu16 ")\n", - rq->sq->cq->id, cqe.cid); + (uint16_t)rq->sq->cq->id, cqe.cid); continue; } From d81e69f12c89f3d550e02f048710e1fa74bf857a Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 15:26:28 +0200 Subject: [PATCH 11/23] gitignore: crc64table.h and config.h These files are normally ignored when compiling with meson on Linux, but on macOS, we need to copy them into our source code. Signed-off-by: Mads Ynddal --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6e92f57d..f18f49d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ tags +crc64table.h +config.h From ff1b9e6843b295c54bdc8ce6822475a0b0581a2a Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Thu, 11 Apr 2024 15:10:12 +0200 Subject: [PATCH 12/23] src/iommu/context.h: Add include guard context.h needs to be included from different places, so an include guard is required to not cause issues. Signed-off-by: Mads Ynddal --- src/iommu/context.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/iommu/context.h b/src/iommu/context.h index d260b561..57b3e934 100644 --- a/src/iommu/context.h +++ b/src/iommu/context.h @@ -10,6 +10,9 @@ * COPYING and LICENSE files for more information. */ +#ifndef LIBVFN_SRC_IOMMU_CONTEXT_H +#define LIBVFN_SRC_IOMMU_CONTEXT_H + #include "util/skiplist.h" struct iommu_ctx; @@ -64,3 +67,5 @@ struct iommu_ctx *iommufd_get_iommu_context(const char *name); void iommu_ctx_init(struct iommu_ctx *ctx); int iommu_iova_range_to_string(struct iommu_iova_range *range, char **str); + +#endif /* LIBVFN_SRC_IOMMU_CONTEXT_H */ From 91b3142f8c05175c6ae3b93f805b38c37ecfaa6b Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Tue, 9 Apr 2024 15:26:52 +0200 Subject: [PATCH 13/23] misc: reduce includes, add C++ ifdef Remove all redundant includes. Introduce ifdef's for __cplusplus. Signed-off-by: Mads Ynddal --- include/vfn/iommu.h | 8 -------- include/vfn/nvme.h | 30 ++++++------------------------ include/vfn/nvme/rq.h | 2 ++ include/vfn/support.h | 27 +++++++++++++++++++++------ include/vfn/trace.h | 4 +++- src/iommu/context.c | 6 ------ src/iommu/dma.c | 12 +++++++----- src/iommu/iommufd.c | 17 ++--------------- src/iommu/vfio.c | 19 ------------------- src/nvme/core.c | 31 ++++++++----------------------- src/nvme/queue.c | 29 ++++------------------------- src/nvme/rq.c | 23 ----------------------- src/nvme/util.c | 25 +------------------------ src/pci/util.c | 23 +++-------------------- src/support/arch/x86_64/rdtsc.c | 12 ++++++++++++ src/support/io.c | 12 ------------ src/support/platform/linux/mem.c | 21 +++------------------ src/support/ticks.c | 16 +++------------- src/vfio/device.c | 16 ---------------- src/vfio/pci.c | 18 +----------------- 20 files changed, 76 insertions(+), 275 deletions(-) diff --git a/include/vfn/iommu.h b/include/vfn/iommu.h index 98542b08..43642ea0 100644 --- a/include/vfn/iommu.h +++ b/include/vfn/iommu.h @@ -13,10 +13,6 @@ #ifndef LIBVFN_IOMMU_H #define LIBVFN_IOMMU_H -#ifdef __cplusplus -extern "C" { -#endif - #include #include #include @@ -26,8 +22,4 @@ extern "C" { #include #include -#ifdef __cplusplus -} -#endif - #endif /* LIBVFN_IOMMU_H */ diff --git a/include/vfn/nvme.h b/include/vfn/nvme.h index a1023c4b..d42f6e8f 100644 --- a/include/vfn/nvme.h +++ b/include/vfn/nvme.h @@ -13,38 +13,20 @@ #ifndef LIBVFN_NVME_H #define LIBVFN_NVME_H -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include -#include +#ifndef __APPLE__ +#include +#else +#include +#endif -#include #include -#include #include -#include #include #include #include #include #include -#ifdef __cplusplus -} -#endif - #endif /* LIBVFN_NVME_H */ diff --git a/include/vfn/nvme/rq.h b/include/vfn/nvme/rq.h index ee90cb51..722f68fb 100644 --- a/include/vfn/nvme/rq.h +++ b/include/vfn/nvme/rq.h @@ -217,6 +217,7 @@ static inline void nvme_rq_exec(struct nvme_rq *rq, union nvme_cmd *cmd) int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, uint64_t iova, size_t len); +#ifndef __APPLE__ /** * nvme_rq_mapv_prp - Set up the Physical Region Pages in the data pointer of * the command from an iovec. @@ -234,6 +235,7 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * */ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, struct iovec *iov, int niov); +#endif /** * nvme_rq_spin - Spin for completion of the command associated with the request diff --git a/include/vfn/support.h b/include/vfn/support.h index 050b3179..eaa86c1f 100644 --- a/include/vfn/support.h +++ b/include/vfn/support.h @@ -17,19 +17,32 @@ extern "C" { #endif +#ifndef __APPLE__ #include #include +#include +#include #include +#include +#include +#include + +#include + +#include +#else +#include +#include +#include +#endif + +#include // PRIx64 used for logging #include #include #include #include #include #include -#include -#include - -#include #include #include @@ -37,13 +50,15 @@ extern "C" { #include #include #include -#include #include #include #include #include -#include #include +#ifndef __APPLE__ +#include +#include +#endif #ifdef __cplusplus } diff --git a/include/vfn/trace.h b/include/vfn/trace.h index 59dd981b..0fdf8bec 100644 --- a/include/vfn/trace.h +++ b/include/vfn/trace.h @@ -13,9 +13,11 @@ #ifndef LIBVFN_TRACE_H #define LIBVFN_TRACE_H +#ifndef __APPLE__ #include +#endif -#ifdef DEBUG +#if defined(DEBUG) && !defined(__APPLE__) extern __thread const char *__trace_event; # define __trace_prefix(fmt) "T %s (%s:%d) " fmt diff --git a/src/iommu/context.c b/src/iommu/context.c index b8c331e7..46ec2186 100644 --- a/src/iommu/context.c +++ b/src/iommu/context.c @@ -12,12 +12,6 @@ #define log_fmt(fmt) "iommu/context: " fmt -#include -#include -#include - -#include - #include "vfn/iommu.h" #include "vfn/support.h" diff --git a/src/iommu/dma.c b/src/iommu/dma.c index de4edb66..36d76c47 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -12,11 +12,9 @@ #define log_fmt(fmt) "iommu/dma: " fmt -#include -#include -#include -#include -#include +#ifdef __cplusplus +extern "C" { +#endif #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" @@ -210,3 +208,7 @@ int iommu_iova_range_to_string(struct iommu_iova_range *r, char **str) { return asprintf(str, "[0x%llx; 0x%llx]", r->start, r->last); } + +#ifdef __cplusplus +} +#endif diff --git a/src/iommu/iommufd.c b/src/iommu/iommufd.c index 8213bddd..64edd709 100644 --- a/src/iommu/iommufd.c +++ b/src/iommu/iommufd.c @@ -12,30 +12,17 @@ #define log_fmt(fmt) "iommu/iommufd: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include - -#include #include -#include #include "vfn/trace.h" #include "vfn/support.h" #include "vfn/pci.h" #include "vfn/iommu.h" +#include "ccan/compiler/compiler.h" #include "ccan/list/list.h" #include "context.h" diff --git a/src/iommu/vfio.c b/src/iommu/vfio.c index 3f497922..0bafa96f 100644 --- a/src/iommu/vfio.c +++ b/src/iommu/vfio.c @@ -12,30 +12,11 @@ #define log_fmt(fmt) "iommu/vfio: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include -#include - #include "ccan/str/str.h" #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" -#include -#include - #include "vfn/support.h" #include "vfn/iommu.h" #include "vfn/trace.h" diff --git a/src/nvme/core.c b/src/nvme/core.c index a936067e..a79278ba 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -12,34 +12,19 @@ #define log_fmt(fmt) "nvme/core: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include #include +#include +#include #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" +#ifndef __APPLE__ +#include #include "ccan/time/time.h" +#else +#include +int errno; +#endif #include "types.h" diff --git a/src/nvme/queue.c b/src/nvme/queue.c index c89d0c26..37b5304b 100644 --- a/src/nvme/queue.c +++ b/src/nvme/queue.c @@ -10,32 +10,11 @@ * COPYING and LICENSE files for more information. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - +#include +#include "iommu/context.h" +#ifndef __APPLE__ #include "ccan/time/time.h" +#endif void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n) { diff --git a/src/nvme/rq.c b/src/nvme/rq.c index 1ada3327..c73e0820 100644 --- a/src/nvme/rq.c +++ b/src/nvme/rq.c @@ -12,32 +12,9 @@ #define log_fmt(fmt) "nvme/rq: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include #include - #include "ccan/minmax/minmax.h" -#include "iommu/context.h" static inline int __map_first(leint64_t *prp1, leint64_t *prplist, uint64_t iova, size_t len, int pageshift) diff --git a/src/nvme/util.c b/src/nvme/util.c index bec90a33..27375c2b 100644 --- a/src/nvme/util.c +++ b/src/nvme/util.c @@ -10,32 +10,9 @@ * COPYING and LICENSE files for more information. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include #include - +#include "iommu/context.h" #include "types.h" - #include "crc64table.h" uint64_t nvme_crc64(uint64_t crc, const unsigned char *buffer, size_t len) diff --git a/src/pci/util.c b/src/pci/util.c index 3a87daeb..de3e549c 100644 --- a/src/pci/util.c +++ b/src/pci/util.c @@ -10,29 +10,12 @@ * COPYING and LICENSE files for more information. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - #include - -#include -#include -#include -#include -#include +#include +#include #include + int pci_unbind(const char *bdf) { char *path = NULL; diff --git a/src/support/arch/x86_64/rdtsc.c b/src/support/arch/x86_64/rdtsc.c index ac1b325f..693de4cc 100644 --- a/src/support/arch/x86_64/rdtsc.c +++ b/src/support/arch/x86_64/rdtsc.c @@ -7,6 +7,12 @@ * From DPDK; modified by the libvfn Authors. */ +#if defined(__x86_64__) + +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -34,3 +40,9 @@ uint64_t get_ticks_freq_arch(void) { return __x86_64_get_tsc_freq(); } + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/support/io.c b/src/support/io.c index da347d9a..b508e8d6 100644 --- a/src/support/io.c +++ b/src/support/io.c @@ -10,18 +10,6 @@ * COPYING and LICENSE files for more information. */ -#include -#ifdef __GLIBC__ -#include -#endif -#include -#include -#include -#include -#include -#include -#include - #include "vfn/support.h" ssize_t writeallfd(int fd, const void *buf, size_t count) diff --git a/src/support/platform/linux/mem.c b/src/support/platform/linux/mem.c index 5d7f2fea..09d34320 100644 --- a/src/support/platform/linux/mem.c +++ b/src/support/platform/linux/mem.c @@ -11,27 +11,12 @@ */ #define log_fmt(fmt) "support/mem: " fmt - -#include +#include #ifdef __GLIBC__ #include #endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include +#include "ccan/compiler/compiler.h" +#include size_t __VFN_PAGESIZE; int __VFN_PAGESHIFT; diff --git a/src/support/ticks.c b/src/support/ticks.c index a8d26c30..4ca07a21 100644 --- a/src/support/ticks.c +++ b/src/support/ticks.c @@ -9,21 +9,11 @@ #define log_fmt(fmt) "support/ticks: " fmt -#include -#include -#include -#include -#include -#include -#include - -#include "vfn/support/align.h" -#include "vfn/support/atomic.h" -#include "vfn/support/log.h" -#include "vfn/support/ticks.h" -#include "vfn/support/timer.h" +#include "vfn/support.h" +#ifndef __APPLE__ #include "ccan/time/time.h" +#endif #define TICKS_PER_10MHZ 10000000ULL diff --git a/src/vfio/device.c b/src/vfio/device.c index 6ad7275a..e36572f6 100644 --- a/src/vfio/device.c +++ b/src/vfio/device.c @@ -12,25 +12,9 @@ #define log_fmt(fmt) "vfio/device: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include -#include #include - #include -#include #include #include "vfn/support.h" diff --git a/src/vfio/pci.c b/src/vfio/pci.c index 76d9c1f3..72f674b4 100644 --- a/src/vfio/pci.c +++ b/src/vfio/pci.c @@ -12,26 +12,10 @@ #define log_fmt(fmt) "vfio/pci: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - #include #include #include +#include #include "vfn/support.h" #include "vfn/iommu.h" From da24e666afb9b237b02481a20aea15ba928953f5 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Tue, 9 Apr 2024 15:34:19 +0200 Subject: [PATCH 14/23] iommu/iova: dma_map, iova_map_add, dma_unmap takes iova_mapping Change dma_map, iova_map_add and dma_unmap to take an iova_mapping instead of individual arguments. This simplified passing the context around, and adding more attributes transparently later. Signed-off-by: Mads Ynddal --- src/iommu/context.h | 25 ++++++++++++------------- src/iommu/dma.c | 38 +++++++++++++++++++------------------- src/iommu/iommufd.c | 38 ++++++++++++++++++++++---------------- src/iommu/vfio.c | 21 +++++++++++---------- 4 files changed, 64 insertions(+), 58 deletions(-) diff --git a/src/iommu/context.h b/src/iommu/context.h index 57b3e934..a45afe70 100644 --- a/src/iommu/context.h +++ b/src/iommu/context.h @@ -17,30 +17,29 @@ struct iommu_ctx; +struct iova_mapping { + void *vaddr; + size_t len; + uint64_t iova; + + unsigned long flags; + + struct skiplist_node list; +}; + struct iommu_ctx_ops { /* container/ioas ops */ int (*iova_reserve)(struct iommu_ctx *ctx, size_t len, uint64_t *iova, unsigned long flags); void (*iova_put_ephemeral)(struct iommu_ctx *ctx); - int (*dma_map)(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, - unsigned long flags); - int (*dma_unmap)(struct iommu_ctx *ctx, uint64_t iova, size_t len); + int (*dma_map)(struct iommu_ctx *ctx, struct iova_mapping *m); + int (*dma_unmap)(struct iommu_ctx *ctx, struct iova_mapping *m); int (*dma_unmap_all)(struct iommu_ctx *ctx); /* device ops */ int (*get_device_fd)(struct iommu_ctx *ctx, const char *bdf); }; -struct iova_mapping { - void *vaddr; - size_t len; - uint64_t iova; - - unsigned long flags; - - struct skiplist_node list; -}; - struct iova_map { pthread_mutex_t lock; struct skiplist list; diff --git a/src/iommu/dma.c b/src/iommu/dma.c index 36d76c47..97046c75 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -36,31 +36,22 @@ static int iova_cmp(const void *vaddr, const struct skiplist_node *n) return 0; } -static int iova_map_add(struct iova_map *map, void *vaddr, size_t len, uint64_t iova, - unsigned long flags) +static int iova_map_add(struct iova_map *map, struct iova_mapping *m) { __autolock(&map->lock); struct skiplist_node *update[SKIPLIST_LEVELS] = {}; - struct iova_mapping *m; - if (!len) { + if (!m->len) { errno = EINVAL; return -1; } - if (skiplist_find(&map->list, vaddr, iova_cmp, update)) { + if (skiplist_find(&map->list, m->vaddr, iova_cmp, update)) { errno = EEXIST; return -1; } - m = znew_t(struct iova_mapping, 1); - - m->vaddr = vaddr; - m->len = len; - m->iova = iova; - m->flags = flags; - skiplist_link(&map->list, &m->list, update); return 0; @@ -115,7 +106,7 @@ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *io unsigned long flags) { uint64_t _iova; - + struct iova_mapping *m; if (iommu_translate_vaddr(ctx, vaddr, &_iova)) goto out; @@ -126,16 +117,25 @@ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *io return -1; } - if (ctx->ops.dma_map(ctx, vaddr, len, &_iova, flags)) { - log_debug("failed to map dma\n"); + m = znew_t(struct iova_mapping, 1); + m->vaddr = vaddr; + m->len = len; + m->iova = 0; + m->flags = flags; + + if (ctx->ops.dma_map(ctx, m)) { + log_error("failed to map dma\n"); + free(m); return -1; } - if (iova_map_add(&ctx->map, vaddr, len, _iova, flags)) { - log_debug("failed to add mapping\n"); + if (iova_map_add(&ctx->map, m)) { + log_error("failed to add mapping\n"); + free(m); return -1; } + _iova = m->iova; out: if (iova) *iova = _iova; @@ -156,7 +156,7 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) if (len) *len = m->len; - if (ctx->ops.dma_unmap(ctx, m->iova, m->len)) { + if (ctx->ops.dma_unmap(ctx, m)) { log_debug("failed to unmap dma\n"); return -1; } @@ -174,7 +174,7 @@ static void __unmap_mapping(void *opaque, struct skiplist_node *n) struct iommu_ctx *ctx = (struct iommu_ctx *)opaque; struct iova_mapping *m = container_of_var(n, m, list); - log_fatal_if(ctx->ops.dma_unmap(ctx, m->len, m->iova), + log_fatal_if(ctx->ops.dma_unmap(ctx, m), "failed to unmap dma (iova 0x%" PRIx64 " len %zu)\n", m->iova, m->len); free(m); diff --git a/src/iommu/iommufd.c b/src/iommu/iommufd.c index 64edd709..c5f24a36 100644 --- a/src/iommu/iommufd.c +++ b/src/iommu/iommufd.c @@ -143,8 +143,7 @@ static int iommufd_get_device_fd(struct iommu_ctx *ctx, const char *bdf) return -1; } -static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, - unsigned long flags) +static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) { struct iommu_ioas *ioas = container_of_var(ctx, ioas, ctx); @@ -152,26 +151,27 @@ static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, .size = sizeof(map), .flags = IOMMU_IOAS_MAP_READABLE | IOMMU_IOAS_MAP_WRITEABLE, .ioas_id = ioas->id, - .user_va = (uint64_t)vaddr, - .length = len, + .user_va = (uint64_t)m->vaddr, + .length = m->len, }; - if (flags & IOMMU_MAP_FIXED_IOVA) { + if (m->flags & IOMMU_MAP_FIXED_IOVA) { map.flags |= IOMMU_IOAS_MAP_FIXED_IOVA; - map.iova = *iova; + map.iova = m->iova; } - if (flags & IOMMU_MAP_NOWRITE) + if (m->flags & IOMMU_MAP_NOWRITE) map.flags &= ~IOMMU_IOAS_MAP_WRITEABLE; - if (flags & IOMMU_MAP_NOREAD) + if (m->flags & IOMMU_MAP_NOREAD) map.flags &= ~IOMMU_IOAS_MAP_READABLE; trace_guard(IOMMUFD_IOAS_MAP_DMA) { - if (flags & IOMMU_MAP_FIXED_IOVA) - trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", vaddr, *iova, len); + if (m->flags & IOMMU_MAP_FIXED_IOVA) + trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", m->vaddr, m->iova, + m->len); else - trace_emit("vaddr %p iova AUTO len %zu\n", vaddr, len); + trace_emit("vaddr %p iova AUTO len %zu\n", m->vaddr, m->len); } if (ioctl(__iommufd, IOMMU_IOAS_MAP, &map)) { @@ -179,21 +179,23 @@ static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, return -1; } - if (flags & IOMMU_MAP_FIXED_IOVA) + if (m->flags & IOMMU_MAP_FIXED_IOVA) return 0; - *iova = map.iova; + m->iova = map.iova; trace_guard(IOMMUFD_IOAS_MAP_DMA) { - trace_emit("allocated iova 0x%" PRIx64 "\n", *iova); + trace_emit("allocated iova 0x%" PRIx64 "\n", m->iova); } return 0; } -static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t len) +static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) { struct iommu_ioas *ioas = container_of_var(ctx, ioas, ctx); + uint64_t iova = m->iova; + size_t len = m->len; struct iommu_ioas_unmap unmap = { .size = sizeof(unmap), @@ -216,7 +218,11 @@ static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t static int iommu_ioas_do_dma_unmap_all(struct iommu_ctx *ctx) { - return iommu_ioas_do_dma_unmap(ctx, 0, UINT64_MAX); + struct iova_mapping m = { + .iova = 0, + .len = UINT64_MAX + }; + return iommu_ioas_do_dma_unmap(ctx, &m); } static const struct iommu_ctx_ops iommufd_ops = { diff --git a/src/iommu/vfio.c b/src/iommu/vfio.c index 0bafa96f..f8931696 100644 --- a/src/iommu/vfio.c +++ b/src/iommu/vfio.c @@ -393,30 +393,29 @@ static int vfio_get_device_fd(struct iommu_ctx *ctx, const char *bdf) return ret_fd; } -static int vfio_iommu_type1_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, - uint64_t *iova, unsigned long flags) +static int vfio_iommu_type1_do_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) { struct vfio_container *vfio = container_of_var(ctx, vfio, ctx); struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(dma_map), - .vaddr = (uintptr_t)vaddr, - .size = len, - .iova = *iova, + .vaddr = (uintptr_t)m->vaddr, + .size = m->len, + .iova = m->iova, .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, }; - if (flags & IOMMU_MAP_NOWRITE) + if (m->flags & IOMMU_MAP_NOWRITE) dma_map.flags &= ~VFIO_DMA_MAP_FLAG_WRITE; - if (flags & IOMMU_MAP_NOREAD) + if (m->flags & IOMMU_MAP_NOREAD) dma_map.flags &= ~VFIO_DMA_MAP_FLAG_READ; trace_guard(VFIO_IOMMU_TYPE1_MAP_DMA) { - trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", vaddr, *iova, len); + trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", m->vaddr, m->iova, m->len); } - if (!ALIGNED(((uintptr_t)vaddr | len | *iova), __VFN_PAGESIZE)) { + if (!ALIGNED(((uintptr_t)m->vaddr | m->len | m->iova), __VFN_PAGESIZE)) { log_debug("vaddr, len or iova not page aligned\n"); errno = EINVAL; return -1; @@ -430,9 +429,11 @@ static int vfio_iommu_type1_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_ return 0; } -static int vfio_iommu_type1_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t len) +static int vfio_iommu_type1_do_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) { struct vfio_container *vfio = container_of_var(ctx, vfio, ctx); + uint64_t iova = m->iova; + size_t len = m->len; struct vfio_iommu_type1_dma_unmap dma_unmap = { .argsz = sizeof(dma_unmap), From 45be9272252a911d70ff02bf4560ab5e5262871c Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 10 Apr 2024 14:58:57 +0200 Subject: [PATCH 15/23] skiplist: replace rand() with arc4random() as it's not present on macOS arc4 is a cryptographically secure RNG, which we don't need, but is what we have on macOS+DriverKit. Signed-off-by: Mads Ynddal --- src/util/skiplist.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/util/skiplist.c b/src/util/skiplist.c index 2decc242..c686ead0 100644 --- a/src/util/skiplist.c +++ b/src/util/skiplist.c @@ -12,6 +12,12 @@ #include +#ifndef __APPLE__ +#define _rand rand +#else +#define _rand arc4random +#endif + #include "skiplist.h" void skiplist_init(struct skiplist *list) @@ -81,7 +87,7 @@ static inline int __skiplist_random_level(void) { int k = 0; - while (k < SKIPLIST_LEVELS - 1 && (rand() > (RAND_MAX / 2))) + while (k < SKIPLIST_LEVELS - 1 && (_rand() > (RAND_MAX / 2))) k++; return k; From a961d1a32edb7fefbea89fba26ce14cf5b46e30e Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 10 Apr 2024 15:42:41 +0200 Subject: [PATCH 16/23] skiplist: Add include guard Added include guard to skiplist.h to avoid duplicate definitions. Signed-off-by: Mads Ynddal --- src/util/skiplist.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/util/skiplist.h b/src/util/skiplist.h index c22ea290..f8906402 100644 --- a/src/util/skiplist.h +++ b/src/util/skiplist.h @@ -10,6 +10,9 @@ * COPYING and LICENSE files for more information. */ +#ifndef LIBVFN_SKIPLIST_H +#define LIBVFN_SKIPLIST_H + #include "ccan/list/list.h" #ifndef SKIPLIST_LEVELS @@ -58,3 +61,5 @@ void skiplist_link(struct skiplist *list, struct skiplist_node *n, struct skiplist_node *update[SKIPLIST_LEVELS]); void skiplist_erase(struct skiplist *list, struct skiplist_node *n, struct skiplist_node *update[SKIPLIST_LEVELS]); + +#endif /* LIBVFN_SKIPLIST_H */ From 290dcd4e6cd2972e7f8d4baf260bc5f058d2b71d Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Thu, 11 Apr 2024 10:17:22 +0200 Subject: [PATCH 17/23] nvme_wait_rdy: timeout for macOS Port timeout to macOS+DriverKit using clock_gettime_nsec_np. Signed-off-by: Mads Ynddal --- src/nvme/core.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/nvme/core.c b/src/nvme/core.c index a79278ba..00809dca 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -19,7 +19,6 @@ #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" #ifndef __APPLE__ -#include #include "ccan/time/time.h" #else #include @@ -382,19 +381,25 @@ int nvme_delete_ioqpair(struct nvme_ctrl *ctrl, int qid) return 0; } +#ifndef __APPLE__ +#define _time_ns time_now().ts.tv_nsec +#else +#define _time_ns clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) +#endif + static int nvme_wait_rdy(struct nvme_ctrl *ctrl, unsigned short rdy) { uint64_t cap; + uint64_t timeout_ns; + uint64_t deadline_ns; uint32_t csts; - unsigned long timeout_ms; - struct timeabs deadline; cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); - timeout_ms = 500 * (NVME_FIELD_GET(cap, CAP_TO) + 1); - deadline = timeabs_add(time_now(), time_from_msec(timeout_ms)); + timeout_ns = 500 * (NVME_FIELD_GET(cap, CAP_TO) + 1) * 1000000; + deadline_ns = _time_ns + timeout_ns; do { - if (time_after(time_now(), deadline)) { + if ((uint64_t)_time_ns > deadline_ns) { log_debug("timed out\n"); errno = ETIMEDOUT; From 3b4bae130a0bcddb7c1e9a1ff656905a278d56b5 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Thu, 11 Apr 2024 11:31:38 +0200 Subject: [PATCH 18/23] nvme_cq_wait_cqes: timeout for macOS and counting ticks Change internal timekeeping to nanoseconds to be universal between Linux and macOS. Signed-off-by: Mads Ynddal --- include/vfn/nvme/queue.h | 4 ++-- src/nvme/queue.c | 7 +++---- src/support/ticks.c | 6 ++++++ tests/device/aer.c | 4 ++-- tests/device/timeout.c | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/include/vfn/nvme/queue.h b/include/vfn/nvme/queue.h index ae5ccc3c..84473970 100644 --- a/include/vfn/nvme/queue.h +++ b/include/vfn/nvme/queue.h @@ -279,7 +279,7 @@ void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n); * @cq: Completion queue * @cqes: Pointer to array of struct cqe to place cqes into * @n: Number of cqes to reap - * @ts: Maximum time to wait for CQEs + * @timeout_ns: Maximum time to wait for CQEs in nanoseconds * * Continuously poll @cq and copy @n cqes into @cqes if not NULL. * @@ -288,6 +288,6 @@ void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n); * Return: ``n`` on success. On timeout, returns the number of cqes reaped * (i.e., less than ``n``) and sets ``errno``. */ -int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, struct timespec *ts); +int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, uint64_t timeout_ns); #endif /* LIBVFN_NVME_QUEUE_H */ diff --git a/src/nvme/queue.c b/src/nvme/queue.c index 37b5304b..cd16b5fd 100644 --- a/src/nvme/queue.c +++ b/src/nvme/queue.c @@ -32,19 +32,18 @@ void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n) } while (n > 0); } -int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, struct timespec *ts) +int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, uint64_t timeout_ns) { struct nvme_cqe *cqe; - struct timerel rel = {.ts = *ts}; uint64_t timeout; - if (!ts) { + if (!timeout_ns) { nvme_cq_get_cqes(cq, cqes, n); return 0; } - timeout = get_ticks() + time_to_usec(rel) * (__vfn_ticks_freq / 1000000ULL); + timeout = get_ticks() + (timeout_ns*1000) * (__vfn_ticks_freq / 1000000ULL); do { cqe = nvme_cq_get_cqe(cq); diff --git a/src/support/ticks.c b/src/support/ticks.c index 4ca07a21..8d183b2f 100644 --- a/src/support/ticks.c +++ b/src/support/ticks.c @@ -19,6 +19,7 @@ uint64_t __vfn_ticks_freq; +#ifndef __APPLE__ static uint64_t measure_ticks_freq(void) { struct timemono t_start, t_end; @@ -62,6 +63,7 @@ static uint64_t estimate_ticks_freq(void) return ROUND(get_ticks() - start, TICKS_PER_10MHZ); } +#endif static void __attribute__((constructor)) init_ticks_freq(void) { @@ -69,11 +71,15 @@ static void __attribute__((constructor)) init_ticks_freq(void) freq = get_ticks_freq_arch(); if (!freq) + #ifndef __APPLE__ freq = measure_ticks_freq(); if (!freq) freq = estimate_ticks_freq(); + #else + abort(); + #endif log_debug("tick frequency is ~%" PRIu64 " Hz\n", freq); __vfn_ticks_freq = freq; diff --git a/tests/device/aer.c b/tests/device/aer.c index 9ffd0b7c..d8a74624 100644 --- a/tests/device/aer.c +++ b/tests/device/aer.c @@ -85,7 +85,7 @@ static int test_aer(void) union nvme_cmd cmd; struct nvme_rq *rq; struct nvme_cqe cqe, cqes[2]; - struct timespec timeout = {.tv_sec = 1}; + uint64_t timeout_ns = 1000000000; int ret; uint32_t temp_thresh; @@ -133,7 +133,7 @@ static int test_aer(void) nvme_rq_exec(rq, &cmd); - ret = nvme_cq_wait_cqes(ctrl.adminq.cq, cqes, 2, &timeout); + ret = nvme_cq_wait_cqes(ctrl.adminq.cq, cqes, 2, timeout_ns); if (ret < 2 && errno == ETIMEDOUT) err(errno, "not enough completions in time"); diff --git a/tests/device/timeout.c b/tests/device/timeout.c index 24a8fb61..e333b487 100644 --- a/tests/device/timeout.c +++ b/tests/device/timeout.c @@ -32,13 +32,13 @@ static int test_timeout(void) { struct nvme_cqe cqe; - struct timespec timeout = {.tv_sec = 1}; + uint64_t timeout_ns = 1000000000; int ret; if (nvme_aer(&ctrl, NULL)) err(1, "could not enable aen"); - ret = nvme_cq_wait_cqes(ctrl.adminq.cq, &cqe, 1, &timeout); + ret = nvme_cq_wait_cqes(ctrl.adminq.cq, &cqe, 1, timeout_ns); if (ret != 1 || errno != ETIMEDOUT) return -1; From 1c48f71fa5e9eb34e3f21b73f7a0df5990e495b5 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Tue, 30 Apr 2024 15:54:22 +0200 Subject: [PATCH 19/23] nvme/core: add serial to nvme_ctrl Added serial string to nvme_ctrl to be able to identify the device. This value is copied from the already issued identify during nvme_init. Signed-off-by: Mads Ynddal --- include/vfn/nvme/ctrl.h | 5 +++++ src/nvme/core.c | 2 ++ src/nvme/types.h | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index b2a686bb..a3c3e1d8 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -45,6 +45,11 @@ struct nvme_ctrl { */ struct vfio_pci_device pci; + /** + * @serial: Serial number from controller + */ + char serial[20]; + /** * @regs: Controller Configuration (``MBAR.CC``) registers */ diff --git a/src/nvme/core.c b/src/nvme/core.c index 00809dca..e8273258 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -617,6 +617,8 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op goto out; } + memcpy(ctrl->serial, ((char *) vaddr) + + NVME_IDENTIFY_CTRL_SERIAL_NUMBER, sizeof(ctrl->serial)); oacs = le16_to_cpu(*(leint16_t *)(((uintptr_t) vaddr) + NVME_IDENTIFY_CTRL_OACS)); if (oacs & NVME_IDENTIFY_CTRL_OACS_DBCONFIG) diff --git a/src/nvme/types.h b/src/nvme/types.h index 5afd0984..7714cfb4 100644 --- a/src/nvme/types.h +++ b/src/nvme/types.h @@ -102,7 +102,8 @@ enum nvme_identify_cns { }; enum nvme_identify_ctrl_offset { - NVME_IDENTIFY_CTRL_OACS = 0x100, + NVME_IDENTIFY_CTRL_SERIAL_NUMBER = 0x4, + NVME_IDENTIFY_CTRL_OACS = 0x100, }; enum nvme_identify_ctrl_oacs { From 87ba60d4629941762340a95a1c7eff6d05577b01 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Fri, 3 May 2024 13:13:20 +0200 Subject: [PATCH 20/23] pgmap/iommu: refactor for opaque pointers Add support for passing opaque pointers around with the pgmap and iommu functions. This is needed for macOS, where the vaddr is not enough to free memory or map with the IOMMU. Signed-off-by: Mads Ynddal --- include/vfn/iommu/dma.h | 4 ++ include/vfn/nvme/queue.h | 3 ++ include/vfn/support.h | 1 - include/vfn/support/mem.h | 20 ++++++++++ include/vfn/support/platform/linux/mem.h | 20 ---------- include/vfn/support/platform/macos/mem.h | 2 - src/iommu/context.h | 1 + src/iommu/dma.c | 14 ++++++- src/nvme/core.c | 49 +++++++++++++++--------- src/nvme/util.c | 3 +- src/support/platform/linux/mem.c | 20 ++++++++++ 11 files changed, 93 insertions(+), 44 deletions(-) diff --git a/include/vfn/iommu/dma.h b/include/vfn/iommu/dma.h index d6abc809..346c72d7 100644 --- a/include/vfn/iommu/dma.h +++ b/include/vfn/iommu/dma.h @@ -31,6 +31,7 @@ enum iommu_map_flags { IOMMU_MAP_NOREAD = 1 << 3, }; +#ifndef __APPLE__ /** * iommu_map_vaddr - Map a virtual memory address to an I/O virtual address * @ctx: &struct iommu_ctx @@ -53,6 +54,9 @@ enum iommu_map_flags { */ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, unsigned long flags); +#endif +int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, + unsigned long flags, void *opaque); /** * iommu_unmap_vaddr - Unmap a virtual memory address in the IOMMU diff --git a/include/vfn/nvme/queue.h b/include/vfn/nvme/queue.h index 84473970..849033d5 100644 --- a/include/vfn/nvme/queue.h +++ b/include/vfn/nvme/queue.h @@ -27,6 +27,7 @@ struct nvme_dbbuf { */ struct nvme_cq { /* private: */ + void *vaddr_opaque; void *vaddr; uint64_t iova; @@ -51,10 +52,12 @@ struct nvme_sq { /* private: */ struct nvme_cq *cq; + void *vaddr_opaque; void *vaddr; uint64_t iova; struct { + void *vaddr_opaque; void *vaddr; uint64_t iova; } pages; diff --git a/include/vfn/support.h b/include/vfn/support.h index eaa86c1f..8839c211 100644 --- a/include/vfn/support.h +++ b/include/vfn/support.h @@ -33,7 +33,6 @@ extern "C" { #else #include #include -#include #endif #include // PRIx64 used for logging diff --git a/include/vfn/support/mem.h b/include/vfn/support/mem.h index 88cea31b..1470fadc 100644 --- a/include/vfn/support/mem.h +++ b/include/vfn/support/mem.h @@ -10,8 +10,28 @@ * COPYING and LICENSE files for more information. */ +#ifndef LIBVFN_SUPPORT_MEM_H +#define LIBVFN_SUPPORT_MEM_H + +ssize_t __pgmap(void **mem, size_t sz, void **opaque); +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque); +void __pgunmap(void *mem, size_t len, void *opaque); + +#define _new_t(t, n, f) \ + ((t *) f(n, sizeof(t))) + +#define new_t(t, n) _new_t(t, n, mallocn) +#define znew_t(t, n) _new_t(t, n, zmallocn) + +ssize_t pgmap(void **mem, size_t sz); +ssize_t pgmapn(void **mem, unsigned int n, size_t sz); + +void pgunmap(void *mem, size_t len); + #ifdef __APPLE__ #include #else #include #endif + +#endif /* LIBVFN_SUPPORT_MEM_H */ diff --git a/include/vfn/support/platform/linux/mem.h b/include/vfn/support/platform/linux/mem.h index 6773c4b1..67b4cf68 100644 --- a/include/vfn/support/platform/linux/mem.h +++ b/include/vfn/support/platform/linux/mem.h @@ -10,9 +10,6 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MEM_H -#define LIBVFN_SUPPORT_MEM_H - extern size_t __VFN_PAGESIZE; extern int __VFN_PAGESHIFT; @@ -108,20 +105,3 @@ static inline void *reallocn(void *mem, unsigned int n, size_t sz) return realloc(mem, n * sz); } - -#define _new_t(t, n, f) \ - ((t *) f(n, sizeof(t))) - -#define new_t(t, n) _new_t(t, n, mallocn) -#define znew_t(t, n) _new_t(t, n, zmallocn) - -ssize_t pgmap(void **mem, size_t sz); -ssize_t pgmapn(void **mem, unsigned int n, size_t sz); - -static inline void pgunmap(void *mem, size_t len) -{ - if (munmap(mem, len)) - backtrace_abort(); -} - -#endif /* LIBVFN_SUPPORT_MEM_H */ diff --git a/include/vfn/support/platform/macos/mem.h b/include/vfn/support/platform/macos/mem.h index 0fe06310..ff6eefca 100644 --- a/include/vfn/support/platform/macos/mem.h +++ b/include/vfn/support/platform/macos/mem.h @@ -10,8 +10,6 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MEM_H -#define LIBVFN_SUPPORT_MEM_H #include #include diff --git a/src/iommu/context.h b/src/iommu/context.h index a45afe70..48080540 100644 --- a/src/iommu/context.h +++ b/src/iommu/context.h @@ -21,6 +21,7 @@ struct iova_mapping { void *vaddr; size_t len; uint64_t iova; + void *opaque[2]; unsigned long flags; diff --git a/src/iommu/dma.c b/src/iommu/dma.c index 97046c75..f8939b12 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -102,8 +102,8 @@ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova) return false; } -int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, - unsigned long flags) +int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, + unsigned long flags, void *opaque) { uint64_t _iova; struct iova_mapping *m; @@ -121,6 +121,8 @@ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *io m->vaddr = vaddr; m->len = len; m->iova = 0; + m->opaque[0] = NULL; + m->opaque[1] = opaque; m->flags = flags; if (ctx->ops.dma_map(ctx, m)) { @@ -143,6 +145,14 @@ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *io return 0; } +#ifndef __APPLE__ +int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, + unsigned long flags) +{ + return _iommu_map_vaddr(ctx, vaddr, len, iova, flags, NULL); +} +#endif + int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) { struct iova_mapping *m; diff --git a/src/nvme/core.c b/src/nvme/core.c index e8273258..e52f3353 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -43,6 +43,7 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec uint64_t cap; uint8_t dstrd; size_t len; + void *opaque; cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); @@ -76,12 +77,12 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec cq->dbbuf.eventidx = cqhdbl(ctrl->dbbuf.eventidxs, qid, dstrd); } - len = pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES); + len = __pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES, &opaque); - if (iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0)) { + if (_iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0, opaque)) { log_debug("failed to map vaddr\n"); - pgunmap(cq->vaddr, len); + __pgunmap(cq->vaddr, len, opaque); return -1; } @@ -98,7 +99,7 @@ static void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq) if (iommu_unmap_vaddr(__iommu_ctx(ctrl), cq->vaddr, &len)) log_debug("failed to unmap vaddr\n"); - pgunmap(cq->vaddr, len); + __pgunmap(cq->vaddr, len, cq->vaddr_opaque); if (ctrl->dbbuf.doorbells) { __STORE_PTR(uint32_t *, cq->dbbuf.doorbell, 0); @@ -152,12 +153,14 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, * Use ctrl->config.mps instead of host page size, as we have the * opportunity to pack the allocations. */ - len = pgmapn(&sq->pages.vaddr, qsize, __mps_to_pagesize(ctrl->config.mps)); + len = __pgmapn(&sq->pages.vaddr, qsize, __mps_to_pagesize(ctrl->config.mps), + &sq->pages.vaddr_opaque); if (len < 0) return -1; - if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, len, &sq->pages.iova, 0x0)) { + if (_iommu_map_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, len, &sq->pages.iova, 0x0, + sq->pages.vaddr_opaque)) { log_debug("failed to map vaddr\n"); goto unmap_pages; } @@ -180,11 +183,11 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, rq->rq_next = &sq->rqs[i - 1]; } - len = pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES); + len = __pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES, &sq->vaddr_opaque); if (len < 0) goto free_sq_rqs; - if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0)) { + if (_iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0, sq->vaddr_opaque)) { log_debug("failed to map vaddr\n"); goto unmap_sq; } @@ -192,14 +195,14 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, return 0; unmap_sq: - pgunmap(sq->vaddr, len); + __pgunmap(sq->vaddr, len, sq->vaddr_opaque); free_sq_rqs: free(sq->rqs); unmap_pages: if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, (size_t *)&len)) log_debug("failed to unmap vaddr\n"); - pgunmap(sq->pages.vaddr, len); + __pgunmap(sq->pages.vaddr, len, sq->pages.vaddr_opaque); return -1; } @@ -214,14 +217,14 @@ static void nvme_discard_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq) if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->vaddr, &len)) log_debug("failed to unmap vaddr\n"); - pgunmap(sq->vaddr, len); + __pgunmap(sq->vaddr, len, sq->vaddr_opaque); free(sq->rqs); if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, &len)) log_debug("failed to unmap vaddr\n"); - pgunmap(sq->pages.vaddr, len); + __pgunmap(sq->pages.vaddr, len, sq->pages.vaddr_opaque); if (ctrl->dbbuf.doorbells) { __STORE_PTR(uint32_t *, sq->dbbuf.doorbell, 0); @@ -455,17 +458,20 @@ static int nvme_init_dbconfig(struct nvme_ctrl *ctrl) { uint64_t prp1, prp2; union nvme_cmd cmd; + void *opaque; - if (pgmap((void **)&ctrl->dbbuf.doorbells, __VFN_PAGESIZE) < 0) + if (__pgmap((void **)&ctrl->dbbuf.doorbells, __VFN_PAGESIZE, &opaque) < 0) return -1; - if (iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.doorbells, __VFN_PAGESIZE, &prp1, 0x0)) + if (_iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.doorbells, __VFN_PAGESIZE, &prp1, 0x0, + opaque)) return -1; - if (pgmap((void **)&ctrl->dbbuf.eventidxs, __VFN_PAGESIZE) < 0) + if (__pgmap((void **)&ctrl->dbbuf.eventidxs, __VFN_PAGESIZE, &opaque) < 0) return -1; - if (iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.eventidxs, __VFN_PAGESIZE, &prp2, 0x0)) + if (_iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.eventidxs, __VFN_PAGESIZE, &prp2, 0x0, + opaque)) return -1; cmd = (union nvme_cmd) { @@ -501,6 +507,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op uint8_t mpsmin, mpsmax; uint16_t oacs; ssize_t len; + void *opaque; void *vaddr; int ret; @@ -602,10 +609,15 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->config.ncqa = min_t(int, ctrl->opts.ncqr, NVME_FIELD_GET(le32_to_cpu(cqe.dw0), FEAT_NRQS_NCQR)); - len = pgmap(&vaddr, NVME_IDENTIFY_DATA_SIZE); + len = __pgmap(&vaddr, NVME_IDENTIFY_DATA_SIZE, &opaque); if (len < 0) return -1; + if (_iommu_map_vaddr(__iommu_ctx(ctrl), vaddr, len, NULL, IOMMU_MAP_EPHEMERAL, opaque)) { + log_error("failed to map vaddr\n"); + return -1; + } + cmd.identify = (struct nvme_cmd_identify) { .opcode = NVME_ADMIN_IDENTIFY, .cns = NVME_IDENTIFY_CNS_CTRL, @@ -625,7 +637,8 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ret = nvme_init_dbconfig(ctrl); out: - pgunmap(vaddr, len); + log_fatal_if(iommu_unmap_vaddr(__iommu_ctx(ctrl), vaddr, NULL), "iommu_unmap_vaddr\n"); + __pgunmap(vaddr, len, opaque); return ret; } diff --git a/src/nvme/util.c b/src/nvme/util.c index 27375c2b..03bd435c 100644 --- a/src/nvme/util.c +++ b/src/nvme/util.c @@ -61,13 +61,14 @@ int nvme_sync(struct nvme_ctrl *ctrl, struct nvme_sq *sq, union nvme_cmd *sqe, v bool do_unmap = false; int ret = 0; + void *opaque = NULL; // Could be null, as we're always mapped? if (buf) { struct iommu_ctx *ctx = __iommu_ctx(ctrl); if (!iommu_translate_vaddr(ctx, buf, &iova)) { do_unmap = true; - if (iommu_map_vaddr(ctx, buf, len, &iova, IOMMU_MAP_EPHEMERAL)) { + if (_iommu_map_vaddr(ctx, buf, len, &iova, IOMMU_MAP_EPHEMERAL, opaque)) { log_debug("failed to map vaddr\n"); return -1; } diff --git a/src/support/platform/linux/mem.c b/src/support/platform/linux/mem.c index 09d34320..9a051e93 100644 --- a/src/support/platform/linux/mem.c +++ b/src/support/platform/linux/mem.c @@ -73,4 +73,24 @@ ssize_t pgmapn(void **mem, unsigned int n, size_t sz) return pgmap(mem, n * sz); } +ssize_t __pgmap(void **mem, size_t sz, void **opaque UNUSED) +{ + return pgmap(mem, sz); +} + +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque UNUSED) +{ + return pgmapn(mem, n, sz); +} + +void __pgunmap(void *mem, size_t len, void *opaque UNUSED) +{ + return pgunmap(mem, len); +} + +void pgunmap(void *mem, size_t len) +{ + if (munmap(mem, len)) + backtrace_abort(); +} From 3b111ddb8e7464d0b7a7fa9509f3d78767c4e654 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Wed, 13 Sep 2023 13:31:49 +0200 Subject: [PATCH 21/23] macos: DriverKit core backend This commit introduces the bulk of macOS support to libvfn. This has to be used together with MacVFN, which introduces a System Extension through DriverKit. Signed-off-by: Mads Ynddal --- include/vfn/driverkit.h | 28 +++++++ include/vfn/driverkit/device.h | 37 ++++++++ include/vfn/driverkit/pci.h | 47 +++++++++++ include/vfn/iommu.h | 2 + include/vfn/iommu/dma.h | 4 + include/vfn/nvme.h | 1 + include/vfn/nvme/ctrl.h | 4 + include/vfn/nvme/queue.h | 4 +- include/vfn/support.h | 1 + src/driverkit/pci.c | 50 +++++++++++ src/iommu/context.c | 10 +++ src/iommu/context.h | 25 ++++++ src/iommu/dma.c | 8 +- src/iommu/driverkit.c | 139 +++++++++++++++++++++++++++++++ src/nvme/core.c | 31 +++++++ src/nvme/rq.c | 6 ++ src/pci/util.c | 26 ++++++ src/support/platform/macos/mem.c | 84 +++++++++++++++++++ 18 files changed, 502 insertions(+), 5 deletions(-) create mode 100644 include/vfn/driverkit.h create mode 100644 include/vfn/driverkit/device.h create mode 100644 include/vfn/driverkit/pci.h create mode 100644 src/driverkit/pci.c create mode 100644 src/iommu/driverkit.c create mode 100644 src/support/platform/macos/mem.c diff --git a/include/vfn/driverkit.h b/include/vfn/driverkit.h new file mode 100644 index 00000000..8d50ffc8 --- /dev/null +++ b/include/vfn/driverkit.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_VFIO_H +#define LIBVFN_VFIO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif /* LIBVFN_VFIO_H */ diff --git a/include/vfn/driverkit/device.h b/include/vfn/driverkit/device.h new file mode 100644 index 00000000..1fb77bbd --- /dev/null +++ b/include/vfn/driverkit/device.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_DRIVERKIT_DEVICE_H +#define LIBVFN_DRIVERKIT_DEVICE_H + +#include +#include +#include "iommu/context.h" + +struct driverkit_bar_info { + uint8_t type; + uint8_t memory_index; + uint64_t size; +}; + +struct driverkit_pci_device { + IOPCIDevice *dev; + struct iommu_ctx ctx; + OSDictionary *iommu_mappings; + const char *bdf; + + struct driverkit_bar_info bar_region_info[6]; +}; + +#define __iommu_ctx(x) (&((struct driverkit_pci_device *)(x))->ctx) + +#endif /* LIBVFN_DRIVERKIT_DEVICE_H */ diff --git a/include/vfn/driverkit/pci.h b/include/vfn/driverkit/pci.h new file mode 100644 index 00000000..01d54415 --- /dev/null +++ b/include/vfn/driverkit/pci.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +#ifndef LIBVFN_VFIO_PCI_H +#define LIBVFN_VFIO_PCI_H + +#define off_t int + +// NOTE: Dummy PROT to satisfy Linux code +enum { + PROT_READ = 0, + PROT_WRITE = 1, +}; + +// NOTE: Faux open for macOS +int vfio_pci_open(struct driverkit_pci_device *pci, const char *bdf); + +/** + * vfio_pci_map_bar - faux mapping in DriverKit + * @pci: &struct driverkit_pci_device + * @idx: the vfio region index to map + * @len: number of bytes to map + * @offset: offset at which to start mapping + * @prot: what accesses to permit to the mapped area (see ``man mmap``). + * + * Used in place of VFIO map device memory region identified by @idx into virtual memory. + * + * Return: On success, returns the virtual memory address mapped. On error, + * returns ``NULL`` and sets ``errno``. + */ +void *vfio_pci_map_bar(struct driverkit_pci_device *pci, int idx, size_t len, uint64_t offset, + int prot); + +/** + * vfio_pci_unmap_bar - faux unmap a region in virtual memory for DriverKit + * @pci: &struct driverkit_pci_device + * @idx: the vfio region index to unmap + * @mem: virtual memory address to unmap + * @len: number of bytes to unmap + * @offset: offset at which to start unmapping + * + * Used in place of VFIO to unmap the virtual memory address, previously mapped + * to the vfio device memory region identified by @idx. + */ +void vfio_pci_unmap_bar(struct driverkit_pci_device *pci, int idx, void *mem, size_t len, + uint64_t offset); + +#endif /* LIBVFN_VFIO_PCI_H */ diff --git a/include/vfn/iommu.h b/include/vfn/iommu.h index 43642ea0..9f78bb41 100644 --- a/include/vfn/iommu.h +++ b/include/vfn/iommu.h @@ -17,7 +17,9 @@ #include #include +#ifndef __APPLE__ #include +#endif #include #include diff --git a/include/vfn/iommu/dma.h b/include/vfn/iommu/dma.h index 346c72d7..e21ab5a0 100644 --- a/include/vfn/iommu/dma.h +++ b/include/vfn/iommu/dma.h @@ -82,12 +82,14 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len); */ int iommu_unmap_all(struct iommu_ctx *ctx); +#ifndef __APPLE__ #ifndef IOMMU_IOAS_IOVA_RANGES struct iommu_iova_range { __aligned_u64 start; __aligned_u64 last; }; #endif +#endif /** * iommu_translate_vaddr - Translate a virtual address into an iova @@ -102,6 +104,7 @@ struct iommu_iova_range { */ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova); +#ifndef __APPLE__ /** * iommu_get_iova_ranges - Get iova ranges * @ctx: &struct iommu_ctx @@ -112,5 +115,6 @@ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova); * Return: the number of elements in the array pointed to. */ int iommu_get_iova_ranges(struct iommu_ctx *ctx, struct iommu_iova_range **ranges); +#endif #endif /* LIBVFN_IOMMU_DMA_H */ diff --git a/include/vfn/nvme.h b/include/vfn/nvme.h index d42f6e8f..0dcb720a 100644 --- a/include/vfn/nvme.h +++ b/include/vfn/nvme.h @@ -18,6 +18,7 @@ #ifndef __APPLE__ #include #else +#define NVME_AQ_QSIZE 32 #include #endif diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index a3c3e1d8..570baf7a 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -43,7 +43,11 @@ struct nvme_ctrl { /** * @pci: vfio pci device state */ + #ifdef __APPLE__ + struct driverkit_pci_device pci; + #else struct vfio_pci_device pci; + #endif /** * @serial: Serial number from controller diff --git a/include/vfn/nvme/queue.h b/include/vfn/nvme/queue.h index 849033d5..54cba926 100644 --- a/include/vfn/nvme/queue.h +++ b/include/vfn/nvme/queue.h @@ -87,7 +87,7 @@ struct nvme_sq { */ static inline void nvme_sq_post(struct nvme_sq *sq, const union nvme_cmd *sqe) { - memcpy(sq->vaddr + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES); + memcpy(((uint8_t *) sq->vaddr) + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES); trace_guard(NVME_SQ_POST) { trace_emit("sqid %d tail %d\n", sq->id, sq->tail); @@ -177,7 +177,7 @@ static inline void nvme_sq_exec(struct nvme_sq *sq, const union nvme_cmd *sqe) */ static inline struct nvme_cqe *nvme_cq_head(struct nvme_cq *cq) { - return (struct nvme_cqe *)(cq->vaddr + (cq->head << NVME_CQES)); + return (struct nvme_cqe *)(((uint8_t *) cq->vaddr) + (cq->head << NVME_CQES)); } /** diff --git a/include/vfn/support.h b/include/vfn/support.h index 8839c211..eaa86c1f 100644 --- a/include/vfn/support.h +++ b/include/vfn/support.h @@ -33,6 +33,7 @@ extern "C" { #else #include #include +#include #endif #include // PRIx64 used for logging diff --git a/src/driverkit/pci.c b/src/driverkit/pci.c new file mode 100644 index 00000000..fe2cb74c --- /dev/null +++ b/src/driverkit/pci.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + + + +#define log_fmt(fmt) "driverkit/pci: " fmt + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// NOTE: Faux open for macOS +int vfio_pci_open(struct driverkit_pci_device *pci, const char *bdf) +{ + return 0; +} + +void *vfio_pci_map_bar(struct driverkit_pci_device *pci, int idx, size_t len, uint64_t offset, + int prot) +{ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) zmallocn(1, + sizeof(struct macvfn_pci_map_bar)); + mapping->pci = pci; + mapping->idx = idx; + mapping->len = len; + mapping->offset = offset; + + return mapping; +} + +void vfio_pci_unmap_bar(struct driverkit_pci_device *pci, int idx, void *mem, size_t len, + uint64_t offset) +{ + free(mem); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/iommu/context.c b/src/iommu/context.c index 46ec2186..79b0261c 100644 --- a/src/iommu/context.c +++ b/src/iommu/context.c @@ -45,11 +45,16 @@ struct iommu_ctx *iommu_get_default_context(void) fallback: #endif +#ifdef __APPLE__ + return driverkit_get_default_iommu_context(); +#else return vfio_get_default_iommu_context(); +#endif } struct iommu_ctx *iommu_get_context(const char *name) { +#ifndef __APPLE__ #ifdef HAVE_VFIO_DEVICE_BIND_IOMMUFD if (__iommufd_broken) goto fallback; @@ -59,8 +64,12 @@ struct iommu_ctx *iommu_get_context(const char *name) fallback: #endif return vfio_get_iommu_context(name); +#else + return driverkit_get_iommu_context(name); +#endif } +#ifndef __APPLE__ void iommu_ctx_init(struct iommu_ctx *ctx) { ctx->nranges = 1; @@ -78,3 +87,4 @@ void iommu_ctx_init(struct iommu_ctx *ctx) skiplist_init(&ctx->map.list); pthread_mutex_init(&ctx->map.lock, NULL); } +#endif diff --git a/src/iommu/context.h b/src/iommu/context.h index 48080540..97d8f034 100644 --- a/src/iommu/context.h +++ b/src/iommu/context.h @@ -13,8 +13,13 @@ #ifndef LIBVFN_SRC_IOMMU_CONTEXT_H #define LIBVFN_SRC_IOMMU_CONTEXT_H + +#ifdef __cplusplus +extern "C" { +#endif #include "util/skiplist.h" + struct iommu_ctx; struct iova_mapping { @@ -42,7 +47,11 @@ struct iommu_ctx_ops { }; struct iova_map { + #ifndef __APPLE__ pthread_mutex_t lock; + #else + IOLock * lock; + #endif struct skiplist list; }; @@ -50,13 +59,19 @@ struct iommu_ctx { struct iova_map map; struct iommu_ctx_ops ops; + #ifndef __APPLE__ pthread_mutex_t lock; int nranges; struct iommu_iova_range *iova_ranges; + #else + IOPCIDevice *pci; + IOLock *lock; + #endif }; struct iommu_ctx *iommu_get_default_context(void); +#ifndef __APPLE__ struct iommu_ctx *vfio_get_default_iommu_context(void); struct iommu_ctx *vfio_get_iommu_context(const char *name); @@ -64,8 +79,18 @@ struct iommu_ctx *vfio_get_iommu_context(const char *name); struct iommu_ctx *iommufd_get_default_iommu_context(void); struct iommu_ctx *iommufd_get_iommu_context(const char *name); #endif +#else +struct iommu_ctx *driverkit_get_default_iommu_context(void); +struct iommu_ctx *driverkit_get_iommu_context(const char *name); +#endif void iommu_ctx_init(struct iommu_ctx *ctx); +#ifndef __APPLE__ int iommu_iova_range_to_string(struct iommu_iova_range *range, char **str); +#endif + +#ifdef __cplusplus +} +#endif #endif /* LIBVFN_SRC_IOMMU_CONTEXT_H */ diff --git a/src/iommu/dma.c b/src/iommu/dma.c index f8939b12..92816021 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -120,9 +120,9 @@ int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *i m = znew_t(struct iova_mapping, 1); m->vaddr = vaddr; m->len = len; - m->iova = 0; - m->opaque[0] = NULL; - m->opaque[1] = opaque; + m->iova = 0; // Physical/IOMMU address + m->opaque[0] = NULL; // IODMACommand + m->opaque[1] = opaque; // IOMemoryDescriptor m->flags = flags; if (ctx->ops.dma_map(ctx, m)) { @@ -208,6 +208,7 @@ int iommu_unmap_all(struct iommu_ctx *ctx) return 0; } +#ifndef __APPLE__ int iommu_get_iova_ranges(struct iommu_ctx *ctx, struct iommu_iova_range **ranges) { *ranges = ctx->iova_ranges; @@ -218,6 +219,7 @@ int iommu_iova_range_to_string(struct iommu_iova_range *r, char **str) { return asprintf(str, "[0x%llx; 0x%llx]", r->start, r->last); } +#endif #ifdef __cplusplus } diff --git a/src/iommu/driverkit.c b/src/iommu/driverkit.c new file mode 100644 index 00000000..a1d0a0cb --- /dev/null +++ b/src/iommu/driverkit.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2023 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define log_fmt(fmt) "iommu/vfio: " fmt + +#include "ccan/str/str.h" +#include "ccan/compiler/compiler.h" +#include "ccan/minmax/minmax.h" + +#include "vfn/driverkit.h" +#include "vfn/support.h" +#include "vfn/iommu.h" +#include "vfn/trace.h" +#include "vfn/pci/util.h" + +#include + +#include "context.h" + +static int driverkit_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) +{ + kern_return_t ret; + + IODMACommand **dmaCommand = (IODMACommand **) &m->opaque[0]; + + log_debug("DriverKit DMA mapping (PrepareForDMA) %p %zu, %p, %p", m->vaddr, m->len, + m->opaque[0], m->opaque[1]); + IODMACommandSpecification dmaSpecification = { + .options = kIODMACommandSpecificationNoOptions, + .maxAddressBits = 64, + }; + struct driverkit_pci_device *dev = container_of(ctx, struct driverkit_pci_device, ctx); + + ret = IODMACommand::Create(dev->dev, kIODMACommandCreateNoOptions, &dmaSpecification, + dmaCommand); + if (ret != kIOReturnSuccess) { + log_error("IODMACommand::Create failed with error code: %x", ret); + return -1; + } + + if (!m->opaque[1]) { + log_error("DriverKit DMA mapping: opaque[1] is NULL"); + return -1; + } + + uint64_t dmaFlags = 0; + uint32_t dmaSegmentCount = 1; + IOAddressSegment physicalAddressSegment; + + ret = (*dmaCommand)->PrepareForDMA( + kIODMACommandPrepareForDMANoOptions, + (IOMemoryDescriptor *) m->opaque[1], + 0, + m->len, + &dmaFlags, + &dmaSegmentCount, + &physicalAddressSegment + ); + if (ret != kIOReturnSuccess) { + log_error("failed to PrepareForDMA %x", ret); + return -1; + } + if (dmaSegmentCount != 1) { + log_error("dmaSegmentCount not 1! %u", dmaSegmentCount); + return -1; + } + assert(m->len == physicalAddressSegment.length); + m->iova = physicalAddressSegment.address; + + log_debug("DriverKit DMA mapping: iova %llx", m->iova); + + return 0; +} + +static int driverkit_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) +{ + log_debug("DriverKit DMA unmapping (CompleteDMA) %p %zu", m->vaddr, m->len); + IODMACommand *dmaCommand = (IODMACommand *) m->opaque[0]; + kern_return_t ret = (int) dmaCommand->CompleteDMA(kIODMACommandCompleteDMANoOptions); + + if (ret != kIOReturnSuccess) { + log_error("DriverKit DMA unmapping failed to complete DMA"); + return -1; + } + OSSafeReleaseNULL(dmaCommand); + m->opaque[0] = NULL; + return 0; +} + +static int driverkit_dma_unmap_all(struct iommu_ctx *ctx) +{ + return 0; +} + +static const struct iommu_ctx_ops driverkit_ops = { + .get_device_fd = NULL, + + .iova_reserve = NULL, + .iova_put_ephemeral = NULL, + + .dma_map = driverkit_dma_map, + .dma_unmap = driverkit_dma_unmap, + .dma_unmap_all = driverkit_dma_unmap_all, +}; + +void iommu_ctx_init(struct iommu_ctx *ctx) +{ + memcpy(&ctx->ops, &driverkit_ops, sizeof(ctx->ops)); + ctx->lock = IOLockAlloc(); + ctx->map.lock = IOLockAlloc(); + skiplist_init(&ctx->map.list); +} + +struct iommu_ctx *driverkit_get_iommu_context(const char *name) +{ + return NULL; +} + +struct iommu_ctx *driverkit_get_default_iommu_context(void) +{ + return NULL; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/nvme/core.c b/src/nvme/core.c index e52f3353..4b247681 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -27,11 +27,42 @@ int errno; #include "types.h" +#ifdef __APPLE__ +inline void *cqhdbl(void *doorbells, int qid, int dstrd) +{ + struct macvfn_pci_map_bar *doorbell_mapping = (struct macvfn_pci_map_bar *) doorbells; + struct macvfn_pci_map_bar *new_mapping = (struct macvfn_pci_map_bar *) zmallocn(1, + sizeof(struct macvfn_pci_map_bar)); + + new_mapping->pci = doorbell_mapping->pci; + new_mapping->idx = doorbell_mapping->idx; + new_mapping->len = doorbell_mapping->len; + new_mapping->offset = doorbell_mapping->offset + (2 * qid + 1) * (4 << dstrd); + + return new_mapping; +} + +inline void *sqtdbl(void *doorbells, int qid, int dstrd) +{ + struct macvfn_pci_map_bar *doorbell_mapping = (struct macvfn_pci_map_bar *) doorbells; + struct macvfn_pci_map_bar *new_mapping = (struct macvfn_pci_map_bar *) zmallocn(1, + sizeof(struct macvfn_pci_map_bar)); + + new_mapping->pci = doorbell_mapping->pci; + new_mapping->idx = doorbell_mapping->idx; + new_mapping->len = doorbell_mapping->len; + new_mapping->offset = doorbell_mapping->offset + (2 * qid) * (4 << dstrd); + + return new_mapping; +} +#else #define cqhdbl(doorbells, qid, dstrd) \ (doorbells + (2 * qid + 1) * (4 << dstrd)) #define sqtdbl(doorbells, qid, dstrd) \ (doorbells + (2 * qid) * (4 << dstrd)) +#endif + enum nvme_ctrl_feature_flags { NVME_CTRL_F_ADMINISTRATIVE = 1 << 0, diff --git a/src/nvme/rq.c b/src/nvme/rq.c index c73e0820..423313cc 100644 --- a/src/nvme/rq.c +++ b/src/nvme/rq.c @@ -96,6 +96,7 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * return 0; } +#ifndef __APPLE__ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, struct iovec *iov, int niov) { @@ -169,6 +170,7 @@ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd errno = EINVAL; return -1; } +#endif int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) { @@ -187,7 +189,11 @@ int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) } if (!nvme_cqe_ok(&cqe)) { + #ifdef __APPLE__ + if (false) { + #else if (logv(LOG_DEBUG)) { + #endif uint16_t status = le16_to_cpu(cqe.sfp) >> 1; log_debug("cqe status 0x%" PRIx16 "\n", (uint16_t)(status & 0x7ff)); diff --git a/src/pci/util.c b/src/pci/util.c index de3e549c..15a7bb2d 100644 --- a/src/pci/util.c +++ b/src/pci/util.c @@ -10,12 +10,33 @@ * COPYING and LICENSE files for more information. */ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef __APPLE__ #include #include +#endif + #include #include +#include + +#ifdef __APPLE__ +int pci_device_info_get_ull(const char *bdf, const char *prop, unsigned long long *v) +{ + if (!strcmp(prop, "class")) { + // TODO: Actually read this from device once it makes sense + *v = 0x010800; + } else { + return -1; + } + return 0; +} +#else int pci_unbind(const char *bdf) { char *path = NULL; @@ -263,3 +284,8 @@ char *pci_get_device_vfio_id(const char *bdf) return vfio_id; } +#endif + +#ifdef __cplusplus +} +#endif diff --git a/src/support/platform/macos/mem.c b/src/support/platform/macos/mem.c new file mode 100644 index 00000000..5d141676 --- /dev/null +++ b/src/support/platform/macos/mem.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +size_t __VFN_PAGESIZE; +int __VFN_PAGESHIFT; + +static void __attribute__((constructor)) init_page_size(void) +{ + __VFN_PAGESIZE = IOVMPageSize; + __VFN_PAGESHIFT = 63u - __builtin_clzl(__VFN_PAGESIZE); + + log_debug("pagesize is %zu (shift %d)\n", __VFN_PAGESIZE, __VFN_PAGESHIFT); +} + +void backtrace_abort(void) +{ + abort(); +} + +ssize_t __pgmap(void **mem, size_t sz, void **opaque) +{ + IOAddressSegment virtualAddressSegment; + ssize_t len = ALIGN_UP(sz, __VFN_PAGESIZE); + + IOBufferMemoryDescriptor **mem_descriptor = (IOBufferMemoryDescriptor **) opaque; + IOBufferMemoryDescriptor::Create( + kIOMemoryDirectionInOut, + len, + __VFN_PAGESIZE, + mem_descriptor + ); + (*mem_descriptor)->SetLength(len); + (*mem_descriptor)->GetAddressRange(&virtualAddressSegment); + bzero((void *) virtualAddressSegment.address, virtualAddressSegment.length); + *mem = (void *) virtualAddressSegment.address; + + return len; +} + +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque) +{ + return __pgmap(mem, n * sz, opaque); +} + +void __pgunmap(void *mem, size_t len, void *opaque) +{ + IOBufferMemoryDescriptor *_mem = (IOBufferMemoryDescriptor *) opaque; + + OSSafeReleaseNULL(_mem); +} + +ssize_t pgmap(void **mem, size_t sz) +{ + log_fatal("Use ::__pgmap on macOS"); +} + +ssize_t pgmapn(void **mem, unsigned int n, size_t sz) +{ + log_fatal("Use ::__pgmapn on macOS"); +} + +void pgunmap(void *mem, size_t len) +{ + log_fatal("Use ::__pgunmap on macOS"); +} + +#ifdef __cplusplus +} +#endif From e2144c2aae6e3fa297195c037ff7b2eb92992f12 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Fri, 3 May 2024 13:43:25 +0200 Subject: [PATCH 22/23] misc/log: change logging on unrecoverable errors Change log-statements from 'log_debug' to 'log_error', when they are in relation to an unrecoverable error. As a rule of thumb, if the log is immediately followed by 'return -1', then it's an error. If we can continue but just log for helping, it's debug. Signed-off-by: Mads Ynddal --- src/iommu/dma.c | 6 ++--- src/iommu/iommufd.c | 22 +++++++++--------- src/iommu/vfio.c | 38 +++++++++++++++---------------- src/nvme/core.c | 54 ++++++++++++++++++++++----------------------- src/nvme/util.c | 2 +- src/pci/util.c | 32 +++++++++++++-------------- src/vfio/device.c | 6 ++--- src/vfio/pci.c | 20 ++++++++--------- 8 files changed, 90 insertions(+), 90 deletions(-) diff --git a/src/iommu/dma.c b/src/iommu/dma.c index 92816021..5a15c25b 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -113,7 +113,7 @@ int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *i if (flags & IOMMU_MAP_FIXED_IOVA) { _iova = *iova; } else if (ctx->ops.iova_reserve && ctx->ops.iova_reserve(ctx, len, &_iova, flags)) { - log_debug("failed to allocate iova\n"); + log_error("failed to allocate iova\n"); return -1; } @@ -167,7 +167,7 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) *len = m->len; if (ctx->ops.dma_unmap(ctx, m)) { - log_debug("failed to unmap dma\n"); + log_error("failed to unmap dma\n"); return -1; } @@ -194,7 +194,7 @@ int iommu_unmap_all(struct iommu_ctx *ctx) { if (ctx->ops.dma_unmap_all) { if (ctx->ops.dma_unmap_all(ctx)) { - log_debug("failed to unmap dma\n"); + log_error("failed to unmap dma\n"); return -1; } diff --git a/src/iommu/iommufd.c b/src/iommu/iommufd.c index c5f24a36..956f9a4c 100644 --- a/src/iommu/iommufd.c +++ b/src/iommu/iommufd.c @@ -51,7 +51,7 @@ static int iommu_ioas_update_iova_ranges(struct iommu_ioas *ioas) if (ioctl(__iommufd, IOMMU_IOAS_IOVA_RANGES, &iova_ranges)) { if (errno != EMSGSIZE) { - log_debug("could not get ioas iova ranges\n"); + log_error("could not get ioas iova ranges\n"); return -1; } @@ -62,7 +62,7 @@ static int iommu_ioas_update_iova_ranges(struct iommu_ioas *ioas) iova_ranges.allowed_iovas = (uintptr_t)ioas->ctx.iova_ranges; if (ioctl(__iommufd, IOMMU_IOAS_IOVA_RANGES, &iova_ranges)) { - log_debug("could not get ioas iova ranges\n"); + log_error("could not get ioas iova ranges\n"); return -1; } } @@ -104,33 +104,33 @@ static int iommufd_get_device_fd(struct iommu_ctx *ctx, const char *bdf) vfio_id = pci_get_device_vfio_id(bdf); if (!vfio_id) { - log_debug("could not determine the vfio device id for %s\n", bdf); + log_error("could not determine the vfio device id for %s\n", bdf); return -1; } if (asprintf(&path, "/dev/vfio/devices/%s", vfio_id) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } devfd = open(path, O_RDWR); if (devfd < 0) { - log_debug("could not open the device cdev\n"); + log_error("could not open the device cdev\n"); return -1; } if (ioctl(devfd, VFIO_DEVICE_BIND_IOMMUFD, &bind)) { - log_debug("could not bind device to iommufd\n"); + log_error("could not bind device to iommufd\n"); goto close_dev; } if (ioctl(devfd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data)) { - log_debug("could not associate device with ioas\n"); + log_error("could not associate device with ioas\n"); goto close_dev; } if (iommu_ioas_update_iova_ranges(ioas)) { - log_debug("could not update iova ranges\n"); + log_error("could not update iova ranges\n"); goto close_dev; } @@ -175,7 +175,7 @@ static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) } if (ioctl(__iommufd, IOMMU_IOAS_MAP, &map)) { - log_debug("failed to map\n"); + log_error("failed to map\n"); return -1; } @@ -209,7 +209,7 @@ static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m } if (ioctl(__iommufd, IOMMU_IOAS_UNMAP, &unmap)) { - log_debug("failed to unmap\n"); + log_error("failed to unmap\n"); return -1; } @@ -241,7 +241,7 @@ static int iommu_ioas_init(struct iommu_ioas *ioas) }; if (ioctl(__iommufd, IOMMU_IOAS_ALLOC, &alloc_data)) { - log_debug("could not allocate ioas\n"); + log_error("could not allocate ioas\n"); return -1; } diff --git a/src/iommu/vfio.c b/src/iommu/vfio.c index f8931696..4f91e1db 100644 --- a/src/iommu/vfio.c +++ b/src/iommu/vfio.c @@ -202,7 +202,7 @@ static int vfio_iommu_type1_iova_reserve(struct iommu_ctx *ctx, size_t len, uint __autolock(&vfio->lock); if (!ALIGNED(len, __VFN_PAGESIZE)) { - log_debug("len is not page aligned\n"); + log_error("len is not page aligned\n"); errno = EINVAL; return -1; } @@ -232,7 +232,7 @@ static int vfio_iommu_type1_init(struct vfio_container *vfio) return 0; if (ioctl(vfio->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)) { - log_debug("failed to set vfio iommu type\n"); + log_error("failed to set vfio iommu type\n"); return -1; } @@ -240,13 +240,13 @@ static int vfio_iommu_type1_init(struct vfio_container *vfio) #ifdef VFIO_IOMMU_INFO_CAPS if (vfio_iommu_type1_get_capabilities(vfio)) { - log_debug("failed to get iommu capabilities\n"); + log_error("failed to get iommu capabilities\n"); return -1; } #endif if (vfio_iommu_type1_iova_reserve(&vfio->ctx, VFIO_IOMMU_TYPE1_IOVA_RESERVED, &iova, 0x0)) { - log_debug("could not reserve iova range\n"); + log_error("could not reserve iova range\n"); return -1; } @@ -270,12 +270,12 @@ static int vfio_group_set_container(struct vfio_group *group, struct vfio_contai log_info("adding group '%s' to container\n", group->path); if (ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &vfio->fd)) { - log_debug("failed to add group to vfio container\n"); + log_error("failed to add group to vfio container\n"); return -1; } if (vfio_iommu_type1_init(vfio)) { - log_debug("failed to configure iommu\n"); + log_error("failed to configure iommu\n"); log_fatal_if(ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER), "unset container\n"); @@ -295,18 +295,18 @@ static int vfio_group_open(const char *path) fd = open(path, O_RDWR); if (fd < 0) { - log_debug("failed to open vfio group file: %s\n", strerror(errno)); + log_error("failed to open vfio group file: %s\n", strerror(errno)); return -1; } if (ioctl(fd, VFIO_GROUP_GET_STATUS, &group_status)) { - log_debug("failed to get vfio group status\n"); + log_error("failed to get vfio group status\n"); goto close_fd; } if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { errno = EINVAL; - log_debug("vfio group is not viable\n"); + log_error("vfio group is not viable\n"); goto close_fd; } @@ -348,7 +348,7 @@ static int vfio_get_group_fd(struct vfio_container *vfio, const char *path) group->fd = vfio_group_open(group->path); if (group->fd < 0) { - log_debug("failed to open vfio group\n"); + log_error("failed to open vfio group\n"); goto free_group_path; } @@ -374,7 +374,7 @@ static int vfio_get_device_fd(struct iommu_ctx *ctx, const char *bdf) group = pci_get_iommu_group(bdf); if (!group) { - log_debug("could not determine iommu group for device %s\n", bdf); + log_error("could not determine iommu group for device %s\n", bdf); errno = EINVAL; return -1; } @@ -386,7 +386,7 @@ static int vfio_get_device_fd(struct iommu_ctx *ctx, const char *bdf) ret_fd = ioctl(gfd, VFIO_GROUP_GET_DEVICE_FD, bdf); if (ret_fd < 0) { - log_debug("failed to get device fd\n"); + log_error("failed to get device fd\n"); return -1; } @@ -416,13 +416,13 @@ static int vfio_iommu_type1_do_dma_map(struct iommu_ctx *ctx, struct iova_mappin } if (!ALIGNED(((uintptr_t)m->vaddr | m->len | m->iova), __VFN_PAGESIZE)) { - log_debug("vaddr, len or iova not page aligned\n"); + log_error("vaddr, len or iova not page aligned\n"); errno = EINVAL; return -1; } if (ioctl(vfio->fd, VFIO_IOMMU_MAP_DMA, &dma_map)) { - log_debug("could not map\n"); + log_error("could not map\n"); return -1; } @@ -446,7 +446,7 @@ static int vfio_iommu_type1_do_dma_unmap(struct iommu_ctx *ctx, struct iova_mapp } if (ioctl(vfio->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap)) { - log_debug("could not unmap\n"); + log_error("could not unmap\n"); return -1; } @@ -484,7 +484,7 @@ static int vfio_iommu_type1_do_dma_unmap_all(struct iommu_ctx *ctx) }; if (ioctl(vfio->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap)) { - log_debug("failed to unmap dma\n"); + log_error("failed to unmap dma\n"); return -1; } @@ -509,17 +509,17 @@ static int vfio_init_container(struct vfio_container *vfio) { vfio->fd = open("/dev/vfio/vfio", O_RDWR); if (vfio->fd < 0) { - log_debug("failed to open vfio device\n"); + log_error("failed to open vfio device\n"); return -1; } if (ioctl(vfio->fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION) { - log_debug("invalid vfio version\n"); + log_error("invalid vfio version\n"); return -1; } if (!ioctl(vfio->fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) { - log_debug("vfio type 1 iommu not supported\n"); + log_error("vfio type 1 iommu not supported\n"); return -1; } diff --git a/src/nvme/core.c b/src/nvme/core.c index 4b247681..81dd17d0 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -80,14 +80,14 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); if (qid && qid > ctrl->config.ncqa + 1) { - log_debug("qid %d invalid; max qid is %d\n", qid, ctrl->config.ncqa + 1); + log_error("qid %d invalid; max qid is %d\n", qid, ctrl->config.ncqa + 1); errno = EINVAL; return -1; } if (qsize < 2) { - log_debug("qsize must be at least 2\n"); + log_error("qsize must be at least 2\n"); errno = EINVAL; return -1; } @@ -111,7 +111,7 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec len = __pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES, &opaque); if (_iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0, opaque)) { - log_debug("failed to map vaddr\n"); + log_error("failed to map vaddr\n"); __pgunmap(cq->vaddr, len, opaque); return -1; @@ -128,7 +128,7 @@ static void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq) return; if (iommu_unmap_vaddr(__iommu_ctx(ctrl), cq->vaddr, &len)) - log_debug("failed to unmap vaddr\n"); + log_error("failed to unmap vaddr\n"); __pgunmap(cq->vaddr, len, cq->vaddr_opaque); @@ -152,14 +152,14 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); if (qid && qid > ctrl->config.nsqa + 1) { - log_debug("qid %d invalid; max qid is %d\n", qid, ctrl->config.nsqa + 1); + log_error("qid %d invalid; max qid is %d\n", qid, ctrl->config.nsqa + 1); errno = EINVAL; return -1; } if (qsize < 2) { - log_debug("qsize must be at least 2\n"); + log_error("qsize must be at least 2\n"); errno = EINVAL; return -1; } @@ -192,7 +192,7 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, if (_iommu_map_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, len, &sq->pages.iova, 0x0, sq->pages.vaddr_opaque)) { - log_debug("failed to map vaddr\n"); + log_error("failed to map vaddr\n"); goto unmap_pages; } @@ -219,7 +219,7 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, goto free_sq_rqs; if (_iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0, sq->vaddr_opaque)) { - log_debug("failed to map vaddr\n"); + log_error("failed to map vaddr\n"); goto unmap_sq; } @@ -231,7 +231,7 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, free(sq->rqs); unmap_pages: if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, (size_t *)&len)) - log_debug("failed to unmap vaddr\n"); + log_error("failed to unmap vaddr\n"); __pgunmap(sq->pages.vaddr, len, sq->pages.vaddr_opaque); @@ -273,12 +273,12 @@ static int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags) struct nvme_sq *sq = &ctrl->sq[NVME_AQ]; if (nvme_configure_cq(ctrl, NVME_AQ, NVME_AQ_QSIZE, 0)) { - log_debug("failed to configure admin completion queue\n"); + log_error("failed to configure admin completion queue\n"); return -1; } if (nvme_configure_sq(ctrl, NVME_AQ, NVME_AQ_QSIZE, cq, sq_flags)) { - log_debug("failed to configure admin submission queue\n"); + log_error("failed to configure admin submission queue\n"); goto discard_cq; } @@ -313,7 +313,7 @@ int nvme_create_iocq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector) uint16_t iv = 0; if (nvme_configure_cq(ctrl, qid, qsize, vector)) { - log_debug("could not configure io completion queue\n"); + log_error("could not configure io completion queue\n"); return -1; } @@ -355,7 +355,7 @@ int nvme_create_iosq(struct nvme_ctrl *ctrl, int qid, int qsize, struct nvme_cq union nvme_cmd cmd; if (nvme_configure_sq(ctrl, qid, qsize, cq, flags)) { - log_debug("could not configure io submission queue\n"); + log_error("could not configure io submission queue\n"); return -1; } @@ -388,12 +388,12 @@ int nvme_delete_iosq(struct nvme_ctrl *ctrl, int qid) int nvme_create_ioqpair(struct nvme_ctrl *ctrl, int qid, int qsize, int vector, unsigned long flags) { if (nvme_create_iocq(ctrl, qid, qsize, vector)) { - log_debug("could not create io completion queue\n"); + log_error("could not create io completion queue\n"); return -1; } if (nvme_create_iosq(ctrl, qid, qsize, &ctrl->cq[qid], flags)) { - log_debug("could not create io submission queue\n"); + log_error("could not create io submission queue\n"); return -1; } @@ -403,12 +403,12 @@ int nvme_create_ioqpair(struct nvme_ctrl *ctrl, int qid, int qsize, int vector, int nvme_delete_ioqpair(struct nvme_ctrl *ctrl, int qid) { if (nvme_delete_iosq(ctrl, qid)) { - log_debug("could not delete io submission queue\n"); + log_error("could not delete io submission queue\n"); return -1; } if (nvme_delete_iocq(ctrl, qid)) { - log_debug("could not delete io completion queue\n"); + log_error("could not delete io completion queue\n"); return -1; } @@ -434,7 +434,7 @@ static int nvme_wait_rdy(struct nvme_ctrl *ctrl, unsigned short rdy) do { if ((uint64_t)_time_ns > deadline_ns) { - log_debug("timed out\n"); + log_error("timed out\n"); errno = ETIMEDOUT; return -1; @@ -551,14 +551,14 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op memcpy(&ctrl->opts, &nvme_ctrl_opts_default, sizeof(*opts)); if (pci_device_info_get_ull(bdf, "class", &classcode)) { - log_debug("could not get device class code\n"); + log_error("could not get device class code\n"); return -1; } log_info("pci class code is 0x%06llx\n", classcode); if ((classcode & 0xffff00) != 0x010800) { - log_debug("%s is not an NVMe device\n", bdf); + log_error("%s is not an NVMe device\n", bdf); errno = EINVAL; return -1; } @@ -571,7 +571,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->regs = vfio_pci_map_bar(&ctrl->pci, 0, 0x1000, 0, PROT_READ | PROT_WRITE); if (!ctrl->regs) { - log_debug("could not map controller registersn\n"); + log_error("could not map controller registersn\n"); return -1; } @@ -593,14 +593,14 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->config.mqes = NVME_FIELD_GET(cap, CAP_MQES); if (nvme_reset(ctrl)) { - log_debug("could not reset controller\n"); + log_error("could not reset controller\n"); return -1; } /* map admin queue doorbells */ ctrl->doorbells = vfio_pci_map_bar(&ctrl->pci, 0, 0x1000, 0x1000, PROT_WRITE); if (!ctrl->doorbells) { - log_debug("could not map doorbells\n"); + log_error("could not map doorbells\n"); return -1; } @@ -609,12 +609,12 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->cq = znew_t(struct nvme_cq, ctrl->opts.ncqr + 2); if (nvme_configure_adminq(ctrl, 0x0)) { - log_debug("could not configure admin queue\n"); + log_error("could not configure admin queue\n"); return -1; } if (nvme_enable(ctrl)) { - log_debug("could not enable controller\n"); + log_error("could not enable controller\n"); return -1; } @@ -631,7 +631,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op NVME_FIELD_SET(ctrl->opts.ncqr, FEAT_NRQS_NCQR)); if (nvme_admin(ctrl, &cmd, NULL, 0, &cqe)) { - log_debug("could not set number of queues\n"); + log_error("could not set number of queues\n"); return -1; } @@ -656,7 +656,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ret = nvme_admin(ctrl, &cmd, vaddr, len, NULL); if (ret) { - log_debug("could not identify\n"); + log_error("could not identify\n"); goto out; } diff --git a/src/nvme/util.c b/src/nvme/util.c index 03bd435c..7684ea28 100644 --- a/src/nvme/util.c +++ b/src/nvme/util.c @@ -69,7 +69,7 @@ int nvme_sync(struct nvme_ctrl *ctrl, struct nvme_sq *sq, union nvme_cmd *sqe, v do_unmap = true; if (_iommu_map_vaddr(ctx, buf, len, &iova, IOMMU_MAP_EPHEMERAL, opaque)) { - log_debug("failed to map vaddr\n"); + log_error("failed to map vaddr\n"); return -1; } } diff --git a/src/pci/util.c b/src/pci/util.c index 15a7bb2d..960aac9a 100644 --- a/src/pci/util.c +++ b/src/pci/util.c @@ -44,7 +44,7 @@ int pci_unbind(const char *bdf) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/devices/%s/driver/unbind", bdf) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } @@ -66,7 +66,7 @@ int pci_bind(const char *bdf, const char *driver) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/drivers/%s/bind", driver) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } @@ -84,12 +84,12 @@ int pci_driver_new_id(const char *driver, uint16_t vid, uint16_t did) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/drivers/%s/new_id", driver) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } if (asprintf(&id, "%x %x", vid, did) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); free(path); return -1; } @@ -109,12 +109,12 @@ int pci_driver_remove_id(const char *driver, uint16_t vid, uint16_t did) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/drivers/%s/remove_id", driver) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } if (asprintf(&id, "%x %x", vid, did) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); free(path); return -1; } @@ -133,7 +133,7 @@ int pci_device_info_get_ull(const char *bdf, const char *prop, unsigned long lon ssize_t ret; if (asprintf(&path, "/sys/bus/pci/devices/%s/%s", bdf, prop) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } @@ -161,7 +161,7 @@ char *pci_get_driver(const char *bdf) if (asprintf(&link, "/sys/bus/pci/devices/%s/driver", bdf) < 0) { link = NULL; - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); goto out; } @@ -172,7 +172,7 @@ char *pci_get_driver(const char *bdf) if (errno == ENOENT) goto out; - log_debug("failed to resolve driver link\n"); + log_error("failed to resolve driver link\n"); goto out; } @@ -180,13 +180,13 @@ char *pci_get_driver(const char *bdf) p = strrchr(driver, '/'); if (!p) { - log_debug("failed to determine driver name\n"); + log_error("failed to determine driver name\n"); goto out; } if (asprintf(&name, "%s", p + 1) < 0) { name = NULL; - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); goto out; } @@ -203,7 +203,7 @@ char *pci_get_iommu_group(const char *bdf) ssize_t ret; if (asprintf(&link, "/sys/bus/pci/devices/%s/iommu_group", bdf) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); goto out; } @@ -211,7 +211,7 @@ char *pci_get_iommu_group(const char *bdf) ret = readlink(link, group, PATH_MAX - 1); if (ret < 0) { - log_debug("failed to resolve iommu group link\n"); + log_error("failed to resolve iommu group link\n"); goto out; } @@ -219,7 +219,7 @@ char *pci_get_iommu_group(const char *bdf) p = strrchr(group, '/'); if (!p) { - log_debug("failed to find iommu group number\n"); + log_error("failed to find iommu group number\n"); goto out; } @@ -243,13 +243,13 @@ char *pci_get_device_vfio_id(const char *bdf) DIR *dp; if (asprintf(&path, "/sys/bus/pci/devices/%s/vfio-dev", bdf) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return NULL; } dp = opendir(path); if (!dp) { - log_debug("could not open directory; is %s bound to vfio-pci?\n", bdf); + log_error("could not open directory; is %s bound to vfio-pci?\n", bdf); return NULL; } diff --git a/src/vfio/device.c b/src/vfio/device.c index e36572f6..40d136d5 100644 --- a/src/vfio/device.c +++ b/src/vfio/device.c @@ -32,7 +32,7 @@ int vfio_set_irq(struct vfio_device *dev, int *eventfds, int count) if (!(dev->irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) { errno = EINVAL; - log_debug("device irq does not support eventfd\n"); + log_error("device irq does not support eventfd\n"); return -1; } @@ -53,7 +53,7 @@ int vfio_set_irq(struct vfio_device *dev, int *eventfds, int count) free(irq_set); if (ret) { - log_debug("failed to set device irq\n"); + log_error("failed to set device irq\n"); return -1; } @@ -74,7 +74,7 @@ int vfio_disable_irq(struct vfio_device *dev) ret = ioctl(dev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); if (ret) { - log_debug("failed to disable device irq\n"); + log_error("failed to disable device irq\n"); return -1; } diff --git a/src/vfio/pci.c b/src/vfio/pci.c index 72f674b4..0eb1bdd8 100644 --- a/src/vfio/pci.c +++ b/src/vfio/pci.c @@ -34,14 +34,14 @@ static int pci_set_bus_master(struct vfio_pci_device *pci) uint16_t pci_cmd; if (vfio_pci_read_config(pci, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND) < 0) { - log_debug("failed to read pci config region\n"); + log_error("failed to read pci config region\n"); return -1; } pci_cmd |= PCI_COMMAND_MASTER; if (vfio_pci_write_config(pci, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND) < 0) { - log_debug("failed to write pci config region\n"); + log_error("failed to write pci config region\n"); return -1; } @@ -58,7 +58,7 @@ static int vfio_pci_init_bar(struct vfio_pci_device *pci, int idx) }; if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_REGION_INFO, &pci->bar_region_info[idx])) { - log_debug("failed to get bar region info\n"); + log_error("failed to get bar region info\n"); return -1; } @@ -76,14 +76,14 @@ static int vfio_pci_init_irq(struct vfio_pci_device *pci) do { if (irq_index < 0) { errno = EINVAL; - log_debug("no supported irq types\n"); + log_error("no supported irq types\n"); return -1; } pci->dev.irq_info.index = irq_index--; if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_IRQ_INFO, &pci->dev.irq_info)) { - log_debug("failed to get device irq info\n"); + log_error("failed to get device irq info\n"); return -1; } } while (!pci->dev.irq_info.count); @@ -132,14 +132,14 @@ int vfio_pci_open(struct vfio_pci_device *pci, const char *bdf) pci->dev.fd = pci->dev.ctx->ops.get_device_fd(pci->dev.ctx, bdf); if (pci->dev.fd < 0) { - log_debug("failed to get device fd\n"); + log_error("failed to get device fd\n"); return -1; } pci->dev.device_info.argsz = sizeof(struct vfio_device_info); if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_INFO, &pci->dev.device_info)) { - log_debug("failed to get device info\n"); + log_error("failed to get device info\n"); return -1; } @@ -151,7 +151,7 @@ int vfio_pci_open(struct vfio_pci_device *pci, const char *bdf) }; if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_REGION_INFO, &pci->config_region_info)) { - log_debug("failed to get config region info\n"); + log_error("failed to get config region info\n"); return -1; } @@ -161,12 +161,12 @@ int vfio_pci_open(struct vfio_pci_device *pci, const char *bdf) } if (pci_set_bus_master(pci)) { - log_debug("failed to set pci bus master\n"); + log_error("failed to set pci bus master\n"); return -1; } if (vfio_pci_init_irq(pci)) { - log_debug("failed to initialize irq\n"); + log_error("failed to initialize irq\n"); return -1; } From 934f84177e4b793a133f914369fdbba98782b150 Mon Sep 17 00:00:00 2001 From: Mads Ynddal Date: Mon, 6 May 2024 09:58:15 +0200 Subject: [PATCH 23/23] ci/macos: add macOS build process By setting up the current PR branch as a submodule in MacVFN, we can build MacVFN and thereby verify the current libvfn branch is valid. No actual device or code test is performed. Signed-off-by: Mads Ynddal --- .github/workflows/build.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef47948d..76581a71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -189,6 +189,33 @@ jobs: name: meson test log path: build/meson-logs/testlog.txt + macos-build: + name: "macOS build" + needs: [sanity] + runs-on: macos-latest + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + repository: samsungds/macvfn + ref: libvfn4 # TODO: 'master' once merged + + - name: Configure PR branch as submodule + run: | + SUBMODULE_PATH="MacVFN/libvfn" + BRANCH_NAME=${GITHUB_HEAD_REF#refs/heads/} + git config -f .gitmodules submodule.$SUBMODULE_PATH.url ${GITHUB_SERVER_URL}/${{github.event.pull_request.head.repo.full_name}} + git config -f .gitmodules submodule.$SUBMODULE_PATH.branch "$BRANCH_NAME" + cat .gitmodules + git submodule update --remote --init $SUBMODULE_PATH + + - name: Install dependencies + run: brew install meson + + - name: Xcode build + run: make build + device-test: name: "run emulated nvme device tests"