diff --git a/include/vfn/driverkit.h b/include/vfn/driverkit.h new file mode 100644 index 00000000..68a2d0b0 --- /dev/null +++ b/include/vfn/driverkit.h @@ -0,0 +1,16 @@ +#ifndef LIBVFN_VFIO_H +#define LIBVFN_VFIO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif /* LIBVFN_VFIO_H */ diff --git a/include/vfn/driverkit/device.h b/include/vfn/driverkit/device.h new file mode 100644 index 00000000..1fb77bbd --- /dev/null +++ b/include/vfn/driverkit/device.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_DRIVERKIT_DEVICE_H +#define LIBVFN_DRIVERKIT_DEVICE_H + +#include +#include +#include "iommu/context.h" + +struct driverkit_bar_info { + uint8_t type; + uint8_t memory_index; + uint64_t size; +}; + +struct driverkit_pci_device { + IOPCIDevice *dev; + struct iommu_ctx ctx; + OSDictionary *iommu_mappings; + const char *bdf; + + struct driverkit_bar_info bar_region_info[6]; +}; + +#define __iommu_ctx(x) (&((struct driverkit_pci_device *)(x))->ctx) + +#endif /* LIBVFN_DRIVERKIT_DEVICE_H */ diff --git a/include/vfn/driverkit/pci.h b/include/vfn/driverkit/pci.h new file mode 100644 index 00000000..fefe1310 --- /dev/null +++ b/include/vfn/driverkit/pci.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +#ifndef LIBVFN_VFIO_PCI_H +#define LIBVFN_VFIO_PCI_H + +#define off_t int + +// NOTE: Dummy PROT to satisfy Linux code +enum { + PROT_READ = 0, + PROT_WRITE = 1, +}; + +// NOTE: Faux open for macOS +int vfio_pci_open(struct driverkit_pci_device *pci, const char *bdf); + +/** + * vfio_pci_map_bar - faux mapping in DriverKit + * @pci: &struct driverkit_pci_device + * @idx: the vfio region index to map + * @len: number of bytes to map + * @offset: offset at which to start mapping + * @prot: what accesses to permit to the mapped area (see ``man mmap``). + * + * Used in place of VFIO map device memory region identified by @idx into virtual memory. + * + * Return: On success, returns the virtual memory address mapped. On error, + * returns ``NULL`` and sets ``errno``. + */ +void *vfio_pci_map_bar(struct driverkit_pci_device *pci, int idx, size_t len, uint64_t offset, + int prot); + +/** + * vfio_pci_unmap_bar - faux unmap a region in virtual memory for DriverKit + * @pci: &struct driverkit_pci_device + * @idx: the vfio region index to unmap + * @mem: virtual memory address to unmap + * @len: number of bytes to unmap + * @offset: offset at which to start unmapping + * + * Used in place of VFIO to unmap the virtual memory address, previously mapped to the vfio device memory + * region identified by @idx. + */ +void vfio_pci_unmap_bar(struct driverkit_pci_device *pci, int idx, void *mem, size_t len, + uint64_t offset); + +#endif /* LIBVFN_VFIO_PCI_H */ \ No newline at end of file diff --git a/include/vfn/iommu.h b/include/vfn/iommu.h index 43642ea0..9f78bb41 100644 --- a/include/vfn/iommu.h +++ b/include/vfn/iommu.h @@ -17,7 +17,9 @@ #include #include +#ifndef __APPLE__ #include +#endif #include #include diff --git a/include/vfn/iommu/dma.h b/include/vfn/iommu/dma.h index 346c72d7..e21ab5a0 100644 --- a/include/vfn/iommu/dma.h +++ b/include/vfn/iommu/dma.h @@ -82,12 +82,14 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len); */ int iommu_unmap_all(struct iommu_ctx *ctx); +#ifndef __APPLE__ #ifndef IOMMU_IOAS_IOVA_RANGES struct iommu_iova_range { __aligned_u64 start; __aligned_u64 last; }; #endif +#endif /** * iommu_translate_vaddr - Translate a virtual address into an iova @@ -102,6 +104,7 @@ struct iommu_iova_range { */ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova); +#ifndef __APPLE__ /** * iommu_get_iova_ranges - Get iova ranges * @ctx: &struct iommu_ctx @@ -112,5 +115,6 @@ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova); * Return: the number of elements in the array pointed to. */ int iommu_get_iova_ranges(struct iommu_ctx *ctx, struct iommu_iova_range **ranges); +#endif #endif /* LIBVFN_IOMMU_DMA_H */ diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index a3c3e1d8..570baf7a 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -43,7 +43,11 @@ struct nvme_ctrl { /** * @pci: vfio pci device state */ + #ifdef __APPLE__ + struct driverkit_pci_device pci; + #else struct vfio_pci_device pci; + #endif /** * @serial: Serial number from controller diff --git a/include/vfn/nvme/queue.h b/include/vfn/nvme/queue.h index 849033d5..4a886957 100644 --- a/include/vfn/nvme/queue.h +++ b/include/vfn/nvme/queue.h @@ -87,7 +87,7 @@ struct nvme_sq { */ static inline void nvme_sq_post(struct nvme_sq *sq, const union nvme_cmd *sqe) { - memcpy(sq->vaddr + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES); + memcpy(((uint8_t *) sq->vaddr) + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES); trace_guard(NVME_SQ_POST) { trace_emit("sqid %d tail %d\n", sq->id, sq->tail); @@ -177,7 +177,7 @@ static inline void nvme_sq_exec(struct nvme_sq *sq, const union nvme_cmd *sqe) */ static inline struct nvme_cqe *nvme_cq_head(struct nvme_cq *cq) { - return (struct nvme_cqe *)(cq->vaddr + (cq->head << NVME_CQES)); + return (struct nvme_cqe *)(((uint8_t*) cq->vaddr) + (cq->head << NVME_CQES)); } /** diff --git a/src/driverkit/pci.c b/src/driverkit/pci.c new file mode 100644 index 00000000..7af11a30 --- /dev/null +++ b/src/driverkit/pci.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + + + +#define log_fmt(fmt) "driverkit/pci: " fmt + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// NOTE: Faux open for macOS +int vfio_pci_open(struct driverkit_pci_device *pci, const char *bdf) +{ + return 0; +} + +void *vfio_pci_map_bar(struct driverkit_pci_device *pci, int idx, size_t len, uint64_t offset, int prot) +{ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) zmallocn(1, sizeof(struct macvfn_pci_map_bar)); + mapping->pci = pci; + mapping->idx = idx; + mapping->len = len; + mapping->offset = offset; + + return mapping; +} + +void vfio_pci_unmap_bar(struct driverkit_pci_device *pci, int idx, void *mem, size_t len, uint64_t offset) +{ + free(mem); +} + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/src/iommu/context.c b/src/iommu/context.c index 793cea4e..d890e277 100644 --- a/src/iommu/context.c +++ b/src/iommu/context.c @@ -47,11 +47,16 @@ struct iommu_ctx *iommu_get_default_context(void) fallback: #endif +#ifdef __APPLE__ + return driverkit_get_default_iommu_context(); +#else return vfio_get_default_iommu_context(); +#endif } struct iommu_ctx *iommu_get_context(const char *name) { +#ifndef __APPLE__ #ifdef HAVE_VFIO_DEVICE_BIND_IOMMUFD if (__iommufd_broken) goto fallback; @@ -61,8 +66,12 @@ struct iommu_ctx *iommu_get_context(const char *name) fallback: #endif return vfio_get_iommu_context(name); +#else + return driverkit_get_iommu_context(name); +#endif } +#ifndef __APPLE__ void iommu_ctx_init(struct iommu_ctx *ctx) { ctx->nranges = 1; @@ -80,3 +89,4 @@ void iommu_ctx_init(struct iommu_ctx *ctx) skiplist_init(&ctx->map.list); pthread_mutex_init(&ctx->map.lock, NULL); } +#endif diff --git a/src/iommu/context.h b/src/iommu/context.h index 865555e5..5aa16aea 100644 --- a/src/iommu/context.h +++ b/src/iommu/context.h @@ -13,10 +13,27 @@ #ifndef LIBVFN_SRC_IOMMU_CONTEXT_H #define LIBVFN_SRC_IOMMU_CONTEXT_H + +#ifdef __cplusplus +extern "C" { +#endif #include "util/skiplist.h" + struct iommu_ctx; +struct iova_mapping { + void *vaddr; + size_t len; + uint64_t iova; + void *opaque[2]; + + unsigned long flags; + + struct skiplist_node list; +}; + + struct iommu_ctx_ops { /* container/ioas ops */ int (*iova_reserve)(struct iommu_ctx *ctx, size_t len, uint64_t *iova, @@ -31,7 +48,11 @@ struct iommu_ctx_ops { }; struct iova_map { + #ifndef __APPLE__ pthread_mutex_t lock; + #else + IOLock* lock; + #endif struct skiplist list; }; @@ -39,13 +60,19 @@ struct iommu_ctx { struct iova_map map; struct iommu_ctx_ops ops; + #ifndef __APPLE__ pthread_mutex_t lock; int nranges; struct iommu_iova_range *iova_ranges; + #else + IOPCIDevice* pci; + IOLock* lock; + #endif }; struct iommu_ctx *iommu_get_default_context(void); +#ifndef __APPLE__ struct iommu_ctx *vfio_get_default_iommu_context(void); struct iommu_ctx *vfio_get_iommu_context(const char *name); @@ -53,8 +80,18 @@ struct iommu_ctx *vfio_get_iommu_context(const char *name); struct iommu_ctx *iommufd_get_default_iommu_context(void); struct iommu_ctx *iommufd_get_iommu_context(const char *name); #endif +#else +struct iommu_ctx *driverkit_get_default_iommu_context(void); +struct iommu_ctx *driverkit_get_iommu_context(const char *name); +#endif void iommu_ctx_init(struct iommu_ctx *ctx); +#ifndef __APPLE__ int iommu_iova_range_to_string(struct iommu_iova_range *range, char **str); +#endif + +#ifdef __cplusplus +} +#endif #endif /* LIBVFN_SRC_IOMMU_CONTEXT_H */ diff --git a/src/iommu/dma.c b/src/iommu/dma.c index 43456d3c..e5b2b488 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -122,9 +122,9 @@ int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *i m = znew_t(struct iova_mapping, 1); m->vaddr = vaddr; m->len = len; - m->iova = 0; - m->opaque[0] = NULL; - m->opaque[1] = opaque; + m->iova = 0; // Physical/IOMMU address + m->opaque[0] = NULL; // IODMACommand + m->opaque[1] = opaque; // IOMemoryDescriptor m->flags = flags; if (ctx->ops.dma_map(ctx, m)) { @@ -210,6 +210,7 @@ int iommu_unmap_all(struct iommu_ctx *ctx) return 0; } +#ifndef __APPLE__ int iommu_get_iova_ranges(struct iommu_ctx *ctx, struct iommu_iova_range **ranges) { *ranges = ctx->iova_ranges; @@ -220,6 +221,7 @@ int iommu_iova_range_to_string(struct iommu_iova_range *r, char **str) { return asprintf(str, "[0x%llx; 0x%llx]", r->start, r->last); } +#endif #ifdef __cplusplus } diff --git a/src/iommu/driverkit.c b/src/iommu/driverkit.c new file mode 100644 index 00000000..6487d2b0 --- /dev/null +++ b/src/iommu/driverkit.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2023 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#include "common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define log_fmt(fmt) "iommu/vfio: " fmt + +#include "ccan/str/str.h" +#include "ccan/compiler/compiler.h" +#include "ccan/minmax/minmax.h" + +#include "vfn/driverkit.h" +#include "vfn/support.h" +#include "vfn/iommu.h" +#include "vfn/trace.h" +#include "vfn/pci/util.h" + +#include "context.h" + +static int driverkit_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) +{ + kern_return_t ret; + IODMACommand **dmaCommand = (IODMACommand **) &m->opaque[0]; + log_debug("driverkit_dma_map (PrepareForDMA) %p %zu, %p, %p", m->vaddr, m->len, m->opaque[0], m->opaque[1]); + IODMACommandSpecification dmaSpecification = { + .options = kIODMACommandSpecificationNoOptions, + .maxAddressBits = 64, + }; + struct driverkit_pci_device *dev = container_of(ctx, struct driverkit_pci_device, ctx); + + ret = IODMACommand::Create(dev->dev, kIODMACommandCreateNoOptions, &dmaSpecification, dmaCommand); + if (ret != kIOReturnSuccess){ + log_error("IODMACommand::Create failed with error code: %x", ret); + return -1; + } + + if (!m->opaque[1]){ + log_error("driverkit_dma_map: opaque[1] is NULL"); + return -1; + } + + uint64_t dmaFlags = 0; + uint32_t dmaSegmentCount = 1; + IOAddressSegment physicalAddressSegment; + ret = (*dmaCommand)->PrepareForDMA( + kIODMACommandPrepareForDMANoOptions, + (IOMemoryDescriptor *) m->opaque[1], + 0, + m->len, + &dmaFlags, + &dmaSegmentCount, + &physicalAddressSegment + ); + if (ret != kIOReturnSuccess){ + log_error("failed to PrepareForDMA %x", ret); + return -1; + } + if (dmaSegmentCount != 1){ + log_error("dmaSegmentCount not 1! %u", dmaSegmentCount); + return -1; + } + assert (m->len == physicalAddressSegment.length); + m->iova = physicalAddressSegment.address; + + log_debug("driverkit_dma_map: iova %llx", m->iova); + + return 0; +} + +static int driverkit_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) +{ + log_debug("driverkit_dma_unmap (CompleteDMA) %p %zu", m->vaddr, m->len); + IODMACommand *dmaCommand = (IODMACommand *) m->opaque[0]; + kern_return_t ret = (int) dmaCommand->CompleteDMA(kIODMACommandCompleteDMANoOptions); + if (ret != kIOReturnSuccess){ + return -1; + } + OSSafeReleaseNULL(dmaCommand); + m->opaque[0] = NULL; + return 0; +} + +static int driverkit_dma_unmap_all(struct iommu_ctx *ctx) +{ + return 0; +} + +static const struct iommu_ctx_ops driverkit_ops = { + .get_device_fd = NULL, + + .iova_reserve = NULL, + .iova_put_ephemeral = NULL, + + .dma_map = driverkit_dma_map, + .dma_unmap = driverkit_dma_unmap, + .dma_unmap_all = driverkit_dma_unmap_all, +}; + +void iommu_ctx_init(struct iommu_ctx *ctx) +{ + memcpy(&ctx->ops, &driverkit_ops, sizeof(ctx->ops)); + ctx->lock = IOLockAlloc(); + ctx->map.lock = IOLockAlloc(); + skiplist_init(&ctx->map.list); +} + +struct iommu_ctx *driverkit_get_iommu_context(const char *name) +{ + return NULL; +} + +struct iommu_ctx *driverkit_get_default_iommu_context(void) +{ + return NULL; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/nvme/core.c b/src/nvme/core.c index d0c62be0..f0ab3123 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -29,11 +29,40 @@ int errno; #include "types.h" +#ifdef __APPLE__ +inline void *cqhdbl(void *doorbells, int qid, int dstrd) +{ + struct macvfn_pci_map_bar *doorbell_mapping = (struct macvfn_pci_map_bar*) doorbells; + struct macvfn_pci_map_bar *new_mapping = (struct macvfn_pci_map_bar*) zmallocn(1, sizeof(struct macvfn_pci_map_bar)); + + new_mapping->pci = doorbell_mapping->pci; + new_mapping->idx = doorbell_mapping->idx; + new_mapping->len = doorbell_mapping->len; + new_mapping->offset = doorbell_mapping->offset + (2 * qid + 1) * (4 << dstrd); + + return new_mapping; +} + +inline void *sqtdbl(void *doorbells, int qid, int dstrd) +{ + struct macvfn_pci_map_bar *doorbell_mapping = (struct macvfn_pci_map_bar*) doorbells; + struct macvfn_pci_map_bar *new_mapping = (struct macvfn_pci_map_bar*) zmallocn(1, sizeof(struct macvfn_pci_map_bar)); + + new_mapping->pci = doorbell_mapping->pci; + new_mapping->idx = doorbell_mapping->idx; + new_mapping->len = doorbell_mapping->len; + new_mapping->offset = doorbell_mapping->offset + (2 * qid) * (4 << dstrd); + + return new_mapping; +} +#else #define cqhdbl(doorbells, qid, dstrd) \ (doorbells + (2 * qid + 1) * (4 << dstrd)) #define sqtdbl(doorbells, qid, dstrd) \ (doorbells + (2 * qid) * (4 << dstrd)) +#endif + enum nvme_ctrl_feature_flags { NVME_CTRL_F_ADMINISTRATIVE = 1 << 0, diff --git a/src/nvme/rq.c b/src/nvme/rq.c index 1591046d..ff5b34e6 100644 --- a/src/nvme/rq.c +++ b/src/nvme/rq.c @@ -97,6 +97,7 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * return 0; } +#ifndef __APPLE__ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, struct iovec *iov, int niov) { @@ -170,6 +171,7 @@ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd errno = EINVAL; return -1; } +#endif int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) { @@ -188,7 +190,11 @@ int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) } if (!nvme_cqe_ok(&cqe)) { + #ifdef __APPLE__ + if (false) { + #else if (logv(LOG_DEBUG)) { + #endif uint16_t status = le16_to_cpu(cqe.sfp) >> 1; log_debug("cqe status 0x%" PRIx16 "\n", (uint16_t)(status & 0x7ff)); diff --git a/src/pci/util.c b/src/pci/util.c index ff529b98..fcef5cc0 100644 --- a/src/pci/util.c +++ b/src/pci/util.c @@ -12,12 +12,32 @@ #include "../common.h" +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef __APPLE__ #include +#endif #include #include +#include + +#ifdef __APPLE__ +int pci_device_info_get_ull(const char *bdf, const char *prop, unsigned long long *v) +{ + if (!strcmp(prop, "class")) { + // TODO: Actually read this from device once it makes sense + *v = 0x010800; + } else { + return -1; + } + return 0; +} +#else int pci_unbind(const char *bdf) { char *path = NULL; @@ -265,3 +285,8 @@ char *pci_get_device_vfio_id(const char *bdf) return vfio_id; } +#endif + +#ifdef __cplusplus +} +#endif diff --git a/src/support/platform/macos/mem.c b/src/support/platform/macos/mem.c new file mode 100644 index 00000000..30bd50f6 --- /dev/null +++ b/src/support/platform/macos/mem.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#include "common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +size_t __VFN_PAGESIZE; +int __VFN_PAGESHIFT; + +static void __attribute__((constructor)) init_page_size(void) +{ + __VFN_PAGESIZE = IOVMPageSize; + __VFN_PAGESHIFT = 63u - __builtin_clzl(__VFN_PAGESIZE); + + log_debug("pagesize is %zu (shift %d)\n", __VFN_PAGESIZE, __VFN_PAGESHIFT); +} + +void backtrace_abort(void) +{ + abort(); +} + +ssize_t __pgmap(void **mem, size_t sz, void **opaque) +{ + IOAddressSegment virtualAddressSegment; + ssize_t len = ALIGN_UP(sz, __VFN_PAGESIZE); + IOBufferMemoryDescriptor **mem_descriptor = (IOBufferMemoryDescriptor **) opaque; + + IOBufferMemoryDescriptor::Create( + kIOMemoryDirectionInOut, + len, + __VFN_PAGESIZE, + mem_descriptor + ); + (*mem_descriptor)->SetLength(len); + (*mem_descriptor)->GetAddressRange(&virtualAddressSegment); + bzero((void*) virtualAddressSegment.address, virtualAddressSegment.length); + *mem = (void*) virtualAddressSegment.address; + + return len; +} + +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque) +{ + return __pgmap(mem, n * sz, opaque); +} + +void __pgunmap(void *mem, size_t len, void *opaque) +{ + IOBufferMemoryDescriptor *_mem = (IOBufferMemoryDescriptor *) opaque; + + OSSafeReleaseNULL(_mem); +} + +ssize_t pgmap(void **mem, size_t sz) +{ + log_fatal("Use ::__pgmap on macOS"); +} + +ssize_t pgmapn(void **mem, unsigned int n, size_t sz) +{ + log_fatal("Use ::__pgmapn on macOS"); +} + +void pgunmap(void *mem, size_t len) +{ + log_fatal("Use ::__pgunmap on macOS"); +} + +#ifdef __cplusplus +} +#endif