fix: avoid car repack (#2551)

Attempt to remove need for car repacking --------- Co-authored-by: Benjamin Goering <[email protected]>
nftstorage · Apr 2, 2024 · b0d622e · b0d622e
1 parent 6a50a70
commit b0d622e
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 42 deletions.
diff --git a/packages/api/src/routes/metaplex-upload.js b/packages/api/src/routes/metaplex-upload.js
@@ -54,6 +54,7 @@ export async function metaplexUpload(event, ctx) {
       files: [],
       structure: 'Unknown',
       meta,
+      car: blob,
     },
     stat
   )

diff --git a/packages/api/src/routes/nfts-store.js b/packages/api/src/routes/nfts-store.js
@@ -4,6 +4,7 @@ import { CID } from 'multiformats'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { CarWriter } from '@ipld/car/writer'
 import { MemoryBlockStore } from 'ipfs-car/blockstore/memory'
+import { createCarCid } from '../utils/car.js'
 import { importer as unixFsImporter } from 'ipfs-unixfs-importer'
 import { checkAuth } from '../utils/auth.js'
 import { setIn } from '../utils/utils.js'
@@ -65,14 +66,14 @@ export async function nftStore(event, ctx) {
 
   const size = totalSize(bs)
   const structure = 'Complete'
-  const carBytes = await exportToCar(rootCid, bs)
+  const car = await exportToCar(rootCid, bs)
 
   /** @type {import('./nfts-upload.js').CarStat} */
   const carStat = {
     rootCid,
     structure,
-    carBytes,
     size,
+    cid: await createCarCid(new Uint8Array(await car.arrayBuffer())),
   }
 
   const upload = await uploadCarWithStat(
@@ -84,6 +85,7 @@ export async function nftStore(event, ctx) {
       uploadType: 'Nft',
       files: [],
       structure,
+      car,
     },
     carStat
   )
@@ -195,10 +197,8 @@ async function exportToCar(rootCid, bs) {
   for await (const part of out) {
     parts.push(part)
   }
-  const car = new Blob(parts)
-  // @ts-expect-error
-  parts = null
-  return new Uint8Array(await car.arrayBuffer())
+
+  return new Blob(parts)
 }
 
 /**

diff --git a/packages/api/src/routes/nfts-upload.js b/packages/api/src/routes/nfts-upload.js
@@ -154,22 +154,22 @@ export async function uploadCar(params) {
 }
 
 /**
- * @param {Omit<UploadCarInput, 'car'>} data
+ * @param {UploadCarInput} data
  * @param {CarStat} stat
  */
 export async function uploadCarWithStat(
-  { event, ctx, user, key, uploadType = 'Car', mimeType, files, meta },
+  { event, ctx, user, key, car, uploadType = 'Car', mimeType, files, meta },
   stat
 ) {
   console.log('UCWS')
   const sourceCid = stat.rootCid.toString()
   const contentCid = stat.rootCid.toV1().toString()
-  const carCid = await createCarCid(stat.carBytes)
+
   const metadata = {
     structure: stat.structure || 'Unknown',
     sourceCid,
     contentCid,
-    carCid: carCid.toString(),
+    carCid: stat.cid.toString(),
   }
 
   /** @type {(() => Promise<void>)|undefined} */
@@ -179,30 +179,13 @@ export async function uploadCarWithStat(
   if (ctx.w3up && w3upFeatureSwitchEnabled(ctx, { user })) {
     console.log('SWITCH ENABLED')
     const { w3up } = ctx
-    // should only be 1 - shard size in w3up is > max upload size in CF
-    /** @type {import('@web3-storage/w3up-client/types').CARLink[]} */
-    const shards = []
-    const carBytesBlobLike = {
-      stream: () =>
-        new ReadableStream({
-          start(c) {
-            c.enqueue(stat.carBytes)
-            c.close()
-          },
-        }),
-    }
+
     console.log('UPLOADING CAR')
-    await w3up.uploadCAR(carBytesBlobLike, {
-      onShardStored: ({ cid }) => {
-        shards.push(cid)
-      },
-      // @ts-expect-error TODO adjust upstream type
-      pieceHasher: null,
-    })
+    await w3up.capability.store.add(car)
     console.log('UPLOADED CAR')
     // register as gateway links to record the CAR CID - we don't have another
     // way to know the location right now.
-    backupUrls.push(...shards.map((s) => new URL(`https://w3s.link/ipfs/${s}`)))
+    backupUrls.push(new URL(`https://w3s.link/ipfs/${stat.cid}`))
 
     if (stat.structure === 'Partial') {
       console.log('PARTIAL')
@@ -223,9 +206,10 @@ export async function uploadCarWithStat(
       }
     }
   } else {
+    const carBytes = new Uint8Array(await car.arrayBuffer())
     const [s3Backup, r2Backup] = await Promise.all([
-      ctx.s3Uploader.uploadCar(stat.carBytes, carCid, user.id, metadata),
-      ctx.r2Uploader.uploadCar(stat.carBytes, carCid, user.id, metadata),
+      ctx.s3Uploader.uploadCar(carBytes, stat.cid, user.id, metadata),
+      ctx.r2Uploader.uploadCar(carBytes, stat.cid, user.id, metadata),
     ])
     backupUrls.push(s3Backup.url, r2Backup.url)
 
@@ -332,16 +316,19 @@ export async function nftUpdateUpload(event, ctx) {
  * @typedef {Object} CarStat
  * @property {number} [size] DAG size in bytes
  * @property {import('multiformats').CID} rootCid Root CID of the DAG
+ * @property {import('multiformats').CID} cid CID of the CAR
  * @property {DagStructure} [structure] Completeness of the DAG within the CAR
- * @property {Uint8Array} carBytes
  *
  * @param {Blob} carBlob
  * @param {Object} [options]
  * @param {DagStructure} [options.structure]
  * @returns {Promise<CarStat>}
  */
 export async function carStat(carBlob, { structure } = {}) {
+  // We load whole thing into memory to derive stats but don't hold a reference
+  // so that it will get GC'd once function returns.
   const carBytes = new Uint8Array(await carBlob.arrayBuffer())
+  const cid = await createCarCid(carBytes)
   const blocksIterator = await CarBlockIterator.fromBytes(carBytes)
   const roots = await blocksIterator.getRoots()
   if (roots.length === 0) {
@@ -406,7 +393,7 @@ export async function carStat(carBlob, { structure } = {}) {
       }
     }
   }
-  return { rootCid, size, structure, carBytes }
+  return { cid, rootCid, size, structure }
 }
 
 /**

diff --git a/packages/api/test/nfts-upload.spec.js b/packages/api/test/nfts-upload.spec.js
@@ -153,14 +153,15 @@ test.serial('should forward uploads to W3UP_URL', async (t) => {
     'this upload sent one valid store/add invocation to w3up'
   )
 
-  const finalW3upUploadAddCount = mockW3upUploadAddCount
-  const uploadAddCountDelta =
-    finalW3upUploadAddCount - initialW3upUploadAddCount
-  t.is(
-    uploadAddCountDelta,
-    1,
-    'this upload sent one valid upload/add invocation to w3up'
-  )
+  // @todo re-enable or remove this. We may not ever need to do an upload/add, but haven't decided yet
+  // const finalW3upUploadAddCount = mockW3upUploadAddCount
+  // const uploadAddCountDelta =
+  //   finalW3upUploadAddCount - initialW3upUploadAddCount
+  // t.is(
+  //   uploadAddCountDelta,
+  //   1,
+  //   'this upload sent one valid upload/add invocation to w3up'
+  // )
 
   // if similar request is made by user not in W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS allow list,
   // that should not result in request to w3up
-Original file line number
+Diff line change
@@ Expand Up / @@ -54,6 +54,7 @@ export async function metaplexUpload(event, ctx) { @@
           files: [],
           structure: 'Unknown',
           meta,
+          car: blob,
         },
         stat
       )
@@ Expand Down @@