feat: POST /uploads can write uploads to w3up for web3 serving and dagcargoless filecoin deal-making

.github/workflows/client.yml

@@ -36,7 +36,6 @@ jobs:
     strategy:
       matrix:
         node_version:
-          - 16
           - 18
           - 20
     steps:

.github/workflows/cron-dagcargo-sizes.yml

@@ -23,7 +23,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-metrics.yml

@@ -23,7 +23,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-nft-ttr.yml

@@ -20,7 +20,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-pins-failed.yml

@@ -27,7 +27,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-pins.yml

@@ -23,7 +23,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: yarn workspace nft.storage prepare
       - run: yarn workspace cron lint
@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: yarn workspace nft.storage prepare
       - run: yarn workspace cron typecheck
@@ -46,7 +46,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Test (ES)
         run: yarn --cwd packages/cron test

.github/workflows/deploy.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Publish app
         uses: cloudflare/[email protected]
@@ -43,7 +43,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: ./packages/tools/cli.js dns --name nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
       - run: echo "::warning::https://nft.storage"
@@ -54,7 +54,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: ./packages/tools/cli.js dns --name staging.nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
       - run: echo "::warning::https://staging.nft.storage"
@@ -65,7 +65,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: ./packages/tools/cli.js dns --name dev.nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
       - run: echo "::warning::https://dev.nft.storage"

.github/workflows/website.yml

@@ -17,11 +17,17 @@ jobs:
   check:
     name: Test
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - 18
+          - 20
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: ${{ matrix.node-version }}
       - uses: bahmutov/npm-install@v1
       - name: Run build
         env:
@@ -42,7 +48,8 @@ jobs:
         os:
           - ubuntu-20.04
         node-version:
-          - '16'
+          - 18
+          - 20
         test_results_path:
           # corresponds to playwright invocation/configuration
           - packages/website/test-results
@@ -97,7 +104,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: 20
           registry-url: https://registry.npmjs.org/
       - uses: bahmutov/npm-install@v1
       - name: Run build

decisions/20240313-try-w3up.md

@@ -48,13 +48,29 @@ Rationale
 - when nft.storage api handles request POST /upload/ (e.g. from the nft.storage ui), the uploaded file should be stored using up.web3.storage in a space associated with an web3.storage account controlled by an administrator of nft.storage
   - initially, this will only happen when a feature switch is enabled
 
+##### Open Architectural Decisions
+
+###### Should write to w3up happen sync or async?
+
+i.e. if the write to w3up fails, should the POST /uploads request that triggered it also fail ('sync'=yes)? Or should we only do the write to w3up after a successful response to POST /uploads ('async')?
+
+Initially, it is simplest to get something working in a sync way, so we'll do that to make sure the w3up-client creation, configuration, authorization, accounting, etc. is set up correctly.
+
+Once that works, we can look at deferring the write to w3up. We'd need to use some kind of job queue (e.g. [Cloudflare Queues](https://developers.cloudflare.com/queues/) or perhaps using the mysql db as a queue and nftstorage/cron as a worker).
+
 #### Data Model Changes
 
 Minimal. Only new configuration.
 
 We'll need to decide which storage space that uploads will be added to.
 To start, we'll just making everything store in a configured storage space, and the app will read that from configuration passed in environment variables, similar to env vars in [add-to-web3](https://github.com/web3-storage/add-to-web3?tab=readme-ov-file#generating-a-secret_key-and-proof).
 
+##### `W3UP_URL` Environment Variable
+
+configures the URL for connecting to w3up.
+
+optional. If not set, no URL will be passed to `@web3-storage/w3up-client` constructors. [Currently that means](https://github.com/web3-storage/w3up/blob/7a6385bef1dd424d5eb952528ae5d86a83837c80/packages/upload-client/src/service.js#L5) the default will be `up.web3.storage`.
+
 ##### `W3_NFTSTORAGE_SPACE` Environment Variable
 
 configures the web3.storage space that nfts will be stored in.
@@ -70,13 +86,13 @@ configures how nft.storage will authenticate to web3.storage when sending invoca
 configures the capabilities that nft.storage has access to when interacting with web3.storage to store nfts. These capabilities will usually be UCAN delegations whose audience is the identifier of `W3_NFTSTORAGE_PRINCIPAL`.
 W3_NFTSTORAGE_PROOF needs to have proof rooted in W3_NFTSTORAGE_SPACE that authorize W3_NFTSTORAGE_PRINCIPAL to store in W3_NFTSTORAGE_SPACE.
 
-##### `NFTSTORAGE_W3S_ALLOW_ACCOUNTS` Environment Variable
+##### `W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS` Environment Variable
 
 configures feature switch for which nftstorage accounts will have new uploads stored in web3.storage.
 
 Note: this environment variable may not be a permanent addition to the codebase. It's only meant to be used as a feature switch that decouples enabling the new functionality from deploying the new code. After testing, we may remove or change this feature switch when it is no longer useful.
 
-Format: JSON Array of Account Identifiers
+Format: JSON Array of email address strings.
 
 #### UI Changes
 
@@ -134,9 +150,12 @@ If we determine it is not desirable for nft.storage to use the entirety of w3up'
 Prototype Happy path
 
 - [x] @gobengo set up nft.storage localdev to be able to hack on packages/api
-- [ ] nft.storage/api: can be configured with `W3_NFTSTORAGE_PRINCIPAL` and `W3_NFTSTORAGE_PROOF` and use them to get a w3up-client at runtime
-- [ ] nft.storage/api: modify POST /upload/ handler to store using w3up-client@12
-- [ ] nft.storage/api: add feature switch requiring allowlist for new uploads to store in web3.storage
+- [x] nft.storage/api: can be configured with W3_URL
+- [x] add deps on w3up-client, ucanto
+  - [ x] upgrade typescript past 5.x and node past 16 as is required by w3up-client+ucanto
+- [x] nft.storage/api: can be configured with `W3_NFTSTORAGE_PRINCIPAL` and `W3_NFTSTORAGE_PROOF` and use them to get a w3up-client at runtime
+- [x] nft.storage/api: modify POST /upload/ handler to store using w3up-client@12
+- [x] nft.storage/api: add feature switch requiring allowlist for new uploads to store in web3.storage
 
 Rollout
 

package.json

@@ -39,14 +39,13 @@
     "prettier": "2.5.1",
     "rimraf": "^3.0.2",
     "simple-git-hooks": "^2.3.1",
-    "typescript": "4.5.3",
+    "typescript": "5.2.2",
     "webpack": "^5.72.0",
     "wrangler": "^2.0.23"
   },
   "resolutions": {
     "prettier": "2.5.1",
-    "@types/react": "^17.0.34",
-    "typescript": "4.5.3"
+    "@types/react": "^17.0.34"
   },
   "engines": {
     "node": ">= 16.0"

packages/api/package.json

@@ -27,7 +27,12 @@
     "@nftstorage/ipfs-cluster": "^5.0.1",
     "@noble/ed25519": "^1.6.1",
     "@supabase/postgrest-js": "^0.34.1",
+    "@ucanto/core": "^9.0.1",
+    "@ucanto/principal": "^9.0.0",
+    "@ucanto/server": "^9.0.1",
+    "@web3-storage/access": "^18.2.0",
     "@web3-storage/car-block-validator": "^1.2.0",
+    "@web3-storage/w3up-client": "^12.5.0",
     "cardex": "^1.0.0",
     "ipfs-car": "^0.6.1",
     "it-last": "^2.0.0",

packages/api/src/bindings.d.ts

@@ -7,6 +7,7 @@ import { UserOutput, UserOutputKey } from './utils/db-client-types.js'
 import { DBClient } from './utils/db-client.js'
 import { LinkdexApi } from './utils/linkdex.js'
 import { Logging } from './utils/logs.js'
+import { Client as W3upClient } from '@web3-storage/w3up-client'
 
 export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
 
@@ -96,6 +97,25 @@ export interface ServiceConfiguration {
 
   /** Slack webhook url */
   SLACK_USER_REQUEST_WEBHOOK_URL: string
+
+  /** w3up connection URL (e.g. https://up.web3.storage) */
+  W3UP_URL?: string
+
+  /** base64 encoded multiformats ed25519 secretKey */
+  W3_NFTSTORAGE_PRINCIPAL?: string
+
+  /** CID (identity codec) of CAR-encoded UCAN DAG */
+  W3_NFTSTORAGE_PROOF?: string
+
+  /** did:key of the w3up space in which to store NFTs */
+  W3_NFTSTORAGE_SPACE?: string
+
+  /**
+   * JSON array of strings that are emails whose uploads should be uploaded via w3up.
+   * This is meant as a feature switch to test new functionality,
+   * and this configuration may be removed once the feature switch isn't needed to limit access.
+   */
+  W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
 }
 
 export interface Ucan {
@@ -128,6 +148,12 @@ export interface RouteContext {
   r2Uploader: Uploader
   ucanService: Service
   auth?: Auth
+  W3UP_URL?: string
+  W3_NFTSTORAGE_PRINCIPAL?: string
+  W3_NFTSTORAGE_PROOF?: string
+  W3_NFTSTORAGE_SPACE?: string
+  W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
+  w3up?: W3upClient
 }
 
 export type Handler = (

packages/api/src/config.js

@@ -61,6 +61,13 @@ export function serviceConfigFromVariables(vars) {
     VERSION: vars.NFT_STORAGE_VERSION || NFT_STORAGE_VERSION,
     // @ts-ignore
     COMMITHASH: vars.NFT_STORAGE_COMMITHASH || NFT_STORAGE_COMMITHASH,
+
+    W3UP_URL: vars.W3UP_URL,
+    W3_NFTSTORAGE_PRINCIPAL: vars.W3_NFTSTORAGE_PRINCIPAL,
+    W3_NFTSTORAGE_PROOF: vars.W3_NFTSTORAGE_PROOF,
+    W3_NFTSTORAGE_SPACE: vars.W3_NFTSTORAGE_SPACE,
+    W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS:
+      vars.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS,
   }
 }
 
@@ -123,6 +130,11 @@ export function loadConfigVariables() {
     'LINKDEX_URL',
     'S3_ENDPOINT',
     'SLACK_USER_REQUEST_WEBHOOK_URL',
+    'W3UP_URL',
+    'W3_NFTSTORAGE_SPACE',
+    'W3_NFTSTORAGE_PRINCIPAL',
+    'W3_NFTSTORAGE_PROOF',
+    'W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS',
   ]
 
   for (const name of optional) {
@@ -150,6 +162,12 @@ function parseRuntimeEnv(s) {
     return 'test'
   }
 
+  if (typeof s !== 'string') {
+    throw new Error(
+      `Unable to parse non-string (${typeof s}) environment name: ${s}`
+    )
+  }
+
   switch (s) {
     case 'test':
     case 'dev':