diff --git a/add-on/src/lib/ipfs-client/index.js b/add-on/src/lib/ipfs-client/index.js index f8be70403..64426ad59 100644 --- a/add-on/src/lib/ipfs-client/index.js +++ b/add-on/src/lib/ipfs-client/index.js @@ -34,7 +34,7 @@ async function initIpfsClient (opts) { const instance = await client.init(opts) easeApiChanges(instance) - _reloadIpfsClientDependents(instance) // async (API is present) + _reloadIpfsClientDependents(instance, opts) // async (API is present) return instance } @@ -71,9 +71,9 @@ async function _reloadIpfsClientDependents (instance, opts) { } } // online only - if (client && instance) { + if (client && instance && opts) { // add important data to local ipfs repo for instant load - setTimeout(() => precache(instance), 10000) + setTimeout(() => precache(instance, opts), 5000) } } diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js index f3722fcd4..d5560e7f4 100644 --- a/add-on/src/lib/ipfs-companion.js +++ b/add-on/src/lib/ipfs-companion.js @@ -311,8 +311,6 @@ module.exports = async function init () { let results try { const dataSrc = await findValueForContext(context, contextType) - log('onAddFromContext.context', context) // TODO - log('onAddFromContext.dataSrc', dataSrc) // TODO if (contextType === 'selection') { // TODO: persist full pageUrl somewhere (eg. append at the end of the content but add toggle to disable it) data = { diff --git a/add-on/src/lib/precache.js b/add-on/src/lib/precache.js index 84e1b5862..24ccf15a0 100644 --- a/add-on/src/lib/precache.js +++ b/add-on/src/lib/precache.js @@ -11,48 +11,86 @@ const debug = require('debug') const log = debug('ipfs-companion:precache') log.error = debug('ipfs-companion:precache:error') -// Web UI release that should be precached -// WARNING: do not remove this constant, as its used in package.json -const webuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' // v2.9.0 -module.exports.precachedWebuiCid = webuiCid - -const PRECACHE_ARCHIVES = [ - { tarPath: '/dist/precache/webui.tar', cid: webuiCid } -] +// TODO: can be removed when embedded:chromesockets is gone +module.exports.braveJsIpfsWebuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' // v2.9.0 for js-ipfs in Brave (import in newer ones does not work) /** * Adds important assets such as Web UI to the local js-ipfs-repo. * This ensures they load instantly, even in offline environments. */ -module.exports.precache = async (ipfs) => { - for (const { cid, tarPath } of PRECACHE_ARCHIVES) { - if (!await inRepo(ipfs, cid)) { - try { - const { body } = await fetch(tarPath) - log(`importing ${tarPath} to js-ipfs-repo`) - await importTar(ipfs, body.getReader(), cid) - log(`${tarPath} successfully cached under CID ${cid}`) - } catch (err) { - if (err.message === 'Error in body stream') { - // This error means .tar is missing from the extension bundle: - // It is the case in Firefox, due to https://github.com/ipfs-shipyard/ipfs-webui/issues/959 - log(`unable to find/read ${tarPath}, skipping`) +module.exports.precache = async (ipfs, state) => { + // simplified prefetch over HTTP when in Brave + if (state.ipfsNodeType === 'embedded:chromesockets') { + return preloadOverHTTP(log, ipfs, state, module.exports.braveJsIpfsWebuiCid) + } + + const roots = [] + // find out the content path of webui, and add it to precache list + try { + let cid, name + if (state.useLatestWebUI) { // resolve DNSLink + cid = await ipfs.dns('webui.ipfs.io', { recursive: true }) + name = 'latest webui from DNSLink at webui.ipfs.io' + } else { // find out safelisted path behind /webui + cid = new URL((await fetch(`${state.apiURLString}webui`)).url).pathname + name = `stable webui hardcoded at ${state.apiURLString}webui` + } + roots.push({ + nodeType: 'external', + name, + cid + }) + } catch (e) { + log.error('unable to find webui content path for precache', e) + } + + // precache each root + for (const { name, cid, nodeType } of roots) { + if (state.ipfsNodeType !== nodeType) continue + if (await inRepo(ipfs, cid)) { + log(`${name} (${cid}) already in local repo, skipping import`) + continue + } + log(`importing ${name} (${cid}) to local ipfs repo`) + + // prefetch over IPFS + try { + for await (const ref of ipfs.refs(cid, { recursive: true })) { + if (ref.err) { + log.error(`error while preloading ${name} (${cid})`, ref.err) continue } - log.error(`error while processing ${tarPath}`, err) } - } else { - log(`${cid} already in local repo, skipping import`) + log(`${name} successfully cached under CID ${cid}`) + } catch (err) { + log.error(`error while processing ${name}`, err) } } } async function inRepo (ipfs, cid) { - for await (const ref of ipfs.refs.local()) { - if (ref.err) return false - if (ref.ref === cid) return true + // dag.get in offline mode will throw block is not present in local repo + // (we also have timeout as a failsafe) + try { + await ipfs.dag.get(cid, { offline: true, timeout: 5000 }) + return true + } catch (_) { + return false + } +} + +// Downloads CID from a public gateway +// (alternative to ipfs.refs -r) +async function preloadOverHTTP (log, ipfs, state, cid) { + const url = `${state.pubGwURLString}api/v0/get?arg=${cid}&archive=true` + try { + log(`importing ${url} (${cid}) to local ipfs repo`) + const { body } = await fetch(url) + await importTar(ipfs, body.getReader(), cid) + log(`successfully fetched TAR from ${url} and cached under CID ${cid}`) + } catch (err) { + log.error(`error while processing ${url}`, err) } - return false } async function importTar (ipfs, tarReader, expectedCid) { @@ -84,7 +122,7 @@ async function importTar (ipfs, tarReader, expectedCid) { const root = results.find(e => e.cid.toString(multibaseName) === expectedCid) if (!root) { - throw new Error('imported CID does not match expected one') + throw new Error(`imported CID (${root}) does not match expected one: ${expectedCid}`) } } diff --git a/add-on/src/lib/state.js b/add-on/src/lib/state.js index 75024895a..2d1ee0caa 100644 --- a/add-on/src/lib/state.js +++ b/add-on/src/lib/state.js @@ -2,7 +2,7 @@ /* eslint-env browser, webextensions */ const { safeURL } = require('./options') -const { precachedWebuiCid } = require('./precache') +const { braveJsIpfsWebuiCid } = require('./precache') const offlinePeerCount = -1 function initState (options, overrides) { @@ -48,12 +48,11 @@ function initState (options, overrides) { // Did user opt-in for rolling release published on DNSLink? if (state.useLatestWebUI) return `${state.gwURLString}ipns/webui.ipfs.io/` - // Below is needed to make webui work for embedded js-ipfs - // TODO: revisit if below is still needed after upgrading to js-ipfs >= 44 - const webuiUrl = state.ipfsNodeType === 'embedded:chromesockets' - ? `${state.gwURLString}ipfs/${precachedWebuiCid}/` - : `${state.apiURLString}webui` - return webuiUrl + // Below is needed to make webui work for embedded js-ipfs in Brave + if (state.ipfsNodeType === 'embedded:chromesockets' && braveJsIpfsWebuiCid) { + return `${state.gwURLString}ipfs/${braveJsIpfsWebuiCid}/` + } + return `${state.apiURLString}webui` } }) // apply optional overrides diff --git a/package.json b/package.json index 0f2a2e9cc..59d63ea7c 100644 --- a/package.json +++ b/package.json @@ -23,22 +23,20 @@ "build:bundle-all": "cross-env RELEASE_CHANNEL=${RELEASE_CHANNEL:=dev} run-s bundle:chromium bundle:brave:$RELEASE_CHANNEL bundle:firefox:$RELEASE_CHANNEL", "build:rename-artifacts": "./scripts/rename-artifacts.js", "precache:clean": "shx rm -rf add-on/dist/precache", - "precache:webui:cid": "shx grep 'const webuiCid' add-on/src/lib/precache.js | shx sed \"s/^const webuiCid = '//\" | shx sed \"s/'.*$//\"", - "precache:webui": "shx mkdir -p add-on/dist/precache && ipfs-or-gateway -c $(npm run -s precache:webui:cid) -p add-on/dist/precache/webui.tar --archive", "bundle": "run-s bundle:*", - "bundle:chromium": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts", - "bundle:firefox": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", + "bundle:chromium": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts", + "bundle:firefox": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", "bundle:firefox:dev": "npm run bundle:firefox", "bundle:firefox:stable": "npm run bundle:firefox", - "bundle:firefox:beta": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", + "bundle:firefox:beta": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", "bundle:fennec": "npm run bundle:firefox", "bundle:fennec:dev": "npm run bundle:firefox:dev", "bundle:fennec:stable": "npm run bundle:firefox:stable", "bundle:fennec:beta": "npm run bundle:firefox:beta", - "bundle:brave": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", + "bundle:brave": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", "bundle:brave:dev": "npm run bundle:brave", "bundle:brave:stable": "npm run bundle:brave", - "bundle:brave:beta": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", + "bundle:brave:beta": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", "watch": "npm-run-all build:copy --parallel watch:*", "watch:js": "run-p watch:js:*", "watch:js:webpack": "webpack --watch --progress -d --devtool inline-source-map --config ./webpack.config.js", @@ -144,12 +142,10 @@ "is-fqdn": "1.0.1", "is-ipfs": "1.0.3", "it-all": "1.0.4", - "it-buffer": "0.1.2", "it-concat": "1.0.1", "it-tar": "1.2.2", "lru-cache": "5.1.1", "merge-options": "2.0.0", - "mortice": "2.0.0", "multiaddr": "7.4.3", "multiaddr-to-uri": "5.0.0", "p-memoize": "4.0.0", diff --git a/yarn.lock b/yarn.lock index 52a6c9863..890023347 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8436,7 +8436,7 @@ it-batch@^1.0.3, it-batch@^1.0.4: resolved "https://registry.yarnpkg.com/it-batch/-/it-batch-1.0.4.tgz#2d68d6e57899e159644aa16c0b4b1c93eb0d5c65" integrity sha512-hZ+gaj5MaECauRd+Ahvo9iAxg90YGVBg7AZ32wOeXJ08IRjfQRMSnZ9oA0JjNeJeSGuVjWf91UUD5y2SYmKlwQ== -it-buffer@0.1.2, it-buffer@^0.1.1, it-buffer@^0.1.2: +it-buffer@^0.1.1, it-buffer@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/it-buffer/-/it-buffer-0.1.2.tgz#2b37e2c66bbbb94479c2e47c1904bd729f04fc39" integrity sha512-NOJ3ogSNq3Y2c75ZDcPs9qlgitWyCkUQdmgqqMw+/LMmHZqwWQw7OBDodonz250nJ4EEBXkRQ+pIwz1sL9Zuyg== @@ -10677,7 +10677,7 @@ moment@^2.19.3: resolved "https://registry.yarnpkg.com/moment/-/moment-2.28.0.tgz#cdfe73ce01327cee6537b0fafac2e0f21a237d75" integrity sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw== -mortice@2.0.0, mortice@^2.0.0: +mortice@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/mortice/-/mortice-2.0.0.tgz#7be171409c2115561ba3fc035e4527f9082eefde" integrity sha512-rXcjRgv2MRhpwGHErxKcDcp5IoA9CPvPFLXmmseQYIuQ2fSVu8tsMKi/eYUXzp/HH1s6y3IID/GwRqlSglDdRA==