feat: recover from DNS failures (#797)

This change builds on top of #783 and:

-  adds support for recovery from DNS lookup failures
-  enables recovery for all HTTP Codes >= 400 
-  recovers `.eth` DNS failures bu reopening website on EthDNS gateway at `.eth.link`
-  simplifies some code paths and adds more tests

Motivation:

When a third-party IPFS gateway is discontinued or censored at DNS level, the IPFS Companion should retry request using currently active gateway set by the user (public or local).

We also want to recover in situation when website with DNSLink has a valid DNS `TXT` record, but HTTP behind `A` record is down or unreachable.

*Note*: right now the only real use for DNS recovery is support of .eth TLD via .eth.link gateway, however in the future this could provide means of working around DNS-based censorship (eg. by executing DNSLink lookups over libp2p as a fallback).

closes #678, closes #640

add-on/src/lib/dnslink.js

@@ -180,8 +180,7 @@ module.exports = function createDnslinkResolver (getState) {
       if (typeof url === 'string') {
         url = new URL(url)
       }
-      const fqdn = url.hostname
-      return `/ipns/${fqdn}${url.pathname}${url.search}${url.hash}`
+      return `/ipns/${url.hostname}${url.pathname}${url.search}${url.hash}`
     },
 
     // Test if URL contains a valid DNSLink FQDN

add-on/src/lib/ipfs-request.js

@@ -9,36 +9,20 @@ const LRU = require('lru-cache')
 const IsIpfs = require('is-ipfs')
 const isFQDN = require('is-fqdn')
 const { pathAtHttpGateway } = require('./ipfs-path')
+
 const redirectOptOutHint = 'x-ipfs-companion-no-redirect'
-const recoverableErrors = new Set([
+const recoverableNetworkErrors = new Set([
   // Firefox
+  'NS_ERROR_UNKNOWN_HOST', // dns failure
   'NS_ERROR_NET_TIMEOUT', // eg. httpd is offline
   'NS_ERROR_NET_RESET', // failed to load because the server kept reseting the connection
   'NS_ERROR_NET_ON_RESOLVED', // no network
   // Chrome
+  'net::ERR_NAME_NOT_RESOLVED', // dns failure
   'net::ERR_CONNECTION_TIMED_OUT', // eg. httpd is offline
   'net::ERR_INTERNET_DISCONNECTED' // no network
 ])
-
-const recoverableErrorCodes = new Set([
-  404,
-  408,
-  410,
-  415,
-  451,
-  500,
-  502,
-  503,
-  504,
-  509,
-  520,
-  521,
-  522,
-  523,
-  524,
-  525,
-  526
-])
+const recoverableHttpError = (code) => code && code >= 400
 
 // Request modifier provides event listeners for the various stages of making an HTTP request
 // API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
@@ -171,11 +155,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     // This is a good place to listen if you want to modify HTTP request headers.
     onBeforeSendHeaders (request) {
       const state = getState()
-
-      // Skip if IPFS integrations are inactive
-      if (!state.active) {
-        return
-      }
+      if (!state.active) return
 
       // Special handling of requests made to API
       if (request.url.startsWith(state.apiURLString)) {
@@ -286,11 +266,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     // You can use this event to modify HTTP response headers or do a very late redirect.
     onHeadersReceived (request) {
       const state = getState()
-
-      // Skip if IPFS integrations are inactive
-      if (!state.active) {
-        return
-      }
+      if (!state.active) return
 
       // Special handling of requests made to API
       if (request.url.startsWith(state.apiURLString)) {
@@ -387,58 +363,53 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     },
 
     // browser.webRequest.onErrorOccurred
-    // Fired when a request could not be processed due to an error:
-    // for example, a lack of Internet connectivity.
+    // Fired when a request could not be processed due to an error on network level.
+    // For example: TCP timeout, DNS lookup failure
     async onErrorOccurred (request) {
       const state = getState()
-
-      // Skip if IPFS integrations are inactive or request is marked as ignored
-      if (!state.active || isIgnored(request.requestId)) {
-        return
+      if (!state.active) return
+
+      // Check if error can be recovered via EthDNS
+      if (isRecoverableViaEthDNS(request, state)) {
+        const url = new URL(request.url)
+        url.hostname = `${url.hostname}.link`
+        const redirect = { redirectUrl: url.toString() }
+        log(`onErrorOccurred: attempting to recover from DNS error (${request.error}) using EthDNS for ${request.url}`, redirect.redirectUrl)
+        return createTabWithURL(redirect, browser)
       }
 
-      // console.log('onErrorOccurred:' + request.error)
-      // console.log('onErrorOccurred', request)
-      // Check if error is final and can be recovered via DNSLink
-      let redirect
-      const recoverableViaDnslink =
-        state.dnslinkPolicy &&
-        request.type === 'main_frame' &&
-        recoverableErrors.has(request.error)
-      if (recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)) {
-        // Explicit call to ignore global DNSLink policy and force DNS TXT lookup
-        const cachedDnslink = dnslinkResolver.readAndCacheDnslink(new URL(request.url).hostname)
-        redirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
-        log(`onErrorOccurred: attempting to recover using dnslink for ${request.url}`, redirect)
+      // Check if error can be recovered via DNSLink
+      if (isRecoverableViaDNSLink(request, state, dnslinkResolver)) {
+        const { hostname } = new URL(request.url)
+        const dnslink = dnslinkResolver.readAndCacheDnslink(hostname)
+        if (dnslink) {
+          const redirect = dnslinkResolver.dnslinkRedirect(request.url, dnslink)
+          log(`onErrorOccurred: attempting to recover from network error (${request.error}) using dnslink for ${request.url}`, redirect.redirectUrl)
+          return createTabWithURL(redirect, browser)
+        }
       }
-      // if error cannot be recovered via DNSLink
-      // direct the request to the public gateway
-      const recoverable = isRecoverable(request, state, ipfsPathValidator)
-      if (!redirect && recoverable) {
+
+      // Check if error can be recovered by opening same content-addresed path
+      // using active gateway (public or local, depending on redirect state)
+      if (isRecoverable(request, state, ipfsPathValidator)) {
         const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
-        redirect = { redirectUrl }
-        log(`onErrorOccurred: attempting to recover failed request for ${request.url}`, redirect)
-      }
-      // We can't redirect in onErrorOccurred, so if DNSLink is present
-      // recover by opening IPNS version in a new tab
-      // TODO: add tests and demo
-      if (redirect) {
-        createTabWithURL(redirect, browser)
+        log(`onErrorOccurred: attempting to recover from network error (${request.error}) for ${request.url}`, redirectUrl)
+        return createTabWithURL({ redirectUrl }, browser)
       }
     },
 
+    // browser.webRequest.onCompleted
+    // Fired when HTTP request is completed (successfully or with an error code)
     async onCompleted (request) {
       const state = getState()
-
-      const recoverable =
-        isRecoverable(request, state, ipfsPathValidator) &&
-        recoverableErrorCodes.has(request.statusCode)
-      if (recoverable) {
+      if (!state.active) return
+      if (request.statusCode === 200) return // finish if no error to recover from
+      if (isRecoverable(request, state, ipfsPathValidator)) {
         const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
         const redirect = { redirectUrl }
         if (redirect) {
-          log(`onCompleted: attempting to recover failed request for ${request.url}`, redirect)
-          createTabWithURL(redirect, browser)
+          log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirect)
+          return createTabWithURL(redirect, browser)
         }
       }
     }
@@ -548,18 +519,41 @@ function findHeaderIndex (name, headers) {
   return headers.findIndex(x => x.name && x.name.toLowerCase() === name.toLowerCase())
 }
 
-// utility functions for handling redirects
-// from onErrorOccurred and onCompleted
+// RECOVERY OF FAILED REQUESTS
+// ===================================================================
+
+// Recovery check for onErrorOccurred (request.error) and onCompleted (request.statusCode)
 function isRecoverable (request, state, ipfsPathValidator) {
   return state.recoverFailedHttpRequests &&
+    request.type === 'main_frame' &&
+    (recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) &&
     ipfsPathValidator.publicIpfsOrIpnsResource(request.url) &&
-    !request.url.startsWith(state.pubGwURLString) &&
-    request.type === 'main_frame'
+    !request.url.startsWith(state.pubGwURLString)
+}
+
+// Recovery check for onErrorOccurred (request.error)
+function isRecoverableViaDNSLink (request, state, dnslinkResolver) {
+  const recoverableViaDnslink =
+    state.recoverFailedHttpRequests &&
+    request.type === 'main_frame' &&
+    state.dnslinkPolicy &&
+    recoverableNetworkErrors.has(request.error)
+  return recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)
+}
+
+// Recovery check for onErrorOccurred (request.error)
+function isRecoverableViaEthDNS (request, state) {
+  return state.recoverFailedHttpRequests &&
+    request.type === 'main_frame' &&
+    recoverableNetworkErrors.has(request.error) &&
+    new URL(request.url).hostname.endsWith('.eth')
 }
 
+// We can't redirect in onErrorOccurred/onCompleted
+// Indead, we recover by opening URL in a new tab that replaces the failed one
 async function createTabWithURL (redirect, browser) {
   const currentTabId = await browser.tabs.query({ active: true, currentWindow: true }).then(tabs => tabs[0].id)
-  await browser.tabs.create({
+  return browser.tabs.create({
     active: true,
     openerTabId: currentTabId,
     url: redirect.redirectUrl

add-on/src/options/forms/experiments-form.js

@@ -59,6 +59,15 @@ function experimentsForm ({
           </label>
           <div>${switchToggle({ id: 'catchUnhandledProtocols', checked: catchUnhandledProtocols, onchange: onCatchUnhandledProtocolsChange })}</div>
         </div>
+        <div>
+          <label for="recoverFailedHttpRequests">
+            <dl>
+              <dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
+              <dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
+            </dl>
+          </label>
+          <div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
+        </div>
         <div>
           <label for="linkify">
             <dl>
@@ -98,15 +107,6 @@ function experimentsForm ({
             </option>
           </select>
         </div>
-        <div>
-          <label for="recoverFailedHttpRequests">
-            <dl>
-              <dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
-              <dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
-            </dl>
-          </label>
-          <div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
-        </div>
         <div>
           <label for="detectIpfsPathHeader">
             <dl>

test/functional/lib/ipfs-request-gateway-recover.test.js

@@ -18,7 +18,11 @@ const urlRequestWithStatus = (url, statusCode = 200, type = 'main_frame') => {
   return { ...url2request(url, type), statusCode }
 }
 
-describe('requestHandler.onCompleted:', function () {
+const urlRequestWithNetworkError = (url, error = 'net::ERR_CONNECTION_TIMED_OUT', type = 'main_frame') => {
+  return { ...url2request(url, type), error }
+}
+
+describe('requestHandler.onCompleted:', function () { // HTTP-level errors
   let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
 
   before(function () {
@@ -40,6 +44,7 @@ describe('requestHandler.onCompleted:', function () {
   describe('with recoverFailedHttpRequests=true', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = true
+      state.dnslinkPolicy = false
     })
     it('should do nothing if broken request is a non-IPFS request', async function () {
       const request = urlRequestWithStatus('https://wikipedia.org', 500)
@@ -61,7 +66,7 @@ describe('requestHandler.onCompleted:', function () {
       await requestHandler.onCompleted(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
-    it('should redirect broken non-default public gateway IPFS request to public gateway', async function () {
+    it('should recover from unreachable third party public gateway by reopening on the public gateway', async function () {
       const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
       await requestHandler.onCompleted(request)
       assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
@@ -71,6 +76,7 @@ describe('requestHandler.onCompleted:', function () {
   describe('with recoverFailedHttpRequests=false', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = false
+      state.dnslinkPolicy = false
     })
     it('should do nothing on broken non-default public gateway IPFS request', async function () {
       const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
@@ -90,7 +96,7 @@ describe('requestHandler.onCompleted:', function () {
   })
 })
 
-describe('requestHandler.onErrorOccurred:', function () {
+describe('requestHandler.onErrorOccurred:', function () { // network errors
   let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
 
   before(function () {
@@ -112,42 +118,83 @@ describe('requestHandler.onErrorOccurred:', function () {
   describe('with recoverFailedHttpRequests=true', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = true
+      state.dnslinkPolicy = false
     })
     it('should do nothing if failed request is a non-IPFS request', async function () {
-      const request = url2request('https://wikipedia.org', 500)
+      const request = urlRequestWithNetworkError('https://wikipedia.org')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
     it('should do nothing if failed request is a non-public IPFS request', async function () {
-      const request = url2request('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
+      const request = urlRequestWithNetworkError('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
     it('should do nothing if failed request is to the default public gateway', async function () {
-      const request = url2request('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
+      const request = urlRequestWithNetworkError('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
     it('should do nothing if failed request is not a \'main_frame\' request', async function () {
-      const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'stylesheet')
+      const requestType = 'stylesheet'
+      const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'net::ERR_NAME_NOT_RESOLVED', requestType)
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
-    it('should redirect failed non-default public gateway IPFS request to public gateway', async function () {
-      const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
+    it('should recover from unreachable third party public gateway by reopening on the public gateway', async function () {
+      const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
     })
+    it('should recover from unreachable HTTP server by reopening DNSLink on the public gateway', async function () {
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')
+      const expectedUrl = 'http://127.0.0.1:8080/ipns/en.wikipedia-on-ipfs.org/'
+      const request = urlRequestWithNetworkError('https://en.wikipedia-on-ipfs.org/')
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.withArgs({ url: expectedUrl, active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with ENS resource on local gateway URL')
+      dnslinkResolver.clearCache()
+    })
+    it('should recover from failed DNS for .eth opening it on EthDNS gateway at .eth.link', async function () {
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('almonit.eth', false)
+      dnslinkResolver.setDnslink('almonit.eth.link', '/ipfs/QmPH7VMnfFKvrr7kLXNRwuxjYRLWnfcxPvnWs8ipyWAQK2')
+      const dnsFailure = 'net::ERR_NAME_NOT_RESOLVED' // chrome code
+      const expectedUrl = 'https://almonit.eth.link/'
+      const request = urlRequestWithNetworkError('https://almonit.eth', dnsFailure)
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.withArgs({ url: expectedUrl, active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with ENS resource on local gateway URL')
+      dnslinkResolver.clearCache()
+    })
   })
 
   describe('with recoverFailedHttpRequests=false', function () {
     beforeEach(function () {
       state.recoverFailedHttpRequests = false
+      state.dnslinkPolicy = false
+    })
+    it('should do nothing on unreachable third party public gateway', async function () {
+      const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+    })
+    it('should do nothing on unreachable HTTP server with DNSLink', async function () {
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')
+      const request = urlRequestWithNetworkError('https://en.wikipedia-on-ipfs.org')
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+      dnslinkResolver.clearCache()
     })
     it('should do nothing on failed non-default public gateway IPFS request', async function () {
-      const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
+      state.dnslinkPolicy = 'best-effort'
+      dnslinkResolver.setDnslink('almonit.eth', false)
+      dnslinkResolver.setDnslink('almonit.eth.link', '/ipfs/QmPH7VMnfFKvrr7kLXNRwuxjYRLWnfcxPvnWs8ipyWAQK2')
+      const dnsFailure = 'net::ERR_NAME_NOT_RESOLVED' // chrome code
+      const request = urlRequestWithNetworkError('https://almonit.eth', dnsFailure)
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+      dnslinkResolver.clearCache()
     })
   })