Merge pull request #434 from sharetribe/private-marketplace

Private marketplace mode.

CHANGELOG.md

@@ -14,6 +14,15 @@ way to update this template, but currently, we follow a pattern:
 
 ## Upcoming version 2024-XX-XX
 
+- [add] Access control: private marketplace mode
+
+  - Fetch a new asset: /general/access-control.json to check private: true/false flag
+  - Make SearchPage, ListingPage, ProfilePage, Styleguide require authentication
+  - Ensure currentUser entity is loaded before loadData on client-side
+  - Restrict data load & add redirections for SearchPage, ListingPage, and ProfilePage
+
+  [#434](https://github.com/sharetribe/web-template/pull/434)
+
 - [add] Access control: 'pending-approval' state for users.
 
   - Users will get "state", which is exposed through currentUser's attribute

server/api-util/sdk.js

@@ -186,3 +186,23 @@ exports.fetchBranding = sdk => {
     return response;
   });
 };
+
+// Fetch branding asset with 'latest' alias.
+// This is needed for generating webmanifest on server-side.
+exports.fetchAccessControlAsset = sdk => {
+  return sdk
+    .assetsByAlias({ paths: ['/general/access-control.json'], alias: 'latest' })
+    .then(response => {
+      // Let's throw an error if we can't fetch branding for some reason
+      const accessControlAsset = response?.data?.data?.[0];
+      if (!accessControlAsset) {
+        const message = 'access-control configuration was not available.';
+        const error = new Error(message);
+        error.status = 404;
+        error.statusText = message;
+        error.data = {};
+        throw error;
+      }
+      return response;
+    });
+};

server/resources/README.md

@@ -29,6 +29,11 @@ from `REACT_APP_MARKETPLACE_ROOT_URL`. This transformation is done by the actual
 **Note**: on localhost, this file is served from the Dev port on apiServer.js (3500 aka
 http://localhost:3500/robots.txt) for debugging purposes.
 
+### Robots.txt on private marketplace
+
+There's a more restrictive version of the robots.txt (robotsPrivateMarketplace.txt), which is used
+if access-control.json asset has private mode set active.
+
 ## Sitemap
 
 A [sitemap](https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview) is a file
@@ -60,6 +65,8 @@ The sitemap-index.xml contains links to 3 different sub sitemaps:
 - sitemap-recent-listings.xml
 - sitemap-recent-pages.xml
 
+**Note**: sitemap-recent-listings.xml is not served if the private marketplace mode is activated.
+
 ### /sitemap-default.xml
 
 This sitemap contains links to public built-in pages of the client app. It also shows
@@ -71,11 +78,15 @@ sitemap. E.g. if you have added a category "hats", you could highlight that for
 by adding `searchHats: { url: '/s?pub_category=hats' }` to variable **defaultPublicPaths** on
 sitemap.js.
 
+**Note**: search page is not served if the private marketplace mode is activated.
+
 ### /sitemap-recent-listings.xml
 
 The recent listings is the heaviest file to be returned. Marketplace API returns max 10,000 recent
 listings, from which the XML syntax is created and the result is compressed.
 
+**Note**: sitemap-recent-listings.xml is not served if the private marketplace mode is activated.
+
 ### /sitemap-recent-pages.xml
 
 This contains (CMS) **Pages**, which are shown from path `/p/:pageId`. However, it doesn't contain

server/resources/robotsPrivateMarketplace.txt

@@ -0,0 +1,21 @@
+#
+# Hello bot! There are some routes that need authentication. You should avoid them :)
+#
+
+User-agent: *
+Disallow: /s
+Disallow: /u
+Disallow: /l
+Disallow: /profile-settings
+Disallow: /inbox
+Disallow: /order
+Disallow: /sale
+Disallow: /listings
+Disallow: /account
+Disallow: /reset-password
+Disallow: /verify-email
+Disallow: /preview
+Disallow: /styleguide
+Crawl-Delay: 5
+
+Sitemap: https://my.marketplace.com/sitemap-index.xml

server/resources/robotsTxt.js

@@ -2,6 +2,9 @@ const fs = require('fs');
 const { Transform, Writable } = require('stream');
 const log = require('../log.js');
 const { getRootURL } = require('../api-util/rootURL.js');
+const sdkUtils = require('../api-util/sdk.js');
+
+const dev = process.env.REACT_APP_ENV === 'development';
 
 // Emulate feature that's part of sitemap dependency
 const streamToPromise = stream => {
@@ -27,9 +30,32 @@ const streamToPromise = stream => {
   });
 };
 
-// Simple variable cache
-// This assumes that robots.txt does not change after first initialization
-let cachedRobotsTxt = null;
+// Time-to-live (ttl) is set to one day aka 86400 seconds
+const ttl = 86400; // seconds
+
+// This creates simple (proxied) memory cache
+const createCacheProxy = ttl => {
+  const cache = {};
+  return new Proxy(cache, {
+    // Get data for the property together with timestamp
+    get(target, property, receiver) {
+      const cachedData = target[property];
+      if (!!cachedData) {
+        // Check if the cached data has expired
+        if (Date.now() - cachedData.timestamp < ttl * 1000) {
+          return cachedData;
+        }
+      }
+      return { data: null, timestamp: cachedData?.timestamp || Date.now() };
+    },
+    // Set given value as data to property accompanied with timestamp
+    set(target, property, value, receiver) {
+      target[property] = { data: value, timestamp: Date.now() };
+    },
+  });
+};
+
+const cache = createCacheProxy(ttl);
 
 // Fallback data if something failes with streams
 const fallbackRobotsTxt = `
@@ -49,20 +75,19 @@ Disallow: /account
 Disallow: /reset-password
 Disallow: /verify-email
 Disallow: /preview
+Disallow: /styleguide
 Crawl-Delay: 5
 `;
 
-// Middleware to generate robots.txt
-// This reads the accompanied robots.txt file and changes the sitemap url on the fly
-module.exports = (req, res) => {
-  res.header('Content-Type', 'text/plain');
-
-  // If we have a cached content send it
-  if (cachedRobotsTxt) {
-    res.send(cachedRobotsTxt);
-    return;
-  }
-
+/**
+ * This processes given robots.txt file (adds correct URL for the sitemap-index.xml)
+ * and sends it as response.
+ *
+ * @param {Object} req
+ * @param {Object} res
+ * @param {String} robotsTxtPath
+ */
+const sendRobotsTxt = (req, res, robotsTxtPath) => {
   const sitemapIndexUrl = `${getRootURL({ useDevApiServerPort: true })}/sitemap-index.xml`;
 
   try {
@@ -77,11 +102,11 @@ module.exports = (req, res) => {
       },
     });
 
-    const readStream = fs.createReadStream('server/resources/robots.txt', { encoding: 'utf8' });
+    const readStream = fs.createReadStream(robotsTxtPath, { encoding: 'utf8' });
     const robotsStream = readStream.pipe(modifiedStream);
 
     // Save the data to a variable cache
-    streamToPromise(robotsStream).then(rs => (cachedRobotsTxt = rs));
+    streamToPromise(robotsStream).then(rs => (cache.robotsTxt = rs));
 
     robotsStream.pipe(res).on('error', e => {
       throw e;
@@ -91,3 +116,46 @@ module.exports = (req, res) => {
     res.send(fallbackRobotsTxt);
   }
 };
+// Middleware to generate robots.txt
+// This reads the accompanied robots.txt file and changes the sitemap url on the fly
+module.exports = (req, res) => {
+  res.set({
+    'Content-Type': 'text/plain',
+    'Cache-Control': `public, max-age=${ttl}`,
+  });
+
+  // If we have a cached content send it
+  const { data, timestamp } = cache.robotsTxt;
+  if (data && timestamp) {
+    const age = Math.floor((Date.now() - timestamp) / 1000);
+    res.set('Age', age);
+    res.send(data);
+    return;
+  }
+
+  const sdk = sdkUtils.getSdk(req, res);
+  sdkUtils
+    .fetchAccessControlAsset(sdk)
+    .then(response => {
+      const accessControlAsset = response.data.data[0];
+
+      const { marketplace } =
+        accessControlAsset?.type === 'jsonAsset' ? accessControlAsset.attributes.data : {};
+      const isPrivateMarketplace = marketplace?.private === true;
+      const robotsTxtPath = isPrivateMarketplace
+        ? 'server/resources/robotsPrivateMarketplace.txt'
+        : 'server/resources/robots.txt';
+
+      sendRobotsTxt(req, res, robotsTxtPath);
+    })
+    .catch(e => {
+      // Log error
+      const is404 = e.status === 404;
+      if (is404 && dev) {
+        console.log('robots-txt-render-failed-no-asset-found');
+      }
+      // TODO: This defaults to more permissive robots.txt due to backward compatibility.
+      // You might want to change that.
+      sendRobotsTxt(req, res, 'server/resources/robots.txt');
+    });
+};