11.0.1

composer.json

@@ -148,7 +148,8 @@
     },
     "scripts": {
         "post-drupal-scaffold-cmd": [
-            "patch -Ns -r \"\" --no-backup-if-mismatch -p1 < ./patches/htaccess.patch"
+            "patch -Ns -r \"\" --no-backup-if-mismatch -p1 < ./patches/htaccess.patch",
+            "patch -Ns -r \"\" --no-backup-if-mismatch -p1 < ./patches/robotstxt.patch"
         ],
         "init-lando": [
             "cp lando/example.local.blt.yml blt/local.blt.yml",

docroot/robots.txt

@@ -14,6 +14,7 @@
 # http://www.robotstxt.org/robotstxt.html
 
 User-agent: *
+Crawl-delay: 30
 # CSS, JS, Images
 Allow: /core/*.css$
 Allow: /core/*.css?
@@ -63,3 +64,16 @@ Disallow: /index.php/user/login
 Disallow: /index.php/user/logout
 Disallow: /index.php/media/oembed
 Disallow: /index.php/*/media/oembed
+# Block saml paths links
+Disallow: /sso/login
+Disallow: /saml_login
+Disallow: /saml/login
+Disallow: /saml/acs
+# Block Femtosearch
+User-agent: FemtosearchBot
+Crawl-Delay: 120
+Disallow: /
+# Block SemrushBot
+User-agent: SemrushBot
+Crawl-Delay: 120
+Disallow: /

patches/robotstxt.patch

@@ -0,0 +1,29 @@
+diff --git a/docroot/robots.txt b/docroot/robots.txt
+index ebcd04b..8927a2b 100644
+--- a/docroot/robots.txt
++++ b/docroot/robots.txt
+@@ -14,6 +14,7 @@
+ # http://www.robotstxt.org/robotstxt.html
+
+ User-agent: *
++Crawl-delay: 30
+ # CSS, JS, Images
+ Allow: /core/*.css$
+ Allow: /core/*.css?
+@@ -63,3 +64,16 @@ Disallow: /index.php/user/login
+ Disallow: /index.php/user/logout
+ Disallow: /index.php/media/oembed
+ Disallow: /index.php/*/media/oembed
++# Block saml paths links
++Disallow: /sso/login
++Disallow: /saml_login
++Disallow: /saml/login
++Disallow: /saml/acs
++# Block Femtosearch
++User-agent: FemtosearchBot
++Crawl-Delay: 120
++Disallow: /
++# Block SemrushBot
++User-agent: SemrushBot
++Crawl-Delay: 120
++Disallow: /