From d98a57f4bb22aa3e9f51189ce2551b08c8b83002 Mon Sep 17 00:00:00 2001
From: Mike Decker <pookmish@gmail.com>
Date: Tue, 31 Oct 2023 17:58:11 -0700
Subject: [PATCH] 11.0.1

---
 composer.json           |  3 ++-
 docroot/robots.txt      | 14 ++++++++++++++
 patches/robotstxt.patch | 29 +++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 patches/robotstxt.patch

diff --git a/composer.json b/composer.json
index 25d8e3d..0a9fb5f 100644
--- a/composer.json
+++ b/composer.json
@@ -148,7 +148,8 @@
     },
     "scripts": {
         "post-drupal-scaffold-cmd": [
-            "patch -Ns -r \"\" --no-backup-if-mismatch -p1 < ./patches/htaccess.patch"
+            "patch -Ns -r \"\" --no-backup-if-mismatch -p1 < ./patches/htaccess.patch",
+            "patch -Ns -r \"\" --no-backup-if-mismatch -p1 < ./patches/robotstxt.patch"
         ],
         "init-lando": [
             "cp lando/example.local.blt.yml blt/local.blt.yml",
diff --git a/docroot/robots.txt b/docroot/robots.txt
index ebcd04b..8927a2b 100644
--- a/docroot/robots.txt
+++ b/docroot/robots.txt
@@ -14,6 +14,7 @@
 # http://www.robotstxt.org/robotstxt.html
 
 User-agent: *
+Crawl-delay: 30
 # CSS, JS, Images
 Allow: /core/*.css$
 Allow: /core/*.css?
@@ -63,3 +64,16 @@ Disallow: /index.php/user/login
 Disallow: /index.php/user/logout
 Disallow: /index.php/media/oembed
 Disallow: /index.php/*/media/oembed
+# Block saml paths links
+Disallow: /sso/login
+Disallow: /saml_login
+Disallow: /saml/login
+Disallow: /saml/acs
+# Block Femtosearch
+User-agent: FemtosearchBot
+Crawl-Delay: 120
+Disallow: /
+# Block SemrushBot
+User-agent: SemrushBot
+Crawl-Delay: 120
+Disallow: /
diff --git a/patches/robotstxt.patch b/patches/robotstxt.patch
new file mode 100644
index 0000000..81587c2
--- /dev/null
+++ b/patches/robotstxt.patch
@@ -0,0 +1,29 @@
+diff --git a/docroot/robots.txt b/docroot/robots.txt
+index ebcd04b..8927a2b 100644
+--- a/docroot/robots.txt
++++ b/docroot/robots.txt
+@@ -14,6 +14,7 @@
+ # http://www.robotstxt.org/robotstxt.html
+ 
+ User-agent: *
++Crawl-delay: 30
+ # CSS, JS, Images
+ Allow: /core/*.css$
+ Allow: /core/*.css?
+@@ -63,3 +64,16 @@ Disallow: /index.php/user/login
+ Disallow: /index.php/user/logout
+ Disallow: /index.php/media/oembed
+ Disallow: /index.php/*/media/oembed
++# Block saml paths links
++Disallow: /sso/login
++Disallow: /saml_login
++Disallow: /saml/login
++Disallow: /saml/acs
++# Block Femtosearch
++User-agent: FemtosearchBot
++Crawl-Delay: 120
++Disallow: /
++# Block SemrushBot
++User-agent: SemrushBot
++Crawl-Delay: 120
++Disallow: /