Add option to optionally skip downloading the contents of a file

README.md

@@ -139,6 +139,17 @@ plugins:
       validate_rendered_template: True
 ```
 
+### `skip_downloads`
+
+Optionally skip downloading of a remote URLs content via GET request. This can
+considerably reduce the time taken to validate URLs.
+
+```yaml
+plugins:
+  - htmlproofer:
+      skip_downloads: True
+```
+
 ## Compatibility with `attr_list` extension
 
 If you need to manually specify anchors make use of the `attr_list` [extension](https://python-markdown.github.io/extensions/attr_list) in the markdown.

htmlproofer/plugin.py

@@ -65,6 +65,7 @@ class HtmlProoferPlugin(BasePlugin):
         ('raise_error', config_options.Type(bool, default=False)),
         ('raise_error_after_finish', config_options.Type(bool, default=False)),
         ('raise_error_excludes', config_options.Type(dict, default={})),
+        ('skip_downloads', config_options.Type(bool, default=False)),
         ('validate_external_urls', config_options.Type(bool, default=True)),
         ('validate_rendered_template', config_options.Type(bool, default=False)),
         ('ignore_urls', config_options.Type(list, default=[])),
@@ -146,7 +147,13 @@ def get_external_url(self, url, scheme, src_path):
     @lru_cache(maxsize=1000)
     def resolve_web_scheme(self, url: str) -> int:
         try:
-            response = self._session.get(url, timeout=URL_TIMEOUT)
+            response = self._session.get(url, timeout=URL_TIMEOUT, stream=True)
+
+            if self.config['skip_downloads'] is False:
+                # Download the entire contents as to not break previous behaviour.
+                for _ in response.iter_content(chunk_size=1024):
+                    pass
+
             return response.status_code
         except requests.exceptions.Timeout:
             return 504

tests/integration/mkdocs.yml

@@ -16,3 +16,4 @@ plugins:
             'page2.html#BAD_ANCHOR',
             '../../../tests',
           ]
+        skip_downloads: True

tests/unit/test_plugin.py

@@ -78,8 +78,10 @@ def test_on_post_page(
     })
 
     # Always raise a 500 error
-    mock_requests.side_effect = [Mock(spec=Response, status_code=500)]
     link_to_500 = '<a href="https://google.com"><a/>'
+    iter_content = Mock()
+    iter_content.side_effect = link_to_500
+    mock_requests.side_effect = [Mock(spec=Response, status_code=500, iter_content=iter_content)]
 
     plugin.files = empty_files
     page = Mock(