Malicious Site Protection Reference Tests

malicious-site-protection/README.md

@@ -0,0 +1,81 @@
+# Feature Name Tests
+
+Privacy Feature: <https://app.asana.com/0/1208717418466383/1199621914667995/f>
+
+## Goals
+
+This set of tests verifies implementation of malicious site protection. In particular it focuses on verifying that:
+- local entries stored in the `hashPrefix` and `filterSet` datasets result in blocked navigations
+- entries in both the `phishing` and `malware` categories are independently blocked
+- entries that should not be blocked are not: ensuring low chance of false positives
+- unexpected situations are handled gracefully, i.e. invalid regular expressions, duplicate entries, overlapping data for different categories, etc. 
+
+## Structure
+
+Test suite specific fields:
+
+- `featureName` - string - name of the privacy feature as defined in the config
+- `siteURL` - string - the URL of the site we are testing protections for
+- `expectURL` - string - the expected canonicalized URL
+- `expectDomain` - string - the expected canonicalized domain
+- `expectBlock` - bool - true if expected to be blocked, false otherwise
+
+## Pseudo-code implementation
+
+### Block Test
+```
+for $testSet in block_tests.json
+  loadReferenceHashPrefixes('hashPrefixes_reference.json')
+  loadReferenceFilterSet('filterSet_reference.json')
+
+  for $test in $testSet
+    if $test.exceptPlatforms includes 'current-platform'
+        skip
+
+    $blocked = MaliciousSiteDetector.evaluate(
+        url=$test.siteURL,
+    )
+
+    expect($blocked === $test.expectBlock)
+```
+
+### Canonicalization Test
+
+#### URL Canonicalization
+
+```
+  $testSet = load_tests(canonicalization_test.json).get('urlTests')
+  for $test in $testSet
+    if $test.exceptPlatforms includes 'current-platform'
+        skip
+
+    $canonicalURL = URL.canonicalize(
+        url=$test.siteURL,
+    )
+
+    expect($canonicalURL === $test.expectURL)
+```
+
+#### Domain Canonicalization
+
+```
+  $testSet = load_tests(canonicalization_test.json).get('domainTests')
+  for $test in $testSet
+    if $test.exceptPlatforms includes 'current-platform'
+        skip
+
+    $canonicalDomain = Domain.canonicalize(
+        url=$test.siteURL,
+    )
+
+    expect($canonicalDomain === $test.expectDomain)
+```
+
+## Platform exceptions
+
+- Currently, malicious site protection is only available on the browser platforms:
+  - macOS
+  - iOS
+  - Android
+  - Windows
+- The feature doesn't make sense to be enabled on extensions, as they will mostly run on browsers that already provide their own malicious site protections.

malicious-site-protection/block_tests.json

@@ -0,0 +1,140 @@
+{
+    "phishingDetectionTests": {
+        "name": "Should block phishing pages",
+        "desc": "All browser platforms should block pages in the phishing datasets, and not block pages not in the dataset.",
+        "tests": [
+            {
+                "name": "phishing wildcard regex test",
+                "siteURL": "https://malicious.example.com/all/paths/should/match/phishing",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "phishing wildcard regex test 2",
+                "siteURL": "https://malicious.example.com/all/other/paths/should/match/phishing/too",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "multiple path phishing match",
+                "siteURL": "https://phishing-site.com/download/transaction/safety/check",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "multiple subdomain phishing match",
+                "siteURL": "https://stealing.your.data.co.uk/loading/login/connect/alert/download/information/your/transaction",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "phishing wildcard and subdomain regex test",
+                "siteURL": "https://impersonating-gov-uk.co.uk/login/protect/account/account/data/account/urgent/service/install/support",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "exclusively not in phishing dataset",
+                "siteURL": "https://not-in-dataset.com/anything.html",
+                "expectBlock": false,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "hash match in malware dataset, but not regex",
+                "siteURL": "https://impersonating-gov-uk.co.uk/path/not/in/dataset",
+                "expectBlock": false,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            }
+        ]
+    },
+    "malwareDetectionTests": {
+        "name": "Should block malware pages",
+        "desc": "All browser platforms should block pages in the malware datasets, and not block pages not in the dataset.",
+        "tests": [
+            {
+                "name": "malware wildcard regex test",
+                "siteURL": "https://drive-by-attack.com/anything.html",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "malware wildcard regex test 2",
+                "siteURL": "https://drive-by-attack.com/anything/else/should/match/too",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "multiple path malware match",
+                "siteURL": "https://malicious.example.com/login/safe/urgent/notification",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "multiple subdomain malware match",
+                "siteURL": "https://stealing.your.data.co.uk/loading/login/connect/alert/download/information/your/transaction",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "malware wildcard and subdomain regex test",
+                "siteURL": "https://danger.malware.distribution.com/support/confirm/safety/login/notification/password/password/information/information",
+                "expectBlock": true,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "exclusively not in malware dataset",
+                "siteURL": "https://not-in-dataset.com/anything.html",
+                "expectBlock": false,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "hash match in malware dataset, but not regex",
+                "siteURL": "https://danger.malware.distribution.com/path/not/in/dataset",
+                "expectBlock": false,
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            }
+        ]
+    }
+}

malicious-site-protection/canonicalization_tests.json

@@ -0,0 +1,152 @@
+{
+    "urlTests": {
+        "name": "Should normalize URLs",
+        "desc": "All browser platforms should sufficiently canonicalize URLs consitently.",
+        "tests": [
+            {
+                "name": "fragment in URL",
+                "siteURL": "https://broken.third-party.site/path/to/resource#fragment",
+                "expectURL": "https://broken.third-party.site/path/to/resource",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "double slashes in path",
+                "siteURL": "https://broken.third-party.site//path//to//resource",
+                "expectURL": "https://broken.third-party.site/path/to/resource",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "trailing spaces in path",
+                "siteURL": "https://broken.third-party.site/path/to/resource%20with%20spaces%20%20%20%20",
+                "expectURL": "https://broken.third-party.site/path/to/resource%20with%20spaces%20%20%20%20",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "encoded fragment",
+                "siteURL": "https://broken.third-party.site/path/to/resource%23encodedfragment",
+                "expectURL": "https://broken.third-party.site/path/to/resource%23encodedfragment",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "dot segments in path",
+                "siteURL": "https://broken.third-party.site/path/./to/./resource",
+                "expectURL": "https://broken.third-party.site/path/to/resource",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "spaces in path without trailing spaces",
+                "siteURL": "https://broken.third-party.site/path/to/resource%20with%20spaces",
+                "expectURL": "https://broken.third-party.site/path/to/resource%20with%20spaces",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "encoded dot segments in path",
+                "siteURL": "https://broken.third-party.site/path/to/%2E%2E/%2E%2E/resource",
+                "expectURL": "https://broken.third-party.site/resource",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            },
+            {
+                "name": "multiple encoded slashes in path",
+                "siteURL": "https://broken.third-party.site/path/to/%2F%2F%2F%2F%2F%2F%2F%2F%2F",
+                "expectURL": "https://broken.third-party.site/path/to",
+                "exceptPlatforms": [
+                    "web-extension",
+                    "safari-extension"
+                ]
+            }
+        ]
+    },
+    "domainTests": {
+        "name": "Domain Canonicalization",
+        "desc": "Domain Canonicalization - tests provide input URI, with expected output",
+        "tests": [
+            {
+                "name": "Simple domain - extract hostname portion from the URL",
+                "siteURL": "http://www.somesite.com",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Simple domain with a port",
+                "siteURL": "http://www.somesite.com:8000/",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Simple domain with a username",
+                "siteURL": "http://user:[email protected]",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Simple domain with a fragment",
+                "siteURL": "http://www.somesite.com#fragment",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Simple domain with a query string",
+                "siteURL": "http://www.somesite.com?some=value",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Decode any %XX escapes present in the hostname",
+                "siteURL": "http://www.%73ome%73ite.com",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Discard any leading and/or trailing full-stops",
+                "siteURL": "http://.www.somesite.com.",
+                "expectDomain": "www.somesite.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "Replace sequences of two or more full-stops with a single full-stop.",
+                "siteURL": "http://www..example...com",
+                "expectDomain": "www.example.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "If the hostname is a numeric IPv4 address, reduce it to the canonical dotted quad form.",
+                "siteURL": "http://192.168.000.001:8000/",
+                "expectDomain": "192.168.0.1",
+                "exceptPlatforms": []
+            },                        
+            {
+                "name": "If there are more than six components in the resulting hostname, discard all but the rightmost six components.",
+                "siteURL": "http://a.b.c.d.e.f.g.h.i.j.example.com",
+                "expectDomain": "g.h.i.j.example.com",
+                "exceptPlatforms": []
+            },
+            {
+                "name": "If the hostname is a numeric IPv4 address, reduce it to the canonical dotted quad form.",
+                "siteURL": "http://192.168.001.001:8080/",
+                "expectDomain": "192.168.1.1",
+                "exceptPlatforms": []
+            }            
+        ]
+    }
+}

malicious-site-protection/reference_malware_filterSet.json

@@ -0,0 +1 @@
+[{"regex": "(?i)^https?\\:\\/\\/malicious.example.com(?:\\:(?:80|443))?\\/login\\/safe\\/urgent\\/notification$", "hash": "e2a8607b1dfd51fc72246aa9a31246e246482441ec26e5838b28c253fe7635c5"}, {"regex": "(?i)^https?\\:\\/\\/\\w+\\.\\w+(?:\\:(?:80|443))?\\/malicious\\.html$", "hash": "a12f78b1a902ccca3491ee69f07d0e68f656c08f49ac41f560c0a25b6f9b2216"}, {"regex": "(?i)^https?\\:\\/\\/malicious-javascript.com(?:\\:(?:80|443))?\\/alert\\/account\\/verify\\/connect\\/file$", "hash": "89ffbd4bab9142ca2d05fed27228bd0b73f02b9980ba8beb023409a0400f7d45"}, {"regex": ".", "hash": "bef8479e07d45839982dab1279cd8eb2181a4a4d62be24f1d2d85baf39dae526"}, {"regex": "(?i)^https?\\:\\/\\/danger.malware.distribution.com(?:\\:(?:80|443))?\\/support\\/confirm\\/safety\\/login\\/notification\\/password\\/password\\/information\\/information$", "hash": "c0af9f76dce48df2408edbcc96860e24848bd918a3c8d3656d6dc361a642397b"}]

malicious-site-protection/reference_malware_hashPrefixes.json

@@ -0,0 +1 @@
+["e2a8607b", "a12f78b1", "89ffbd4b", "bef8479e", "c0af9f76"]

malicious-site-protection/reference_phishing_filterSet.json

@@ -0,0 +1 @@
+[{"regex": ".", "hash": "e2a8607b1dfd51fc72246aa9a31246e246482441ec26e5838b28c253fe7635c5"}, {"regex": "(?i)^https?\\:\\/\\/stealing.your.data.co.uk(?:\\:(?:80|443))?\\/loading\\/login\\/connect\\/alert\\/download\\/information\\/your\\/transaction$", "hash": "41be5794b4ce63c0c529f6d87eb0cdd1887b65fdfaef99cf18d90c5ef8f9d768"}, {"regex": "(?i)^https?\\:\\/\\/phishing-site.com(?:\\:(?:80|443))?\\/download\\/transaction\\/safety\\/check$", "hash": "1be002268941e7c2e63394701793e9526f5d0dd64393ea9712317f2e70348c08"}, {"regex": "(?i)^https?\\:\\/\\/impersonating-gov-uk.co.uk(?:\\:(?:80|443))?\\/login\\/protect\\/account\\/account\\/data\\/account\\/urgent\\/service\\/install\\/support$", "hash": "aab7d9fcb9a26e65b4feec8216a7bdb179f884ab7f14096c95ec2cd96f3999bf"}, {"regex": ".", "hash": "5dd01a6a826584303a07cf3eebe027b0827275943488930c95c4718ffe34a56d"}]

malicious-site-protection/reference_phishing_hashPrefixes.json

@@ -0,0 +1 @@
+["e2a8607b", "41be5794", "1be00226", "aab7d9fc", "5dd01a6a"]