For #1837 - Bring Shavar Lists up to date (#1855) (#1915)

* For #1837 - Bring Shavar Lists up to date * remove commented line * use same has as firefox-ios * fix XCUITest * point to commit on v89 branch Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> (cherry picked from commit 0e0b3d4) Co-authored-by: isabelrios <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
mozilla-mobile · Jun 10, 2021 · 2e3c747 · 2e3c747
1 parent 37c1202
commit 2e3c747
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 40 deletions.
diff --git a/checkout.sh b/checkout.sh
@@ -7,6 +7,6 @@
 git clone https://github.com/mozilla-services/shavar-prod-lists.git || exit 1
 
 # This revision is taken from the original Cartfile.resolved
-(cd shavar-prod-lists && git checkout -q c938da47c4880a48ac40d535caff74dac1d4d77b)
+(cd shavar-prod-lists && git checkout -q 3910527004252af3aa9dd701566a2cb3b78e5c3a)
 
 (cd content-blocker-lib-ios/ContentBlockerGen && swift run)
diff --git a/content-blocker-lib-ios/ContentBlockerGen/Sources/ContentBlockerGen/main.swift b/content-blocker-lib-ios/ContentBlockerGen/Sources/ContentBlockerGen/main.swift
@@ -12,7 +12,6 @@ let execIsFromCorrectDir = fm.fileExists(atPath: fm.currentDirectoryPath + "/Pac
 let rootdir = execIsFromCorrectDir ? fm.currentDirectoryPath : fallbackPath
 let blacklist = "\(rootdir)/../../shavar-prod-lists/disconnect-blacklist.json"
 let entityList = "\(rootdir)/../../shavar-prod-lists/disconnect-entitylist.json"
-let googleMappingList = "\(rootdir)/../../shavar-prod-lists/google_mapping.json"
 let fingerprintingList = "\(rootdir)/../../shavar-prod-lists/normalized-lists/base-fingerprinting-track.json"
 
 func jsonFrom(filename: String) -> [String: Any] {
@@ -21,7 +20,7 @@ func jsonFrom(filename: String) -> [String: Any] {
     return try! JSONSerialization.jsonObject(with: data, options: []) as! [String: Any]
 }
 
-let gen = ContentBlockerGenLib(entityListJson: jsonFrom(filename: entityList), googleMappingJson: jsonFrom(filename: googleMappingList))
+let gen = ContentBlockerGenLib(entityListJson: jsonFrom(filename: entityList))
 
 let outputDir = URL(fileURLWithPath: "\(rootdir)/../../Lists")
 

diff --git a/...blocker-lib-ios/ContentBlockerGen/Sources/ContentBlockerGenLib/ContentBlockerGenLib.swift b/...blocker-lib-ios/ContentBlockerGen/Sources/ContentBlockerGenLib/ContentBlockerGenLib.swift
@@ -21,20 +21,19 @@ public enum CategoryTitle: String, CaseIterable {
 
 public class ContentBlockerGenLib {
     var companyToRelatedDomains = [String: [String]]()
-    let googleMappingJson: [String: Any]
 
-    public init(entityListJson: [String: Any], googleMappingJson: [String: Any]) {
-        self.googleMappingJson = googleMappingJson["categories"]! as! [String: Any]
+    public init(entityListJson: [String: Any]) {
         parseEntityList(json: entityListJson)
     }
 
     func parseEntityList(json: [String: Any]) {
-        json.forEach {
-            let company = $0.key
-            let related = ($0.value as! [String: [String]])["properties"]!
-            companyToRelatedDomains[company] = related
+            let entities = json["entities"]! as! [String: Any]
+            entities.forEach {
+                let company = $0.key
+                let related = ($0.value as! [String: [String]])["properties"]!
+                companyToRelatedDomains[company] = related
+            }
         }
-    }
 
     func buildUnlessDomain(_ domains: [String]) -> String {
         guard domains.count > 0 else { return "" }
@@ -82,26 +81,6 @@ public class ContentBlockerGenLib {
         category.forEach {
             result += handleCategoryItem($0, action: action)
         }
-
-        // Special handling for Social, pull in lists from Disconnect category
-        if categoryTitle == .Social {
-            let category = categories[CategoryTitle.Disconnect.rawValue] as! [Any]
-            category.forEach {
-                let item = $0 as! [String: Any]
-                let companyName = item.first!.key
-                if ["Facebook", "Twitter"].contains(companyName) {
-                    result += handleCategoryItem($0, action: action)
-                }
-            }
-        }
-
-        // Google properties exist in a special list that gets appended per-category
-        if let cat = googleMappingJson[categoryTitle.rawValue] as? [Any] {
-            cat.forEach {
-                result += handleCategoryItem($0, action: action)
-            }
-        }
-
         return result
     }
 

diff --git a/...ocker-lib-ios/ContentBlockerGen/Tests/ContentBlockerGenTests/ContentBlockerGenTests.swift b/...ocker-lib-ios/ContentBlockerGen/Tests/ContentBlockerGenTests/ContentBlockerGenTests.swift
@@ -24,18 +24,10 @@ let entitylist = """
 }
 """
 
-let googlemapping = """
-{ "categories": {
-    "Analytics": [{"Google": {"http://www.google.com/": ["google-analytics.com", "postrank.com"]}}]
-}}
-"""
-
 final class ContentBlockerGenTests: XCTestCase {
     func testParsing() throws {
         let entityJson = try! JSONSerialization.jsonObject(with: entitylist.data(using: .utf8)!, options: []) as! [String: Any]
-        let googleJson = try! JSONSerialization.jsonObject(with: googlemapping.data(using: .utf8)!, options: []) as! [String: Any]
-
-        let contentBlocker = ContentBlockerGenLib(entityListJson: entityJson, googleMappingJson: googleJson)
+        let contentBlocker = ContentBlockerGenLib(entityListJson: entityJson)
 
         let json = try! JSONSerialization.jsonObject(with: blacklist.data(using: .utf8)!, options: []) as! [String: Any]
         let categories = json["categories"]! as! [String: Any]