Implement EXCLUSIONS config value

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python Debugger: Module",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "sweeper",
+      "args": [
+        "sweep",
+        "--workspace",
+        "C:\\temp\\locators.gdb"
+      ],
+      "cwd": "${workspaceFolder}"
+    }
+  ]
+}

config.sample.json

@@ -3,5 +3,8 @@
   "TO_ADDRESSES": ["[email protected]"],
   "CONNECTIONS_FOLDER": "path-to-folder",
   "CHANGE_DETECTION_CONNECTION": "something.sde",
-  "CHANGE_DETECTION_TABLE": "SGID.META.ChangeDetection"
+  "CHANGE_DETECTION_TABLE": "SGID.META.ChangeDetection",
+  "EXCLUSIONS": {
+    "duplicates": []
+  }
 }

readme.md

@@ -116,6 +116,10 @@ A normalized string representing the entire address that was passed into the con
 > - `--change-detect` argument
 > - using user-specific connection files via the `CONNECTIONS_FOLDER` config value
 
+## Exclusions
+
+Tables can be skipped by adding values to the `EXCLUSIONS.<sweeper_key>` config array. These values are matched against table names using [fnmatch](https://docs.python.org/3/library/fnmatch.html#fnmatch.fnmatch). Note that these do not apply when using the `--table-name` argument.
+
 ## Development
 
 1. clone arcgis conda environment

src/sweeper/__main__.py

@@ -34,7 +34,7 @@
 from supervisor.message_handlers import SendGridHandler
 from supervisor.models import MessageDetails, Supervisor
 
-from . import backup, config, report, workspace_info
+from . import backup, config, report, utilities, workspace_info
 from .sweepers.addresses import AddressTest
 from .sweepers.duplicates import DuplicateTest
 from .sweepers.empties import EmptyTest
@@ -129,6 +129,7 @@
 
     log.info(f"running {len(closet)} sweepers. try fix: {try_fix}")
     for tool in closet:
+        log.info(f"running sweeper: {tool.key}")
         if tool.table_name:
             run_tool(tool)
 
@@ -145,6 +146,16 @@
                 if any("SGID." in fc for fc in feature_class_names):
                     feature_class_names = [fc.split("SGID.", 1)[1] for fc in feature_class_names if "SGID." in fc]
 
+            #: apply exclusions
+            if config.has_config():
+                try:
+                    exclusions_config = config.get_config("EXCLUSIONS")
+                except KeyError:
+                    exclusions_config = {}
+
+                exclusions = exclusions_config.get(tool.key, [])
+            feature_class_names = utilities.apply_exclusions(feature_class_names, exclusions)
+
         log.info(f"feature_class_names is: {feature_class_names}")
 
         if using_change_detection and feature_class_names is None:

src/sweeper/config.py

@@ -7,6 +7,7 @@
 try:
     with open("config.json") as f:
         config = json.load(f)
+    log.info("config.json loaded successfully")
 except FileNotFoundError:
     log.debug("A config.json not found in current working directory. This will prevent some features from working.")
 
@@ -20,5 +21,16 @@
         )
 
 
+def has_config():
+    """
+    Check if a config.json file has been loaded into the config module.
+
+    Returns:
+        bool: True if a config.json file has been loaded, False otherwise.
+    """
+
+    return config is not None
+
+
 # set this to the current working directory
 LOG_FILE_PATH = Path(Path.cwd(), "sweeper.log")

src/sweeper/sweepers/addresses.py

@@ -7,12 +7,16 @@
 """
 
 import arcpy
+
+
 from ..address_parser import Address
 
 
 class AddressTest:
     """A class that validates address data"""
 
+    key = "addresses"
+
     def __init__(self, workspace, table_name, field_name):
         self.workspace = workspace
         self.table_name = table_name

src/sweeper/sweepers/duplicates.py

@@ -14,6 +14,8 @@
 class DuplicateTest(SweeperBase):
     """A class that finds and removes duplicate geometries or attributes or both"""
 
+    key = "duplicates"
+
     def __init__(self, workspace, table_name):
         self.workspace = workspace
         self.table_name = table_name

src/sweeper/sweepers/empties.py

@@ -14,6 +14,8 @@
 class EmptyTest(SweeperBase):
     """A class to find empty geometries"""
 
+    key = "empties"
+
     def __init__(self, workspace, table_name):
         self.workspace = workspace
         self.table_name = table_name

src/sweeper/sweepers/invalids.py

@@ -5,6 +5,8 @@
 class InvalidSweeper:
     """A class that identifies invalid geometry objects and returns a report dictionary"""
 
+    key = "invalids"
+
     def __init__(self, workspace, table_name):
         self.report = {}
         self.workspace = workspace

src/sweeper/sweepers/metadata.py

@@ -175,6 +175,8 @@ def get_description_text_only(html):
 class MetadataTest(SweeperBase):
     """A class that validates geodatabase metadata"""
 
+    key = "metadata"
+
     def __init__(self, workspace, table_name):
         self.workspace = workspace
         self.table_name = table_name

src/sweeper/utilities.py

@@ -0,0 +1,27 @@
+import logging
+from fnmatch import fnmatchcase
+
+log = logging.getLogger("sweeper")
+
+
+def apply_exclusions(list, exclusions):
+    """
+    Apply exclusions to a list of items. Matches are checked using the fnmatch module.
+
+    Args:
+        list (list): The list of items to apply exclusions to.
+        exclusions (list): The list of exclusion fnmatch-compatible patterns.
+
+    Returns:
+        list: The updated list after applying exclusions.
+    """
+    for exclusion in exclusions:
+        new_list = []
+        for item in list:
+            if fnmatchcase(item.casefold(), exclusion.casefold()):
+                log.info(f"Excluding {item} based on exclusion {exclusion}")
+            else:
+                new_list.append(item)
+        list = new_list
+
+    return list

tests/test_utilities.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# * coding: utf8 *
+"""
+test_utilities.py
+tests for the utilities module
+"""
+
+from sweeper import utilities
+
+
+class TestApplyExclusions:
+    def test_simple_exclusions(self):
+        exclusions = [
+            "test1",
+            "test2",
+            "test3",
+        ]
+        test_list = [
+            "test1",
+            "test2",
+            "test4",
+            "test5",
+        ]
+        result = utilities.apply_exclusions(test_list, exclusions)
+        assert result == ["test4", "test5"]
+
+    def test_glob_exclusions(self):
+        exclusions = [
+            "test*",
+        ]
+        test_list = ["test1", "test2", "test4", "test5", "hello"]
+        result = utilities.apply_exclusions(test_list, exclusions)
+        assert result == ["hello"]
+
+    def test_no_exclusions(self):
+        exclusions = []
+        test_list = ["test1", "test2", "test4", "test5", "hello"]
+        result = utilities.apply_exclusions(test_list, exclusions)
+        assert result == test_list
+
+    def test_case_insensitive(self):
+        exclusions = [
+            "TEST*",
+        ]
+        test_list = ["test1", "test2", "test4", "test5", "hello"]
+        result = utilities.apply_exclusions(test_list, exclusions)
+        assert result == ["hello"]