Source Github: decrease give up rate (airbytehq#24457)

* added validation and transformation for repositories config * added changelog, updated regex and tests * updated regex, fixed test_check_connection_repos_only * auto-bump connector version --------- Co-authored-by: Octavia Squidington III <[email protected]>
shipyardapp · Mar 30, 2023 · 836e4f4 · 836e4f4
1 parent 0b156a8
commit 836e4f4
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 9 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -705,7 +705,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.4.6
+  dockerImageTag: 0.4.7
   documentationUrl: https://docs.airbyte.com/integrations/sources/github
   icon: github.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4978,7 +4978,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.4.6"
+- dockerImage: "airbyte/source-github:0.4.7"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.4.6
+LABEL io.airbyte.version=0.4.7
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py
@@ -2,7 +2,8 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, Dict, List, Mapping, Tuple
+import re
+from typing import Any, Dict, List, Mapping, Set, Tuple
 
 from airbyte_cdk import AirbyteLogger
 from airbyte_cdk.models import FailureType, SyncMode
@@ -57,6 +58,40 @@
 
 
 class SourceGithub(AbstractSource):
+    @staticmethod
+    def _is_repositories_config_valid(config_repositories: Set[str]) -> bool:
+        """
+        _is_repositories_config_valid validates that each repo config matches regex to highlight problem in provided config.
+        Valid examples: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte
+        Args:
+            config_repositories: set of provided repositories
+        Returns:
+            True if config valid, False if it's not
+        """
+        pattern = re.compile(r"^(?:[\w-]+/)+(?:\*|[\w-]+)$")
+
+        for repo in config_repositories:
+            if not pattern.match(repo):
+                return False
+        return True
+
+    @staticmethod
+    def _get_and_prepare_repositories_config(config: Mapping[str, Any]) -> Set[str]:
+        """
+        _get_and_prepare_repositories_config gets set of repositories names from config and removes simple errors that user could provide
+        Args:
+            config: Dict representing connector's config
+        Returns:
+            set of provided repositories
+        """
+        config_repositories = set(filter(None, config["repository"].split(" ")))
+        # removing spaces
+        config_repositories = {repo.strip() for repo in config_repositories}
+        # removing redundant / in the end
+        config_repositories = {repo[:-1] if repo.endswith("/") else repo for repo in config_repositories}
+
+        return config_repositories
+
     @staticmethod
     def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator) -> Tuple[List[str], List[str]]:
         """
@@ -65,7 +100,13 @@ def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleToke
             config (dict): Dict representing connector's config
             authenticator(MultipleTokenAuthenticator): authenticator object
         """
-        config_repositories = set(filter(None, config["repository"].split(" ")))
+        config_repositories = SourceGithub._get_and_prepare_repositories_config(config)
+        if not SourceGithub._is_repositories_config_valid(config_repositories):
+            raise Exception(
+                f"You provided invalid format of repositories config: {' ' .join(config_repositories)}."
+                f" Valid examples: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte"
+            )
+
         if not config_repositories:
             raise Exception("Field `repository` required to be provided for connect to Github API")
 
@@ -170,7 +211,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
             authenticator = self._get_authenticator(config)
             _, repositories = self._get_org_repositories(config=config, authenticator=authenticator)
             if not repositories:
-                return False, "no valid repositories found"
+                return False, "Invalid repositories. Valid examples: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte"
             return True, None
 
         except Exception as e:

diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py
@@ -19,9 +19,9 @@ def check_source(repo_line: str) -> AirbyteConnectionStatus:
 
 @responses.activate
 def test_check_connection_repos_only():
-    responses.add("GET", "https://api.github.com/repos/airbyte", json={"full_name": "airbyte"})
+    responses.add("GET", "https://api.github.com/repos/airbytehq/airbyte", json={"full_name": "airbytehq/airbyte"})
 
-    status = check_source("airbyte airbyte airbyte")
+    status = check_source("airbytehq/airbyte airbytehq/airbyte airbytehq/airbyte")
     assert not status.message
     assert status.status == Status.SUCCEEDED
     # Only one request since 3 repos have same name
@@ -136,3 +136,31 @@ def test_organization_or_repo_available():
         config = {"access_token": "test_token", "repository": ""}
         source.streams(config=config)
     assert exc_info.value.args[0] == "No streams available. Please check permissions"
+
+
+@pytest.mark.parametrize(
+    ("repos_config", "expected"),
+    [
+        (("airbytehq/airbyte/", "airbytehq/", "airbytehq", "airbyte hq", "airbytehq/*/", "adc/ff*f", "akj*/jahsd"), False),
+        (("airbytehq/airbyte", ), True),
+        (("airbytehq/airbyte-test", "airbytehq/airbyte_test", "airbytehq/airbyte-test/another-repo"), True),
+        (("air232bytehq/air32byte", "airbyte_hq/another-repo", "airbytehq/*", "airbytehq/airbyte"), True),
+    ],
+)
+def test_config_validation(repos_config, expected):
+    actual = SourceGithub._is_repositories_config_valid(repos_config)
+    assert actual == expected
+
+
+@pytest.mark.parametrize(
+    ("config", "expected"),
+    [
+        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test"}, {"airbytehq/airbyte-test", }),
+        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test "}, {"airbytehq/airbyte-test", }),
+        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test/"}, {"airbytehq/airbyte-test", }),
+        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test/  airbytehq/airbyte"}, {"airbytehq/airbyte", "airbytehq/airbyte-test"}),
+    ],
+)
+def tests_get_and_prepare_repositories_config(config, expected):
+    actual = SourceGithub._get_and_prepare_repositories_config(config)
+    assert actual == expected
diff --git a/connectors.md b/connectors.md
@@ -84,7 +84,7 @@
 | **GNews** | <img alt="GNews icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/gnews.svg" height="30" height="30"/> | Source | airbyte/source-gnews:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/sources/gnews) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-gnews) | <small>`ce38aec4-5a77-439a-be29-9ca44fd4e811`</small> |
 | **Genesys** | <img alt="Genesys icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/genesys.svg" height="30" height="30"/> | Source | airbyte/source-genesys:0.1.0 | unknown | [link](https://docs.airbyte.com/integrations/sources/genesys) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-genesys) | <small>`5ea4459a-8f1a-452a-830f-a65c38cc438d`</small> |
 | **GetLago** | <img alt="GetLago icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/getlago.svg" height="30" height="30"/> | Source | airbyte/source-getlago:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/getlago) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-getlago) | <small>`e1a3866b-d3b2-43b6-b6d7-8c1ee4d7f53f`</small> |
-| **GitHub** | <img alt="GitHub icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/github.svg" height="30" height="30"/> | Source | airbyte/source-github:0.4.6 | generally_available | [link](https://docs.airbyte.com/integrations/sources/github) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-github) | <small>`ef69ef6e-aa7f-4af1-a01d-ef775033524e`</small> |
+| **GitHub** | <img alt="GitHub icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/github.svg" height="30" height="30"/> | Source | airbyte/source-github:0.4.7 | generally_available | [link](https://docs.airbyte.com/integrations/sources/github) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-github) | <small>`ef69ef6e-aa7f-4af1-a01d-ef775033524e`</small> |
 | **Gitlab** | <img alt="Gitlab icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/gitlab.svg" height="30" height="30"/> | Source | airbyte/source-gitlab:1.0.3 | beta | [link](https://docs.airbyte.com/integrations/sources/gitlab) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-gitlab) | <small>`5e6175e5-68e1-4c17-bff9-56103bbb0d80`</small> |
 | **Glassfrog** | <img alt="Glassfrog icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/glassfrog.svg" height="30" height="30"/> | Source | airbyte/source-glassfrog:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/glassfrog) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-glassfrog) | <small>`cf8ff320-6272-4faa-89e6-4402dc17e5d5`</small> |
 | **GoCardless** | <img alt="GoCardless icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/gocardless.svg" height="30" height="30"/> | Source | airbyte/source-gocardless:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/gocardless) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-gocardless) | <small>`ba15ac82-5c6a-4fb2-bf24-925c23a1180c`</small> |

diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md
@@ -163,6 +163,7 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.4.7   | 2023-03-24 | [24457](https://github.com/airbytehq/airbyte/pull/24457)                                                          | Add validation and transformation for repositories config                                                                                                           |
 | 0.4.6   | 2023-03-24 | [24398](https://github.com/airbytehq/airbyte/pull/24398)                                                          | Fix caching for `get_starting_point` in stream "Commits"                                                                                                            |
 | 0.4.5   | 2023-03-23 | [24417](https://github.com/airbytehq/airbyte/pull/24417)                                                          | Add pattern_descriptors to fields with an expected format                                                                                                           |
 | 0.4.4   | 2023-03-17 | [24255](https://github.com/airbytehq/airbyte/pull/24255)                                                          | Add field groups and titles to improve display of connector setup form                                                                                              |