DEV 2445 gitlab ci (#285)

* add gitlab-ci.yml * pre-commit run -a
NCI-GDC · Jan 31, 2024 · 254c55b · 254c55b
1 parent b62fd75
commit 254c55b
Show file tree

Hide file tree

Showing 27 changed files with 273 additions and 120 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,29 @@
+---
+include:
+  - project: nci-gdc/gitlab-templates
+    ref: 0.6.2
+    file:
+      - templates/global/full.yaml
+      - templates/python/full.yaml
+      - templates/common/python.yaml
+
+tox:
+  parallel:
+    matrix:
+      - BUILD_PY_VERSION:
+          - python3.6
+          - python3.7
+  script:
+    - tox -r -e py
+
+
+release:
+  before_script:
+    # unshallow the git repo to resolve version with setuptools_scm.
+    - git fetch --unshallow || true
+    - |
+      if [ ${CI_COMMIT_TAG+x} ]; then
+        export TWINE_REPOSITORY_URL=https://nexus.osdc.io/repository/pypi-releases/
+      fi
+  variables:
+    RELEASE_PY_VERSION: python3.6
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,15 +1,17 @@
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.5.0
-  hooks:
-    - id: end-of-file-fixer
-    - id: no-commit-to-branch
-      args: [--branch, develop, --branch, master, --pattern, release/.*]
-- repo: https://github.com/psf/black
-  rev: 19.10b0
-  hooks:
-    - id: black
--   repo: [email protected]:Yelp/detect-secrets
-    rev: v0.13.0
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
     hooks:
-    -   id: detect-secrets
+      - id: end-of-file-fixer
+      - id: no-commit-to-branch
+        args: [--branch, develop, --branch, master, --pattern, release/.*]
+  - repo: https://github.com/psf/black
+    rev: 24.1.1
+    hooks:
+      - id: black
+  - repo: [email protected]:Yelp/detect-secrets
+    rev: v1.4.0
+    hooks:
+      - id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -1,33 +1,29 @@
 {
-  "exclude": {
-    "files": null,
-    "lines": null
-  },
-  "generated_at": "2020-05-07T19:50:44Z",
+  "version": "1.4.0",
   "plugins_used": [
     {
-      "name": "AWSKeyDetector"
+      "name": "ArtifactoryDetector"
     },
     {
-      "name": "ArtifactoryDetector"
+      "name": "AWSKeyDetector"
     },
     {
-      "base64_limit": 4.5,
-      "name": "Base64HighEntropyString"
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
     },
     {
       "name": "BasicAuthDetector"
     },
     {
-      "hex_limit": 3,
-      "name": "HexHighEntropyString"
+      "name": "HexHighEntropyString",
+      "limit": 3
     },
     {
       "name": "JwtTokenDetector"
     },
     {
-      "keyword_exclude": null,
-      "name": "KeywordDetector"
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
     },
     {
       "name": "MailchimpDetector"
@@ -45,10 +41,46 @@
       "name": "StripeDetector"
     }
   ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
   "results": {},
-  "version": "0.13.0",
-  "word_list": {
-    "file": null,
-    "hash": null
-  }
+  "generated_at": "2024-01-30T20:56:48Z"
 }
diff --git a/bin/gdc-client b/bin/gdc-client
@@ -46,10 +46,14 @@ def log_version_header(log):
 
 if __name__ == "__main__":
 
-    parser = GDCClientArgumentParser(description=DESCRIPTION,)
+    parser = GDCClientArgumentParser(
+        description=DESCRIPTION,
+    )
 
     parser.add_argument(
-        "--version", action="version", version=version.__version__,
+        "--version",
+        action="version",
+        version=version.__version__,
     )
 
     conf_parser = argparse.ArgumentParser(add_help=False)
@@ -62,7 +66,9 @@ if __name__ == "__main__":
     # Generate template parser for use by all sub-commands. This will
     # contain any shared, top-level arguments and flags that will be
     # needed by most, if not all, subcommands.
-    template = GDCClientArgumentParser(add_help=False,)
+    template = GDCClientArgumentParser(
+        add_help=False,
+    )
 
     logger.parser.config(template)
     auth.parser.config(template)
@@ -76,17 +82,22 @@ if __name__ == "__main__":
     subparsers.required = True
 
     download_subparser = subparsers.add_parser(
-        "download", parents=[template], help="download data from the GDC",
+        "download",
+        parents=[template],
+        help="download data from the GDC",
     )
     download.parser.config(download_subparser, config_loader.to_dict("download"))
 
     upload_subparser = subparsers.add_parser(
-        "upload", parents=[template], help="upload data to the GDC",
+        "upload",
+        parents=[template],
+        help="upload data to the GDC",
     )
     upload.parser.config(upload_subparser, config_loader.to_dict("upload"))
 
     settings_subparser = subparsers.add_parser(
-        "settings", help="display default settings",
+        "settings",
+        help="display default settings",
     )
     settings.parser.config(settings_subparser, conf_parser_args.config)
 

diff --git a/gdc_client/auth/auth.py b/gdc_client/auth/auth.py
@@ -2,8 +2,7 @@
 
 
 class GDCTokenAuth(requests.auth.AuthBase):
-    """ GDC Token Authentication
-    """
+    """GDC Token Authentication"""
 
     def __init__(self, token):
         self.token = token

diff --git a/gdc_client/auth/parser.py b/gdc_client/auth/parser.py
@@ -27,8 +27,7 @@
 
 
 def read_token_file(path):
-    """ Safely open, read and close a token file.
-    """
+    """Safely open, read and close a token file."""
 
     # there's a circular dependency on setting up logging to process this arg
     # but also needing the logs to be set up before you can process args
@@ -54,7 +53,9 @@ def read_token_file(path):
     )
 
     if invalid_permissions:
-        permissions_msg = PERMISSIONS_MSG.format(token_file=abspath,)
+        permissions_msg = PERMISSIONS_MSG.format(
+            token_file=abspath,
+        )
         log.warning(permissions_msg)
         # FIXME convert to error after investigation on windows
         # raise argparse.ArgumentTypeError(permissions_msg)
@@ -69,9 +70,11 @@ def read_token_file(path):
 
 
 def config(parser):
-    """ Configure argparse parser for GDC auth token parsing.
-    """
+    """Configure argparse parser for GDC auth token parsing."""
 
     parser.add_argument(
-        "-t", "--token-file", type=read_token_file, help="GDC API auth token file",
+        "-t",
+        "--token-file",
+        type=read_token_file,
+        help="GDC API auth token file",
     )
diff --git a/gdc_client/client/client.py b/gdc_client/client/client.py
@@ -12,8 +12,7 @@
 
 
 class GDCClient(object):
-    """ GDC API Requests Client
-    """
+    """GDC API Requests Client"""
 
     def __init__(self, host=GDC_API_HOST, port=GDC_API_PORT, token=None):
         self.host = host
@@ -35,12 +34,13 @@ def __init__(self, host=GDC_API_HOST, port=GDC_API_PORT, token=None):
 
     @contextmanager
     def request(self, verb, path, **kwargs):
-        """ Make a request to the GDC API.
-        """
+        """Make a request to the GDC API."""
         res = self.session.request(
             verb,
             "https://{host}:{port}{path}".format(
-                host=self.host, port=self.port, path=path,
+                host=self.host,
+                port=self.port,
+                path=path,
             ),
             auth=auth.GDCTokenAuth(self.token),
             **kwargs

diff --git a/gdc_client/client/parser.py b/gdc_client/client/parser.py
@@ -5,8 +5,7 @@
 
 
 def config(parser):
-    """ Configure an argparse parser for use with the GDC Client.
-    """
+    """Configure an argparse parser for use with the GDC Client."""
     parser.add_argument(
         "-H",
         "--host",

diff --git a/gdc_client/download/client.py b/gdc_client/download/client.py
@@ -20,10 +20,10 @@
 
 def fix_url(url):
     # type: (str) -> str
-    """ Fix a url to be used in the rest of the program
+    """Fix a url to be used in the rest of the program
 
-        example:
-            api.gdc.cancer.gov -> https://api.gdc.cancer.gov/
+    example:
+        api.gdc.cancer.gov -> https://api.gdc.cancer.gov/
     """
     if not url.endswith("/"):
         url = "{0}/".format(url)
@@ -47,7 +47,7 @@ def __init__(
         *args,
         **kwargs
     ):
-        """ GDC parcel client that overrides parallel download
+        """GDC parcel client that overrides parallel download
         Args:
             uri (str):
             download_related_files (bool):
@@ -135,7 +135,7 @@ def download_annotations(self, file_id):
 
     def _untar_file(self, tarfile_name):
         # type: (str) -> list[str]
-        """ untar the file and return all the file names inside the tarfile """
+        """untar the file and return all the file names inside the tarfile"""
 
         t = tarfile.open(tarfile_name)
         members = [m for m in t.getmembers() if m.name != "MANIFEST.txt"]
@@ -149,7 +149,7 @@ def _untar_file(self, tarfile_name):
 
     def _md5_members(self, members):
         # type: (list[str]) -> list[str]
-        """ Calculate md5 hash and compare them with values given by the API """
+        """Calculate md5 hash and compare them with values given by the API"""
 
         errors = []
         for m in members:
@@ -175,7 +175,7 @@ def _md5_members(self, members):
 
     def _post(self, path, headers=None, json=None, stream=True):
         # type: (str, dict[str,str], dict[str,object], bool) -> requests.models.Response
-        """ custom post request that will query both active and legacy api
+        """custom post request that will query both active and legacy api
 
         return a python requests object to be handled by the method calling self._post
         """
@@ -210,7 +210,7 @@ def _post(self, path, headers=None, json=None, stream=True):
 
     def _download_tarfile(self, small_files):
         # type: (list[str]) -> tuple[str, object]
-        """ Make the request to the API for the tarfile downloads """
+        """Make the request to the API for the tarfile downloads"""
 
         errors = []
         headers = {
@@ -249,7 +249,8 @@ def _download_tarfile(self, small_files):
 
         if content_filename:
             tarfile_name = os.path.join(
-                self.base_directory, content_filename.split("=")[1],
+                self.base_directory,
+                content_filename.split("=")[1],
             )
         else:
             tarfile_name = time.strftime("gdc-client-%Y%m%d-%H%M%S.tar")
@@ -264,7 +265,7 @@ def _download_tarfile(self, small_files):
 
     def download_small_groups(self, smalls):
         # type: (list[str]) -> tuple[list[str], int]
-        """ Download small groups
+        """Download small groups
 
         Smalls are predetermined groupings of smaller file size files.
         They are grouped to reduce the number of open connections per download.