Merge pull request #18 from lowes/development

Release 1.1.2

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main, development ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main, development ]
+  schedule:
+    - cron: '42 3 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/python-package.yml

@@ -0,0 +1,46 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ main, development ]
+  pull_request:
+    branches: [ main, development ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.8", "3.9"]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        # Kerberos must be installed for kerberos support
+        sudo apt-get install -f libkrb5-dev
+        # Minio support
+        wget https://dl.min.io/server/minio/release/linux-amd64/minio
+        chmod +x minio
+        sudo mv minio /usr/bin/minio
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest flit
+        flit install
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

.github/workflows/python-publish.yml

@@ -0,0 +1,36 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/uploadandscan.yml

@@ -4,7 +4,13 @@
 name: Vericode Scan
 
 on:
-  workflow_dispatch
+  push:
+    branches: [ main, development ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main, development ]
+  schedule:
+    - cron: '42 3 * * 5'
 
 jobs:
   build:

README.md

@@ -105,7 +105,7 @@ credentials:
   # In Yaml the - indiciates this is a list element.
   # Any additional stanzas will start with a similar -
   - realm:
-  	  # LVFS uses realms to determine which set of credentials to use for each URL.
+      # LVFS uses realms to determine which set of credentials to use for each URL.
       # Every credential stanza gets one realm.
       # LVFS will use the first stanza that matches the URL.
       # In this case we match anything using HDFSOverSSH
@@ -134,7 +134,7 @@ processing within the cluster. In that case, configuration is a lot easier.
 ```yaml
 credentials:
   - realm:
-  	  # For historic reasons, all HDFS connections use the HDFSOverSSH connection class
+      # For historic reasons, all HDFS connections use the HDFSOverSSH connection class
       # because SSH will be disabled when you don't configure it here.
       classname: HDFSOverSSH
     # This is the Hadoop username; there is no jumpbox
@@ -207,11 +207,11 @@ At any rate, this is not been tested with LVFS.
 For later reference, these are the possible modes for HDFSOverSSH:
 
 ssh_jump_host | ssh_username | username | password | webhdfs_root | use case
---------------|--------------|----------|----------|--------------------------------------
-*             | *            | *        | *        | None         | Invalid, unconfigured
-not None      | None         | *        | *        | *            | Invalid
-None          | not None     | *        | *        | *            | Invalid
-*             | *            | None     | not None | *            | Invalid
+------------- | ------------ | -------- | -------- | ------------ | --------
+(any)         | (any)        | (any)    | (any)    | None         | Invalid, unconfigured
+not None      | None         | (any)    | (any)    | (any)        | Invalid
+None          | not None     | (any)    | (any)    | (any)        | Invalid
+(any)         | (any)        | None     | not None | (any)        | Invalid
 not None      | not None     | not None | None     | not None     | HDP2 with SSH
 None          | None         | not None | None     | not None     | HDP2 without SSH
 not None      | not None     | not None | not None | not None     | HDP3+AD with SSH

lvfs/__init__.py

@@ -11,7 +11,7 @@
 * Artifactory, for model and code reading and writing
 """
 
-__version__ = "1.1.1"
+__version__ = "1.1.2"
 __all__ = ["URL"]
 import copyreg
 import urllib

lvfs/credentials.py

@@ -26,7 +26,7 @@ def init_register(cls, search_paths: List[Path] = None):
             location: List of Paths to try to read the configuration from (must be local)
 
             If location is not specified, the following locations will be searched:
-            * `./lvfs.conf`
+            * `./lvfs.yml`
             * `~/.config/lvfs.yml`
             * `/etc/creds/lvfs.yml`
             * `/etc/secret/lvfs.yml`

lvfs/minio.py

@@ -58,7 +58,7 @@ def __bucket(self, required=True):
             raise ValueError(f"No bucket specified for Minio URL {self}")
         return bucket
 
-    def __path_without_bucket(self, required=True):
+    def __path_without_bucket(self):
         """ Get the path without the bucket from this URL.
 
             Accepts
@@ -67,22 +67,18 @@ def __path_without_bucket(self, required=True):
         """
         path = self.path
         path = path[1:] if path.startswith("/") else path
-        path = path.split("/", 1)[1] if "/" in path else None
-        path = path or None
-        if required and not path:
-            raise ValueError(f"No path specified for Minio URL {self}")
-        return path
+        return path.split("/", 1)[1] if "/" in path else ""
 
     def __connect(self):
-        creds = self.__creds
-        if creds not in self.__clients:
-            self.__clients[creds] = minio.Minio(
+        host, creds = self.host, self.__creds
+        if (host, creds) not in self.__clients:
+            self.__clients[(host, creds)] = minio.Minio(
                 self.host,
                 access_key=creds[0],
                 secret_key=creds[1],
                 secure=creds[2]
             )
-        return self.__clients[creds]
+        return self.__clients[(host, creds)]
 
     @_wrap_error
     async def read_binary(self) -> bytes:
@@ -122,12 +118,12 @@ async def ls(self, recursive: bool = False) -> List[URL]:
         """ Get the list of files in this directory, if it is one
 
             Returns a list of URL objects. Results are always absolute.
-            *DO NOT `root.join(file)`*
         """
         bucket = self.__bucket()
         if bucket:
             prefix = self.__path_without_bucket()
-            prefix = prefix if prefix.endswith("/") else prefix + "/"
+            if prefix and not prefix.endswith("/"):
+                prefix = prefix + "/"
             return [
                 # These paths are relative to the bucket, but join()
                 # is relative to this prefix. So slice it off.
@@ -175,13 +171,38 @@ async def mkdir(self, ignore_if_exists: bool = False):
             ignore_if_exists: boolean: DEPRECATED
                 Included for backward compatibility. Existing directories are always ignored.
         """
-        # This doesn't really exist
+        # This doesn't really exist, but it sorta does for buckets
+
+    @_wrap_error
+    async def make_bucket(self):
+        """ Create a bucket.
+
+            The path is not used and no folders are created.
+            For filesystems without buckets, this method has no effect.
+            In other filesystems you may need special permissions to create buckets.
+            Creating buckets programmatically may be unwise on account of billing.
+        """
+        self.__connect().make_bucket(self.__bucket())
 
     def supports_permissions(self) -> bool:
         """ Some implementations, like blobs, do not always support permissions,
             If this method returns true, the file system supports permissions
         """
         return False
+
+    def supports_directories(self) -> bool:
+        """ Return whether the protocol supports first-class directories.
+
+            Notes
+            -----
+            If the filesystems support directories, then:
+                - mkdir() and isdir() have meaning
+                - mkdir() followed by isdir() should be True
+            Otherwise:
+                - mkdir() has no effect and isdir() degrades to best-effort
+                  which usually means it will only be True if the directory has content
+        """
+        return False
 
     @_wrap_error
     async def unlink(self, ignore_if_missing: bool = False):