Integrate with new data transfer client (#436)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Oliver Koenig <[email protected]>
Co-authored-by: Federico Negri <[email protected]>

.github/actions/hps_services/action.yml

@@ -8,6 +8,10 @@ inputs:
     description: 'HPS version'
     required: true
 
+  feature:
+    description: 'HPS feature (only for latest-dev version)'
+    required: false
+
   token:
     description: 'Token'
     required: true
@@ -44,14 +48,17 @@ runs:
         token: ${{ inputs.token }}
 
     - if: ${{ inputs.version == 'latest-dev' }}
-      uses: dsaltares/fetch-gh-release-asset@master
+      uses: dawidd6/action-download-artifact@v3
       name: Download Internal Services Artifact
       with:
-        file: 'docker-compose-internal.tar.gz'
-        target: 'docker-compose-artifact/docker-compose-internal.tar.gz'
+        workflow: main.yaml
+        name: docker-compose-internal
+        branch: ${{ inputs.feature || 'main'}}
         repo: ansys-internal/rep-deployments
-        version: tags/latest-dev
-        token: ${{ inputs.token }}
+        workflow_conclusion: success           
+        search_artifacts: false
+        path: ./docker-compose-artifact
+        github_token: ${{ inputs.token }}
 
     - uses: KengoTODA/actions-setup-docker-compose@main
       env:
@@ -80,17 +87,19 @@ runs:
       if: ${{ inputs.version == 'latest-dev' }}
       shell: bash
       run: |
+        echo "$(pwd)"
+        pwd
         ls -la
         tar -xvzf docker-compose-internal.tar.gz
         cd docker-compose
-        FSGATEWAY_TAG=latest-dev JMS_TAG=latest-dev docker-compose pull
-        FSGATEWAY_TAG=latest-dev JMS_TAG=latest-dev docker-compose up -d
+        docker-compose pull
+        docker-compose up -d
       working-directory: ./docker-compose-artifact
 
     - name: Wait for services
       shell: bash
       run: |
-        curl -k --head -X GET --retry 30 --retry-connrefused --retry-delay 1 https://localhost:8443/hps/jms/api/v1
+        curl -k --head -X GET --retry 30 --retry-connrefused --retry-delay 5 https://localhost:8443/hps/jms/api/v1
         curl -k --head -X GET --retry 30 --retry-connrefused --retry-delay 1 https://localhost:8443/hps/fs/api/v1
         curl -k --head -X GET --retry 30 --retry-connrefused --retry-delay 1 https://localhost:8443/hps/rms/api/v1
 
@@ -100,4 +109,4 @@ runs:
       run: |
         echo "url=https://localhost:8443/hps" >> $GITHUB_OUTPUT
         echo "path=$(pwd)" >> $GITHUB_OUTPUT
-      working-directory: ./docker-compose-artifact/docker-compose
+      working-directory: ./docker-compose-artifact/docker-compose

.github/workflows/ci_cd.yml

@@ -10,6 +10,10 @@ on:
           - 'v1.0.2'
           - 'v1.0.3'
           - 'latest-dev'
+      hps-feature:
+        description: HPS feature to test against
+        type: 'string'
+        default: 'main'
 
   pull_request:
   push:
@@ -57,6 +61,7 @@ jobs:
       toxenv: ${{ matrix.cfg.toxenv }}
       runner: ${{ matrix.os }}
       hps-version: ${{ inputs.hps-version || 'latest-dev' }}
+      hps-feature: ${{ inputs.hps-feature || 'main' }}
 
   docs:
     name: Documentation
@@ -69,6 +74,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.MAIN_PYTHON_VERSION }}
+
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: pyansys-ci-bot
+          password: ${{ secrets.PYANSYS_CI_BOT_PACKAGE_TOKEN }}
 
       - name: Install dependencies
         run: |
@@ -120,6 +131,12 @@ jobs:
             python-version: '3.12'
 
     steps:
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: pyansys-ci-bot
+          password: ${{ secrets.PYANSYS_CI_BOT_PACKAGE_TOKEN }}
+
       - name: Build wheelhouse and perform smoke test
         uses: ansys/actions/build-wheelhouse@v7
         with:
@@ -146,6 +163,12 @@ jobs:
     steps:
       - name: "Install Git and clone project"
         uses: actions/checkout@v4
+
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: pyansys-ci-bot
+          password: ${{ secrets.PYANSYS_CI_BOT_PACKAGE_TOKEN }}
 
       - name: "Set up Python ${{ env.MAIN_PYTHON_VERSION }}"
         uses: ansys/actions/_setup-python@main

.github/workflows/tests.yml

@@ -19,6 +19,10 @@ on:
         description: HPS version to test against
         type: string
         default: 'latest-dev'
+      hps-feature:
+        description: HPS Feature to test against (only for latest-dev version)
+        type: string
+        default: 'main'
 
 jobs:
 
@@ -31,6 +35,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python-version }}
+
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: pyansys-ci-bot
+          password: ${{ secrets.PYANSYS_CI_BOT_PACKAGE_TOKEN }}
 
       - name: Install dependencies
         run: |
@@ -45,7 +55,8 @@ jobs:
           ghcr-username: ${{ secrets.PYANSYS_CI_BOT_USERNAME }}
           ghcr-token: ${{ secrets.PYANSYS_CI_BOT_PACKAGE_TOKEN }}
           version: ${{ inputs.hps-version }}
-
+          feature: ${{ inputs.hps-feature }}
+
       - name: Test with tox
         run: tox -e ${{ inputs.toxenv }}-coverage
         env:

pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
     "marshmallow_oneofschema>=2.0.1",
     "backoff>=2.0.0",
     "pydantic>=1.10.0",
-    "PyJWT>=2.8.0"
+    "PyJWT>=2.8.0",
+    "ansys-hps-data-transfer-client@git+https://github.com/ansys-internal/hps-data-transfer-client.git@main#egg=ansys-hps-data-transfer-client"
 ]
 
 [project.optional-dependencies]

src/ansys/hps/client/client.py

@@ -19,12 +19,18 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+
 """Module providing the Python client to the HPS APIs."""
 
+import atexit
 import logging
+import os
+import platform
 from typing import Union
 import warnings
 
+from ansys.hps.data_transfer.client import Client as DTClient
+from ansys.hps.data_transfer.client import DataTransferApi
 import jwt
 import requests
 
@@ -156,6 +162,8 @@ def __init__(
         self.client_id = client_id
         self.client_secret = client_secret
         self.verify = verify
+        self.data_transfer_url = url + f"/dt/api/v1"
+        self.dt_client = None
 
         if self.verify is None:
             self.verify = False
@@ -256,6 +264,13 @@ def __init__(
         self._unauthorized_num_retry = 0
         self._unauthorized_max_retry = 1
 
+        def exit_handler():
+            if self.dt_client is not None:
+                log.info("Stopping the data transfer client gracefully.")
+                self.dt_client.stop()
+
+        atexit.register(exit_handler)
+
     def _get_username(self, decoded_token):
         parsed_username = decoded_token.get("preferred_username", None)
         if not parsed_username:
@@ -280,6 +295,62 @@ def rep_url(self) -> str:
         log.warning(msg)
         return self.url
 
+    def _start_dt_worker(self):
+
+        if self.dt_client is None:
+            try:
+                log.info("Starting Data Transfer client.")
+                # start Data transfer client
+                self.dt_client = DTClient(download_dir=self._get_download_dir("Ansys"))
+
+                self.dt_client.binary_config.update(
+                    verbosity=3,
+                    debug=False,
+                    insecure=True,
+                    token=self.access_token,
+                    data_transfer_url=self.data_transfer_url,
+                )
+                self.dt_client.start()
+
+                self.dt_api = DataTransferApi(self.dt_client)
+                self.dt_api.status(wait=True)
+            except Exception as ex:
+                log.debug(ex)
+                raise HPSError("Error occurred when starting Data Transfer client.")
+
+    def _get_download_dir(self, company=None):
+        """
+        Returns download directory platform dependent
+
+        :Parameters:
+        -`company`: Company name of the software provider
+
+        Resulting paths:
+        `Linux`: /home/user/.ansys/binaries
+        `Windows`: C:\\Users\\user\\AppData\\Local\\Ansys\\binaries
+
+        Note that on Windows we use AppData\Local for this,
+        not AppData\Roaming, as the data stored for an application should typically be kept local.
+
+        """
+
+        environment_variable = "HOME"
+        if platform.uname()[0].lower() == "windows":
+            environment_variable = "LOCALAPPDATA"
+        path = os.environ.get(environment_variable, None)
+
+        app_dir = ""
+        if company:
+            app_dir = os.path.join(app_dir, company)
+        if app_dir:
+            if platform.uname()[0].lower() != "windows":
+                app_dir = "." + app_dir.lower()
+            path = os.path.join(path, app_dir)
+
+        path = os.path.join(path, "binaries")
+
+        return path
+
     @property
     def auth_api_url(self) -> str:
         msg = f"The client 'auth_api_url' property is deprecated. \
@@ -306,6 +377,8 @@ def _auto_refresh_token(self, response, *args, **kwargs):
             response.request.headers.update(
                 {"Authorization": self.session.headers["Authorization"]}
             )
+            if self.dt_client is not None:
+                self.dt_client.binary_config.update(token=self.access_token)
             log.debug(f"Retrying request with updated access token.")
             return self.session.send(response.request)