diff --git a/.github/workflows/build.js.yml b/.github/workflows/build.js.yml index bc4016d91..a9b2b91ba 100644 --- a/.github/workflows/build.js.yml +++ b/.github/workflows/build.js.yml @@ -395,143 +395,6 @@ jobs: name: logs path: logs - end_to_end_beta: - needs: [build_backend, build_frontend] - runs-on: ubuntu-latest - - steps: - # Clone repository - - uses: actions/checkout@v3 - - - name: Free up disk space - run: | - # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - echo "Check free disk space before cleanup." - df -h - echo "Removing non-essential tools and libraries." - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /opt/ghc - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/share/boost - # Delete libraries for Android (12G), PowerShell (1.3G), Swift (1.7G) - sudo rm -rf /usr/local/lib/android - sudo rm -rf /usr/local/share/powershell - sudo rm -rf /usr/share/swift - echo "Check free disk space after cleanup." - df -h - - - name: Use Node.js 18 - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: "npm" - - # Setup BuildX - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # Get Bailo Docker images - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: bailo_backend - path: /tmp - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: bailo_frontend - path: /tmp - - - name: Load image - run: | - docker load --input /tmp/bailo_frontend.tar - docker load --input /tmp/bailo_backend.tar - docker image ls -a - - rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar - - # Create logs directory - - run: mkdir logs - - # Create certs - - run: sudo chown -R runner:docker backend/certs - - run: npm run certs - - # Configure application - - name: Configure Bailo - run: | - cat <> backend/config/local.cjs - module.exports = { - minio: { - accessKey: 'minioadmin', - secretKey: 'minioadmin', - }, - - smtp: { - auth: { - user: 'mailuser', - pass: 'mailpass', - }, - tls: { - rejectUnauthorized: false, - }, - }, - - logging: { - file: { - path: '/home/runner/work/Bailo/Bailo/logs/out.log', - }, - }, - } - EOT - - - run: docker-compose -f docker-compose-prod.yml up -d - - # Install dependencies - - run: npm ci - - # wait for app to start - - run: | - chmod +x ./backend/src/scripts/waitForIt.sh - ./backend/src/scripts/waitForIt.sh localhost:8080 -t 10 - - # wait for mongo to start - - run: | - chmod +x ./backend/src/scripts/waitForIt.sh - ./backend/src/scripts/waitForIt.sh localhost:27017 -t 10 - - - uses: cypress-io/github-action@v5 - with: - working-directory: ./frontend - install: false - spec: cypress/e2e/beta/**/* - - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: cypress-screenshots-e2e - path: frontend/cypress/screenshots - - # - uses: actions/upload-artifact@v3 - # if: always() - # with: - # name: cypress-videos - # path: cypress/videos - - - name: Dump docker-compose logs - if: always() - run: docker-compose logs > logs/stack.log - - - name: Docker Compose Logs - if: always() - run: cat logs/stack.log - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: logs - integration_python: needs: [build_backend, build_frontend] runs-on: ubuntu-latest diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 26e927c60..d8334f34f 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -21,33 +21,9 @@ permissions: id-token: write jobs: - build-sphinx: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - - name: Install pandoc - run: sudo apt install pandoc - - - name: Build Sphinx docs - run: | - python -m pip install --upgrade pip - pwd - cd lib/python-beta/docs - python -m pip install -r requirements.txt - make dirhtml - # Build job build: runs-on: ubuntu-latest - needs: build-sphinx defaults: run: working-directory: lib/landing @@ -89,7 +65,6 @@ jobs: run: npm i - name: Run generator run: npm run generate - - name: Build and export run: npm run export env: diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 0e060943c..735d20e59 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -139,6 +139,26 @@ module.exports = { schema: require('../src/scripts/example_schemas/minimal_deployment_schema.json'), }, ], + v2: { + modelCards: [ + { + name: 'Minimal Schema v10', + id: 'minimal-general-v10', + description: + "This is the latest version of the default model card for users from West. It complies with all requirements laid out in the [AI Policy](https://example.com) as well as best practices recommended by 'Science and Research'.\n\nIf you're unsure which model card to pick, you'll likely want this one!", + jsonSchema: require('../src/scripts/example_schemas/minimal_model_schema.json'), + }, + ], + accessRequests: [ + { + name: 'Minimal Access Request Schema v10', + id: 'minimal-access-request-general-v10', + description: + 'This access request should be used for models that are being deployed by the same organisation that created it and MAY be being used for operational use cases.\n\n ✔ Development Work \n ✔ Operational Deployments \n ✖ Second Party Sharing', + jsonSchema: require('../src/scripts/example_schemas/minimal_access_request_schema.json'), + }, + ], + }, }, session: { @@ -191,32 +211,11 @@ module.exports = { host: 'localhost:8080', }, - // A configurable checkbox on the last page when uploading a model - uploadWarning: { - showWarning: true, - checkboxText: 'By checking here you confirm that the information is correct', - }, - - // A configurable checkbox on the last page when requesting a deployment - deploymentWarning: { - showWarning: true, - checkboxText: 'By checking here you confirm that the information is correct', - }, - // Used by some admin pages (e.g. the logs) to directly open the correct page in your IDE // Not needed in production development: { logUrl: 'vscode://file/home/ec2-user/git/Bailo/', }, - - // The available seldon versions that can be used to build images - seldonVersions: [ - { - name: 'seldonio - 1.10.0', - image: 'seldonio/seldon-core-s2i-python37:1.10.0', - }, - ], - maxModelSizeGB: 50, }, connectors: { diff --git a/backend/python-docs/_build/dirhtml/.buildinfo b/backend/python-docs/_build/dirhtml/.buildinfo index b8c0d63a8..12dadcadf 100644 --- a/backend/python-docs/_build/dirhtml/.buildinfo +++ b/backend/python-docs/_build/dirhtml/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: db1993bf4fe87f6a0d251d9236d6e7ea +config: dbafcd98063260b5743f60ed267116f6 tags: d77d1c0d9ca2f4c8421862c7c5a0d620 diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/core/agent/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/core/agent/index.html index 3df1e1531..661633c11 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/core/agent/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/core/agent/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/core/client/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/core/client/index.html index 29023c2cb..37af240c8 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/core/client/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/core/client/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
@@ -422,24 +424,48 @@

Source code for bailo.core.client

         self,
         model_id: str,
         file_id: str,
-        buffer: BytesIO,
     ):
-        """Download a specific file.
+        """Download a specific file by it's id.
 
         :param model_id: Unique model ID
         :param file_id: Unique file ID
-        :param buffer: BytesIO object for bailo to write to
         :return: The unique file ID
         """
         if isinstance(self.agent, TokenAgent):
-            req = self.agent.get(
+            return self.agent.get(
                 f"{self.url}/v2/token/model/{model_id}/file/{file_id}/download", stream=True, timeout=10_000
             )
         else:
-            req = self.agent.get(f"{self.url}/v2/model/{model_id}/file/{file_id}/download", stream=True, timeout=10_000)
+            return self.agent.get(
+                f"{self.url}/v2/model/{model_id}/file/{file_id}/download", stream=True, timeout=10_000
+            )
+ + +
+[docs] + def get_download_by_filename( + self, + model_id: str, + semver: str, + filename: str, + ): + """Download a specific file. - shutil.copyfileobj(req.raw, buffer) - return file_id
+ :param model_id: Unique model ID + :param semver: Semver of the release + :param filename: The filename trying to download from + :return: The filename + """ + if isinstance(self.agent, TokenAgent): + return self.agent.get( + f"{self.url}/v2/token/model/{model_id}/release/{semver}/file/{filename}/download", + stream=True, + timeout=10_000, + ) + else: + return self.agent.get( + f"{self.url}/v2/model/{model_id}/release/{semver}/file/{filename}/download", stream=True, timeout=10_000 + )
diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/core/enums/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/core/enums/index.html index e11ae8f65..b875ba66c 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/core/enums/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/core/enums/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/core/exceptions/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/core/exceptions/index.html index 797d20161..4be458b91 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/core/exceptions/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/core/exceptions/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/access_request/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/access_request/index.html index 0477c139d..f73926a0a 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/access_request/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/access_request/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/model/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/model/index.html index 7935059ec..b093f374a 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/model/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/model/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/release/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/release/index.html index 0b80927a4..e959f87db 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/release/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/release/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
@@ -229,19 +231,18 @@

Source code for bailo.helper.release

 
 

 [docs]
-    def download(self, file_id: str, buffer: BytesIO) -> str:
+    def download(self, filename: str) -> Any:
         """Give returns a Reading object given the file id.
 
-        :param file_name: The name of the file to retrieve
-        :param buffer: A BytesIO object
+        :param filename: The name of the file to retrieve
         :return: A JSON response object
         """
-        return self.client.get_download_file(self.model_id, file_id, buffer)

+        return self.client.get_download_by_filename(self.model_id, str(self.version), filename)
[docs] - def upload(self, name: str, file: BytesIO) -> Any: + def upload(self, name: str, file: BytesIO) -> str: """Upload files in a given directory to the release. :param name: The name of the file to upload to bailo diff --git a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/schema/index.html b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/schema/index.html index 37712ad62..8acfc3400 100644 --- a/backend/python-docs/_build/dirhtml/_modules/bailo/helper/schema/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/bailo/helper/schema/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_modules/index.html b/backend/python-docs/_build/dirhtml/_modules/index.html index d27fb13a3..5f5f7a899 100644 --- a/backend/python-docs/_build/dirhtml/_modules/index.html +++ b/backend/python-docs/_build/dirhtml/_modules/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
diff --git a/backend/python-docs/_build/dirhtml/_sources/notebooks/models_and_releases_demo.ipynb.txt b/backend/python-docs/_build/dirhtml/_sources/notebooks/models_and_releases_demo.ipynb.txt index 6acb00da1..37330632a 100644 --- a/backend/python-docs/_build/dirhtml/_sources/notebooks/models_and_releases_demo.ipynb.txt +++ b/backend/python-docs/_build/dirhtml/_sources/notebooks/models_and_releases_demo.ipynb.txt @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -92,7 +92,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -132,9 +132,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "model.card_from_schema(schema_id='minimal-general-v10-beta')\n", "\n", @@ -155,9 +166,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "new_card = {\n", " 'overview': {\n", @@ -203,9 +225,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'YOLOv5 model for object detection.'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "model = Model.from_id(client=client, model_id=model_id)\n", "\n", @@ -253,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -280,9 +313,18 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "yolov5-wb9i5i\n", + "yolov5-wb9i5i\n" + ] + } + ], "source": [ "print(release_one.model_id)\n", "print(model.model_id)\n", @@ -291,6 +333,35 @@ " release_one.upload(\"demo_file.txt\", f)" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Downloading files from release\n", + "\n", + "Similarly you can also download files from release using a response object. Using the same release you can view the contents of **demo_file** as follows." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'This is a demo file.\\n'\n" + ] + } + ], + "source": [ + "res = release_one.download(\"demo_file.txt\")\n", + "\n", + "print(res.content)" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -309,9 +380,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Successfully retrieved identical releases!\n" + ] + } + ], "source": [ "release_latest = model.get_latest_release()\n", "release_one = model.get_release(version='1.0.0')\n", diff --git a/backend/python-docs/_build/dirhtml/_static/favicon.png b/backend/python-docs/_build/dirhtml/_static/favicon.png new file mode 100644 index 000000000..b30701666 Binary files /dev/null and b/backend/python-docs/_build/dirhtml/_static/favicon.png differ diff --git a/backend/python-docs/_build/dirhtml/_static/logo-vertical-dark-transparent.png b/backend/python-docs/_build/dirhtml/_static/logo-vertical-dark-transparent.png new file mode 100644 index 000000000..a70cef28d Binary files /dev/null and b/backend/python-docs/_build/dirhtml/_static/logo-vertical-dark-transparent.png differ diff --git a/backend/python-docs/_build/dirhtml/_static/vertical-white.png b/backend/python-docs/_build/dirhtml/_static/vertical-white.png new file mode 100644 index 000000000..78f29d6da Binary files /dev/null and b/backend/python-docs/_build/dirhtml/_static/vertical-white.png differ diff --git a/backend/python-docs/_build/dirhtml/bailo.core/index.html b/backend/python-docs/_build/dirhtml/bailo.core/index.html index 8478d2f69..963c905cd 100644 --- a/backend/python-docs/_build/dirhtml/bailo.core/index.html +++ b/backend/python-docs/_build/dirhtml/bailo.core/index.html @@ -9,6 +9,7 @@ + @@ -36,6 +37,7 @@ Bailo Python Client Docs +
@@ -88,6 +90,7 @@
  • Client.get_all_releases()
  • Client.get_all_schemas()
  • Client.get_all_teams()
    • +
  • Client.get_download_by_filename()
  • Client.get_download_file()
  • Client.get_files()
  • Client.get_model()
    • @@ -453,15 +456,32 @@

    bailo.core package -
    -get_download_file(model_id: str, file_id: str, buffer: BytesIO)[source]
    +
    +get_download_by_filename(model_id: str, semver: str, filename: str)[source]

    Download a specific file.

    Parameters:

    • model_id – Unique model ID

      • +

    • semver – Semver of the release

      • +

    • filename – The filename trying to download from

      • +
    +
    +
    Returns:
    +

    The filename

    +
    +
    +
    + +
    +
    +get_download_file(model_id: str, file_id: str)[source]
    +

    Download a specific file by it’s id.

    +
    +
    Parameters:
    +
      +

    • model_id – Unique model ID

    • file_id – Unique file ID

      • -

    • buffer – BytesIO object for bailo to write to

    Returns:
    diff --git a/backend/python-docs/_build/dirhtml/bailo.helper/index.html b/backend/python-docs/_build/dirhtml/bailo.helper/index.html index 8b8e141b3..4f53a63d0 100644 --- a/backend/python-docs/_build/dirhtml/bailo.helper/index.html +++ b/backend/python-docs/_build/dirhtml/bailo.helper/index.html @@ -9,6 +9,7 @@ + @@ -36,6 +37,7 @@ Bailo Python Client Docs +
    @@ -481,14 +483,11 @@
    -download(file_id: str, buffer: BytesIO) → str[source]
    +download(filename: str) → Any[source]

    Give returns a Reading object given the file id.

    Parameters:
    -
      -

    • file_name – The name of the file to retrieve

      • -

    • buffer – A BytesIO object

      • -
    +

    filename – The name of the file to retrieve

    Returns:

    A JSON response object

    @@ -524,7 +523,7 @@
    -upload(name: str, file: BytesIO) → Any[source]
    +upload(name: str, file: BytesIO) → str[source]

    Upload files in a given directory to the release.

    Parameters:
    diff --git a/backend/python-docs/_build/dirhtml/genindex/index.html b/backend/python-docs/_build/dirhtml/genindex/index.html index f6dc3faf9..27f90b2c6 100644 --- a/backend/python-docs/_build/dirhtml/genindex/index.html +++ b/backend/python-docs/_build/dirhtml/genindex/index.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
    @@ -301,17 +303,19 @@

    G

  • get_card_latest() (bailo.helper.model.Model method)
  • get_card_revision() (bailo.helper.model.Model method) +
    • +
  • get_download_by_filename() (bailo.core.client.Client method)
  • get_download_file() (bailo.core.client.Client method)
  • get_files() (bailo.core.client.Client method)
  • get_image() (bailo.helper.model.Model method) -
    • -
  • get_images() (bailo.helper.model.Model method)
    • + +
    +
    +get_download_file(model_id: str, file_id: str)[source]
    +

    Download a specific file by it’s id.

    +
    +
    Parameters:
    +
      +

    • model_id – Unique model ID

    • file_id – Unique file ID

      • -

    • buffer – BytesIO object for bailo to write to

    Returns:
    diff --git a/backend/python-docs/_build/html/bailo.helper.html b/backend/python-docs/_build/html/bailo.helper.html index 97ab5ea49..4dd168bd3 100644 --- a/backend/python-docs/_build/html/bailo.helper.html +++ b/backend/python-docs/_build/html/bailo.helper.html @@ -9,6 +9,7 @@ + @@ -36,6 +37,7 @@ Bailo Python Client Docs +
    @@ -481,14 +483,11 @@
    -download(file_id: str, buffer: BytesIO) → str[source]
    +download(filename: str) → Any[source]

    Give returns a Reading object given the file id.

    Parameters:
    -
      -

    • file_name – The name of the file to retrieve

      • -

    • buffer – A BytesIO object

      • -
    +

    filename – The name of the file to retrieve

    Returns:

    A JSON response object

    @@ -524,7 +523,7 @@
    -upload(name: str, file: BytesIO) → Any[source]
    +upload(name: str, file: BytesIO) → str[source]

    Upload files in a given directory to the release.

    Parameters:
    diff --git a/backend/python-docs/_build/html/genindex.html b/backend/python-docs/_build/html/genindex.html index cbfdcaaeb..7348689d6 100644 --- a/backend/python-docs/_build/html/genindex.html +++ b/backend/python-docs/_build/html/genindex.html @@ -8,6 +8,7 @@ + @@ -33,6 +34,7 @@ Bailo Python Client Docs +
    @@ -301,17 +303,19 @@

    G

  • get_card_latest() (bailo.helper.model.Model method)
  • get_card_revision() (bailo.helper.model.Model method) +
    • +
  • get_download_by_filename() (bailo.core.client.Client method)
  • get_download_file() (bailo.core.client.Client method)
  • get_files() (bailo.core.client.Client method)
  • get_image() (bailo.helper.model.Model method) -
    • -
  • get_images() (bailo.helper.model.Model method)