29 feat apply changes to use workflow pipelines270

.github/workflows/ci.yml

@@ -53,10 +53,24 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'poetry'
+    -
+      name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    -
+      name: Run services
+      run: |
+        docker-compose -f docker-compose-tutorial.yml up -d --build
     -
       name: Installing workflow dependencies
       run: |
         poetry install
+    -
+      name: Set workspace for testing
+      run: |
+        poetry run workflow workspace set development
     -
       name: Run workflow tests
       run: |
@@ -67,3 +81,8 @@ jobs:
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         path-to-lcov: "coverage.lcov"
+    -
+      name: Kill services
+      if: always()
+      run: |
+        docker-compose -f docker-compose-tutorial.yml down -v

.pre-commit-config.yaml

@@ -23,11 +23,11 @@ repos:
           - --py36-plus
         id: pyupgrade
     repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.15.2
   - hooks:
       - id: black
     repo: https://github.com/psf/black
-    rev: 24.1.1
+    rev: 24.4.2
   - hooks:
       - additional_dependencies:
           - types-attrs
@@ -40,7 +40,7 @@ repos:
           - --no-implicit-optional
         id: mypy
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.10.0
   - hooks:
       - args:
           - --convention=google
@@ -70,14 +70,14 @@ repos:
         args:
           - --autofix
     repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
   - hooks:
       - args:
           - -iii
           - -lll
         id: bandit
     repo: https://github.com/PyCQA/bandit
-    rev: 1.7.7
+    rev: 1.7.8
   - hooks:
       - additional_dependencies:
           - radon
@@ -96,4 +96,4 @@ repos:
         stages:
           - commit-msg
     repo: https://github.com/commitizen-tools/commitizen
-    rev: v3.13.0
+    rev: v3.27.0

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [0.4.0](https://github.com/CHIMEFRB/workflow/compare/v0.3.0...v0.4.0) (2024-05-28)
+
+
+### Features
+
+* add schedule cli ([#25](https://github.com/CHIMEFRB/workflow/issues/25)) ([56dcd6e](https://github.com/CHIMEFRB/workflow/commit/56dcd6e8f53853234de069bac4c77255c448cceb))
+* **cli/main.py:** adding logging on top level function ([#37](https://github.com/CHIMEFRB/workflow/issues/37)) ([7820c62](https://github.com/CHIMEFRB/workflow/commit/7820c623406f582e7fceef22785cb040d1be04b3))
+* **docker-compose-tutorial.yml:** adding docker compose for tutorial ([#35](https://github.com/CHIMEFRB/workflow/issues/35)) ([7b17e2a](https://github.com/CHIMEFRB/workflow/commit/7b17e2a987e09d8a950d9b4202b29c068321971f))
+
 ## [0.3.0](https://github.com/CHIMEFRB/workflow/compare/v0.2.0...v0.3.0) (2024-03-06)
 
 

docker-compose-tutorial.yml

@@ -0,0 +1,139 @@
+services:
+  pipelines_api:
+    image: chimefrb/pipelines:latest
+    container_name: pipelines_api
+    command: python -m pipelines.server
+    ports:
+      - "8001:8001"
+    expose:
+      - 8001
+    environment:
+      - SANIC_HOSTNAME=0.0.0.0
+      - SANIC_PORT=8001
+      - SANIC_ACCESS_LOG=true
+      - SANIC_AUTO_RELOAD=true
+      - SANIC_DEBUG=true
+      - SANIC_MONGODB_HOSTNAME=mongo
+      - SANIC_MONGODB_PORT=27017
+      - SANIC_START_MANAGER_URL=http://pipelines_managers:8002/v2/start
+      - SANIC_PAUSE_MANAGER_URL=http://pipelines_managers:8002/v2/pause
+      - SANIC_STOP_MANAGER_URL=http://pipelines_managers:8002/v2/stop
+      - SANIC_HEALTH_MANAGERS_URL=http://pipelines_managers:8002/__health__
+      - SANIC_HEALTH_MANAGERS_CHECK_TIMES=10
+      - SANIC_HEALTH_MANAGERS_CHECK_INTERVAL_SECONDS=30
+      - SANIC_LISTENERS_THRESHOLD_SECONDS=120
+      - TZ=Etc/UTC
+    networks:
+      - workflow
+    restart: always
+
+  pipelines_managers:
+    image: chimefrb/pipelines:latest
+    container_name: pipelines_managers
+    command: python -m managers.server
+    ports:
+      - "8002:8002"
+    environment:
+      - SANIC_HOSTNAME=0.0.0.0
+      - SANIC_PORT=8002
+      - SANIC_ACCESS_LOG=true
+      - SANIC_AUTO_RELOAD=true
+      - SANIC_DEBUG=true
+      - SANIC_MONGODB_HOSTNAME=mongo
+      - SANIC_MONGODB_PORT=27017
+      - SANIC_BUCKETS_URL=http://buckets:8004
+      - SANIC_RESULTS_URL=http://results:8005
+      - SANIC_UPDATE_INTERVAL_SECONDS=40
+      - SANIC_SLEEP_INTERVAL_SECONDS=30
+      - SANIC_PURGE_TIME_SECONDS=3600
+      - DOCKER_HOST=unix:///var/run/docker.sock # Replace with production address or dind
+      - TZ=Etc/UTC
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - workflow
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:8002/__health__"
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    restart: always
+
+  buckets:
+    image: chimefrb/buckets:latest
+    container_name: buckets
+    command: [ "/bin/bash", "-c", "python -m buckets.server" ]
+    expose:
+      - 8004
+    ports:
+      - "8004:8004"
+    environment:
+      - SANIC_HOSTNAME=0.0.0.0
+      - SANIC_PORT=8004
+      - SANIC_ACCESS_LOG=true
+      - SANIC_AUTO_RELOAD=true
+      - SANIC_DEBUG=true
+      - SANIC_MONGODB_HOSTNAME=mongo
+      - SANIC_MONGODB_PORT=27017
+      - SANIC_CORS_ORIGINS=*
+    networks:
+      - workflow
+
+  results:
+    image: chimefrb/results:latest
+    container_name: results
+    command: [ "/bin/bash", "-c", "python -m results.server" ]
+    expose:
+      - 8005
+    ports:
+      - "8005:8005"
+    environment:
+      - SANIC_HOSTNAME=0.0.0.0
+      - SANIC_PORT=8005
+      - SANIC_ACCESS_LOG=true
+      - SANIC_AUTO_RELOAD=true
+      - SANIC_DEBUG=true
+      - SANIC_MONGODB_HOSTNAME=mongo
+      - SANIC_MONGODB_PORT=27017
+      - SANIC_CORS_ORIGINS=*
+    networks:
+      - workflow
+
+  mongo:
+    image: mongo:latest
+    command: mongod --bind_ip_all
+    container_name: mongo
+    ports:
+      - "27017:27017"
+    networks:
+      - workflow
+
+  local:
+    image: docker:dind
+    container_name: local
+    command:
+      - "/bin/sh"
+      - "-c"
+      - |
+        dockerd -H tcp://0.0.0.0:4444 --tls=false \
+        & while(! docker -H tcp://0.0.0.0:4444 info >/dev/null 2>&1); \
+        do sleep 1; \
+        done \
+        && docker -H tcp://0.0.0.0:4444 swarm init && tail -f /dev/null
+    ports:
+      - "4444:4444"
+    expose:
+      - 4444
+    privileged: true
+    networks:
+      - workflow
+
+networks:
+  workflow:
+    driver: bridge

docs/workspaces.md

@@ -6,7 +6,7 @@ A workspace is,
 
     - Project-Specific: Each project can have their own workspace, while sharing the same installation.
     - YAML-Based: Workspaces are defined in YAML files, which can be version controlled.
-    - Stored in the client/user's home directory under the path `~/.workflow/workspaces/`
+    - Stored in the client/user's home directory under the path `~/.config/workflow`
 
 ## How do I activate a workspace?
 
@@ -32,13 +32,13 @@ To remove an active workspace,
     workflow workspace rm
     ```
 
-This will only remove the active workspace, i.e. `~/.workflow/workspaces/active.yml`.
+This will only remove the active workspace, i.e. `~/.config/workflow/workspace.yml`.
 
 !!! Important
 
     Running workflow without a workspace set will result in an runtime error.
 
-In order to purge all workspaces, from `~/.workflow/workspaces/` run:
+In order to purge all workspaces, from `~/.config/workflow/` run:
 
     ```bash
     workflow workspace purge

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "workflow"
-version = "0.3.0"
+version = "0.4.0"
 description = "Workflow Core"
 authors = ["Shiny Brar <[email protected]>"]
 license = "MIT"

tests/conftest.py

@@ -1,16 +1,14 @@
 """pytest configuration file."""
 
+import pytest
 from click.testing import CliRunner
 
 from workflow.cli.main import cli as workflow
 
 
-def pytest_configure(config):
-    """Initailize pytest configuration.
-
-    Allows plugins and conftest files to perform initial configuration.
-    This hook is called for every plugin and initial conftest
-    file after command line options have been parsed.
-    """
+@pytest.fixture(autouse=True, scope="function")
+def set_testing_workspace():
+    """Initailize testing workspace."""
     runner = CliRunner()
     runner.invoke(workflow, ["workspace", "set", "development"])
+    return True

tests/test_cli.py

@@ -0,0 +1,35 @@
+"""Test the workspace CLI commands."""
+
+import os
+
+from click.testing import CliRunner
+
+from workflow import CONFIG_PATH, DEFAULT_WORKSPACE_PATH
+from workflow.cli.workspace import ls, set
+
+
+class TestWorkspaceCLI:
+    def test_workspace_ls(self):
+        runner = CliRunner()
+        result = runner.invoke(ls)
+        assert result.exit_code == 0
+        assert "From Workflow Python Module" in result.output
+        assert "development" in result.output
+
+    def test_workspace_set(self):
+        runner = CliRunner()
+        result = runner.invoke(set, ["development"])
+        assert result.exit_code == 0
+        assert "Locating workspace development" in result.output
+        assert "Workspace development set to active" in result.output
+        # ? Check the default folder only contains the active workspace file
+        entries = os.listdir(CONFIG_PATH)
+        files = [
+            CONFIG_PATH / entry
+            for entry in entries
+            if os.path.isfile(os.path.join(CONFIG_PATH, entry))
+        ]
+        files = [f.as_posix() for f in files]
+        assert files == [DEFAULT_WORKSPACE_PATH.as_posix()]
+        # ? Re set workspace for other tests
+        result = runner.invoke(set, ["development"])