Re-Organize Folders (#717)

The old structure: ``` docs lumigator frontend infra mzai helm lumigator python mzai backend jobs sample_data schemas sdk ``` Re-organized to ``` docs infra helm lumigator lumigator frontend backend jobs sample_data schemas sdk ``` --------- Signed-off-by: Nathan Brake <[email protected]> Co-authored-by: Kyle White <[email protected]>
mozilla-ai · Jan 30, 2025 · 77c0931 · 77c0931
1 parent 884a772
commit 77c0931
Show file tree

Hide file tree

Showing 305 changed files with 788 additions and 806 deletions.
diff --git a/.devcontainer/docker-compose.override.yaml b/.devcontainer/docker-compose.override.yaml
@@ -13,20 +13,20 @@ services:
       - "5678:5678"
     develop:
       watch:
-        - path: lumigator/python/mzai/backend/
-          target: /mzai/lumigator/python/mzai/backend
+        - path: lumigator/backend/
+          target: /mzai/lumigator/backend
           action: sync
           ignore:
             - .venv/
-        - path: lumigator/python/mzai/jobs/
-          target: /mzai/lumigator/python/mzai/jobs
+        - path: lumigator/jobs/
+          target: /mzai/lumigator/jobs
           action: sync
           ignore:
             - .venv/
-        - path: lumigator/python/mzai/schemas/
-          target: /mzai/lumigator/python/mzai/schemas
+        - path: lumigator/schemas/
+          target: /mzai/lumigator/schemas
           action: sync
           ignore:
             - .venv/
-        - path: lumigator/python/mzai/backend/pyproject.toml
+        - path: lumigator/backend/pyproject.toml
           action: rebuild
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -1,9 +1,9 @@
 api:
 - changed-files:
-  - any-glob-to-any-file: 'lumigator/python/mzai/backend/backend/api/**'
+  - any-glob-to-any-file: 'lumigator/backend/backend/api/**'
 backend:
 - changed-files:
-  - any-glob-to-any-file: 'lumigator/python/mzai/backend/**'
+  - any-glob-to-any-file: 'lumigator/backend/**'
 dependencies:
 - changed-files:
   - any-glob-to-any-file: 'pyproject.toml'
@@ -18,7 +18,7 @@ gha:
   - any-glob-to-any-file: '.github/workflows/**'
 schemas:
 - changed-files:
-  - any-glob-to-any-file: 'lumigator/python/mzai/schemas/**'
+  - any-glob-to-any-file: 'lumigator/schemas/**'
 sdk:
 - changed-files:
-  - any-glob-to-any-file: 'lumigator/python/mzai/sdk/**'
+  - any-glob-to-any-file: 'lumigator/sdk/**'
diff --git a/.github/workflows/build_and_publish_docs.yaml b/.github/workflows/build_and_publish_docs.yaml
@@ -28,7 +28,8 @@ jobs:
       uses: lycheeverse/lychee-action@v2
       with:
         # Documentation: https://github.com/lycheeverse/lychee?tab=readme-ov-file#commandline-parameters
-        args: --base . --verbose --no-progress './**/*.md'  './**/*.rst' --exclude 'mailto:|localhost'
+        # Excluding the docs folder because these links will be checked by the sphinx linkcheck builder later
+        args: --base . --verbose --no-progress './**/*.md'  './**/*.rst' --exclude 'mailto:|localhost' --exclude-path 'docs'
 
     - name: Check for modified paths
       uses: dorny/paths-filter@v3

diff --git a/.github/workflows/lumigator_pipeline.yaml b/.github/workflows/lumigator_pipeline.yaml
@@ -370,32 +370,32 @@ jobs:
 
       - name: Check tag consistency (SDK)
         run: check_version.sh
-        working-directory: lumigator/python/mzai/sdk
+        working-directory: lumigator/sdk
 
       - name: Check tag consistency (schemas)
         run: check_version.sh
-        working-directory: lumigator/python/mzai/schemas
+        working-directory: lumigator/schemas
 
       - name: Install python (SDK)
         run: uv python install
-        working-directory: lumigator/python/mzai/sdk
+        working-directory: lumigator/sdk
 
       - name: Install python (schemas)
         run: uv python install
-        working-directory: lumigator/python/mzai/schemas
+        working-directory: lumigator/schemas
 
       - name: Package SDK
         run: uv build
-        working-directory: lumigator/python/mzai/sdk
+        working-directory: lumigator/sdk
 
       - name: Package schemas
         run: uv build
-        working-directory: lumigator/python/mzai/schemas
+        working-directory: lumigator/schemas
 
       - name: Upload SDK packages to PyPI
-        working-directory: lumigator/python/mzai/sdk
+        working-directory: lumigator/sdk
         run: uv publish --token ${{ secrets.PYPI_TOKEN }} --verbose
 
       - name: Upload schemas packages to PyPI
-        working-directory: lumigator/python/mzai/schemas
-        run: uv publish --token ${{ secrets.PYPI_TOKEN }} --verbose
+        working-directory: lumigator/schemas
+        run: uv publish --token ${{ secrets.PYPI_TOKEN }} --verbose
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -10,7 +10,7 @@
     },
     "python.testing.pytestEnabled": true,
     "python.analysis.extraPaths": [
-        "./lumigator/python/mzai/schemas",
-        "./lumigator/python/mzai/jobs"
+        "./lumigator/schemas",
+        "./lumigator/jobs"
     ]
 }
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -10,7 +10,7 @@ We want to encourage both suggestions and contributions from the community in th
 
 * Check out the [roadmap](https://github.com/orgs/mozilla-ai/projects/17): This will give you an overview of the project's direction and help you identify areas where your contributions can have the greatest impact.
 * Take a look into the existing issues: Check if your issue or challenge has already been discussed or reported. This helps avoid duplication of effort.
-* Learn about the code: The Lumigator project consists of several packages, each with its own pyproject.toml file. Understanding the project structure will make it easier to contribute effectively: the `lumigator/python/mzai/backend` package that powers the lumigator server functionality, the `lumigator/python/mzai/schemas` package containing the formal schemas for communication with the server through the REST API, and the `lumigator/python/mzai/sdk` package abstracting the REST API for Python applications. Each package holds its own `pyproject.toml` definition.
+* Learn about the code: The Lumigator project consists of several packages, each with its own pyproject.toml file. Understanding the project structure will make it easier to contribute effectively: the `lumigator/backend` package that powers the lumigator server functionality, the `lumigator/schemas` package containing the formal schemas for communication with the server through the REST API, and the `lumigator/sdk` package abstracting the REST API for Python applications. Each package holds its own `pyproject.toml` definition.
 
 ### **Disclaimer**
 
@@ -22,7 +22,7 @@ We will carefully consider all submissions and provide feedback whenever possibl
 
 **Reporting Bugs**
 
-If you find an error while running Lumigator, we would appreciate you opening an [issue](https://github.com/mozilla-ai/lumigator/issues) to report it. Follow the corresponding template in Github providing a description of the bug, a detailed description on how to reproduce the reported behavior, a detailed description on what you would expect to happen and your system info. 
+If you find an error while running Lumigator, we would appreciate you opening an [issue](https://github.com/mozilla-ai/lumigator/issues) to report it. Follow the corresponding template in Github providing a description of the bug, a detailed description on how to reproduce the reported behavior, a detailed description on what you would expect to happen and your system info.
 
 **Suggesting features**
 
@@ -54,5 +54,3 @@ To contribute to Lumigator, first identify a need for a new feature or bug fix.
 * [Lumigator documentation](https://mozilla-ai.github.io/lumigator/): Describes all the components and includes usage guides.
 * [Open issues](https://github.com/mozilla-ai/lumigator/issues): Browse the open issues to check if the topic you want to raise is already being discussed.
 * [Contribute](https://github.com/mozilla-ai/lumigator/contribute): Explore the list of "good first issues" to start contributing.
-
-
diff --git a/Dockerfile b/Dockerfile
@@ -4,8 +4,8 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 # Copy the project into the image
 ADD . /mzai
 
-WORKDIR /mzai/lumigator/python/mzai/backend
-ENV PYTHONPATH=/mzai/lumigator/python/mzai/backend:/mzai/lumigator/python/mzai/jobs
+WORKDIR /mzai/lumigator/backend
+ENV PYTHONPATH=/mzai/lumigator/backend:/mzai/lumigator/jobs
 
 FROM base AS main_image
 
@@ -23,8 +23,8 @@ RUN uv sync --frozen
 CMD [\
     "uv","run", "-m", "debugpy", "--listen", "0.0.0.0:5678", \
     "-m", "uvicorn", "backend.main:app", "--reload", \
-    "--reload-dir", "/mzai/lumigator/python/mzai/jobs", \
-    "--reload-dir", "/mzai/lumigator/python/mzai/schemas", \
+    "--reload-dir", "/mzai/lumigator/jobs", \
+    "--reload-dir", "/mzai/lumigator/schemas", \
     "--host", "0.0.0.0", \
     "--port", "8000" \
 ]
diff --git a/Makefile b/Makefile
@@ -153,11 +153,11 @@ clean-all: clean-docker-buildcache clean-docker-containers
 # `test-sdk-integration-containers` is usually called and this will either
 # start them if they are not present or use the currently running ones.
 test-sdk-unit:
-	cd lumigator/python/mzai/sdk/tests; \
+	cd lumigator/sdk/tests; \
 	uv run $(DEBUGPY_ARGS) -m pytest -o python_files="unit/*/test_*.py unit/test_*.py"
 
 test-sdk-integration:
-	cd lumigator/python/mzai/sdk/tests; \
+	cd lumigator/sdk/tests; \
 	uv run $(DEBUGPY_ARGS) -m pytest -s -o python_files="integration/test_*.py integration/*/test_*.py"
 
 test-sdk-integration-containers:
@@ -176,7 +176,7 @@ test-sdk: test-sdk-unit test-sdk-integration-containers
 # `test-sdk-integration-containers` is usually called and this will either
 # start them if they are not present or use the currently running ones.
 test-backend-unit:
-	cd lumigator/python/mzai/backend/; \
+	cd lumigator/backend/; \
 	S3_BUCKET=lumigator-storage \
 	RAY_HEAD_NODE_HOST=localhost \
 	RAY_DASHBOARD_PORT=8265 \
@@ -185,7 +185,7 @@ test-backend-unit:
 	uv run $(DEBUGPY_ARGS) -m pytest -s -o python_files="backend/tests/unit/*/test_*.py backend/tests/unit/test_*.py"
 
 test-backend-integration:
-	cd lumigator/python/mzai/backend/; \
+	cd lumigator/backend/; \
 	docker container list --all; \
 	S3_BUCKET=lumigator-storage \
 	RAY_HEAD_NODE_HOST=localhost \
@@ -217,11 +217,11 @@ test-backend: test-backend-unit test-backend-integration-containers
 # be running, but they will set up a different, volatile python environment
 # with all the deps specified in their respective `requirements.txt` files.
 test-jobs-evaluation-unit:
-	cd lumigator/python/mzai/jobs/evaluator_lite; \
+	cd lumigator/jobs/evaluator_lite; \
 	uv run $(DEBUGPY_ARGS) --with pytest --with-requirements requirements.txt --isolated pytest
 
 test-jobs-inference-unit:
-	cd lumigator/python/mzai/jobs/inference; \
+	cd lumigator/jobs/inference; \
 	uv run $(DEBUGPY_ARGS) --with pytest --with-requirements requirements.txt --isolated pytest
 
 test-jobs-unit: test-jobs-evaluation-unit test-jobs-inference-unit

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -125,14 +125,14 @@ services:
       - AWS_DEFAULT_REGION
       - AWS_ENDPOINT_URL
       - S3_BUCKET
-      - EVALUATOR_PIP_REQS=/mzai/lumigator/python/mzai/jobs/evaluator/requirements.txt
-      - EVALUATOR_WORK_DIR=/mzai/lumigator/python/mzai/jobs/evaluator
+      - EVALUATOR_PIP_REQS=/mzai/lumigator/jobs/evaluator/requirements.txt
+      - EVALUATOR_WORK_DIR=/mzai/lumigator/jobs/evaluator
       # TODO: the following two rows should be renamed to EVALUATOR_*
       #       and the two above should be removed when we depreate evaluator
-      - EVALUATOR_LITE_PIP_REQS=/mzai/lumigator/python/mzai/jobs/evaluator_lite/requirements.txt
-      - EVALUATOR_LITE_WORK_DIR=/mzai/lumigator/python/mzai/jobs/evaluator_lite
-      - INFERENCE_PIP_REQS=/mzai/lumigator/python/mzai/jobs/inference/requirements.txt
-      - INFERENCE_WORK_DIR=/mzai/lumigator/python/mzai/jobs/inference
+      - EVALUATOR_LITE_PIP_REQS=/mzai/lumigator/jobs/evaluator_lite/requirements.txt
+      - EVALUATOR_LITE_WORK_DIR=/mzai/lumigator/jobs/evaluator_lite
+      - INFERENCE_PIP_REQS=/mzai/lumigator/jobs/inference/requirements.txt
+      - INFERENCE_WORK_DIR=/mzai/lumigator/jobs/inference
       - RAY_DASHBOARD_PORT
       - RAY_HEAD_NODE_HOST
       - MISTRAL_API_KEY=$MISTRAL_API_KEY

diff --git a/docs/Makefile b/docs/Makefile
@@ -7,6 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build
+GIT_COMMIT    ?= $(shell git rev-parse HEAD)
 
 # Put it first so that "make" without argument is like "make help".
 help: