diff --git a/.github/release-please-config.json b/.github/release-please-config.json index 6cf564cdb..8f475204d 100644 --- a/.github/release-please-config.json +++ b/.github/release-please-config.json @@ -26,6 +26,11 @@ "path": "**/zarf.yaml", "glob": true }, + { + "type": "generic", + "path": "**/zarf-config.yaml", + "glob": true + }, { "type": "generic", "path": "**/uds-bundle.yaml", diff --git a/.github/workflows/e2e-llama-cpp-python.yaml b/.github/workflows/e2e-llama-cpp-python.yaml index 416146b78..b3019819f 100644 --- a/.github/workflows/e2e-llama-cpp-python.yaml +++ b/.github/workflows/e2e-llama-cpp-python.yaml @@ -57,6 +57,11 @@ jobs: runs-on: ai-ubuntu-big-boy-8-core if: ${{ !github.event.pull_request.draft }} + permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -70,6 +75,7 @@ jobs: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Setup API and Supabase uses: ./.github/actions/lfai-core diff --git a/.github/workflows/e2e-playwright.yaml b/.github/workflows/e2e-playwright.yaml index cf2db1d47..ddf9da1c8 100644 --- a/.github/workflows/e2e-playwright.yaml +++ b/.github/workflows/e2e-playwright.yaml @@ -58,6 +58,11 @@ jobs: runs-on: ai-ubuntu-big-boy-8-core if: ${{ !github.event.pull_request.draft }} + permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -83,6 +88,7 @@ jobs: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Create Test User run: | diff --git a/.github/workflows/e2e-text-backend-full-cpu.yaml b/.github/workflows/e2e-text-backend-full-cpu.yaml index 7cb282d47..9e7faf01f 100644 --- a/.github/workflows/e2e-text-backend-full-cpu.yaml +++ b/.github/workflows/e2e-text-backend-full-cpu.yaml @@ -58,6 +58,11 @@ jobs: runs-on: ai-ubuntu-big-boy-8-core if: ${{ !github.event.pull_request.draft }} + permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -70,6 +75,8 @@ jobs: with: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Setup LFAI-API and Supabase uses: ./.github/actions/lfai-core @@ -98,5 +105,7 @@ jobs: # Test ########## - name: Test Text Backend + env: + LEAPFROGAI_MODEL: llama-cpp-python run: | python -m pytest ./tests/e2e/test_text_backend_full.py -v diff --git a/.github/workflows/e2e-text-embeddings.yaml b/.github/workflows/e2e-text-embeddings.yaml index f60919724..3742de352 100644 --- a/.github/workflows/e2e-text-embeddings.yaml +++ b/.github/workflows/e2e-text-embeddings.yaml @@ -59,6 +59,11 @@ jobs: runs-on: ai-ubuntu-big-boy-8-core if: ${{ !github.event.pull_request.draft }} + permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -72,6 +77,7 @@ jobs: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Setup LFAI-API and Supabase uses: ./.github/actions/lfai-core diff --git a/.github/workflows/e2e-vllm.yaml b/.github/workflows/e2e-vllm.yaml index e808f15ea..ace153006 100644 --- a/.github/workflows/e2e-vllm.yaml +++ b/.github/workflows/e2e-vllm.yaml @@ -59,6 +59,11 @@ jobs: runs-on: ai-ubuntu-big-boy-8-core if: ${{ !github.event.pull_request.draft }} + permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -74,7 +79,7 @@ jobs: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} - udsCliVersion: 0.14.0 + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} ########## # vLLM diff --git a/.github/workflows/e2e-whisper.yaml b/.github/workflows/e2e-whisper.yaml index a7cd17e14..90e94106e 100644 --- a/.github/workflows/e2e-whisper.yaml +++ b/.github/workflows/e2e-whisper.yaml @@ -57,6 +57,11 @@ jobs: runs-on: ai-ubuntu-big-boy-8-core if: ${{ !github.event.pull_request.draft }} + permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -72,6 +77,7 @@ jobs: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Setup LFAI-API and Supabase uses: ./.github/actions/lfai-core diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 21d2e1985..fec72192b 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -31,7 +31,10 @@ on: - "!packages/ui/**" # Declare default permissions as read only. -permissions: read-all +permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. concurrency: group: pytest-integration-${{ github.ref }} @@ -98,6 +101,7 @@ jobs: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Setup API and Supabase uses: ./.github/actions/lfai-core diff --git a/.github/workflows/e2e-registry1-weekly.yaml b/.github/workflows/weekly-registry1-flavor-test.yaml similarity index 56% rename from .github/workflows/e2e-registry1-weekly.yaml rename to .github/workflows/weekly-registry1-flavor-test.yaml index 65f4c5897..21d799c9b 100644 --- a/.github/workflows/e2e-registry1-weekly.yaml +++ b/.github/workflows/weekly-registry1-flavor-test.yaml @@ -1,8 +1,8 @@ -name: e2e-registry1-weekly +name: weekly-registry1-flavor-test on: schedule: - - cron: "0 0 * * 6" # Run every Sunday at 12 AM EST + - cron: "0 8 * * 0" # Run every Sunday at 12 AM PST workflow_dispatch: # trigger manually as needed pull_request: types: @@ -12,11 +12,11 @@ on: - ready_for_review # don't run on draft PRs - milestoned # allows us to trigger on bot PRs paths: - - .github/workflows/e2e-registry1-weekly.yaml + - .github/workflows/weekly-registry1-flavor-test.yaml - bundles/latest/** concurrency: - group: e2e-registry1-weekly-${{ github.ref }} + group: weekly-registry1-flavor-test-${{ github.ref }} cancel-in-progress: true defaults: @@ -24,67 +24,98 @@ defaults: shell: bash jobs: - test-flavors: + registry1-flavor-test: runs-on: ai-ubuntu-big-boy-8-core - name: e2e_registry1_weekly + name: weekly_registry1_flavor_test if: ${{ !github.event.pull_request.draft }} permissions: contents: read - packages: write + packages: read id-token: write # This is needed for OIDC federation. steps: - - name: Checkout Repo + # Checkout main just to see the latest release in the release-please manifest + - name: Checkout Repo (main) uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - # x-release-please-start-version - ref: "caf4f9c3093a55a003b49fcbf05c03221be6a232" # 0.12.2 w/ integration tests turned-on - # x-release-please-end + ref: main - - name: Setup Python - uses: ./.github/actions/python + - name: Get Latest Release Version + id: get_version + run: | + LFAI_VERSION=$(jq -r '.["."]' .github/.release-please-manifest.json) + echo "LFAI_VERSION=$LFAI_VERSION" >> $GITHUB_OUTPUT - - name: Install API and SDK Dev Dependencies - run : | - make install + ################ + # LATEST RELEASE + ################ + + - name: Checkout Repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-tags: true + ref: v${{ steps.get_version.outputs.LFAI_VERSION }} - - name: Setup UDS Cluster - uses: ./.github/actions/uds-cluster + - name: Setup UDS Environment + uses: defenseunicorns/uds-common/.github/actions/setup@24c8a2a48eeb33773b76b3587c489cb17496c9e0 # v0.12.0 with: registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} ghToken: ${{ secrets.GITHUB_TOKEN }} - udsCliVersion: 0.14.0 + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - - name: Create UDS Cluster - shell: bash + - name: Setup Python + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c #v5.0.0 + with: + python-version-file: "pyproject.toml" + + - name: Install Python Dependencies + run: pip install ".[dev]" "src/leapfrogai_api" "src/leapfrogai_sdk" --no-cache-dir + + - name: Mutation of the Zarf Packages run: | - UDS_CONFIG=.github/config/uds-config.yaml make create-uds-cpu-cluster + uds zarf tools yq -i ' + .components[].images[0] |= sub(":v[0-9\.]+$", ":v${{ steps.get_version.outputs.LFAI_VERSION }}") + ' packages/api/zarf.yaml + uds zarf tools yq -i '.api.image.tag = "v${{ steps.get_version.outputs.LFAI_VERSION }}"' packages/api/values/registry1-values.yaml - - name: Setup Playwright + - name: Print the Modified Zarf Packages run: | - npm --prefix src/leapfrogai_ui ci - npx --prefix src/leapfrogai_ui playwright install + cat packages/api/zarf.yaml + cat packages/api/values/registry1-values.yaml - - name: Create Registry1 Packages + - name: Create Registry1 Zarf Packages run: | - LOCAL_VERSION=registry1 FLAVOR=registry1 make build-api + uds zarf package create packages/api --set image_version="${{ steps.get_version.outputs.LFAI_VERSION }}" --flavor registry1 -a amd64 --confirm # Mutate UDS bundle definition to use Registry1 packages - - name: Mutation to Registry1 Bundle - # TODO: fix bundle path + # Mutate non-Registry1 packages to be the current tagged version + - name: Mutation of the UDS Bundle run: | - uds zarf tools yq -i '.packages[1] |= del(.repository)' bundles/latest/cpu/uds-bundle.yaml - uds zarf tools yq -i '.packages[1] |= .ref = "registry1"' bundles/latest/cpu/uds-bundle.yaml - uds zarf tools yq -i '.packages[1] |= .path = "../../../packages/api"' bundles/latest/cpu/uds-bundle.yaml uds zarf tools yq -i '.metadata.version = "registry1"' bundles/latest/cpu/uds-bundle.yaml - - name: Create and Deploy Bundle + uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ steps.get_version.outputs.LFAI_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml + + uds zarf tools yq -i '.packages[1] |= del(.repository)' bundles/latest/cpu/uds-bundle.yaml + uds zarf tools yq -i '.packages[1] |= .ref = "${{ steps.get_version.outputs.LFAI_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml + uds zarf tools yq -i '.packages[1] |= .path = "../../../"' bundles/latest/cpu/uds-bundle.yaml + + - name: Print the Modified UDS Bundle + run: | + cat bundles/latest/cpu/uds-config.yaml + cat bundles/latest/cpu/uds-bundle.yaml + + - name: Create UDS Cluster + shell: bash + run: | + UDS_CONFIG=.github/config/uds-config.yaml make create-uds-cpu-cluster + + - name: Create and Deploy Registry1 Bundle run: | cd bundles/latest/cpu uds create . --confirm && \ - uds deploy uds-bundle-leapfrogai-amd64-registry1.tar.zst --confirm --no-progress && \ + uds deploy uds-bundle-leapfrogai-amd64-registry1.tar.zst --confirm --no-progress --log-level debug && \ rm -rf uds-bundle-leapfrogai-amd64-registry1.tar.zst && \ docker system prune -af @@ -107,32 +138,19 @@ jobs: echo "ANON_KEY is set: ${{ steps.generate_secrets.outputs.ANON_KEY != '' }}" echo "SERVICE_KEY is set: ${{ steps.generate_secrets.outputs.SERVICE_KEY != '' }}" - - name: Run Integration Tests - env: - SUPABASE_ANON_KEY: ${{ steps.generate_secrets.outputs.ANON_KEY }} - SUPABASE_PASS: ${{ steps.generate_secrets.outputs.FAKE_PASSWORD }} - SUPABASE_EMAIL: integration@uds.dev - SUPABASE_URL: https://supabase-kong.uds.dev - # Turn off NIAH tests that are not applicable for integration testing using the Repeater model - LFAI_RUN_NIAH_TESTS: "false" - run: | - uds zarf connect --name=llama-cpp-python-model --namespace=leapfrogai --local-port=50051 --remote-port=50051 & - while ! nc -z localhost 50051; do sleep 1; done - - make test-user-pipeline - env $(cat .env | xargs) python -m pytest -v -s tests/integration/api - # Backends - name: Run Backend E2E Tests env: ANON_KEY: ${{ steps.generate_secrets.outputs.ANON_KEY }} SERVICE_KEY: ${{ steps.generate_secrets.outputs.SERVICE_KEY }} + LEAPFROGAI_MODEL: llama-cpp-python + run: | + python -m pytest -vvv -s ./tests/e2e + + - name: Setup Playwright run: | - python -m pytest ./tests/e2e/test_llama.py -vv - python -m pytest ./tests/e2e/test_text_embeddings.py -vv - python -m pytest ./tests/e2e/test_whisper.py -vv - python -m pytest ./tests/e2e/test_supabase.py -vv - python -m pytest ./tests/e2e/test_api.py -vv + npm --prefix src/leapfrogai_ui ci + npx --prefix src/leapfrogai_ui playwright install - name: Run Playwright E2E Tests env: @@ -156,3 +174,12 @@ jobs: name: playwright-report path: src/leapfrogai_ui/e2e-report/ retention-days: 30 + + - name: Get Cluster Debug Information + id: debug + if: ${{ !cancelled() }} + uses: defenseunicorns/uds-common/.github/actions/debug-output@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + + - name: Get Cluster Debug Information + if: ${{ !cancelled() && steps.debug.conclusion == 'success' }} + uses: defenseunicorns/uds-common/.github/actions/save-logs@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 diff --git a/Makefile b/Makefile index bf8afb315..ed74b5ccf 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ ARCH ?= amd64 +FLAVOR ?= upstream REG_PORT ?= 5000 REG_NAME ?= registry LOCAL_VERSION ?= $(shell git rev-parse --short HEAD) DOCKER_FLAGS := ZARF_FLAGS := -FLAVOR := upstream SILENT_DOCKER_FLAGS := --quiet SILENT_ZARF_FLAGS := --no-progress -l warn --no-color MAX_JOBS := 4 @@ -55,24 +55,34 @@ build-supabase: local-registry docker-supabase docker-api: local-registry sdk-wheel @echo $(DOCKER_FLAGS) @echo $(ZARF_FLAGS) -ifeq ($(FLAVOR),upstream) + ## Build the API image (and tag it for the local registry) docker build ${DOCKER_FLAGS} --platform=linux/${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -f packages/api/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -endif + ## Build the migration container for this version of the API docker build ${DOCKER_FLAGS} --platform=linux/${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" . docker tag ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -build-api: local-registry docker-api ## Build the leapfrogai_api container and Zarf package +## If registry1, don't locally Docker-build anything +ifeq ($(FLAVOR),upstream) + DOCKER_TARGETS := local-registry docker-api +else + DOCKER_TARGETS := +endif + +build-api: $(DOCKER_TARGETS) ## Build the leapfrogai_api container and Zarf package + ## Only push to local registry and build if this is an upstream-flavored package ifeq ($(FLAVOR),upstream) ## Push the images to the local registry (Zarf is super slow if the image is only in the local daemon) docker push ${DOCKER_FLAGS} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -endif docker push ${DOCKER_FLAGS} localhost:${REG_PORT}/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} - ## Build the Zarf package uds zarf package create packages/api --flavor ${FLAVOR} -a ${ARCH} -o packages/api --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} ${ZARF_FLAGS} --confirm +else + ## Build the registry1 Zarf package + ZARF_CONFIG=packages/api/zarf-config.yaml uds zarf package create packages/api --flavor ${FLAVOR} -a ${ARCH} -o packages/api ${ZARF_FLAGS} --confirm +endif docker-ui: ## Build the UI image (and tag it for the local registry) diff --git a/bundles/latest/cpu/uds-bundle.yaml b/bundles/latest/cpu/uds-bundle.yaml index 747645ae3..00327dbec 100644 --- a/bundles/latest/cpu/uds-bundle.yaml +++ b/bundles/latest/cpu/uds-bundle.yaml @@ -4,35 +4,35 @@ kind: UDSBundle metadata: name: leapfrogai description: A UDS bundle for deploying LeapfrogAI - version: 0.12.2-upstream + version: 0.13.1-upstream packages: # Supabase backend for the UI and API to interface with Postgresql - name: supabase repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/supabase - ref: 0.12.2-upstream + ref: 0.13.1-upstream # API - name: leapfrogai-api repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/leapfrogai-api - ref: 0.12.2-upstream + ref: 0.13.1-upstream # Chat Model - name: llama-cpp-python repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/llama-cpp-python - ref: 0.12.2-upstream + ref: 0.13.1-upstream # Text Embeddings Model - name: text-embeddings repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/text-embeddings - ref: 0.12.2-upstream + ref: 0.13.1-upstream # Transcription Model - name: whisper repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/whisper - ref: 0.12.2-upstream + ref: 0.13.1-upstream # UI - name: leapfrogai-ui repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/leapfrogai-ui - ref: 0.12.2-upstream + ref: 0.13.1-upstream diff --git a/bundles/latest/gpu/uds-bundle.yaml b/bundles/latest/gpu/uds-bundle.yaml index 3867749a4..ab2a9e0f5 100644 --- a/bundles/latest/gpu/uds-bundle.yaml +++ b/bundles/latest/gpu/uds-bundle.yaml @@ -4,35 +4,35 @@ kind: UDSBundle metadata: name: leapfrogai description: A UDS bundle for deploying LeapfrogAI - version: 0.12.2-upstream + version: 0.13.1-upstream packages: # Supabase backend for the UI and API to interface with Postgresql - name: supabase repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/supabase - ref: 0.12.2-upstream + ref: 0.13.1-upstream # OpenAI-like API - name: leapfrogai-api repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/leapfrogai-api - ref: 0.12.2-upstream + ref: 0.13.1-upstream # Model for generic chat and summarization - name: vllm repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/vllm - ref: 0.12.2-upstream + ref: 0.13.1-upstream # Model for providing vector embeddings for text - name: text-embeddings repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/text-embeddings - ref: 0.12.2-upstream + ref: 0.13.1-upstream # Model for converting audio to text - name: whisper repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/whisper - ref: 0.12.2-upstream + ref: 0.13.1-upstream # UI - name: leapfrogai-ui repository: ghcr.io/defenseunicorns/packages/uds/leapfrogai/leapfrogai-ui - ref: 0.12.2-upstream + ref: 0.13.1-upstream diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 7aa8d5c02..9fefb8a7e 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -127,6 +127,7 @@ uds zarf tools registry prune --confirm # create and deploy the new package # FLAVOR can be upstream (default) or registry1 - see README for availability details +# See individual sub-directories for any flavor-specific instructions (e.g., packages/api/README.md) LOCAL_VERSION=dev FLAVOR=upstream REGISTRY_PORT=5000 ARCH=amd64 make build-api LOCAL_VERSION=dev FLAVOR=upstream REGISTRY_PORT=5000 ARCH=amd64 make deploy-api ``` @@ -153,6 +154,7 @@ uds zarf package deploy zarf-package-*.tar.zst --confirm ```bash # FLAVOR can be upstream (default) or registry1 - see README for availability details + # See individual sub-directories for any flavor-specific instructions (e.g., packages/api/README.md) LOCAL_VERSION=dev FLAVOR=upstream ARCH=amd64 make build-cpu # ui, api, llama-cpp-python, text-embeddings, whisper, supabase # OR LOCAL_VERSION=dev FLAVOR=upstream ARCH=amd64 make build-gpu # ui, api, vllm, text-embeddings, whisper, supabase @@ -166,6 +168,7 @@ uds zarf package deploy zarf-package-*.tar.zst --confirm ```bash # FLAVOR can be upstream (default) or registry1 - see README for availability details + # See individual sub-directories for any flavor-specific instructions (e.g., packages/api/README.md) LOCAL_VERSION=dev FLAVOR=upstream ARCH=amd64 make build-ui LOCAL_VERSION=dev FLAVOR=upstream ARCH=amd64 make build-api LOCAL_VERSION=dev FLAVOR=upstream ARCH=amd64 make build-supabase @@ -200,7 +203,7 @@ Although not provided in the example UDS bundle manifests found in this reposito - name: leapfrogai-api repository: ghcr.io/defenseunicorns/packages/leapfrogai/leapfrogai-api # x-release-please-start-version - ref: 0.12.2 + ref: 0.13.1 # x-release-please-end # THE BELOW LINES WERE ADDED FOR DEMONSTRATION PURPOSES @@ -234,6 +237,7 @@ To demonstrate what this would look like for an Apple Silicon Mac: ```bash # FLAVOR can be upstream (default) or registry1 - see README for availability details +# See individual sub-directories for any flavor-specific instructions (e.g., packages/api/README.md) REG_PORT=5001 ARCH=arm64 LOCAL_VERSION=dev FLAVOR=upstream make build-cpu ``` @@ -241,6 +245,7 @@ To demonstrate what this would look like for an older Intel Mac: ```bash # FLAVOR can be upstream (default) or registry1 - see README for availability details +# See individual sub-directories for any flavor-specific instructions (e.g., packages/api/README.md) REG_PORT=5001 ARCH=arm64 LOCAL_VERSION=dev FLAVOR=upstream make build-cpu ``` diff --git a/mk-clean.mk b/mk-clean.mk index ff7e8c61d..4ca00ae89 100644 --- a/mk-clean.mk +++ b/mk-clean.mk @@ -15,8 +15,8 @@ clean-artifacts: # Zarf packages, UDS bundles, Python build artifacts, etc. clean-cache: -rm -rf ./**/__pycache__ ./**/*/__pycache__ ./**/**/*/__pycache__ - -rm -rf ./**/*/.ruff_cache ./**/.ruff_cache - -rm -rf ./**/.pytest_cache ./**/*/.pytest_cache + -rm -rf ./.ruff_cache ./**/*/.ruff_cache ./**/.ruff_cache + -rm -rf ./.pytest_cache ./**/.pytest_cache ./**/*/.pytest_cache -rm -rf ./.mypy_cache clean-env: diff --git a/packages/api/README.md b/packages/api/README.md index aa2b34690..2d68d67f8 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -27,6 +27,13 @@ make build-api LOCAL_VERSION=dev FLAVOR=upstream uds zarf package deploy packages/api/zarf-package-leapfrogai-api-*-dev.tar.zst --confirm ``` +For other package flavors, use the following example: + +```bash +make build-api FLAVOR=registry1 +uds zarf package deploy packages/api/zarf-package-leapfrogai-api-*-dev.tar.zst --confirm +``` + ### Local Development See the [source code documentation](../../src/leapfrogai_api/README.md) for running the API from the source code for local Python environment development. diff --git a/packages/api/values/registry1-values.yaml b/packages/api/values/registry1-values.yaml index 4bd35ee39..91f92b168 100644 --- a/packages/api/values/registry1-values.yaml +++ b/packages/api/values/registry1-values.yaml @@ -1,9 +1,7 @@ api: image: repository: "registry1.dso.mil/ironbank/opensource/defenseunicorns/leapfrogai/api" - # x-release-please-start-version - tag: v0.12.2 - # x-release-please-end + tag: v###ZARF_CONST_IMAGE_VERSION### expose: "###ZARF_VAR_EXPOSE_API###" diff --git a/packages/api/zarf-config.yaml b/packages/api/zarf-config.yaml new file mode 100644 index 000000000..475ac2d48 --- /dev/null +++ b/packages/api/zarf-config.yaml @@ -0,0 +1,6 @@ +package: + create: + set: + # x-release-please-start-version + image_version: "0.13.1" + # x-release-please-end diff --git a/packages/api/zarf.yaml b/packages/api/zarf.yaml index 92b3c8123..51e0b5f38 100644 --- a/packages/api/zarf.yaml +++ b/packages/api/zarf.yaml @@ -50,7 +50,7 @@ components: valuesFiles: - "values/registry1-values.yaml" images: - - "registry1.dso.mil/ironbank/opensource/defenseunicorns/leapfrogai/api:v0.12.2" + - "registry1.dso.mil/ironbank/opensource/defenseunicorns/leapfrogai/api:v###ZARF_PKG_TMPL_IMAGE_VERSION###" # TODO: replace with Ironbank image once hardened: registry1.dso.mil/ironbank/opensource/defenseunicorns/leapfrogai/api/migrations - "ghcr.io/defenseunicorns/leapfrogai/api-migrations:###ZARF_PKG_TMPL_IMAGE_VERSION###" - "registry1.dso.mil/ironbank/kiwigrid/k8s-sidecar:1.23.3" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 4f498b102..580034011 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,12 +1,14 @@ +from openai import OpenAI import pytest -from openai import OpenAI +from tests.utils.client import leapfrogai_client, get_leapfrogai_model -from .utils import create_test_user + +@pytest.fixture(scope="module") +def client() -> OpenAI: + return leapfrogai_client() @pytest.fixture(scope="module") -def client(): - return OpenAI( - base_url="https://leapfrogai-api.uds.dev/openai/v1", api_key=create_test_user() - ) +def model_name() -> str: + return get_leapfrogai_model() diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index b556954e0..44e533645 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -5,7 +5,7 @@ import pytest as pytest import requests -from .utils import create_test_user +from tests.utils.client import create_test_user logger = logging.getLogger(__name__) test_id = str(uuid.uuid4()) diff --git a/tests/e2e/test_llm_generation.py b/tests/e2e/test_llm_generation.py index badb0dd3e..cb309d597 100644 --- a/tests/e2e/test_llm_generation.py +++ b/tests/e2e/test_llm_generation.py @@ -1,41 +1,28 @@ -import os from typing import Iterable -import warnings import pytest from openai import InternalServerError, OpenAI from openai.types.chat import ChatCompletionMessageParam from tests.utils.data_path import data_path, WAV_FILE -DEFAULT_LEAPFROGAI_MODEL = "llama-cpp-python" - - -def get_model_name(): - model_name = os.getenv("LEAPFROGAI_MODEL") - if model_name is None: - warnings.warn( - f"LEAPFROGAI_MODEL environment variable not set. Defaulting to '{DEFAULT_LEAPFROGAI_MODEL}'.\n" - "Consider setting LEAPFROGAI_MODEL explicitly. Examples: 'vllm', 'repeater', 'llama-cpp-python'." - ) - model_name = DEFAULT_LEAPFROGAI_MODEL - return model_name - - -@pytest.fixture -def model_name(): - return get_model_name() +# Test generation parameters +SYSTEM_PROMPT = "You are a helpful assistant." +USER_PROMPT = "Only return 1 word" +MAX_TOKENS = 128 +TEMPERATURE = 0 def test_chat_completions(client: OpenAI, model_name: str): messages: Iterable[ChatCompletionMessageParam] = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"}, + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": USER_PROMPT}, ] chat_completion = client.chat.completions.create( model=model_name, messages=messages, - max_tokens=128, + max_tokens=MAX_TOKENS, + temperature=TEMPERATURE, ) assert chat_completion.model == model_name assert len(chat_completion.choices) == 1 @@ -51,8 +38,9 @@ def test_chat_completions(client: OpenAI, model_name: str): def test_completions(client: OpenAI, model_name: str): completion = client.completions.create( model=model_name, - prompt="Only return 1 word", - max_tokens=128, + prompt=USER_PROMPT, + max_tokens=MAX_TOKENS, + temperature=TEMPERATURE, ) assert completion.model == model_name assert len(completion.choices) == 1 diff --git a/tests/e2e/test_supabase.py b/tests/e2e/test_supabase.py index 1e98f2ec4..c9302c6be 100644 --- a/tests/e2e/test_supabase.py +++ b/tests/e2e/test_supabase.py @@ -17,7 +17,7 @@ from leapfrogai_api.data.crud_vector_store_file import CRUDVectorStoreFile -from .utils import ANON_KEY, create_test_user, SERVICE_KEY +from tests.utils.client import ANON_KEY, create_test_user, SERVICE_KEY from openai.types import FileObject health_urls = { diff --git a/tests/e2e/test_text_backend_full.py b/tests/e2e/test_text_backend_full.py index fdee17172..d1f28bcf4 100644 --- a/tests/e2e/test_text_backend_full.py +++ b/tests/e2e/test_text_backend_full.py @@ -21,7 +21,7 @@ def download_arxiv_pdf(): ) -def test_run_with_background_task(client: OpenAI): +def test_run_with_background_task(client: OpenAI, model_name: str): """ This test confirms whether a vector store for an assistant can index files while chatting at the same time. @@ -52,7 +52,7 @@ def test_run_with_background_task(client: OpenAI): # Create an assistant assistant = client.beta.assistants.create( - model="llama-cpp-python", + model=model_name, name="Test Assistant", instructions="You are a helpful assistant with access to a knowledge base about AI and machine learning.", tools=[{"type": "file_search"}], diff --git a/tests/e2e/utils.py b/tests/e2e/utils.py deleted file mode 100644 index 32eb8daff..000000000 --- a/tests/e2e/utils.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -import logging -import os -import traceback -import pytest -import requests - -# This is the anon_key for supabase, it provides access to the endpoints that would otherwise be inaccessible -ANON_KEY = os.environ["ANON_KEY"] -SERVICE_KEY = os.environ["SERVICE_KEY"] -DEFAULT_TEST_EMAIL = "fakeuser1@test.com" -DEFAULT_TEST_PASSWORD = "password" - - -def create_test_user( - anon_key: str = ANON_KEY, - email: str = DEFAULT_TEST_EMAIL, - password: str = DEFAULT_TEST_PASSWORD, -) -> str: - headers = { - "apikey": f"{anon_key}", - "Authorization": f"Bearer {anon_key}", - "Content-Type": "application/json", - } - - try: - requests.post( - url="https://supabase-kong.uds.dev/auth/v1/signup", - headers=headers, - json={ - "email": email, - "password": password, - "confirmPassword": password, - }, - ) - except Exception: - logging.error( - "Error creating user (likely because the user already exists): %s", - traceback.format_exc(), - ) - - return get_jwt_token(anon_key, email, password) - - -def get_jwt_token( - api_key: str, - test_email: str = DEFAULT_TEST_EMAIL, - test_password: str = DEFAULT_TEST_PASSWORD, -) -> str: - url = "https://supabase-kong.uds.dev/auth/v1/token?grant_type=password" - headers = {"apikey": f"{api_key}", "Content-Type": "application/json"} - data = {"email": test_email, "password": test_password} - - response = requests.post(url, headers=headers, json=data) - if response.status_code != 200: - pytest.fail( - f"Request for the JWT token failed with status code {response.status_code} expected 200", - False, - ) - - return json.loads(response.content)["access_token"] diff --git a/tests/integration/api/test_vector_stores.py b/tests/integration/api/test_vector_stores.py index 5427a0943..9a3be72a4 100644 --- a/tests/integration/api/test_vector_stores.py +++ b/tests/integration/api/test_vector_stores.py @@ -1,7 +1,6 @@ """Test the API endpoints for assistants.""" import json -import os import time import pytest @@ -19,6 +18,7 @@ ) from leapfrogai_api.routers.openai.vector_stores import router as vector_store_router from leapfrogai_api.routers.openai.files import router as files_router +from tests.utils.client import create_test_user from tests.utils.data_path import data_path, TXT_FILE INSTRUCTOR_XL_EMBEDDING_SIZE: int = 768 @@ -37,11 +37,11 @@ class MissingEnvironmentVariable(Exception): headers: dict[str, str] = {} try: - headers = {"Authorization": f"Bearer {os.environ['SUPABASE_USER_JWT']}"} + headers = {"Authorization": f"Bearer {create_test_user()}"} except KeyError as exc: raise MissingEnvironmentVariable( "SUPABASE_USER_JWT must be defined for the test to pass. " - "Please check the api README for instructions on obtaining this token." + "Please check the packages/api and src/leapfrogai_api READMEs for instructions on obtaining this token." ) from exc vector_store_client = TestClient(vector_store_router, headers=headers) diff --git a/tests/utils/client.py b/tests/utils/client.py index 6fe598514..0016f8c4c 100644 --- a/tests/utils/client.py +++ b/tests/utils/client.py @@ -1,8 +1,113 @@ +import json +import logging +import traceback from urllib.parse import urljoin from openai import OpenAI import os +import pytest import requests from requests import Response +from fastapi import status + +ANON_KEY = os.environ["ANON_KEY"] +SERVICE_KEY = os.environ["SERVICE_KEY"] +DEFAULT_TEST_EMAIL = "test-user@test.com" +DEFAULT_TEST_PASSWORD = "password" + + +def get_supabase_url() -> str: + """Get the URL for Supabase. + + Returns: + str: The URL for Supabase. (default: "https://supabase-kong.uds.dev") + """ + + return os.getenv("SUPABASE_URL", "https://supabase-kong.uds.dev") + + +def create_test_user( + anon_key: str = ANON_KEY, + email: str = DEFAULT_TEST_EMAIL, + password: str = DEFAULT_TEST_PASSWORD, +) -> str: + """ + Create a test user in the authentication system. + + This function attempts to create a new user with the given email and password using the specified + anonymous API key. If the user already exists, the error is logged. It returns the JWT token + for the created or existing user. + + Args: + anon_key (str): The anonymous API key for authentication service. + email (str): The email address of the test user. Default is "fakeuser1@test.com". + password (str): The password for the test user. Default is "password". + + Returns: + str: The JWT token for the created or existing user. + """ + supabase_base_url = get_supabase_url() + + headers = { + "apikey": f"{anon_key}", + "Authorization": f"Bearer {anon_key}", + "Content-Type": "application/json", + } + + try: + requests.post( + url=f"{supabase_base_url}/auth/v1/signup", + headers=headers, + json={ + "email": email, + "password": password, + "confirmPassword": password, + }, + ) + except Exception: + logging.error( + "Error creating user (likely because the user already exists): %s", + traceback.format_exc(), + ) + + return get_jwt_token(supabase_base_url, anon_key, email, password) + + +def get_jwt_token( + supabase_base_url: str, + api_key: str, + test_email: str = DEFAULT_TEST_EMAIL, + test_password: str = DEFAULT_TEST_PASSWORD, +) -> str: + """ + Retrieve a JWT token for a test user using email and password. + + This function sends a request to the authentication service to obtain a JWT token using + the provided API key, email, and password. + + Args: + api_key (str): The API key for the authentication service. + test_email (str): The email address of the test user. Default is "fakeuser1@test.com". + test_password (str): The password for the test user. Default is "password". + + Returns: + str: The JWT access token for the authenticated user. + + Raises: + AssertionError: If the request fails or the response status code is not 200. + """ + + url = f"{supabase_base_url}/auth/v1/token?grant_type=password" + headers = {"apikey": f"{api_key}", "Content-Type": "application/json"} + data = {"email": test_email, "password": test_password} + + response = requests.post(url, headers=headers, json=data) + if response.status_code != status.HTTP_200_OK: + pytest.fail( + f"Request for the JWT token failed with status code {response.status_code} expected 200", + False, + ) + + return json.loads(response.content)["access_token"] def get_leapfrogai_model() -> str: @@ -12,7 +117,15 @@ def get_leapfrogai_model() -> str: str: The model to use for LeapfrogAI. (default: "vllm") """ - return os.getenv("LEAPFROGAI_MODEL", "vllm") + model = os.getenv("LEAPFROGAI_MODEL") + + if not model: + model = "vllm" + logging.warning( + f"LEAPFROGAI_MODEL is not set, using default model of `{model}`" + ) + + return model def get_openai_key() -> str: @@ -49,14 +162,18 @@ def get_leapfrogai_api_key() -> str: Returns: str: The API key for the LeapfrogAI API. + Raises: ValueError: If LEAPFROGAI_API_KEY or SUPABASE_USER_JWT is not set. """ api_key = os.getenv("LEAPFROGAI_API_KEY") or os.getenv("SUPABASE_USER_JWT") - if api_key is None: - raise ValueError("LEAPFROGAI_API_KEY or SUPABASE_USER_JWT not set") + if not api_key: + logging.warning( + "LEAPFROGAI_API_KEY or SUPABASE_USER_JWT not set, automatically generating test user." + ) + return create_test_user() return api_key @@ -74,9 +191,9 @@ def get_leapfrogai_api_url() -> str: def get_leapfrogai_api_url_base() -> str: """Get the base URL for the LeapfrogAI API. - Set via the LEAPFRAGAI_API_URL environment variable. + Set via the LEAPFROGAI_API_URL environment variable. - If LEAPFRAGAI_API_URL is set to "https://leapfrogai-api.uds.dev/openai/v1", this will trim off the "/openai/v1" part. + If LEAPFROGAI_API_URL is set to "https://leapfrogai-api.uds.dev/openai/v1", this will trim off the "/openai/v1" part. Returns: str: The base URL for the LeapfrogAI API. (default: "https://leapfrogai-api.uds.dev")