diff --git a/.ci/compose.ci.yaml b/.ci/compose.ci.yaml deleted file mode 100644 index dd6f3a0e..00000000 --- a/.ci/compose.ci.yaml +++ /dev/null @@ -1,25 +0,0 @@ -services: - redash: - build: ../ - command: manage version - depends_on: - - postgres - - redis - ports: - - "5000:5000" - environment: - PYTHONUNBUFFERED: 0 - REDASH_LOG_LEVEL: "INFO" - REDASH_REDIS_URL: "redis://redis:6379/0" - POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb" - REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres" - REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF" - redis: - image: redis:7-alpine - restart: unless-stopped - postgres: - image: pgautoupgrade/pgautoupgrade:15-alpine3.8 - command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" - restart: unless-stopped - environment: - POSTGRES_HOST_AUTH_METHOD: "trust" diff --git a/.ci/compose.cypress.yaml b/.ci/compose.cypress.yaml deleted file mode 100644 index e47ec92e..00000000 --- a/.ci/compose.cypress.yaml +++ /dev/null @@ -1,73 +0,0 @@ -x-redash-service: &redash-service - build: - context: ../ - args: - install_groups: "main" - code_coverage: ${CODE_COVERAGE} -x-redash-environment: &redash-environment - REDASH_LOG_LEVEL: "INFO" - REDASH_REDIS_URL: "redis://redis:6379/0" - POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb" - REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres" - REDASH_RATELIMIT_ENABLED: "false" - REDASH_ENFORCE_CSRF: "true" - REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF" -services: - server: - <<: *redash-service - command: server - depends_on: - - postgres - - redis - ports: - - "5000:5000" - environment: - <<: *redash-environment - PYTHONUNBUFFERED: 0 - scheduler: - <<: *redash-service - command: scheduler - depends_on: - - server - environment: - <<: *redash-environment - worker: - <<: *redash-service - command: worker - depends_on: - - server - environment: - <<: *redash-environment - PYTHONUNBUFFERED: 0 - cypress: - ipc: host - build: - context: ../ - dockerfile: .ci/Dockerfile.cypress - depends_on: - - server - - worker - - scheduler - environment: - CYPRESS_baseUrl: "http://server:5000" - CYPRESS_coverage: ${CODE_COVERAGE} - PERCY_TOKEN: ${PERCY_TOKEN} - PERCY_BRANCH: ${CIRCLE_BRANCH} - PERCY_COMMIT: ${CIRCLE_SHA1} - PERCY_PULL_REQUEST: ${CIRCLE_PR_NUMBER} - COMMIT_INFO_BRANCH: ${CIRCLE_BRANCH} - COMMIT_INFO_MESSAGE: ${COMMIT_INFO_MESSAGE} - COMMIT_INFO_AUTHOR: ${CIRCLE_USERNAME} - COMMIT_INFO_SHA: ${CIRCLE_SHA1} - COMMIT_INFO_REMOTE: ${CIRCLE_REPOSITORY_URL} - CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID} - CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY} - redis: - image: redis:7-alpine - restart: unless-stopped - postgres: - image: pgautoupgrade/pgautoupgrade:15-alpine3.8 - command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" - restart: unless-stopped - environment: - POSTGRES_HOST_AUTH_METHOD: "trust" diff --git a/.ci/docker_build b/.ci/docker_build deleted file mode 100755 index 324c7e99..00000000 --- a/.ci/docker_build +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# This script only needs to run on the main Redash repo - -if [ "${GITHUB_REPOSITORY}" != "getredash/redash" ]; then - echo "Skipping image build for Docker Hub, as this isn't the main Redash repository" - exit 0 -fi - -if [ "${GITHUB_REF_NAME}" != "master" ] && [ "${GITHUB_REF_NAME}" != "preview-image" ]; then - echo "Skipping image build for Docker Hub, as this isn't the 'master' nor 'preview-image' branch" - exit 0 -fi - -if [ "x${DOCKER_USER}" = "x" ] || [ "x${DOCKER_PASS}" = "x" ]; then - echo "Skipping image build for Docker Hub, as the login details aren't available" - exit 0 -fi - -set -e -VERSION=$(jq -r .version package.json) -VERSION_TAG="$VERSION.b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}" - -export DOCKER_BUILDKIT=1 -export COMPOSE_DOCKER_CLI_BUILD=1 - -docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" - -DOCKERHUB_REPO="redash/redash" -DOCKER_TAGS="-t redash/redash:preview -t redash/preview:${VERSION_TAG}" - -# Build the docker container -docker build --build-arg install_groups="main,all_ds,dev" ${DOCKER_TAGS} . - -# Push the container to the preview build locations -docker push "${DOCKERHUB_REPO}:preview" -docker push "redash/preview:${VERSION_TAG}" - -echo "Built: ${VERSION_TAG}" diff --git a/.ci/pack b/.ci/pack deleted file mode 100755 index 16223c5a..00000000 --- a/.ci/pack +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -NAME=redash -VERSION=$(jq -r .version package.json) -FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM -FILENAME=$NAME.$FULL_VERSION.tar.gz - -mkdir -p /tmp/artifacts/ - -tar -zcv -f /tmp/artifacts/$FILENAME --exclude=".git" --exclude="optipng*" --exclude="cypress" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" * diff --git a/.ci/update_version b/.ci/update_version deleted file mode 100755 index 53b53720..00000000 --- a/.ci/update_version +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VERSION=$(jq -r .version package.json) -FULL_VERSION=${VERSION}+b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER} - -sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py -sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json diff --git a/.dockerignore b/.dockerignore index 8e3dfae1..b5a2c33e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ client/.tmp/ -client/dist/ node_modules/ viz-lib/node_modules/ .tmp/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c42929c..798fb4aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,20 @@ on: - master env: NODE_VERSION: 18 + REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF" + COMPOSE_DOCKER_CLI_BUILD: 1 + DOCKER_BUILDKIT: 1 + FRONTEND_BUILD_MODE: 1 + INSTALL_GROUPS: main,all_ds,dev + SERVER_MOUNT: /ignore + PERCY_BRANCH: ${{github.head_ref || github.ref_name}} + PERCY_COMMIT: ${{github.sha}} + PERCY_PULL_REQUEST: ${{github.event.number}} + COMMIT_INFO_BRANCH: ${{github.head_ref || github.ref_name}} + COMMIT_INFO_MESSAGE: ${{github.event.head_commit.message}} + COMMIT_INFO_AUTHOR: ${{github.event.pull_request.user.login}} + COMMIT_INFO_SHA: ${{github.sha}} + COMMIT_INFO_REMOTE: ${{github.server_url}}/${{github.repository}} jobs: backend-lint: runs-on: ubuntu-22.04 @@ -19,10 +33,10 @@ jobs: with: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - - run: sudo pip install black==23.1.0 ruff==0.0.287 + - run: sudo pip install black==23.12.1 ruff==0.1.9 - run: ruff check . - run: black --check . @@ -30,10 +44,7 @@ jobs: runs-on: ubuntu-22.04 needs: backend-lint env: - COMPOSE_FILE: .ci/compose.ci.yaml - COMPOSE_PROJECT_NAME: redash - COMPOSE_DOCKER_CLI_BUILD: 1 - DOCKER_BUILDKIT: 1 + FRONTEND_BUILD_MODE: 0 steps: - if: github.event.pull_request.mergeable == 'false' name: Exit if PR is not mergeable @@ -45,15 +56,15 @@ jobs: - name: Build Docker Images run: | set -x - docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true + docker compose build docker compose up -d sleep 10 - name: Create Test Database - run: docker compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;" + run: docker compose run --rm postgres psql -h postgres -U postgres -c "create database tests;" - name: List Enabled Query Runners - run: docker compose -p redash run --rm redash manage ds list_types + run: docker compose run --rm server manage ds list_types - name: Run Tests - run: docker compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/ + run: docker compose run --name tests server tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/ - name: Copy Test Results run: | mkdir -p /tmp/test-results/unit-tests @@ -62,14 +73,14 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 - name: Store Test Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: test-results + name: backend-test-results path: /tmp/test-results - name: Store Coverage Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: coverage + name: backend-coverage path: coverage.xml frontend-lint: @@ -82,20 +93,21 @@ jobs: with: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} cache: 'yarn' - name: Install Dependencies run: | npm install --global --force yarn@1.22.19 - yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + yarn cache clean + yarn --frozen-lockfile --network-concurrency 1 - name: Run Lint run: yarn lint:ci - name: Store Test Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: test-results + name: frontend-test-results path: /tmp/test-results frontend-unit-tests: @@ -109,31 +121,32 @@ jobs: with: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} cache: 'yarn' - name: Install Dependencies run: | npm install --global --force yarn@1.22.19 - yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + yarn cache clean + yarn --frozen-lockfile --network-concurrency 1 - name: Run App Tests run: yarn test - name: Run Visualizations Tests - run: cd viz-lib && yarn test + run: | + cd viz-lib + yarn test - run: yarn lint frontend-e2e-tests: runs-on: ubuntu-22.04 needs: frontend-lint env: - COMPOSE_FILE: .ci/compose.cypress.yaml - COMPOSE_PROJECT_NAME: cypress CYPRESS_INSTALL_BINARY: 0 PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + INSTALL_GROUPS: main + COMPOSE_PROFILES: e2e + REDASH_PRODUCTION: "true" steps: - if: github.event.pull_request.mergeable == 'false' name: Exit if PR is not mergeable @@ -142,7 +155,7 @@ jobs: with: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} cache: 'yarn' @@ -153,10 +166,12 @@ jobs: - name: Install Dependencies run: | npm install --global --force yarn@1.22.19 - yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + yarn cache clean + yarn --frozen-lockfile --network-concurrency 1 - name: Setup Redash Server run: | set -x + yarn build yarn cypress build yarn cypress start -- --skip-db-seed docker compose run cypress yarn cypress db-seed @@ -168,7 +183,12 @@ jobs: - name: Copy Code Coverage Results run: docker cp cypress:/usr/src/app/coverage ./coverage || true - name: Store Coverage Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: coverage + name: frontend-coverage path: coverage + - uses: actions/upload-artifact@v4 + with: + name: frontend + path: client/dist + retention-days: 1 diff --git a/.github/workflows/preview-image.yml b/.github/workflows/preview-image.yml index cf8b0241..a7b3abe2 100644 --- a/.github/workflows/preview-image.yml +++ b/.github/workflows/preview-image.yml @@ -1,12 +1,11 @@ name: Preview Image on: - push: + workflow_run: + workflows: + - Tests tags: - '*-dev' -env: - NODE_VERSION: 18 - jobs: build-skip-check: runs-on: ubuntu-22.04 @@ -32,56 +31,97 @@ jobs: runs-on: ubuntu-22.04 needs: - build-skip-check + outputs: + version: ${{ steps.version.outputs.VERSION_TAG }} if: needs.build-skip-check.outputs.skip == 'false' + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 steps: - uses: actions/checkout@v4 with: fetch-depth: 1 ref: ${{ github.event.push.after }} - - - uses: actions/setup-node@v4 + - uses: actions/download-artifact@v4 with: - node-version: ${{ env.NODE_VERSION }} - cache: 'yarn' - - - name: Install Dependencies - run: | - npm install --global --force yarn@1.22.19 - yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - + name: frontend + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ github.event.workflow_run }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ vars.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} - - name: Set version id: version run: | set -x - .ci/update_version - VERSION_TAG=$(jq -r .version package.json) - echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT" - + VERSION=$(jq -r .version package.json) + FULL_VERSION=${VERSION}+b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER} + sed -ri "s/^__version__ = ([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py + sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json + echo "VERSION_TAG=$FULL_VERSION" >> "$GITHUB_OUTPUT" - name: Build and push preview image to Docker Hub - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: push: true - tags: | - redash/redash:preview - redash/preview:${{ steps.version.outputs.VERSION_TAG }} context: . - build-args: | - test_all_deps=true cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64 + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + build-args: | + FRONTEND_BUILD_MODE=1 env: DOCKER_CONTENT_TRUST: true + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 - - name: "Failure: output container logs to console" - if: failure() - run: docker compose logs + publish-docker-manifest: + runs-on: ubuntu-22.04 + needs: + - build-skip-check + - build-docker-image + if: needs.build-skip-check.outputs.skip == 'false' + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + name: digests + path: /tmp/digests + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + tags: preview + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} + - name: Push image to redash/preview + run: | + docker buildx imagetools create \ + --tag redash/preview:${{ needs.build-docker-image.outputs.version }} \ + ${{ env.REGISTRY_IMAGE }}:preview diff --git a/Dockerfile b/Dockerfile index 95b82915..84812919 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,38 @@ -FROM node:18-bookworm as frontend-builder +# Controls whether to build the frontend assets +ARG FRONTEND_BUILD_MODE=0 -RUN npm install --global --force yarn@1.22.19 +# MODE 0: create empty files. useful for backend tests +FROM alpine:3.19 as frontend-builder-0 +RUN \ + mkdir -p /frontend/client/dist && \ + touch /frontend/client/dist/multi_org.html && \ + touch /frontend/client/dist/index.html -# Controls whether to build the frontend assets -ARG skip_frontend_build +# MODE 1: copy static frontend from host, useful for CI to ignore building static content multiple times +FROM alpine:3.19 as frontend-builder-1 +COPY client/dist /frontend/client/dist +# MODE 2: build static content in docker, can be used for a local development +FROM node:18.1-bookworm as frontend-builder-2 +RUN npm install --global --force yarn@1.22.19 ENV CYPRESS_INSTALL_BINARY=0 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 - RUN useradd -m -d /frontend redash USER redash - WORKDIR /frontend COPY --chown=redash package.json yarn.lock .yarnrc /frontend/ COPY --chown=redash viz-lib /frontend/viz-lib COPY --chown=redash scripts /frontend/scripts # Controls whether to instrument code for coverage information -ARG code_coverage -ENV BABEL_ENV=${code_coverage:+test} - -RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn --frozen-lockfile --network-concurrency 1; fi - +ARG CODE_COVERAGE +ENV BABEL_ENV=${CODE_COVERAGE:+test} +RUN yarn --frozen-lockfile --network-concurrency 1; COPY --chown=redash client /frontend/client COPY --chown=redash webpack.config.js /frontend/ -RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi +RUN yarn build + +FROM frontend-builder-${FRONTEND_BUILD_MODE} as frontend-builder FROM python:3.8-slim-bookworm @@ -91,8 +99,8 @@ COPY pyproject.toml poetry.lock ./ ARG POETRY_OPTIONS="--no-root --no-interaction --no-ansi" # for LDAP authentication, install with `ldap3` group # disabled by default due to GPL license conflict -ARG install_groups="main,all_ds,dev" -RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS +ARG INSTALL_GROUPS="main,all_ds,dev" +RUN /etc/poetry/bin/poetry install --only $INSTALL_GROUPS $POETRY_OPTIONS COPY --chown=redash . /app COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist diff --git a/.ci/Dockerfile.cypress b/Dockerfile.cypress similarity index 100% rename from .ci/Dockerfile.cypress rename to Dockerfile.cypress diff --git a/Makefile b/Makefile index aa9d750e..166595d7 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,13 @@ .PHONY: compose_build up test_db create_database clean down tests lint backend-unit-tests frontend-unit-tests test build watch start redis-cli bash +export COMPOSE_DOCKER_CLI_BUILD=1 +export DOCKER_BUILDKIT=1 + compose_build: .env - COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose build + docker compose build up: - COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose up -d --build + docker compose up -d --build test_db: @for i in `seq 1 5`; do \ diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 5e777c10..575365b8 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -4,22 +4,37 @@ set -e scheduler() { echo "Starting RQ scheduler..." - exec /app/manage.py rq scheduler -} - -dev_scheduler() { - echo "Starting dev RQ scheduler..." - - exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq scheduler + case $REDASH_PRODUCTION in + true) + echo "Starting RQ scheduler in production mode" + exec ./manage.py rq scheduler + ;; + *) + echo "Starting RQ scheduler in dev mode" + exec watchmedo auto-restart \ + --directory=./redash/ \ + --pattern=*.py \ + --recursive -- ./manage.py rq scheduler $QUEUES + ;; + esac } worker() { - echo "Starting RQ worker..." - export WORKERS_COUNT=${WORKERS_COUNT:-2} export QUEUES=${QUEUES:-} - - exec supervisord -c worker.conf + case $REDASH_PRODUCTION in + true) + echo "Starting RQ worker in production mode" + exec supervisord -c worker.conf + ;; + *) + echo "Starting RQ worker in dev mode" + exec watchmedo auto-restart \ + --directory=./redash/ \ + --pattern=*.py \ + --recursive -- ./manage.py rq worker $QUEUES + ;; + esac } workers_healthcheck() { @@ -35,18 +50,28 @@ workers_healthcheck() { fi } -dev_worker() { - echo "Starting dev RQ worker..." - - exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq worker $QUEUES -} - server() { # Recycle gunicorn workers every n-th request. See http://docs.gunicorn.org/en/stable/settings.html#max-requests for more details. - MAX_REQUESTS=${MAX_REQUESTS:-1000} - MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100} - TIMEOUT=${REDASH_GUNICORN_TIMEOUT:-60} - exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app --max-requests $MAX_REQUESTS --max-requests-jitter $MAX_REQUESTS_JITTER --timeout $TIMEOUT + case $REDASH_PRODUCTION in + true) + echo "Starting Redash Server in production mode" + MAX_REQUESTS=${MAX_REQUESTS:-1000} + MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100} + TIMEOUT=${REDASH_GUNICORN_TIMEOUT:-60} + exec /usr/local/bin/gunicorn \ + -b 0.0.0.0:5000 \ + --name redash \ + -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app \ + --max-requests $MAX_REQUESTS \ + --max-requests-jitter $MAX_REQUESTS_JITTER \ + --timeout $TIMEOUT + ;; + *) + echo "Starting Redash Server in a dev mode" + export FLASK_DEBUG=1 + exec /app/manage.py runserver --debugger --reload -h 0.0.0.0 + ;; + esac } create_db() { @@ -61,12 +86,9 @@ help() { echo "server -- start Redash server (with gunicorn)" echo "worker -- start a single RQ worker" - echo "dev_worker -- start a single RQ worker with code reloading" echo "scheduler -- start an rq-scheduler instance" - echo "dev_scheduler -- start an rq-scheduler instance with code reloading" echo "" echo "shell -- open shell" - echo "dev_server -- start Flask development server with debugger and auto reload" echo "debug -- start Flask development server with remote debugger via ptvsd" echo "create_db -- create database tables" echo "manage -- CLI to manage redash" @@ -101,22 +123,10 @@ case "$1" in shift scheduler ;; - dev_scheduler) - shift - dev_scheduler - ;; - dev_worker) - shift - dev_worker - ;; celery_healthcheck) shift echo "DEPRECATED: Celery has been replaced with RQ and now performs healthchecks autonomously as part of the 'worker' entrypoint." ;; - dev_server) - export FLASK_DEBUG=1 - exec /app/manage.py runserver --debugger --reload -h 0.0.0.0 - ;; debug) export FLASK_DEBUG=1 export REMOTE_DEBUG=1 diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js index 320402f1..ce6cb49e 100644 --- a/client/cypress/cypress.js +++ b/client/cypress/cypress.js @@ -43,18 +43,18 @@ function seedDatabase(seedValues) { function buildServer() { console.log("Building the server..."); - execSync("docker compose -p cypress build", { stdio: "inherit" }); + execSync("docker compose build", { stdio: "inherit" }); } function startServer() { console.log("Starting the server..."); - execSync("docker compose -p cypress up -d", { stdio: "inherit" }); - execSync("docker compose -p cypress run server create_db", { stdio: "inherit" }); + execSync("docker compose up -d", { stdio: "inherit" }); + execSync("docker compose run server create_db", { stdio: "inherit" }); } function stopServer() { console.log("Stopping the server..."); - execSync("docker compose -p cypress down", { stdio: "inherit" }); + execSync("docker compose down", { stdio: "inherit" }); } function runCypressCI() { @@ -68,7 +68,7 @@ function runCypressCI() { } execSync( - "COMMIT_INFO_MESSAGE=$(git show -s --format=%s) docker compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run $CYPRESS_OPTIONS", + "docker compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run $CYPRESS_OPTIONS", { stdio: "inherit" } ); } diff --git a/compose.base.yml b/compose.base.yml new file mode 100644 index 00000000..b743d7c2 --- /dev/null +++ b/compose.base.yml @@ -0,0 +1,23 @@ +version: "3.8" +services: + .redash: + build: + context: . + args: + FRONTEND_BUILD_MODE: ${FRONTEND_BUILD_MODE:-2} + INSTALL_GROUPS: ${INSTALL_GROUPS:-main,all_ds,dev} + CODE_COVERAGE: ${CODE_COVERAGE:-"false"} + volumes: + - $PWD:${SERVER_MOUNT:-/app} + command: manage version + environment: + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres" + REDASH_RATELIMIT_ENABLED: "false" + REDASH_MAIL_DEFAULT_SENDER: "redash@example.com" + REDASH_MAIL_SERVER: "email" + REDASH_MAIL_PORT: "1025" + REDASH_ENFORCE_CSRF: "true" + REDASH_COOKIE_SECRET: ${REDASH_COOKIE_SECRET} + REDASH_PRODUCTION: ${REDASH_PRODUCTION} diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index ba1b0e17..00000000 --- a/compose.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# This configuration file is for the **development** setup. -# For a production example please refer to getredash/setup repository on GitHub. -x-redash-service: &redash-service - build: - context: . - args: - skip_frontend_build: "true" # set to empty string to build - volumes: - - .:/app - env_file: - - .env -x-redash-environment: &redash-environment - REDASH_LOG_LEVEL: "INFO" - REDASH_REDIS_URL: "redis://redis:6379/0" - REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres" - REDASH_RATELIMIT_ENABLED: "false" - REDASH_MAIL_DEFAULT_SENDER: "redash@example.com" - REDASH_MAIL_SERVER: "email" - REDASH_MAIL_PORT: 1025 - REDASH_ENFORCE_CSRF: "true" - REDASH_GUNICORN_TIMEOUT: 60 - # Set secret keys in the .env file -services: - server: - <<: *redash-service - command: dev_server - depends_on: - - postgres - - redis - ports: - - "5001:5000" - - "5678:5678" - environment: - <<: *redash-environment - PYTHONUNBUFFERED: 0 - scheduler: - <<: *redash-service - command: dev_scheduler - depends_on: - - server - environment: - <<: *redash-environment - worker: - <<: *redash-service - command: dev_worker - depends_on: - - server - environment: - <<: *redash-environment - PYTHONUNBUFFERED: 0 - redis: - image: redis:7-alpine - restart: unless-stopped - postgres: - image: pgautoupgrade/pgautoupgrade:15-alpine3.8 - ports: - - "15432:5432" - # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3 - # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for - # tests. - command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" - restart: unless-stopped - environment: - POSTGRES_HOST_AUTH_METHOD: "trust" - email: - image: maildev/maildev - ports: - - "1080:1080" - - "1025:1025" - restart: unless-stopped diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..6cc5dbda --- /dev/null +++ b/compose.yml @@ -0,0 +1,80 @@ +services: + server: + extends: + file: compose.base.yml + service: .redash + command: server + depends_on: + - postgres + - redis + ports: + - "5000:5000" + - "5678:5678" + environment: + PYTHONUNBUFFERED: 0 + scheduler: + extends: + file: compose.base.yml + service: .redash + profiles: + - e2e + - local + command: scheduler + depends_on: + - server + worker: + extends: + file: compose.base.yml + service: .redash + profiles: + - e2e + - local + command: worker + depends_on: + - server + environment: + PYTHONUNBUFFERED: 0 + redis: + image: redis:7-alpine + restart: unless-stopped + postgres: + image: pgautoupgrade/pgautoupgrade:15-alpine3.8 + ports: + - "15432:5432" + # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3 + # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for + # tests. + command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" + restart: unless-stopped + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" + email: + image: maildev/maildev + ports: + - "1080:1080" + - "1025:1025" + restart: unless-stopped + cypress: + ipc: host + build: + context: . + dockerfile: Dockerfile.cypress + profiles: + - e2e + depends_on: + - server + - worker + - scheduler + environment: + CYPRESS_baseUrl: "http://server:5000" + PERCY_TOKEN: ${PERCY_TOKEN} + PERCY_BRANCH: ${PERCY_BRANCH} + PERCY_COMMIT: ${PERCY_COMMIT} + PERCY_PULL_REQUEST: ${PERCY_PULL_REQUEST} + COMMIT_INFO_BRANCH: ${COMMIT_INFO_BRANCH} + COMMIT_INFO_MESSAGE: ${COMMIT_INFO_MESSAGE} + COMMIT_INFO_AUTHOR: ${COMMIT_INFO_AUTHOR} + COMMIT_INFO_SHA: ${COMMIT_INFO_SHA} + COMMIT_INFO_REMOTE: ${COMMIT_INFO_REMOTE} + CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID} + CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY}