push.yml

name: Build

on:
  push:
    paths:
      - '.github/actions/smoke.sh'
      - '.github/workflows/push.yml'
      - '.github/workflows/master.yml'
      - 'packages/**'
      - 'rust/cubestore/js-wrapper/**'
      - 'rust/cubestore/tsconfig.json'
      - 'rust/cubestore/package.json'
      - 'rust/cubesql/**'
      - '.eslintrc.js'
      - '.prettierrc'
      - 'package.json'
      - 'lerna.json'
      - 'rollup.config.js'
      - 'yarn.lock'
    branches:
      - 'master'
  pull_request:
    paths:
      - '.github/workflows/push.yml'
      - '.github/workflows/master.yml'
      - 'packages/**'
      - 'rust/cubestore/js-wrapper/**'
      - 'rust/cubestore/tsconfig.json'
      - 'rust/cubestore/package.json'
      - 'rust/cubesql/**'
      - '.eslintrc.js'
      - '.prettierrc'
      - 'package.json'
      - 'lerna.json'
      - 'rollup.config.js'
      - 'yarn.lock'

jobs:
  unit:
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    needs: latest-tag-sha
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        node-version: [16.x, 18.x]
      fail-fast: false

    steps:
      - id: get-tag-out
        run: echo "$OUT"
        env:
          OUT: ${{ needs['latest-tag-sha'].outputs.sha }}
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # pulls all commits (needed for codecov)
          fetch-depth: 2
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-12-13
          override: true
          components: rustfmt
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Build client
        run: yarn build
      - name: Lerna test
        run: yarn lerna run --concurrency 1 --stream --no-prefix unit
      - uses: codecov/codecov-action@v1
        if: (matrix.node-version == '16.x')
        with:
          files: ./packages/*/coverage/clover.xml
          flags: cube-backend
          verbose: true # optional (default = false)

  lint:
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    needs: latest-tag-sha
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-12-13
          override: true
          components: rustfmt
      - name: Install Node.js 16.x
        uses: actions/setup-node@v3
        with:
          node-version: 16.x
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: NPM lint
        run: yarn lint:npm
      - name: Lerna lint
        run: yarn lerna run --concurrency 1 lint

  build:
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    needs: latest-tag-sha
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-12-13
          override: true
          components: rustfmt
      - name: Install Node.js 16.x
        uses: actions/setup-node@v3
        with:
          node-version: 16.x
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Check Yarn lock wasn't modified
        run: if [ "$(git status | grep nothing)x" = "x" ]; then echo "Non empty changeset after lerna bootstrap"; git status; exit 1; else echo "Nothing to commit. Proceeding"; fi;
      - name: Build Core Client libraries
        run: yarn build
      - name: Build other packages
        run: yarn lerna run --concurrency 1 build
        env:
          NODE_OPTIONS: --max_old_space_size=4096

  build-cubestore:
    needs: [latest-tag-sha]
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ./rust/cubestore -> target
          # Separate path for release key to protect cache bloating
          shared-key: cubestore-release
          key: ubuntu-20.04
      - name: Build cubestore
        uses: actions-rs/cargo@v1
        with:
          command: build
          args: --manifest-path rust/cubestore/Cargo.toml -j 4 --release
      - name: 'Upload cubestored-x86_64-unknown-linux-gnu-release artifact'
        uses: actions/upload-artifact@v3
        with:
          name: cubestored-x86_64-unknown-linux-gnu-release
          path: ./rust/cubestore/target/release/cubestored
          retention-days: 5

  integration-cubestore:
    needs: [latest-tag-sha, build-cubestore]
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        node-version: [16.x]
      fail-fast: false

    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
        uses: actions/download-artifact@v3
        with:
          path: ./rust/cubestore/target/release/
          name: cubestored-x86_64-unknown-linux-gnu-release
      - name: Run Cube Store in background
        run: |
          chmod +x ./rust/cubestore/target/release/cubestored
          RUNNER_TRACKING_ID="" && ./rust/cubestore/target/release/cubestored &
      - name: Run Cubestore Integration
        timeout-minutes: 10
        run: |
          yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore

  integration:
    needs: [unit, lint, latest-tag-sha]
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        node-version: [16.x]
        db: [
            'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
            'mysql-aurora-serverless', 'crate', 'mongobi'
        ]
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-12-13
          override: true
          components: rustfmt
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Run Integration tests for ${{ matrix.db }} matrix
        timeout-minutes: 30
        run: ./.github/actions/integration/${{ matrix.db }}.sh

  integration-smoke:
    needs: [ latest-tag-sha, build-cubestore ]
    runs-on: ubuntu-20.04
    timeout-minutes: 90
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        node-version: [ 16.x ]
      fail-fast: false

    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Install instant client for Oracle
        uses: GoodManWEN/oracle-client-action@main
      - name: Build client
        run: yarn build
      - name: Lerna tsc
        run: yarn tsc
      - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
        uses: actions/download-artifact@v3
        with:
          path: rust/cubestore/downloaded/latest/bin/
          name: cubestored-x86_64-unknown-linux-gnu-release
      - name: Chmod +x for cubestored
        run: |
          chmod +x ./rust/cubestore/downloaded/latest/bin/cubestored
      - name: Run Integration smoke tests
        timeout-minutes: 30
        run: ./.github/actions/smoke.sh

  docker-image-latest-set-tag:
    # At least git should be completed pushed up until this moment
    needs: [lint, latest-tag-sha]
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    runs-on: ubuntu-20.04
    outputs:
      tag: ${{ steps.get-tag.outputs.tag }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - id: get-tag
        run: echo "::set-output name=tag::$(git tag --contains $GITHUB_SHA)"
        env:
          GITHUB_SHA: ${{ github.sha }}

  latest-tag-sha:
    runs-on: ubuntu-20.04
    outputs:
      sha: ${{ steps.get-tag.outputs.sha }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - id: git-log
        run: git log HEAD~30..HEAD
      - id: get-tag-test
        run: echo "$SHA $(git rev-list -n 1 $(git tag --contains $SHA))"
        env:
          SHA: ${{ github.sha }}
      - id: get-tag
        run: echo "::set-output name=sha::$(git rev-list -n 1 $(git tag --contains $SHA))"
        env:
          SHA: ${{ github.sha }}
      - id: get-tag-out
        run: echo "$OUT"
        env:
          OUT: ${{ steps.get-tag.outputs.sha }}

  docker-dev:
    needs: [latest-tag-sha]
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    name: Build & Test :dev for ${{ matrix.name }} without pushing
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    strategy:
      matrix:
        dockerfile:
          - dev.Dockerfile
        include:
          - dockerfile: dev.Dockerfile
            name: Debian
            tag: tmp-dev
      fail-fast: false
    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: v0.9.1
          driver-opts: network=host
      - name: Build image
        uses: docker/build-push-action@v3
        timeout-minutes: 30
        with:
          context: .
          file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
          platforms: linux/amd64
          push: true
          tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
      - name: Use Node.js 16.x
        uses: actions/setup-node@v3
        with:
          node-version: 16.x
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v3
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.19
      - name: Yarn install
        uses: nick-invision/retry@v2
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Build client
        run: yarn build
      - name: Lerna tsc
        run: yarn tsc
      - name: Testing CubeJS (container mode) via BirdBox
        run: |
          cd packages/cubejs-testing/
          export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
          export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
          export DEBUG=testcontainers
          yarn run dataset:minimal
          yarn run birdbox:postgresql
          yarn run birdbox:postgresql-pre-aggregations
      # - name: Testing Athena driver (container mode) via BirdBox
      #   env:
      #     CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
      #     CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
      #     CUBEJS_AWS_REGION: us-east-1
      #     CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output
      #     CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export
      #   run: |
      #     cd packages/cubejs-testing/
      #     export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
      #     export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
      #     export DEBUG=testcontainers
      #     yarn run driver:athena --log=ignore --mode=docker
      # - name: Testing BigQuery driver (container mode) via BirdBox
      #   env:
      #     CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
      #     CUBEJS_DB_BQ_PROJECT_ID: cube-open-source
      #     CUBEJS_DB_EXPORT_BUCKET: cube-open-source-export-bucket
      #   run: |
      #     cd packages/cubejs-testing/
      #     export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
      #     export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
      #     export DEBUG=testcontainers
      #     yarn run driver:bigquery --log=ignore --mode=docker
      - name: Testing PostgreSQL driver (container mode) via BirdBox
        env:
          CUBEJS_DB_TYPE: postgres
          CUBEJS_DB_USER: postgres
          CUBEJS_DB_PASS: postgres
        run: |
          cd packages/cubejs-testing/
          export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
          export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
          export DEBUG=testcontainers
          yarn run driver:postgres --log=ignore --mode=docker
      - name: Testing Docker image via Cypress (Chrome)
        env:
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
          BIRDBOX_CYPRESS_UPDATE_SCREENSHOTS: ${{ contains(github.event.head_commit.message, '[update screenshots]') }}
        run: |
          cd packages/cubejs-testing/
          export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
          export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
          export BIRDBOX_CYPRESS_BROWSER=chrome
          export BIRDBOX_CYPRESS_TARGET=postgresql
          export DEBUG=testcontainers
          yarn run cypress:install
          yarn run cypress:birdbox
      - name: Upload screenshots on failure
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: cypress-screenshots-docker-dev-${{ matrix.name }}
          path: packages/cubejs-testing/cypress/screenshots