From 524675b725cf784900999ea6e221eb7cd6a359a8 Mon Sep 17 00:00:00 2001 From: Paul Fisher Date: Fri, 15 Jul 2022 16:41:51 -0400 Subject: [PATCH] Unify TileDB installation scripts and cache outputs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change unifies the build scripts between Mac OS and Linux, and caches build outputs. This means that future builds of the same version don’t need to rerun the entire build process; they can use the previously-built version. We do a few things to accomplish this: - Move compilation/installation into the `~/tiledb-core/install` directory. This allows us to cache it. - Set appropriate `CGO_*FLAGS` variables to point to that directory. This includes the `rpath` directive, which embeds the full path to the dynamic library in the binary, so it knows it needs to look in `~/tiledb-core/install/lib` for `libtiledb.so`. - Matrixes things out to reduce repetition. --- .github/scripts/env.sh | 16 ++ .github/scripts/install_tiledb_binary.sh | 8 + .github/scripts/install_tiledb_linux.sh | 4 - .github/scripts/install_tiledb_linux_debug.sh | 8 - .github/scripts/install_tiledb_macos.sh | 3 - .github/scripts/install_tiledb_source.sh | 23 ++ .../scripts/install_tiledb_source_linux.sh | 8 - .../scripts/install_tiledb_source_macos.sh | 7 - .github/scripts/post_install.sh | 12 + .github/workflows/tiledb-go.yml | 232 ++++++++---------- 10 files changed, 157 insertions(+), 164 deletions(-) create mode 100644 .github/scripts/env.sh create mode 100755 .github/scripts/install_tiledb_binary.sh delete mode 100755 .github/scripts/install_tiledb_linux.sh delete mode 100755 .github/scripts/install_tiledb_linux_debug.sh delete mode 100755 .github/scripts/install_tiledb_macos.sh create mode 100755 .github/scripts/install_tiledb_source.sh delete mode 100755 .github/scripts/install_tiledb_source_linux.sh delete mode 100755 .github/scripts/install_tiledb_source_macos.sh create mode 100755 .github/scripts/post_install.sh diff --git a/.github/scripts/env.sh b/.github/scripts/env.sh new file mode 100644 index 00000000..efbde5eb --- /dev/null +++ b/.github/scripts/env.sh @@ -0,0 +1,16 @@ +# Subroutine to set up environment variables. + +CORE_ROOT="$HOME/tiledb-core" + +case "$(uname -s)" in + Linux) + OS="linux" + ;; + Darwin) + OS="macos" + ;; + *) + echo 'Unknown OS!' + exit 1 + ;; +esac diff --git a/.github/scripts/install_tiledb_binary.sh b/.github/scripts/install_tiledb_binary.sh new file mode 100755 index 00000000..f8837573 --- /dev/null +++ b/.github/scripts/install_tiledb_binary.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" +mkdir "$CORE_ROOT" +cd "$CORE_ROOT" +mkdir install +curl --location -o tiledb.tar.gz "https://github.com/TileDB-Inc/TileDB/releases/download/${CORE_VERSION}/tiledb-${OS}-x86_64-${CORE_VERSION}-${CORE_HASH}.tar.gz" +tar -C ./install -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_linux.sh b/.github/scripts/install_tiledb_linux.sh deleted file mode 100755 index 049beda3..00000000 --- a/.github/scripts/install_tiledb_linux.sh +++ /dev/null @@ -1,4 +0,0 @@ -set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.10.2/tiledb-linux-x86_64-2.10.2-9ab84f9.tar.gz \ -&& sudo tar -C /usr/local -xf tiledb.tar.gz -sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_linux_debug.sh b/.github/scripts/install_tiledb_linux_debug.sh deleted file mode 100755 index 301ec0f1..00000000 --- a/.github/scripts/install_tiledb_linux_debug.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j4 -sudo make -C tiledb install -sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_macos.sh b/.github/scripts/install_tiledb_macos.sh deleted file mode 100755 index 57ec38cb..00000000 --- a/.github/scripts/install_tiledb_macos.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.10.2/tiledb-macos-x86_64-2.10.2-9ab84f9.tar.gz \ -&& sudo tar -C /usr/local -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_source.sh b/.github/scripts/install_tiledb_source.sh new file mode 100755 index 00000000..007a200b --- /dev/null +++ b/.github/scripts/install_tiledb_source.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" +mkdir "$CORE_ROOT" +cd "$CORE_ROOT" +case "$OS" in + linux) + NPROC="$(nproc)" + ;; + macos) + NPROC="$(sysctl -n hw.ncpu)" + ;; +esac +git clone --depth 1 -b "$CORE_VERSION" https://github.com/TileDB-Inc/TileDB.git +cd TileDB +mkdir build +cd build +# BUILD_FLAGS is unquoted because it needs to expand into multiple strings. +cmake $BUILD_FLAGS -DCMAKE_INSTALL_PREFIX="${CORE_ROOT}/install" .. +make -j"$NPROC" + +mkdir "${CORE_ROOT}/install" +make -C tiledb install diff --git a/.github/scripts/install_tiledb_source_linux.sh b/.github/scripts/install_tiledb_source_linux.sh deleted file mode 100755 index 146dc263..00000000 --- a/.github/scripts/install_tiledb_source_linux.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j$(nproc) -sudo make -C tiledb install -sudo ldconfig diff --git a/.github/scripts/install_tiledb_source_macos.sh b/.github/scripts/install_tiledb_source_macos.sh deleted file mode 100755 index 0e5e616c..00000000 --- a/.github/scripts/install_tiledb_source_macos.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j$(sysctl -n hw.ncpu) -sudo make -C tiledb install diff --git a/.github/scripts/post_install.sh b/.github/scripts/post_install.sh new file mode 100755 index 00000000..e0838bd6 --- /dev/null +++ b/.github/scripts/post_install.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" + +# Sets up the environment after installing or extracting TileDB. + +if [ "$OS" = "linux" ]; then + /sbin/ldconfig -n "${CORE_ROOT}/install/lib" +fi + +go env -w "CGO_CFLAGS=-I${CORE_ROOT}/install/include" +go env -w "CGO_LDFLAGS=-L${CORE_ROOT}/install/lib/ -Wl,-rpath,${CORE_ROOT}/install/lib/" diff --git a/.github/workflows/tiledb-go.yml b/.github/workflows/tiledb-go.yml index bb799473..f576a602 100644 --- a/.github/workflows/tiledb-go.yml +++ b/.github/workflows/tiledb-go.yml @@ -7,157 +7,121 @@ on: pull_request: branches: [ master ] +env: + # The version of TileDB to test against. + CORE_VERSION: "2.10.2" + # The first seven characters of that version's commit hash + # (the same ones that GitHub displays on a release page). + CORE_HASH: "9ab84f9" + jobs: golangci: name: lint runs-on: ubuntu-20.04 steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux.sh - shell: bash - - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - continue-on-error: true - with: - # Required: the version of golangci-lint is required and must be - # specified without patch version: we always use the latest patch version. - version: v1.35.2 - - Linux_Test: - runs-on: ubuntu-20.04 + # Checks out repository + - uses: actions/checkout@v2 + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: golangci-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_binary.sh') }} + + - name: Install TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_binary.sh' + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + continue-on-error: true + with: + # Required: the version of golangci-lint is required and must be + # specified without patch version: we always use the latest patch version. + version: v1.35.2 + + binary-test: + name: binary-${{ matrix.runner }}-go${{ matrix.go }} + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - # Will be checking following versions go: ["1.18"] - steps: + runner: ["macos-11", "ubuntu-20.04"] - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} + steps: - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -v ./... + - uses: actions/checkout@v2 - Macos_Test: - runs-on: macos-11 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] - steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_macos.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -v ./... - - Linux_Experimental_Test: - runs-on: ubuntu-20.04 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] - steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} - # Checks out repository - - uses: actions/checkout@v2 + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: golangci-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_binary.sh') }} - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_source_linux.sh - shell: bash + - name: Install TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_binary.sh' - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} + - name: Post-install + run: ./.github/scripts/post_install.sh - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -tags=experimental -v ./... + - name: Run tests + run: go test -v ./... - Macos_Experimental_Test: - runs-on: macos-11 + source-test: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - # Will be checking following versions - go: ["1.18"] - steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_source_macos.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - # Tests experimental bindings of TileDB-Go - - name: Test TileDB-Go - run: go test -tags=experimental -v ./... - - Linux_Address_Sanitizer: - runs-on: ubuntu-20.04 - strategy: - matrix: - # Will be checking following versions - go: [1.18] + include: + - name: macos-experimental + runner: macos-11 + flags: "-DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release" + test-command: go test -tags=experimental -v ./... + - name: linux-experimental + runner: ubuntu-20.04 + flags: "-DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release" + test-command: go test -tags=experimental -v ./... + - name: linux-asan + runner: ubuntu-20.04 + flags: "-DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug" + test-command: go run -tags asan ./cmd/tiledb-go-examples + steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux_debug.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - # Tests TileDB-Go - - name: Running examples using address sanitizer flags - continue-on-error: true - run: go run -tags asan ./cmd/tiledb-go-examples + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "1.18" + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: tiledb-binary-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_source.sh') }}-${{ matrix.flags }} + + - name: Compile TileDB Core + if: steps.build-cache.output.cache-hit != 'true' + run: './.github/scripts/install_tiledb_source.sh' + env: + BUILD_FLAGS: ${{ matrix.flags }} + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: Run test + run: ${{ matrix.test-command }}