From bd158ed2e333b002a9075ee507c6e7dd33c5e61b Mon Sep 17 00:00:00 2001 From: Paul Fisher Date: Fri, 15 Jul 2022 13:54:39 -0400 Subject: [PATCH 1/3] Remove Go dependency caching since we now vendor deps. --- .github/workflows/tiledb-go.yml | 66 --------------------------------- 1 file changed, 66 deletions(-) diff --git a/.github/workflows/tiledb-go.yml b/.github/workflows/tiledb-go.yml index 2955584c..bb799473 100644 --- a/.github/workflows/tiledb-go.yml +++ b/.github/workflows/tiledb-go.yml @@ -52,19 +52,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - # Tests TileDB-Go - name: Test TileDB-Go run: go test -v ./... @@ -91,19 +78,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - # Tests TileDB-Go - name: Test TileDB-Go run: go test -v ./... @@ -131,19 +105,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-experimental - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - # Tests TileDB-Go - name: Test TileDB-Go run: go test -tags=experimental -v ./... @@ -170,19 +131,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-experimental - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - # Tests experimental bindings of TileDB-Go - name: Test TileDB-Go run: go test -tags=experimental -v ./... @@ -209,20 +157,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-asan - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - # Tests TileDB-Go - name: Running examples using address sanitizer flags continue-on-error: true From 95977bb61c480ff951fdc22efb6cd9c8907e11af Mon Sep 17 00:00:00 2001 From: Paul Fisher Date: Fri, 15 Jul 2022 13:59:24 -0400 Subject: [PATCH 2/3] Remove unused `build_with_sanitizer_and_run` script. --- .github/scripts/build_with_sanitizer_and_run.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 .github/scripts/build_with_sanitizer_and_run.sh diff --git a/.github/scripts/build_with_sanitizer_and_run.sh b/.github/scripts/build_with_sanitizer_and_run.sh deleted file mode 100755 index 8291f7e1..00000000 --- a/.github/scripts/build_with_sanitizer_and_run.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -e -x -CGO_ENABLED=1 CGO_LDFLAGS='-fsanitize=address' CGO_CFLAGS='-fsanitize=address' go build cmd/tiledb-go/main.go -./main \ No newline at end of file From 1f9718e2183ed06ea74565cbf7b463a95e8564de Mon Sep 17 00:00:00 2001 From: Paul Fisher Date: Fri, 15 Jul 2022 16:41:51 -0400 Subject: [PATCH 3/3] Unify TileDB installation scripts and cache outputs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change unifies the build scripts between Mac OS and Linux, and caches build outputs. This means that future builds of the same version don’t need to rerun the entire build process; they can use the previously-built version. We do a few things to accomplish this: - Move compilation/installation into the `~/tiledb-core/install` directory. This allows us to cache it. - Set appropriate `CGO_*FLAGS` variables to point to that directory. This includes the `rpath` directive, which embeds the full path to the dynamic library in the binary, so it knows it needs to look in `~/tiledb-core/install/lib` for `libtiledb.so`. - Matrixes things out to reduce repetition. This also fixes the ASAN step, which was never actually compiling. --- .github/scripts/env.sh | 16 ++ .github/scripts/install_tiledb_binary.sh | 8 + .github/scripts/install_tiledb_linux.sh | 4 - .github/scripts/install_tiledb_linux_debug.sh | 8 - .github/scripts/install_tiledb_macos.sh | 3 - .github/scripts/install_tiledb_source.sh | 23 ++ .../scripts/install_tiledb_source_linux.sh | 8 - .../scripts/install_tiledb_source_macos.sh | 7 - .github/scripts/post_install.sh | 12 + .github/workflows/tiledb-go.yml | 232 ++++++++---------- cmd/tiledb-go-examples/asan_on.go | 7 + 11 files changed, 164 insertions(+), 164 deletions(-) create mode 100644 .github/scripts/env.sh create mode 100755 .github/scripts/install_tiledb_binary.sh delete mode 100755 .github/scripts/install_tiledb_linux.sh delete mode 100755 .github/scripts/install_tiledb_linux_debug.sh delete mode 100755 .github/scripts/install_tiledb_macos.sh create mode 100755 .github/scripts/install_tiledb_source.sh delete mode 100755 .github/scripts/install_tiledb_source_linux.sh delete mode 100755 .github/scripts/install_tiledb_source_macos.sh create mode 100755 .github/scripts/post_install.sh diff --git a/.github/scripts/env.sh b/.github/scripts/env.sh new file mode 100644 index 00000000..efbde5eb --- /dev/null +++ b/.github/scripts/env.sh @@ -0,0 +1,16 @@ +# Subroutine to set up environment variables. + +CORE_ROOT="$HOME/tiledb-core" + +case "$(uname -s)" in + Linux) + OS="linux" + ;; + Darwin) + OS="macos" + ;; + *) + echo 'Unknown OS!' + exit 1 + ;; +esac diff --git a/.github/scripts/install_tiledb_binary.sh b/.github/scripts/install_tiledb_binary.sh new file mode 100755 index 00000000..f8837573 --- /dev/null +++ b/.github/scripts/install_tiledb_binary.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" +mkdir "$CORE_ROOT" +cd "$CORE_ROOT" +mkdir install +curl --location -o tiledb.tar.gz "https://github.com/TileDB-Inc/TileDB/releases/download/${CORE_VERSION}/tiledb-${OS}-x86_64-${CORE_VERSION}-${CORE_HASH}.tar.gz" +tar -C ./install -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_linux.sh b/.github/scripts/install_tiledb_linux.sh deleted file mode 100755 index 049beda3..00000000 --- a/.github/scripts/install_tiledb_linux.sh +++ /dev/null @@ -1,4 +0,0 @@ -set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.10.2/tiledb-linux-x86_64-2.10.2-9ab84f9.tar.gz \ -&& sudo tar -C /usr/local -xf tiledb.tar.gz -sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_linux_debug.sh b/.github/scripts/install_tiledb_linux_debug.sh deleted file mode 100755 index 301ec0f1..00000000 --- a/.github/scripts/install_tiledb_linux_debug.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j4 -sudo make -C tiledb install -sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_macos.sh b/.github/scripts/install_tiledb_macos.sh deleted file mode 100755 index 57ec38cb..00000000 --- a/.github/scripts/install_tiledb_macos.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.10.2/tiledb-macos-x86_64-2.10.2-9ab84f9.tar.gz \ -&& sudo tar -C /usr/local -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_source.sh b/.github/scripts/install_tiledb_source.sh new file mode 100755 index 00000000..007a200b --- /dev/null +++ b/.github/scripts/install_tiledb_source.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" +mkdir "$CORE_ROOT" +cd "$CORE_ROOT" +case "$OS" in + linux) + NPROC="$(nproc)" + ;; + macos) + NPROC="$(sysctl -n hw.ncpu)" + ;; +esac +git clone --depth 1 -b "$CORE_VERSION" https://github.com/TileDB-Inc/TileDB.git +cd TileDB +mkdir build +cd build +# BUILD_FLAGS is unquoted because it needs to expand into multiple strings. +cmake $BUILD_FLAGS -DCMAKE_INSTALL_PREFIX="${CORE_ROOT}/install" .. +make -j"$NPROC" + +mkdir "${CORE_ROOT}/install" +make -C tiledb install diff --git a/.github/scripts/install_tiledb_source_linux.sh b/.github/scripts/install_tiledb_source_linux.sh deleted file mode 100755 index 146dc263..00000000 --- a/.github/scripts/install_tiledb_source_linux.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j$(nproc) -sudo make -C tiledb install -sudo ldconfig diff --git a/.github/scripts/install_tiledb_source_macos.sh b/.github/scripts/install_tiledb_source_macos.sh deleted file mode 100755 index 0e5e616c..00000000 --- a/.github/scripts/install_tiledb_source_macos.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j$(sysctl -n hw.ncpu) -sudo make -C tiledb install diff --git a/.github/scripts/post_install.sh b/.github/scripts/post_install.sh new file mode 100755 index 00000000..e0838bd6 --- /dev/null +++ b/.github/scripts/post_install.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" + +# Sets up the environment after installing or extracting TileDB. + +if [ "$OS" = "linux" ]; then + /sbin/ldconfig -n "${CORE_ROOT}/install/lib" +fi + +go env -w "CGO_CFLAGS=-I${CORE_ROOT}/install/include" +go env -w "CGO_LDFLAGS=-L${CORE_ROOT}/install/lib/ -Wl,-rpath,${CORE_ROOT}/install/lib/" diff --git a/.github/workflows/tiledb-go.yml b/.github/workflows/tiledb-go.yml index bb799473..752ffdd4 100644 --- a/.github/workflows/tiledb-go.yml +++ b/.github/workflows/tiledb-go.yml @@ -7,157 +7,121 @@ on: pull_request: branches: [ master ] +env: + # The version of TileDB to test against. + CORE_VERSION: "2.10.2" + # The first seven characters of that version's commit hash + # (the same ones that GitHub displays on a release page). + CORE_HASH: "9ab84f9" + jobs: golangci: name: lint runs-on: ubuntu-20.04 steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux.sh - shell: bash - - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - continue-on-error: true - with: - # Required: the version of golangci-lint is required and must be - # specified without patch version: we always use the latest patch version. - version: v1.35.2 - - Linux_Test: - runs-on: ubuntu-20.04 + # Checks out repository + - uses: actions/checkout@v2 + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: golangci-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_binary.sh') }} + + - name: Install TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_binary.sh' + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + continue-on-error: true + with: + # Required: the version of golangci-lint is required and must be + # specified without patch version: we always use the latest patch version. + version: v1.35.2 + + binary-test: + name: binary-${{ matrix.runner }}-go${{ matrix.go }} + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - # Will be checking following versions go: ["1.18"] - steps: + runner: ["macos-11", "ubuntu-20.04"] - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} + steps: - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -v ./... + - uses: actions/checkout@v2 - Macos_Test: - runs-on: macos-11 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] - steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_macos.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -v ./... - - Linux_Experimental_Test: - runs-on: ubuntu-20.04 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] - steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} - # Checks out repository - - uses: actions/checkout@v2 + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: golangci-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_binary.sh') }} - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_source_linux.sh - shell: bash + - name: Install TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_binary.sh' - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} + - name: Post-install + run: ./.github/scripts/post_install.sh - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -tags=experimental -v ./... + - name: Run tests + run: go test -v ./... - Macos_Experimental_Test: - runs-on: macos-11 + source-test: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - # Will be checking following versions - go: ["1.18"] - steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_source_macos.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - # Tests experimental bindings of TileDB-Go - - name: Test TileDB-Go - run: go test -tags=experimental -v ./... - - Linux_Address_Sanitizer: - runs-on: ubuntu-20.04 - strategy: - matrix: - # Will be checking following versions - go: [1.18] + include: + - name: macos-experimental + runner: macos-11 + flags: "-DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release" + test-command: go test -tags=experimental -v ./... + - name: linux-experimental + runner: ubuntu-20.04 + flags: "-DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release" + test-command: go test -tags=experimental -v ./... + - name: linux-asan + runner: ubuntu-20.04 + flags: "-DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug" + test-command: go run -asan ./cmd/tiledb-go-examples + steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux_debug.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - # Tests TileDB-Go - - name: Running examples using address sanitizer flags - continue-on-error: true - run: go run -tags asan ./cmd/tiledb-go-examples + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "1.18" + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: tiledb-binary-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_source.sh') }}-${{ matrix.flags }} + + - name: Compile TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_source.sh' + env: + BUILD_FLAGS: ${{ matrix.flags }} + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: Run test + run: ${{ matrix.test-command }} diff --git a/cmd/tiledb-go-examples/asan_on.go b/cmd/tiledb-go-examples/asan_on.go index 694fb284..064c9b3c 100644 --- a/cmd/tiledb-go-examples/asan_on.go +++ b/cmd/tiledb-go-examples/asan_on.go @@ -8,7 +8,14 @@ package main // void __lsan_do_leak_check(void); import "C" +import ( + "runtime" +) + // maybeASAN runs ASAN if the ASAN build tag is enabled. func maybeASAN() { + // Aggressively GC so that we can minimize false positives. + runtime.GC() + runtime.GC() C.__lsan_do_leak_check() }