diff --git a/.github/scripts/build_with_sanitizer_and_run.sh b/.github/scripts/build_with_sanitizer_and_run.sh deleted file mode 100755 index 8291f7e1..00000000 --- a/.github/scripts/build_with_sanitizer_and_run.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -e -x -CGO_ENABLED=1 CGO_LDFLAGS='-fsanitize=address' CGO_CFLAGS='-fsanitize=address' go build cmd/tiledb-go/main.go -./main \ No newline at end of file diff --git a/.github/scripts/env.sh b/.github/scripts/env.sh new file mode 100644 index 00000000..efbde5eb --- /dev/null +++ b/.github/scripts/env.sh @@ -0,0 +1,16 @@ +# Subroutine to set up environment variables. + +CORE_ROOT="$HOME/tiledb-core" + +case "$(uname -s)" in + Linux) + OS="linux" + ;; + Darwin) + OS="macos" + ;; + *) + echo 'Unknown OS!' + exit 1 + ;; +esac diff --git a/.github/scripts/install_tiledb_binary.sh b/.github/scripts/install_tiledb_binary.sh new file mode 100755 index 00000000..f8837573 --- /dev/null +++ b/.github/scripts/install_tiledb_binary.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" +mkdir "$CORE_ROOT" +cd "$CORE_ROOT" +mkdir install +curl --location -o tiledb.tar.gz "https://github.com/TileDB-Inc/TileDB/releases/download/${CORE_VERSION}/tiledb-${OS}-x86_64-${CORE_VERSION}-${CORE_HASH}.tar.gz" +tar -C ./install -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_linux.sh b/.github/scripts/install_tiledb_linux.sh deleted file mode 100755 index 049beda3..00000000 --- a/.github/scripts/install_tiledb_linux.sh +++ /dev/null @@ -1,4 +0,0 @@ -set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.10.2/tiledb-linux-x86_64-2.10.2-9ab84f9.tar.gz \ -&& sudo tar -C /usr/local -xf tiledb.tar.gz -sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_linux_debug.sh b/.github/scripts/install_tiledb_linux_debug.sh deleted file mode 100755 index 301ec0f1..00000000 --- a/.github/scripts/install_tiledb_linux_debug.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j4 -sudo make -C tiledb install -sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_macos.sh b/.github/scripts/install_tiledb_macos.sh deleted file mode 100755 index 57ec38cb..00000000 --- a/.github/scripts/install_tiledb_macos.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.10.2/tiledb-macos-x86_64-2.10.2-9ab84f9.tar.gz \ -&& sudo tar -C /usr/local -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_source.sh b/.github/scripts/install_tiledb_source.sh new file mode 100755 index 00000000..007a200b --- /dev/null +++ b/.github/scripts/install_tiledb_source.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" +mkdir "$CORE_ROOT" +cd "$CORE_ROOT" +case "$OS" in + linux) + NPROC="$(nproc)" + ;; + macos) + NPROC="$(sysctl -n hw.ncpu)" + ;; +esac +git clone --depth 1 -b "$CORE_VERSION" https://github.com/TileDB-Inc/TileDB.git +cd TileDB +mkdir build +cd build +# BUILD_FLAGS is unquoted because it needs to expand into multiple strings. +cmake $BUILD_FLAGS -DCMAKE_INSTALL_PREFIX="${CORE_ROOT}/install" .. +make -j"$NPROC" + +mkdir "${CORE_ROOT}/install" +make -C tiledb install diff --git a/.github/scripts/install_tiledb_source_linux.sh b/.github/scripts/install_tiledb_source_linux.sh deleted file mode 100755 index 146dc263..00000000 --- a/.github/scripts/install_tiledb_source_linux.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j$(nproc) -sudo make -C tiledb install -sudo ldconfig diff --git a/.github/scripts/install_tiledb_source_macos.sh b/.github/scripts/install_tiledb_source_macos.sh deleted file mode 100755 index 0e5e616c..00000000 --- a/.github/scripts/install_tiledb_source_macos.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.10.2 -cd TileDB -mkdir build && cd build -cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -make -j$(sysctl -n hw.ncpu) -sudo make -C tiledb install diff --git a/.github/scripts/post_install.sh b/.github/scripts/post_install.sh new file mode 100755 index 00000000..e0838bd6 --- /dev/null +++ b/.github/scripts/post_install.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -exo pipefail +. "${BASH_SOURCE%/*}/env.sh" + +# Sets up the environment after installing or extracting TileDB. + +if [ "$OS" = "linux" ]; then + /sbin/ldconfig -n "${CORE_ROOT}/install/lib" +fi + +go env -w "CGO_CFLAGS=-I${CORE_ROOT}/install/include" +go env -w "CGO_LDFLAGS=-L${CORE_ROOT}/install/lib/ -Wl,-rpath,${CORE_ROOT}/install/lib/" diff --git a/.github/workflows/tiledb-go.yml b/.github/workflows/tiledb-go.yml index 2955584c..752ffdd4 100644 --- a/.github/workflows/tiledb-go.yml +++ b/.github/workflows/tiledb-go.yml @@ -7,223 +7,121 @@ on: pull_request: branches: [ master ] +env: + # The version of TileDB to test against. + CORE_VERSION: "2.10.2" + # The first seven characters of that version's commit hash + # (the same ones that GitHub displays on a release page). + CORE_HASH: "9ab84f9" + jobs: golangci: name: lint runs-on: ubuntu-20.04 steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux.sh - shell: bash - - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - continue-on-error: true - with: - # Required: the version of golangci-lint is required and must be - # specified without patch version: we always use the latest patch version. - version: v1.35.2 - - Linux_Test: - runs-on: ubuntu-20.04 + # Checks out repository + - uses: actions/checkout@v2 + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: golangci-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_binary.sh') }} + + - name: Install TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_binary.sh' + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + continue-on-error: true + with: + # Required: the version of golangci-lint is required and must be + # specified without patch version: we always use the latest patch version. + version: v1.35.2 + + binary-test: + name: binary-${{ matrix.runner }}-go${{ matrix.go }} + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - # Will be checking following versions go: ["1.18"] - steps: + runner: ["macos-11", "ubuntu-20.04"] - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -v ./... - - Macos_Test: - runs-on: macos-11 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] - steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_macos.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -v ./... - - Linux_Experimental_Test: - runs-on: ubuntu-20.04 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_source_linux.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-experimental - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - - # Tests TileDB-Go - - name: Test TileDB-Go - run: go test -tags=experimental -v ./... - - Macos_Experimental_Test: - runs-on: macos-11 - strategy: - matrix: - # Will be checking following versions - go: ["1.18"] - steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_source_macos.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-experimental - restore-keys: | - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - - # Tests experimental bindings of TileDB-Go - - name: Test TileDB-Go - run: go test -tags=experimental -v ./... - - Linux_Address_Sanitizer: - runs-on: ubuntu-20.04 + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: golangci-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_binary.sh') }} + + - name: Install TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_binary.sh' + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: Run tests + run: go test -v ./... + + source-test: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - # Will be checking following versions - go: [1.18] + include: + - name: macos-experimental + runner: macos-11 + flags: "-DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release" + test-command: go test -tags=experimental -v ./... + - name: linux-experimental + runner: ubuntu-20.04 + flags: "-DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release" + test-command: go test -tags=experimental -v ./... + - name: linux-asan + runner: ubuntu-20.04 + flags: "-DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug" + test-command: go run -asan ./cmd/tiledb-go-examples + steps: - # Checks out repository - - uses: actions/checkout@v2 - - # Downloads TileDB-Core from release assets and install - - name: Run TileDB install script - run: ./.github/scripts/install_tiledb_linux_debug.sh - shell: bash - - # Following action sets up Go and uses the strategy matrix to test on - # specific versions - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-asan - restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - ${{ runner.os }}-go- - - - name: Install dependencies - run: go get -t . - - # Tests TileDB-Go - - name: Running examples using address sanitizer flags - continue-on-error: true - run: go run -tags asan ./cmd/tiledb-go-examples + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "1.18" + + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: ~/tiledb-core/install + key: tiledb-binary-${{ runner.os }}-${{ env.CORE_VERSION }}-${{ env.CORE_HASH }}-${{ hashFiles('./.github/scripts/install_tiledb_source.sh') }}-${{ matrix.flags }} + + - name: Compile TileDB Core + if: steps.build-cache.outputs.cache-hit != 'true' + run: './.github/scripts/install_tiledb_source.sh' + env: + BUILD_FLAGS: ${{ matrix.flags }} + + - name: Post-install + run: ./.github/scripts/post_install.sh + + - name: Run test + run: ${{ matrix.test-command }} diff --git a/cmd/tiledb-go-examples/asan_on.go b/cmd/tiledb-go-examples/asan_on.go index 694fb284..064c9b3c 100644 --- a/cmd/tiledb-go-examples/asan_on.go +++ b/cmd/tiledb-go-examples/asan_on.go @@ -8,7 +8,14 @@ package main // void __lsan_do_leak_check(void); import "C" +import ( + "runtime" +) + // maybeASAN runs ASAN if the ASAN build tag is enabled. func maybeASAN() { + // Aggressively GC so that we can minimize false positives. + runtime.GC() + runtime.GC() C.__lsan_do_leak_check() }