diff --git a/.github/workflows/ExtensionTemplate.yml b/.github/workflows/ExtensionTemplate.yml index 42265dd..2f630d2 100644 --- a/.github/workflows/ExtensionTemplate.yml +++ b/.github/workflows/ExtensionTemplate.yml @@ -11,7 +11,7 @@ concurrency: jobs: linux: name: Linux - if: ${{ vars.RUN_RENAME_TEST == 'true' }} + if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} runs-on: ubuntu-latest container: ubuntu:18.04 strategy: @@ -76,7 +76,7 @@ jobs: macos: name: MacOS - if: ${{ vars.RUN_RENAME_TEST == 'true' }} + if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} runs-on: macos-latest strategy: matrix: @@ -129,7 +129,7 @@ jobs: windows: name: Windows - if: ${{ vars.RUN_RENAME_TEST == 'true' }} + if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} runs-on: windows-latest strategy: matrix: diff --git a/.gitmodules b/.gitmodules index bb8e599..cd15846 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = duckdb url = https://github.com/duckdb/duckdb branch = main +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = git@github.com:duckdb/extension-ci-tools.git + branch = main \ No newline at end of file diff --git a/Makefile b/Makefile index 0862e89..e91db43 100644 --- a/Makefile +++ b/Makefile @@ -1,152 +1,8 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads - -all: release - -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -PROJ_DIR := $(dir $(MKFILE_PATH)) - -TEST_PATH="/test/unittest" -DUCKDB_PATH="/duckdb" - -# For non-MinGW windows the path is slightly different -ifeq ($(OS),Windows_NT) -ifneq ($(CXX),g++) - TEST_PATH="/test/Release/unittest.exe" - DUCKDB_PATH="/Release/duckdb.exe" -endif -endif - -#### OSX config -OSX_BUILD_FLAG= -ifneq (${OSX_BUILD_ARCH}, "") - OSX_BUILD_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH} -endif - -#### VCPKG config -VCPKG_TOOLCHAIN_PATH?= -ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' -endif -ifneq ("${VCPKG_TARGET_TRIPLET}", "") - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' -endif - -#### Enable Ninja as generator -ifeq ($(GEN),ninja) - GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 -endif +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +# Configuration of extension EXT_NAME=quack +EXT_CONFIG=${PROJ_DIR}extension_config.cmake -#### Configuration for this extension -EXTENSION_NAME=QUACK -EXTENSION_FLAGS=\ --DDUCKDB_EXTENSION_NAMES="${EXT_NAME}" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)src/include" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql" - -#### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed) -EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="tpch" - -BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' -CLIENT_FLAGS:= - -#### Main build -# For regular CLI build, we link the quack extension directly into the DuckDB executable -CLIENT_FLAGS=-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 - -debug: - mkdir -p build/debug && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \ - cmake --build build/debug --config Debug - -release: - mkdir -p build/release && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ - cmake --build build/release --config Release - -##### Client build -JS_BUILD_FLAGS=-DBUILD_NODE=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 -PY_BUILD_FLAGS=-DBUILD_PYTHON=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 - -debug_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS) -debug_js: debug -debug_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS) -debug_python: debug -release_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS) -release_js: release -release_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS) -release_python: release - -# Main tests -test: test_release - -test_release: release - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" - -test_debug: debug - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" - -#### Client tests -DEBUG_EXT_PATH='$(PROJ_DIR)build/debug/extension/quack/quack.duckdb_extension' -RELEASE_EXT_PATH='$(PROJ_DIR)build/release/extension/quack/quack.duckdb_extension' - -test_js: test_debug_js - -test_debug_js: debug_js - cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js" - -test_release_js: release_js - cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js" - -test_python: test_debug_python - -test_debug_python: debug_python - cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) python3 -m pytest - -test_release_python: release_python - cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) python3 -m pytest - -#### Misc -format: - find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i - cmake-format -i CMakeLists.txt -update: - git submodule update --remote --merge -pull: - git submodule init - git submodule update --recursive --remote - -clean: - rm -rf build - rm -rf testext - cd duckdb && make clean - cd duckdb && make clean-python - - -VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS) -DDUCKDB_CUSTOM_PLATFORM='${DUCKDB_PLATFORM}' -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' -WASM_CXX_MVP_FLAGS= -WASM_CXX_EH_FLAGS=$(WASM_CXX_MVP_FLAGS) -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -WASM_CXX_THREADS_FLAGS=$(WASM_COMPILE_TIME_EH_FLAGS) -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread -WASM_LINK_TIME_FLAGS=-O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_${EXT_NAME}_version,_${EXT_NAME}_init" - -wasm_mvp: - mkdir -p build/wasm_mvp - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="$(WASM_CXX_MVP_FLAGS)" -S duckdb - emmake make -j8 -Cbuild/wasm_mvp - cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) - -wasm_eh: - mkdir -p build/wasm_eh - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="$(WASM_CXX_EH_FLAGS)" -S duckdb - emmake make -j8 -Cbuild/wasm_eh - cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) - -wasm_threads: - mkdir -p ./build/wasm_threads - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="$(WASM_CXX_THREADS_FLAGS)" -S duckdb - emmake make -j8 -Cbuild/wasm_threads - cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -o ../../${EXT_NAME}.duckdb_extension.wasm ${EXT_NAME}.duckdb_extension.wasm.lib $(WASM_LINK_TIME_FLAGS) -sSHARED_MEMORY=1 -pthread +# Include the Makefile from extension-ci-tools +include extension-ci-tools/makefiles/duckdb_extension.Makefile \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 1b092e4..ad51929 100644 --- a/docs/README.md +++ b/docs/README.md @@ -100,16 +100,6 @@ CLI: duckdb -unsigned ``` -Python: -```python -con = duckdb.connect(':memory:', config={'allow_unsigned_extensions' : 'true'}) -``` - -NodeJS: -```js -db = new duckdb.Database(':memory:', {"allow_unsigned_extensions": "true"}); -``` - Secondly, you will need to set the repository endpoint in DuckDB to the HTTP url of your bucket + version of the extension you want to install. To do this run the following SQL query in DuckDB: ```sql diff --git a/docs/UPDATING.md b/docs/UPDATING.md new file mode 100644 index 0000000..67afed8 --- /dev/null +++ b/docs/UPDATING.md @@ -0,0 +1,12 @@ +# Extension updating +When cloning this template, the target version of DuckDB should be the latest stable release of DuckDB. However, there +will inevitably come a time when a new DuckDB is released and the extension repository needs updating. This process goes +as follows: + +- Bump submodules + - `./duckdb` should be set to latest tagged release + - `./extension-ci-tools` should be set to updated branch corresponding to latest DuckDB release +- Bump versions in `./github/workflows` + - `duckdb_version` input in `MainDistributionPipeline.yml` should be set to latest tagged release + - reusable workflow `_extension_distribution.yml` should be set to updated branch corresponding to latest DuckDB release + diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 0000000..f0c2c59 --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit f0c2c59b4963a7684effba724168e03b4251f04b diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 0000000..959e702 --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,10 @@ +# This file is included by DuckDB's build system. It specifies which extension to load + +# Extension from this repo +duckdb_extension_load(quack + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + LOAD_TESTS +) + +# Any extra extensions that should be built +# e.g.: duckdb_extension_load(json) \ No newline at end of file diff --git a/scripts/bootstrap-template.py b/scripts/bootstrap-template.py index bae21b8..9d3dca3 100755 --- a/scripts/bootstrap-template.py +++ b/scripts/bootstrap-template.py @@ -4,6 +4,8 @@ from pathlib import Path shutil.copyfile(f'docs/NEXT_README.md', f'README.md') +os.remove(f'docs/NEXT_README.md') +os.remove(f'docs/README.md') if (len(sys.argv) != 2): raise Exception('usage: python3 bootstrap-template.py ') @@ -38,9 +40,7 @@ def replace(file_name, to_find, to_replace): files_to_search = [] files_to_search.extend(Path('./.github').rglob('./**/*.yml')) -files_to_search.extend(Path('./test').rglob('./**/*.py')) files_to_search.extend(Path('./test').rglob('./**/*.test')) -files_to_search.extend(Path('./test').rglob('./**/*.js')) files_to_search.extend(Path('./src').rglob('./**/*.hpp')) files_to_search.extend(Path('./src').rglob('./**/*.cpp')) files_to_search.extend(Path('./src').rglob('./**/*.txt')) @@ -57,19 +57,22 @@ def replace_everywhere(to_find, to_replace): replace("./Makefile", to_find.capitalize(), to_camel_case(to_replace)) replace("./Makefile", to_find.upper(), to_replace.upper()) replace("./README.md", to_find, to_replace) + replace("./extension_config.cmake", to_find, to_replace) replace_everywhere("quack", name_extension) +replace_everywhere("Quack", name_extension.capitalize()) replace_everywhere("", name_extension) string_to_replace = name_extension string_to_find = "quack" # rename files -os.rename(f'test/python/{string_to_find}_test.py', f'test/python/{string_to_replace}_test.py') os.rename(f'test/sql/{string_to_find}.test', f'test/sql/{string_to_replace}.test') os.rename(f'src/{string_to_find}_extension.cpp', f'src/{string_to_replace}_extension.cpp') os.rename(f'src/include/{string_to_find}_extension.hpp', f'src/include/{string_to_replace}_extension.hpp') -os.rename(f'test/nodejs/{string_to_find}_test.js', f'test/nodejs/{string_to_replace}_test.js') # remove template-specific files -os.remove('.github/workflows/ExtensionTemplate.yml') \ No newline at end of file +os.remove('.github/workflows/ExtensionTemplate.yml') + +# finally, remove this bootstrap file +os.remove(__file__) \ No newline at end of file diff --git a/test/README.md b/test/README.md index f2625b8..fb5e514 100644 --- a/test/README.md +++ b/test/README.md @@ -1,14 +1,11 @@ -# Testing the quack extension -This directory contains all the tests for the quack extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. However, client specific testing is also available. +# Testing this extension +This directory contains all the tests for this extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: ```bash make test ``` - -To run the python tests: -```sql -make test_python -``` - -For other client tests check the makefile in the root of this repository. \ No newline at end of file +or +```bash +make test_debug +``` \ No newline at end of file diff --git a/test/nodejs/quack_test.js b/test/nodejs/quack_test.js deleted file mode 100644 index b6b09c9..0000000 --- a/test/nodejs/quack_test.js +++ /dev/null @@ -1,31 +0,0 @@ -var duckdb = require('../../duckdb/tools/nodejs'); -var assert = require('assert'); - -describe(`quack extension`, () => { - let db; - let conn; - before((done) => { - db = new duckdb.Database(':memory:', {"allow_unsigned_extensions":"true"}); - conn = new duckdb.Connection(db); - conn.exec(`LOAD '${process.env.QUACK_EXTENSION_BINARY_PATH}';`, function (err) { - if (err) throw err; - done(); - }); - }); - - it('quack function should return expected string', function (done) { - db.all("SELECT quack('Sam') as value;", function (err, res) { - if (err) throw err; - assert.deepEqual(res, [{value: "Quack Sam 🐥"}]); - done(); - }); - }); - - it('quack_openssl_version function should return expected string', function (done) { - db.all("SELECT quack_openssl_version('Michael') as value;", function (err, res) { - if (err) throw err; - assert(res[0].value.startsWith('Quack Michael, my linked OpenSSL version is OpenSSL')); - done(); - }); - }); -}); \ No newline at end of file diff --git a/test/python/quack_test.py b/test/python/quack_test.py deleted file mode 100644 index e2be276..0000000 --- a/test/python/quack_test.py +++ /dev/null @@ -1,23 +0,0 @@ -import duckdb -import os -import pytest - -# Get a fresh connection to DuckDB with the quack extension binary loaded -@pytest.fixture -def duckdb_conn(): - extension_binary = os.getenv('QUACK_EXTENSION_BINARY_PATH') - if (extension_binary == ''): - raise Exception('Please make sure the `QUACK_EXTENSION_BINARY_PATH` is set to run the python tests') - conn = duckdb.connect('', config={'allow_unsigned_extensions': 'true'}) - conn.execute(f"load '{extension_binary}'") - return conn - -def test_quack(duckdb_conn): - duckdb_conn.execute("SELECT quack('Sam') as value;"); - res = duckdb_conn.fetchall() - assert res[0][0] == "Quack Sam 🐥" - -def test_quack_openssl_version_test(duckdb_conn): - duckdb_conn.execute("SELECT quack_openssl_version('Michael');"); - res = duckdb_conn.fetchall() - assert res[0][0][0:51] == "Quack Michael, my linked OpenSSL version is OpenSSL"