From 8ed495a6d0e8f1b38c567f16c4eb87fe09fec699 Mon Sep 17 00:00:00 2001 From: Hugues Chocart Date: Wed, 11 Sep 2024 21:59:08 +0100 Subject: [PATCH] feat: add data warehouse settings (#548) --- package-lock.json | 593 +++++++++++++++++- packages/backend/package.json | 2 + .../src/api/v1/data-warehouse/index.ts | 36 ++ .../src/api/v1/data-warehouse/utils.ts | 247 ++++++++ packages/backend/src/api/v1/index.ts | 2 + packages/db/0045.sql | 10 + packages/db/0046.sql | 1 + .../components/settings/data-warehouse.tsx | 87 +++ .../settings/data-warehouse/bigquery.tsx | 93 +++ packages/frontend/pages/settings/index.tsx | 16 +- .../public/assets/amazon-redshift.svg | 11 + .../public/assets/azure-synapse-analytics.svg | 32 + packages/frontend/public/assets/bigquery.svg | 1 + .../frontend/public/assets/databricks.svg | 7 + packages/frontend/public/assets/snowflake.svg | 29 + .../utils/dataHooks/data-warehouse.ts | 17 + packages/frontend/utils/dataHooks/index.ts | 5 +- 17 files changed, 1170 insertions(+), 19 deletions(-) create mode 100644 packages/backend/src/api/v1/data-warehouse/index.ts create mode 100644 packages/backend/src/api/v1/data-warehouse/utils.ts create mode 100644 packages/db/0045.sql create mode 100644 packages/db/0046.sql create mode 100644 packages/frontend/components/settings/data-warehouse.tsx create mode 100644 packages/frontend/pages/settings/data-warehouse/bigquery.tsx create mode 100644 packages/frontend/public/assets/amazon-redshift.svg create mode 100644 packages/frontend/public/assets/azure-synapse-analytics.svg create mode 100644 packages/frontend/public/assets/bigquery.svg create mode 100644 packages/frontend/public/assets/databricks.svg create mode 100644 packages/frontend/public/assets/snowflake.svg create mode 100644 packages/frontend/utils/dataHooks/data-warehouse.ts diff --git a/package-lock.json b/package-lock.json index 6445337b..873b0ee0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1182,6 +1182,161 @@ "integrity": "sha512-X8R8Oj771YRl/w+c1HqAC1szL8zWQRwFvgDwT129k9ACdBoud/+/rX9V0qiMl6LWUdP9voC2nDVZYPMQQsb6eA==", "license": "MIT" }, + "node_modules/@google-cloud/bigquery": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.9.0.tgz", + "integrity": "sha512-KJTimGLDlAR1IfZ4Y8xhIVfoZ+XBXd0GGuJttLSXxtR0g+4vNsUt0xS33PRVa5TXey97374yU+uWNlCb5bHwBw==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.2", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", + "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "html-entities": "^2.5.2", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/datastream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@google-cloud/datastream/-/datastream-3.2.0.tgz", + "integrity": "sha512-QcQNPpBXUnt6q6Dh17oU26HJMizL+S76LMTOCd0V6R6eYnqOzKauvMk5VG9MgAW4HoS4xOb93ZHUaCRwdq4kTA==", + "license": "Apache-2.0", + "dependencies": { + "google-gax": "^4.0.3" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.11.2.tgz", + "integrity": "sha512-DWp92gDD7/Qkj7r8kus6/HCINeo3yPZWZ3paKgDgsbKbSpoxKg1yvN8xe2Q8uE3zOsPe3bX8FQX2+XValq2yTw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.7.13", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", + "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", + "license": "Apache-2.0" + }, + "node_modules/@grpc/proto-loader/node_modules/protobufjs": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", + "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@hapi/bourne": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-3.0.0.tgz", @@ -1352,6 +1507,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@json2csv/formatters": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/@json2csv/formatters/-/formatters-7.0.6.tgz", @@ -2803,6 +2968,15 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/@types/accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.7.tgz", @@ -2834,6 +3008,12 @@ "@types/node": "*" } }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -3160,6 +3340,32 @@ "@types/react": "*" } }, + "node_modules/@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/scheduler": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.23.0.tgz", @@ -3190,6 +3396,12 @@ "@types/send": "*" } }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, "node_modules/@xenova/transformers": { "version": "2.17.2", "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.17.2.tgz", @@ -3376,6 +3588,15 @@ "node": ">=8" } }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3508,6 +3729,28 @@ "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==", "license": "MIT" }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -3584,6 +3827,12 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/bundle-require": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.0.0.tgz", @@ -4376,6 +4625,18 @@ "url": "https://dotenvx.com" } }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -4383,6 +4644,15 @@ "dev": true, "license": "MIT" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -4564,6 +4834,12 @@ "node": ">=6" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4848,6 +5124,60 @@ "node": ">=10" } }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4982,6 +5312,76 @@ "node": ">=8" } }, + "node_modules/google-auth-library": { + "version": "9.14.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.14.1.tgz", + "integrity": "sha512-Rj+PMjoNFGFTmtItH7gHfbHpGVSb3vmnGK3nwNBqxQF9NoBpttSZI/rc0WiM63ma2uGDQtYEkMHkK9U6937NiA==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-gax": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.4.1.tgz", + "integrity": "sha512-Phyp9fMfA00J3sZbJxbbB4jC55b7DBjE3F6poyL3wKMEBVKA79q6BGuHcTiM28yOzVql0NDbRL8MLLh8Iwk9Dg==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.10.9", + "@grpc/proto-loader": "^0.7.13", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^4.0.0", + "google-auth-library": "^9.3.0", + "node-fetch": "^2.7.0", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^2.0.2", + "protobufjs": "^7.3.2", + "retry-request": "^7.0.0", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-gax/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", + "license": "Apache-2.0" + }, + "node_modules/google-gax/node_modules/protobufjs": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", + "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -5009,6 +5409,19 @@ "node": ">=8.0.0" } }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/guid-typescript": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", @@ -5099,6 +5512,22 @@ "integrity": "sha512-BXUKIkUuh6cmmxzi5OIbUJxrG8OAk2MqoL1DtO3Wo9D2faJg2ph5ntyuQeLqaHJmzER6H5tllCDA9ZnNe9BVGg==", "license": "MIT" }, + "node_modules/html-entities": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", + "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/http-assert": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/http-assert/-/http-assert-1.5.0.tgz", @@ -5137,6 +5566,20 @@ "node": ">= 0.6" } }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -5272,6 +5715,15 @@ "loose-envify": "^1.0.0" } }, + "node_modules/is": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/is-arrayish": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", @@ -5366,7 +5818,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5453,6 +5904,15 @@ "node": ">= 16" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-stable-stringify": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.1.1.tgz", @@ -5501,6 +5961,27 @@ "jsonrepair": "bin/cli.js" } }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -5791,6 +6272,12 @@ "deprecated": "This package is discontinued. Use lodash@^4.0.0.", "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -6423,6 +6910,15 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", @@ -7117,6 +7613,48 @@ "react-is": "^16.13.1" } }, + "node_modules/proto3-json-serializer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", + "integrity": "sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ==", + "license": "Apache-2.0", + "dependencies": { + "protobufjs": "^7.2.5" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/proto3-json-serializer/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", + "license": "Apache-2.0" + }, + "node_modules/proto3-json-serializer/node_modules/protobufjs": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", + "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/protobufjs": { "version": "6.11.4", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", @@ -7575,6 +8113,20 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -7967,6 +8519,21 @@ "node": ">= 0.6" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -8092,6 +8659,12 @@ "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", "license": "MIT" }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, "node_modules/styled-jsx": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz", @@ -8293,6 +8866,22 @@ "node": ">=8" } }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/text-decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz", @@ -8991,6 +9580,8 @@ "dependencies": { "@authenio/samlify-node-xmllint": "^2.0.0", "@aws-sdk/client-comprehend": "^3.645.0", + "@google-cloud/bigquery": "^7.9.0", + "@google-cloud/datastream": "^3.2.0", "@json2csv/plainjs": "^7.0.6", "@koa/cors": "^5.0.0", "@sentry/node": "^7.99.0", diff --git a/packages/backend/package.json b/packages/backend/package.json index f10f4d6f..a00c887b 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -13,6 +13,8 @@ "dependencies": { "@authenio/samlify-node-xmllint": "^2.0.0", "@aws-sdk/client-comprehend": "^3.645.0", + "@google-cloud/bigquery": "^7.9.0", + "@google-cloud/datastream": "^3.2.0", "@json2csv/plainjs": "^7.0.6", "@koa/cors": "^5.0.0", "@sentry/node": "^7.99.0", diff --git a/packages/backend/src/api/v1/data-warehouse/index.ts b/packages/backend/src/api/v1/data-warehouse/index.ts new file mode 100644 index 00000000..2a5dba88 --- /dev/null +++ b/packages/backend/src/api/v1/data-warehouse/index.ts @@ -0,0 +1,36 @@ +import Context from "@/src/utils/koa"; +import Router from "koa-router"; +import { z } from "zod"; +import { createNewDatastream } from "./utils"; +import sql from "@/src/utils/db"; + +const dataWarehouse = new Router({ + prefix: "/data-warehouse", +}); + +dataWarehouse.get("/bigquery", async (ctx: Context) => { + const { projectId } = ctx.state; + const [connector] = + await sql`select * from _data_warehouse_connector where project_id = ${projectId}`; + + console.log(connector); + ctx.body = connector; +}); + +dataWarehouse.post("/bigquery", async (ctx: Context) => { + const bodySchema = z.object({ + apiKey: z.string().transform((apiKey) => JSON.parse(apiKey)), + }); + + // TODO: validate apiKey first with Zod + + const { apiKey } = bodySchema.parse(ctx.request.body); + + await createNewDatastream(apiKey, process.env.DATABASE_URL!, ctx); + + ctx.body = {}; +}); + +dataWarehouse.patch("/bigquery", async (ctx: Context) => {}); + +export default dataWarehouse; diff --git a/packages/backend/src/api/v1/data-warehouse/utils.ts b/packages/backend/src/api/v1/data-warehouse/utils.ts new file mode 100644 index 00000000..36dc1d9c --- /dev/null +++ b/packages/backend/src/api/v1/data-warehouse/utils.ts @@ -0,0 +1,247 @@ +import { protos, DatastreamClient } from "@google-cloud/datastream"; +import { parse as parseUrl } from "url"; +import fs from "fs"; +import Context from "@/src/utils/koa"; +import sql from "@/src/utils/db"; +import { BigQuery } from "@google-cloud/bigquery"; + +type ConnectionProfile = protos.google.cloud.datastream.v1.IConnectionProfile; + +interface ParsedPostgresUri { + hostname: string; + port: number; + username: string; + password: string; + database: string; +} + +function parsePostgresUri(uri: string): ParsedPostgresUri { + const parsed = parseUrl(uri); + + if (!parsed.hostname || !parsed.auth || !parsed.pathname) { + throw new Error("Invalid PostgreSQL connection URI"); + } + + const [username, password] = parsed.auth.split(":"); + + if (!username || !password) { + throw new Error("Username or password is missing from the URI"); + } + + return { + hostname: parsed.hostname, + port: parseInt(parsed.port, 10), + username, + password, + database: parsed.pathname.slice(1), + }; +} + +function createBigQueryConnectionProfile( + displayName: string, +): ConnectionProfile { + const connectionProfile: ConnectionProfile = { + displayName: displayName, + bigqueryProfile: {}, + staticServiceIpConnectivity: {}, + }; + + return connectionProfile; +} + +type PostgresqlSourceConfig = + protos.google.cloud.datastream.v1.IPostgresqlSourceConfig; + +type BigQueryDestinationConfig = + protos.google.cloud.datastream.v1.IBigQueryDestinationConfig; + +type Stream = protos.google.cloud.datastream.v1.IStream; + +const location = "us-east1"; + +export async function createNewDatastream( + apiKey: string, + postgresURI: string, + ctx: Context, +) { + const projectId = apiKey.project_id as string; + + const datastreamClient = new DatastreamClient({ + credentials: apiKey, + projectId, + }); + + const parsedUri = parsePostgresUri(postgresURI); + + const postgresConnectionProfile: ConnectionProfile = { + displayName: "Lunary Data Warehouse Source", + postgresqlProfile: { + hostname: parsedUri.hostname, + port: parsedUri.port || 5432, + username: parsedUri.username, + password: parsedUri.password, + database: parsedUri.database, + }, + staticServiceIpConnectivity: {}, + }; + + const request1 = { + parent: datastreamClient.locationPath(projectId, location), + connectionProfileId: "lunary-data-warehouse-source", + connectionProfile: postgresConnectionProfile, + }; + + try { + await sql`revoke all privileges on all tables in schema public from lunary_bigquery_connector`; + await sql`revoke all privileges on schema public from lunary_bigquery_connector`; + await sql`revoke all privileges on all sequences in schema public from lunary_bigquery_connector`; + await sql`revoke all privileges on all functions in schema public from lunary_bigquery_connector`; + await sql`alter default privileges in schema public revoke all on tables from lunary_bigquery_connector`; + await sql`alter default privileges in schema public revoke all on sequences from lunary_bigquery_connector`; + await sql`alter default privileges in schema public revoke all on functions from lunary_bigquery_connector`; + await sql`drop publication if exists lunary_bigquery_connector`; + await sql`select pg_drop_replication_slot('lunary_bigquery_connector')`.catch( + console.error, + ); + await sql`drop user if exists lunary_bigquery_connector`; + + await sql`create publication lunary_bigquery_connector for all tables`; + await sql`select pg_create_logical_replication_slot('lunary_bigquery_connector', 'pgoutput')`; + await sql.unsafe( + `create user lunary_bigquery_connector with encrypted password '${process.env.JWT_SECRET}'`, + ); + await sql`grant select on all tables in schema public to lunary_bigquery_connector`; + await sql`grant usage on schema public to lunary_bigquery_connector`; + await sql`alter default privileges in schema public grant select on tables to lunary_bigquery_connector`; + await sql`grant rds_replication to lunary_bigquery_connector`.catch( + () => {}, + ); + } catch (error) { + console.error(error); + ctx.throw( + 500, + "Could not configure the PostgreSQL source database. Have you read the tutorial at https://lunary.ai/docs/enterprise/bigquery#setup-postgtresl-source", + ); + } + + await sql`grant RDS_REPLICATION to lunary_bigquery_connector`.catch(() => {}); + + // TODO: delete connections and stream if they already exist + + try { + const [operation1] = + await datastreamClient.createConnectionProfile(request1); + const [response1] = await operation1.promise(); + } catch (error) { + if (error.code === 6) { + console.info("Source Connection already exists. Skipping."); + } else { + throw error; + } + } + + const bigQueryConnectionProfile = createBigQueryConnectionProfile( + "Lunary Data Warehouse Destination", + ); + + const request2 = { + parent: datastreamClient.locationPath(apiKey.project_id, location), + connectionProfileId: "lunary-data-warehouse-destination", + connectionProfile: bigQueryConnectionProfile, + }; + + try { + const [operation2] = + await datastreamClient.createConnectionProfile(request2); + const [response2] = await operation2.promise(); + } catch (error) { + if (error.code === 6) { + console.info("Destination Connection already exists. Skipping."); + } else { + throw error; + } + } + + const postgresSourceConfig: PostgresqlSourceConfig = { + includeObjects: { + postgresqlSchemas: [{ schema: "public" }], + }, + replicationSlot: "lunary_bigquery_connector", + publication: "lunary_bigquery_connector", + }; + + const bigqueryDestinationConfig: BigQueryDestinationConfig = { + dataFreshness: { seconds: 300 }, + singleTargetDataset: { + datasetId: `${projectId}:lunary`, + }, + }; + + const bigquery = new BigQuery({ + credentials: apiKey, + projectId, + }); + + try { + await bigquery.createDataset("lunary", { + location: "US", + }); + } catch (error) { + if (error.code === 403) { + ctx.throw(500, "You do not have the suh"); + } + console.error(error); + console.info("Dataset already exist. Skipping"); + } + + const streamConfig: Stream = { + sourceConfig: { + sourceConnectionProfile: `projects/${projectId}/locations/${location}/connectionProfiles/lunary-data-warehouse-source`, + postgresqlSourceConfig: postgresSourceConfig, + }, + destinationConfig: { + destinationConnectionProfile: `projects/${projectId}/locations/${location}/connectionProfiles/lunary-data-warehouse-destination`, + bigqueryDestinationConfig: bigqueryDestinationConfig, + }, + displayName: `PostgreSQL to BigQuery Stream`, + backfillAll: {}, + }; + + const request = { + parent: `projects/${projectId}/locations/${location}`, + streamId: "lunary-data-warehouse-stream", + stream: streamConfig, + }; + + const [operation] = await datastreamClient.createStream(request); + console.log(`Stream creation initiated. Operation name: ${operation.name}`); + + const [response] = await operation.promise(); + console.log("Stream created successfully:", response); + + if (typeof operation.result?.name !== "string") { + throw new Error("Stream creation failed:", response); + } + + const updateStreamRequest = { + stream: { + name: operation.result.name, + state: protos.google.cloud.datastream.v1.Stream.State.RUNNING, + }, + updateMask: { + paths: ["state"], + }, + }; + + const [updateOperation] = + await datastreamClient.updateStream(updateStreamRequest); + console.log( + `Stream update initiated. Operation name: ${updateOperation.name}`, + ); + await sql` + insert into _data_warehouse_connector (project_id, type, status) + values (${ctx.state.projectId}, 'BigQuery', 'created') + on conflict (project_id) + do update set type = 'BigQuery', status = 'created', updated_at = now() + `; +} diff --git a/packages/backend/src/api/v1/index.ts b/packages/backend/src/api/v1/index.ts index 497fbdca..d08d1be5 100644 --- a/packages/backend/src/api/v1/index.ts +++ b/packages/backend/src/api/v1/index.ts @@ -17,6 +17,7 @@ import evaluators from "./evaluator"; import analytics from "./analytics"; import views from "./views"; import models from "./models"; +import dataWarehouse from "./data-warehouse"; const v1 = new Router({ prefix: "/v1", @@ -50,5 +51,6 @@ v1.use(checklists.routes()); v1.use(analytics.routes()); v1.use(views.routes()); v1.use(models.routes()); +v1.use(dataWarehouse.routes()); export default v1; diff --git a/packages/db/0045.sql b/packages/db/0045.sql new file mode 100644 index 00000000..34a34e84 --- /dev/null +++ b/packages/db/0045.sql @@ -0,0 +1,10 @@ +create table _data_warehouse_connector( + id uuid default uuid_generate_v4() primary key, + created_at timestamp with time zone default now(), + updated_at timestamp with time zone default now(), + project_id uuid not null, + type text, + status text, + constraint fk_checklist_project_id foreign key (project_id) references project(id) on delete cascade +); +create unique index on _data_warehouse_connector(project_id); \ No newline at end of file diff --git a/packages/db/0046.sql b/packages/db/0046.sql new file mode 100644 index 00000000..e31950d6 --- /dev/null +++ b/packages/db/0046.sql @@ -0,0 +1 @@ +alter table api_key add primary key (id); \ No newline at end of file diff --git a/packages/frontend/components/settings/data-warehouse.tsx b/packages/frontend/components/settings/data-warehouse.tsx new file mode 100644 index 00000000..9279057e --- /dev/null +++ b/packages/frontend/components/settings/data-warehouse.tsx @@ -0,0 +1,87 @@ +import { useOrg } from "@/utils/dataHooks"; +import { SettingsCard } from "../blocks/SettingsCard"; +import { IconDatabaseShare } from "@tabler/icons-react"; +import AmazonRedshiftIconSrc from "public/assets/amazon-redshift.svg"; +import AzureSynapseAnalyticsIconSrc from "public/assets/azure-synapse-analytics.svg"; +import BigQueryIconSrc from "public/assets/bigquery.svg"; +import DatabricksIconSrc from "public/assets/databricks.svg"; +import SnowflakeIconSrc from "public/assets/snowflake.svg"; +import Image from "next/image"; +import config from "@/utils/config"; +import { ActionIcon, Group, Text, Tooltip } from "@mantine/core"; +import { useRouter } from "next/router"; + +const PROVIDERS = [ + { + name: "Amazon Redshift", + iconSrc: AmazonRedshiftIconSrc, + disabled: true, + }, + { + name: "Azure Synapse Analytics", + iconSrc: AzureSynapseAnalyticsIconSrc, + disabled: true, + }, + { + name: "BigQuery", + settingsPath: "bigquery", + iconSrc: BigQueryIconSrc, + }, + { + name: "Databricks", + iconSrc: DatabricksIconSrc, + disabled: true, + }, + { + name: "Snowflake", + iconSrc: SnowflakeIconSrc, + disabled: true, + }, +]; + +export default function DataWarehouseCard() { + const { org } = useOrg(); + const router = useRouter(); + + return ( + + Synchronize your data with a data warehouse provider + + {PROVIDERS.map(({ name, iconSrc, settingsPath, disabled }) => ( + + + router.push(`/settings/data-warehouse/${settingsPath}`) + } + > + {name} + + + ))} + + + ); +} diff --git a/packages/frontend/pages/settings/data-warehouse/bigquery.tsx b/packages/frontend/pages/settings/data-warehouse/bigquery.tsx new file mode 100644 index 00000000..46cff8ab --- /dev/null +++ b/packages/frontend/pages/settings/data-warehouse/bigquery.tsx @@ -0,0 +1,93 @@ +import { useBigQuery } from "@/utils/dataHooks/data-warehouse"; +import { + Alert, + Anchor, + Button, + Container, + Loader, + PasswordInput, + Title, +} from "@mantine/core"; +import { IconAlertTriangle, IconCheck } from "@tabler/icons-react"; +import { useState } from "react"; + +export default function BigQuery() { + const [apiKey, setApiKey] = useState(""); + const [isLoading, setIsLoading] = useState(false); + const { connector, insert, isLoading: isBigQueryLoading } = useBigQuery(); + + if (isBigQueryLoading) { + return ( + + + + ); + } + + async function submit() { + try { + setIsLoading(true); + await insert(apiKey); + } catch (error) { + console.error(error); + } finally { + setIsLoading(false); + } + } + + return ( + + BigQuery Configuration + + {!connector && ( + <> + } + > + The connector requires prerequisites to be fulfilled to run + successfully:{" "} + + Prerequisites + + + + setApiKey(e.target.value)} + /> + + + + )} + + {connector && ( + } + > + A new BigQuery dataset "lunary" will be populated in a few minutes, if + you're setting it up for the first time. + + )} + + ); +} diff --git a/packages/frontend/pages/settings/index.tsx b/packages/frontend/pages/settings/index.tsx index 2e896c32..baf9acda 100644 --- a/packages/frontend/pages/settings/index.tsx +++ b/packages/frontend/pages/settings/index.tsx @@ -45,6 +45,7 @@ import Link from "next/link"; import { useEffect, useState } from "react"; import { CheckLogic, hasAccess } from "shared"; import useSWR from "swr"; +import DataWarehouseCard from "@/components/settings/data-warehouse"; function Keys() { const [regenerating, setRegenerating] = useState(false); @@ -356,20 +357,7 @@ export default function AppAnalytics() { - + {user && hasAccess(user.role, "projects", "delete") && ( diff --git a/packages/frontend/public/assets/amazon-redshift.svg b/packages/frontend/public/assets/amazon-redshift.svg new file mode 100644 index 00000000..52449a5d --- /dev/null +++ b/packages/frontend/public/assets/amazon-redshift.svg @@ -0,0 +1,11 @@ + + + + + + + + \ No newline at end of file diff --git a/packages/frontend/public/assets/azure-synapse-analytics.svg b/packages/frontend/public/assets/azure-synapse-analytics.svg new file mode 100644 index 00000000..8b9fe9ed --- /dev/null +++ b/packages/frontend/public/assets/azure-synapse-analytics.svg @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/frontend/public/assets/bigquery.svg b/packages/frontend/public/assets/bigquery.svg new file mode 100644 index 00000000..34e84de1 --- /dev/null +++ b/packages/frontend/public/assets/bigquery.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/frontend/public/assets/databricks.svg b/packages/frontend/public/assets/databricks.svg new file mode 100644 index 00000000..17f4afe5 --- /dev/null +++ b/packages/frontend/public/assets/databricks.svg @@ -0,0 +1,7 @@ + + download (2)-svg + + + \ No newline at end of file diff --git a/packages/frontend/public/assets/snowflake.svg b/packages/frontend/public/assets/snowflake.svg new file mode 100644 index 00000000..c8db487e --- /dev/null +++ b/packages/frontend/public/assets/snowflake.svg @@ -0,0 +1,29 @@ + + logo-blue-svg copy-svg + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/frontend/utils/dataHooks/data-warehouse.ts b/packages/frontend/utils/dataHooks/data-warehouse.ts new file mode 100644 index 00000000..91f7732a --- /dev/null +++ b/packages/frontend/utils/dataHooks/data-warehouse.ts @@ -0,0 +1,17 @@ +import { useProjectMutation, useProjectSWR } from "."; +import { fetcher } from "../fetcher"; + +export function useBigQuery() { + const { data, isLoading, mutate } = useProjectSWR("/data-warehouse/bigquery"); + const { trigger: insertMutation } = useProjectMutation( + `/data-warehouse/bigquery`, + fetcher.post, + ); + + async function insert(apiKey: string) { + await insertMutation({ apiKey }); + await mutate(); + } + + return { insert, connector: data, isLoading }; +} diff --git a/packages/frontend/utils/dataHooks/index.ts b/packages/frontend/utils/dataHooks/index.ts index 851663a3..bf1fe19b 100644 --- a/packages/frontend/utils/dataHooks/index.ts +++ b/packages/frontend/utils/dataHooks/index.ts @@ -1,18 +1,15 @@ -import { useContext } from "react"; +import { useContext, useMemo } from "react"; import useSWR, { SWRConfiguration, useSWRConfig } from "swr"; import useSWRInfinite from "swr/infinite"; import useSWRMutation, { SWRMutationConfiguration } from "swr/mutation"; import { getUserColor } from "../colors"; import { ProjectContext } from "../context"; -import { useMemo } from "react"; import { useComputedColorScheme } from "@mantine/core"; import { useAuth } from "../auth"; import { fetcher } from "../fetcher"; -import { CheckLogic } from "shared"; - type KeyType = string | ((...args: any[]) => string); function generateKey(