🎉 New Destination: Starburst Galaxy (airbytehq#25399)

Co-authored-by: Mayank Vadariya <[email protected]>
shipyardapp · Apr 21, 2023 · 123a31e · 123a31e
1 parent 62f3c0b
commit 123a31e
Show file tree

Hide file tree

Showing 57 changed files with 3,841 additions and 1 deletion.
diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/starburst-galaxy.svg b/airbyte-config-oss/init-oss/src/main/resources/icons/starburst-galaxy.svg
diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/destination_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/destination_definitions.yaml
@@ -116,6 +116,13 @@
   documentationUrl: https://docs.airbyte.io/integrations/destinations/convex
   icon: convex.svg
   releaseStage: alpha
+- name: Starburst Galaxy
+  destinationDefinitionId: 4528e960-6f7b-4412-8555-7e0097e1da17
+  dockerRepository: airbyte/destination-starburst-galaxy
+  dockerImageTag: 0.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/destinations/starburst-galaxy
+  icon: starburst-galaxy.svg
+  releaseStage: alpha
 - name: Databricks Lakehouse
   destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496
   dockerRepository: airbyte/destination-databricks

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/destination_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/destination_specs.yaml
@@ -1820,6 +1820,158 @@
     - "overwrite"
     - "append"
     - "append_dedup"
+- dockerImage: "airbyte/destination-starburst-galaxy:0.0.1"
+  spec:
+    documentationUrl: "https://docs.airbyte.com/integrations/destinations/starburst-galaxy"
+    connectionSpecification:
+      $schema: "http://json-schema.org/draft-07/schema#"
+      title: "Starburst Galaxy Destination Spec"
+      type: "object"
+      required:
+      - "accept_terms"
+      - "server_hostname"
+      - "username"
+      - "password"
+      - "catalog"
+      - "staging_object_store"
+      properties:
+        accept_terms:
+          title: "Agree to the Starburst Galaxy terms & conditions"
+          type: "boolean"
+          description: "You must agree to the Starburst Galaxy <a href=\"https://www.starburst.io/terms/\"\
+            >terms & conditions</a> to use this connector."
+          default: false
+          order: 1
+        server_hostname:
+          title: "Hostname"
+          type: "string"
+          description: "Starburst Galaxy cluster hostname."
+          examples:
+          - "abc-12345678-wxyz.trino.galaxy-demo.io"
+          order: 2
+        port:
+          title: "Port"
+          type: "string"
+          description: "Starburst Galaxy cluster port."
+          default: "443"
+          examples:
+          - "443"
+          order: 3
+        username:
+          title: "User"
+          type: "string"
+          description: "Starburst Galaxy user."
+          examples:
+          - "[email protected]"
+          order: 4
+        password:
+          title: "Password"
+          type: "string"
+          description: "Starburst Galaxy password for the specified user."
+          examples:
+          - "password"
+          airbyte_secret: true
+          order: 5
+        catalog:
+          title: "Amazon S3 catalog"
+          type: "string"
+          description: "Name of the Starburst Galaxy Amazon S3 catalog."
+          examples:
+          - "sample_s3_catalog"
+          order: 6
+        catalog_schema:
+          title: "Amazon S3 catalog schema"
+          type: "string"
+          description: "The default Starburst Galaxy Amazon S3 catalog schema where\
+            \ tables are written to if the source does not specify a namespace. Defaults\
+            \ to \"public\"."
+          default: "public"
+          examples:
+          - "public"
+          order: 7
+        staging_object_store:
+          title: "Staging object store"
+          type: "object"
+          description: "Temporary storage on which temporary Iceberg table is created."
+          oneOf:
+          - title: "Amazon S3"
+            required:
+            - "object_store_type"
+            - "s3_bucket_name"
+            - "s3_bucket_path"
+            - "s3_bucket_region"
+            - "s3_access_key_id"
+            - "s3_secret_access_key"
+            properties:
+              object_store_type:
+                type: "string"
+                enum:
+                - "S3"
+                default: "S3"
+                order: 1
+              s3_bucket_name:
+                title: "S3 bucket name"
+                type: "string"
+                description: "Name of the S3 bucket"
+                examples:
+                - "airbyte_staging"
+                order: 1
+              s3_bucket_path:
+                title: "S3 bucket path"
+                type: "string"
+                description: "Directory in the S3 bucket where staging data is stored."
+                examples:
+                - "temp_airbyte__sync/test"
+                order: 2
+              s3_bucket_region:
+                title: "S3 bucket region"
+                type: "string"
+                default: "us-east-1"
+                description: "The region of the S3 bucket."
+                enum:
+                - "ap-northeast-1"
+                - "ap-southeast-1"
+                - "ap-southeast-2"
+                - "ca-central-1"
+                - "eu-central-1"
+                - "eu-west-1"
+                - "eu-west-2"
+                - "eu-west-3"
+                - "us-east-1"
+                - "us-east-2"
+                - "us-west-1"
+                - "us-west-2"
+                order: 3
+              s3_access_key_id:
+                title: "Access key"
+                type: "string"
+                description: "Access key with access to the bucket. Airbyte requires\
+                  \ read and write permissions to a given bucket."
+                examples:
+                - "A012345678910EXAMPLE"
+                airbyte_secret: true
+                order: 4
+              s3_secret_access_key:
+                title: "Secret key"
+                type: "string"
+                description: "Secret key used with the specified access key."
+                examples:
+                - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
+                airbyte_secret: true
+                order: 5
+          order: 8
+        purge_staging_table:
+          title: "Purge staging Iceberg table"
+          type: "boolean"
+          description: "Defaults to 'true'. Switch to 'false' for debugging purposes."
+          default: true
+          order: 9
+    supportsIncremental: true
+    supportsNormalization: false
+    supportsDBT: false
+    supported_destination_sync_modes:
+    - "overwrite"
+    - "append"
 - dockerImage: "airbyte/destination-databricks:1.0.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/databricks"

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json
@@ -1694,6 +1694,147 @@
     "public": true,
     "custom": false,
     "releaseStage": "alpha"
+  }, {
+    "destinationDefinitionId": "4528e960-6f7b-4412-8555-7e0097e1da17",
+    "name": "Starburst Galaxy",
+    "dockerRepository": "airbyte/destination-starburst-galaxy",
+    "dockerImageTag": "0.0.1",
+    "documentationUrl": "https://docs.airbyte.com/integrations/destinations/starburst-galaxy",
+    "icon": "starburst-galaxy.svg",
+    "spec": {
+      "documentationUrl": "https://docs.airbyte.com/integrations/destinations/starburst-galaxy",
+      "connectionSpecification": {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "title": "Starburst Galaxy Destination Spec",
+        "type": "object",
+        "required": [ "accept_terms", "server_hostname", "username", "password", "catalog", "staging_object_store" ],
+        "properties": {
+          "accept_terms": {
+            "title": "Agree to the Starburst Galaxy terms & conditions",
+            "type": "boolean",
+            "description": "You must agree to the Starburst Galaxy <a href=\"https://www.starburst.io/terms/\">terms & conditions</a> to use this connector.",
+            "default": false,
+            "order": 1
+          },
+          "server_hostname": {
+            "title": "Hostname",
+            "type": "string",
+            "description": "Starburst Galaxy cluster hostname.",
+            "examples": [ "abc-12345678-wxyz.trino.galaxy-demo.io" ],
+            "order": 2
+          },
+          "port": {
+            "title": "Port",
+            "type": "string",
+            "description": "Starburst Galaxy cluster port.",
+            "default": "443",
+            "examples": [ "443" ],
+            "order": 3
+          },
+          "username": {
+            "title": "User",
+            "type": "string",
+            "description": "Starburst Galaxy user.",
+            "examples": [ "[email protected]" ],
+            "order": 4
+          },
+          "password": {
+            "title": "Password",
+            "type": "string",
+            "description": "Starburst Galaxy password for the specified user.",
+            "examples": [ "password" ],
+            "airbyte_secret": true,
+            "order": 5
+          },
+          "catalog": {
+            "title": "Amazon S3 catalog",
+            "type": "string",
+            "description": "Name of the Starburst Galaxy Amazon S3 catalog.",
+            "examples": [ "sample_s3_catalog" ],
+            "order": 6
+          },
+          "catalog_schema": {
+            "title": "Amazon S3 catalog schema",
+            "type": "string",
+            "description": "The default Starburst Galaxy Amazon S3 catalog schema where tables are written to if the source does not specify a namespace. Defaults to \"public\".",
+            "default": "public",
+            "examples": [ "public" ],
+            "order": 7
+          },
+          "staging_object_store": {
+            "title": "Staging object store",
+            "type": "object",
+            "description": "Temporary storage on which temporary Iceberg table is created.",
+            "oneOf": [ {
+              "title": "Amazon S3",
+              "required": [ "object_store_type", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "s3_access_key_id", "s3_secret_access_key" ],
+              "properties": {
+                "object_store_type": {
+                  "type": "string",
+                  "enum": [ "S3" ],
+                  "default": "S3",
+                  "order": 1
+                },
+                "s3_bucket_name": {
+                  "title": "S3 bucket name",
+                  "type": "string",
+                  "description": "Name of the S3 bucket",
+                  "examples": [ "airbyte_staging" ],
+                  "order": 1
+                },
+                "s3_bucket_path": {
+                  "title": "S3 bucket path",
+                  "type": "string",
+                  "description": "Directory in the S3 bucket where staging data is stored.",
+                  "examples": [ "temp_airbyte__sync/test" ],
+                  "order": 2
+                },
+                "s3_bucket_region": {
+                  "title": "S3 bucket region",
+                  "type": "string",
+                  "default": "us-east-1",
+                  "description": "The region of the S3 bucket.",
+                  "enum": [ "ap-northeast-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "us-east-1", "us-east-2", "us-west-1", "us-west-2" ],
+                  "order": 3
+                },
+                "s3_access_key_id": {
+                  "title": "Access key",
+                  "type": "string",
+                  "description": "Access key with access to the bucket. Airbyte requires read and write permissions to a given bucket.",
+                  "examples": [ "A012345678910EXAMPLE" ],
+                  "airbyte_secret": true,
+                  "order": 4
+                },
+                "s3_secret_access_key": {
+                  "title": "Secret key",
+                  "type": "string",
+                  "description": "Secret key used with the specified access key.",
+                  "examples": [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ],
+                  "airbyte_secret": true,
+                  "order": 5
+                }
+              }
+            } ],
+            "order": 8
+          },
+          "purge_staging_table": {
+            "title": "Purge staging Iceberg table",
+            "type": "boolean",
+            "description": "Defaults to 'true'. Switch to 'false' for debugging purposes.",
+            "default": true,
+            "order": 9
+          }
+        }
+      },
+      "supportsIncremental": true,
+      "supportsNormalization": false,
+      "supportsDBT": false,
+      "supported_destination_sync_modes": [ "overwrite", "append" ]
+    },
+    "tombstone": false,
+    "public": true,
+    "custom": false,
+    "releaseStage": "alpha"
   }, {
     "destinationDefinitionId": "072d5540-f236-4294-ba7c-ade8fd918496",
     "name": "Databricks Lakehouse",

diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java
@@ -12,6 +12,7 @@ public enum DatabaseDriver {
   CLICKHOUSE("com.clickhouse.jdbc.ClickHouseDriver", "jdbc:clickhouse:%s://%s:%d/%s"),
   DATABRICKS("com.databricks.client.jdbc.Driver", "jdbc:databricks://%s:%s;HttpPath=%s;SSL=1;UserAgentEntry=Airbyte"),
   DB2("com.ibm.db2.jcc.DB2Driver", "jdbc:db2://%s:%d/%s"),
+  STARBURST("io.trino.jdbc.TrinoDriver", "jdbc:trino://%s:%s/%s?SSL=true&source=airbyte"),
   MARIADB("org.mariadb.jdbc.Driver", "jdbc:mariadb://%s:%d/%s"),
   MSSQLSERVER("com.microsoft.sqlserver.jdbc.SQLServerDriver", "jdbc:sqlserver://%s:%d/%s"),
   MYSQL("com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%d/%s"),

diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/.dockerignore b/airbyte-integrations/connectors/destination-starburst-galaxy/.dockerignore
@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build
diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/BOOTSTRAP.md b/airbyte-integrations/connectors/destination-starburst-galaxy/BOOTSTRAP.md
@@ -0,0 +1,8 @@
+# Starburst Galaxy destination connector bootstrap
+
+This destination syncs data to Amazon S3 catalog in [Starburst Galaxy](https://www.starburst.io/platform/starburst-galaxy/) by completing the following steps:
+
+1. Persist source stream data to S3 staging storage in the Iceberg table format.
+2. Create a destination Iceberg table in Amazon S3 catalog in Starburst Galaxy from the staged Iceberg table.
+
+Learn more from [the Airbyte documentation](https://docs.airbyte.io/integrations/destinations/starburst-galaxy).
diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/Dockerfile b/airbyte-integrations/connectors/destination-starburst-galaxy/Dockerfile
@@ -0,0 +1,18 @@
+FROM airbyte/integration-base-java:dev AS build
+
+WORKDIR /airbyte
+ENV APPLICATION destination-starburst-galaxy
+
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+
+RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
+
+FROM airbyte/integration-base-java:dev
+
+WORKDIR /airbyte
+ENV APPLICATION destination-starburst-galaxy
+
+COPY --from=build /airbyte /airbyte
+
+LABEL io.airbyte.version=0.0.1
+LABEL io.airbyte.name=airbyte/destination-starburst-galaxy