Skip to content

Commit

Permalink
Shuffled destination-jdbc into bases folder (airbytehq#25120)
Browse files Browse the repository at this point in the history
* Testing changes for reshuffle

* Resolves dependency/race-condition with base classes and docker creation

* Removes dockerfile

* Updates java connectors gradle dependencies
  • Loading branch information
ryankfu authored Apr 17, 2023
1 parent f2f788f commit 0dcc653
Show file tree
Hide file tree
Showing 63 changed files with 42 additions and 57 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
# Java-based destination connectors
/airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations
/airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations
/airbyte-integrations/connectors/destination-jdbc/ @airbytehq/destinations
/airbyte-integrations/bases/bases-destination-jdbc/ @airbytehq/destinations
/airbyte-integrations/connectors/destination-bigquery/ @airbytehq/destinations
/airbyte-integrations/connectors/destination-bigquery-denormalized/ @airbytehq/destinations
/airbyte-integrations/connectors/destination-azure-blob-storage/ @airbytehq/destinations
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,18 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**Strategy:
* <p>1. Create a final table for each stream
* <p>2. Accumulate records in a buffer. One buffer per stream
* <p>3. As records accumulate write them in batch to the database. We set a minimum numbers of records
/**
* Strategy:
* <p>
* 1. Create a final table for each stream
* <p>
* 2. Accumulate records in a buffer. One buffer per stream
* <p>
* 3. As records accumulate write them in batch to the database. We set a minimum numbers of records
* before writing to avoid wasteful record-wise writes. In the case with slow syncs this will be
* superseded with a periodic record flush from {@link BufferedStreamConsumer#periodicBufferFlush()}
* <p>4. Once all records have been written to buffer, flush the buffer and write any remaining records
* <p>
* 4. Once all records have been written to buffer, flush the buffer and write any remaining records
* to the database (regardless of how few are left)
*/
public class JdbcBufferedConsumerFactory {
Expand Down Expand Up @@ -118,9 +123,12 @@ private static String getOutputSchema(final AirbyteStream stream,

/**
* Sets up destination storage through:
* <p>1. Creates Schema (if not exists)
* <p>2. Creates airybte_raw table (if not exists)
* <p>3. <Optional>Truncates table if sync mode is in OVERWRITE
* <p>
* 1. Creates Schema (if not exists)
* <p>
* 2. Creates airybte_raw table (if not exists)
* <p>
* 3. <Optional>Truncates table if sync mode is in OVERWRITE
*
* @param database JDBC database to connect to
* @param sqlOperations interface for execution SQL queries
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-config-oss:config-models-oss')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

implementation 'com.azure:azure-storage-blob:12.20.2'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-clickhouse')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ dependencies {
implementation project(':airbyte-config-oss:config-models-oss')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ dependencies {
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:bases:base-java-s3')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-s3')
implementation project(':airbyte-integrations:connectors:destination-azure-blob-storage')
implementation group: 'com.databricks', name: 'databricks-jdbc', version: '2.6.25'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-config-oss:config-models-oss')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.12.47'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')

implementation 'com.exasol:exasol-jdbc:7.1.17'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ dependencies {
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:bases:base-java-s3')
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

implementation platform('com.amazonaws:aws-java-sdk-bom:1.12.14')
Expand Down

This file was deleted.

20 changes: 0 additions & 20 deletions airbyte-integrations/connectors/destination-jdbc/Dockerfile

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ dependencies {
implementation project(':airbyte-config-oss:config-models-oss')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

implementation 'org.mariadb.jdbc:mariadb-java-client:2.7.4'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ dependencies {
implementation project(':airbyte-config-oss:config-models-oss')
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-mssql')
implementation project(':airbyte-test-utils')

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-test-utils')

implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-mysql')

implementation 'mysql:mysql-connector-java:8.0.22'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')

implementation 'mysql:mysql-connector-java:8.0.22'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-oracle')

implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')

implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-postgres')

integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')

testImplementation project(':airbyte-test-utils')

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:bases:base-java-s3')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ dependencies {
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')

implementation group: 'com.rockset', name: 'rockset-java', version: '0.9.0'
implementation group: 'org.awaitility', name: 'awaitility', version: '4.1.1'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ dependencies {
implementation project(':airbyte-config-oss:config-models-oss')
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-integrations:connectors:destination-gcs')
implementation project(':airbyte-integrations:bases:base-java-s3')
implementation libs.airbyte.protocol
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation 'com.teradata.jdbc:terajdbc4:17.20.00.12'

integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ dependencies {
implementation project(':airbyte-db:db-lib')
implementation project(':airbyte-integrations:bases:base-java')
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)

implementation 'mysql:mysql-connector-java:8.0.30'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ dependencies {
implementation libs.airbyte.protocol
implementation project(':airbyte-integrations:bases:base-java')
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
implementation project(':airbyte-integrations:connectors:destination-jdbc')
implementation project(':airbyte-integrations:bases:bases-destination-jdbc')
implementation project(':airbyte-db:db-lib')

implementation 'com.yugabyte:jdbc-yugabytedb:42.3.5-yb-1'
Expand Down
2 changes: 1 addition & 1 deletion settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
include ':airbyte-integrations:bases:base-java'
include ':airbyte-integrations:bases:base-java-s3'
include ':airbyte-integrations:bases:base-normalization'
include ':airbyte-integrations:bases:bases-destination-jdbc' // needs to be lexicographically after base-java and base-normalization to avoid race condition
include ':airbyte-integrations:bases:base-standard-source-test-file'
include ':airbyte-integrations:bases:connector-acceptance-test'
include ':airbyte-integrations:bases:standard-destination-test'
Expand All @@ -115,7 +116,6 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"

// Needed by normalization integration tests
include ':airbyte-integrations:connectors:destination-bigquery'
include ':airbyte-integrations:connectors:destination-jdbc'
include ':airbyte-integrations:connectors:destination-mysql'
include ':airbyte-integrations:connectors:destination-postgres'
include ':airbyte-integrations:connectors:destination-redshift'
Expand Down
2 changes: 1 addition & 1 deletion tools/bin/ci_check_dependency.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
re.compile(".*-strict-encrypt$"),
re.compile("^destination-dev-null$"),
re.compile("^destination-scaffold-destination-python$"),
re.compile("^destination-jdbc$")
re.compile("^bases-destination-jdbc$")
]
COMMENT_TEMPLATE_PATH = ".github/comment_templates/connector_dependency_template.md"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ class GradleTask(Step, ABC):
]

DESTINATION_BUILD_INCLUDE = [
"airbyte-integrations/connectors/destination-jdbc",
"airbyte-integrations/bases/bases-destination-jdbc",
# destination-bigquery uses utils from destination gcs
"airbyte-integrations/connectors/destination-gcs",
]
Expand Down

0 comments on commit 0dcc653

Please sign in to comment.