From 893b1a6727af8b59e7d4cec79b4dc0311d6e6134 Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Mon, 23 Dec 2024 17:33:08 -0800 Subject: [PATCH] Rebase with HierarchicalTableIdentifier changes --- .../catalog/TestCatalogConversionFactory.java | 7 ++--- .../conversion/TestConversionController.java | 8 ++---- .../apache/xtable/testutil/ITTestUtils.java | 4 +-- .../xtable/utilities/RunCatalogSync.java | 28 ++++++++++++++++--- .../src/test/resources/catalogConfig.yaml | 26 +++++++---------- 5 files changed, 41 insertions(+), 32 deletions(-) diff --git a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java index 53e68923..3d75221a 100644 --- a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java +++ b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java @@ -27,7 +27,7 @@ import org.apache.xtable.conversion.ExternalCatalogConfig; import org.apache.xtable.conversion.TargetCatalogConfig; -import org.apache.xtable.model.catalog.CatalogTableIdentifier; +import org.apache.xtable.model.catalog.HierarchicalTableIdentifier; import org.apache.xtable.spi.extractor.CatalogConversionSource; import org.apache.xtable.spi.sync.CatalogSyncClient; import org.apache.xtable.testutil.ITTestUtils.TestCatalogConversionSourceImpl; @@ -61,10 +61,7 @@ void createForCatalog() { .catalogProperties(Collections.emptyMap()) .build()) .catalogTableIdentifier( - CatalogTableIdentifier.builder() - .databaseName("target-database") - .tableName("target-tableName") - .build()) + new HierarchicalTableIdentifier("target-database", "target-tableName")) .build(); CatalogSyncClient catalogSyncClient = CatalogConversionFactory.getInstance() diff --git a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java index 0600c5ee..2a6461de 100644 --- a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java +++ b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java @@ -57,7 +57,7 @@ import org.apache.xtable.model.InternalSnapshot; import org.apache.xtable.model.InternalTable; import org.apache.xtable.model.TableChange; -import org.apache.xtable.model.catalog.CatalogTableIdentifier; +import org.apache.xtable.model.catalog.HierarchicalTableIdentifier; import org.apache.xtable.model.metadata.TableSyncMetadata; import org.apache.xtable.model.storage.TableFormat; import org.apache.xtable.model.sync.SyncMode; @@ -594,10 +594,8 @@ private TargetCatalogConfig getTargetCatalog(String suffix) { .catalogProperties(Collections.emptyMap()) .build()) .catalogTableIdentifier( - CatalogTableIdentifier.builder() - .databaseName("target-database" + suffix) - .tableName("target-tableName" + suffix) - .build()) + new HierarchicalTableIdentifier( + "target-database" + suffix, "target-tableName" + suffix)) .build(); } diff --git a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java index 04be989d..d194558a 100644 --- a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java +++ b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java @@ -67,12 +67,12 @@ public String getStorageLocation(Object o) { } @Override - public boolean hasDatabase(String databaseName) { + public boolean hasDatabase(CatalogTableIdentifier tableIdentifier) { return false; } @Override - public void createDatabase(String databaseName) {} + public void createDatabase(CatalogTableIdentifier tableIdentifier) {} @Override public Object getTable(CatalogTableIdentifier tableIdentifier) { diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java index a423fbc2..4d12d088 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java @@ -57,10 +57,12 @@ import org.apache.xtable.conversion.TargetCatalogConfig; import org.apache.xtable.conversion.TargetTable; import org.apache.xtable.model.catalog.CatalogTableIdentifier; +import org.apache.xtable.model.catalog.HierarchicalTableIdentifier; import org.apache.xtable.model.sync.SyncMode; import org.apache.xtable.reflection.ReflectionUtils; import org.apache.xtable.spi.extractor.CatalogConversionSource; import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.StorageIdentifier; +import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.TableIdentifier; import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.TargetTableIdentifier; /** @@ -156,7 +158,8 @@ public static void main(String[] args) throws Exception { } else { sourceTable = catalogConversionSource.getSourceTable( - dataset.getSourceCatalogTableIdentifier().getCatalogTableIdentifier()); + getCatalogTableIdentifier( + dataset.getSourceCatalogTableIdentifier().getTableIdentifier())); } List targetTables = new ArrayList<>(); Map> targetCatalogs = new HashMap<>(); @@ -178,7 +181,8 @@ public static void main(String[] args) throws Exception { .add( TargetCatalogConfig.builder() .catalogTableIdentifier( - targetCatalogTableIdentifier.getCatalogTableIdentifier()) + getCatalogTableIdentifier( + targetCatalogTableIdentifier.getTableIdentifier())) .catalogConfig(catalogsById.get(targetCatalogTableIdentifier.getCatalogId())) .build()); } @@ -229,6 +233,13 @@ static Map getConversionSourceProviders( return CONVERSION_SOURCE_PROVIDERS; } + static CatalogTableIdentifier getCatalogTableIdentifier(TableIdentifier tableIdentifier) { + if (tableIdentifier.getHierarchicalId() != null) { + return new HierarchicalTableIdentifier(tableIdentifier.getHierarchicalId()); + } + throw new IllegalArgumentException("Invalid tableIdentifier configuration provided"); + } + @Data public static class DatasetConfig { /** @@ -258,7 +269,7 @@ public static class Dataset { @Data public static class SourceTableIdentifier { /** Specifies the table identifier in the source catalog. */ - CatalogTableIdentifier catalogTableIdentifier; + TableIdentifier tableIdentifier; /** * (Optional) Provides direct storage details such as a table’s base path (like an S3 * location) and the partition specification. This allows reading from a source even if it is @@ -280,7 +291,16 @@ public static class TargetTableIdentifier { */ String tableFormat; /** Specifies the table identifier in the target catalog. */ - CatalogTableIdentifier catalogTableIdentifier; + TableIdentifier tableIdentifier; + } + + @Data + public static class TableIdentifier { + /** + * Specifics the three level hierarchical table identifier for {@link + * HierarchicalTableIdentifier} + */ + String hierarchicalId; } /** diff --git a/xtable-utilities/src/test/resources/catalogConfig.yaml b/xtable-utilities/src/test/resources/catalogConfig.yaml index 84773235..05b2df4b 100644 --- a/xtable-utilities/src/test/resources/catalogConfig.yaml +++ b/xtable-utilities/src/test/resources/catalogConfig.yaml @@ -43,20 +43,17 @@ targetCatalogs: key33: "value3" datasets: - sourceCatalogTableIdentifier: - catalogTableIdentifier: - databaseName: "source-database-1" - tableName: "source-1" + tableIdentifier: + hierarchicalId: "source-database-1.source-1" targetCatalogTableIdentifiers: - catalogId: "target-1" tableFormat: "DELTA" - catalogTableIdentifier: - databaseName: "target-database-1" - tableName: "target-tableName-1" + tableIdentifier: + hierarchicalId: "target-database-1.target-tableName-1" - catalogId: "target-2" tableFormat: "HUDI" - catalogTableIdentifier: - databaseName: "target-database-2" - tableName: "target-tableName-2-delta" + tableIdentifier: + hierarchicalId: "target-database-2.target-tableName-2-delta" - sourceCatalogTableIdentifier: storageIdentifier: tableBasePath: s3://tpc-ds-datasets/1GB/hudi/catalog_sales @@ -66,12 +63,9 @@ datasets: targetCatalogTableIdentifiers: - catalogId: "target-2" tableFormat: "ICEBERG" - catalogTableIdentifier: - databaseName: "target-database-2" - tableName: "target-tableName-2" + tableIdentifier: + hierarchicalId: "target-database-2.target-tableName-2" - catalogId: "target-3" tableFormat: "HUDI" - catalogTableIdentifier: - catalogName: "default-catalog-2" - databaseName: "target-database-3" - tableName: "target-tableName-3" + tableIdentifier: + hierarchicalId: "default-catalog-2.target-database-3.target-tableName-3"