Skip to content

Commit

Permalink
Merge pull request #692 from bci-oss/622-sql-generator-handling-colle…
Browse files Browse the repository at this point in the history 
…ctions-of-entities

Update SQL generator for handling collections of entities
  • Loading branch information
@Yauhenikapl
Yauhenikapl authored Jan 13, 2025
2 parents aa29771 + 6d2920b commit dfee56f
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 17 deletions.
66 changes: 60 additions & 6 deletions ...smf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.eclipse.esmf.metamodel.characteristic.Collection;
import org.eclipse.esmf.metamodel.characteristic.Either;
import org.eclipse.esmf.metamodel.characteristic.Trait;
import org.eclipse.esmf.metamodel.characteristic.impl.DefaultList;
import org.eclipse.esmf.metamodel.vocabulary.SAMM;
import org.eclipse.esmf.samm.KnownVersion;

Expand Down Expand Up @@ -170,6 +171,7 @@ private String escapeComment( final String comment ) {

@Override
public String visitAspect( final Aspect aspect, final Context context ) {

final String columnDeclarations = visitStructureElement( aspect, context );
final String comment = config.includeTableComment()
? Optional.ofNullable( aspect.getDescription( config.commentLanguage() ) ).map( description ->
Expand All @@ -194,7 +196,7 @@ public String visitProperty( final Property property, final Context context ) {
}

return property.getCharacteristic().get().accept( this, context.copy()
.prefix( ( context.prefix().isEmpty() ? "" : context.prefix() + LEVEL_DELIMITER ) + columnName( property ) )
.prefix( (context.prefix().isEmpty() ? "" : context.prefix() + LEVEL_DELIMITER) + columnName( property ) )
.currentProperty( property )
.build() );
}
Expand Down Expand Up @@ -256,11 +258,63 @@ public String visitCollection( final Collection collection, final Context contex
? Optional.ofNullable( Optional.ofNullable( context.forceDescriptionFromElement() ).orElse( property )
.getDescription( config.commentLanguage() ) )
: Optional.empty();
final String typeDef = type.isComplexType()
? entityToStruct( type.as( ComplexType.class ), false ).toString()
: type.accept( this, context );
return column( context.prefix(), "ARRAY<" + typeDef + ">", property.isOptional() || context.forceOptional(),
comment );

if ( type.isComplexType() ) {
// Flattening collections of complex types
final ComplexType complexType = type.as( ComplexType.class );
collection.as( Collection.class ).getCollectionType();
return processComplexType( complexType, context, context.prefix(), collection.is( DefaultList.class ) );
} else {
// Handle scalar types normally
final String typeDef = type.accept( this, context );
return column(
context.prefix(),
"ARRAY<" + typeDef + ">",
property.isOptional() || context.forceOptional(),
comment
);
}
}

private String processComplexType( final ComplexType entity, final Context context, final String parentPrefix,
final boolean isDefaultList ) {
StringBuilder columns = new StringBuilder();
final String lineDelimiter = ",\n ";

entity.getAllProperties().forEach( property -> {
if ( property.getDataType().isEmpty() || property.isNotInPayload() ) {
return; // Skip properties with no data type or not in payload
}

final Type type = property.getDataType().get();
String columnPrefix = columnName( property );

if ( parentPrefix.contains( LEVEL_DELIMITER ) ) {
columnPrefix = parentPrefix + LEVEL_DELIMITER + columnName( property );

columns.append( column( columnPrefix + "_id", "BIGINT", false, Optional.empty() ) )
.append( lineDelimiter );
}

if ( isDefaultList && !parentPrefix.contains( LEVEL_DELIMITER ) ) {
columnPrefix = parentPrefix + LEVEL_DELIMITER + columnName( property );
}

if ( type instanceof Scalar ) {
final String typeDef = type.accept( this, context );
columns.append( column( columnPrefix, typeDef, property.isOptional(),
Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) )
.append( lineDelimiter );
} else if ( type instanceof ComplexType ) {
columns.append( processComplexType( type.as( ComplexType.class ), context, columnPrefix, type.is( DefaultList.class ) ) );
}
} );

if ( !columns.isEmpty() && columns.toString().endsWith( ",\n " ) ) {
columns.setLength( columns.length() - 4 ); // Remove last ",\n "
}

return columns.toString();
}

private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity, final boolean isInsideNestedType ) {
Expand Down
42 changes: 32 additions & 10 deletions ...aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_collections_with_element_characteristic_a
void testAspectWithCollectionAndElementCharacteristic() {
assertThat( sql( TestAspect.ASPECT_WITH_COLLECTION_AND_ELEMENT_CHARACTERISTIC ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_collection_and_element_characteristic (
items ARRAY<STRUCT<test_property: STRING NOT NULL>> NOT NULL
test_property STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithCollectionAndElementCharacteristic');
""" );
Expand Down Expand Up @@ -106,7 +106,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_complex_collection_enum (
void testAspectWithComplexEntityCollectionEnum() {
assertThat( sql( TestAspect.ASPECT_WITH_COMPLEX_ENTITY_COLLECTION_ENUM ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_complex_entity_collection_enum (
my_property_one__entity_property_one ARRAY<STRUCT<entity_property_two: STRING NOT NULL>> NOT NULL
my_property_one__entity_property_one__entity_property_two_id BIGINT NOT NULL,
my_property_one__entity_property_one__entity_property_two STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexEntityCollectionEnum');
""" );
Expand Down Expand Up @@ -198,7 +199,11 @@ CREATE TABLE IF NOT EXISTS aspect_with_entity_instance_with_scalar_list_property
void testAspectWithEntityList() {
assertThat( sql( TestAspect.ASPECT_WITH_ENTITY_LIST ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_entity_list (
test_list ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL>> NOT NULL
test_list__test_string STRING NOT NULL,
test_list__test_int INT NOT NULL,
test_list__test_float FLOAT NOT NULL,
test_list__test_local_date_time TIMESTAMP NOT NULL,
test_list__random_value STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithEntityList');
""" );
Expand All @@ -209,7 +214,8 @@ void testAspectWithEntityWithNestedEntityListProperty() {
assertThat( sql( TestAspect.ASPECT_WITH_ENTITY_WITH_NESTED_ENTITY_LIST_PROPERTY ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_entity_with_nested_entity_list_property (
test_property__code SMALLINT NOT NULL,
test_property__test_list ARRAY<STRUCT<nested_entity_property: STRING NOT NULL>> NOT NULL
test_property__test_list__nested_entity_property_id BIGINT NOT NULL,
test_property__test_list__nested_entity_property STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithEntityWithNestedEntityListProperty');
""" );
Expand All @@ -219,7 +225,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_entity_with_nested_entity_list_property (
void testAspectWithExtendedEntity() {
assertThat( sql( TestAspect.ASPECT_WITH_EXTENDED_ENTITY ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_extended_entity (
test_property ARRAY<STRUCT<parent_string: STRING NOT NULL, parent_of_parent_string: STRING NOT NULL>> NOT NULL COMMENT 'This is a test property.'
parent_string STRING NOT NULL,
parent_of_parent_string STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithExtendedEntity');
""" );
Expand Down Expand Up @@ -308,8 +315,16 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entities_on_multiple_levels (
void testAspectWithMultipleEntityCollections() {
assertThat( sql( TestAspect.ASPECT_WITH_MULTIPLE_ENTITY_COLLECTIONS ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections (
test_list_one ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL>> NOT NULL,
test_list_two ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL>> NOT NULL
test_list_one__test_string STRING NOT NULL,
test_list_one__test_int INT NOT NULL,
test_list_one__test_float FLOAT NOT NULL,
test_list_one__test_local_date_time TIMESTAMP NOT NULL,
test_list_one__random_value STRING NOT NULL,
test_list_two__test_string STRING NOT NULL,
test_list_two__test_int INT NOT NULL,
test_list_two__test_float FLOAT NOT NULL,
test_list_two__test_local_date_time TIMESTAMP NOT NULL,
test_list_two__random_value STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithMultipleEntityCollections');
""" );
Expand All @@ -319,7 +334,13 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections (
void testAspectWithNestedEntityList() {
assertThat( sql( TestAspect.ASPECT_WITH_NESTED_ENTITY_LIST ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_nested_entity_list (
test_list ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_second_list: STRUCT<test_local_date_time: TIMESTAMP, random_value: STRING> NOT NULL>> NOT NULL
test_list__test_string STRING NOT NULL,
test_list__test_int INT NOT NULL,
test_list__test_float FLOAT NOT NULL,
test_list__test_second_list__test_local_date_time_id BIGINT NOT NULL,
test_list__test_second_list__test_local_date_time TIMESTAMP NOT NULL,
test_list__test_second_list__random_value_id BIGINT NOT NULL,
test_list__test_second_list__random_value STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithNestedEntityList');
""" );
Expand Down Expand Up @@ -471,7 +492,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_sorted_set (
void testAspectWithTimeSeries() {
assertThat( sql( TestAspect.ASPECT_WITH_TIME_SERIES ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_time_series (
test_property ARRAY<STRUCT<value: STRING NOT NULL COMMENT 'The value that was recorded and is part of a time series.', timestamp: TIMESTAMP NOT NULL COMMENT 'The specific point in time when the corresponding value was recorded.'>> NOT NULL COMMENT 'This is a test property.'
value STRING NOT NULL COMMENT 'The value that was recorded and is part of a time series.',
timestamp TIMESTAMP NOT NULL COMMENT 'The specific point in time when the corresponding value was recorded.'
)
COMMENT 'This is a test description'
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithTimeSeries');
Expand All @@ -482,7 +504,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_time_series (
void testAspectWithComplexSet() {
assertThat( sql( TestAspect.ASPECT_WITH_COMPLEX_SET ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_complex_set (
test_property ARRAY<STRUCT<product_id: STRING NOT NULL>> NOT NULL COMMENT 'This is a test property.'
product_id STRING NOT NULL
)
COMMENT 'This is a test description'
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexSet');
Expand Down
2 changes: 1 addition & 1 deletion ...lipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;

public class DatabricksColumnDefinitionParserTest extends DatabricksTestBase {
class DatabricksColumnDefinitionParserTest extends DatabricksTestBase {
@Test
void testMinimalDefinition() {
final DatabricksColumnDefinition definition = new DatabricksColumnDefinitionParser( "abc STRING" ).get();
Expand Down

0 comments on commit dfee56f

Please sign in to comment.