diff --git a/backend/pom.xml b/backend/pom.xml index d2987fa93b..0b75eb8fee 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -54,20 +54,6 @@ - - default-jar - - jar - - - - - java.base/java.lang ALL-UNNAMED java.base/java.util ALL-UNNAMED java.base/java.nio ALL-UNNAMED java.base/sun.nio.ch ALL-UNNAMED - - - - - diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 2ceee255fb..96e6205e84 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.apiv1; import java.net.URI; - import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -15,6 +14,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.SearchIndexId; import com.bakdata.conquery.models.index.FrontendValueIndex; import com.bakdata.conquery.models.index.FrontendValueIndexKey; +import com.bakdata.conquery.models.index.IndexCreationException; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.index.search.SearchIndex; import com.bakdata.conquery.util.io.FileUtil; @@ -41,8 +41,6 @@ @CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class) public class FilterTemplate extends IdentifiableImpl implements Searchable, SearchIndex { - private static final long serialVersionUID = 1L; - @NotNull @NsIdRef private Dataset dataset; @@ -89,7 +87,7 @@ public boolean isSearchDisabled() { return false; } - public TrieSearch createTrieSearch(IndexConfig config) { + public TrieSearch createTrieSearch(IndexConfig config) throws IndexCreationException { final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath()); log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index d1c5686040..d48094fbad 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -85,7 +85,7 @@ public void run(Manager manager) throws InterruptedException { this.manager = manager; final ObjectMapper objectMapper = environment.getObjectMapper(); - customizeApiObjectMapper(objectMapper); + customizeApiObjectMapper(objectMapper, getDatasetRegistry(), getMetaStorage(), config, validator); // FormScanner needs to be instantiated before plugins are initialized @@ -182,7 +182,12 @@ protected void configure() { * * @param objectMapper to be configured (should be a JSON mapper) */ - public void customizeApiObjectMapper(ObjectMapper objectMapper) { + public static void customizeApiObjectMapper( + ObjectMapper objectMapper, + DatasetRegistry datasetRegistry, + MetaStorage metaStorage, + ConqueryConfig config, + Validator validator) { // Set serialization config SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); @@ -200,18 +205,18 @@ public void customizeApiObjectMapper(ObjectMapper objectMapper) { final MutableInjectableValues injectableValues = new MutableInjectableValues(); objectMapper.setInjectableValues(injectableValues); - injectableValues.add(Validator.class, getValidator()); + injectableValues.add(Validator.class, validator); - getDatasetRegistry().injectInto(objectMapper); - getMetaStorage().injectInto(objectMapper); - getConfig().injectInto(objectMapper); + datasetRegistry.injectInto(objectMapper); + metaStorage.injectInto(objectMapper); + config.injectInto(objectMapper); } /** * Create a new internal object mapper for binary (de-)serialization that is equipped with {@link ManagerNode} related injectables. * * @return a preconfigured binary object mapper - * @see ManagerNode#customizeApiObjectMapper(ObjectMapper) + * @see ManagerNode#customizeApiObjectMapper(ObjectMapper, DatasetRegistry, MetaStorage, ConqueryConfig, Validator) */ public ObjectMapper createInternalObjectMapper(Class viewClass) { return getInternalObjectMapperCreator().createInternalObjectMapper(viewClass); @@ -219,7 +224,7 @@ public ObjectMapper createInternalObjectMapper(Class viewClass) private void loadMetaStorage() { log.info("Opening MetaStorage"); - getMetaStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); + getMetaStorage().openStores(createInternalObjectMapper(View.Persistence.Manager.class)); log.info("Loading MetaStorage"); getMetaStorage().loadData(); log.info("MetaStorage loaded {}", getMetaStorage()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java new file mode 100644 index 0000000000..e70ceda844 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.io.jackson; + +import com.fasterxml.jackson.databind.util.StdConverter; + +/** + * Interface for class instances that need initialization after deserialization and value injection. + * Let the class implement this interface and annotate the class with: + * 
+ *  {@code
+ *  @JsonDeserialize(converter = Initializing.Converter.class )
+ *  }
+ *
+ * @param + * @implNote Every class that inherits from an initializing class needs to define its own Converter. + * Otherwise, the class is parsed as the super class, and its overrides are not called. + */ +public interface Initializing { + + void init(); + + class Converter extends StdConverter { + + @Override + public T convert(T value) { + value.init(); + return value; + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index 7de472e7b3..f22cc675e9 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -3,6 +3,8 @@ import java.util.Collection; import java.util.Objects; +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.xodus.stores.CachedStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.config.StoreFactory; @@ -16,10 +18,12 @@ import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; @Slf4j -public class NamespaceStorage extends NamespacedStorage { +@ToString +public class NamespaceStorage extends NamespacedStorage implements Injectable { protected IdentifiableStore internToExternMappers; protected IdentifiableStore searchIndexes; @@ -40,11 +44,6 @@ private void decorateIdMapping(SingletonStore idMapping) { .onAdd(mapping -> mapping.setStorage(this)); } - private void decorateInternToExternMappingStore(IdentifiableStore store) { - // We don't call internToExternMapper::init this is done by the first select that needs the mapping - } - - @Override public void openStores(ObjectMapper objectMapper) { super.openStores(objectMapper); @@ -57,7 +56,6 @@ public void openStores(ObjectMapper objectMapper) { preview = getStorageFactory().createPreviewStore(super.getPathName(), getCentralRegistry(), objectMapper); entity2Bucket = getStorageFactory().createEntity2BucketStore(super.getPathName(), objectMapper); - decorateInternToExternMappingStore(internToExternMappers); decorateIdMapping(idMapping); } @@ -169,4 +167,10 @@ public PreviewConfig getPreviewConfig() { public void removePreviewConfig() { preview.remove(); } + + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return super.inject(values).add(NamespaceStorage.class, this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index a9a7378760..25baf5df05 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -4,6 +4,8 @@ import java.util.Collection; import java.util.List; +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.datasets.Column; @@ -35,7 +37,7 @@ */ @Slf4j @ToString(onlyExplicitlyIncluded = true) -public abstract class NamespacedStorage extends ConqueryStorage { +public abstract class NamespacedStorage extends ConqueryStorage implements Injectable { @Getter protected final CentralRegistry centralRegistry = new CentralRegistry(); @@ -59,6 +61,7 @@ public NamespacedStorage(StoreFactory storageFactory, String pathName) { public void openStores(ObjectMapper objectMapper) { // Before we start to parse the stores we need to replace the injected value for the IdResolveContext (from DatasetRegistry to this centralRegistry) new SingletonNamespaceCollection(centralRegistry).injectInto(objectMapper); + this.injectInto(objectMapper); dataset = storageFactory.createDatasetStore(pathName, objectMapper); secondaryIds = storageFactory.createSecondaryIdDescriptionStore(centralRegistry, pathName, objectMapper); @@ -67,9 +70,7 @@ public void openStores(ObjectMapper objectMapper) { concepts = storageFactory.createConceptStore(centralRegistry, pathName, objectMapper); decorateDatasetStore(dataset); - decorateSecondaryIdDescriptionStore(secondaryIds); decorateTableStore(tables); - decorateImportStore(imports); decorateConceptStore(concepts); } @@ -88,10 +89,6 @@ private void decorateDatasetStore(SingletonStore store) { store.onAdd(centralRegistry::register).onRemove(centralRegistry::remove); } - private void decorateSecondaryIdDescriptionStore(IdentifiableStore store) { - // Nothing to decorate - } - private void decorateTableStore(IdentifiableStore store) { store.onAdd(table -> { for (Column column : table.getColumns()) { @@ -109,13 +106,13 @@ private void decorateTableStore(IdentifiableStore
store) { private void decorateConceptStore(IdentifiableStore> store) { store.onAdd(concept -> { - if (concept.getDataset() != null && !concept.getDataset().equals(dataset.get())) { - throw new IllegalStateException("Concept is not for this dataset."); + if (concept.getDataset() == null) { + throw new IllegalStateException("Concept had no dataset set"); } - concept.setDataset(dataset.get()); - - concept.initElements(); + if (!concept.getDataset().equals(dataset.get())) { + throw new IllegalStateException("Concept is not for this dataset."); + } concept.getSelects().forEach(centralRegistry::register); for (Connector connector : concept.getConnectors()) { @@ -145,11 +142,6 @@ private void decorateConceptStore(IdentifiableStore> store) { }); } - private void decorateImportStore(IdentifiableStore store) { - // Intentionally left blank - } - - public void addImport(Import imp) { imports.add(imp); } @@ -231,4 +223,9 @@ public Collection> getAllConcepts() { return concepts.getAll(); } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return values.add(NamespacedStorage.class, this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java index 84ff4e90d6..ac51d97c4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java @@ -2,6 +2,8 @@ import java.util.Collection; +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.datasets.concepts.Concept; @@ -13,19 +15,18 @@ import com.bakdata.conquery.models.worker.WorkerInformation; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import jakarta.validation.Validator; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @Slf4j @ToString(of = "worker") -public class WorkerStorage extends NamespacedStorage { +public class WorkerStorage extends NamespacedStorage implements Injectable { private SingletonStore worker; private IdentifiableStore buckets; private IdentifiableStore cBlocks; - public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) { + public WorkerStorage(StoreFactory storageFactory, String pathName) { super(storageFactory, pathName); } @@ -36,10 +37,6 @@ public void openStores(ObjectMapper objectMapper) { worker = getStorageFactory().createWorkerInformationStore(getPathName(), objectMapper); buckets = getStorageFactory().createBucketStore(centralRegistry, getPathName(), objectMapper); cBlocks = getStorageFactory().createCBlockStore(centralRegistry, getPathName(), objectMapper); - - decorateWorkerStore(worker); - decorateBucketStore(buckets); - decorateCBlockStore(cBlocks); } @Override @@ -58,19 +55,6 @@ public ImmutableList getStores() { } - private void decorateWorkerStore(SingletonStore store) { - // Nothing to decorate - } - - private void decorateBucketStore(IdentifiableStore store) { - // Nothing to decorate - } - - private void decorateCBlockStore(IdentifiableStore baseStoreCreator) { - // Nothing to decorate - } - - public void addCBlock(CBlock cBlock) { log.trace("Adding CBlock[{}]", cBlock.getId()); cBlocks.add(cBlock); @@ -129,4 +113,9 @@ public void removeConcept(ConceptId id) { log.debug("Removing Concept[{}]", id); concepts.remove(id); } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return super.inject(values).add(WorkerStorage.class, this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index fef5334580..af83f42f13 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -33,14 +33,17 @@ public interface NamespaceHandler { static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator, IndexService indexService) { List injectables = new ArrayList<>(); injectables.add(indexService); + injectables.add(storage); ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class); ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null); - injectables.forEach(i -> i.injectInto(persistenceMapper)); - injectables.forEach(i -> i.injectInto(communicationMapper)); - injectables.forEach(i -> i.injectInto(preprocessMapper)); + injectables.forEach(i -> { + i.injectInto(persistenceMapper); + i.injectInto(communicationMapper); + i.injectInto(preprocessMapper); + }); // Each store needs its own mapper because each injects its own registry diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 7529e92934..797df2caab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -16,8 +16,11 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; - import javax.annotation.Nullable; +import jakarta.validation.Valid; +import jakarta.validation.Validator; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.storage.IdentifiableStore; @@ -63,10 +66,6 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.Sets; import io.dropwizard.util.Duration; -import jakarta.validation.Valid; -import jakarta.validation.Validator; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import jetbrains.exodus.env.Environment; import lombok.AllArgsConstructor; import lombok.Getter; @@ -195,7 +194,7 @@ public Collection discoverNamespaceStorages() { @Override public Collection discoverWorkerStorages() { - return loadNamespacedStores("worker_", (storePath) -> new WorkerStorage(this, validator, storePath), WORKER_STORES); + return loadNamespacedStores("worker_", (storePath) -> new WorkerStorage(this, storePath), WORKER_STORES); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index b7f3d6a27d..f50f0b2375 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import jakarta.validation.Valid; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; @@ -15,8 +16,6 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.exceptions.ConfigurationException; -import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; @@ -29,7 +28,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.validation.Valid; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -74,11 +72,6 @@ public List, NamespacedIdentifiable { public static final int[] NOT_CONTAINED = new int[]{-1}; - private static final long serialVersionUID = 1L; @Nullable @JsonAlias("validityDatesTooltip") @@ -140,8 +133,4 @@ public static boolean isNotContained(int[] mostSpecificChildren) { public Dataset getDataset() { return getConcept().getDataset(); } - - public void init() { - getSelects().forEach(Select::init); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java index c6839d49e7..3401ab38ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.models.datasets.concepts; -import java.util.List; +import jakarta.validation.constraints.Min; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.models.config.IndexConfig; +import com.bakdata.conquery.models.index.IndexCreationException; import com.bakdata.conquery.models.query.FilterSearch; import com.bakdata.conquery.util.search.TrieSearch; -import com.fasterxml.jackson.annotation.JsonIgnore; -import jakarta.validation.constraints.Min; /** * @implNote This class is tightly coupled with {@link FilterSearch} and {@link com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter}. @@ -19,7 +18,7 @@ public interface Searchable { /** * All available {@link FrontendValue}s for searching in a {@link TrieSearch}. */ - TrieSearch createTrieSearch(IndexConfig config); + TrieSearch createTrieSearch(IndexConfig config) throws IndexCreationException; /** * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create. diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 0a5031840e..dae1514d87 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -32,13 +32,13 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase @Slf4j +@Getter +@Setter @EqualsAndHashCode(callSuper = true) public abstract class Select extends Labeled implements NamespacedIdentifiable { @EqualsAndHashCode.Exclude @JsonBackReference - @Getter - @Setter private SelectHolder holder; @JsonIgnore @@ -47,15 +47,11 @@ public Dataset getDataset() { return getHolder().findConcept().getDataset(); } - @Setter - @Getter private String description; /** * When set, the Frontend will preselect the Select for the User. */ - @Setter - @Getter @JsonProperty("default") private boolean isDefault = false; @@ -75,9 +71,6 @@ public SelectId createId() { return new ConceptSelectId(holder.findConcept().getId(), getName()); } - public void init() { - } - @NotNull @JsonIgnore public String getColumnName() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 8ab6a7f733..97bf335aed 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -3,8 +3,8 @@ import java.util.HashSet; import java.util.Set; import java.util.function.BiFunction; - import javax.annotation.Nullable; +import jakarta.validation.Valid; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.jackson.View; @@ -17,7 +17,6 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; -import jakarta.validation.Valid; import lombok.Getter; public abstract class MappableSingleColumnSelect extends SingleColumnSelect { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java index 0c2c90d8fd..492001c953 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java @@ -6,25 +6,24 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.Initializing; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.SelectHolder; import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; -import com.bakdata.conquery.models.exceptions.ConfigurationException; -import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Getter; -import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -33,7 +32,8 @@ */ @Slf4j @CPSType(id = "TREE", base = Concept.class) -public class TreeConcept extends Concept implements ConceptTreeNode, SelectHolder { +@JsonDeserialize(converter = TreeConcept.TreeConceptInitializer.class) +public class TreeConcept extends Concept implements ConceptTreeNode, SelectHolder, Initializing { @JsonIgnore @Getter @@ -90,21 +90,22 @@ public boolean matchesPrefix(int[] conceptPrefix) { return conceptPrefix != null && conceptPrefix[0] == 0; } - @Override - public void initElements() throws ConfigurationException, JSONException { - super.initElements(); + public void init() { setLocalId(0); localIdMap.add(this); - final List openList = new ArrayList<>(); - openList.addAll(getChildren()); + final List openList = new ArrayList<>(getChildren()); for (ConceptTreeConnector con : getConnectors()) { if (con.getCondition() == null) { continue; } - con.getCondition().init(this); + try { + con.getCondition().init(this); + } catch (ConceptConfigurationException e) { + throw new RuntimeException("Unable to init condition", e); + } } for (int i = 0; i < openList.size(); i++) { @@ -117,8 +118,7 @@ public void initElements() throws ConfigurationException, JSONException { ctc.init(); - } - catch (Exception e) { + } catch (Exception e) { throw new RuntimeException("Error trying to consolidate the node " + ctc.getLabel() + " in " + getLabel(), e); } @@ -200,7 +200,7 @@ public void removeImportCache(Import imp) { * @param ids the local id array to look for * @return the element matching the most specific local id in the array */ - public ConceptTreeNode getElementByLocalIdPath(@NonNull int[] ids) { + public ConceptTreeNode getElementByLocalIdPath(int[] ids) { final int mostSpecific = ids[ids.length - 1]; return getElementByLocalId(mostSpecific); } @@ -236,4 +236,6 @@ public String printConceptLocalId(Object rawValue, PrintSettings printSettings) return node.getLabel() + " - " + node.getDescription(); } + + public static class TreeConceptInitializer extends Initializing.Converter {} } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java new file mode 100644 index 0000000000..f8b85731d1 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexCreationException.java @@ -0,0 +1,8 @@ +package com.bakdata.conquery.models.index; + +public class IndexCreationException extends Exception { + + public IndexCreationException(IndexKey key, Throwable cause) { + super(String.format("Unable to build index from index configuration: %s", key), cause); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java index 4da2c358e2..5bc17b61a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.index; -import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; @@ -40,6 +39,7 @@ public class IndexService implements Injectable { private final String emptyDefaultLabel; private final LoadingCache, Index> mappings = CacheBuilder.newBuilder().recordStats().build(new CacheLoader<>() { + @NotNull @Override public Index load(@NotNull IndexKey key) throws Exception { @@ -86,10 +86,6 @@ public Index load(@NotNull IndexKey key) throws Exception { } } } - catch (IOException ioException) { - log.warn("Failed to open `{}`", key.getCsv(), ioException); - throw ioException; - } // Run finalizing operations on the index int2ext.finalizer(); @@ -150,13 +146,21 @@ public void evictCache() { } + /** + * Returns an index mapping from the information in the given key. + * If the index is not yet present, it is loaded. + * @param key the key describing the requested index + * @return the index mapping + * @throws IndexCreationException if the index mapping could not be loaded. + */ @SuppressWarnings("unchecked") - public , I extends Index> I getIndex(K key) { + @NotNull + public , I extends Index> I getIndex(@NotNull K key) throws IndexCreationException { try { return (I) mappings.get(key); } catch (ExecutionException e) { - throw new IllegalStateException(String.format("Unable to build index from index configuration: %s)", key), e); + throw new IndexCreationException(key, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java index 28aed27004..7a7572830b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java @@ -1,8 +1,6 @@ package com.bakdata.conquery.models.index; import com.bakdata.conquery.io.cps.CPSBase; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.Named; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; @@ -12,13 +10,11 @@ @JsonTypeInfo(property = "type", use = JsonTypeInfo.Id.CUSTOM) public interface InternToExternMapper extends NamespacedIdentifiable, Named { - void init(); - boolean initialized(); - String external(String internalValue); + void init(); - void setDataset(Dataset dataset); + String external(String internalValue); @Override InternToExternMapperId getId(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index 2fc9927e1a..1ed80b263e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -2,12 +2,14 @@ import java.net.URI; - +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.jackson.serializer.NsIdRef; +import com.bakdata.conquery.io.jackson.Initializing; +import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.identifiable.NamedImpl; @@ -17,6 +19,8 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -31,20 +35,34 @@ @ToString(onlyExplicitlyIncluded = true) @FieldNameConstants @Getter -public class MapInternToExternMapper extends NamedImpl implements InternToExternMapper, NamespacedIdentifiable { +@JsonDeserialize(converter = MapInternToExternMapper.Initializer.class ) +@EqualsAndHashCode(callSuper = true) +public class MapInternToExternMapper extends NamedImpl implements InternToExternMapper, NamespacedIdentifiable, Initializing { // We inject the service as a non-final property so, jackson will never try to create a serializer for it (in contrast to constructor injection) @JsonIgnore @JacksonInject(useInput = OptBoolean.FALSE) + @NotNull + @Setter(onMethod_ = @TestOnly) + @EqualsAndHashCode.Exclude private IndexService mapIndex; @JsonIgnore @JacksonInject(useInput = OptBoolean.FALSE) + @NotNull + @Setter(onMethod_ = @TestOnly) + @EqualsAndHashCode.Exclude private ConqueryConfig config; - @NsIdRef - @Setter + @JsonIgnore + @JacksonInject(useInput = OptBoolean.FALSE) + @NotNull + @Setter(onMethod_ = @TestOnly) + @EqualsAndHashCode.Exclude + private NamespaceStorage storage; + + @JsonIgnore @NotNull private Dataset dataset; @@ -65,35 +83,70 @@ public class MapInternToExternMapper extends NamedImpl i //Manager only @JsonIgnore @Getter(onMethod_ = {@TestOnly}) - private MapIndex int2ext = null; + @EqualsAndHashCode.Exclude + private CompletableFuture int2ext = null; @Override public synchronized void init() { + if (mapIndex == null && config == null) { + log.trace("Injections were null. Skipping init, because class was deserialized by a test object mapper"); + return; + } + + dataset = storage.getDataset(); + final URI resolvedURI = FileUtil.getResolvedUri(config.getIndex().getBaseUrl(), csv); log.trace("Resolved mapping reference csv url '{}': {}", this.getId(), resolvedURI); - int2ext = mapIndex.getIndex(new MapIndexKey(resolvedURI, internalColumn, externalTemplate)); + MapIndexKey key = new MapIndexKey(resolvedURI, internalColumn, externalTemplate); + + int2ext = CompletableFuture.supplyAsync(() -> { + try { + return mapIndex.getIndex(key); + } + catch (IndexCreationException e) { + throw new IllegalStateException(e); + } + }).whenComplete((m, e) -> { + if (e != null) { + log.warn("Unable to get index: {} (enable TRACE for exception)", key, (Exception) (log.isTraceEnabled() ? e : null)); + } + }); } @Override public boolean initialized() { - return int2ext != null; + return int2ext != null && int2ext.isDone(); } @Override public String external(String internalValue) { if(!initialized()){ + log.trace("Skip mapping for value '{}', because mapper is not initialized", internalValue); + return internalValue; + } + + if (int2ext.isCompletedExceptionally() || int2ext.isCancelled()) { + log.trace("Skip mapping for value '{}', because mapper could not be initialized", internalValue); return internalValue; } - return int2ext.getOrDefault(internalValue, internalValue); + try { + return int2ext.get().getOrDefault(internalValue, internalValue); + } catch (InterruptedException | ExecutionException e) { + // Should never be reached + log.warn("Unable to resolve mapping for internal value {} (enable TRACE for exception)", internalValue, (Exception) (log.isTraceEnabled() ? e : null)); + return internalValue; + } } @Override public InternToExternMapperId createId() { return new InternToExternMapperId(getDataset().getId(), getName()); } + + public static class Initializer extends Initializing.Converter {} } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java index b79c379006..35b60f3416 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.models.config.IndexConfig; import com.bakdata.conquery.models.datasets.concepts.Searchable; import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; +import com.bakdata.conquery.models.index.IndexCreationException; import com.bakdata.conquery.util.search.TrieSearch; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; @@ -91,12 +92,19 @@ public synchronized void addSearches(Map> /** - * Adds new values to a search. If there is no search yet for the searchable, it is created. + * Adds new values to a search. If there is no search for the searchable yet, it is created. * In order for this to work an existing search is not allowed to be shrunken yet, because shrinking * prevents from adding new values. */ public void registerValues(Searchable searchable, Collection values) { - TrieSearch search = searchCache.computeIfAbsent(searchable, (ignored) -> searchable.createTrieSearch(indexConfig)); + TrieSearch search = searchCache.computeIfAbsent(searchable, (ignored) -> { + try { + return searchable.createTrieSearch(indexConfig); + } + catch (IndexCreationException e) { + throw new IllegalStateException(e); + } + }); synchronized (search) { values.stream() diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java index 52ccbfbea9..be49168550 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java @@ -299,7 +299,7 @@ public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) { @Override public String typeInfo() { - return "MONEY"; + return MajorTypeId.MONEY.name(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index dcee848050..0c33d482d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -41,7 +41,7 @@ public abstract class Namespace extends IdResolveContext { @ToString.Include private final NamespaceStorage storage; - private final ExecutionManager executionManager; + private final ExecutionManager executionManager; // TODO: 01.07.2020 FK: This is not used a lot, as NamespacedMessages are highly convoluted and hard to decouple as is. private final JobManager jobManager; @@ -62,16 +62,14 @@ public Dataset getDataset() { public void close() { try { jobManager.close(); - } - catch (Exception e) { + } catch (Exception e) { log.error("Unable to close namespace jobmanager of {}", this, e); } try { log.info("Closing namespace storage of {}", getStorage().getDataset().getId()); storage.close(); - } - catch (IOException e) { + } catch (IOException e) { log.error("Unable to close namespace storage of {}.", this, e); } } @@ -79,8 +77,7 @@ public void close() { public void remove() { try { jobManager.close(); - } - catch (Exception e) { + } catch (Exception e) { log.error("Unable to close namespace jobmanager of {}", this, e); } @@ -96,19 +93,6 @@ public int getNumberOfEntities() { return getStorage().getNumberOfEntities(); } - public void updateInternToExternMappings() { - storage.getAllConcepts().stream() - .flatMap(c -> c.getConnectors().stream()) - .flatMap(con -> con.getSelects().stream()) - .filter(MappableSingleColumnSelect.class::isInstance) - .map(MappableSingleColumnSelect.class::cast) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); - - storage.getSecondaryIds().stream() - .filter(desc -> desc.getMapping() != null) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); - } - public void clearIndexCache() { indexService.evictCache(); } @@ -125,6 +109,19 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep return storage.getCentralRegistry(); } + public void updateInternToExternMappings() { + storage.getAllConcepts().stream() + .flatMap(c -> c.getConnectors().stream()) + .flatMap(con -> con.getSelects().stream()) + .filter(MappableSingleColumnSelect.class::isInstance) + .map(MappableSingleColumnSelect.class::cast) + .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); + + storage.getSecondaryIds().stream() + .filter(desc -> desc.getMapping() != null) + .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); + } + /** * Issues a job that initializes the search that is used by the frontend for recommendations in the filter interface of a concept. */ @@ -157,9 +154,9 @@ public void postprocessData() { getJobManager().addSlowJob(new SimpleJob( "Initiate Update Matching Stats and FilterSearch", () -> { + updateInternToExternMappings(); updateMatchingStats(); updateFilterSearch(); - updateInternToExternMappings(); } )); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java index 096554e5c5..3e20e742fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java @@ -26,7 +26,6 @@ import com.bakdata.conquery.models.messages.network.specific.ForwardToNamespace; import com.bakdata.conquery.models.query.QueryExecutor; import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.validation.Validator; import lombok.Getter; import lombok.NonNull; import lombok.Setter; @@ -84,13 +83,12 @@ public static Worker newWorker( @NonNull ExecutorService jobsExecutorService, @NonNull StoreFactory config, @NonNull String directory, - @NonNull Validator validator, boolean failOnError, int entityBucketSize, ObjectMapper persistenceMapper, ObjectMapper communicationMapper, int secondaryIdSubPlanLimit) { - WorkerStorage workerStorage = new WorkerStorage(config, validator, directory); + WorkerStorage workerStorage = new WorkerStorage(config, directory); // On the worker side we don't have to set the object writer for ForwardToWorkerMessages in WorkerInformation WorkerInformation info = new WorkerInformation(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index cb7f603021..0a734708b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -93,7 +93,7 @@ public Worker createWorker(Dataset dataset, StoreFactory storageConfig, @NonNull final Worker worker = - Worker.newWorker(dataset, queryThreadPoolDefinition, jobsThreadPool, storageConfig, name, validator, failOnError, entityBucketSize, persistenceMapper, communicationMapper, secondaryIdSubPlanRetention); + Worker.newWorker(dataset, queryThreadPoolDefinition, jobsThreadPool, storageConfig, name, failOnError, entityBucketSize, persistenceMapper, communicationMapper, secondaryIdSubPlanRetention); addWorker(worker); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java index 82b90a7a51..78952dd7ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java @@ -3,10 +3,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.CONCEPT; import static com.bakdata.conquery.resources.ResourceConstants.DATASET; -import com.bakdata.conquery.io.jersey.ExtraMimeTypes; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.worker.Namespace; import jakarta.annotation.PostConstruct; import jakarta.inject.Inject; import jakarta.ws.rs.Consumes; @@ -15,6 +11,11 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; + +import com.bakdata.conquery.io.jersey.ExtraMimeTypes; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.worker.Namespace; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -34,7 +35,7 @@ public class AdminConceptsResource { private Dataset dataset; private Namespace namespace; @PathParam(CONCEPT) - private Concept concept; + private Concept concept; @PostConstruct public void init() { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 8248842abc..875b65abd4 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -8,6 +8,12 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import jakarta.inject.Inject; +import jakarta.validation.Validator; +import jakarta.ws.rs.ForbiddenException; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.ImportHandler; @@ -36,12 +42,6 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.univocity.parsers.csv.CsvParser; -import jakarta.inject.Inject; -import jakarta.validation.Validator; -import jakarta.ws.rs.ForbiddenException; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.WebApplicationException; -import jakarta.ws.rs.core.Response; import lombok.Data; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -327,8 +327,6 @@ public EntityIdMap getIdMapping(Namespace namespace) { } public void addInternToExternMapping(Namespace namespace, InternToExternMapper internToExternMapper) { - internToExternMapper.setDataset(namespace.getDataset()); - ValidatorHelper.failOnError(log, validator.validate(internToExternMapper)); if (namespace.getStorage().getInternToExternMapper(internToExternMapper.getId()) != null) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java index cefa4caf5c..f886d29a39 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.IdColumnConfig; diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 8d25bc0197..11ead26f2c 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -1,8 +1,10 @@ package com.bakdata.conquery.io; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.io.IOException; import jakarta.validation.Validator; import com.bakdata.conquery.commands.ManagerNode; @@ -14,30 +16,48 @@ import com.bakdata.conquery.mode.cluster.ClusterNamespaceHandler; import com.bakdata.conquery.mode.cluster.ClusterState; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.NonPersistentStoreFactory; +import com.codahale.metrics.SharedMetricRegistries; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.jersey.validation.Validators; import lombok.Getter; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @Getter public abstract class AbstractSerializationTest { private final Validator validator = Validators.newValidator(); - private final ConqueryConfig config = new ConqueryConfig(); + private final ConqueryConfig config = new ConqueryConfig() {{ + this.setStorage(new NonPersistentStoreFactory()); + }}; private DatasetRegistry datasetRegistry; + private Namespace namespace; private MetaStorage metaStorage; - private ObjectMapper managerInternalMapper; + + private ObjectMapper managerMetaInternalMapper; + private ObjectMapper namespaceInternalMapper; private ObjectMapper shardInternalMapper; private ObjectMapper apiMapper; + @BeforeAll + public static void beforeAll() { + // Some components need the shared registry, and it might have been set already by another test + if (SharedMetricRegistries.tryGetDefault() == null) { + SharedMetricRegistries.setDefault(AbstractSerializationTest.class.getSimpleName()); + } + } + @BeforeEach - public void before() { + public void before() throws IOException { + metaStorage = new MetaStorage(new NonPersistentStoreFactory()); InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, metaStorage, validator); final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel"); @@ -45,20 +65,17 @@ public void before() { datasetRegistry = new DatasetRegistry<>(0, config, creator, clusterNamespaceHandler, indexService); creator.init(datasetRegistry); - // Prepare manager node internal mapper - final ManagerNode managerNode = mock(ManagerNode.class); - when(managerNode.getConfig()).thenReturn(config); - when(managerNode.getValidator()).thenReturn(validator); - doReturn(datasetRegistry).when(managerNode).getDatasetRegistry(); - when(managerNode.getMetaStorage()).thenReturn(metaStorage); - when(managerNode.getInternalObjectMapperCreator()).thenReturn(creator); + namespace = datasetRegistry.createNamespace(new Dataset("serialization_test"), metaStorage); - when(managerNode.createInternalObjectMapper(any())).thenCallRealMethod(); - managerInternalMapper = managerNode.createInternalObjectMapper(View.Persistence.Manager.class); - - metaStorage.openStores(managerInternalMapper); + // Prepare manager meta internal mapper + managerMetaInternalMapper = creator.createInternalObjectMapper(View.Persistence.Manager.class); + metaStorage.openStores(managerMetaInternalMapper); metaStorage.loadData(); + // Prepare namespace internal mapper + namespaceInternalMapper = creator.createInternalObjectMapper(View.Persistence.Manager.class); + namespace.getInjectables().forEach(injectable -> injectable.injectInto(namespaceInternalMapper)); + // Prepare shard node internal mapper final ShardNode shardNode = mock(ShardNode.class); when(shardNode.getConfig()).thenReturn(config); @@ -67,11 +84,9 @@ public void before() { when(shardNode.createInternalObjectMapper(any())).thenCallRealMethod(); shardInternalMapper = shardNode.createInternalObjectMapper(View.Persistence.Shard.class); - // Prepare api response mapper - doCallRealMethod().when(managerNode).customizeApiObjectMapper(any(ObjectMapper.class)); + // Prepare api mapper with a Namespace injected (usually done by PathParamInjector) apiMapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); - managerNode.customizeApiObjectMapper(apiMapper); + ManagerNode.customizeApiObjectMapper(apiMapper, datasetRegistry, metaStorage, config, validator); + namespace.getInjectables().forEach(i -> i.injectInto(apiMapper)); } - - } diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java index bec921c38e..11db3ada69 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java @@ -6,6 +6,10 @@ import java.io.IOException; import java.lang.ref.SoftReference; import java.lang.ref.WeakReference; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.UnaryOperator; import jakarta.validation.Validator; @@ -48,14 +52,15 @@ public class SerializationTestUtil { ThreadLocal.class, User.ShiroUserAdapter.class, Validator.class, - WeakReference.class + WeakReference.class, + CompletableFuture.class }; private final JavaType type; private final Validator validator = Validators.newValidator(); @Setter private CentralRegistry registry; - private ObjectMapper[] objectMappers; + private List objectMappers = Collections.emptyList(); @NonNull private Injectable[] injectables = {}; @@ -76,7 +81,7 @@ public static SerializationTestUtil forArrayType(TypeReference eleme } public SerializationTestUtil objectMappers(ObjectMapper... objectMappers) { - this.objectMappers = objectMappers; + this.objectMappers = Arrays.asList(objectMappers); return this; } @@ -96,7 +101,7 @@ public SerializationTestUtil customizingAssertion(UnaryOperator identifiableValue) { assertThat(((IdentifiableImpl) copy).getId()).as("the serialized value").isEqualTo(identifiableValue.getId()); } RecursiveComparisonAssert ass = assertThat(copy) .as("Unequal after copy.") .usingRecursiveComparison() + .usingOverriddenEquals() .ignoringFieldsOfTypes(TYPES_TO_IGNORE); // Apply assertion customizations diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 99e0d08580..a540a9a655 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -4,17 +4,13 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.time.LocalDate; import java.time.ZonedDateTime; -import java.util.Arrays; -import java.util.BitSet; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.UUID; +import java.util.*; import java.util.stream.Stream; +import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.MeProcessor; @@ -31,8 +27,6 @@ import com.bakdata.conquery.io.AbstractSerializationTest; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.external.form.FormBackendVersion; -import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.serializer.SerializationTestUtil; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; @@ -47,6 +41,7 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.error.ConqueryError; @@ -74,6 +69,8 @@ import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.index.InternToExternMapper; +import com.bakdata.conquery.models.index.MapInternToExternMapper; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.results.EntityResult; @@ -92,7 +89,6 @@ import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntMaps; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; -import jakarta.validation.Validator; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.RecursiveComparisonAssert; import org.junit.jupiter.api.Tag; @@ -111,7 +107,7 @@ public void dataset() throws IOException, JSONException { SerializationTestUtil .forType(Dataset.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper()) .test(dataset); } @@ -121,7 +117,7 @@ public void passwordCredential() throws IOException, JSONException { SerializationTestUtil .forType(PasswordCredential.class) - .objectMappers(getManagerInternalMapper()) + .objectMappers(getManagerMetaInternalMapper()) .test(credential); } @@ -131,7 +127,7 @@ public void role() throws IOException, JSONException { SerializationTestUtil .forType(Role.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .test(mandator); } @@ -151,7 +147,7 @@ public void user() throws IOException, JSONException { SerializationTestUtil .forType(User.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .registry(registry) .test(user); } @@ -174,7 +170,7 @@ public void group() throws IOException, JSONException { SerializationTestUtil .forType(Group.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .test(group); } @@ -240,14 +236,9 @@ public void bucketCompoundDateRange() throws JSONException, IOException { SerializationTestUtil .forType(Bucket.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper()) .registry(registry) - .injectables(new Injectable() { - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(Validator.class, validator); - } - }) + .injectables(values -> values.add(Validator.class, validator)) .test(bucket); } @@ -281,7 +272,7 @@ public void table() throws JSONException, IOException { SerializationTestUtil .forType(Table.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .registry(registry) .test(table); } @@ -293,10 +284,11 @@ public void treeConcept() throws IOException, JSONException { Dataset dataset = createDataset(registry); TreeConcept concept = createConcept(registry, dataset); + concept.init(); SerializationTestUtil - .forType(TreeConcept.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .forType(Concept.class) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .registry(registry) .test(concept); } @@ -305,7 +297,7 @@ public void treeConcept() throws IOException, JSONException { @Test public void persistentIdMap() throws JSONException, IOException { SerializationTestUtil.forType(EntityIdMap.class) - .objectMappers(getManagerInternalMapper()) + .objectMappers(getManagerMetaInternalMapper()) .test(IdMapSerialisationTest.createTestPersistentMap()); } @@ -328,7 +320,7 @@ public void formConfig() throws JSONException, IOException { SerializationTestUtil .forType(FormConfig.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .registry(registry) .test(formConfig); } @@ -351,7 +343,7 @@ public void managedQuery() throws JSONException, IOException { execution.setTags(new String[]{"test-tag"}); SerializationTestUtil.forType(ManagedExecution.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .registry(registry) .test(execution); } @@ -369,7 +361,7 @@ public void testExportForm() throws JSONException, IOException { final ExportForm exportForm = createExportForm(registry, dataset); SerializationTestUtil.forType(QueryDescription.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .registry(registry) .checkHashCode() .test(exportForm); @@ -390,7 +382,7 @@ public void managedForm() throws JSONException, IOException { execution.setTags(new String[]{"test-tag"}); SerializationTestUtil.forType(ManagedExecution.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .registry(registry) .test(execution); } @@ -414,7 +406,7 @@ public void testExternalExecution() throws IOException, JSONException { final ExternalExecution execution = new ExternalExecution(form, user, dataset, getMetaStorage()); SerializationTestUtil.forType(ManagedExecution.class) - .objectMappers(getManagerInternalMapper()) + .objectMappers(getManagerMetaInternalMapper()) .registry(centralRegistry) .test(execution); @@ -452,7 +444,7 @@ public void cqConcept() throws JSONException, IOException { SerializationTestUtil .forType(CQConcept.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .registry(registry) .test(cqConcept); } @@ -466,7 +458,7 @@ public void executionCreationPlanError() throws JSONException, IOException { SerializationTestUtil .forType(ConqueryError.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .test(error); } @@ -476,7 +468,7 @@ public void executionCreationResolveError() throws JSONException, IOException { SerializationTestUtil .forType(ConqueryError.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .test(error); } @@ -488,13 +480,13 @@ public void executionQueryJobError() throws JSONException, IOException { SerializationTestUtil .forType(ConqueryError.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .test(error); } @Test public void meInformation() throws IOException, JSONException { - User user = new User("name", "labe", getMetaStorage()); + User user = new User("name", "label", getMetaStorage()); MeProcessor.FrontendMeInformation info = MeProcessor.FrontendMeInformation.builder() .userName(user.getLabel()) @@ -505,7 +497,7 @@ public void meInformation() throws IOException, JSONException { SerializationTestUtil .forType(MeProcessor.FrontendMeInformation.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getApiMapper()) .test(info); } @@ -553,7 +545,7 @@ public void testFormQuery() throws IOException, JSONException { SerializationTestUtil .forType(AbsoluteFormQuery.class) - .objectMappers(getManagerInternalMapper(), getShardInternalMapper(), getApiMapper()) + .objectMappers(getManagerMetaInternalMapper(), getShardInternalMapper(), getApiMapper()) .registry(centralRegistry) .test(query); } @@ -609,25 +601,25 @@ public void testBiMapSerialization() throws JSONException, IOException { SerializationTestUtil .forType(new TypeReference>() { }) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test(map); } @Test public void testNonStrictNumbers() throws JSONException, IOException { SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.NaN, null); + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()).test(Double.NaN, null); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.NEGATIVE_INFINITY, null); + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()).test(Double.NEGATIVE_INFINITY, null); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.POSITIVE_INFINITY, null); + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()).test(Double.POSITIVE_INFINITY, null); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.MAX_VALUE); + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()).test(Double.MAX_VALUE); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.MIN_VALUE); + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()).test(Double.MIN_VALUE); SerializationTestUtil .forType(EntityResult.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test( new MultilineEntityResult("4", List.of( new Object[]{0, 1, 2}, @@ -670,21 +662,21 @@ public void test(Range range) throws IOException, JSONException { SerializationTestUtil .forType(new TypeReference>() { }) - .objectMappers(getApiMapper(), getManagerInternalMapper(), getShardInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper(), getShardInternalMapper()) .test(range); } @Test public void locale() throws JSONException, IOException { SerializationTestUtil.forType(Locale.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test(Locale.GERMANY); } @Test public void localeArray() throws JSONException, IOException { SerializationTestUtil.forType(Locale[].class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test(new Locale[]{Locale.GERMANY, Locale.ROOT, Locale.ENGLISH, Locale.US, Locale.UK}); } @@ -707,7 +699,7 @@ public void externalForm() throws IOException, JSONException { ExternalForm externalForm = getApiMapper().readerFor(QueryDescription.class).readValue(externalFormString); SerializationTestUtil.forType(QueryDescription.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test(externalForm); } @@ -742,7 +734,7 @@ public void externalFormArray() throws IOException, JSONException { ExternalForm externalForm = getApiMapper().readerFor(QueryDescription.class).readValue(externalFormString); ExternalForm externalForm2 = getApiMapper().readerFor(QueryDescription.class).readValue(externalFormString2); SerializationTestUtil.forType(QueryDescription[].class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test(new QueryDescription[]{externalForm, externalForm2}); } @@ -789,7 +781,7 @@ public void object2IntEmpty() throws JSONException, IOException { SerializationTestUtil.forType(new TypeReference>() { }) - .objectMappers(getApiMapper(), getShardInternalMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getShardInternalMapper(), getManagerMetaInternalMapper()) .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) .test(empty); @@ -805,7 +797,7 @@ public void object2IntString() throws JSONException, IOException { map.put("two", 2); SerializationTestUtil.forType(new TypeReference>() { }) - .objectMappers(getApiMapper(), getShardInternalMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getShardInternalMapper(), getManagerMetaInternalMapper()) .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) .test(map); @@ -829,7 +821,7 @@ public void arrayObject2Int() throws JSONException, IOException { }} }; SerializationTestUtil.forArrayType(new TypeReference>() { - }).objectMappers(getApiMapper(), getShardInternalMapper(), getManagerInternalMapper()) + }).objectMappers(getApiMapper(), getShardInternalMapper(), getManagerMetaInternalMapper()) .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) .test(map); @@ -840,8 +832,29 @@ public void formBackendVersion() throws JSONException, IOException { final FormBackendVersion version = new FormBackendVersion("3.45.45-g85ut85u43t8", ZonedDateTime.parse("2007-12-03T10:15:30+00:00")); SerializationTestUtil.forType(FormBackendVersion.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) + .objectMappers(getApiMapper(), getManagerMetaInternalMapper()) .test(version); } + + @Test + public void mapInternToExternMapper() throws JSONException, IOException, URISyntaxException { + final MapInternToExternMapper mapper = new MapInternToExternMapper( + "test1", + new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), + "internal", + "{{external}}" + ); + + mapper.setStorage(getNamespace().getStorage()); + mapper.setConfig(getConfig()); + mapper.setMapIndex(getNamespace().getIndexService()); + + + mapper.init(); + + SerializationTestUtil.forType(InternToExternMapper.class) + .objectMappers(getApiMapper(), getNamespaceInternalMapper()) + .test(mapper); + } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java index 97cef7a5df..50783215aa 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java @@ -98,12 +98,10 @@ public static void init() throws IOException, JSONException, ConfigurationExcept indexedConcept = conceptReader.readValue(node); indexedConcept.setDataset(dataset); - indexedConcept.initElements(); oldConcept = conceptReader.readValue(node); oldConcept.setDataset(dataset); - oldConcept.initElements(); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java index 24f0198dd9..828c76cb7c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.index; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockserver.model.HttpRequest.request; import java.io.IOException; @@ -8,9 +9,13 @@ import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; +import java.util.concurrent.ExecutionException; +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.github.powerlibraries.io.In; import com.univocity.parsers.csv.CsvParserSettings; import lombok.SneakyThrows; @@ -29,6 +34,7 @@ @Slf4j public class IndexServiceTest { + private static final NamespaceStorage NAMESPACE_STORAGE = new NamespaceStorage(new NonPersistentStoreFactory(), IndexServiceTest.class.getName()); private static final Dataset DATASET = new Dataset("dataset"); private static final ConqueryConfig CONFIG = new ConqueryConfig(); private static final ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); @@ -37,6 +43,9 @@ public class IndexServiceTest { @BeforeAll @SneakyThrows public static void beforeAll() { + NAMESPACE_STORAGE.openStores(Jackson.MAPPER); + + NAMESPACE_STORAGE.updateDataset(DATASET); CONFIG.getIndex().setBaseUrl(new URI(String.format("http://localhost:%d/", REF_SERVER.getPort()))); @@ -50,7 +59,7 @@ public static void afterAll() { @Test @Order(0) - void testLoading() throws NoSuchFieldException, IllegalAccessException, URISyntaxException, IOException { + void testLoading() throws NoSuchFieldException, IllegalAccessException, URISyntaxException, IOException, ExecutionException, InterruptedException { log.info("Test loading of mapping"); try (InputStream inputStream = In.resource("/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv").asStream()) { @@ -80,14 +89,19 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta ); - injectComponents(mapper, indexService, CONFIG); - injectComponents(mapperUrlAbsolute, indexService, CONFIG); - injectComponents(mapperUrlRelative, indexService, CONFIG); + injectComponents(mapper, indexService); + injectComponents(mapperUrlAbsolute, indexService); + injectComponents(mapperUrlRelative, indexService); mapper.init(); mapperUrlAbsolute.init(); mapperUrlRelative.init(); + // Wait for future + mapper.getInt2ext().get(); + mapperUrlAbsolute.getInt2ext().get(); + mapperUrlRelative.getInt2ext().get(); + assertThat(mapper.external("int1")).as("Internal Value").isEqualTo("hello"); assertThat(mapper.external("int2")).as("Internal Value").isEqualTo("int2"); @@ -99,24 +113,25 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta } - private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService, ConqueryConfig config) + private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService) throws NoSuchFieldException, IllegalAccessException { + mapInternToExternMapper.setStorage(NAMESPACE_STORAGE); + final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.mapIndex); indexServiceField.setAccessible(true); indexServiceField.set(mapInternToExternMapper, indexService); final Field configField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.config); configField.setAccessible(true); - configField.set(mapInternToExternMapper, config); + configField.set(mapInternToExternMapper, IndexServiceTest.CONFIG); - mapInternToExternMapper.setDataset(DATASET); } @Test @Order(2) void testEvictOnMapper() - throws NoSuchFieldException, IllegalAccessException, URISyntaxException { + throws NoSuchFieldException, IllegalAccessException, URISyntaxException, ExecutionException, InterruptedException { log.info("Test evicting of mapping on mapper"); final MapInternToExternMapper mapInternToExternMapper = new MapInternToExternMapper( "test1", @@ -125,14 +140,17 @@ void testEvictOnMapper() "{{external}}" ); - injectComponents(mapInternToExternMapper, indexService, CONFIG); + injectComponents(mapInternToExternMapper, indexService); mapInternToExternMapper.init(); + // Wait for future + mapInternToExternMapper.getInt2ext().get(); + // Before eviction the result should be the same assertThat(mapInternToExternMapper.external("int1")).as("Internal Value").isEqualTo("hello"); - final MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext(); + final MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext().get(); indexService.evictCache(); @@ -141,11 +159,31 @@ void testEvictOnMapper() mapInternToExternMapper.init(); - final MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext(); + final MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext().get(); // Check that the mapping reinitialized assertThat(mappingBeforeEvict).as("Mapping before and after eviction") .isNotSameAs(mappingAfterEvict); } + @Test + void testFailedLoading() throws NoSuchFieldException, IllegalAccessException, URISyntaxException { + final MapInternToExternMapper mapInternToExternMapper = new MapInternToExternMapper( + "test1", + new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/not_existing_mapping.csv"), + "internal", + "{{external}}" + ); + + injectComponents(mapInternToExternMapper, indexService); + mapInternToExternMapper.init(); + + // Wait for future + assertThatThrownBy(() -> mapInternToExternMapper.getInt2ext().get()).as("Not existent CSV").hasCauseInstanceOf(IllegalStateException.class); + + + // Before eviction the result should be the same + assertThat(mapInternToExternMapper.external("int1")).as("Internal Value").isEqualTo("int1"); + } + } diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index 6af316d6a3..d99223f1f1 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -266,7 +266,7 @@ public static TestConcept create(int countConnectors, BiFunction { + for (Searchable searchable : filter.getSearchReferences()) { search.addSearches(Map.of(searchable, searchable.createTrieSearch(indexConfig))); - }); + } search.registerValues(column, List.of( "a", @@ -69,7 +71,7 @@ public void totals() { } @Test - public void totalsEmptyFiler() { + public void totalsEmptyFiler() throws IndexCreationException { final IndexConfig indexConfig = new IndexConfig(); FilterSearch search = new FilterSearch(indexConfig); @@ -96,9 +98,10 @@ public void totalsEmptyFiler() { filter.setConnector(connector); // Register - filter.getSearchReferences().forEach(searchable -> { + for (Searchable searchable : filter.getSearchReferences()) { search.addSearches(Map.of(searchable, searchable.createTrieSearch(indexConfig))); - }); + } + search.shrinkSearch(column); assertThat(search.getTotal(filter)).isEqualTo(0); diff --git a/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java b/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java new file mode 100644 index 0000000000..b01d28f2ca --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java @@ -0,0 +1,25 @@ +package com.bakdata.conquery.util.extensions; + +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.util.NonPersistentStoreFactory; +import lombok.Getter; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +@Getter +public class NamespaceStorageExtension implements BeforeAllCallback, BeforeEachCallback { + + private final NamespaceStorage storage = new NamespaceStorage(new NonPersistentStoreFactory(), "test_path"); + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + storage.openStores(Jackson.MAPPER); + } + + @Override + public void beforeEach(ExtensionContext context) { + storage.clear(); + } +} \ No newline at end of file diff --git a/executable/pom.xml b/executable/pom.xml index 12762e1c6c..83307d34ca 100644 --- a/executable/pom.xml +++ b/executable/pom.xml @@ -1,28 +1,47 @@ - 4.0.0 + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - - com.bakdata.conquery - parent - ${revision} - - executable + + com.bakdata.conquery + parent + ${revision} + + executable - - - - org.apache.maven.plugins - maven-shade-plugin - 3.4.1 - - false - false - - - *:* - + + + + org.apache.maven.plugins + maven-jar-plugin + + + default-jar + + jar + + + + + java.base/java.lang ALL-UNNAMED java.base/java.util ALL-UNNAMED java.base/java.nio ALL-UNNAMED java.base/sun.nio.ch ALL-UNNAMED + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + + false + false + + + *:* + META-INF/*.SF META-INF/*.DSA META-INF/*.RSA @@ -50,48 +69,48 @@ - - - - package - - shade - - - - - false - - - - + + + package + + shade + + + + + false + + + + com.bakdata.conquery.ConqueryServer true - - - - - - - + + + + + + + - - - com.bakdata.conquery - backend - ${project.version} - - - org.assertj - assertj-core - test - - + + + com.bakdata.conquery + backend + ${project.version} + + + org.assertj + assertj-core + test + +