diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 0d7b3b25a8..2ceee255fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; -import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.IndexConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Searchable; @@ -40,7 +39,7 @@ @ToString @Slf4j @CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class) -public class FilterTemplate extends IdentifiableImpl implements Searchable, SearchIndex { +public class FilterTemplate extends IdentifiableImpl implements Searchable, SearchIndex { private static final long serialVersionUID = 1L; @@ -90,7 +89,7 @@ public boolean isSearchDisabled() { return false; } - public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config) { final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath()); log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/LabelMap.java b/backend/src/main/java/com/bakdata/conquery/apiv1/LabelMap.java new file mode 100644 index 0000000000..51663d3dfd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/LabelMap.java @@ -0,0 +1,71 @@ +package com.bakdata.conquery.apiv1; + +import java.util.List; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.frontend.FrontendValue; +import com.bakdata.conquery.models.config.IndexConfig; +import com.bakdata.conquery.models.datasets.concepts.Searchable; +import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; +import com.bakdata.conquery.models.query.FilterSearch; +import com.bakdata.conquery.util.search.TrieSearch; +import com.google.common.collect.BiMap; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.StopWatch; + +@Getter +@RequiredArgsConstructor +@Slf4j +@EqualsAndHashCode +public class LabelMap implements Searchable { + + private final FilterId id; + @Delegate + private final BiMap delegate; + private final int minSuffixLength; + private final boolean generateSearchSuffixes; + + @Override + public TrieSearch createTrieSearch(IndexConfig config) { + + final TrieSearch search = config.createTrieSearch(true); + + final List collected = delegate.entrySet().stream() + .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + + if (log.isTraceEnabled()) { + log.trace("Labels for {}: `{}`", getId(), collected.stream().map(FrontendValue::toString).collect(Collectors.toList())); + } + + StopWatch timer = StopWatch.createStarted(); + log.trace("START-SELECT ADDING_ITEMS for {}", getId()); + + collected.forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + + log.trace("DONE-SELECT ADDING_ITEMS for {} in {}", getId(), timer); + + timer.reset(); + log.trace("START-SELECT SHRINKING for {}", getId()); + + search.shrinkToFit(); + + log.trace("DONE-SELECT SHRINKING for {} in {}", getId(), timer); + + return search; + } + + @Override + public boolean isGenerateSuffixes() { + return generateSearchSuffixes; + } + + @Override + public boolean isSearchDisabled() { + return false; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 163c5c9bc5..1fc4bae819 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -554,15 +554,14 @@ public Stream> resolveEntities(Subject subject, List resultInfos = query.getResultInfos(); + public ResultStatistics getResultStatistics(SingleTableResult managedQuery) { + final List resultInfos = managedQuery.getResultInfos(); final Optional dateInfo = - query.getResultInfos().stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst(); + resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst(); - final int dateIndex = dateInfo.map(resultInfos::indexOf).orElse(0 /*Discarded if dateInfo is not present*/); + final Optional dateIndex = dateInfo.map(resultInfos::indexOf); final Locale locale = I18n.LOCALE.get(); final NumberFormat decimalFormat = NumberFormat.getNumberInstance(locale); diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java b/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java index e5d4f91673..c82b8f5dfa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java @@ -13,25 +13,29 @@ import lombok.extern.slf4j.Slf4j; import org.apache.mina.core.future.WriteFuture; import org.apache.mina.core.session.IoSession; +import org.jetbrains.annotations.NotNull; @RequiredArgsConstructor @Slf4j public class NetworkSession implements MessageSender> { + public static final int MAX_MESSAGE_LENGTH = 30; + public static final int MAX_QUEUE_LENGTH = 20; @Getter private final IoSession session; - private final LinkedBlockingQueue> queuedMessages = new LinkedBlockingQueue<>(20); + private final LinkedBlockingQueue> queuedMessages = new LinkedBlockingQueue<>(MAX_QUEUE_LENGTH); @Override public WriteFuture send(final NetworkMessage message) { try { while (!queuedMessages.offer(message, 2, TimeUnit.MINUTES)) { - log.debug( - "Waiting for full writing queue for {}\n\tcurrently filled by: {}", - message, - new ArrayList<>(queuedMessages) - .stream() - .map(Objects::toString) - .collect(Collectors.joining("\n\t\t")) + log.debug("Waiting for full writing queue for {} currently filled by:\n\t- {}", + message, + log.isTraceEnabled() + ? new ArrayList<>(queuedMessages).stream() + .map(Objects::toString) + .map(NetworkSession::shorten) + .collect(Collectors.joining("\n\t\t- ")) + : "%s messages".formatted(queuedMessages.size()) ); } } @@ -45,6 +49,16 @@ public WriteFuture send(final NetworkMessage message) { return future; } + @NotNull + private static String shorten(String desc) { + if (desc.length() <= MAX_MESSAGE_LENGTH) { + return desc; + } + + return desc.substring(0, MAX_MESSAGE_LENGTH) + "…"; + + } + @Override public void trySend(final NetworkMessage message) { if (isConnected()) { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index 2d5740e4d1..8f3e4eb1e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -46,7 +46,7 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError()); - FilterSearch filterSearch = new FilterSearch(storage, jobManager, config.getCsv(), config.getIndex()); + FilterSearch filterSearch = new FilterSearch(config.getIndex()); return new NamespaceSetupData(injectables, indexService, communicationMapper, preprocessMapper, jobManager, filterSearch); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index b692a8e708..bd20ea676d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; -import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.IndexConfig; import com.bakdata.conquery.models.datasets.concepts.Searchable; import com.bakdata.conquery.models.events.MajorTypeId; @@ -26,7 +25,7 @@ @Setter @NoArgsConstructor @Slf4j -public class Column extends Labeled implements NamespacedIdentifiable, Searchable { +public class Column extends Labeled implements NamespacedIdentifiable, Searchable { public static final int UNKNOWN_POSITION = -1; @@ -75,7 +74,7 @@ public Dataset getDataset() { * We create only an empty search here, because the content is provided through {@link com.bakdata.conquery.models.messages.namespaces.specific.RegisterColumnValues} and filled by the caller. */ @Override - public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config) { return config.createTrieSearch(isGenerateSuffixes()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java index 579b373449..c6839d49e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java @@ -3,11 +3,7 @@ import java.util.List; import com.bakdata.conquery.apiv1.frontend.FrontendValue; -import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.IndexConfig; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.identifiable.Identifiable; -import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.query.FilterSearch; import com.bakdata.conquery.util.search.TrieSearch; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -18,25 +14,12 @@ *

* Searchable classes describe how a search should be constructed, and provide the values with getSearchValues. */ -public interface Searchable>> extends Identifiable { - - public Dataset getDataset(); +public interface Searchable { /** * All available {@link FrontendValue}s for searching in a {@link TrieSearch}. */ - TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage); - - /** - * The actual Searchables to use, if there is potential for deduplication/pooling. - * - * @implSpec The order of objects returned is used to also sort search results from different sources. - */ - @JsonIgnore - default List> getSearchReferences() { - //Hopefully the only candidate will be Column - return List.of(this); - } + TrieSearch createTrieSearch(IndexConfig config); /** * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create. diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index a3aab711a1..d8087f2aa6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -7,20 +7,16 @@ import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.FilterTemplate; +import com.bakdata.conquery.apiv1.LabelMap; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; -import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.IndexConfig; import com.bakdata.conquery.models.datasets.concepts.Searchable; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; -import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; -import com.bakdata.conquery.models.query.FilterSearch; -import com.bakdata.conquery.util.search.TrieSearch; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.common.collect.BiMap; @@ -30,7 +26,6 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.time.StopWatch; import org.jetbrains.annotations.NotNull; @Setter @@ -38,7 +33,7 @@ @NoArgsConstructor @Slf4j @JsonIgnoreProperties({"searchType"}) -public abstract class SelectFilter extends SingleColumnFilter implements Searchable { +public abstract class SelectFilter extends SingleColumnFilter { /** * user given mapping from the values in the columns to shown labels @@ -71,19 +66,27 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig @JsonIgnore public abstract String getFilterType(); - @Override - public List> getSearchReferences() { - final List> out = new ArrayList<>(); - if (getTemplate() != null) { + /** + * The actual Searchables to use, if there is potential for deduplication/pooling. + * + * @implSpec The order of objects returned is used to also sort search results from different sources. + */ + @JsonIgnore + public List getSearchReferences() { + final List out = new ArrayList<>(); + + if (getTemplate() != null && !getTemplate().isSearchDisabled()) { out.add(getTemplate()); } if (!labels.isEmpty()) { - out.add(this); + out.add(new LabelMap(getId(), labels, searchMinSuffixLength, generateSearchSuffixes)); } - out.addAll(getColumn().getSearchReferences()); + if (!getColumn().isSearchDisabled()) { + out.add(getColumn()); + } return out; } @@ -105,51 +108,4 @@ public boolean isNotUsingTemplateAndLabels() { return (getTemplate() == null) != labels.isEmpty(); } - - @Override - @JsonIgnore - public boolean isGenerateSuffixes() { - return generateSearchSuffixes; - } - - @Override - @JsonIgnore - public int getMinSuffixLength() { - return searchMinSuffixLength; - } - - /** - * Does not make sense to distinguish at Filter level since it's only referenced when labels are set. - */ - @Override - @JsonIgnore - public boolean isSearchDisabled() { - return false; - } - - @Override - public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { - - final TrieSearch search = config.createTrieSearch(true); - - if(log.isTraceEnabled()) { - log.trace("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); - } - - StopWatch timer = StopWatch.createStarted(); - log.trace("START-SELECT ADDING_ITEMS for {}", getId()); - - collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); - - log.trace("DONE-SELECT ADDING_ITEMS for {} in {}", getId(), timer); - - timer.reset(); - log.trace("START-SELECT SHRINKING for {}", getId()); - - search.shrinkToFit(); - - log.trace("DONE-SELECT SHRINKING for {} in {}", getId(), timer); - - return search; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java index d4f885e098..bca5009799 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java @@ -27,13 +27,20 @@ public static StringStoreString create(int size) { @JsonCreator public static StringStoreString withInternedStrings(String[] values) { - for (int index = 0; index < values.length; index++) { - values[index] = values[index] != null ? values[index].intern() : null; + if(shouldIntern()) { + for (int index = 0; index < values.length; index++) { + values[index] = values[index] != null ? values[index].intern() : null; + } } return new StringStoreString(values); } + private static boolean shouldIntern() { + //TODO use mixin or properly wire this property + return "yes".equals(System.getProperty("cq.intern", "no")); + } + @Override public boolean has(int event) { return values[event] != null; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 5e4170944a..367fec4626 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -31,7 +31,6 @@ import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.FormShardResult; import com.bakdata.conquery.models.worker.DistributedNamespace; -import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; @@ -111,8 +110,8 @@ public void start() { } @Override - public List generateColumnDescriptions(boolean isInitialized, Namespace namespace, ConqueryConfig config) { - return subQueries.values().iterator().next().generateColumnDescriptions(isInitialized, namespace, config); + public List generateColumnDescriptions(boolean isInitialized, ConqueryConfig config) { + return subQueries.values().iterator().next().generateColumnDescriptions(isInitialized, config); } @@ -131,7 +130,7 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject return; } ManagedQuery subQuery = subQueries.entrySet().iterator().next().getValue(); - status.setColumnDescriptions(subQuery.generateColumnDescriptions(isInitialized(), getNamespace(), getConfig())); + status.setColumnDescriptions(subQuery.generateColumnDescriptions(isInitialized(), getConfig())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index c23d065b9b..023633f288 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.jobs; -import java.time.Duration; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -12,7 +11,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.frontend.FrontendValue; @@ -28,6 +26,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; +import org.jetbrains.annotations.NotNull; /** * Job that initializes the filter search for the frontend. @@ -60,24 +59,16 @@ public void execute() throws Exception { log.info("BEGIN loading SourceSearch"); - // collect all SelectFilters to the create searches for them + // collect all SelectFilters to create searches for them final List> allSelectFilters = - storage.getAllConcepts().stream() - .flatMap(c -> c.getConnectors().stream()) - .flatMap(co -> co.collectAllFilters().stream()) - .filter(SelectFilter.class::isInstance) - .map(f -> ((SelectFilter) f)) - .collect(Collectors.toList()); + getAllSelectFilters(storage); // Unfortunately the is no ClassToInstanceMultimap yet - final Map, Set>> collectedSearchables = + final Map, Set> collectedSearchables = allSelectFilters.stream() .map(SelectFilter::getSearchReferences) .flatMap(Collection::stream) - // Disabling search is only a last resort for when columns are too big to store in memory or process for indexing. - // TODO FK: We want no Searchable to be disabled, better scaling searches or mechanisms to fill search. - .filter(Predicate.not(Searchable::isSearchDisabled)) // Group Searchables into "Columns" and other "Searchables" .collect(Collectors.groupingBy(s -> s instanceof Column ? Column.class : Searchable.class, Collectors.toSet())); @@ -85,13 +76,13 @@ public void execute() throws Exception { // Most computations are cheap but data intensive: we fork here to use as many cores as possible. final ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1); - final HashMap, TrieSearch> searchCache = new HashMap<>(); - final Map, TrieSearch> synchronizedResult = Collections.synchronizedMap(searchCache); + final HashMap> searchCache = new HashMap<>(); + final Map> synchronizedResult = Collections.synchronizedMap(searchCache); log.debug("Found {} searchable Objects.", collectedSearchables.values().stream().mapToLong(Set::size).sum()); - for (Searchable searchable : collectedSearchables.getOrDefault(Searchable.class, Collections.emptySet())) { - if (searchable instanceof Column column) { + for (Searchable searchable : collectedSearchables.getOrDefault(Searchable.class, Collections.emptySet())) { + if (searchable instanceof Column) { throw new IllegalStateException("Columns should have been grouped out previously"); } @@ -99,17 +90,17 @@ public void execute() throws Exception { final StopWatch watch = StopWatch.createStarted(); - log.info("BEGIN collecting entries for `{}`", searchable.getId()); + log.info("BEGIN collecting entries for `{}`", searchable); try { - final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); + final TrieSearch search = searchable.createTrieSearch(indexConfig); synchronizedResult.put(searchable, search); log.debug( "DONE collecting {} entries for `{}`, within {}", search.calculateSize(), - searchable.getId(), + searchable, watch ); } @@ -120,7 +111,7 @@ public void execute() throws Exception { }); } - // The following cast is save + // The following cast is safe final Set searchableColumns = (Set) collectedSearchables.getOrDefault(Column.class, Collections.emptySet()); log.debug("Start collecting column values: {}", Arrays.toString(searchableColumns.toArray())); registerColumnValuesInSearch.accept(searchableColumns); @@ -146,6 +137,16 @@ public void execute() throws Exception { } + @NotNull + public static List> getAllSelectFilters(NamespaceStorage storage) { + return storage.getAllConcepts().stream() + .flatMap(c -> c.getConnectors().stream()) + .flatMap(co -> co.collectAllFilters().stream()) + .filter(SelectFilter.class::isInstance) + .map(f -> ((SelectFilter) f)) + .collect(Collectors.toList()); + } + @Override public String getLabel() { return "UpdateFilterSearchJob"; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java index 58058b92f9..3df723a45b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java @@ -14,8 +14,11 @@ import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; +import com.bakdata.conquery.models.jobs.Job; +import com.bakdata.conquery.models.jobs.UpdateFilterSearchJob; import com.bakdata.conquery.models.messages.namespaces.ActionReactionMessage; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; @@ -93,15 +96,65 @@ public void react(Worker context) throws Exception { log.debug("Still waiting for jobs: {} of {} done", done.get(), futures.size()); } } - + log.info("Finished collecting values from these columns: {}", Arrays.toString(columns.toArray())); context.send(new FinalizeReactionMessage(getMessageId(), context.getInfo().getId())); } @Override public void afterAllReaction() { - log.debug("{} shrinking searches", this); - final FilterSearch filterSearch = namespace.getFilterSearch(); - columns.forEach(filterSearch::shrinkSearch); + + // Run this in a job, so it is definitely processed after UpdateFilterSearchJob + namespace.getJobManager().addSlowJob(new SearchShrinker()); + } + + private class SearchShrinker extends Job { + + @Override + public void execute() { + + final List> allSelectFilters = UpdateFilterSearchJob.getAllSelectFilters(namespace.getStorage()); + final FilterSearch filterSearch = namespace.getFilterSearch(); + + getProgressReporter().setMax(allSelectFilters.size() + columns.size()); + + log.debug("{} shrinking searches", this); + + for (Column column : columns) { + try { + filterSearch.shrinkSearch(column); + } + catch (Exception e) { + log.warn("Unable to shrink search for {}", column, e); + } + finally { + getProgressReporter().report(1); + } + } + + log.info("BEGIN counting search totals on {}", namespace.getDataset().getId()); + + for (SelectFilter filter : allSelectFilters) { + log.trace("Calculate totals for filter: {}", filter.getId()); + try { + final long total = namespace.getFilterSearch().getTotal(filter); + log.trace("Filter '{}' totals: {}", filter, total); + } + catch (Exception e) { + log.warn("Unable to calculate totals for filter '{}'", filter.getId(), e); + } + finally { + getProgressReporter().report(1); + } + } + + getProgressReporter().done(); + log.debug("FINISHED counting search totals on {}", namespace.getDataset().getId()); + } + + @Override + public String getLabel() { + return "Finalize Search update"; + } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/FinalizeReactionMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/FinalizeReactionMessage.java index fb383ccacc..e047a56963 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/FinalizeReactionMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/FinalizeReactionMessage.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.messages.ReactionMessage; +import com.bakdata.conquery.models.messages.namespaces.ActionReactionMessage; import com.bakdata.conquery.models.messages.namespaces.NamespaceMessage; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.worker.DistributedNamespace; @@ -14,12 +15,16 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; + +/** + * Use {@link ActionReactionMessage#afterAllReaction()} to processing on initiator side after all reactions where collected. + */ @CPSType(id = "FINALIZE_REACTION_MESSAGE", base = NamespacedMessage.class) @AllArgsConstructor(onConstructor_ = @JsonCreator) @Getter @Slf4j @ToString -public class FinalizeReactionMessage extends NamespaceMessage implements ReactionMessage { +public final class FinalizeReactionMessage extends NamespaceMessage implements ReactionMessage { private UUID callerId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java index 86cdef9d2f..32587e4bdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.worker.DistributedNamespace; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; @@ -34,15 +35,23 @@ public class RegisterColumnValues extends NamespaceMessage implements ReactionMe @NsIdRef private final Column column; + + @ToString.Exclude private final Collection values; + @JsonIgnore + @ToString.Include + public int size() { + return values.size(); + } + @Override public void react(DistributedNamespace context) throws Exception { if (log.isTraceEnabled()) { - log.trace("Registering values for column '{}': {}", column.getId(), Arrays.toString(values.toArray())); + log.trace("Registering {} values for column '{}': {}", size(), column.getId(), Arrays.toString(values.toArray())); } else { - log.debug("Registering {} values for column '{}'", values.size(), column.getId()); + log.debug("Registering {} values for column '{}'", size(), column.getId()); } context.getFilterSearch().registerValues(column, values); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java index 8c6522eb3c..46273c979f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java @@ -9,29 +9,23 @@ import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.frontend.FrontendValue; -import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.config.IndexConfig; import com.bakdata.conquery.models.datasets.concepts.Searchable; import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; -import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.util.search.TrieSearch; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.base.Functions; import it.unimi.dsi.fastutil.objects.Object2LongMap; -import it.unimi.dsi.fastutil.objects.Object2LongMaps; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@Data +@RequiredArgsConstructor public class FilterSearch { - private final NamespaceStorage storage; - private final JobManager jobManager; - private final CSVConfig parserConfig; + @Getter private final IndexConfig indexConfig; /** @@ -40,8 +34,8 @@ public class FilterSearch { * In the code below, the keys of this map will usually be called "reference". */ @JsonIgnore - private Map, TrieSearch> searchCache = new HashMap<>(); - private Object2LongMap> totals = Object2LongMaps.emptyMap(); + private Map> searchCache = new HashMap<>(); + private Object2LongMap> totals = new Object2LongOpenHashMap<>(); /** * From a given {@link FrontendValue} extract all relevant keywords. @@ -62,11 +56,11 @@ public static List extractKeywords(FrontendValue value) { /** * For a {@link SelectFilter} collect all relevant {@link TrieSearch}. */ - public final List> getSearchesFor(Searchable searchable) { - final List> references = searchable.getSearchReferences(); + public final List> getSearchesFor(SelectFilter searchable) { + final List references = searchable.getSearchReferences(); if (log.isTraceEnabled()) { - log.trace("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + log.trace("Got {} as searchables for {}", references.stream().map(Searchable::toString).collect(Collectors.toList()), searchable.getId()); } return references.stream() @@ -75,20 +69,21 @@ public final List> getSearchesFor(Searchable search .collect(Collectors.toList()); } - public long getTotal(Searchable searchable) { - return totals.getOrDefault(searchable, 0); + public long getTotal(SelectFilter filter) { + return totals.computeIfAbsent(filter, (f) -> filter.getSearchReferences().stream() + .map(searchCache::get) + .flatMap(TrieSearch::stream) + .distinct() + .count()); } /** * Add ready searches to the cache. This assumes that the search already has been shrunken. */ - public synchronized void addSearches(Map, TrieSearch> searchCache) { + public synchronized void addSearches(Map> searchCache) { this.searchCache.putAll(searchCache); - log.debug("BEGIN counting Search totals."); - // Precompute totals as that can be slow when doing it on-demand. - updateTotals(searchCache); } @@ -97,44 +92,21 @@ public synchronized void addSearches(Map, TrieSearch searchable, Collection values) { - TrieSearch search = searchCache.computeIfAbsent(searchable, (ignored) -> searchable.createTrieSearch(indexConfig, storage)); + public void registerValues(Searchable searchable, Collection values) { + TrieSearch search = searchCache.computeIfAbsent(searchable, (ignored) -> searchable.createTrieSearch(indexConfig)); synchronized (search) { values.stream() .map(value -> new FrontendValue(value, value)) .forEach(value -> search.addItem(value, extractKeywords(value))); - - // Update totals for search - totals.put(searchable, getTotalFor(searchable)); } } - private void updateTotals(Map, TrieSearch> searchCache) { - final Object2LongOpenHashMap> partialTotals = new Object2LongOpenHashMap<>(searchCache.keySet().stream() - .collect(Collectors.toMap( - Functions.identity(), - this::getTotalFor - ))); - totals.putAll(partialTotals); - } - - private long getTotalFor(Searchable searchable) { - return searchable.getSearchReferences() - .stream() - .map(searchCache::get) - .filter(Objects::nonNull) // Failed or disabled searches are null - .flatMap(TrieSearch::stream) - .mapToInt(FrontendValue::hashCode) - .distinct() - .count(); - } - /** * Shrink the memory footprint of a search. After this action, no values can be registered anymore to a search. */ - public void shrinkSearch(Searchable searchable) { - final TrieSearch search = getSearchCache().get(searchable); + public void shrinkSearch(Searchable searchable) { + final TrieSearch search = searchCache.get(searchable); if (search == null) { log.warn("Searchable has no search associated: {}", searchable); @@ -145,6 +117,6 @@ public void shrinkSearch(Searchable searchable) { public synchronized void clearSearch() { totals = new Object2LongOpenHashMap<>(); - searchCache.clear(); + searchCache = new HashMap<>(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 460fefa906..d1215721e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -110,7 +110,7 @@ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus sta protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { if (columnDescriptions == null) { - columnDescriptions = generateColumnDescriptions(isInitialized(), getNamespace(), getConfig()); + columnDescriptions = generateColumnDescriptions(isInitialized(), getConfig()); } status.setColumnDescriptions(columnDescriptions); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java index 64fd65be1d..87f7c625d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java @@ -18,13 +18,13 @@ public interface SingleTableResult { - default List generateColumnDescriptions(boolean isInitialized, Namespace namespace, ConqueryConfig config) { + default List generateColumnDescriptions(boolean isInitialized, ConqueryConfig config) { Preconditions.checkArgument(isInitialized, "The execution must have been initialized first"); List columnDescriptions = new ArrayList<>(); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings(true, locale, namespace, config, null); + PrintSettings settings = new PrintSettings(true, locale, getNamespace(), config, null); UniqueNamer uniqNamer = new UniqueNamer(settings); @@ -53,4 +53,8 @@ default List generateColumnDescriptions(boolean isInitialized, @JsonIgnore long resultRowCount(); + @JsonIgnore + Namespace getNamespace(); + + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 40eb592659..df62608dfb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -42,7 +42,6 @@ import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; -import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; @@ -397,12 +396,12 @@ private Object[] getCompleteLine(EntityResult entityResult) { } protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { - status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getNamespace(), getConfig())); + status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } @Override - public List generateColumnDescriptions(boolean isInitialized, Namespace namespace, ConqueryConfig config) { - final List descriptors = getValuesQuery().generateColumnDescriptions(isInitialized, namespace, config); + public List generateColumnDescriptions(boolean isInitialized, ConqueryConfig config) { + final List descriptors = getValuesQuery().generateColumnDescriptions(isInitialized, config); for (ColumnDescriptor descriptor : descriptors) { // Add grouping semantics to secondaryIds to group by @@ -432,7 +431,7 @@ private ManagedQuery getValuesQuery() { @Override protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status) { - status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getNamespace(), getConfig())); + status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java index ba6ea6e54d..2e6b4894ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -35,28 +34,30 @@ public record ResultStatistics(int entities, int total, List statistics, Range dateRange) { @SneakyThrows @NotNull - public static ResultStatistics collectResultStatistics(ManagedQuery managedQuery, List resultInfos, Optional dateInfo, int dateIndex, PrintSettings printSettings, UniqueNamer uniqueNamer, ConqueryConfig conqueryConfig) { - + public static ResultStatistics collectResultStatistics(SingleTableResult managedQuery, List resultInfos, Optional dateInfo, Optional dateIndex, PrintSettings printSettings, UniqueNamer uniqueNamer, ConqueryConfig conqueryConfig) { //TODO pull inner executor service from ManagerNode - final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)); + final ListeningExecutorService + executorService = + MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)); // Yes, we are actually iterating the result for every job. // Span date-column final ListenableFuture> futureSpan; - if (managedQuery.isContainsDates()) { - futureSpan = executorService.submit(() -> calculateDateSpan(managedQuery, dateInfo, dateIndex)); + final boolean containsDates = dateInfo.isPresent(); + + if (containsDates) { + futureSpan = executorService.submit(() -> calculateDateSpan(managedQuery, dateInfo, dateIndex.get())); } else { futureSpan = Futures.immediateFuture(CDateRange.all().toSimpleRange()); } // Count result lines and entities (may differ in case of form or SecondaryIdQuery) - final ListenableFuture futureLines = - executorService.submit(() -> (int) managedQuery.getQuery().countResults(managedQuery.streamResults(OptionalLong.empty()))); + final ListenableFuture futureLines = executorService.submit(() -> (int) managedQuery.resultRowCount()); final ListenableFuture futureEntities = executorService.submit(() -> (int) managedQuery.streamResults(OptionalLong.empty()).count()); @@ -66,7 +67,7 @@ public static ResultStatistics collectResultStatistics(ManagedQuery managedQuery futureDescriptions = IntStream.range(0, resultInfos.size()) // If the query doesn't contain dates, we can skip the dates-column. - .filter(col -> !resultInfos.get(col).getSemantics().contains(new SemanticType.EventDateT()) || managedQuery.isContainsDates()) + .filter(col -> !resultInfos.get(col).getSemantics().contains(new SemanticType.EventDateT()) || containsDates) .mapToObj(col -> (Callable) () -> { final StopWatch started = StopWatch.createStarted(); @@ -76,7 +77,10 @@ public static ResultStatistics collectResultStatistics(ManagedQuery managedQuery log.trace("BEGIN stats collection for {}", info); - managedQuery.streamResults(OptionalLong.empty()).map(EntityResult::listResultLines).flatMap(List::stream).forEach(line -> statsCollector.consume(line[col])); + managedQuery.streamResults(OptionalLong.empty()) + .map(EntityResult::listResultLines) + .flatMap(List::stream) + .forEach(line -> statsCollector.consume(line[col])); log.trace("DONE collecting values for {}, in {}", info, started); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 2d1d9a0722..e6dd64867e 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -27,8 +27,6 @@ import com.bakdata.conquery.models.datasets.PreviewConfig; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.FrontEndConceptBuilder; -import com.bakdata.conquery.models.datasets.concepts.Searchable; -import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; @@ -78,14 +76,14 @@ public FrontendList load(Concept concept) { /** * Cache of all search results on SelectFilters. */ - private final LoadingCache, String>, List> + private final LoadingCache, String>, List> searchResults = CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { @Override - public List load(Pair, String> filterAndSearch) { + public List load(Pair, String> filterAndSearch) { final String searchTerm = filterAndSearch.getValue(); - final Searchable searchable = filterAndSearch.getKey(); + final SelectFilter searchable = filterAndSearch.getKey(); log.trace("Calculating a new search cache for the term \"{}\" on Searchable[{}]", searchTerm, searchable.getId()); @@ -97,9 +95,9 @@ public List load(Pair, String> filterAndSearch) { * Cache of raw listing of values on a filter. * We use Cursor here to reduce strain on memory and increase response time. */ - private final LoadingCache, CursorAndLength> listResults = CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { + private final LoadingCache, CursorAndLength> listResults = CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { @Override - public CursorAndLength load(Searchable searchable) { + public CursorAndLength load(SelectFilter searchable) { log.trace("Creating cursor for `{}`", searchable.getId()); return new CursorAndLength(listAllValues(searchable), countAllValues(searchable)); } @@ -159,7 +157,7 @@ public FrontendPreviewConfig getEntityPreviewFrontendConfig(Dataset dataset) { * Search for all search terms at once, with stricter scoring. * The user will upload a file and expect only well-corresponding resolutions. */ - public ResolvedFilterValues resolveFilterValues(Searchable searchable, List searchTerms) { + public ResolvedFilterValues resolveFilterValues(SelectFilter searchable, List searchTerms) { // search in the full text engine final Set openSearchTerms = new HashSet<>(searchTerms); @@ -183,13 +181,17 @@ public ResolvedFilterValues resolveFilterValues(Searchable searchable, List searchable, Optional maybeText, OptionalInt pageNumberOpt, OptionalInt itemsPerPageOpt) { + public AutoCompleteResult autocompleteTextFilter( + SelectFilter searchable, + Optional maybeText, + OptionalInt pageNumberOpt, + OptionalInt itemsPerPageOpt + ) { final int pageNumber = pageNumberOpt.orElse(0); final int itemsPerPage = itemsPerPageOpt.orElse(50); @@ -225,7 +227,7 @@ public AutoCompleteResult autocompleteTextFilter(Searchable searchable, Optio } } - private Cursor listAllValues(Searchable searchable) { + private Cursor listAllValues(SelectFilter searchable) { final Namespace namespace = namespaces.get(searchable.getDataset().getId()); /* Don't worry, I am as confused as you are! @@ -250,7 +252,7 @@ private Cursor listAllValues(Searchable searchable) { return new Cursor<>(Iterators.filter(iterators, seen::add)); } - private long countAllValues(Searchable searchable) { + private long countAllValues(SelectFilter searchable) { final Namespace namespace = namespaces.get(searchable.getDataset().getId()); return namespace.getFilterSearch().getTotal(searchable); @@ -260,7 +262,7 @@ private long countAllValues(Searchable searchable) { * Autocompletion for search terms. For values of {@link SelectFilter }. * Is used by the serach cache to load missing items */ - private List autocompleteTextFilter(Searchable searchable, String text) { + private List autocompleteTextFilter(SelectFilter searchable, String text) { final Namespace namespace = namespaces.get(searchable.getDataset().getId()); // Note that FEValues is equals/hashcode only on value: diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index 268fa9f201..599c06500b 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.SingleTableResult; import io.dropwizard.auth.Auth; import io.dropwizard.jersey.PATCH; @@ -73,7 +72,7 @@ public Response getDescription(@Auth Subject subject, @PathParam(QUERY) ManagedE return Response.status(Response.Status.CONFLICT.getStatusCode(), "Query is still running.").build(); // Request was submitted too early. } - return Response.ok((processor.getResultStatistics(((ManagedQuery) query)))).build(); + return Response.ok((processor.getResultStatistics(((SingleTableResult) query)))).build(); } @PATCH diff --git a/backend/src/test/java/com/bakdata/conquery/service/FilterSearchTest.java b/backend/src/test/java/com/bakdata/conquery/service/FilterSearchTest.java new file mode 100644 index 0000000000..825c13df7c --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/service/FilterSearchTest.java @@ -0,0 +1,70 @@ +package com.bakdata.conquery.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; + +import com.bakdata.conquery.models.config.IndexConfig; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SingleSelectFilter; +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.query.FilterSearch; +import com.google.common.collect.ImmutableBiMap; +import org.junit.jupiter.api.Test; + +public class FilterSearchTest { + + @Test + public void totals() { + final IndexConfig indexConfig = new IndexConfig(); + FilterSearch search = new FilterSearch(indexConfig); + + // Column Searchable + SelectFilter filter = new SingleSelectFilter(); + ConceptTreeConnector connector = new ConceptTreeConnector(); + TreeConcept concept = new TreeConcept(); + Column column = new Column(); + Table table = new Table(); + Dataset dataset = new Dataset("test_dataset"); + + table.setName("test_table"); + table.setDataset(dataset); + concept.setDataset(dataset); + concept.setName("test_concept"); + concept.setConnectors(List.of(connector)); + connector.setName("test_connector"); + connector.setFilters(List.of(filter)); + connector.setConcept(concept); + column.setTable(table); + column.setName("test_column"); + filter.setColumn(column); + filter.setConnector(connector); + + + // Map Searchable + filter.setLabels(ImmutableBiMap.of( + "mm", "MM", + "nn", "NN" + )); + + // Register + filter.getSearchReferences().forEach(searchable -> { + search.addSearches(Map.of(searchable, searchable.createTrieSearch(indexConfig))); + }); + + search.registerValues(column, List.of( + "a", + "bb", + "cc", + "mm" + )); + search.shrinkSearch(column); + + assertThat(search.getTotal(filter)).isEqualTo(5); + } +}