Merge Release

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up JDK
       uses: actions/setup-java@v4

.github/workflows/helm_chart.yml

@@ -0,0 +1,48 @@
+name: Helm
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - develop
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.12.0
+
+      - name: Lint Helm Chart
+        run: helm lint charts/conquery
+
+      - name: Validate Helm Templates
+        run: helm template conquery charts/conquery --debug
+
+  push-to-ghcr:
+    name: Push Helm Chart to GHCR (OCI)
+    runs-on: ubuntu-latest
+    needs: [lint]
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Helm
+        uses: azure/setup-helm@v3
+
+      - name: Login to GitHub Container Registry (GHCR)
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io --username $GITHUB_ACTOR --password-stdin
+
+      - name: Package Helm Chart
+        run: helm package charts/conquery
+
+      - name: Push Helm Chart to GHCR
+        run: |
+          helm push conquery-*.tgz oci://ghcr.io/${{ github.repository_owner }}/helm-charts

.github/workflows/lint_openapi.yml

@@ -11,7 +11,7 @@ jobs:
     container: dshanley/vacuum
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: "Lint OpenApi spec for /api"
         run: vacuum lint --details ./openapi.yaml
       - name: "Lint OpenApi spec for external form backends"

.github/workflows/prepare-merge-release.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 3
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: pull-request
         uses: repo-sync/pull-request@v2

.github/workflows/release_backend.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: true
           # This should be deep enough to fetch at least one light weight tag
@@ -25,7 +25,7 @@ jobs:
           java-version: '21'
           overwrite-settings: false
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

.github/workflows/run_autodoc.yml

@@ -18,13 +18,13 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
           submodules: true

.github/workflows/test_backend.yml

@@ -17,13 +17,13 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Set up JDK

.gitignore

@@ -77,4 +77,3 @@ tutorial/mimic_iii_demo/data/**
 /node_modules
 cypress/screenshots/
 cypress/videos/
-/cypress/support/test_data/table.cqpp

backend/pom.xml

@@ -15,7 +15,7 @@
         <apache-poi.version>5.2.2</apache-poi.version>
 		<apache-arrow.version>16.0.0</apache-arrow.version>
         <apache-parquet.version>1.12.3</apache-parquet.version>
-        <prometheus-metrics.version>1.2.1</prometheus-metrics.version>
+        <prometheus-metrics.version>1.3.5</prometheus-metrics.version>
         <openapi-generator.version>7.9.0</openapi-generator.version>
     </properties>
 

backend/src/main/java/com/bakdata/conquery/Conquery.java

@@ -17,6 +17,7 @@
 import com.bakdata.conquery.mode.cluster.ClusterManagerProvider;
 import com.bakdata.conquery.mode.local.LocalManagerProvider;
 import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.codahale.metrics.MetricRegistry;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.dropwizard.configuration.JsonConfigurationFactory;
 import io.dropwizard.configuration.SubstitutingSourceProvider;
@@ -59,12 +60,21 @@ public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
 		bootstrap.addCommand(new RecodeStoreCommand());
 		bootstrap.addCommand(new MigrateCommand());
 
-		((MutableInjectableValues) confMapper.getInjectableValues()).add(Validator.class, bootstrap.getValidatorFactory().getValidator());
+		MutableInjectableValues injectableValues = (MutableInjectableValues) confMapper.getInjectableValues();
+		injectableValues.add(Validator.class, bootstrap.getValidatorFactory().getValidator());
+		injectableValues.add(MetricRegistry.class, bootstrap.getMetricRegistry());
 
 		// do some setup in other classes after initialization but before running a
 		// command
 		bootstrap.addBundle(new ConfiguredBundle<>() {
 
+			@Override
+			public void initialize(Bootstrap<?> bootstrap) {
+				// Allow overriding of config from environment variables.
+				bootstrap.setConfigurationSourceProvider(new SubstitutingSourceProvider(
+						bootstrap.getConfigurationSourceProvider(), StringSubstitutor.createInterpolator()));
+			}
+
 			@Override
 			public void run(ConqueryConfig configuration, Environment environment) {
 				configuration.configureObjectMapper(environment.getObjectMapper());
@@ -77,13 +87,6 @@ protected void configure() {
 					}
 				});
 			}
-
-			@Override
-			public void initialize(Bootstrap<?> bootstrap) {
-				// Allow overriding of config from environment variables.
-				bootstrap.setConfigurationSourceProvider(new SubstitutingSourceProvider(
-						bootstrap.getConfigurationSourceProvider(), StringSubstitutor.createInterpolator()));
-			}
 		});
 
 		bootstrap.addBundle(new PrometheusBundle());

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -107,10 +107,10 @@ public class QueryProcessor {
 	private Validator validator;
 
 
-	public Stream<? extends ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
-		final Stream<ManagedExecution> allQueries = storage.getAllExecutions();
-
-		return getQueriesFiltered(dataset.getId(), RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders);
+	public List<? extends ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
+		try(Stream<ManagedExecution> allQueries = storage.getAllExecutions()) {
+			return getQueriesFiltered(dataset.getId(), RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders).toList();
+		}
 	}
 
 	public Stream<? extends ExecutionStatus> getQueriesFiltered(DatasetId datasetId, UriBuilder uriBuilder, Subject subject, Stream<ManagedExecution> allQueries, boolean allProviders) {
@@ -216,7 +216,7 @@ public void cancel(Subject subject, ManagedExecution query) {
 
 		log.info("User[{}] cancelled Query[{}]", subject.getId(), query.getId());
 
-		executionManager.cancelQuery(query);
+		executionManager.cancelExecution(query);
 	}
 
 	public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatch patch) {
@@ -269,18 +269,18 @@ public void reexecute(Subject subject, ManagedExecution query) {
 		if (!query.getState().equals(ExecutionState.RUNNING)) {
 			final Namespace namespace = query.getNamespace();
 
-			namespace.getExecutionManager().execute(query, config);
+			namespace.getExecutionManager().execute(query);
 		}
 	}
 
-	public void deleteQuery(Subject subject, ManagedExecution execution) {
-		log.info("User[{}] deleted Query[{}]", subject.getId(), execution.getId());
+	public void deleteQuery(Subject subject, ManagedExecutionId execution) {
+		log.info("User[{}] deleted Query[{}]", subject.getId(), execution);
 
 		datasetRegistry.get(execution.getDataset())
 					   .getExecutionManager() // Don't go over execution#getExecutionManager() as that's only set when query is initialized
 					   .clearQueryResults(execution);
 
-		storage.removeExecution(execution.getId());
+		storage.removeExecution(execution);
 	}
 
 	public ExecutionState awaitDone(ManagedExecution query, int time, TimeUnit unit) {
@@ -291,7 +291,7 @@ public ExecutionState awaitDone(ManagedExecution query, int time, TimeUnit unit)
 	public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) {
 		final Namespace namespace = datasetRegistry.get(query.getDataset());
 
-		query.initExecutable(config);
+		query.initExecutable();
 
 		final FullExecutionStatus status = query.buildStatusFull(subject, namespace);
 
@@ -338,7 +338,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext
 			execution.setLabel(upload.getLabel());
 		}
 
-		execution.initExecutable(config);
+		execution.initExecutable();
 
 		return new ExternalUploadResult(execution.getId(), statistic.getResolved().size(), statistic.getUnresolvedId(), statistic.getUnreadableDate());
 	}
@@ -375,9 +375,12 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri
 			throw new ConqueryError.ExecutionProcessingError();
 		}
 
+		// Workaround update our execution as the lastresultcount was set in the background
+		final EntityPreviewExecution executionFinished = (EntityPreviewExecution) execution.getId().resolve();
+		executionFinished.initExecutable();
 
 		final FullExecutionStatus status = execution.buildStatusFull(subject, namespace);
-		status.setResultUrls(getResultAssets(config.getResultProviders(), execution, uriBuilder, false));
+		status.setResultUrls(getResultAssets(config.getResultProviders(), executionFinished, uriBuilder, false));
 		return status;
 	}
 
@@ -434,21 +437,21 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje
 
 			final Optional<ManagedExecution>
 					execution =
-					executionId.map(id -> tryReuse(query, id, namespace, config, executionManager, subject.getUser()));
+					executionId.map(id -> tryReuse(query, id, namespace, executionManager, subject.getUser()));
 
 			if (execution.isPresent()) {
 				return execution.get();
 			}
 		}
 
 		// Execute the query
-		return executionManager.runQuery(namespace, query, subject.getId(), config, system);
+		return executionManager.runQuery(namespace, query, subject.getId(), system);
 	}
 
 	/**
 	 * Determine if the submitted query does reuse ONLY another query and restart that instead of creating another one.
 	 */
-	private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ConqueryConfig config, ExecutionManager executionManager, User user) {
+	private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ExecutionManager executionManager, User user) {
 
 		ManagedExecution execution = storage.getExecution(executionId);
 
@@ -491,7 +494,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe
 
 		log.trace("Re-executing Query {}", execution);
 
-		executionManager.execute(execution, config);
+		executionManager.execute(execution);
 
 		return execution;
 
@@ -577,7 +580,7 @@ public Stream<Map<String, String>> resolveEntities(Subject subject, List<FilterV
 					 .filter(Predicate.not(Map::isEmpty));
 	}
 
-	public ResultStatistics getResultStatistics(SingleTableResult managedQuery) {
+	public <E extends ManagedExecution & SingleTableResult> ResultStatistics getResultStatistics(E managedQuery) {
 
 		final Locale locale = I18n.LOCALE.get();
 		final NumberFormat decimalFormat = NumberFormat.getNumberInstance(locale);
@@ -590,6 +593,8 @@ public ResultStatistics getResultStatistics(SingleTableResult managedQuery) {
 				new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat);
 		final UniqueNamer uniqueNamer = new UniqueNamer(printSettings);
 
+		managedQuery.initExecutable();
+
 		final List<ResultInfo> resultInfos = managedQuery.getResultInfos();
 
 		final Optional<ResultInfo>

backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java

@@ -16,6 +16,7 @@
 import com.bakdata.conquery.io.cps.CPSTypeIdResolver;
 import com.bakdata.conquery.io.cps.SubTyped;
 import com.bakdata.conquery.io.storage.MetaStorage;
+import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormType;
@@ -133,8 +134,8 @@ public String getFormType() {
 	}
 
 	@Override
-	public ManagedExecution toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry<?> datasetRegistry) {
-		return new ExternalExecution(this, user, submittedDataset, storage, datasetRegistry);
+	public ManagedExecution toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry<?> datasetRegistry, ConqueryConfig config) {
+		return new ExternalExecution(this, user, submittedDataset, storage, datasetRegistry, config);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java

@@ -24,6 +24,7 @@
 import com.bakdata.conquery.internationalization.ExportFormC10n;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.storage.MetaStorage;
+import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.forms.managed.ManagedInternalForm;
 import com.bakdata.conquery.models.forms.util.Alignment;
 import com.bakdata.conquery.models.forms.util.Resolution;
@@ -136,8 +137,9 @@ public String getLocalizedTypeLabel() {
 	}
 
 	@Override
-	public ManagedInternalForm<ExportForm> toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry<?> datasetRegistry) {
-		return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry);
+	public ManagedInternalForm<ExportForm> toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry<?> datasetRegistry,
+															  ConqueryConfig config) {
+		return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry, config);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java

@@ -24,6 +24,7 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.common.Range;
+import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.forms.managed.ManagedInternalForm;
 import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
@@ -126,7 +127,8 @@ public String getLocalizedTypeLabel() {
 
 
 	@Override
-	public ManagedInternalForm<FullExportForm> toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry<?> datasetRegistry) {
-		return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry);
+	public ManagedInternalForm<FullExportForm> toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry<?> datasetRegistry,
+																  ConqueryConfig config) {
+		return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry, config);
 	}
 }