fix: Fetch in batches instead of using Query.max()

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
   release:
     types: [ published ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, 2025-01-29-eclipse-edc-0.7.2 ]
 
 env:
   REGISTRY_URL: ghcr.io

CHANGELOG.md

@@ -22,6 +22,7 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md).
 
 - Refactoring: Config as Java Code ([#1051](https://github.com/sovity/edc-ce/pull/1051))
 - Fix issues with the Create Data Offer Endpoint ([PR#1055](https://github.com/sovity/edc-ce/pull/1055))
+- Fix issue when the number of items is greater than 5000
 
 ### Deployment Migration Notes
 

extensions/wrapper/clients/benchmark/build.gradle.kts

@@ -0,0 +1,19 @@
+plugins {
+    id("java")
+}
+
+group = "org.example"
+version = "unspecified"
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    testImplementation(platform("org.junit:junit-bom:5.10.0"))
+    testImplementation("org.junit.jupiter:junit-jupiter")
+}
+
+tasks.test {
+    useJUnitPlatform()
+}

extensions/wrapper/clients/java-client-example/src/test/java/de/sovity/edc/client/examples/Benchmark.java

@@ -0,0 +1,142 @@
+package de.sovity.edc.client.examples;
+
+import de.sovity.edc.client.EdcClient;
+import de.sovity.edc.client.gen.model.ContractDefinitionRequest;
+import de.sovity.edc.client.gen.model.ContractNegotiationRequest;
+import de.sovity.edc.client.gen.model.DataSourceType;
+import de.sovity.edc.client.gen.model.OperatorDto;
+import de.sovity.edc.client.gen.model.PolicyDefinitionCreateDto;
+import de.sovity.edc.client.gen.model.UiAssetCreateRequest;
+import de.sovity.edc.client.gen.model.UiCriterion;
+import de.sovity.edc.client.gen.model.UiCriterionLiteral;
+import de.sovity.edc.client.gen.model.UiCriterionLiteralType;
+import de.sovity.edc.client.gen.model.UiCriterionOperator;
+import de.sovity.edc.client.gen.model.UiDataSource;
+import de.sovity.edc.client.gen.model.UiDataSourceHttpData;
+import de.sovity.edc.client.gen.model.UiPolicyConstraint;
+import de.sovity.edc.client.gen.model.UiPolicyExpression;
+import de.sovity.edc.client.gen.model.UiPolicyExpressionType;
+import de.sovity.edc.client.gen.model.UiPolicyLiteral;
+import de.sovity.edc.client.gen.model.UiPolicyLiteralType;
+import de.sovity.edc.utils.jsonld.vocab.Prop;
+
+import java.time.OffsetDateTime;
+import java.util.List;
+
+public class Benchmark {
+
+    private static String PROVIDER_DSP = "https://load-test-1.stage-sovity.azure.sovity.io/control/api/v1/dsp";
+    private static String PROVIDER_ID = "BPNL0000000A0UNJ";
+
+    public static void main(String[] args) {
+        var provider = EdcClient.builder()
+            .managementApiUrl("https://load-test-1.stage-sovity.azure.sovity.io/control/data")
+            .managementApiKey("LzyGxc8AMWrY+WKIrjd6yVzragE0vztZttzjWcopsBs=")
+            .build();
+
+        provider.testConnection();
+
+        var consumer = EdcClient.builder()
+            .managementApiUrl("https://load-test-1-consumer.stage-sovity.azure.sovity.io/control/data")
+            .managementApiKey("lOtQEZ/prbEtoGMlNjFsB3SNV1HSO00e7iSIyTKcrEo=")
+            .build();
+
+        consumer.testConnection();
+
+        create(provider, consumer, 0, 1);
+    }
+
+    private static void create(EdcClient provider, EdcClient consumer, int offset, int amount) {
+        int index = offset;
+
+        createAsset(provider, index);
+        createPolicy(provider, index);
+        createContractDefinition(provider, index);
+        negotiateContract(consumer, index);
+    }
+
+    private static void negotiateContract(EdcClient client, int index) {
+        client.uiApi().initiateContractNegotiation(
+            ContractNegotiationRequest.builder()
+                .counterPartyId(PROVIDER_ID)
+                .counterPartyAddress(PROVIDER_DSP)
+                .assetId("asset-" + index)
+                .contractOfferId("contract-offer-" + index)
+                .policyJsonLd("policy-" + index)
+                .build()
+        );
+    }
+
+    private static void createContractDefinition(EdcClient client, int index) {
+        client.uiApi().createContractDefinition(
+            ContractDefinitionRequest.builder()
+                .contractDefinitionId("contract-definition-" + index)
+                .accessPolicyId("policy-" + index)
+                .contractPolicyId("policy-" + index)
+                .assetSelector(List.of(UiCriterion.builder()
+                    .operandLeft(Prop.Edc.ID)
+                    .operator(UiCriterionOperator.EQ)
+                    .operandRight(UiCriterionLiteral.builder()
+                        .type(UiCriterionLiteralType.VALUE)
+                        .value("asset-" + index)
+                        .build())
+                    .build()))
+                .build()
+        );
+    }
+
+    private static void createPolicy(EdcClient client, int index) {
+        var afterYesterday = UiPolicyExpression.builder()
+            .type(UiPolicyExpressionType.CONSTRAINT)
+            .constraint(UiPolicyConstraint.builder()
+                .left("POLICY_EVALUATION_TIME")
+                .operator(OperatorDto.GT)
+                .right(UiPolicyLiteral.builder()
+                    .type(UiPolicyLiteralType.STRING)
+                    .value(OffsetDateTime.now().minusDays(1).toString())
+                    .build())
+                .build())
+            .build();
+
+        var beforeTomorrow = UiPolicyExpression.builder()
+            .type(UiPolicyExpressionType.CONSTRAINT)
+            .constraint(UiPolicyConstraint.builder()
+                .left("POLICY_EVALUATION_TIME")
+                .operator(OperatorDto.LT)
+                .right(UiPolicyLiteral.builder()
+                    .type(UiPolicyLiteralType.STRING)
+                    .value(OffsetDateTime.now().plusDays(1).toString())
+                    .build())
+                .build())
+            .build();
+
+        var expression = UiPolicyExpression.builder()
+            .type(UiPolicyExpressionType.AND)
+            .expressions(List.of(afterYesterday, beforeTomorrow))
+            .build();
+
+        client.uiApi().createPolicyDefinitionV2(
+            PolicyDefinitionCreateDto.builder()
+                .policyDefinitionId("policy-" + index)
+                .expression(expression)
+                .build()
+        );
+    }
+
+    private static void createAsset(EdcClient client, int index) {
+        client.uiApi().createAsset(
+            UiAssetCreateRequest.builder()
+                .title("Test Asset " + index)
+                .id("asset-" + index)
+                .dataSource(
+                    UiDataSource.builder()
+                        .type(DataSourceType.HTTP_DATA)
+                        .httpData(UiDataSourceHttpData.builder()
+                            .baseUrl("http://example.com")
+                            .build())
+                        .build()
+                )
+                .build()
+        );
+    }
+}

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java

@@ -252,11 +252,7 @@ public static WrapperExtensionContext buildContext(
         var cxDidConfigService = new CxDidConfigService(config);
         var dapsConfigService = new DapsConfigService(config);
         var dashboardDataFetcher = new DashboardDataFetcher(
-            contractNegotiationStore,
-            transferProcessService,
-            assetIndex,
-            policyDefinitionService,
-            contractDefinitionService
+            transferProcessService
         );
         var dashboardApiService = new DashboardPageApiService(
             dashboardDataFetcher,
@@ -315,7 +311,8 @@ public static WrapperExtensionContext buildContext(
         var useCaseResource = new UseCaseResourceImpl(
             kpiApiService,
             supportedPolicyApiService,
-            useCaseCatalogApiService
+            useCaseCatalogApiService,
+            dslContextFactory
         );
         val placeholderEndpointController = new PlaceholderEndpointController();
 

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResourceImpl.java

@@ -77,7 +77,7 @@ public class UiResourceImpl implements UiResource {
 
     @Override
     public DashboardPage getDashboardPage() {
-        return dashboardPageApiService.dashboardPage();
+        return dslContextFactory.transactionResult(dashboardPageApiService::dashboardPage);
     }
 
     @Override
@@ -159,13 +159,13 @@ public UiContractNegotiation getContractNegotiation(String contractNegotiationId
     @Override
     public ContractAgreementPage getContractAgreementPage(@Nullable ContractAgreementPageQuery contractAgreementPageQuery) {
         return dslContextFactory.transactionResult(dsl ->
-            contractAgreementApiService.contractAgreementPage(dsl, contractAgreementPageQuery));
+                contractAgreementApiService.contractAgreementPage(dsl, contractAgreementPageQuery));
     }
 
     @Override
     public ContractAgreementCard getContractAgreementCard(String contractAgreementId) {
         return dslContextFactory.transactionResult(dsl ->
-            contractAgreementApiService.contractAgreement(dsl, contractAgreementId));
+                contractAgreementApiService.contractAgreement(dsl, contractAgreementId));
     }
 
     @Override
@@ -180,12 +180,12 @@ public IdResponseDto initiateCustomTransfer(InitiateCustomTransferRequest reques
 
     @Override
     public IdResponseDto terminateContractAgreement(
-        String contractAgreementId,
-        ContractTerminationRequest contractTerminationRequest
+            String contractAgreementId,
+            ContractTerminationRequest contractTerminationRequest
     ) {
         validate(contractTerminationRequest);
         return dslContextFactory.transactionResult(dsl ->
-            contractAgreementTerminationApiService.terminate(dsl, contractAgreementId, contractTerminationRequest));
+                contractAgreementTerminationApiService.terminate(dsl, contractAgreementId, contractTerminationRequest));
     }
 
     @Override
@@ -201,18 +201,18 @@ public UiAsset getTransferProcessAsset(String transferProcessId) {
     @Override
     public IdAvailabilityResponse isPolicyIdAvailable(String policyId) {
         return dslContextFactory.transactionResult(dsl ->
-            dataOfferPageApiService.checkIfPolicyIdAvailable(dsl, policyId));
+                dataOfferPageApiService.checkIfPolicyIdAvailable(dsl, policyId));
     }
 
     @Override
     public IdAvailabilityResponse isAssetIdAvailable(String assetId) {
         return dslContextFactory.transactionResult(dsl ->
-            dataOfferPageApiService.checkIfAssetIdAvailable(dsl, assetId));
+                dataOfferPageApiService.checkIfAssetIdAvailable(dsl, assetId));
     }
 
     @Override
     public IdAvailabilityResponse isContractDefinitionIdAvailable(String contractDefinitionId) {
         return dslContextFactory.transactionResult(dsl ->
-            dataOfferPageApiService.checkIfContractDefinitionIdAvailable(dsl, contractDefinitionId));
+                dataOfferPageApiService.checkIfContractDefinitionIdAvailable(dsl, contractDefinitionId));
     }
 }

extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/asset/AssetApiService.java

@@ -14,21 +14,20 @@
 
 package de.sovity.edc.ext.wrapper.api.ui.pages.asset;
 
-import de.sovity.edc.ext.db.jooq.Tables;
 import de.sovity.edc.ext.wrapper.api.ServiceException;
 import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
 import de.sovity.edc.ext.wrapper.api.common.model.UiAsset;
 import de.sovity.edc.ext.wrapper.api.common.model.UiAssetCreateRequest;
 import de.sovity.edc.ext.wrapper.api.common.model.UiAssetEditRequest;
 import de.sovity.edc.ext.wrapper.api.ui.model.IdResponseDto;
 import de.sovity.edc.ext.wrapper.api.ui.pages.dashboard.services.SelfDescriptionService;
+import de.sovity.edc.ext.wrapper.utils.QueryUtils;
 import lombok.RequiredArgsConstructor;
 import lombok.val;
 import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset;
 import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService;
 import org.eclipse.edc.spi.query.QuerySpec;
 import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
 
 import java.util.Comparator;
 import java.util.List;
@@ -75,14 +74,10 @@ public IdResponseDto deleteAsset(String assetId) {
     }
 
     private List<Asset> getAllAssets() {
-        return assetService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList();
-    }
-
-    public boolean assetExists(DSLContext dsl, String assetId) {
-        val a = Tables.EDC_ASSET;
-        return dsl.selectCount()
-            .from(a)
-            .where(a.ASSET_ID.eq(assetId))
-            .fetchSingleInto(Integer.class) > 0;
+        return QueryUtils.fetchAllInBatches((offset, size) ->
+            assetService.search(
+                QuerySpec.Builder.newInstance().offset(offset).limit(size).build()
+            ).orElseThrow(ServiceException::new)
+        );
     }
 }