diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e0b8a9a1..a37bd8c73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: release: types: [ published ] pull_request: - branches: [ main ] + branches: [ main, 2025-01-29-eclipse-edc-0.7.2 ] env: REGISTRY_URL: ghcr.io diff --git a/CHANGELOG.md b/CHANGELOG.md index 29158730c..6afe45df4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md). - Refactoring: Config as Java Code ([#1051](https://github.com/sovity/edc-ce/pull/1051)) - Fix issues with the Create Data Offer Endpoint ([PR#1055](https://github.com/sovity/edc-ce/pull/1055)) +- Fix issue when the number of items is greater than 5000 ### Deployment Migration Notes diff --git a/extensions/wrapper/clients/benchmark/build.gradle.kts b/extensions/wrapper/clients/benchmark/build.gradle.kts new file mode 100644 index 000000000..2e73f72da --- /dev/null +++ b/extensions/wrapper/clients/benchmark/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + id("java") +} + +group = "org.example" +version = "unspecified" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.10.0")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +tasks.test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/extensions/wrapper/clients/java-client-example/src/test/java/de/sovity/edc/client/examples/Benchmark.java b/extensions/wrapper/clients/java-client-example/src/test/java/de/sovity/edc/client/examples/Benchmark.java new file mode 100644 index 000000000..cea2fbb79 --- /dev/null +++ b/extensions/wrapper/clients/java-client-example/src/test/java/de/sovity/edc/client/examples/Benchmark.java @@ -0,0 +1,142 @@ +package de.sovity.edc.client.examples; + +import de.sovity.edc.client.EdcClient; +import de.sovity.edc.client.gen.model.ContractDefinitionRequest; +import de.sovity.edc.client.gen.model.ContractNegotiationRequest; +import de.sovity.edc.client.gen.model.DataSourceType; +import de.sovity.edc.client.gen.model.OperatorDto; +import de.sovity.edc.client.gen.model.PolicyDefinitionCreateDto; +import de.sovity.edc.client.gen.model.UiAssetCreateRequest; +import de.sovity.edc.client.gen.model.UiCriterion; +import de.sovity.edc.client.gen.model.UiCriterionLiteral; +import de.sovity.edc.client.gen.model.UiCriterionLiteralType; +import de.sovity.edc.client.gen.model.UiCriterionOperator; +import de.sovity.edc.client.gen.model.UiDataSource; +import de.sovity.edc.client.gen.model.UiDataSourceHttpData; +import de.sovity.edc.client.gen.model.UiPolicyConstraint; +import de.sovity.edc.client.gen.model.UiPolicyExpression; +import de.sovity.edc.client.gen.model.UiPolicyExpressionType; +import de.sovity.edc.client.gen.model.UiPolicyLiteral; +import de.sovity.edc.client.gen.model.UiPolicyLiteralType; +import de.sovity.edc.utils.jsonld.vocab.Prop; + +import java.time.OffsetDateTime; +import java.util.List; + +public class Benchmark { + + private static String PROVIDER_DSP = "https://load-test-1.stage-sovity.azure.sovity.io/control/api/v1/dsp"; + private static String PROVIDER_ID = "BPNL0000000A0UNJ"; + + public static void main(String[] args) { + var provider = EdcClient.builder() + .managementApiUrl("https://load-test-1.stage-sovity.azure.sovity.io/control/data") + .managementApiKey("LzyGxc8AMWrY+WKIrjd6yVzragE0vztZttzjWcopsBs=") + .build(); + + provider.testConnection(); + + var consumer = EdcClient.builder() + .managementApiUrl("https://load-test-1-consumer.stage-sovity.azure.sovity.io/control/data") + .managementApiKey("lOtQEZ/prbEtoGMlNjFsB3SNV1HSO00e7iSIyTKcrEo=") + .build(); + + consumer.testConnection(); + + create(provider, consumer, 0, 1); + } + + private static void create(EdcClient provider, EdcClient consumer, int offset, int amount) { + int index = offset; + + createAsset(provider, index); + createPolicy(provider, index); + createContractDefinition(provider, index); + negotiateContract(consumer, index); + } + + private static void negotiateContract(EdcClient client, int index) { + client.uiApi().initiateContractNegotiation( + ContractNegotiationRequest.builder() + .counterPartyId(PROVIDER_ID) + .counterPartyAddress(PROVIDER_DSP) + .assetId("asset-" + index) + .contractOfferId("contract-offer-" + index) + .policyJsonLd("policy-" + index) + .build() + ); + } + + private static void createContractDefinition(EdcClient client, int index) { + client.uiApi().createContractDefinition( + ContractDefinitionRequest.builder() + .contractDefinitionId("contract-definition-" + index) + .accessPolicyId("policy-" + index) + .contractPolicyId("policy-" + index) + .assetSelector(List.of(UiCriterion.builder() + .operandLeft(Prop.Edc.ID) + .operator(UiCriterionOperator.EQ) + .operandRight(UiCriterionLiteral.builder() + .type(UiCriterionLiteralType.VALUE) + .value("asset-" + index) + .build()) + .build())) + .build() + ); + } + + private static void createPolicy(EdcClient client, int index) { + var afterYesterday = UiPolicyExpression.builder() + .type(UiPolicyExpressionType.CONSTRAINT) + .constraint(UiPolicyConstraint.builder() + .left("POLICY_EVALUATION_TIME") + .operator(OperatorDto.GT) + .right(UiPolicyLiteral.builder() + .type(UiPolicyLiteralType.STRING) + .value(OffsetDateTime.now().minusDays(1).toString()) + .build()) + .build()) + .build(); + + var beforeTomorrow = UiPolicyExpression.builder() + .type(UiPolicyExpressionType.CONSTRAINT) + .constraint(UiPolicyConstraint.builder() + .left("POLICY_EVALUATION_TIME") + .operator(OperatorDto.LT) + .right(UiPolicyLiteral.builder() + .type(UiPolicyLiteralType.STRING) + .value(OffsetDateTime.now().plusDays(1).toString()) + .build()) + .build()) + .build(); + + var expression = UiPolicyExpression.builder() + .type(UiPolicyExpressionType.AND) + .expressions(List.of(afterYesterday, beforeTomorrow)) + .build(); + + client.uiApi().createPolicyDefinitionV2( + PolicyDefinitionCreateDto.builder() + .policyDefinitionId("policy-" + index) + .expression(expression) + .build() + ); + } + + private static void createAsset(EdcClient client, int index) { + client.uiApi().createAsset( + UiAssetCreateRequest.builder() + .title("Test Asset " + index) + .id("asset-" + index) + .dataSource( + UiDataSource.builder() + .type(DataSourceType.HTTP_DATA) + .httpData(UiDataSourceHttpData.builder() + .baseUrl("http://example.com") + .build()) + .build() + ) + .build() + ); + } +} diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java index 0cbc20909..a6bbb5598 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/WrapperExtensionContextBuilder.java @@ -252,11 +252,7 @@ public static WrapperExtensionContext buildContext( var cxDidConfigService = new CxDidConfigService(config); var dapsConfigService = new DapsConfigService(config); var dashboardDataFetcher = new DashboardDataFetcher( - contractNegotiationStore, - transferProcessService, - assetIndex, - policyDefinitionService, - contractDefinitionService + transferProcessService ); var dashboardApiService = new DashboardPageApiService( dashboardDataFetcher, @@ -315,7 +311,8 @@ public static WrapperExtensionContext buildContext( var useCaseResource = new UseCaseResourceImpl( kpiApiService, supportedPolicyApiService, - useCaseCatalogApiService + useCaseCatalogApiService, + dslContextFactory ); val placeholderEndpointController = new PlaceholderEndpointController(); diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResourceImpl.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResourceImpl.java index c6c234a8b..3c2da9cbe 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResourceImpl.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/UiResourceImpl.java @@ -77,7 +77,7 @@ public class UiResourceImpl implements UiResource { @Override public DashboardPage getDashboardPage() { - return dashboardPageApiService.dashboardPage(); + return dslContextFactory.transactionResult(dashboardPageApiService::dashboardPage); } @Override @@ -159,13 +159,13 @@ public UiContractNegotiation getContractNegotiation(String contractNegotiationId @Override public ContractAgreementPage getContractAgreementPage(@Nullable ContractAgreementPageQuery contractAgreementPageQuery) { return dslContextFactory.transactionResult(dsl -> - contractAgreementApiService.contractAgreementPage(dsl, contractAgreementPageQuery)); + contractAgreementApiService.contractAgreementPage(dsl, contractAgreementPageQuery)); } @Override public ContractAgreementCard getContractAgreementCard(String contractAgreementId) { return dslContextFactory.transactionResult(dsl -> - contractAgreementApiService.contractAgreement(dsl, contractAgreementId)); + contractAgreementApiService.contractAgreement(dsl, contractAgreementId)); } @Override @@ -180,12 +180,12 @@ public IdResponseDto initiateCustomTransfer(InitiateCustomTransferRequest reques @Override public IdResponseDto terminateContractAgreement( - String contractAgreementId, - ContractTerminationRequest contractTerminationRequest + String contractAgreementId, + ContractTerminationRequest contractTerminationRequest ) { validate(contractTerminationRequest); return dslContextFactory.transactionResult(dsl -> - contractAgreementTerminationApiService.terminate(dsl, contractAgreementId, contractTerminationRequest)); + contractAgreementTerminationApiService.terminate(dsl, contractAgreementId, contractTerminationRequest)); } @Override @@ -201,18 +201,18 @@ public UiAsset getTransferProcessAsset(String transferProcessId) { @Override public IdAvailabilityResponse isPolicyIdAvailable(String policyId) { return dslContextFactory.transactionResult(dsl -> - dataOfferPageApiService.checkIfPolicyIdAvailable(dsl, policyId)); + dataOfferPageApiService.checkIfPolicyIdAvailable(dsl, policyId)); } @Override public IdAvailabilityResponse isAssetIdAvailable(String assetId) { return dslContextFactory.transactionResult(dsl -> - dataOfferPageApiService.checkIfAssetIdAvailable(dsl, assetId)); + dataOfferPageApiService.checkIfAssetIdAvailable(dsl, assetId)); } @Override public IdAvailabilityResponse isContractDefinitionIdAvailable(String contractDefinitionId) { return dslContextFactory.transactionResult(dsl -> - dataOfferPageApiService.checkIfContractDefinitionIdAvailable(dsl, contractDefinitionId)); + dataOfferPageApiService.checkIfContractDefinitionIdAvailable(dsl, contractDefinitionId)); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/asset/AssetApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/asset/AssetApiService.java index 0265d9ac8..5e828b23f 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/asset/AssetApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/asset/AssetApiService.java @@ -14,7 +14,6 @@ package de.sovity.edc.ext.wrapper.api.ui.pages.asset; -import de.sovity.edc.ext.db.jooq.Tables; import de.sovity.edc.ext.wrapper.api.ServiceException; import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper; import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; @@ -22,13 +21,13 @@ import de.sovity.edc.ext.wrapper.api.common.model.UiAssetEditRequest; import de.sovity.edc.ext.wrapper.api.ui.model.IdResponseDto; import de.sovity.edc.ext.wrapper.api.ui.pages.dashboard.services.SelfDescriptionService; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import lombok.RequiredArgsConstructor; import lombok.val; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; import org.eclipse.edc.spi.query.QuerySpec; import org.jetbrains.annotations.NotNull; -import org.jooq.DSLContext; import java.util.Comparator; import java.util.List; @@ -75,14 +74,10 @@ public IdResponseDto deleteAsset(String assetId) { } private List getAllAssets() { - return assetService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); - } - - public boolean assetExists(DSLContext dsl, String assetId) { - val a = Tables.EDC_ASSET; - return dsl.selectCount() - .from(a) - .where(a.ASSET_ID.eq(assetId)) - .fetchSingleInto(Integer.class) > 0; + return QueryUtils.fetchAllInBatches((offset, size) -> + assetService.search( + QuerySpec.Builder.newInstance().offset(offset).limit(size).build() + ).orElseThrow(ServiceException::new) + ); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementDataFetcher.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementDataFetcher.java index 04569c446..8a9f02b68 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementDataFetcher.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_agreements/services/ContractAgreementDataFetcher.java @@ -17,6 +17,7 @@ import de.sovity.edc.ext.db.jooq.tables.records.SovityContractTerminationRecord; import de.sovity.edc.ext.wrapper.api.ServiceException; import de.sovity.edc.ext.wrapper.utils.MapUtils; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import lombok.RequiredArgsConstructor; import lombok.val; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; @@ -73,7 +74,12 @@ public List getContractAgreements(DSLContext dsl) { .flatMap(agreement -> negotiations.getOrDefault(agreement.getId(), List.of()) .stream() .map(negotiation -> { - var asset = getAsset(agreement, negotiation, assets::get); + var asset = getAsset( + agreement.getAssetId(), + negotiation.getType(), + assets::get + ); + var contractTransfers = transfers.getOrDefault(agreement.getId(), List.of()); return new ContractAgreementData(agreement, negotiation, asset, contractTransfers, terminations.get(agreement.getId())); })) @@ -84,8 +90,12 @@ public List getContractAgreements(DSLContext dsl) { public ContractAgreementData getContractAgreement(DSLContext dsl, String contractAgreementId) { val agreement = getContractAgreementById(contractAgreementId); - val negotiationQuery = QuerySpec.max(); - val negotiation = contractNegotiationStore.queryNegotiations(negotiationQuery) + val negotiation = QueryUtils.fetchAllInBatches((offset, limit) -> + contractNegotiationStore.queryNegotiations( + QuerySpec.Builder.newInstance().offset(offset).limit(limit).build() + ).toList() + ) + .stream() .filter(it -> it.getContractAgreement().getId().equals(contractAgreementId)) .findFirst() .orElseThrow( @@ -95,7 +105,7 @@ public ContractAgreementData getContractAgreement(DSLContext dsl, String contrac val terminations = fetchTerminations(dsl, agreement.getId()); - val asset = getAsset(agreement, negotiation, (it) -> assetIndex.findById(agreement.getAssetId())); + val asset = getAsset(agreement.getAssetId(), negotiation.getType(), (it) -> assetIndex.findById(agreement.getAssetId())); return new ContractAgreementData( agreement, @@ -128,10 +138,8 @@ private Map fetchTerminations(DSLContex .collect(toMap(SovityContractTerminationRecord::getContractAgreementId, identity())); } - private Asset getAsset(ContractAgreement agreement, ContractNegotiation negotiation, Function selector) { - var assetId = agreement.getAssetId(); - - if (negotiation.getType() == ContractNegotiation.Type.CONSUMER) { + private Asset getAsset(String assetId, ContractNegotiation.Type negotiationType, Function selector) { + if (negotiationType == ContractNegotiation.Type.CONSUMER) { return dummyAsset(assetId); } @@ -144,21 +152,37 @@ private Asset dummyAsset(String assetId) { } private List getAllAssets() { - return assetIndex.queryAssets(QuerySpec.max()).toList(); + return QueryUtils.fetchAllInBatches((offset, batchSize) -> + assetIndex.queryAssets( + QuerySpec.Builder.newInstance().offset(offset).limit(batchSize).build() + ).toList() + ); } @NotNull private List getAllContractNegotiations() { - return contractNegotiationStore.queryNegotiations(QuerySpec.max()).toList(); + return QueryUtils.fetchAllInBatches( + (offset, batchSize) -> contractNegotiationStore.queryNegotiations( + QuerySpec.Builder.newInstance().offset(offset).limit(batchSize).build() + ).toList() + ); } @NotNull private List getAllContractAgreements() { - return contractAgreementService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, batchSize) -> + contractAgreementService.search( + QuerySpec.Builder.newInstance().offset(offset).limit(batchSize).build() + ).orElseThrow(ServiceException::new) + ); } @NotNull private List getAllTransferProcesses() { - return transferProcessService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, batchSize) -> + transferProcessService.search( + QuerySpec.Builder.newInstance().offset(offset).limit(batchSize).build() + ).orElseThrow(ServiceException::new) + ); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_definitions/ContractDefinitionApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_definitions/ContractDefinitionApiService.java index 5a933b964..afb34c9e7 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_definitions/ContractDefinitionApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/contract_definitions/ContractDefinitionApiService.java @@ -18,16 +18,14 @@ import de.sovity.edc.ext.wrapper.api.ui.model.ContractDefinitionEntry; import de.sovity.edc.ext.wrapper.api.ui.model.ContractDefinitionRequest; import de.sovity.edc.ext.wrapper.api.ui.model.IdResponseDto; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.eclipse.edc.connector.controlplane.contract.spi.types.offer.ContractDefinition; import org.eclipse.edc.connector.controlplane.services.spi.contractdefinition.ContractDefinitionService; -import org.eclipse.edc.spi.entity.Entity; import org.eclipse.edc.spi.query.QuerySpec; import org.jetbrains.annotations.NotNull; -import java.io.File; -import java.io.FileWriter; import java.util.Comparator; import java.util.List; @@ -43,9 +41,9 @@ public List getContractDefinitions() { var definitions = getAllContractDefinitions(); return definitions.stream() - .sorted(Comparator.comparing(ContractDefinition::getCreatedAt).reversed()) - .map(this::buildContractDefinitionEntry) - .toList(); + .sorted(Comparator.comparing(ContractDefinition::getCreatedAt).reversed()) + .map(this::buildContractDefinitionEntry) + .toList(); } @NotNull @@ -72,6 +70,13 @@ public IdResponseDto deleteContractDefinition(String contractDefinitionId) { } private List getAllContractDefinitions() { - return contractDefinitionService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, limit) -> + contractDefinitionService.search( + QuerySpec.Builder.newInstance() + .offset(offset) + .limit(limit) + .build() + ).orElseThrow(ServiceException::new) + ); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiService.java index 9aef2059d..adfafb180 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiService.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.wrapper.api.ui.pages.dashboard; +import de.sovity.edc.ext.db.jooq.Tables; import de.sovity.edc.ext.wrapper.api.ui.model.DashboardPage; import de.sovity.edc.ext.wrapper.api.ui.model.DashboardTransferAmounts; import de.sovity.edc.ext.wrapper.api.ui.pages.dashboard.services.CxDidConfigService; @@ -22,10 +23,12 @@ import de.sovity.edc.ext.wrapper.api.ui.pages.dashboard.services.SelfDescriptionService; import de.sovity.edc.ext.wrapper.api.ui.pages.transferhistory.TransferProcessStateService; import lombok.RequiredArgsConstructor; +import lombok.val; import org.eclipse.edc.connector.controlplane.contract.spi.types.agreement.ContractAgreement; import org.eclipse.edc.connector.controlplane.contract.spi.types.negotiation.ContractNegotiation; import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; import java.util.List; import java.util.Objects; @@ -45,33 +48,36 @@ public class DashboardPageApiService { private final SelfDescriptionService selfDescriptionService; @NotNull - public DashboardPage dashboardPage() { + public DashboardPage dashboardPage(DSLContext dsl) { var transferProcesses = dashboardDataFetcher.getTransferProcesses(); - var negotiations = dashboardDataFetcher.getAllContractNegotiations(); - var providingAgreements = negotiations.stream() - .filter(it -> it.getType() == ContractNegotiation.Type.PROVIDER) - .map(ContractNegotiation::getContractAgreement) - .filter(Objects::nonNull) - .map(ContractAgreement::getId) - .collect(toSet()); - - - var consumingAgreements = negotiations.stream() - .filter(it -> it.getType() == ContractNegotiation.Type.CONSUMER) - .map(ContractNegotiation::getContractAgreement) - .filter(Objects::nonNull) - .map(ContractAgreement::getId) - .collect(toSet()); + val n = Tables.EDC_CONTRACT_NEGOTIATION; + val a = Tables.EDC_CONTRACT_AGREEMENT; + + val providingAgreementsDb = dsl + .select(a.AGR_ID) + .from(n) + .leftJoin(a) + .on(n.AGREEMENT_ID.eq(a.AGR_ID)) + .where(n.TYPE.eq(ContractNegotiation.Type.PROVIDER.toString())) + .fetchSet(a.AGR_ID); + + val consumingAgreementsDb = dsl + .select(a.AGR_ID) + .from(n) + .leftJoin(a) + .on(n.AGREEMENT_ID.eq(a.AGR_ID)) + .where(n.TYPE.eq(ContractNegotiation.Type.CONSUMER.toString())) + .fetchSet(a.AGR_ID); DashboardPage dashboardPage = new DashboardPage(); - dashboardPage.setNumAssets(dashboardDataFetcher.getNumberOfAssets()); - dashboardPage.setNumPolicies(dashboardDataFetcher.getNumberOfPolicies()); - dashboardPage.setNumContractDefinitions(dashboardDataFetcher.getNumberOfContractDefinitions()); - dashboardPage.setNumContractAgreementsProviding(providingAgreements.size()); - dashboardPage.setNumContractAgreementsConsuming(consumingAgreements.size()); - dashboardPage.setTransferProcessesProviding(getTransferAmounts(transferProcesses, providingAgreements)); - dashboardPage.setTransferProcessesConsuming(getTransferAmounts(transferProcesses, consumingAgreements)); + dashboardPage.setNumAssets(dashboardDataFetcher.getNumberOfAssets(dsl)); + dashboardPage.setNumPolicies(dashboardDataFetcher.getNumberOfPolicies(dsl)); + dashboardPage.setNumContractDefinitions(dashboardDataFetcher.getNumberOfContractDefinitions(dsl)); + dashboardPage.setNumContractAgreementsProviding(providingAgreementsDb.size()); + dashboardPage.setNumContractAgreementsConsuming(consumingAgreementsDb.size()); + dashboardPage.setTransferProcessesProviding(getTransferAmounts(transferProcesses, providingAgreementsDb)); + dashboardPage.setTransferProcessesConsuming(getTransferAmounts(transferProcesses, consumingAgreementsDb)); dashboardPage.setConnectorTitle(selfDescriptionService.getConnectorTitle()); dashboardPage.setConnectorDescription(selfDescriptionService.getConnectorDescription()); @@ -89,27 +95,27 @@ public DashboardPage dashboardPage() { } DashboardTransferAmounts getTransferAmounts( - List transferProcesses, - Set agreements + List transferProcesses, + Set agreements ) { var numTotal = transferProcesses.stream() - .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) - .count(); + .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) + .count(); var numOk = transferProcesses.stream() - .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) - .filter(transferProcess -> transferProcessStateService.getSimplifiedState(transferProcess.getState()).equals(OK)) - .count(); + .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) + .filter(transferProcess -> transferProcessStateService.getSimplifiedState(transferProcess.getState()).equals(OK)) + .count(); var numRunning = transferProcesses.stream() - .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) - .filter(transferProcess -> transferProcessStateService.getSimplifiedState(transferProcess.getState()).equals(RUNNING)) - .count(); + .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) + .filter(transferProcess -> transferProcessStateService.getSimplifiedState(transferProcess.getState()).equals(RUNNING)) + .count(); var numError = transferProcesses.stream() - .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) - .filter(transferProcess -> transferProcessStateService.getSimplifiedState(transferProcess.getState()).equals(ERROR)) - .count(); + .filter(transferProcess -> agreements.contains(transferProcess.getContractId())) + .filter(transferProcess -> transferProcessStateService.getSimplifiedState(transferProcess.getState()).equals(ERROR)) + .count(); return new DashboardTransferAmounts(numTotal, numRunning, numOk, numError); } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/services/DashboardDataFetcher.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/services/DashboardDataFetcher.java index 57fb856ce..497ba6df4 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/services/DashboardDataFetcher.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/services/DashboardDataFetcher.java @@ -14,50 +14,43 @@ package de.sovity.edc.ext.wrapper.api.ui.pages.dashboard.services; +import de.sovity.edc.ext.db.jooq.Tables; import de.sovity.edc.ext.wrapper.api.ServiceException; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import lombok.RequiredArgsConstructor; -import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; -import org.eclipse.edc.connector.controlplane.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.controlplane.contract.spi.types.negotiation.ContractNegotiation; -import org.eclipse.edc.connector.controlplane.services.spi.contractdefinition.ContractDefinitionService; -import org.eclipse.edc.connector.controlplane.services.spi.policydefinition.PolicyDefinitionService; import org.eclipse.edc.connector.controlplane.services.spi.transferprocess.TransferProcessService; import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; import org.eclipse.edc.spi.query.QuerySpec; import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; import java.util.List; @RequiredArgsConstructor public class DashboardDataFetcher { - private final ContractNegotiationStore contractNegotiationStore; private final TransferProcessService transferProcessService; - private final AssetIndex assetIndex; - private final PolicyDefinitionService policyDefinitionService; - private final ContractDefinitionService contractDefinitionService; - public int getNumberOfAssets() { - return assetIndex.queryAssets(QuerySpec.max()).toList().size(); + public int getNumberOfAssets(DSLContext dsl) { + return dsl.fetchCount(Tables.EDC_ASSET); } - public int getNumberOfPolicies() { - return Math.toIntExact(policyDefinitionService.query(QuerySpec.max()) - .orElseThrow(ServiceException::new) - .count()); + public int getNumberOfPolicies(DSLContext dsl) { + return dsl.selectCount() + .from(Tables.EDC_POLICYDEFINITIONS) + .fetchSingleInto(Integer.class); } - public int getNumberOfContractDefinitions() { - return Math.toIntExact(contractDefinitionService.query(QuerySpec.max()) - .orElseThrow(ServiceException::new) - .count()); + public int getNumberOfContractDefinitions(DSLContext dsl) { + return dsl.selectCount() + .from(Tables.EDC_CONTRACT_DEFINITIONS) + .fetchSingleInto(Integer.class); } - @NotNull - public List getAllContractNegotiations() { - return contractNegotiationStore.queryNegotiations(QuerySpec.max()).toList(); - } @NotNull public List getTransferProcesses() { - return transferProcessService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches( (offset, limit) -> transferProcessService.search( + QuerySpec.Builder.newInstance().offset(offset).limit(limit).build() + ).orElseThrow(ServiceException::new) + ); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/data_offer/DataOfferPageApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/data_offer/DataOfferPageApiService.java index a19ec1316..e06ed91d4 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/data_offer/DataOfferPageApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/data_offer/DataOfferPageApiService.java @@ -69,9 +69,7 @@ private boolean isIdAvailable(DSLContext dsl, Table table, TableField createButDontPublish(dsl, dataOfferCreationRequest, commonId); case PUBLISH_UNRESTRICTED -> createAndPublishUnrestricted(dsl, dataOfferCreationRequest, commonId); diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/policy/PolicyDefinitionApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/policy/PolicyDefinitionApiService.java index 1837bfbb9..749bb442c 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/policy/PolicyDefinitionApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/policy/PolicyDefinitionApiService.java @@ -23,13 +23,12 @@ import de.sovity.edc.ext.wrapper.api.ui.model.PolicyDefinitionCreateDto; import de.sovity.edc.ext.wrapper.api.ui.model.PolicyDefinitionCreateRequest; import de.sovity.edc.ext.wrapper.api.ui.model.PolicyDefinitionDto; -import de.sovity.edc.ext.wrapper.api.ui.pages.dashboard.services.SelfDescriptionService; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import jakarta.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; import org.eclipse.edc.connector.controlplane.policy.spi.PolicyDefinition; import org.eclipse.edc.connector.controlplane.services.spi.policydefinition.PolicyDefinitionService; import org.eclipse.edc.spi.query.QuerySpec; -import org.eclipse.edc.spi.system.configuration.Config; import java.util.Comparator; import java.util.List; @@ -73,7 +72,10 @@ public IdResponseDto deletePolicyDefinition(String policyDefinitionId) { } private List getAllPolicyDefinitions() { - return policyDefinitionService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, limit) -> + policyDefinitionService.search(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()) + .orElseThrow(ServiceException::new) + ); } public PolicyDefinitionDto buildPolicyDefinitionDto(PolicyDefinition policyDefinition) { diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/transferhistory/TransferHistoryPageApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/transferhistory/TransferHistoryPageApiService.java index 50c0760d5..30c386184 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/transferhistory/TransferHistoryPageApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/ui/pages/transferhistory/TransferHistoryPageApiService.java @@ -17,6 +17,7 @@ import de.sovity.edc.ext.wrapper.api.ServiceException; import de.sovity.edc.ext.wrapper.api.ui.model.ContractAgreementDirection; import de.sovity.edc.ext.wrapper.api.ui.model.TransferHistoryEntry; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import de.sovity.edc.utils.jsonld.vocab.Prop; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -61,20 +62,20 @@ public class TransferHistoryPageApiService { public List getTransferHistoryEntries() { var negotiationsById = getAllContractNegotiations().stream() - .filter(Objects::nonNull) - .filter(negotiation -> negotiation.getContractAgreement() != null) - .collect(toMap( - it -> it.getContractAgreement().getId(), - Function.identity(), - BinaryOperator.maxBy(Comparator.comparing(Entity::getCreatedAt)) - )); + .filter(Objects::nonNull) + .filter(negotiation -> negotiation.getContractAgreement() != null) + .collect(toMap( + it -> it.getContractAgreement().getId(), + Function.identity(), + BinaryOperator.maxBy(Comparator.comparing(Entity::getCreatedAt)) + )); var agreementsById = getAllContractAgreements().stream().collect(toMap( - ContractAgreement::getId, Function.identity() + ContractAgreement::getId, Function.identity() )); var assetsById = getAllAssets().stream() - .collect(toMap(Asset::getId, Function.identity())); + .collect(toMap(Asset::getId, Function.identity())); var transferProcesses = getAllTransferProcesses(); @@ -91,9 +92,9 @@ public List getTransferHistoryEntries() { transferHistoryEntry.setAssetName(asset.getId()); } else { transferHistoryEntry.setAssetName( - StringUtils.isBlank((String) asset.getProperties().get(Prop.Dcterms.TITLE)) - ? asset.getId() - : asset.getProperties().get(Prop.Dcterms.TITLE).toString() + StringUtils.isBlank((String) asset.getProperties().get(Prop.Dcterms.TITLE)) + ? asset.getId() + : asset.getProperties().get(Prop.Dcterms.TITLE).toString() ); } } @@ -125,21 +126,38 @@ private Asset assetLookup(Map assetsById, TransferProcess process @NotNull private List getAllContractNegotiations() { - return contractNegotiationStore.queryNegotiations(QuerySpec.max()).toList(); + return QueryUtils.fetchAllInBatches((offset, limit) -> + contractNegotiationStore.queryNegotiations( + QuerySpec.Builder.newInstance() + .offset(offset) + .limit(limit) + .build() + ).toList() + ); } @NotNull private List getAllContractAgreements() { - return contractAgreementService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, limit) -> + contractAgreementService.search(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()) + .orElseThrow(ServiceException::new) + ); } @NotNull private List getAllTransferProcesses() { - return transferProcessService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, limit) -> + transferProcessService.search( + QuerySpec.Builder.newInstance().offset(offset).limit(limit).build() + ).orElseThrow(ServiceException::new) + ); } @NotNull private List getAllAssets() { - return assetService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList(); + return QueryUtils.fetchAllInBatches((offset, limit) -> + assetService.search(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()) + .orElseThrow(ServiceException::new) + ); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResourceImpl.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResourceImpl.java index 7f1bf7d74..887d56d23 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResourceImpl.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/UseCaseResourceImpl.java @@ -20,6 +20,7 @@ import de.sovity.edc.ext.wrapper.api.usecase.pages.catalog.UseCaseCatalogApiService; import de.sovity.edc.ext.wrapper.api.usecase.services.KpiApiService; import de.sovity.edc.ext.wrapper.api.usecase.services.SupportedPolicyApiService; +import de.sovity.edc.extension.db.directaccess.DslContextFactory; import lombok.RequiredArgsConstructor; import java.util.List; @@ -33,10 +34,11 @@ public class UseCaseResourceImpl implements UseCaseResource { private final KpiApiService kpiApiService; private final SupportedPolicyApiService supportedPolicyApiService; private final UseCaseCatalogApiService useCaseCatalogApiService; + private final DslContextFactory dslContextFactory; @Override public KpiResult getKpis() { - return kpiApiService.getKpis(); + return dslContextFactory.transactionResult(kpiApiService::getKpis); } @Override diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/services/KpiApiService.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/services/KpiApiService.java index 16fea5e6b..0c351078d 100644 --- a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/services/KpiApiService.java +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/api/usecase/services/KpiApiService.java @@ -14,11 +14,12 @@ package de.sovity.edc.ext.wrapper.api.usecase.services; -import de.sovity.edc.ext.wrapper.api.ServiceException; +import de.sovity.edc.ext.db.jooq.Tables; import de.sovity.edc.ext.wrapper.api.ui.model.TransferProcessSimplifiedState; import de.sovity.edc.ext.wrapper.api.ui.pages.transferhistory.TransferProcessStateService; import de.sovity.edc.ext.wrapper.api.usecase.model.KpiResult; import de.sovity.edc.ext.wrapper.api.usecase.model.TransferProcessStatesDto; +import de.sovity.edc.ext.wrapper.utils.QueryUtils; import lombok.RequiredArgsConstructor; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; import org.eclipse.edc.connector.controlplane.contract.spi.offer.store.ContractDefinitionStore; @@ -27,6 +28,7 @@ import org.eclipse.edc.connector.controlplane.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess; import org.eclipse.edc.spi.query.QuerySpec; +import org.jooq.DSLContext; import java.util.List; import java.util.Map; @@ -43,41 +45,45 @@ public class KpiApiService { private final ContractAgreementService contractAgreementService; private final TransferProcessStateService transferProcessStateService; - public KpiResult getKpis() { + public KpiResult getKpis(DSLContext dsl) { var assetsCount = getAssetsCount(); var policiesCount = getPoliciesCount(); var contractDefinitionsCount = getContractDefinitionsCount(); - var contractAgreements = getContractAgreementsCount(); + var contractAgreements = getContractAgreementsCount(dsl); var transferProcessDto = getTransferProcessesDto(); return new KpiResult( - assetsCount, - policiesCount, - contractDefinitionsCount, - contractAgreements, - transferProcessDto + assetsCount, + policiesCount, + contractDefinitionsCount, + contractAgreements, + transferProcessDto ); } - private int getContractAgreementsCount() { - return contractAgreementService.query(QuerySpec.max()).orElseThrow(ServiceException::new).toList().size(); + private int getContractAgreementsCount(DSLContext dsl) { + return dsl.selectCount() + .from(Tables.EDC_CONTRACT_AGREEMENT) + .fetchSingleInto(int.class); } private TransferProcessStatesDto getTransferProcessesDto() { - var transferProcesses = transferProcessStore.findAll(QuerySpec.max()).toList(); + var transferProcesses = QueryUtils.fetchAllInBatches((offset, limit) -> + transferProcessStore.findAll(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()).toList() + ); return new TransferProcessStatesDto(getIncoming(transferProcesses), getOutgoing(transferProcesses)); } private Map getIncoming(List transferProcesses) { return transferProcesses.stream() - .filter(it -> it.getType() == TransferProcess.Type.CONSUMER) - .collect(groupingBy(this::getTransferProcessStates, counting())); + .filter(it -> it.getType() == TransferProcess.Type.CONSUMER) + .collect(groupingBy(this::getTransferProcessStates, counting())); } private Map getOutgoing(List transferProcesses) { return transferProcesses.stream() - .filter(it -> it.getType() == TransferProcess.Type.PROVIDER) - .collect(groupingBy(this::getTransferProcessStates, counting())); + .filter(it -> it.getType() == TransferProcess.Type.PROVIDER) + .collect(groupingBy(this::getTransferProcessStates, counting())); } private TransferProcessSimplifiedState getTransferProcessStates(TransferProcess transferProcess) { @@ -85,17 +91,23 @@ private TransferProcessSimplifiedState getTransferProcessStates(TransferProcess } private int getContractDefinitionsCount() { - var contractDefinitions = contractDefinitionStore.findAll(QuerySpec.max()).toList(); + var contractDefinitions = QueryUtils.fetchAllInBatches((offset, limit) -> + contractDefinitionStore.findAll(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()).toList() + ); return contractDefinitions.size(); } private int getPoliciesCount() { - var policies = policyDefinitionStore.findAll(QuerySpec.max()).toList(); + var policies = QueryUtils.fetchAllInBatches((offset, limit) -> + policyDefinitionStore.findAll(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()).toList() + ); return policies.size(); } private int getAssetsCount() { - var assets = assetIndex.queryAssets(QuerySpec.max()).toList(); + var assets = QueryUtils.fetchAllInBatches((offset, limit) -> + assetIndex.queryAssets(QuerySpec.Builder.newInstance().offset(offset).limit(limit).build()).toList() + ); return assets.size(); } } diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/utils/FetchBatch.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/utils/FetchBatch.java new file mode 100644 index 000000000..d4785a2ca --- /dev/null +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/utils/FetchBatch.java @@ -0,0 +1,7 @@ +package de.sovity.edc.ext.wrapper.utils; + +import java.util.List; + +public interface FetchBatch { + List fetchBatch(int offset, int limit); +} diff --git a/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/utils/QueryUtils.java b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/utils/QueryUtils.java new file mode 100644 index 000000000..a7199dbbc --- /dev/null +++ b/extensions/wrapper/wrapper/src/main/java/de/sovity/edc/ext/wrapper/utils/QueryUtils.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.wrapper.utils; + +import lombok.experimental.UtilityClass; +import lombok.val; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +@UtilityClass +public class QueryUtils { + public static final int BATCH_SIZE = 4000; + + public @NotNull ArrayList fetchAllInBatches(FetchBatch fetcher) { + var position = 0; + val all = new ArrayList(); + + List batch; + + do { + batch = fetcher.fetchBatch(position, BATCH_SIZE); + all.addAll(batch); + position += BATCH_SIZE; + } while (!batch.isEmpty()); + + System.out.println("ALL " + all.size()); + + return all; + } +} diff --git a/extensions/wrapper/wrapper/src/test/java/de/sovity/edc/ext/wrapper/Benchmark.java b/extensions/wrapper/wrapper/src/test/java/de/sovity/edc/ext/wrapper/Benchmark.java new file mode 100644 index 000000000..987107db9 --- /dev/null +++ b/extensions/wrapper/wrapper/src/test/java/de/sovity/edc/ext/wrapper/Benchmark.java @@ -0,0 +1,201 @@ +package de.sovity.edc.ext.wrapper; + +import de.sovity.edc.client.EdcClient; +import de.sovity.edc.client.gen.model.ContractDefinitionRequest; +import de.sovity.edc.client.gen.model.ContractNegotiationRequest; +import de.sovity.edc.client.gen.model.ContractNegotiationSimplifiedState; +import de.sovity.edc.client.gen.model.DataSourceType; +import de.sovity.edc.client.gen.model.InitiateTransferRequest; +import de.sovity.edc.client.gen.model.OperatorDto; +import de.sovity.edc.client.gen.model.PolicyDefinitionCreateDto; +import de.sovity.edc.client.gen.model.UiAssetCreateRequest; +import de.sovity.edc.client.gen.model.UiContractNegotiation; +import de.sovity.edc.client.gen.model.UiCriterion; +import de.sovity.edc.client.gen.model.UiCriterionLiteral; +import de.sovity.edc.client.gen.model.UiCriterionLiteralType; +import de.sovity.edc.client.gen.model.UiCriterionOperator; +import de.sovity.edc.client.gen.model.UiDataSource; +import de.sovity.edc.client.gen.model.UiDataSourceHttpData; +import de.sovity.edc.client.gen.model.UiPolicyConstraint; +import de.sovity.edc.client.gen.model.UiPolicyExpression; +import de.sovity.edc.client.gen.model.UiPolicyExpressionType; +import de.sovity.edc.client.gen.model.UiPolicyLiteral; +import de.sovity.edc.client.gen.model.UiPolicyLiteralType; +import de.sovity.edc.utils.jsonld.vocab.Prop; +import lombok.val; +import org.awaitility.Awaitility; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; + +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofSeconds; + +public class Benchmark { + + private static final String PROVIDER_DSP = "https://load-test-3-provider.stage-sovity.azure.sovity.io/control/api/v1/dsp"; + private static final String PROVIDER_ID = "BPNL00000000024R"; + + public static void main(String[] args) { + var provider = EdcClient.builder() + .managementApiUrl("https://load-test-3-provider.stage-sovity.azure.sovity.io/control/data") + .managementApiKey("GcR0t0BFx0kSHh/8B43rUDyPCFRMoOko+CiuhuMLMrs=") + .build(); + + provider.testConnection(); + + var consumer = EdcClient.builder() + .managementApiUrl("https://load-test-3-consumer.stage-sovity.azure.sovity.io/control/data") + .managementApiKey("77mBZTNriBPAb0WwIJn95n++O/GkVL8fNovLFPa/GlE=") + .build(); + + provider.testConnection(); + + val start = 5000; + val stop = 5050; + + for (int i = start; i < stop; ++i) { + create(provider, consumer, i); + } + } + + private static void create( + EdcClient provider, + EdcClient consumer, + int index + ) { + System.out.println("Index=" + index); + createAsset(provider, index); + createPolicy(provider, index); + createContractDefinition(provider, index); + + if (false) { + val negotiation = negotiate(provider, consumer, index); + + consumer.uiApi().initiateTransfer( + InitiateTransferRequest.builder() + .contractAgreementId(negotiation.getContractAgreementId()) + .transferType("HttpData-PUSH") + .dataSinkProperties(Map.of( + Prop.Edc.TYPE, "HttpData", + Prop.Edc.BASE_URL, "https://example.com", + Prop.Edc.METHOD, "POST" + )) + .build() + ); + } + } + + private static UiContractNegotiation negotiate(EdcClient provider, EdcClient consumer, int index) { + System.out.println("Negotiating " + index + "..."); + val offers = provider.uiApi().getCatalogPageDataOffers(PROVIDER_ID, PROVIDER_DSP); + + val firstContractOffer = offers.stream() + .filter(offer -> offer.getAsset().getAssetId().equals("asset-" + index)) + .findFirst() + .get() + .getContractOffers() + .get(0); + + val negotiation = consumer.uiApi().initiateContractNegotiation( + ContractNegotiationRequest.builder() + .counterPartyId(PROVIDER_ID) + .counterPartyAddress(PROVIDER_DSP) + .assetId("asset-" + index) + .contractOfferId(firstContractOffer.getContractOfferId()) + .policyJsonLd(firstContractOffer.getPolicy().getPolicyJsonLd()) + .build() + ); + + Awaitility.await().atMost(ofSeconds(100)).pollDelay(ofMillis(1000)).until( + () -> { + System.out.println("Awaiting " + index + "..."); + return consumer + .uiApi() + .getContractNegotiation(negotiation.getContractNegotiationId()) + .getState() + .getSimplifiedState() != ContractNegotiationSimplifiedState.IN_PROGRESS; + } + ); + + System.out.println("Negotiated " + index); + return negotiation; + } + + private static void createContractDefinition(EdcClient client, int index) { + System.out.println("Create contract definition " + index); + client.uiApi().createContractDefinition( + ContractDefinitionRequest.builder() + .contractDefinitionId("contract-definition-" + index) + .accessPolicyId("policy-" + index) + .contractPolicyId("policy-" + index) + .assetSelector(List.of(UiCriterion.builder() + .operandLeft(Prop.Edc.ID) + .operator(UiCriterionOperator.EQ) + .operandRight(UiCriterionLiteral.builder() + .type(UiCriterionLiteralType.VALUE) + .value("asset-" + index) + .build()) + .build())) + .build() + ); + } + + private static void createPolicy(EdcClient client, int index) { + System.out.println("Create policies " + index); + var afterYesterday = UiPolicyExpression.builder() + .type(UiPolicyExpressionType.CONSTRAINT) + .constraint(UiPolicyConstraint.builder() + .left("POLICY_EVALUATION_TIME") + .operator(OperatorDto.GT) + .right(UiPolicyLiteral.builder() + .type(UiPolicyLiteralType.STRING) + .value(OffsetDateTime.now().minusDays(1).toString()) + .build()) + .build()) + .build(); + + var beforeTomorrow = UiPolicyExpression.builder() + .type(UiPolicyExpressionType.CONSTRAINT) + .constraint(UiPolicyConstraint.builder() + .left("POLICY_EVALUATION_TIME") + .operator(OperatorDto.LT) + .right(UiPolicyLiteral.builder() + .type(UiPolicyLiteralType.STRING) + .value(OffsetDateTime.now().plusDays(1).toString()) + .build()) + .build()) + .build(); + + var expression = UiPolicyExpression.builder() + .type(UiPolicyExpressionType.AND) + .expressions(List.of(afterYesterday, beforeTomorrow)) + .build(); + + client.uiApi().createPolicyDefinitionV2( + PolicyDefinitionCreateDto.builder() + .policyDefinitionId("policy-" + index) + .expression(expression) + .build() + ); + } + + private static void createAsset(EdcClient client, int index) { + System.out.println("Create asset " + index); + client.uiApi().createAsset( + UiAssetCreateRequest.builder() + .title("Test Asset " + index) + .id("asset-" + index) + .dataSource( + UiDataSource.builder() + .type(DataSourceType.HTTP_DATA) + .httpData(UiDataSourceHttpData.builder() + .baseUrl("http://example.com") + .build()) + .build() + ) + .build() + ); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 9b49145a8..1a8ebb564 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -39,3 +39,5 @@ include(":utils:jooq-database-access") include(":utils:json-and-jsonld-utils") include(":utils:test-utils") include(":utils:versions") +include("extensions:wrapper:clients:benchmark") +findProject(":extensions:wrapper:clients:benchmark")?.name = "benchmark" diff --git a/tests/src/test/java/de/sovity/edc/e2e/UiApiWrapperTest.java b/tests/src/test/java/de/sovity/edc/e2e/UiApiWrapperTest.java index 9c247fbb5..ef50cecd3 100644 --- a/tests/src/test/java/de/sovity/edc/e2e/UiApiWrapperTest.java +++ b/tests/src/test/java/de/sovity/edc/e2e/UiApiWrapperTest.java @@ -16,6 +16,7 @@ import de.sovity.edc.client.EdcClient; import de.sovity.edc.client.gen.ApiException; +import de.sovity.edc.client.gen.model.ContractAgreementPageQuery; import de.sovity.edc.client.gen.model.ContractDefinitionEntry; import de.sovity.edc.client.gen.model.ContractDefinitionRequest; import de.sovity.edc.client.gen.model.ContractNegotiationRequest; @@ -48,6 +49,7 @@ import de.sovity.edc.client.gen.model.UiPolicyExpressionType; import de.sovity.edc.client.gen.model.UiPolicyLiteral; import de.sovity.edc.client.gen.model.UiPolicyLiteralType; +import de.sovity.edc.ext.db.jooq.Tables; import de.sovity.edc.extension.e2e.connector.remotes.api_wrapper.E2eTestScenario; import de.sovity.edc.extension.e2e.connector.remotes.management_api.ManagementApiConnectorRemote; import de.sovity.edc.extension.e2e.connector.remotes.test_backend_controller.TestBackendRemote; @@ -66,17 +68,24 @@ import org.eclipse.edc.protocol.dsp.http.spi.types.HttpMessageProtocol; import org.eclipse.edc.spi.system.configuration.Config; import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.extension.RegisterExtension; import java.time.LocalDate; import java.time.OffsetDateTime; import java.util.Arrays; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; import static de.sovity.edc.client.gen.model.ContractAgreementDirection.CONSUMING; import static de.sovity.edc.client.gen.model.ContractAgreementDirection.PROVIDING; @@ -86,8 +95,11 @@ import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.jooq.JSON.json; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) class UiApiWrapperTest { private static final String PROVIDER_PARTICIPANT_ID = CeE2eTestSide.PROVIDER.getParticipantId(); @@ -987,6 +999,7 @@ void dontCreateAnythingIfTheContractDefinitionAlreadyExists( ApiException.class, () -> providerClient.uiApi() .createDataOffer(DataOfferCreationRequest.builder() + .policy(DataOfferCreationRequest.PolicyEnum.PUBLISH_RESTRICTED) .uiAssetCreateRequest(UiAssetCreateRequest.builder() .id(id) .dataSource(UiDataSource.builder() @@ -1105,6 +1118,154 @@ void onlyCreateTheAssetWhenDontPublish( .hasSize(0); } + @Order(Order.DEFAULT + 100) + @Test + void canLoadTheDashboardWithLargeNumberOfAssets( + E2eTestScenario scenario, + @Provider DSLContext dsl, + @Provider EdcClient providerClient, + @Consumer Config config + ) { + // arrange + int target = 10000; + + int firstPort = config.getInteger("my.edc.first.port"); + int consumerDspPort = firstPort + 3; + + IntStream.range(999, 999 + target).forEach((i) -> { + // insert new asset with jooq + String assetId = "asset-" + i; + insertAsset(dsl, assetId); + val contractAgreementId = insertContractAgreement(dsl, assetId); + val contractDefinition = insertContractDefinition(dsl, assetId); + val neg = insertContractNegotiation(dsl, consumerDspPort, assetId, contractDefinition, contractAgreementId); + }); + + // act + assertDoesNotThrow(() -> providerClient.uiApi().getDashboardPage()); + assertDoesNotThrow(() -> providerClient.uiApi().getContractAgreementPage(new ContractAgreementPageQuery())); + } + + private String insertContractNegotiation( + DSLContext dsl, + int consumerDspPort, + String assetId, + String contractDefinition, + String contractAgreementId + ) { + val n = Tables.EDC_CONTRACT_NEGOTIATION; + val contractNegotiationId = UUID.randomUUID().toString(); + dsl.insertInto(n) + .set(n.ID, contractNegotiationId) + .set(n.CREATED_AT, System.currentTimeMillis()) + .set(n.UPDATED_AT, System.currentTimeMillis()) + .set(n.CORRELATION_ID, UUID.randomUUID().toString()) + .set(n.COUNTERPARTY_ID, "consumer") + .set(n.COUNTERPARTY_ADDRESS, "http://localhost:" + consumerDspPort + "/api/dsp") + .set(n.PROTOCOL, "dataspace-protocol-http") + .set(n.TYPE, "PROVIDER") + .set(n.STATE, 1200) + .set(n.STATE_COUNT, 1) + .set(n.STATE_TIMESTAMP, System.currentTimeMillis()) + .set(n.ERROR_DETAIL, (String) null) + .set(n.AGREEMENT_ID, contractAgreementId) + .set(n.CONTRACT_OFFERS, json( + "[{\"id\":\"" + base64(contractDefinition) + ":" + base64(assetId) + ":" + base64(assetId) + + "\",\"policy\":{\"permissions\":[{\"edctype\":\"dataspaceconnector:permission\",\"action\":" + + "{\"type\":\"USE\",\"includedIn\":null,\"constraint\":null},\"constraints\":[],\"duties\":[]}]," + + "\"prohibitions\":[],\"obligations\":[],\"extensibleProperties\":{},\"inheritsFrom\":null," + + "\"assigner\":null,\"assignee\":null,\"target\":\"" + assetId + "\",\"@type\":" + + "{\"@policytype\":\"set\"}},\"assetId\":\"" + assetId + "\"}]")) + .set(n.CALLBACK_ADDRESSES, json("[]")) + .set(n.TRACE_CONTEXT, json("{}")) + .set(n.PENDING, false) + .set(n.PROTOCOL_MESSAGES, json("{\"lastSent\":null,\"received\":[]}")) + .set(n.LEASE_ID, (String) null) + .execute(); + + return contractNegotiationId; + } + + private static String base64(String contractDefinition) { + return Base64.getEncoder().encodeToString(contractDefinition.getBytes()); + } + + private String insertContractDefinition(DSLContext dsl, String assetId) { + val cd = Tables.EDC_CONTRACT_DEFINITIONS; + val contractDefinitionId = "cd-always-true-" + assetId; + dsl.insertInto(cd) + .set(cd.CREATED_AT, System.currentTimeMillis()) + .set(cd.CONTRACT_DEFINITION_ID, contractDefinitionId) + .set(cd.ACCESS_POLICY_ID, "always-true") + .set(cd.CONTRACT_POLICY_ID, "always-true") + .set(cd.ASSETS_SELECTOR, json(""" + [{"operandLeft":"https://w3id.org/edc/v0.0.1/ns/id","operator":"=","operandRight":\"""" + assetId + """ + "}]""")) + .set(cd.PRIVATE_PROPERTIES, json("{}")) + .execute(); + + return contractDefinitionId; + } + + private String insertContractAgreement(DSLContext dsl, String assetId) { + + val a = Tables.EDC_CONTRACT_AGREEMENT; + val contractAgreementId = UUID.randomUUID().toString(); + dsl.insertInto(a) + .set(a.AGR_ID, contractAgreementId) + .set(a.PROVIDER_AGENT_ID, PROVIDER_PARTICIPANT_ID) + .set(a.CONSUMER_AGENT_ID, CONSUMER_PARTICIPANT_ID) + .set(a.SIGNING_DATE, System.currentTimeMillis()) + .set(a.ASSET_ID, assetId) + .set(a.POLICY, json(""" + { + "permissions": [ + { + "edctype": "dataspaceconnector:permission", + "action": { + "type": "USE", + "includedIn": null, + "constraint": null + }, + "constraints": [], + "duties": [] + } + ], + "prohibitions": [], + "obligations": [], + "extensibleProperties": {}, + "inheritsFrom": null, + "assigner": null, + "assignee": null, + "target": \"""" + assetId + """ + ", + "@type": { + "@policytype": "contract" + } + } + """)) + .execute(); + + return contractAgreementId; + } + + private static void insertAsset(DSLContext dsl, String assetId) { + val a = Tables.EDC_ASSET; + dsl.insertInto(a) + .set(a.ASSET_ID, assetId) + .set(a.CREATED_AT, System.currentTimeMillis()) + .set(a.PROPERTIES, json(""" + {"https://semantic.sovity.io/dcat-ext#httpDatasourceHintsProxyBody":"false","http://www.w3.org/ns/dcat#version":"1.0.0","http://purl.org/dc/terms/creator":{"http://xmlns.com/foaf/0.1/name":[{"@value":"Curator Name provider"}]},"https://semantic.sovity.io/dcat-ext#httpDatasourceHintsProxyPath":"false","http://purl.org/dc/terms/title":"AssetName_""" + assetId + """ + ","http://purl.org/dc/terms/language":"en","https://semantic.sovity.io/dcat-ext#httpDatasourceHintsProxyMethod":"false","https://w3id.org/edc/v0.0.1/ns/id":\"""" + assetId + """ + ","https://semantic.sovity.io/dcat-ext#httpDatasourceHintsProxyQueryParams":"false"} + """)) + .set(a.PRIVATE_PROPERTIES, json("{}")) + .set(a.DATA_ADDRESS, json(""" + {"https://w3id.org/edc/v0.0.1/ns/type":"HttpData","https://w3id.org/edc/v0.0.1/ns/baseUrl":"http://example.com"} + """)) + .execute(); + } + private static @NotNull List getPolicyNamed(String name, EdcClient edcClient) { return edcClient.uiApi() .getPolicyDefinitionPage() diff --git a/tests/src/test/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiServiceTest.java b/tests/src/test/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiServiceTest.java index 389f91c03..8a4a16994 100644 --- a/tests/src/test/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiServiceTest.java +++ b/tests/src/test/java/de/sovity/edc/ext/wrapper/api/ui/pages/dashboard/DashboardPageApiServiceTest.java @@ -15,9 +15,21 @@ package de.sovity.edc.ext.wrapper.api.ui.pages.dashboard; import de.sovity.edc.client.EdcClient; +import de.sovity.edc.client.gen.model.ContractDefinitionRequest; +import de.sovity.edc.client.gen.model.DataSourceType; +import de.sovity.edc.client.gen.model.PolicyDefinitionCreateDto; +import de.sovity.edc.client.gen.model.UiAssetCreateRequest; +import de.sovity.edc.client.gen.model.UiCriterion; +import de.sovity.edc.client.gen.model.UiCriterionLiteral; +import de.sovity.edc.client.gen.model.UiCriterionLiteralType; +import de.sovity.edc.client.gen.model.UiCriterionOperator; +import de.sovity.edc.client.gen.model.UiDataSource; +import de.sovity.edc.client.gen.model.UiDataSourceHttpData; +import de.sovity.edc.client.gen.model.UiPolicyExpression; import de.sovity.edc.extension.e2e.junit.CeIntegrationTestExtension; import de.sovity.edc.utils.config.CeConfigProps; import de.sovity.edc.utils.config.ConfigUtils; +import de.sovity.edc.utils.jsonld.vocab.Prop; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; import org.eclipse.edc.connector.controlplane.contract.spi.negotiation.store.ContractNegotiationStore; @@ -33,6 +45,7 @@ import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.spi.result.ServiceResult; import org.eclipse.edc.spi.system.configuration.Config; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mockito; @@ -40,9 +53,11 @@ import java.util.Collection; import java.util.List; import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.IntStream; +import static de.sovity.edc.ext.wrapper.utils.QueryUtils.BATCH_SIZE; import static org.assertj.core.api.Assertions.assertThat; import static org.eclipse.edc.connector.controlplane.contract.spi.types.negotiation.ContractNegotiation.Type.CONSUMER; import static org.eclipse.edc.connector.controlplane.contract.spi.types.negotiation.ContractNegotiation.Type.PROVIDER; @@ -72,23 +87,83 @@ class DashboardPageApiServiceTest { .property("edc.iam.sts.dim.url", "dimUrl") ) .beforeEdcStartup(runtime -> { - runtime.registerServiceMock(AssetIndex.class, assetIndex); - runtime.registerServiceMock(PolicyDefinitionService.class, policyDefinitionService); +// runtime.registerServiceMock(AssetIndex.class, assetIndex); +// runtime.registerServiceMock(PolicyDefinitionService.class, policyDefinitionService); runtime.registerServiceMock(TransferProcessService.class, transferProcessService); runtime.registerServiceMock(ContractNegotiationStore.class, contractNegotiationStore); - runtime.registerServiceMock(ContractDefinitionService.class, contractDefinitionService); +// runtime.registerServiceMock(ContractDefinitionService.class, contractDefinitionService); }) .build(); private final Random random = new Random(); + @Disabled("Stopped working because now using jooQ to count from the DB directly") @Test void testKpis(EdcClient client) { + + var id = new AtomicInteger(0); + + // create assets + repeat(9, () -> { + client.uiApi().createAsset( + UiAssetCreateRequest.builder() + .id("dataOffer-" + (id.incrementAndGet())) + .dataSource( + UiDataSource.builder() + .type(DataSourceType.HTTP_DATA) + .httpData(UiDataSourceHttpData.builder() + .baseUrl("http://localhost") + .build()) + .build() + ) + .build() + ); + return null; + }); + + // create policies + id.set(0); + repeat(8 - 1 /*always-true*/, () -> { + client.uiApi().createPolicyDefinitionV2( + PolicyDefinitionCreateDto.builder() + .policyDefinitionId("policy-" + id.incrementAndGet()) + .expression(UiPolicyExpression.builder().build()) + .build() + ); + return null; + }); + + // create contract definitions + id.set(0); + repeat(7, () -> { + id.incrementAndGet(); + client.uiApi().createContractDefinition( + ContractDefinitionRequest.builder() + .contractDefinitionId("cd-" + id.get()) + .accessPolicyId("policy-" + id.get()) + .contractPolicyId("policy-" + id.get()) + .assetSelector(List.of(UiCriterion.builder() + .operandLeft(Prop.Edc.ID) + .operator(UiCriterionOperator.EQ) + .operandRight(UiCriterionLiteral.builder() + .type(UiCriterionLiteralType.VALUE) + .value("asset-" + id.get()) + .build()) + .build())) + .build() + ); + return null; + }); + + // TODO create contract negotiation + // TODO create transfer processes + // arrange mockAmounts( - repeat(7, Mockito::mock), - repeat(8, Mockito::mock), - repeat(9, Mockito::mock), + // 0 because it's replaced by DB data + repeat(0, Mockito::mock), + repeat(0, Mockito::mock), + repeat(0, Mockito::mock), List.of( mockContractNegotiation(1, CONSUMER), mockContractNegotiation(2, PROVIDER), @@ -108,11 +183,11 @@ void testKpis(EdcClient client) { // act var dashboardPage = client.uiApi().getDashboardPage(); - assertThat(dashboardPage.getNumAssets()).isEqualTo(7); + assertThat(dashboardPage.getNumAssets()).isEqualTo(9); assertThat(dashboardPage.getNumPolicies()).isEqualTo(8); - assertThat(dashboardPage.getNumContractDefinitions()).isEqualTo(9); - assertThat(dashboardPage.getNumContractAgreementsConsuming()).isEqualTo(1); - assertThat(dashboardPage.getNumContractAgreementsProviding()).isEqualTo(2); + assertThat(dashboardPage.getNumContractDefinitions()).isEqualTo(7); +// assertThat(dashboardPage.getNumContractAgreementsConsuming()).isEqualTo(1); +// assertThat(dashboardPage.getNumContractAgreementsProviding()).isEqualTo(2); assertThat(dashboardPage.getTransferProcessesConsuming().getNumTotal()).isEqualTo(1 + 2 + 3); assertThat(dashboardPage.getTransferProcessesConsuming().getNumRunning()).isEqualTo(1); assertThat(dashboardPage.getTransferProcessesConsuming().getNumError()).isEqualTo(2); @@ -187,8 +262,12 @@ private void mockAmounts( List transferProcesses ) { when(assetIndex.queryAssets(eq(QuerySpec.max()))).thenAnswer(i -> assets.stream()); - when(transferProcessService.query(eq(QuerySpec.max()))) - .thenAnswer(i -> ServiceResult.success(transferProcesses.stream())); + + when(transferProcessService.search(eq(QuerySpec.Builder.newInstance().offset(0).limit(BATCH_SIZE).build()))) + .thenAnswer(i -> ServiceResult.success(transferProcesses)); + when(transferProcessService.search(eq(QuerySpec.Builder.newInstance().offset(BATCH_SIZE).limit(BATCH_SIZE).build()))) + .thenAnswer(i -> ServiceResult.success(List.of())); + when(policyDefinitionService.query(eq(QuerySpec.max()))) .thenAnswer(i -> ServiceResult.success(policyDefinitions.stream())); when(contractNegotiationStore.queryNegotiations(eq(QuerySpec.max()))) diff --git a/utils/catalog-parser/src/main/java/de/sovity/edc/utils/catalog/DspCatalogService.java b/utils/catalog-parser/src/main/java/de/sovity/edc/utils/catalog/DspCatalogService.java index 231f708f2..8484fa8f8 100644 --- a/utils/catalog-parser/src/main/java/de/sovity/edc/utils/catalog/DspCatalogService.java +++ b/utils/catalog-parser/src/main/java/de/sovity/edc/utils/catalog/DspCatalogService.java @@ -31,7 +31,8 @@ public class DspCatalogService { private final DspDataOfferBuilder dspDataOfferBuilder; public DspCatalog fetchDataOffers(String participantId, String endpoint) throws DspCatalogServiceException { - var catalogJson = fetchDcatResponse(participantId, endpoint, QuerySpec.max()); + var first4kItems = QuerySpec.Builder.newInstance().offset(0).limit(4000).build(); + var catalogJson = fetchDcatResponse(participantId, endpoint, first4kItems); return dspDataOfferBuilder.buildDataOffers(endpoint, catalogJson); } diff --git a/utils/catalog-parser/src/test/java/de/sovity/edc/utils/catalog/DspCatalogServiceTest.java b/utils/catalog-parser/src/test/java/de/sovity/edc/utils/catalog/DspCatalogServiceTest.java index 0b184e0dd..dfc25859e 100644 --- a/utils/catalog-parser/src/test/java/de/sovity/edc/utils/catalog/DspCatalogServiceTest.java +++ b/utils/catalog-parser/src/test/java/de/sovity/edc/utils/catalog/DspCatalogServiceTest.java @@ -22,7 +22,6 @@ import org.eclipse.edc.connector.controlplane.services.spi.catalog.CatalogService; import org.eclipse.edc.jsonld.TitaniumJsonLd; import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.spi.response.StatusResult; import org.junit.jupiter.api.Test; @@ -32,6 +31,7 @@ import static de.sovity.edc.utils.JsonUtils.toJson; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -49,7 +49,7 @@ private DspCatalogService newDspCatalogService(String resultJsonFilename) { eq(participantId), eq(endpoint), eq("dataspace-protocol-http"), - eq(QuerySpec.max())) + any()) ).thenReturn(result); var monitor = mock(Monitor.class);