From 58febf8c5ae2131caf08e76545d9eeac721fda96 Mon Sep 17 00:00:00 2001 From: OMPRAKASH MISHRA Date: Mon, 27 Jan 2025 18:53:21 -0800 Subject: [PATCH] chore: more logic for attachment processing, fix dockerfile --- backend-el/.gitignore | 2 +- backend-el/Dockerfile | 11 +- .../fta/el/services/ApplicationService.java | 593 +++++++++--------- .../fta/el/services/S3UploaderService.java | 31 +- .../src/main/resources/application.properties | 3 +- 5 files changed, 340 insertions(+), 300 deletions(-) diff --git a/backend-el/.gitignore b/backend-el/.gitignore index f150a06b..09468e84 100644 --- a/backend-el/.gitignore +++ b/backend-el/.gitignore @@ -31,4 +31,4 @@ build/ ### VS Code ### .vscode/ -backend-el/application-local.properties +src/main/resources/application-local.properties diff --git a/backend-el/Dockerfile b/backend-el/Dockerfile index beb47e92..88315368 100644 --- a/backend-el/Dockerfile +++ b/backend-el/Dockerfile @@ -1,18 +1,15 @@ # Stage 1: Build the application -FROM maven:3.8.4-eclipse-temurin-21 AS build +FROM maven:3-eclipse-temurin-21 AS build WORKDIR /app COPY pom.xml . RUN mvn dependency:go-offline COPY src ./src RUN mvn package -DskipTests - # Stage 2: Run the application FROM eclipse-temurin:21 WORKDIR /app +ARG DEPENDENCY=/app/target/ +COPY --from=build ${DEPENDENCY}/*.jar /app/app.jar RUN useradd -ms /bin/bash appuser USER appuser - -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 CMD curl -f http://localhost:8090/actuator/health || exit 1 -COPY --from=build /app/target/*.jar /app/app.jar -RUN java -Djarmode=layertools -jar app.jar extract -ENTRYPOINT ["java", "-cp", "BOOT-INF/classes:BOOT-INF/lib/*", "com.example.YourMainClass"] +ENTRYPOINT ["java","-Duser.name=REC-ORACLE-EL","-Xms512m","-Xmx512m" ,"-jar","app.jar"] diff --git a/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/ApplicationService.java b/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/ApplicationService.java index c80ae386..83ac4ec0 100644 --- a/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/ApplicationService.java +++ b/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/ApplicationService.java @@ -12,6 +12,7 @@ import java.io.FileWriter; import java.io.IOException; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -69,52 +70,52 @@ public class ApplicationService { private final RecreatnEventRepository recreatnEventRepository; public ApplicationService(S3UploaderService s3UploaderService, - RecreationAccessRepository recreationAccessRepository, - RecreationAccessCodeRepository recreationAccessCodeRepository, - RecreationAccessXrefRepository recreationAccessXrefRepository, - RecreationMapFeatureCodeRepository recreationMapFeatureCodeRepository, - RecreationMapFeatureGeomRepository recreationMapFeatureGeomRepository, - RecreationMapFeatureRepository recreationMapFeatureRepository, - RecreationMapFeatureXguidRepository recreationMapFeatureXguidRepository, - RecreationActivityCodeRepository recreationActivityCodeRepository, - RecreationActivityRepository recreationActivityRepository, - RecreationAgreementHolderRepository recreationAgreementHolderRepository, - RecreationAttachmentRepository recreationAttachmentRepository, - RecreationAttachmentContentRepository recreationAttachmentContentRepository, - RecreationCommentRepository recreationCommentRepository, - RecreationControlAccessCodeRepository recreationControlAccessCodeRepository, - RecreationDefCsRprHistoryRepository recreationDefCsRprHistoryRepository, - RecreationDefinedCampsiteRepository recreationDefinedCampsiteRepository, - RecreationDistrictCodeRepository recreationDistrictCodeRepository, - RecreationDistrictXrefRepository recreationDistrictXrefRepository, - RecreationFeatureCodeRepository recreationFeatureCodeRepository, - RecreationFeeCodeRepository recreationFeeCodeRepository, - RecreationFeeRepository recreationFeeRepository, - RecreationFileStatusCodeRepository recreationFileStatusCodeRepository, - RecreationFileTypeCodeRepository recreationFileTypeCodeRepository, - RecreationInspectionReportRepository recreationInspectionReportRepository, - RecreationMaintainStdCodeRepository recreationMaintainStdCodeRepository, - RecreationMapFeatureRepository recreationMapFeatureRepository1, - RecreationObjectiveRepository recreationObjectiveRepository, - RecreationOccupancyCodeRepository recreationOccupancyCodeRepository, - RecreationPlanRepository recreationPlanRepository, - RecreationProjectRepository recreationProjectRepository, - RecreationRemedRepairCodeRepository recreationRemedRepairCodeRepository, - RecreationRiskEvaluationRepository recreationRiskEvaluationRepository, - RecreationRiskRatingCodeRepository recreationRiskRatingCodeRepository, - RecreationRiskSiteRepository recreationRiskSiteRepository, - RecreationSearchResultRepository recreationSearchResultRepository, - RecreationSitePointRepository recreationSitePointRepository, - RecreationSiteRepository recreationSiteRepository, - RecreationStructDimenCodeRepository recreationStructDimenCodeRepository, - RecreationStructDimenXrefRepository recreationStructDimenXrefRepository, - RecreationStructureCodeRepository recreationStructureCodeRepository, - RecreationStructureRepository recreationStructureRepository, - RecreationStructureValueRepository recreationStructureValueRepository, - RecreationSubAccessCodeRepository recreationSubAccessCodeRepository, - RecreationTrailSegmentRepository recreationTrailSegmentRepository, - RecreationUserDaysCodeRepository recreationUserDaysCodeRepository, - RecreatnEventRepository recreatnEventRepository) { + RecreationAccessRepository recreationAccessRepository, + RecreationAccessCodeRepository recreationAccessCodeRepository, + RecreationAccessXrefRepository recreationAccessXrefRepository, + RecreationMapFeatureCodeRepository recreationMapFeatureCodeRepository, + RecreationMapFeatureGeomRepository recreationMapFeatureGeomRepository, + RecreationMapFeatureRepository recreationMapFeatureRepository, + RecreationMapFeatureXguidRepository recreationMapFeatureXguidRepository, + RecreationActivityCodeRepository recreationActivityCodeRepository, + RecreationActivityRepository recreationActivityRepository, + RecreationAgreementHolderRepository recreationAgreementHolderRepository, + RecreationAttachmentRepository recreationAttachmentRepository, + RecreationAttachmentContentRepository recreationAttachmentContentRepository, + RecreationCommentRepository recreationCommentRepository, + RecreationControlAccessCodeRepository recreationControlAccessCodeRepository, + RecreationDefCsRprHistoryRepository recreationDefCsRprHistoryRepository, + RecreationDefinedCampsiteRepository recreationDefinedCampsiteRepository, + RecreationDistrictCodeRepository recreationDistrictCodeRepository, + RecreationDistrictXrefRepository recreationDistrictXrefRepository, + RecreationFeatureCodeRepository recreationFeatureCodeRepository, + RecreationFeeCodeRepository recreationFeeCodeRepository, + RecreationFeeRepository recreationFeeRepository, + RecreationFileStatusCodeRepository recreationFileStatusCodeRepository, + RecreationFileTypeCodeRepository recreationFileTypeCodeRepository, + RecreationInspectionReportRepository recreationInspectionReportRepository, + RecreationMaintainStdCodeRepository recreationMaintainStdCodeRepository, + RecreationMapFeatureRepository recreationMapFeatureRepository1, + RecreationObjectiveRepository recreationObjectiveRepository, + RecreationOccupancyCodeRepository recreationOccupancyCodeRepository, + RecreationPlanRepository recreationPlanRepository, + RecreationProjectRepository recreationProjectRepository, + RecreationRemedRepairCodeRepository recreationRemedRepairCodeRepository, + RecreationRiskEvaluationRepository recreationRiskEvaluationRepository, + RecreationRiskRatingCodeRepository recreationRiskRatingCodeRepository, + RecreationRiskSiteRepository recreationRiskSiteRepository, + RecreationSearchResultRepository recreationSearchResultRepository, + RecreationSitePointRepository recreationSitePointRepository, + RecreationSiteRepository recreationSiteRepository, + RecreationStructDimenCodeRepository recreationStructDimenCodeRepository, + RecreationStructDimenXrefRepository recreationStructDimenXrefRepository, + RecreationStructureCodeRepository recreationStructureCodeRepository, + RecreationStructureRepository recreationStructureRepository, + RecreationStructureValueRepository recreationStructureValueRepository, + RecreationSubAccessCodeRepository recreationSubAccessCodeRepository, + RecreationTrailSegmentRepository recreationTrailSegmentRepository, + RecreationUserDaysCodeRepository recreationUserDaysCodeRepository, + RecreatnEventRepository recreatnEventRepository) { this.s3UploaderService = s3UploaderService; this.recreationAccessRepository = recreationAccessRepository; this.recreationAccessCodeRepository = recreationAccessCodeRepository; @@ -215,16 +216,16 @@ private void extractAndUploadRecreationMapFeature() { var results = this.recreationMapFeatureRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationMapFeature.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getSectionId(), - item.getAmendmentId(), - item.getRecreationMapFeatureCode(), item.getCurrentInd(), - item.getAmendStatusDate(), item.getRetirementDate(), - item.getRevisionCount(), item.getEntryUserid(), - item.getEntryTimestamp(), item.getUpdateUserid(), - item.getUpdateTimestamp(), item.getRecreationMapFeatureGuid()); + item.getAmendmentId(), + item.getRecreationMapFeatureCode(), item.getCurrentInd(), + item.getAmendStatusDate(), item.getRetirementDate(), + item.getRevisionCount(), item.getEntryUserid(), + item.getEntryTimestamp(), item.getUpdateUserid(), + item.getUpdateTimestamp(), item.getRecreationMapFeatureGuid()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -237,13 +238,13 @@ private void extractAndUploadRecreatnEvent() { var results = this.recreatnEventRepository.findAll(); var entityMetadata = getEntityMetadata(RecreatnEvent.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getOrgUnitNo(), item.getForestFileId(), - item.getRecProjectId(), item.getEventTypeCode(), item.getEntryTimestamp(), - item.getDistrictAdmnZone(), item.getEventDueDate(), item.getEventDate(), - item.getEventRemarks(), item.getWorkAssignmentCd(), item.getEnteredBy()); + item.getRecProjectId(), item.getEventTypeCode(), item.getEntryTimestamp(), + item.getDistrictAdmnZone(), item.getEventDueDate(), item.getEventDate(), + item.getEventRemarks(), item.getWorkAssignmentCd(), item.getEnteredBy()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -256,11 +257,11 @@ private void extractAndUploadRecreationUserDaysCode() { var results = this.recreationUserDaysCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationUserDaysCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationUserDaysCode(), item.getDescription(), - item.getEffectiveDate(), item.getExpiryDate(), item.getUpdateTimestamp()); + item.getEffectiveDate(), item.getExpiryDate(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -273,16 +274,16 @@ private void extractAndUploadRecreationTrailSegment() { var results = this.recreationTrailSegmentRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationTrailSegment.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRecreationTrailSegId(), - item.getTrailSegmentName(), item.getStartStation(), - item.getEndStation(), item.getRecreationRemedRepairCode(), - item.getEstimatedRepairCost(), item.getActualRepairCost(), - item.getRepairCompletedDate(), item.getWheelchairAccessibleInd(), - item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), - item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getTrailSegmentName(), item.getStartStation(), + item.getEndStation(), item.getRecreationRemedRepairCode(), + item.getEstimatedRepairCost(), item.getActualRepairCost(), + item.getRepairCompletedDate(), item.getWheelchairAccessibleInd(), + item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), + item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -295,11 +296,11 @@ private void extractAndUploadRecreationSubAccessCode() { var results = this.recreationSubAccessCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationSubAccessCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationSubAccessCode(), item.getDescription(), - item.getEffectiveDate(), item.getExpiryDate(), item.getUpdateTimestamp()); + item.getEffectiveDate(), item.getExpiryDate(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -312,14 +313,14 @@ private void extractAndUploadRecreationStructureValue() { var results = this.recreationStructureValueRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationStructureValue.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationStructureCode(), item.getStructureValue(), - item.getDimension(), - item.getRevisionCount(), item.getEntryUserid(), - item.getEntryTimestamp(), item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getDimension(), + item.getRevisionCount(), item.getEntryUserid(), + item.getEntryTimestamp(), item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -332,18 +333,18 @@ private void extractAndUploadRecreationStructure() { var results = this.recreationStructureRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationStructure.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getForestFile(), item.getCampSiteForestFileId(), - item.getRecreationStructureCode(), item.getStructureName(), - item.getStructureCount(), item.getStructureLength(), - item.getStructureWidth(), item.getStructureArea(), - item.getActualValue(), item.getCampSiteNumber(), item.getRecreationRemedRepairCode(), - item.getEstimatedRepairCost(), item.getRepairCompletedDate(), - item.getRevisionCount(), - item.getEntryUserid(), - item.getUpdateUserid(), item.getUpdateTimestamp(), item.getEntryTimestamp()); + item.getRecreationStructureCode(), item.getStructureName(), + item.getStructureCount(), item.getStructureLength(), + item.getStructureWidth(), item.getStructureArea(), + item.getActualValue(), item.getCampSiteNumber(), item.getRecreationRemedRepairCode(), + item.getEstimatedRepairCost(), item.getRepairCompletedDate(), + item.getRevisionCount(), + item.getEntryUserid(), + item.getUpdateUserid(), item.getUpdateTimestamp(), item.getEntryTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -356,12 +357,12 @@ private void extractAndUploadRecreationStructureCode() { var results = this.recreationStructureCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationStructureCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationStructureCode(), - item.getDescription(), item.getEffectiveDate(), item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), item.getEffectiveDate(), item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -374,13 +375,13 @@ private void extractAndUploadRecreationStructDimenXref() { var results = this.recreationStructDimenXrefRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationStructDimenXref.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationStructureCode(), - item.getRecreationStructDimenCode(), item.getRevisionCount(), - item.getEntryUserid(), item.getEntryTimestamp(), - item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getRecreationStructDimenCode(), item.getRevisionCount(), + item.getEntryUserid(), item.getEntryTimestamp(), + item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -393,12 +394,12 @@ private void extractAndUploadRecreationStructDimenCode() { var results = this.recreationStructDimenCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationStructDimenCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationStructDimenCode(), - item.getDescription(), item.getEffectiveDate(), item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), item.getEffectiveDate(), item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -411,8 +412,8 @@ private void extractAndUploadRecreationSite() { var results = this.recreationSiteRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationSite.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRecSiteName()); } @@ -427,13 +428,13 @@ private void extractAndUploadRecreationSitePoint() { var results = this.recreationSitePointRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationSitePoint.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRevisionCount(), - item.getEntryUserid(), item.getEntryTimestamp(), - item.getUpdateUserid(), item.getUpdateTimestamp(), - item.getGeometry()); + item.getEntryUserid(), item.getEntryTimestamp(), + item.getUpdateUserid(), item.getUpdateTimestamp(), + item.getGeometry()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -446,12 +447,12 @@ private void extractAndUploadRecreationSearchResult() { var results = this.recreationSearchResultRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationSearchResult.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getOrgUnitCode(), item.getOrgUnitName(), - item.getFileStatusCode(), item.getProjectName(), item.getProjectType(), - item.getRecreationProjectCode(), item.getRecreationProjectCodeDesc()); + item.getFileStatusCode(), item.getProjectName(), item.getProjectType(), + item.getRecreationProjectCode(), item.getRecreationProjectCodeDesc()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -465,12 +466,12 @@ private void extractAndUploadRecreationRiskSite() { var results = this.recreationRiskSiteRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationRiskSite.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getForestFile(), item.getEntryTimestamp(), - item.getEntryUserid(), - item.getUpdateTimestamp(), item.getUpdateUserid()); + item.getEntryUserid(), + item.getUpdateTimestamp(), item.getUpdateUserid()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -483,12 +484,12 @@ private void extractAndUploadRecreationRiskRatingCode() { var results = this.recreationRiskRatingCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationRiskRatingCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationRiskRatingCode(), - item.getDescription(), item.getEffectiveDate(), - item.getExpiryDate(), item.getUpdateTimestamp()); + item.getDescription(), item.getEffectiveDate(), + item.getExpiryDate(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -501,13 +502,13 @@ private void extractAndUploadRecreationRiskEvaluation() { var results = this.recreationRiskEvaluationRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationRiskEvaluation.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getRecreationUserDaysCode(), - item.getRecreationOccupancyCode(), item.getEntryTimestamp(), - item.getEntryUserid(), item.getUpdateTimestamp(), - item.getUpdateUserid()); + item.getRecreationOccupancyCode(), item.getEntryTimestamp(), + item.getEntryUserid(), item.getUpdateTimestamp(), + item.getUpdateUserid()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -520,14 +521,14 @@ private void extractAndUploadRecreationRemedRepairCode() { var results = this.recreationRemedRepairCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationRemedRepairCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationRemedRepairCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -540,24 +541,24 @@ private void extractAndUploadRecreationProject() { var results = this.recreationProjectRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationProject.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getProjectName(), - item.getRecreationControlAccessCode(), item.getRecreationFeatureCode(), - item.getRecreationMaintainStdCode(), item.getRecreationRiskRatingCode(), - item.getUtmZone(), item.getLastRecInspectionDate(), - item.getRecProjectSkey(), item.getResourceFeatureInd(), - item.getLastHzrdTreeAssessDate(), item.getSiteDescription(), - item.getRecreationUserDaysCode(), item.getOverflowCampsites(), - item.getUtmNorthing(), item.getUtmEasting(), - item.getRightOfWay(), item.getArchImpactAssessInd(), - item.getSiteLocation(), item.getProjectEstablishedDate(), - item.getRecreationViewInd(), item.getRevisionCount(), - item.getEntryUserid(), item.getEntryTimestamp(), - item.getUpdateUserid(), item.getUpdateTimestamp(), - item.getArchImpactDate(), item.getBordenNo(), - item.getCampHostInd(), item.getLowMobilityAccessInd()); + item.getRecreationControlAccessCode(), item.getRecreationFeatureCode(), + item.getRecreationMaintainStdCode(), item.getRecreationRiskRatingCode(), + item.getUtmZone(), item.getLastRecInspectionDate(), + item.getRecProjectSkey(), item.getResourceFeatureInd(), + item.getLastHzrdTreeAssessDate(), item.getSiteDescription(), + item.getRecreationUserDaysCode(), item.getOverflowCampsites(), + item.getUtmNorthing(), item.getUtmEasting(), + item.getRightOfWay(), item.getArchImpactAssessInd(), + item.getSiteLocation(), item.getProjectEstablishedDate(), + item.getRecreationViewInd(), item.getRevisionCount(), + item.getEntryUserid(), item.getEntryTimestamp(), + item.getUpdateUserid(), item.getUpdateTimestamp(), + item.getArchImpactDate(), item.getBordenNo(), + item.getCampHostInd(), item.getLowMobilityAccessInd()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -570,11 +571,11 @@ private void extractAndUploadRecreationPlan() { var results = this.recreationPlanRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationPlan.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRecProjectSkey(), - item.getPlanTypeCode(), item.getRemarks()); + item.getPlanTypeCode(), item.getRemarks()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -587,14 +588,14 @@ private void extractAndUploadRecreationOccupancyCode() { var results = this.recreationOccupancyCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationOccupancyCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationOccupancyCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -607,19 +608,19 @@ private void extractAndUploadRecreationObjective() { var results = this.recreationObjectiveRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationObjective.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getForestFile(), - item.getObjectiveDescription(), - item.getObjectiveEstablishedDate(), - item.getObjectiveAmendedDate(), - item.getObjectiveCancelledDate(), - item.getRevisionCount(), - item.getEntryUserid(), - item.getEntryTimestamp(), - item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getObjectiveDescription(), + item.getObjectiveEstablishedDate(), + item.getObjectiveAmendedDate(), + item.getObjectiveCancelledDate(), + item.getRevisionCount(), + item.getEntryUserid(), + item.getEntryTimestamp(), + item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -632,14 +633,14 @@ private void extractAndUploadRecreationMaintainStdCode() { var results = this.recreationMaintainStdCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationMaintainStdCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationMaintainStdCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -653,17 +654,17 @@ private void extractAndUploadRecreationInspectionReport() { var results = this.recreationInspectionReportRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationInspectionReport.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getForestFile(), item.getSiteName(), - item.getLocation(), item.getInspectedBy(), item.getCampsiteNo(), - item.getOccupiedCampsiteNo(), item.getVehicleNo(), item.getCampingPartyNo(), - item.getDayUsePartyNo(), item.getWithPassNo(), item.getWithoutPassNo(), - item.getAbsentOwnerNo(), item.getTotalInspectedNo(), item.getPurchasedPassNo(), - item.getRefusedPassNo(), item.getContractId(), item.getContractor(), - item.getRecProjectSkey(), item.getEntryTimestamp(), item.getEntryUserid(), - item.getUpdateTimestamp(), item.getUpdateUserid()); + item.getLocation(), item.getInspectedBy(), item.getCampsiteNo(), + item.getOccupiedCampsiteNo(), item.getVehicleNo(), item.getCampingPartyNo(), + item.getDayUsePartyNo(), item.getWithPassNo(), item.getWithoutPassNo(), + item.getAbsentOwnerNo(), item.getTotalInspectedNo(), item.getPurchasedPassNo(), + item.getRefusedPassNo(), item.getContractId(), item.getContractor(), + item.getRecProjectSkey(), item.getEntryTimestamp(), item.getEntryUserid(), + item.getUpdateTimestamp(), item.getUpdateUserid()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -676,14 +677,14 @@ private void extractAndUploadRecreationFileTypeCode() { var results = this.recreationFileTypeCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationFileTypeCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationFileTypeCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -696,14 +697,14 @@ private void extractAndUploadRecreationFileStatusCode() { var results = this.recreationFileStatusCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationFileStatusCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationFileStatusCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -716,15 +717,15 @@ private void extractAndUploadRecreationFee() { var results = this.recreationFeeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationFee.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getForestFile(), item.getFeeAmount(), - item.getFeeStartDate(), item.getFeeEndDate(), item.getMondayInd(), - item.getTuesdayInd(), item.getWednesdayInd(), item.getThursdayInd(), - item.getFridayInd(), item.getSaturdayInd(), item.getSundayInd(), - item.getRecreationFeeCode(), item.getRevisionCount(), item.getEntryUserid(), - item.getEntryTimestamp(), item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getFeeStartDate(), item.getFeeEndDate(), item.getMondayInd(), + item.getTuesdayInd(), item.getWednesdayInd(), item.getThursdayInd(), + item.getFridayInd(), item.getSaturdayInd(), item.getSundayInd(), + item.getRecreationFeeCode(), item.getRevisionCount(), item.getEntryUserid(), + item.getEntryTimestamp(), item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -737,14 +738,14 @@ private void extractAndUploadRecreationFeeCode() { var results = this.recreationFeeCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationFeeCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationFeeCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -757,14 +758,14 @@ private void extractAndUploadRecreationFeatureCode() { var results = this.recreationFeatureCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationFeatureCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationFeatureCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -777,16 +778,16 @@ private void extractAndUploadRecreationDistrictXref() { var results = this.recreationDistrictXrefRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationDistrictXref.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationDistrictCode(), - item.getForestFileId(), - item.getRevisionCount(), - item.getEntryUserid(), - item.getEntryTimestamp(), - item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getForestFileId(), + item.getRevisionCount(), + item.getEntryUserid(), + item.getEntryTimestamp(), + item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -799,14 +800,14 @@ private void extractAndUploadRecreationDistrictCode() { var results = this.recreationDistrictCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationDistrictCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationDistrictCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -819,14 +820,14 @@ private void extractAndUploadRecreationDefinedCampsite() { var results = this.recreationDefinedCampsiteRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationDefinedCampsite.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getCampsiteNumber(), - item.getEstimatedRepairCost(), item.getRecreationRemedRepairCode(), - item.getRepairCompleteDate(), item.getRevisionCount(), - item.getEntryUserid(), item.getEntryTimestamp(), - item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getEstimatedRepairCost(), item.getRecreationRemedRepairCode(), + item.getRepairCompleteDate(), item.getRevisionCount(), + item.getEntryUserid(), item.getEntryTimestamp(), + item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -839,14 +840,14 @@ private void extractAndUploadRecreationDefCsRprHistory() { var results = this.recreationDefCsRprHistoryRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationDefCsRprHistory.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getCampsiteNumber(), - item.getRecreationRemedRepairCode(), item.getEstimatedRepairCost(), - item.getRepairCompleteDate(), item.getRevisionCount(), - item.getEntryUserid(), item.getEntryTimestamp(), - item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getRecreationRemedRepairCode(), item.getEstimatedRepairCost(), + item.getRepairCompleteDate(), item.getRevisionCount(), + item.getEntryUserid(), item.getEntryTimestamp(), + item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -859,14 +860,14 @@ private void extractAndUploadRecreationControlAccessCode() { var results = this.recreationControlAccessCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationControlAccessCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationControlAccessCode(), - item.getDescription(), - item.getEffectiveDate(), - item.getExpiryDate(), - item.getUpdateTimestamp()); + item.getDescription(), + item.getEffectiveDate(), + item.getExpiryDate(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -880,12 +881,12 @@ private void extractAndUploadRecreationComment() { var results = this.recreationCommentRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationComment.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationCommentId(), item.getForestFileId(), item.getClosureInd(), - item.getProjectComment(), item.getCommentDate(), item.getRevisionCount(), item.getEntryUserid(), - item.getEntryTimestamp(), item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getProjectComment(), item.getCommentDate(), item.getRevisionCount(), item.getEntryUserid(), + item.getEntryTimestamp(), item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -901,6 +902,24 @@ private void extractAndUploadRecreationComment() { * @param recAttachments all the attachment objects from db */ private void extractAndUploadRecreationAttachmentContent(final List recAttachments) { + var isAttachmentProcessCompletedOnceEarlier = this.s3UploaderService.isAttachmentProcessCompletedEarlier(); + if (isAttachmentProcessCompletedOnceEarlier) { + // find the recAttachments of last 2 weeks and run through them. + var recAttachmentsOfLastTwoWeeks = recAttachments.stream() + .filter(recAttachment -> recAttachment.getUpdateTimestamp() + .isAfter(LocalDateTime.now().minusWeeks(2))) + .toList(); + logger.info("total attachments to be processed : {}", recAttachmentsOfLastTwoWeeks.size()); + if (!recAttachmentsOfLastTwoWeeks.isEmpty()) { + processRecAttachmentsAndUploadToS3IfNotPresent(recAttachmentsOfLastTwoWeeks); + } + } else { + processRecAttachmentsAndUploadToS3IfNotPresent(recAttachments); + } + + } + + private void processRecAttachmentsAndUploadToS3IfNotPresent(List recAttachments) { for (var recAttachment : recAttachments) { // for each attachment, check if it is already present in s3, if not query db // and upload. @@ -931,12 +950,12 @@ private List extractAndUploadRecreationAttachment() { var results = this.recreationAttachmentRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationAttachment.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRecreationAttachmentId(), item.getAttachmentFileName(), - item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -951,13 +970,13 @@ private void extractAndUploadRecreationAgreementHolder() { var results = this.recreationAgreementHolderRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationAgreementHolder.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getForestFile(), item.getClientNumber(), - item.getClientLocationCode(), item.getAgreementStartDate(), item.getAgreementEndDate(), - item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getClientLocationCode(), item.getAgreementStartDate(), item.getAgreementEndDate(), + item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -972,11 +991,11 @@ private void extractAndUploadRecreationActivityCode() { var results = this.recreationActivityCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationActivityCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationActivityCode(), item.getDescription(), item.getEffectiveDate(), - item.getExpiryDate(), item.getUpdateTimestamp()); + item.getExpiryDate(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -990,12 +1009,12 @@ private void extractAndUploadRecreationActivity() { var results = this.recreationActivityRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationActivity.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRecreationActivityCode(), item.getActivityRank(), - item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -1010,14 +1029,14 @@ private void extractAndUploadRecreationMapFeatureGeom() { var results = this.recreationMapFeatureGeomRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationMapFeatureGeom.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getGeometryTypeCode(), item.getMapFeatureId(), item.getFeatureArea(), - item.getFeatureLength(), item.getFeaturePerimeter(), item.getRevisionCount(), item.getEntryUserid(), - item.getEntryTimestamp(), item.getUpdateUserid(), item - .getUpdateTimestamp(), - item.getGeometry()); + item.getFeatureLength(), item.getFeaturePerimeter(), item.getRevisionCount(), item.getEntryUserid(), + item.getEntryTimestamp(), item.getUpdateUserid(), item + .getUpdateTimestamp(), + item.getGeometry()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -1031,11 +1050,11 @@ private void extractAndUploadRecreationMapFeatureCode() { var results = this.recreationMapFeatureCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationMapFeatureCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationMapFeatureCode(), item.getDescription(), item.getEffectiveDate(), - item.getExpiryDate(), item.getUpdateTimestamp()); + item.getExpiryDate(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -1049,8 +1068,8 @@ private void extractAndUploadRecreationMapFeatureXguid() { var results = this.recreationMapFeatureXguidRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationMapFeatureXguid.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getId(), item.getRmfSkey()); } @@ -1067,11 +1086,11 @@ private void extractAndUploadRecreationAccessXref() { var results = this.recreationAccessXrefRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationAccessXref.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationAccessCode(), item.getRecreationSubAccessCode(), item.getRevisionCount(), - item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), item.getUpdateTimestamp()); + item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -1085,12 +1104,12 @@ private void extractAndUploadRecreationAccessCode() { var results = this.recreationAccessCodeRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationAccessCode.class); try ( - var out = new FileWriter(entityMetadata.filePath()); - var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + var out = new FileWriter(entityMetadata.filePath()); + var printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getRecreationAccessCode(), item.getDescription(), item.getEffectiveDate(), - item.getExpiryDate(), item.getUpdateTimestamp()); + item.getExpiryDate(), item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); @@ -1104,13 +1123,13 @@ private void extractAndUploadRecreationAccess() { var results = this.recreationAccessRepository.findAll(); var entityMetadata = getEntityMetadata(RecreationAccess.class); try ( - FileWriter out = new FileWriter(entityMetadata.filePath()); - CSVPrinter printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { + FileWriter out = new FileWriter(entityMetadata.filePath()); + CSVPrinter printer = new CSVPrinter(out, entityMetadata.csvFormatBuilder().build());) { for (var item : results) { printer.printRecord(item.getForestFileId(), item.getRecreationSubAccessCode(), item.getRecreationAccessCode(), - item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), - item.getUpdateTimestamp()); + item.getRevisionCount(), item.getEntryUserid(), item.getEntryTimestamp(), item.getUpdateUserid(), + item.getUpdateTimestamp()); } printer.flush(); this.s3UploaderService.uploadFileToS3(entityMetadata.filePath(), entityMetadata.fileName()); diff --git a/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/S3UploaderService.java b/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/S3UploaderService.java index 8b46d6e8..a399937b 100644 --- a/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/S3UploaderService.java +++ b/backend-el/src/main/java/ca/bc/gov/nrs/environment/fta/el/services/S3UploaderService.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Service; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @Service @@ -18,7 +19,8 @@ public class S3UploaderService { private final S3Client s3Client; @Value("${ca.bc.gov.nrs.environment.fta.el.s3.bucket}") private String bucket; - + @Value("${ca.bc.gov.nrs.environment.fta.el.s3.bucket.attachment.threshold}") + private Integer attachmentTheshold; public S3UploaderService(S3Client s3Client) { this.s3Client = s3Client; } @@ -62,11 +64,11 @@ public void uploadBytesToS3(String filePath, String fileName, byte[] fileContent public boolean checkIfFileExistInTheBucketPath(String filePath, String fileName) { try { var keyToCheck = filePath + "/" + fileName; - var listObjectsRequest = software.amazon.awssdk.services.s3.model.ListObjectsRequest.builder() + var listObjectsRequest = software.amazon.awssdk.services.s3.model.ListObjectsV2Request.builder() .bucket(bucket) .prefix(filePath) .build(); - var listObjectsResponse = this.s3Client.listObjects(listObjectsRequest); + var listObjectsResponse = this.s3Client.listObjectsV2(listObjectsRequest); for (var listObject : listObjectsResponse.contents()) { logger.debug("Key from s3 {}",listObject.key()); logger.debug("Key to check {}", keyToCheck); @@ -79,4 +81,27 @@ public boolean checkIfFileExistInTheBucketPath(String filePath, String fileName) } return false; } + + /** + * This method will check for the count, if the number of objects in the bucket are more than x(a value from property file), + * then it is considered that the attachment process is completed earlier and getting only one week back of data would be sufficient. + * @return true or false based on whether initial load has been done. + */ + public boolean isAttachmentProcessCompletedEarlier() { + var countObjectsRequest= software.amazon.awssdk.services.s3.model.ListObjectsV2Request.builder() + .bucket(bucket) + .prefix("uploads/attachments") + .maxKeys(attachmentTheshold) + .build(); + var countObjectsResponse = this.s3Client.listObjectsV2Paginator(countObjectsRequest); + long totalObjects = 0; + + for (ListObjectsV2Response page : countObjectsResponse) { + long retrievedPageSize = page.contents().stream() + .reduce(0, (subtotal, element) -> subtotal + 1, Integer::sum); + totalObjects += retrievedPageSize; + } + logger.info("Total objects in the bucket: {}, attachment threshold {}", totalObjects, attachmentTheshold); + return totalObjects > attachmentTheshold; + } } diff --git a/backend-el/src/main/resources/application.properties b/backend-el/src/main/resources/application.properties index c54e14bb..fffa09ef 100644 --- a/backend-el/src/main/resources/application.properties +++ b/backend-el/src/main/resources/application.properties @@ -1,6 +1,4 @@ spring.application.name=fta-rst-exporter -logging.level.org.springframework.boot.autoconfigure.logging=${SPRING_BOOT_AUTOCONFIG_LOG_LEVEL:INFO} -spring.mvc.log-request-details=${SPRING_SHOW_REQUEST_DETAILS:true} logging.pattern.console=%highlight(%d{yyyy-MM-dd HH:mm:ss.SSS}) | %highlight([%5p]) | %green([%t]) | %cyan([%logger]) | %white([%replace(%msg){'[\n\r\"]',''}]) | %replace(%rEx{10}){'[\n\r\"]',''}%nopex%n spring.jpa.show-sql=${SPRING_JPA_SHOW_SQL:true} spring.jackson.deserialization.fail-on-unknown-properties=false @@ -20,6 +18,7 @@ spring.docker.compose.enabled=false spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl ca.bc.gov.nrs.environment.fta.el.file-base-path=${FILE_BASE_PATH:/tmp} ca.bc.gov.nrs.environment.fta.el.s3.bucket=${AWS_S3_BUCKET:fta-rec-rst-uploads} +ca.bc.gov.nrs.environment.fta.el.s3.bucket.attachment.threshold=${ATTACHMENT_THRESHOLD:4000} cloud.aws.region.static=${AWS_REGION:ca-central-1} cloud.aws.credentials.accessKey=${AWS_ACCESS_KEY} cloud.aws.credentials.secretKey=${AWS_SECRET_KEY}