From 95d0dcaf69a6175f8c49ce1747364e9f41c886db Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 13:20:53 -0700 Subject: [PATCH 01/29] Fix school reports performance --- api/pom.xml | 5 ++ .../config/EducGradBusinessApiConfig.java | 2 +- .../gradbusiness/config/TokenUtilsConfig.java | 24 +++++++ .../gradbusiness/model/dto/ResponseObj.java | 10 +++ .../model/dto/ResponseObjCache.java | 42 ++++++++++++ .../service/GradBusinessService.java | 55 ++++++++++++---- .../util/EducGraduationApiConstants.java | 12 +++- .../api/gradbusiness/util/TokenUtils.java | 65 +++++++++++++++++++ api/src/main/resources/application.yaml | 7 ++ api/src/test/resources/application.yaml | 7 ++ 10 files changed, 215 insertions(+), 14 deletions(-) create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/TokenUtilsConfig.java create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObj.java create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObjCache.java create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java diff --git a/api/pom.xml b/api/pom.xml index 1078740..ded3762 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -170,6 +170,11 @@ pdfbox 2.0.26 + + org.json + json + 20220320 + diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java index 7cd745b..bcb5b39 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java @@ -40,7 +40,7 @@ public WebClient webClient() { return WebClient.builder().exchangeStrategies(ExchangeStrategies.builder() .codecs(configurer -> configurer .defaultCodecs() - .maxInMemorySize(20 * 1024 * 1024)) + .maxInMemorySize(100 * 1024 * 1024)) .build()).build(); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/TokenUtilsConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/TokenUtilsConfig.java new file mode 100644 index 0000000..84b5c0d --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/TokenUtilsConfig.java @@ -0,0 +1,24 @@ +package ca.bc.gov.educ.api.gradbusiness.config; + +import ca.bc.gov.educ.api.gradbusiness.model.dto.ResponseObjCache; +import ca.bc.gov.educ.api.gradbusiness.util.EducGraduationApiConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class TokenUtilsConfig { + + EducGraduationApiConstants constants; + + @Autowired + public TokenUtilsConfig(EducGraduationApiConstants constants) { + this.constants = constants; + } + + @Bean + public ResponseObjCache createResponseObjCache() { + return new ResponseObjCache(constants.getTokenExpiryOffset()); + } + +} diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObj.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObj.java new file mode 100644 index 0000000..2ca6913 --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObj.java @@ -0,0 +1,10 @@ +package ca.bc.gov.educ.api.gradbusiness.model.dto; + +import lombok.Data; + +@Data +public class ResponseObj { + + private String access_token; + private String refresh_token; +} diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObjCache.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObjCache.java new file mode 100644 index 0000000..d414d25 --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/model/dto/ResponseObjCache.java @@ -0,0 +1,42 @@ +package ca.bc.gov.educ.api.gradbusiness.model.dto; + +import org.json.JSONObject; + +import java.util.Base64; + +public class ResponseObjCache { + + private long tokenExpiry = 0; + private ResponseObj responseObj; + + // tokenExpiry-[seconds] provides a slight offset, if token WILL expire in + // [seconds], obtain a new one + private int offset; + + public ResponseObjCache(int offset) { + this.offset = offset; + } + + public ResponseObj getResponseObj() { + return responseObj; + } + + public void setResponseObj(ResponseObj responseObj) { + this.setTokenExpiry(responseObj); + this.responseObj = responseObj; + } + + public boolean isExpired(){ + // tokenExpiry-[seconds] provides a slight offset, if token WILL expire in + // 10 seconds, obtain a new one + return (responseObj == null) || (tokenExpiry-offset) < (System.currentTimeMillis() / 1000); + } + + private void setTokenExpiry(ResponseObj responseObj){ + String[] parts = responseObj.getAccess_token().split("\\."); + JSONObject payload = new JSONObject(new String(Base64.getUrlDecoder().decode(parts[1]))); + this.tokenExpiry = payload.getLong("exp"); + } + + +} diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index b740b69..5c87f42 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -4,6 +4,7 @@ import ca.bc.gov.educ.api.gradbusiness.util.EducGradBusinessApiConstants; import ca.bc.gov.educ.api.gradbusiness.util.EducGradBusinessUtil; import ca.bc.gov.educ.api.gradbusiness.util.EducGraduationApiConstants; +import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils; import io.github.resilience4j.retry.annotation.Retry; import jakarta.transaction.Transactional; import org.apache.commons.lang3.StringUtils; @@ -23,6 +24,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; /** * The type Grad business service. @@ -41,6 +44,12 @@ public class GradBusinessService { */ final WebClient webClient; + /** + * Utility service to obtain access token + * */ + @Autowired + TokenUtils tokenUtils; + /** * The Educ grad student api constants. */ @@ -234,17 +243,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc logger.debug("******** Fetched Student List ******"); List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { - for (UUID studentID : studentList) { - InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentID, "ACHV")).headers(h -> h.setBearerAuth(accessToken)).retrieve().bodyToMono(InputStreamResource.class).block(); - if (result != null) { - try { - locations.add(result.getInputStream()); - } catch (IOException e) { - logger.debug("Error {}",e.getLocalizedMessage()); - } - } - } - logger.debug("******** Fetched Achievement Reports ******"); + getStudentAchievementReports(studentList, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); String month = "00"; @@ -284,7 +283,6 @@ public ResponseEntity getStudentCredentialPDFByType(String pen, String t } } - @Transactional public ResponseEntity getStudentTranscriptPDFByType(String pen, String type, String interim, String accessToken) { try { @@ -316,4 +314,37 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t return getInternalServerErrorResponse(e); } } + + private void getStudentAchievementReports(List studentList, List locations) { + logger.debug("******** Getting Achievement Reports ******"); + List> futures = studentList.stream() + .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) + .collect(Collectors.toList()); + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); + CompletableFuture> result = allFutures.thenApply(v -> futures.stream() + .map(CompletableFuture::join) + .collect(Collectors.toList())); + locations.addAll(result.join()); +// for (UUID studentID : studentList) { +// InputStream result = getStudentAchievementReport(studentID); +// if (result != null) { +// locations.add(result); +// } +// } + logger.debug("******** Fetched Achievement Reports ******"); + } + + private InputStream getStudentAchievementReport(UUID studentGuid) { + String accessTokenNext = tokenUtils.getAccessToken(); + InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); + if (result != null) { + try { + logger.debug("******** Fetched Achievement Report for {} ******", studentGuid); + return result.getInputStream(); + } catch (IOException e) { + logger.debug("Error extracting bytes from binary stream: {}", e.getLocalizedMessage()); + } + } + return null; + } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGraduationApiConstants.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGraduationApiConstants.java index 4539f55..703e4ec 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGraduationApiConstants.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGraduationApiConstants.java @@ -54,5 +54,15 @@ public class EducGraduationApiConstants { @Value("${endpoint.grad-report-api.transcript-by-request.url}") private String studentTranscriptReportByRequest; - + @Value("${authorization.user}") + private String userName; + + @Value("${authorization.password}") + private String password; + + @Value("${endpoint.keycloak.getToken}") + private String tokenUrl; + + @Value("${authorization.token-expiry-offset}") + private int tokenExpiryOffset; } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java new file mode 100644 index 0000000..8ffc42c --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java @@ -0,0 +1,65 @@ +package ca.bc.gov.educ.api.gradbusiness.util; + +import ca.bc.gov.educ.api.gradbusiness.model.dto.ResponseObj; +import ca.bc.gov.educ.api.gradbusiness.model.dto.ResponseObjCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.reactive.function.BodyInserters; +import org.springframework.web.reactive.function.client.WebClient; + +@Component +public class TokenUtils { + private static Logger logger = LoggerFactory.getLogger(TokenUtils.class); + + private final EducGraduationApiConstants constants; + private final WebClient webClient; + private final ResponseObjCache responseObjCache; + + @Autowired + public TokenUtils(EducGraduationApiConstants constants, WebClient webClient, ResponseObjCache responseObjCache) { + this.constants = constants; + this.webClient = webClient; + this.responseObjCache = responseObjCache; + } + + private String fetchAccessToken() { + return this.getTokenResponseObject().getAccess_token(); + } + + public String getAccessToken() { + return this.fetchAccessToken(); + } + + private ResponseObj getTokenResponseObject() { + if(responseObjCache.isExpired()){ + responseObjCache.setResponseObj(getResponseObj()); + } + return responseObjCache.getResponseObj(); + } + + private ResponseObj getResponseObj() { + HttpHeaders httpHeaders = getHeaders( + constants.getUserName(), constants.getPassword()); + MultiValueMap map= new LinkedMultiValueMap<>(); + map.add("grant_type", "client_credentials"); + return this.webClient.post().uri(constants.getTokenUrl()) + .headers(h -> h.addAll(httpHeaders)) + .contentType(MediaType.APPLICATION_FORM_URLENCODED) + .body(BodyInserters.fromFormData(map)) + .retrieve() + .bodyToMono(ResponseObj.class).block(); + } + + private HttpHeaders getHeaders (String username,String password) { + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + httpHeaders.setBasicAuth(username, password); + return httpHeaders; + } +} diff --git a/api/src/main/resources/application.yaml b/api/src/main/resources/application.yaml index 24257d0..3b122d0 100644 --- a/api/src/main/resources/application.yaml +++ b/api/src/main/resources/application.yaml @@ -49,6 +49,11 @@ logging: autoconfigure: logging: ${SPRING_BOOT_AUTOCONFIG_LOG_LEVEL} +authorization: + user: ${STS_CLIENT_NAME} + password: ${STS_CLIENT_SECRET} + token-expiry-offset: '30' + #API Documentation springdoc: api-docs: @@ -98,6 +103,8 @@ resilience4j.retry: #Endpoint properties endpoint: + keycloak: + getToken: ${KEYCLOAK_TOKEN_URL}auth/realms/master/protocol/openid-connect/token pen-student-api: by-studentid: url: ${PEN_API}api/v1/student/%s diff --git a/api/src/test/resources/application.yaml b/api/src/test/resources/application.yaml index 35f1c1d..89040b7 100644 --- a/api/src/test/resources/application.yaml +++ b/api/src/test/resources/application.yaml @@ -51,6 +51,11 @@ logging: autoconfigure: logging: INFO +authorization: + user: grad-business-api + password: password + token-expiry-offset: '30' + #API Documentation springdoc: api-docs: @@ -62,6 +67,8 @@ springdoc: #Endpoints #Endpoint properties endpoint: + keycloak: + getToken: https://soam-dev.apps.silver.devops.gov.bc.ca/auth/realms/master/protocol/openid-connect/token grad-graduation-report-api: school-report-by-mincode: url: https://educ-grad-graduation-report-api-77c02f-dev.apps.silver.devops.gov.bc.ca/api/v1/graduationreports/schoolreport?mincode=%s&reportType=%s From 49aedbeac77c2d7ef859c149c4a4d86d8fd49795 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 14:37:14 -0700 Subject: [PATCH 02/29] Fix school reports performance --- .../service/GradBusinessService.java | 102 +++++++++--------- .../api/gradbusiness/util/TokenUtils.java | 8 +- api/src/main/resources/application.yaml | 17 ++- .../EducGradBusinessApiApplicationTests.java | 27 +++++ .../EducGradBusinessApiControllerTests.java | 9 +- 5 files changed, 103 insertions(+), 60 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index 5c87f42..c5ea2d9 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -175,48 +175,6 @@ public ResponseEntity getStudentDemographicsByPen(String pen, String acc } } - /** - * Gets internal server error response. - * - * @param t the t - * @return the internal server error response - */ - protected ResponseEntity getInternalServerErrorResponse(Throwable t) { - ResponseEntity result; - - Throwable tmp = t; - String message; - if (tmp.getCause() != null) { - tmp = tmp.getCause(); - message = tmp.getMessage(); - } else { - message = tmp.getMessage(); - } - if(message == null) { - message = tmp.getClass().getName(); - } - - result = ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(message.getBytes()); - return result; - } - - private ResponseEntity handleBinaryResponse(byte[] resultBinary, String reportFile, MediaType contentType) { - ResponseEntity response; - - if(resultBinary.length > 0) { - HttpHeaders headers = new HttpHeaders(); - headers.add("Content-Disposition", "inline; filename=" + reportFile); - response = ResponseEntity - .ok() - .headers(headers) - .contentType(contentType) - .body(resultBinary); - } else { - response = ResponseEntity.status(HttpStatus.NO_CONTENT).build(); - } - return response; - } - public ResponseEntity getSchoolReportPDFByMincode(String mincode, String type, String accessToken) { Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); @@ -238,9 +196,10 @@ public ResponseEntity getSchoolReportPDFByMincode(String mincode, String } public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String mincode, String type, String accessToken) { + logger.debug("******** Retrieve List of Students for Amalgamated School Report ******"); List studentList = webClient.get().uri(String.format(educGradStudentApiConstants.getStudentsForAmalgamatedReport(), mincode, type)).headers(h -> h.setBearerAuth(accessToken)).retrieve().bodyToMono(new ParameterizedTypeReference>() { }).block(); - logger.debug("******** Fetched Student List ******"); + logger.debug("******** Fetched {} students ******", studentList.size()); List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { getStudentAchievementReports(studentList, locations); @@ -249,8 +208,9 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc String month = "00"; String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type); try { - logger.debug("******** Merged Documents ******"); + logger.debug("******** Merging Documents Started ******"); byte[] res = EducGradBusinessUtil.mergeDocuments(locations); + logger.debug("******** Merged {} Documents ******", locations.size()); HttpHeaders headers = new HttpHeaders(); headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken)); headers.put(HttpHeaders.ACCEPT, Collections.singletonList(APPLICATION_PDF)); @@ -316,7 +276,7 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t } private void getStudentAchievementReports(List studentList, List locations) { - logger.debug("******** Getting Achievement Reports ******"); + logger.debug("******** Getting Student Achievement Reports ******"); List> futures = studentList.stream() .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) .collect(Collectors.toList()); @@ -325,13 +285,7 @@ private void getStudentAchievementReports(List studentList, List getInternalServerErrorResponse(Throwable t) { + ResponseEntity result; + + Throwable tmp = t; + String message; + if (tmp.getCause() != null) { + tmp = tmp.getCause(); + message = tmp.getMessage(); + } else { + message = tmp.getMessage(); + } + if(message == null) { + message = tmp.getClass().getName(); + } + + result = ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(message.getBytes()); + return result; + } + + private ResponseEntity handleBinaryResponse(byte[] resultBinary, String reportFile, MediaType contentType) { + ResponseEntity response; + + if(resultBinary.length > 0) { + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-Disposition", "inline; filename=" + reportFile); + response = ResponseEntity + .ok() + .headers(headers) + .contentType(contentType) + .body(resultBinary); + } else { + response = ResponseEntity.status(HttpStatus.NO_CONTENT).build(); + } + return response; + } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java index 8ffc42c..87e8088 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; @@ -42,7 +43,7 @@ private ResponseObj getTokenResponseObject() { } return responseObjCache.getResponseObj(); } - + private ResponseObj getResponseObj() { HttpHeaders httpHeaders = getHeaders( constants.getUserName(), constants.getPassword()); @@ -62,4 +63,9 @@ private HttpHeaders getHeaders (String username,String password) { httpHeaders.setBasicAuth(username, password); return httpHeaders; } + + public ResponseObj getTokenFallback(HttpServerErrorException exception){ + logger.error("{} NOT REACHABLE after many attempts.", constants.getTokenUrl(), exception); + return null; + } } diff --git a/api/src/main/resources/application.yaml b/api/src/main/resources/application.yaml index 3b122d0..98d4fd5 100644 --- a/api/src/main/resources/application.yaml +++ b/api/src/main/resources/application.yaml @@ -65,36 +65,43 @@ springdoc: #Resilience resilience4j.retry: instances: + gettoken: + max-attempts: ${MAX_RETRY_ATTEMPTS} + waitDuration: 5s + retryExceptions: + - org.springframework.web.client.HttpServerErrorException + ignoreExceptions: + - java.lang.NullPointerException searchbypen: - maxRetryAttempts: ${MAX_RETRY_ATTEMPTS} + max-attempts: ${MAX_RETRY_ATTEMPTS} waitDuration: 5s retryExceptions: - org.springframework.web.client.HttpServerErrorException ignoreExceptions: - java.lang.NullPointerException advancedsearch: - maxRetryAttempts: ${MAX_RETRY_ATTEMPTS} + max-attempts: ${MAX_RETRY_ATTEMPTS} waitDuration: 30s retryExceptions: - org.springframework.web.client.HttpServerErrorException ignoreExceptions: - java.lang.NullPointerException searchbyid: - maxRetryAttempts: ${MAX_RETRY_ATTEMPTS} + max-attempts: ${MAX_RETRY_ATTEMPTS} waitDuration: 5s retryExceptions: - org.springframework.web.client.HttpServerErrorException ignoreExceptions: - java.lang.NullPointerException generalpostcall: - maxRetryAttempts: ${MAX_RETRY_ATTEMPTS} + max-attempts: ${MAX_RETRY_ATTEMPTS} waitDuration: 3s retryExceptions: - org.springframework.web.client.HttpServerErrorException ignoreExceptions: - java.lang.NullPointerException generalgetcall: - maxRetryAttempts: ${MAX_RETRY_ATTEMPTS} + max-attempts: ${MAX_RETRY_ATTEMPTS} waitDuration: 3s retryExceptions: - org.springframework.web.client.HttpServerErrorException diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java index fdaf090..733c306 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java @@ -4,6 +4,7 @@ import ca.bc.gov.educ.api.gradbusiness.service.GradBusinessService; import ca.bc.gov.educ.api.gradbusiness.util.EducGradBusinessApiConstants; import ca.bc.gov.educ.api.gradbusiness.util.EducGraduationApiConstants; +import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils; import org.junit.FixMethodOrder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -46,6 +47,8 @@ class EducGradBusinessApiApplicationTests { @MockBean WebClient webClient; + @MockBean + private TokenUtils tokenUtils; @Mock private WebClient.RequestHeadersSpec requestHeadersMock; @@ -88,6 +91,8 @@ void testReportDataByPen() throws Exception { String reportData = readFile("json/studentTranscriptReportData.json"); assertNotNull(reportData); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getGraduateReportDataByPenUrl(),"128385861") + "?type=")).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); @@ -146,6 +151,8 @@ void testReportDataByGraduationData() throws Exception { String reportData = readFile("json/studentTranscriptReportData.json"); assertNotNull(reportData); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.post()).thenReturn(this.requestBodyUriMock); when(this.requestBodyUriMock.uri(educGraduationApiConstants.getGraduateReportDataByGraduation() + "?type=")).thenReturn(this.requestBodyUriMock); when(this.requestBodyUriMock.headers(any(Consumer.class))).thenReturn(this.requestBodyMock); @@ -226,6 +233,8 @@ void testXmlTranscriptReportData() throws Exception { String xmlTranscriptReportData = readFile("json/xml_report_sample.xml"); assertNotNull(xmlTranscriptReportData); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.post()).thenReturn(this.requestBodyUriMock); when(this.requestBodyUriMock.uri(educGraduationApiConstants.getXmlTranscriptReportData())).thenReturn(this.requestBodyUriMock); when(this.requestBodyUriMock.headers(any(Consumer.class))).thenReturn(this.requestBodyMock); @@ -264,6 +273,8 @@ void testSchoolReportPDFByMincode() throws Exception { byte[] samplePdf = readBinaryFile("data/sample.pdf"); InputStreamResource pdf = new InputStreamResource(new ByteArrayInputStream(samplePdf)); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getSchoolReportByMincode(),mincode,type))).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); @@ -284,6 +295,8 @@ void testgetAmalgamatedSchoolReportPDFByMincode() throws Exception { byte[] samplePdf = readBinaryFile("data/sample.pdf"); InputStreamResource pdf = new InputStreamResource(new ByteArrayInputStream(samplePdf)); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + UUID studentID = UUID.randomUUID(); when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGradStudentApiConstants.getStudentsForAmalgamatedReport(),mincode,type))).thenReturn(this.requestHeadersMock); @@ -325,6 +338,7 @@ void testSchoolReportPDFByMincode_NotFound() throws Exception { byte[] samplePdf = new byte[0]; InputStreamResource pdf = new InputStreamResource(new ByteArrayInputStream(samplePdf)); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getSchoolReportByMincode(),mincode,type))).thenReturn(this.requestHeadersMock); @@ -344,6 +358,7 @@ void testSchoolReportPDFByMincode_Error500() throws Exception { String type = "NONGRADPRJ"; InputStream is = getClass().getClassLoader().getResourceAsStream("json/xmlTranscriptReportRequest.json"); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getSchoolReportByMincode(),mincode,type))).thenReturn(this.requestHeadersMock); @@ -364,6 +379,8 @@ void testGetStudentDemographicsByPen() throws Exception { InputStream is = getClass().getClassLoader().getResourceAsStream("json/xmlTranscriptReportRequest.json"); byte[] barr = is.readAllBytes(); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGradStudentApiConstants.getPenDemographicStudentApiUrl(),pen))).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); @@ -410,6 +427,8 @@ void testStudentCredentialPDFByType() throws Exception { sObj.setPen(pen); sObj.setMincode("123123112"); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getStudentCredentialByType(),sObj.getStudentID(),type))).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); @@ -438,6 +457,8 @@ void testStudentTranscriptPDFByTypeByPen() throws Exception { String reportData = readFile("json/studentTranscriptReportData.json"); assertNotNull(reportData); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getGraduateReportDataByPenUrl(),"128385861") + "?type=")).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); @@ -460,6 +481,8 @@ void testStudentTranscriptPDFByTypeByPen() throws Exception { when(this.requestHeadersMock.retrieve()).thenReturn(this.responseMock); when(this.responseMock.bodyToMono(byte[].class)).thenReturn(Mono.just(transcriptPdfSample)); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + ResponseEntity transcriptPdf = gradBusinessService.getStudentTranscriptPDFByType(pen, "xml", null,"accessToken"); assertNotNull(transcriptPdf.getBody()); assertEquals(transcriptPdfSample,transcriptPdf.getBody()); @@ -479,6 +502,8 @@ void testStudentCredentialPDFByType_NotFound() throws Exception { sObj.setPen(pen); sObj.setMincode("123123112"); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGraduationApiConstants.getStudentCredentialByType(),sObj.getStudentID(),type))).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); @@ -518,6 +543,8 @@ void testStudentCredentialPDFByType_Error500() throws Exception { final ParameterizedTypeReference> responseType = new ParameterizedTypeReference<>() { }; + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + when(this.webClient.get()).thenReturn(this.requestHeadersUriMock); when(this.requestHeadersUriMock.uri(String.format(educGradStudentApiConstants.getPenStudentApiByPenUrl(),pen))).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.headers(any(Consumer.class))).thenReturn(this.requestHeadersMock); diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiControllerTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiControllerTests.java index fbe0ade..e40b22b 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiControllerTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiControllerTests.java @@ -3,6 +3,7 @@ import ca.bc.gov.educ.api.gradbusiness.controller.GradBusinessController; import ca.bc.gov.educ.api.gradbusiness.model.dto.Student; import ca.bc.gov.educ.api.gradbusiness.service.GradBusinessService; +import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils; import org.junit.FixMethodOrder; import org.junit.jupiter.api.Test; import org.junit.runner.RunWith; @@ -11,6 +12,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -25,6 +27,7 @@ import java.util.UUID; import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; @FixMethodOrder(MethodSorters.NAME_ASCENDING) @RunWith(SpringRunner.class) @@ -32,13 +35,15 @@ @ActiveProfiles("test") class EducGradBusinessApiControllerTests { + @MockBean + private TokenUtils tokenUtils; + @Mock private GradBusinessService gradBusinessService; @InjectMocks private GradBusinessController gradBusinessController; - @Test void testReportDataByPen() throws Exception { @@ -234,6 +239,8 @@ void testStudentTranscriptPDFByType() throws Exception { .contentType(MediaType.APPLICATION_XML) .body(greBPack); + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + Mockito.when(gradBusinessService.getStudentTranscriptPDFByType("12312321","xml", null,"accessToken")).thenReturn(response); gradBusinessController.studentTranscriptByType("12312321", "xml", null, "accessToken"); Mockito.verify(gradBusinessService).getStudentTranscriptPDFByType("12312321","xml", null,"accessToken"); From c959cd88b667a88d5fb59f02c4c8163539944896 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 14:48:27 -0700 Subject: [PATCH 03/29] Code smells fix --- .../service/GradBusinessService.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index c5ea2d9..6e34e2d 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -25,7 +25,6 @@ import java.io.InputStream; import java.util.*; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; /** * The type Grad business service. @@ -47,20 +46,17 @@ public class GradBusinessService { /** * Utility service to obtain access token * */ - @Autowired - TokenUtils tokenUtils; + final TokenUtils tokenUtils; /** * The Educ grad student api constants. */ - @Autowired - EducGradBusinessApiConstants educGradStudentApiConstants; + final EducGradBusinessApiConstants educGradStudentApiConstants; /** * The Educ graduation api constants. */ - @Autowired - EducGraduationApiConstants educGraduationApiConstants; + final EducGraduationApiConstants educGraduationApiConstants; /** * Instantiates a new Grad business service. @@ -68,8 +64,11 @@ public class GradBusinessService { * @param webClient the web client */ @Autowired - public GradBusinessService(WebClient webClient) { + public GradBusinessService(WebClient webClient, TokenUtils tokenUtils, EducGradBusinessApiConstants educGradStudentApiConstants, EducGraduationApiConstants educGraduationApiConstants) { this.webClient = webClient; + this.tokenUtils = tokenUtils; + this.educGradStudentApiConstants = educGradStudentApiConstants; + this.educGraduationApiConstants = educGraduationApiConstants; } /** @@ -199,9 +198,9 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc logger.debug("******** Retrieve List of Students for Amalgamated School Report ******"); List studentList = webClient.get().uri(String.format(educGradStudentApiConstants.getStudentsForAmalgamatedReport(), mincode, type)).headers(h -> h.setBearerAuth(accessToken)).retrieve().bodyToMono(new ParameterizedTypeReference>() { }).block(); - logger.debug("******** Fetched {} students ******", studentList.size()); List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { + logger.debug("******** Fetched {} students ******", studentList.size()); getStudentAchievementReports(studentList, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); @@ -279,11 +278,11 @@ private void getStudentAchievementReports(List studentList, List> futures = studentList.stream() .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) - .collect(Collectors.toList()); + .toList(); CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); CompletableFuture> result = allFutures.thenApply(v -> futures.stream() .map(CompletableFuture::join) - .collect(Collectors.toList())); + .toList()); locations.addAll(result.join()); logger.debug("******** Fetched All Student Achievement Reports ******"); } From 811945eaa7cfe2b3a4142a676f108c5006acd580 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Thu, 23 Nov 2023 14:58:00 -0700 Subject: [PATCH 04/29] Update pom.xml --- api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/pom.xml b/api/pom.xml index ded3762..ecb4c05 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -11,7 +11,7 @@ ca.bc.gov educ-grad-business-api - 1.8.25 + 1.8.26 educ-grad-business-api GRAD Business API for external clients From 8281383e8973f7e80a8a29cdbe2558b035728f33 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Thu, 23 Nov 2023 15:29:19 -0700 Subject: [PATCH 05/29] Update application.yaml --- api/src/main/resources/application.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/main/resources/application.yaml b/api/src/main/resources/application.yaml index 98d4fd5..32041d6 100644 --- a/api/src/main/resources/application.yaml +++ b/api/src/main/resources/application.yaml @@ -50,8 +50,8 @@ logging: logging: ${SPRING_BOOT_AUTOCONFIG_LOG_LEVEL} authorization: - user: ${STS_CLIENT_NAME} - password: ${STS_CLIENT_SECRET} + user: ${GRAD_STS_CLIENT_NAME} + password: ${GRAD_STS_CLIENT_SECRET} token-expiry-offset: '30' #API Documentation @@ -111,7 +111,7 @@ resilience4j.retry: #Endpoint properties endpoint: keycloak: - getToken: ${KEYCLOAK_TOKEN_URL}auth/realms/master/protocol/openid-connect/token + getToken: ${TOKEN_ISSUER_URL}/protocol/openid-connect/token pen-student-api: by-studentid: url: ${PEN_API}api/v1/student/%s @@ -145,4 +145,4 @@ endpoint: #Splunk LogHelper splunk: log-helper: - enabled: ${ENABLE_SPLUNK_LOG_HELPER} \ No newline at end of file + enabled: ${ENABLE_SPLUNK_LOG_HELPER} From 512517062de621375ae753d2f04af3fec6734f92 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Thu, 23 Nov 2023 15:31:23 -0700 Subject: [PATCH 06/29] Update api.dc.yaml --- tools/openshift/api.dc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/openshift/api.dc.yaml b/tools/openshift/api.dc.yaml index 9cc301c..c8eee3d 100644 --- a/tools/openshift/api.dc.yaml +++ b/tools/openshift/api.dc.yaml @@ -62,6 +62,8 @@ objects: name: educ-grad-api-config-map - configMapRef: name: educ-grad-business-api-config-map + - secretRef: + name: grad-client-secret resources: requests: cpu: "${MIN_CPU}" From 11b2959a83d3e786d4514806c2f7bf1943c880cb Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Thu, 23 Nov 2023 15:46:46 -0700 Subject: [PATCH 07/29] Update api.dc.yaml --- tools/openshift/api.dc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/openshift/api.dc.yaml b/tools/openshift/api.dc.yaml index c8eee3d..3fc174e 100644 --- a/tools/openshift/api.dc.yaml +++ b/tools/openshift/api.dc.yaml @@ -63,7 +63,7 @@ objects: - configMapRef: name: educ-grad-business-api-config-map - secretRef: - name: grad-client-secret + name: grad-sts-client-secret resources: requests: cpu: "${MIN_CPU}" From 4ca9169e4a6c18b339dbc3443aafe441e0759911 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 16:37:35 -0700 Subject: [PATCH 08/29] Code smells fix --- .../api/gradbusiness/config/EducGradBusinessApiConfig.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java index bcb5b39..b2ea285 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java @@ -5,6 +5,7 @@ import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.client.RestTemplate; import org.springframework.web.reactive.function.client.ExchangeStrategies; import org.springframework.web.reactive.function.client.WebClient; @@ -37,7 +38,9 @@ public ModelMapper modelMapper() { public WebClient webClient() { HttpClient client = HttpClient.create(); client.warmup().block(); - return WebClient.builder().exchangeStrategies(ExchangeStrategies.builder() + return WebClient.builder() + .clientConnector(new ReactorClientHttpConnector(HttpClient.newConnection().compress(true))) + .exchangeStrategies(ExchangeStrategies.builder() .codecs(configurer -> configurer .defaultCodecs() .maxInMemorySize(100 * 1024 * 1024)) From 8615a06a270a27dff2e62b74f7141717347132c4 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Thu, 23 Nov 2023 17:13:21 -0700 Subject: [PATCH 09/29] Create build.from.developer.branch.deploy.to.dev.yml --- ...ld.from.developer.branch.deploy.to.dev.yml | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 .github/workflows/build.from.developer.branch.deploy.to.dev.yml diff --git a/.github/workflows/build.from.developer.branch.deploy.to.dev.yml b/.github/workflows/build.from.developer.branch.deploy.to.dev.yml new file mode 100644 index 0000000..e25f942 --- /dev/null +++ b/.github/workflows/build.from.developer.branch.deploy.to.dev.yml @@ -0,0 +1,132 @@ +name: Build & Deploy to DEV from Developer Branch + +env: + # 🖊️ EDIT your repository secrets to log into your OpenShift cluster and set up the context. + # See https://github.com/redhat-actions/oc-login#readme for how to retrieve these values. + # To get a permanent token, refer to https://github.com/redhat-actions/oc-login/wiki/Using-a-Service-Account-for-GitHub-Actions + OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} + OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} + OPENSHIFT_NAMESPACE: ${{ secrets.GRAD_NAMESPACE }}-dev + + # 🖊️ EDIT to change the image registry settings. + # Registries such as GHCR, Quay.io, and Docker Hub are supported. + IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }} + IMAGE_REGISTRY_USER: ${{ github.actor }} + IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + + SPRING_BOOT_IMAGE_NAME: educ-grad-business-api-dc + DOCKER_ARTIFACTORY_REPO: artifacts.developer.gov.bc.ca/docker-remote + + REPO_NAME: "educ-grad-business-api" + APP_DOMAIN: ${{ secrets.APP_DOMAIN }} + TAG: "latest" + #GRAD2-1947 Resource optimization + MIN_CPU: "20m" + MAX_CPU: "100m" + MIN_MEM: "256Mi" + MAX_MEM: "700Mi" + MIN_REPLICAS: "3" + MAX_REPLICAS: "5" + +on: + # https://docs.github.com/en/actions/reference/events-that-trigger-workflows + workflow_dispatch: + inputs: + choice: + type: choice + description: Choose branch to build from + options: + - develop/alex + - develop/chris + - develop/jinil + - develop/km + +jobs: + openshift-ci-cd: + name: Build and deploy to OpenShift DEV from Developer branch + # ubuntu-20.04 can also be used. + runs-on: ubuntu-20.04 + environment: dev + + outputs: + ROUTE: ${{ steps.deploy-and-expose.outputs.route }} + SELECTOR: ${{ steps.deploy-and-expose.outputs.selector }} + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.choice }} + + - name: Determine image tags + if: env.TAG == '' + run: | + echo "TAG=latest ${GITHUB_SHA::12}" | tee -a $GITHUB_ENV + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ${{ env.DOCKER_ARTIFACTORY_REPO }} + username: ${{ secrets.DOCKER_ARTIFACTORY_USERNAME }} + password: ${{ secrets.DOCKER_ARTIFACTORY_ACCESS_TOKEN }} + + # https://github.com/redhat-actions/buildah-build#readme + - name: Build from Dockerfile + id: build-image + uses: redhat-actions/buildah-build@v2 + with: + image: ${{ env.REPO_NAME }} + tags: ${{ env.TAG }} + + # If you don't have a Dockerfile/Containerfile, refer to https://github.com/redhat-actions/buildah-build#scratch-build-inputs + # Or, perform a source-to-image build using https://github.com/redhat-actions/s2i-build + # Otherwise, point this to your Dockerfile/Containerfile relative to the repository root. + dockerfiles: | + ./Dockerfile + + # https://github.com/redhat-actions/push-to-registry#readme + - name: Push to registry + id: push-image + uses: redhat-actions/push-to-registry@v2 + with: + image: ${{ steps.build-image.outputs.image }} + tags: ${{ steps.build-image.outputs.tags }} + registry: ${{ env.IMAGE_REGISTRY }} + username: ${{ env.IMAGE_REGISTRY_USER }} + password: ${{ env.IMAGE_REGISTRY_PASSWORD }} + + # The path the image was pushed to is now stored in ${{ steps.push-image.outputs.registry-path }} + - name: Install oc + uses: redhat-actions/openshift-tools-installer@v1 + with: + oc: 4 + + # https://github.com/redhat-actions/oc-login#readme + - name: Deploy + run: | + set -eux + # Login to OpenShift and select project + oc login --token=${{ env.OPENSHIFT_TOKEN }} --server=${{ env.OPENSHIFT_SERVER }} + oc project ${{ env.OPENSHIFT_NAMESPACE }} + # Cancel any rollouts in progress + oc rollout cancel dc/${{ env.SPRING_BOOT_IMAGE_NAME }} 2> /dev/null \ + || true && echo "No rollout in progress" + # tag image stream + oc -n ${{ env.OPENSHIFT_NAMESPACE }} tag ${{ steps.push-image.outputs.registry-path }} ${{ env.REPO_NAME }}:${{ env.TAG }} + + # Process and apply deployment template + oc process -f tools/openshift/api.dc.yaml -p IS_NAMESPACE=${{ env.OPENSHIFT_NAMESPACE }} -p REPO_NAME=${{ env.REPO_NAME }} -p TAG_NAME=${{ env.TAG }} -p HOST_ROUTE=${{ env.REPO_NAME }}-${{ env.OPENSHIFT_NAMESPACE }}.${{ env.APP_DOMAIN }} -p MIN_REPLICAS=${{ env.MIN_REPLICAS }} -p MAX_REPLICAS=${{ env.MAX_REPLICAS }} -p MIN_CPU=${{ env.MIN_CPU }} -p MAX_CPU=${{ env.MAX_CPU }} -p MIN_MEM=${{ env.MIN_MEM }} -p MAX_MEM=${{ env.MAX_MEM }} \ + | oc apply -f - + + # Start rollout (if necessary) and follow it + oc rollout latest dc/${{ env.SPRING_BOOT_IMAGE_NAME }} 2> /dev/null \ + || true && echo "Rollout in progress" + oc logs -f dc/${{ env.SPRING_BOOT_IMAGE_NAME }} + # Get status, returns 0 if rollout is successful + oc rollout status dc/${{ env.SPRING_BOOT_IMAGE_NAME }} + + # now hit it with a zap scan + - name: ZAP Scan + uses: zaproxy/action-api-scan@v0.1.0 + with: + target: 'https://${{ env.REPO_NAME }}-${{ env.OPENSHIFT_NAMESPACE }}-dev.apps.silver.devops.gov.bc.ca/api/v1/api-docs' From 20666560873390c99b35be3859f22e2ef8441013 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 19:03:59 -0700 Subject: [PATCH 10/29] Fix --- api/pom.xml | 6 +++++ .../service/GradBusinessService.java | 24 +++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index ecb4c05..9375ac3 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -175,6 +175,12 @@ json 20220320 + + + org.apache.commons + commons-collections4 + 4.4 + diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index 6e34e2d..d3125de 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -7,6 +7,7 @@ import ca.bc.gov.educ.api.gradbusiness.util.TokenUtils; import io.github.resilience4j.retry.annotation.Retry; import jakarta.transaction.Transactional; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -201,7 +202,8 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - getStudentAchievementReports(studentList, locations); + List> partitions = ListUtils.partition(studentList, 50); + getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); String month = "00"; @@ -274,16 +276,18 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t } } - private void getStudentAchievementReports(List studentList, List locations) { + private void getStudentAchievementReports(List> partitions, List locations) { logger.debug("******** Getting Student Achievement Reports ******"); - List> futures = studentList.stream() - .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) - .toList(); - CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); - CompletableFuture> result = allFutures.thenApply(v -> futures.stream() - .map(CompletableFuture::join) - .toList()); - locations.addAll(result.join()); + for(List studentList: partitions) { + List> futures = studentList.stream() + .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) + .toList(); + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])); + CompletableFuture> result = allFutures.thenApply(v -> futures.stream() + .map(CompletableFuture::join) + .toList()); + locations.addAll(result.join()); + } logger.debug("******** Fetched All Student Achievement Reports ******"); } From 1606419dd511458052d75c0a53e68a605c255976 Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 19:21:25 -0700 Subject: [PATCH 11/29] Fix --- .../gov/educ/api/gradbusiness/service/GradBusinessService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index d3125de..6d3f7dd 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -202,7 +202,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - List> partitions = ListUtils.partition(studentList, 50); + List> partitions = ListUtils.partition(studentList, 100); getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); From c8b0a2006e46ccfd18a2b90688df9badc21d3e9f Mon Sep 17 00:00:00 2001 From: arybakov Date: Thu, 23 Nov 2023 19:39:17 -0700 Subject: [PATCH 12/29] Fix --- .../educ/api/gradbusiness/service/GradBusinessService.java | 5 +++-- .../ca/bc/gov/educ/api/gradbusiness/util/TokenUtils.java | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index 6d3f7dd..a4f9ed5 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -202,7 +202,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc List locations = new ArrayList<>(); if (studentList != null && !studentList.isEmpty()) { logger.debug("******** Fetched {} students ******", studentList.size()); - List> partitions = ListUtils.partition(studentList, 100); + List> partitions = ListUtils.partition(studentList, 200); getStudentAchievementReports(partitions, locations); Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); @@ -279,6 +279,7 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t private void getStudentAchievementReports(List> partitions, List locations) { logger.debug("******** Getting Student Achievement Reports ******"); for(List studentList: partitions) { + logger.debug("******** Run partition with {} students ******", studentList.size()); List> futures = studentList.stream() .map(studentGuid -> CompletableFuture.supplyAsync(() -> getStudentAchievementReport(studentGuid))) .toList(); @@ -288,7 +289,7 @@ private void getStudentAchievementReports(List> partitions, List map= new LinkedMultiValueMap<>(); map.add("grant_type", "client_credentials"); + logger.debug("******** Fetch Access Token ********"); return this.webClient.post().uri(constants.getTokenUrl()) .headers(h -> h.addAll(httpHeaders)) .contentType(MediaType.APPLICATION_FORM_URLENCODED) From f83867f574b179f220f10774005e30693704729e Mon Sep 17 00:00:00 2001 From: arybakov Date: Fri, 24 Nov 2023 11:08:33 -0700 Subject: [PATCH 13/29] Fix --- api/pom.xml | 5 ++ .../config/EducGradBusinessApiConfig.java | 2 +- .../service/GradBusinessService.java | 13 ++-- .../util/EducGradBusinessUtil.java | 30 +++++++++ .../educ/api/gradbusiness/util/IOUtils.java | 65 +++++++++++++++++++ 5 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java diff --git a/api/pom.xml b/api/pom.xml index 9375ac3..3436e57 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -181,6 +181,11 @@ commons-collections4 4.4 + + org.apache.pdfbox + pdfbox + 2.0.27 + diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java index b2ea285..bac2109 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/config/EducGradBusinessApiConfig.java @@ -43,7 +43,7 @@ public WebClient webClient() { .exchangeStrategies(ExchangeStrategies.builder() .codecs(configurer -> configurer .defaultCodecs() - .maxInMemorySize(100 * 1024 * 1024)) + .maxInMemorySize(300 * 1024 * 1024)) .build()).build(); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index a4f9ed5..b6f7d12 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -22,7 +22,6 @@ import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; -import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -210,7 +209,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type); try { logger.debug("******** Merging Documents Started ******"); - byte[] res = EducGradBusinessUtil.mergeDocuments(locations); + byte[] res = EducGradBusinessUtil.mergeDocumentsPDFs(locations); logger.debug("******** Merged {} Documents ******", locations.size()); HttpHeaders headers = new HttpHeaders(); headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken)); @@ -294,14 +293,14 @@ private void getStudentAchievementReports(List> partitions, List h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); - if (result != null) { - try { + try { + InputStreamResource result = webClient.get().uri(String.format(educGraduationApiConstants.getStudentCredentialByType(), studentGuid, "ACHV")).headers(h -> h.setBearerAuth(accessTokenNext)).retrieve().bodyToMono(InputStreamResource.class).block(); + if (result != null) { logger.debug("******** Fetched Achievement Report for {} ******", studentGuid); return result.getInputStream(); - } catch (IOException e) { - logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage()); } + } catch (Exception e) { + logger.debug("Error extracting report binary from stream: {}", e.getLocalizedMessage()); } return null; } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java index d5edf3e..b3cfb69 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java @@ -5,6 +5,11 @@ import com.itextpdf.text.pdf.PdfCopy; import com.itextpdf.text.pdf.PdfReader; import com.itextpdf.text.pdf.PdfSmartCopy; +import org.apache.pdfbox.io.MemoryUsageSetting; +import org.apache.pdfbox.multipdf.PDFMergerUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -15,11 +20,36 @@ public class EducGradBusinessUtil { + private static final Logger logger = LoggerFactory.getLogger(EducGradBusinessUtil.class); + + public static final String TMP_DIR = "/tmp"; + private EducGradBusinessUtil() {} private static final int BUFFER_SIZE = 250000; + public static byte[] mergeDocumentsPDFs(List locations) throws IOException { + File bufferDirectory = null; + ByteArrayOutputStream result = new ByteArrayOutputStream(); + try { + bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer"); + PDFMergerUtility pdfMergerUtility = new PDFMergerUtility(); + pdfMergerUtility.setDestinationStream(result); + pdfMergerUtility.addSources(locations); + MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000) + .setTempDir(bufferDirectory); + pdfMergerUtility.mergeDocuments(memoryUsageSetting); + } catch (Exception e) { + logger.error("Error {}", e.getLocalizedMessage()); + } finally { + if (bufferDirectory != null) { + IOUtils.removeFileOrDirectory(bufferDirectory); + } + } + return result.toByteArray(); + } + public static byte[] mergeDocuments(List locations) throws IOException { final byte[] result; diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java new file mode 100644 index 0000000..8d54886 --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/IOUtils.java @@ -0,0 +1,65 @@ +package ca.bc.gov.educ.api.gradbusiness.util; + +import org.apache.commons.lang3.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.FileSystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +public class IOUtils { + + private static final Logger logger = LoggerFactory.getLogger(IOUtils.class); + + private IOUtils(){} + + /** + * Creates a secured temp dir for processing files, it is up to + * calling method to also remove directory (see removeFileOrDirectory + * method in this class) + * + * @param location + * @param prefix + * @return + * @throws IOException + */ + public static File createTempDirectory(String location, String prefix) throws IOException { + File temp; + Path loc = Paths.get(location); + if (SystemUtils.IS_OS_UNIX) { + FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + temp = Files.createTempDirectory(loc, prefix, attr).toFile(); // Compliant + } else { + temp = Files.createTempDirectory(loc, prefix).toFile(); // Compliant + temp.setReadable(true, true); + temp.setWritable(true, true); + temp.setExecutable(true, true); + } + return temp; + } + + /** + * Removes a directory or file recursively + * @param file + */ + public static void removeFileOrDirectory(File file) { + try { + if(file.isDirectory() && file.exists()){ + FileSystemUtils.deleteRecursively(file); + } else { + Files.deleteIfExists(Path.of(file.getAbsolutePath())); + } + } catch (IOException e) { + logger.error("Unable to delete file or folder {}", file.getAbsolutePath()); + } + } + +} From d28cf1fe37cdb8920d1e4fa7f457abeee3a4e2ca Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:23:40 -0700 Subject: [PATCH 14/29] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8b4e81..2e83317 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG DEPENDENCY=/workspace/app/target/dependency COPY --from=build ${DEPENDENCY}/BOOT-INF/lib /app/lib COPY --from=build ${DEPENDENCY}/META-INF /app/META-INF COPY --from=build ${DEPENDENCY}/BOOT-INF/classes /app -ENTRYPOINT ["java","-Duser.name=EDUC_GRAD_BUSINESS_API","-Xms500m","-Xmx500m","-noverify","-XX:TieredStopAtLevel=1",\ +ENTRYPOINT ["java","-Duser.name=EDUC_GRAD_BUSINESS_API","-Xms700m","-Xmx700m","-noverify","-XX:TieredStopAtLevel=1",\ "-XX:+UseParallelGC","-XX:MinHeapFreeRatio=20","-XX:MaxHeapFreeRatio=40","-XX:GCTimeRatio=4",\ "-XX:AdaptiveSizePolicyWeight=90","-XX:MaxMetaspaceSize=300m","-XX:ParallelGCThreads=1",\ "-Djava.util.concurrent.ForkJoinPool.common.parallelism=1","-XX:CICompilerCount=2",\ From ae9ebdc80930f429898607eb10a28ea241f0a0b9 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:26:11 -0700 Subject: [PATCH 15/29] Update build.from.developer.branch.deploy.to.dev.yml --- .../build.from.developer.branch.deploy.to.dev.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.from.developer.branch.deploy.to.dev.yml b/.github/workflows/build.from.developer.branch.deploy.to.dev.yml index e25f942..8e79bee 100644 --- a/.github/workflows/build.from.developer.branch.deploy.to.dev.yml +++ b/.github/workflows/build.from.developer.branch.deploy.to.dev.yml @@ -21,10 +21,10 @@ env: APP_DOMAIN: ${{ secrets.APP_DOMAIN }} TAG: "latest" #GRAD2-1947 Resource optimization - MIN_CPU: "20m" - MAX_CPU: "100m" - MIN_MEM: "256Mi" - MAX_MEM: "700Mi" + MIN_CPU: "30m" + MAX_CPU: "120m" + MIN_MEM: "350Mi" + MAX_MEM: "900Mi" MIN_REPLICAS: "3" MAX_REPLICAS: "5" From f0d08a4a5f1e025f6aee08c9fbfd20c3e1305193 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:26:56 -0700 Subject: [PATCH 16/29] Update build.from.main.branch.deploy.to.dev.yml --- .../workflows/build.from.main.branch.deploy.to.dev.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.from.main.branch.deploy.to.dev.yml b/.github/workflows/build.from.main.branch.deploy.to.dev.yml index 36d78aa..1082f7c 100644 --- a/.github/workflows/build.from.main.branch.deploy.to.dev.yml +++ b/.github/workflows/build.from.main.branch.deploy.to.dev.yml @@ -22,10 +22,10 @@ env: APP_DOMAIN: ${{ secrets.APP_DOMAIN }} TAG: "latest" #GRAD2-1947 Resource optimization - MIN_CPU: "20m" - MAX_CPU: "100m" - MIN_MEM: "256Mi" - MAX_MEM: "700Mi" + MIN_CPU: "30m" + MAX_CPU: "120m" + MIN_MEM: "350Mi" + MAX_MEM: "900Mi" MIN_REPLICAS: "3" MAX_REPLICAS: "5" From 3cc7557cbcba1ce64c608d7d0533ad1fce11c188 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:27:10 -0700 Subject: [PATCH 17/29] Update build.from.release.branch.deploy.to.dev.yml --- .../workflows/build.from.release.branch.deploy.to.dev.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.from.release.branch.deploy.to.dev.yml b/.github/workflows/build.from.release.branch.deploy.to.dev.yml index 35ec97c..1e231a3 100644 --- a/.github/workflows/build.from.release.branch.deploy.to.dev.yml +++ b/.github/workflows/build.from.release.branch.deploy.to.dev.yml @@ -21,10 +21,10 @@ env: APP_DOMAIN: ${{ secrets.APP_DOMAIN }} TAG: "latest" #GRAD2-1947 Resource optimization - MIN_CPU: "20m" - MAX_CPU: "100m" - MIN_MEM: "256Mi" - MAX_MEM: "700Mi" + MIN_CPU: "30m" + MAX_CPU: "120m" + MIN_MEM: "350Mi" + MAX_MEM: "900Mi" MIN_REPLICAS: "3" MAX_REPLICAS: "5" From ede78c6505fcaf5d6209938341a5e60d3844c31e Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:27:32 -0700 Subject: [PATCH 18/29] Update deploy_prod.yml --- .github/workflows/deploy_prod.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy_prod.yml b/.github/workflows/deploy_prod.yml index be2c8df..d3511e1 100644 --- a/.github/workflows/deploy_prod.yml +++ b/.github/workflows/deploy_prod.yml @@ -16,10 +16,10 @@ env: TAG: "latest" #GRAD2-1947 Resource optimization - MIN_CPU: "20m" - MAX_CPU: "100m" - MIN_MEM: "256Mi" - MAX_MEM: "700Mi" + MIN_CPU: "30m" + MAX_CPU: "120m" + MIN_MEM: "350Mi" + MAX_MEM: "900Mi" MIN_REPLICAS: "3" MAX_REPLICAS: "5" From ff76619681063244adcdb56afb3844cad015c86d Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:27:48 -0700 Subject: [PATCH 19/29] Update deploy_test.yml --- .github/workflows/deploy_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy_test.yml b/.github/workflows/deploy_test.yml index 27a8912..ecd23cf 100644 --- a/.github/workflows/deploy_test.yml +++ b/.github/workflows/deploy_test.yml @@ -16,10 +16,10 @@ env: TAG: "latest" #GRAD2-1947 Resource optimization - MIN_CPU: "20m" - MAX_CPU: "100m" - MIN_MEM: "256Mi" - MAX_MEM: "700Mi" + MIN_CPU: "30m" + MAX_CPU: "120m" + MIN_MEM: "350Mi" + MAX_MEM: "900Mi" MIN_REPLICAS: "3" MAX_REPLICAS: "5" @@ -75,4 +75,4 @@ jobs: || true && echo "Rollout in progress" oc logs -f dc/${{ env.SPRING_BOOT_IMAGE_NAME }} # Get status, returns 0 if rollout is successful - oc rollout status dc/${{ env.SPRING_BOOT_IMAGE_NAME }} \ No newline at end of file + oc rollout status dc/${{ env.SPRING_BOOT_IMAGE_NAME }} From 9f25cd0f2acf45f001242f3b75983004572e21b0 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 11:30:30 -0700 Subject: [PATCH 20/29] Update api.dc.yaml --- tools/openshift/api.dc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/openshift/api.dc.yaml b/tools/openshift/api.dc.yaml index 3fc174e..3de6e99 100644 --- a/tools/openshift/api.dc.yaml +++ b/tools/openshift/api.dc.yaml @@ -148,7 +148,7 @@ objects: resource: name: cpu target: - averageUtilization: 400 + averageUtilization: 350 type: Utilization - type: Resource resource: From 6ec38cd12ae95808a84a77395c4417e4306410ff Mon Sep 17 00:00:00 2001 From: arybakov Date: Fri, 24 Nov 2023 11:49:52 -0700 Subject: [PATCH 21/29] Fix --- .../gradbusiness/util/EducGradBusinessUtil.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java index b3cfb69..60fbb7a 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java @@ -10,10 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -31,23 +28,27 @@ private EducGradBusinessUtil() {} public static byte[] mergeDocumentsPDFs(List locations) throws IOException { File bufferDirectory = null; - ByteArrayOutputStream result = new ByteArrayOutputStream(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteArrayInputStream result; try { bufferDirectory = IOUtils.createTempDirectory(TMP_DIR, "buffer"); PDFMergerUtility pdfMergerUtility = new PDFMergerUtility(); - pdfMergerUtility.setDestinationStream(result); + pdfMergerUtility.setDestinationStream(outputStream); pdfMergerUtility.addSources(locations); MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMixed(50000000) .setTempDir(bufferDirectory); pdfMergerUtility.mergeDocuments(memoryUsageSetting); + result = new ByteArrayInputStream(outputStream.toByteArray()); + return result.readAllBytes(); } catch (Exception e) { logger.error("Error {}", e.getLocalizedMessage()); } finally { if (bufferDirectory != null) { IOUtils.removeFileOrDirectory(bufferDirectory); } + outputStream.close(); } - return result.toByteArray(); + return new byte[0]; } public static byte[] mergeDocuments(List locations) throws IOException { From c4047a863d7a07eb3751e3dd76066411c9226206 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Fri, 24 Nov 2023 12:07:26 -0700 Subject: [PATCH 22/29] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2e83317..9561aa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG DEPENDENCY=/workspace/app/target/dependency COPY --from=build ${DEPENDENCY}/BOOT-INF/lib /app/lib COPY --from=build ${DEPENDENCY}/META-INF /app/META-INF COPY --from=build ${DEPENDENCY}/BOOT-INF/classes /app -ENTRYPOINT ["java","-Duser.name=EDUC_GRAD_BUSINESS_API","-Xms700m","-Xmx700m","-noverify","-XX:TieredStopAtLevel=1",\ +ENTRYPOINT ["java","-Duser.name=EDUC_GRAD_BUSINESS_API","-Xms700m","-Xmx700m","-XX:TieredStopAtLevel=1",\ "-XX:+UseParallelGC","-XX:MinHeapFreeRatio=20","-XX:MaxHeapFreeRatio=40","-XX:GCTimeRatio=4",\ "-XX:AdaptiveSizePolicyWeight=90","-XX:MaxMetaspaceSize=300m","-XX:ParallelGCThreads=1",\ "-Djava.util.concurrent.ForkJoinPool.common.parallelism=1","-XX:CICompilerCount=2",\ From bc9da42f751f8aa3e73bf3b64b3b717052d74395 Mon Sep 17 00:00:00 2001 From: arybakov Date: Fri, 24 Nov 2023 12:29:16 -0700 Subject: [PATCH 23/29] Tests fix --- .../api/gradbusiness/EducGradBusinessApiApplicationTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java index 733c306..993d1f8 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java @@ -324,8 +324,7 @@ void testgetAmalgamatedSchoolReportPDFByMincode() throws Exception { byteData = gradBusinessService.getAmalgamatedSchoolReportPDFByMincode(mincode, type, "accessToken"); assertNotNull(byteData); - assertNotNull(byteData.getBody()); - assertTrue(byteData.getStatusCode().is5xxServerError()); + assertNull(byteData.getBody()); } From 035ab20014828bf30fd5bcbfcf11418656a9bd8a Mon Sep 17 00:00:00 2001 From: arybakov Date: Mon, 27 Nov 2023 12:11:07 -0700 Subject: [PATCH 24/29] Save generated PDF in the /tmp folder --- .../service/GradBusinessService.java | 16 ++++++++++++++-- .../gradbusiness/util/EducGradBusinessUtil.java | 5 +++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index b6f7d12..b647d93 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -22,7 +22,9 @@ import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; +import java.io.FileOutputStream; import java.io.InputStream; +import java.io.OutputStream; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -38,6 +40,7 @@ public class GradBusinessService { private static final String APPLICATION_JSON = "application/json"; private static final String APPLICATION_PDF = "application/pdf"; private static final String ACCEPT = "*/*"; + private static final String TMP = "/tmp"; /** * The Web client. */ @@ -188,7 +191,7 @@ public ResponseEntity getSchoolReportPDFByMincode(String mincode, String if(result != null) { res = result.getInputStream().readAllBytes(); } - return handleBinaryResponse(res, EducGradBusinessUtil.getFileNameSchoolReports(mincode,year,month,type), MediaType.APPLICATION_PDF); + return handleBinaryResponse(res, EducGradBusinessUtil.getFileNameSchoolReports(mincode,year,month,type,MediaType.APPLICATION_PDF), MediaType.APPLICATION_PDF); } catch (Exception e) { return getInternalServerErrorResponse(e); } @@ -206,7 +209,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("PST"), Locale.CANADA); int year = cal.get(Calendar.YEAR); String month = "00"; - String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type); + String fileName = EducGradBusinessUtil.getFileNameSchoolReports(mincode, year, month, type, MediaType.APPLICATION_PDF); try { logger.debug("******** Merging Documents Started ******"); byte[] res = EducGradBusinessUtil.mergeDocumentsPDFs(locations); @@ -215,6 +218,7 @@ public ResponseEntity getAmalgamatedSchoolReportPDFByMincode(String minc headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken)); headers.put(HttpHeaders.ACCEPT, Collections.singletonList(APPLICATION_PDF)); headers.put(HttpHeaders.CONTENT_TYPE, Collections.singletonList(APPLICATION_PDF)); + saveBinaryResponseToFile(res, fileName); return handleBinaryResponse(res, fileName, MediaType.APPLICATION_PDF); } catch (Exception e) { return getInternalServerErrorResponse(e); @@ -346,4 +350,12 @@ private ResponseEntity handleBinaryResponse(byte[] resultBinary, String } return response; } + + private void saveBinaryResponseToFile(byte[] resultBinary, String reportFile) throws Exception { + if(resultBinary.length > 0) { + try (OutputStream out = new FileOutputStream(TMP + "/" + reportFile)) { + out.write(resultBinary); + } + } + } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java index 60fbb7a..d44b0b8 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/util/EducGradBusinessUtil.java @@ -9,6 +9,7 @@ import org.apache.pdfbox.multipdf.PDFMergerUtility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; import java.io.*; import java.util.ArrayList; @@ -98,8 +99,8 @@ public static String getTempDirPath() { return Optional.ofNullable(System.getProperty("java.io.tmpdir")).orElse("/tmp").concat(File.pathSeparator); } - public static String getFileNameSchoolReports(String mincode, int year, String month, String type) { - return mincode + "_" + year + month + "_" + type + ".pdf"; + public static String getFileNameSchoolReports(String mincode, int year, String month, String type, MediaType mediaType) { + return mincode + "_" + year + month + "_" + type + "." + mediaType.getSubtype(); } public static String getFileNameStudentCredentials(String mincode, String pen, String type) { From b89ce9a2f0b033effaf37e6c434b367b109096e2 Mon Sep 17 00:00:00 2001 From: arybakov Date: Mon, 27 Nov 2023 12:15:12 -0700 Subject: [PATCH 25/29] Save generated PDF in the /tmp folder --- .../gov/educ/api/gradbusiness/service/GradBusinessService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index b647d93..d208502 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -23,6 +23,7 @@ import org.springframework.web.reactive.function.client.WebClient; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.*; @@ -351,7 +352,7 @@ private ResponseEntity handleBinaryResponse(byte[] resultBinary, String return response; } - private void saveBinaryResponseToFile(byte[] resultBinary, String reportFile) throws Exception { + private void saveBinaryResponseToFile(byte[] resultBinary, String reportFile) throws IOException { if(resultBinary.length > 0) { try (OutputStream out = new FileOutputStream(TMP + "/" + reportFile)) { out.write(resultBinary); From f9e8abc2d6c8e2901f027b1ee39d40b85c9f5914 Mon Sep 17 00:00:00 2001 From: arybakov Date: Mon, 27 Nov 2023 16:09:44 -0700 Subject: [PATCH 26/29] GRAD2-2306 HD-21559-GRAD - P3 - GRAD is creating blank transcripts --- .../exception/ServiceException.java | 39 +++++++++++++++++++ .../service/GradBusinessService.java | 11 +++++- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java new file mode 100644 index 0000000..dc86376 --- /dev/null +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java @@ -0,0 +1,39 @@ +package ca.bc.gov.educ.api.gradbusiness.exception; + +import lombok.Data; + +@Data +public class ServiceException extends RuntimeException { + + private int statusCode; + + public ServiceException() { + super(); + } + + public ServiceException(String message) { + super(message); + } + + public ServiceException(String message, Throwable cause) { + super(message, cause); + } + + public ServiceException(Throwable cause) { + super(cause); + } + + protected ServiceException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public ServiceException(String message, int value) { + super(message); + this.statusCode = value; + } + + public ServiceException(String s, int value, Exception e) { + super(s, e); + this.statusCode = value; + } +} diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java index d208502..3c8690c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/service/GradBusinessService.java @@ -1,5 +1,6 @@ package ca.bc.gov.educ.api.gradbusiness.service; +import ca.bc.gov.educ.api.gradbusiness.exception.ServiceException; import ca.bc.gov.educ.api.gradbusiness.model.dto.Student; import ca.bc.gov.educ.api.gradbusiness.util.EducGradBusinessApiConstants; import ca.bc.gov.educ.api.gradbusiness.util.EducGradBusinessUtil; @@ -272,9 +273,17 @@ public ResponseEntity getStudentTranscriptPDFByType(String pen, String t headers.put(HttpHeaders.AUTHORIZATION, Collections.singletonList(BEARER + accessToken)); headers.put(HttpHeaders.ACCEPT, Collections.singletonList(ACCEPT)); headers.put(HttpHeaders.CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)); - byte[] result = webClient.post().uri(educGraduationApiConstants.getStudentTranscriptReportByRequest()).headers(h -> h.addAll(headers)).body(BodyInserters.fromValue(reportRequest.toString())).retrieve().bodyToMono(byte[].class).block(); + byte[] result = webClient.post().uri(educGraduationApiConstants.getStudentTranscriptReportByRequest()) + .headers(h -> h.addAll(headers)).body(BodyInserters.fromValue(reportRequest.toString())).retrieve() + .onStatus( + HttpStatus.NO_CONTENT::equals, + response -> response.bodyToMono(String.class).thenReturn(new ServiceException("NO_CONTENT", response.statusCode().value())) + ) + .bodyToMono(byte[].class).block(); assert result != null; return handleBinaryResponse(result, pen + " Transcript Report.pdf", MediaType.APPLICATION_PDF); + } catch (ServiceException e) { + return handleBinaryResponse(new byte[0], pen + " Transcript Report.pdf", MediaType.APPLICATION_PDF); } catch (Exception e) { return getInternalServerErrorResponse(e); } From 3428a9745205ed91a504dc00dfa1bf1642bff551 Mon Sep 17 00:00:00 2001 From: arybakov Date: Mon, 27 Nov 2023 16:19:31 -0700 Subject: [PATCH 27/29] GRAD2-2306 HD-21559-GRAD - P3 - GRAD is creating blank transcripts --- .../api/gradbusiness/EducGradBusinessApiApplicationTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java index 993d1f8..6e7b616 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java @@ -478,6 +478,7 @@ void testStudentTranscriptPDFByTypeByPen() throws Exception { when(this.requestBodyMock.contentType(any())).thenReturn(this.requestBodyMock); when(this.requestBodyMock.body(any(BodyInserter.class))).thenReturn(this.requestHeadersMock); when(this.requestHeadersMock.retrieve()).thenReturn(this.responseMock); + when(this.responseMock.onStatus(any(), any())).thenReturn(this.responseMock); when(this.responseMock.bodyToMono(byte[].class)).thenReturn(Mono.just(transcriptPdfSample)); when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); From f9c536265cf721fb7b124890d660e4356a809cdd Mon Sep 17 00:00:00 2001 From: arybakov Date: Mon, 27 Nov 2023 16:23:58 -0700 Subject: [PATCH 28/29] Fix code smell --- .../exception/ServiceException.java | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java index dc86376..29f3f31 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java +++ b/api/src/main/java/ca/bc/gov/educ/api/gradbusiness/exception/ServiceException.java @@ -5,35 +5,11 @@ @Data public class ServiceException extends RuntimeException { - private int statusCode; - - public ServiceException() { - super(); - } - - public ServiceException(String message) { - super(message); - } - - public ServiceException(String message, Throwable cause) { - super(message, cause); - } - - public ServiceException(Throwable cause) { - super(cause); - } - - protected ServiceException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } + private final int statusCode; public ServiceException(String message, int value) { super(message); this.statusCode = value; } - public ServiceException(String s, int value, Exception e) { - super(s, e); - this.statusCode = value; - } } From 4fef7a7e1684aac706eae35b61593dbec4775dad Mon Sep 17 00:00:00 2001 From: arybakov Date: Mon, 27 Nov 2023 16:34:41 -0700 Subject: [PATCH 29/29] Fix code smell --- .../EducGradBusinessApiApplicationTests.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java index 6e7b616..1a0f2ec 100644 --- a/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java +++ b/api/src/test/java/ca/bc/gov/educ/api/gradbusiness/EducGradBusinessApiApplicationTests.java @@ -1,5 +1,6 @@ package ca.bc.gov.educ.api.gradbusiness; +import ca.bc.gov.educ.api.gradbusiness.exception.ServiceException; import ca.bc.gov.educ.api.gradbusiness.model.dto.Student; import ca.bc.gov.educ.api.gradbusiness.service.GradBusinessService; import ca.bc.gov.educ.api.gradbusiness.util.EducGradBusinessApiConstants; @@ -486,6 +487,21 @@ void testStudentTranscriptPDFByTypeByPen() throws Exception { ResponseEntity transcriptPdf = gradBusinessService.getStudentTranscriptPDFByType(pen, "xml", null,"accessToken"); assertNotNull(transcriptPdf.getBody()); assertEquals(transcriptPdfSample,transcriptPdf.getBody()); + + when(this.webClient.post()).thenReturn(this.requestBodyUriMock); + when(this.requestBodyUriMock.uri(educGraduationApiConstants.getStudentTranscriptReportByRequest())).thenReturn(this.requestBodyUriMock); + when(this.requestBodyUriMock.headers(any(Consumer.class))).thenReturn(this.requestBodyMock); + when(this.requestBodyMock.contentType(any())).thenReturn(this.requestBodyMock); + when(this.requestBodyMock.body(any(BodyInserter.class))).thenReturn(this.requestHeadersMock); + when(this.requestHeadersMock.retrieve()).thenReturn(this.responseMock); + when(this.responseMock.onStatus(any(), any())).thenReturn(this.responseMock); + when(this.responseMock.bodyToMono(byte[].class)).thenThrow(new ServiceException("NO_CONTENT", 204)); + + when(this.tokenUtils.getAccessToken()).thenReturn("accessToken"); + + transcriptPdf = gradBusinessService.getStudentTranscriptPDFByType(pen, "xml", null,"accessToken"); + assertNotNull(transcriptPdf); + assertNull(transcriptPdf.getBody()); } @Test