✨🔧 452 - [Feature branch] - Add ega permissions integration

.env.example

@@ -26,7 +26,7 @@ JWT_TOKEN_PUBLIC_KEY=
 ############
 # true or false
 VAULT_ENABLED=false
-VAULT_SECRETS_PATH=/service/secrets_v1
+VAULT_SECRETS_PATH=
 VAULT_URL=
 VAULT_ROLE=
 # for local development/testing
@@ -48,34 +48,34 @@ DACO_REVIEW_POLICY_NAME=DACO-REVIEW
 ############
 # Storage  #
 ############
-OBJECT_STORAGE_ENDPOINT=https://object.cancercollaboratory.org:9080
-OBJECT_STORAGE_REGION=
-OBJECT_STORAGE_BUCKET=
-OBJECT_STORAGE_KEY=
-OBJECT_STORAGE_SECRET=
+OBJECT_STORAGE_ENDPOINT=http://localhost:8085
+OBJECT_STORAGE_REGION=nova
+OBJECT_STORAGE_BUCKET=daco
+OBJECT_STORAGE_KEY=minio
+OBJECT_STORAGE_SECRET=minio123
 OBJECT_STORAGE_TIMEOUT_MILLIS=5000
 
 
 ############
 # EMAIL    #
 ############
-EMAIL_HOST=smtp.gmail.com
-EMAIL_PORT=587
+EMAIL_HOST=localhost
+EMAIL_PORT=1025
 EMAIL_USER=
 EMAIL_PASSWORD=
-EMAIL_FROM_ADDRESS=
-EMAIL_FROM_NAME=
-EMAIL_DACO_ADDRESS=
+EMAIL_FROM_ADDRESS=[email protected]
+EMAIL_FROM_NAME=DacoAdmin
+EMAIL_DACO_ADDRESS=[email protected]
 # for emails directed to daco reviewers
-EMAIL_REVIEWER_FIRSTNAME=
-EMAIL_REVIEWER_LASTNAME=
+EMAIL_REVIEWER_FIRSTNAME=DACO
+EMAIL_REVIEWER_LASTNAME=ADMIN
 DCC_MAILING_LIST=
 DACO_SURVEY_URL=
 
 ##############
 # UI         #
 ##############
-DACO_UI_BASE_URL=https://dac.dev.argo.cancercollaboratory.org
+DACO_UI_BASE_URL=http://localhost:3000
 DACO_UI_APPLICATION_SECTION_PATH=/applications/{id}?section={section}
 
 ##############
@@ -88,16 +88,16 @@ FILE_UPLOAD_LIMIT=#in bytes x * 1024 * 1024
 ##############
 
 # ATTESTATION
-ATTESTATION_UNIT_COUNT=
-ATTESTATION_UNIT_OF_TIME=
-DAYS_TO_ATTESTATION=
+ATTESTATION_UNIT_COUNT=1
+ATTESTATION_UNIT_OF_TIME=years
+DAYS_TO_ATTESTATION=45
 
 # EXPIRY
-DAYS_TO_EXPIRY_1=
-DAYS_TO_EXPIRY_2=
-DAYS_POST_EXPIRY=
-EXPIRY_UNIT_COUNT=
-EXPIRY_UNIT_OF_TIME=
+DAYS_TO_EXPIRY_1=90
+DAYS_TO_EXPIRY_2=45
+DAYS_POST_EXPIRY=90
+EXPIRY_UNIT_COUNT=2
+EXPIRY_UNIT_OF_TIME=years
 
 #############
 # Daco Encryption
@@ -109,3 +109,19 @@ DACO_ENCRYPTION_KEY=
 #############
 FEATURE_RENEWAL_ENABLED=false
 FEATURE_ADMIN_PAUSE_ENABLED=false
+FEATURE_EGA_RECONCILIATION_ENABLED=false
+
+#############
+# EGA
+#############
+EGA_CLIENT_ID=
+EGA_AUTH_HOST=
+EGA_AUTH_REALM_NAME=
+EGA_API_URL=
+EGA_USERNAME=
+EGA_PASSWORD=
+DAC_ID=
+EGA_MAX_REQUEST_LIMIT=3;
+EGA_MAX_REQUEST_INTERVAL=1000; # in milliseconds
+EGA_MAX_REQUEST_RETRIES=3
+EGA_MAX_ACCESS_TOKEN_REQUEST_RETRIES=5

.gitignore

@@ -62,4 +62,5 @@ typings/
 dist/
 
 # misc
-.DS_Store
+.DS_Store
+.vscode/settings.json

Makefile

@@ -4,15 +4,15 @@ debug: dcompose
 
 #run the docker compose file
 dcompose:
-	docker-compose -f dac-api-compose/docker-compose.yaml up -d
+	docker compose -f dac-api-compose/docker-compose.yaml up -d
 
 # run all tests
 verify:
 	npm run test
 
 stop:
-	docker-compose  -f dac-api-compose/docker-compose.yaml down --remove-orphans 
+	docker compose  -f dac-api-compose/docker-compose.yaml down --remove-orphans
 
 # delete. everything.
 nuke:
-	docker-compose  -f dac-api-compose/docker-compose.yaml down --volumes --remove-orphans 
+	docker compose  -f dac-api-compose/docker-compose.yaml down --volumes --remove-orphans

README.md

@@ -8,9 +8,26 @@ Development of the Data Access Control API
 | ---------- | ------------- | ----------------------------------------------------------------------------------------- | --------------------------------- | ------- |
 | `NODE_ENV` | isDevelopment | Enables `'/applications/:id'` DELETE endpoint. Enables `debug.log` file in Logger options | set `NODE_ENV` to `"development"` | `false` |
 
+## Environment Variables
+
+| Name                                 | Description                                                                                                                                                                                              | Type     | Required | Default |
+| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | ------- |
+| EGA_CLIENT_ID                        | Client ID for EGA API                                                                                                                                                                                    | `string` | true     |         |
+| EGA_AUTH_HOST                        | Root URL for EGA authentication server                                                                                                                                                                   | `string` | true     |         |
+| EGA_AUTH_REALM_NAME                  | Realm name for EGA authentication server                                                                                                                                                                 | `string` | true     |         |
+| EGA_API_URL                          | Root URL for EGA API                                                                                                                                                                                     | `string` | true     |         |
+| EGA_USERNAME                         | Username for account used to gain access token from EGA authentication server                                                                                                                            | `string` | true     |         |
+| EGA_PASSWORD                         | Password for account used to gain access token from EGA authentication server                                                                                                                            | `string` | true     |         |
+| DAC_ID                               | AccessionId for ICGC DAC                                                                                                                                                                                 | `string` | true     |         |
+| EGA_MAX_REQUEST_LIMIT                | For EGA API rate limiting. The max number of API requests per interval value `EGA_MAX_REQUEST_INTERVAL`                                                                                                  | `number` | true     | 3       |
+| EGA_MAX_REQUEST_INTERVAL             | For EGA API rate limiting. Interval of time for API request limit `EGA_MAX_REQUEST_LIMIT`, in milliseconds                                                                                               | `number` | true     | 1000    |
+| EGA_MAX_REQUEST_RETRIES              | Maximum number of API requests allowed before rejecting the original request. Used in the `axios-retry` config for the [EGA Axios client](./src/jobs/ega/axios/egaClient.ts)                             | `number` | true     | 3       |
+| EGA_MAX_ACCESS_TOKEN_REQUEST_RETRIES | Maximum number of API requests allowed to the EGA IDP server, before rejecting the original request. Used in the `axios-retry` config for the [EGA Auth Axios client](./src/jobs/ega/axios/idpClient.ts) | `number` | true     | 5       |
+
 ## Feature Flags
 
-| Name                        | Config Path                      | Description                                                                                                                                                                                                                                                                                                               | Trigger                   | Default |
-| --------------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | ------- |
-| FEATURE_RENEWAL_ENABLED     | `featureFlags.renewalEnabled`    | enables Renewal and Expiry features, incl. `/applications/{id}/renew` `POST` endpoint for creating a renewal application, and batch jobs triggered by `/jobs/batch-transitions` endpoint: `"FIRST EXPIRY NOTIFICATIONS"`, `"SECOND EXPIRY NOTIFICATIONS"`, `"EXPIRING APPLICATIONS"` and `"CLOSING UNSUBMITTED RENEWALS"` | set env value to `"true"` | `false` |
-| FEATURE_ADMIN_PAUSE_ENABLED | `featureFlags.adminPauseEnabled` | enables manual PAUSE transition of applications, using the Admin scope with the `/applications/{id}` `PATCH` or `/applications/:id/admin-pause` endpoints. Normally pausing is done only by the System role as a batch job. Intended for testing purposes only, **do not enable in production**                           | set env value to `"true"` | `false` |
+| Name                               | Config Path                             | Description                                                                                                                                                                                                                                                                                                               | Trigger                   | Default |
+| ---------------------------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | ------- |
+| FEATURE_RENEWAL_ENABLED            | `featureFlags.renewalEnabled`           | enables Renewal and Expiry features, incl. `/applications/{id}/renew` `POST` endpoint for creating a renewal application, and batch jobs triggered by `/jobs/batch-transitions` endpoint: `"FIRST EXPIRY NOTIFICATIONS"`, `"SECOND EXPIRY NOTIFICATIONS"`, `"EXPIRING APPLICATIONS"` and `"CLOSING UNSUBMITTED RENEWALS"` | set env value to `"true"` | `false` |
+| FEATURE_ADMIN_PAUSE_ENABLED        | `featureFlags.adminPauseEnabled`        | enables manual PAUSE transition of applications, using the Admin scope with the `/applications/{id}` `PATCH` or `/applications/:id/admin-pause` endpoints. Normally pausing is done only by the System role as a batch job. Intended for testing purposes only, **do not enable in production**                           | set env value to `"true"` | `false` |
+| FEATURE_EGA_RECONCILIATION_ENABLED | `featureFlags.egaReconciliationEnabled` | **Enables** [EGA reconciliation job process](./src/jobs/ega/egaPermissionsReconciliation.ts) triggered by `/jobs/batch-transitions` endpoint. **Disables** [Approved users email job](./src/jobs/approvedUsersEmail.ts) triggered in same endpoint.                                                                       | set env value to `"true"` | `false` |

dac-api-compose/docker-compose.yaml

@@ -1,8 +1,6 @@
-version: '3.8'
-
 services:
   vault:
-    image: vault
+    image: vault:1.13.3
     volumes:
       - $PWD/logs/:/tmp/logs
       - ./vault:/scripts
@@ -47,8 +45,8 @@ services:
 
   # for email services
   mailhog:
-    image: mailhog/mailhog
-    container_name: 'mailhog'
+    image: jcalonso/mailhog:latest
+    container_name: mailhog
     ports:
       - '1025:1025'
       - '8025:8025'

pThrottle/index.d.ts

@@ -0,0 +1,119 @@
+export class AbortError extends Error {
+  readonly name: 'AbortError';
+
+  private constructor();
+}
+
+type AnyFunction = (...arguments_: readonly any[]) => unknown;
+
+export type ThrottledFunction<F extends AnyFunction> = F & {
+  /**
+	Whether future function calls should be throttled or count towards throttling thresholds.
+
+	@default true
+	*/
+  isEnabled: boolean;
+
+  /**
+	The number of queued items waiting to be executed.
+	*/
+  readonly queueSize: number;
+
+  /**
+	Abort pending executions. All unresolved promises are rejected with a `pThrottle.AbortError` error.
+	*/
+  abort(): void;
+};
+
+export type Options = {
+  /**
+	The maximum number of calls within an `interval`.
+	*/
+  readonly limit: number;
+
+  /**
+	The timespan for `limit` in milliseconds.
+	*/
+  readonly interval: number;
+
+  /**
+	Use a strict, more resource intensive, throttling algorithm. The default algorithm uses a windowed approach that will work correctly in most cases, limiting the total number of calls at the specified limit per interval window. The strict algorithm throttles each call individually, ensuring the limit is not exceeded for any interval.
+
+	@default false
+	*/
+  readonly strict?: boolean;
+
+  /**
+	Get notified when function calls are delayed due to exceeding the `limit` of allowed calls within the given `interval`. The delayed call arguments are passed to the `onDelay` callback.
+
+ 	Can be useful for monitoring the throttling efficiency.
+
+	@example
+	```
+	import pThrottle from 'p-throttle';
+
+	const throttle = pThrottle({
+		limit: 2,
+		interval: 1000,
+		onDelay: (a, b) => {
+			console.log(`Reached interval limit, call is delayed for ${a} ${b}`);
+		},
+	});
+
+	const throttled = throttle((a, b) => {
+ 		console.log(`Executing with ${a} ${b}...`);
+   	});
+
+	await throttled(1, 2);
+	await throttled(3, 4);
+	await throttled(5, 6);
+	//=> Executing with 1 2...
+	//=> Executing with 3 4...
+	//=> Reached interval limit, call is delayed for 5 6
+	//=> Executing with 5 6...
+	```
+	*/
+  readonly onDelay?: (...arguments_: readonly any[]) => void;
+};
+
+/**
+Throttle promise-returning/async/normal functions.
+
+It rate-limits function calls without discarding them, making it ideal for external API interactions where avoiding call loss is crucial.
+
+@returns A throttle function.
+
+Both the `limit` and `interval` options must be specified.
+
+@example
+```
+import pThrottle from 'p-throttle';
+
+const now = Date.now();
+
+const throttle = pThrottle({
+	limit: 2,
+	interval: 1000
+});
+
+const throttled = throttle(async index => {
+	const secDiff = ((Date.now() - now) / 1000).toFixed();
+	return `${index}: ${secDiff}s`;
+});
+
+for (let index = 1; index <= 6; index++) {
+	(async () => {
+		console.log(await throttled(index));
+	})();
+}
+//=> 1: 0s
+//=> 2: 0s
+//=> 3: 1s
+//=> 4: 1s
+//=> 5: 2s
+//=> 6: 2s
+```
+*/
+export default function pThrottle(
+  options: Options,
+): <F extends AnyFunction>(function_: F) => ThrottledFunction<F>;