diff --git a/framework/API.md b/framework/API.md index 372933a10..556544a47 100644 --- a/framework/API.md +++ b/framework/API.md @@ -8213,6 +8213,9 @@ new utils.Utils() | **Name** | **Description** | | --- | --- | +| generateHash | Generate an 8 character hash from a string based on HMAC algorithm. | +| generateScopeHash | Generate an 8 characters hash of the CDK scope using its path. | +| generateScopeIdHash | Generate an 8 characters hash of the CDK scope and ID using its path. | | loadYaml | Take a document stored as string and load it as YAML. | | randomize | Create a random string to be used as a seed for IAM User password. | | readYamlDocument | Read a YAML file from the path provided and return it. | @@ -8221,6 +8224,68 @@ new utils.Utils() --- +##### `generateHash` + +```typescript +import { utils } from 'aws-dsf' + +utils.Utils.generateHash(text: string) +``` + +Generate an 8 character hash from a string based on HMAC algorithm. + +###### `text`Required + +- *Type:* string + +the text to hash. + +--- + +##### `generateScopeHash` + +```typescript +import { utils } from 'aws-dsf' + +utils.Utils.generateScopeHash(scope: Construct) +``` + +Generate an 8 characters hash of the CDK scope using its path. + +###### `scope`Required + +- *Type:* constructs.Construct + +the CDK construct scope. + +--- + +##### `generateScopeIdHash` + +```typescript +import { utils } from 'aws-dsf' + +utils.Utils.generateScopeIdHash(scope: Construct, id: string) +``` + +Generate an 8 characters hash of the CDK scope and ID using its path. + +###### `scope`Required + +- *Type:* constructs.Construct + +the CDK construct scope. + +--- + +###### `id`Required + +- *Type:* string + +the id of the construct. + +--- + ##### `loadYaml` ```typescript diff --git a/framework/src/governance/lib/data-catalog-database.ts b/framework/src/governance/lib/data-catalog-database.ts index a243f2391..cd6eb3a8b 100644 --- a/framework/src/governance/lib/data-catalog-database.ts +++ b/framework/src/governance/lib/data-catalog-database.ts @@ -1,13 +1,13 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT-0 -import { Names, Stack } from 'aws-cdk-lib'; +import { Stack } from 'aws-cdk-lib'; import { CfnCrawler, CfnDatabase, CfnSecurityConfiguration } from 'aws-cdk-lib/aws-glue'; import { AddToPrincipalPolicyResult, Effect, IPrincipal, IRole, PolicyDocument, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { IKey, Key } from 'aws-cdk-lib/aws-kms'; import { Construct } from 'constructs'; import { DataCatalogDatabaseProps } from './data-catalog-database-props'; -import { Context, TrackedConstruct, TrackedConstructProps } from '../../utils'; +import { Context, TrackedConstruct, TrackedConstructProps, Utils } from '../../utils'; /** * An AWS Glue Data Catalog Database configured with the location and a crawler. @@ -65,9 +65,10 @@ export class DataCatalogDatabase extends TrackedConstruct { super(scope, id, trackedConstructProps); this.dataCatalogDatabaseProps = props; - const removalPolicy = Context.revertRemovalPolicy(scope, props.removalPolicy); + const removalPolicy = Context.revertRemovalPolicy(this, props.removalPolicy); - this.databaseName = props.name + '_' + Names.uniqueResourceName(scope, {}).toLowerCase(); + const hash = Utils.generateScopeHash(this); + this.databaseName = props.name + '_' + hash.toLowerCase(); let locationPrefix = props.locationPrefix; @@ -155,7 +156,7 @@ export class DataCatalogDatabase extends TrackedConstruct { this.crawlerLogEncryptionKey.grantEncryptDecrypt(crawlerRole); this.crawlerSecurityConfiguration = new CfnSecurityConfiguration(this, 'CrawlerSecConfiguration', { - name: `${props.name}-secconfig-${Names.uniqueResourceName(this, {}).toLowerCase()}`, + name: `${props.name}-${hash.toLowerCase()}-secconfig`, encryptionConfiguration: { cloudWatchEncryption: { cloudWatchEncryptionMode: 'SSE-KMS', @@ -169,7 +170,7 @@ export class DataCatalogDatabase extends TrackedConstruct { }, }); - const crawlerName = `${this.databaseName}-crawler-${Names.uniqueResourceName(this, {})}`; + const crawlerName = `${props.name}-${hash.toLowerCase()}-crawler`; this.crawler = new CfnCrawler(this, 'DatabaseAutoCrawler', { role: crawlerRole.roleArn, targets: { diff --git a/framework/src/processing/lib/spark-job/spark-job.ts b/framework/src/processing/lib/spark-job/spark-job.ts index 477a5de27..2987e95c6 100644 --- a/framework/src/processing/lib/spark-job/spark-job.ts +++ b/framework/src/processing/lib/spark-job/spark-job.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT-0 -import { Duration, Names, RemovalPolicy } from 'aws-cdk-lib'; +import { Duration, RemovalPolicy } from 'aws-cdk-lib'; import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; import { SfnStateMachine } from 'aws-cdk-lib/aws-events-targets'; import { IRole } from 'aws-cdk-lib/aws-iam'; @@ -12,7 +12,7 @@ import { Choice, Condition, DefinitionBody, Fail, FailProps, LogLevel, StateMach import { CallAwsService, CallAwsServiceProps } from 'aws-cdk-lib/aws-stepfunctions-tasks'; import { Construct } from 'constructs'; import { SparkJobProps } from './spark-job-props'; -import { Context, TrackedConstruct, TrackedConstructProps } from '../../../utils'; +import { Context, TrackedConstruct, TrackedConstructProps, Utils } from '../../../utils'; /** * A base construct to run Spark Jobs. @@ -155,7 +155,7 @@ export abstract class SparkJob extends TrackedConstruct { // Enable CloudWatch Logs for the state machine this.stateMachineLogGroup = new LogGroup(this, 'LogGroup', { removalPolicy: this.removalPolicy, - logGroupName: `/aws/vendedlogs/states/${Names.uniqueId(this)}`, + logGroupName: `/aws/vendedlogs/states/${Utils.generateScopeHash(this)}`, }); // StepFunctions state machine diff --git a/framework/src/utils/lib/bucket-utils.ts b/framework/src/utils/lib/bucket-utils.ts index 7138bada8..2fe0f7928 100644 --- a/framework/src/utils/lib/bucket-utils.ts +++ b/framework/src/utils/lib/bucket-utils.ts @@ -1,9 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT-0 -import { createHmac } from 'crypto'; import { Aws } from 'aws-cdk-lib'; import { Construct } from 'constructs'; +import { Utils } from './utils'; /** * Utils for working with Amazon S3 buckets. @@ -23,25 +23,6 @@ export class BucketUtils { if (name.length > 26) { throw new Error('Bucket name is too long, maximum length for bucketName is 26'); } - return name + '-' + Aws.ACCOUNT_ID + '-' + Aws.REGION + '-' + BucketUtils.generateHash(scope, id); - } - - /** - * Generate an 8 characters hash of the CDK scope using its path. - * @param scope the CDK construct scope - * @returns the hash - */ - private static generateHash(scope: Construct, id: string): string { - const node = scope.node; - - const components = node.scopes.slice(1).map(c => c.node.id).join('-').concat(id); - - const secret = 'Data Solutions Framework on AWS'; - const hash = createHmac('sha256', secret) - .update(components) - .digest('hex') - .slice(0, 8); - - return hash; + return name + '-' + Aws.ACCOUNT_ID + '-' + Aws.REGION + '-' + Utils.generateScopeIdHash(scope, id); } } \ No newline at end of file diff --git a/framework/src/utils/lib/utils.ts b/framework/src/utils/lib/utils.ts index daf3afe5c..d23ab555a 100644 --- a/framework/src/utils/lib/utils.ts +++ b/framework/src/utils/lib/utils.ts @@ -1,6 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT-0 + +import { createHmac } from 'crypto'; import * as fs from 'fs'; +import { Construct } from 'constructs'; import * as yaml from 'js-yaml'; /** @@ -69,4 +72,42 @@ export class Utils { return text; } + /** + * Generate an 8 character hash from a string based on HMAC algorithm + * @param {string} text the text to hash + * @return {string} the hash + */ + public static generateHash(text: string): string { + return createHmac('sha256', 'Data Solutions Framework on AWS') + .update(text) + .digest('hex') + .slice(0, 8); + } + + /** + * Generate an 8 characters hash of the CDK scope and ID using its path. + * @param {Construct} scope the CDK construct scope + * @param {string} id the id of the construct + * @returns {string} the hash + */ + public static generateScopeIdHash(scope: Construct, id: string): string { + const node = scope.node; + + const components = node.scopes.slice(1).map(c => c.node.id).join('-').concat(id); + + return this.generateHash(components); + } + + /** + * Generate an 8 characters hash of the CDK scope using its path. + * @param {Construct} scope the CDK construct scope + * @returns {string} the hash + */ + public static generateScopeHash(scope: Construct): string { + const node = scope.node; + + const components = node.scopes.slice(1).map(c => c.node.id).join('-'); + + return this.generateHash(components); + } } diff --git a/framework/src/utils/lib/vpc-helper.ts b/framework/src/utils/lib/vpc-helper.ts index d04b8191f..f3200a2bb 100644 --- a/framework/src/utils/lib/vpc-helper.ts +++ b/framework/src/utils/lib/vpc-helper.ts @@ -1,13 +1,14 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT-0 -import { Names, RemovalPolicy, Stack, Tags } from 'aws-cdk-lib'; +import { RemovalPolicy, Stack, Tags } from 'aws-cdk-lib'; import { FlowLogDestination, GatewayVpcEndpoint, GatewayVpcEndpointAwsService, IVpc, IVpcEndpoint, IpAddresses, SubnetType, Vpc } from 'aws-cdk-lib/aws-ec2'; import { Effect, IRole, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { IKey } from 'aws-cdk-lib/aws-kms'; import { ILogGroup, LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; import { Construct } from 'constructs'; import { Context } from './context'; +import { Utils } from './utils'; /** * A network configuration created by the vpcBootstrap function. @@ -76,7 +77,7 @@ export function vpcBootstrap( }); //Create a loggroup name based on the purpose of the VPC, either used by emr on eks or emr serverless app - const logGroupName = eksClusterName ? `/aws/emr-eks-vpc-flow/${eksClusterName}`:`/aws/emr-serverless-vpc/${Names.nodeUniqueId(scope.node)}` ; + const logGroupName = eksClusterName ? `/aws/emr-eks-vpc-flow/${eksClusterName}`:`/aws/emr-serverless-vpc/${Utils.generateScopeHash(scope)}` ; const logGroupResourceId = eksClusterName ? 'EmrEksVpcFlowLog' : 'EmrServerlessVpcFlowLog' ; diff --git a/framework/test/unit/governance/data-catalog-database.test.ts b/framework/test/unit/governance/data-catalog-database.test.ts index bf1b130e9..e81ad4717 100644 --- a/framework/test/unit/governance/data-catalog-database.test.ts +++ b/framework/test/unit/governance/data-catalog-database.test.ts @@ -279,7 +279,7 @@ describe('DataCatalogDatabase default construct', () => { { Ref: 'AWS::AccountId', }, - ':database/sample_stack', + Match.stringLikeRegexp('database\/sample_.*'), ], ], }, @@ -295,7 +295,7 @@ describe('DataCatalogDatabase default construct', () => { { Ref: 'AWS::AccountId', }, - ':table/sample_stack/*', + Match.stringLikeRegexp('table\/sample_.*\/\*'), ], ], },