diff --git a/.github/workflows/terraform-ci-cd.yml b/.github/workflows/terraform-ci-cd.yml index b0d1d8c9..4d70212a 100644 --- a/.github/workflows/terraform-ci-cd.yml +++ b/.github/workflows/terraform-ci-cd.yml @@ -32,7 +32,7 @@ jobs: go-version: "1.14" # The Go version to download (if necessary) and use. - name: "Terraform Code Standards Check" run: | - GO111MODULE="off" go get github.com/segmentio/terraform-docs + GO111MODULE="on" go get github.com/segmentio/terraform-docs export PATH=$HOME/go/bin:$PATH pip3 install --pre slalom.dataops s-infra check_tf_metadata ./catalog \ diff --git a/README.md b/README.md index 4bace3b4..5257b839 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ For information on how to request enhancements, submit bug reports, or contribut ### Workstation Setup -1. Follow the steps in [Windows Development Quickstart](https://docs.dataops.tk/docs/windows_development.html) or [Mac Development Quistart](https://docs.dataops.tk/docs/mac_development.html), which will automatically install all of the following required tools: Terraform, Docker, VS Code, Python 3, and Git. +1. Follow the steps in [Windows Development QuickStart](https://docs.dataops.tk/setup/windows.html) or [Mac Development Quistart](https://docs.dataops.tk/setup/mac.html), which will automatically install all of the following required tools: Terraform, Docker, VS Code, Python 3, and Git. 2. Clone this repo to your local machine. ### Deploying from the Infrastructure Catalog diff --git a/catalog/aws/data-lake/main.tf b/catalog/aws/data-lake/main.tf index 1536630c..00fa71ce 100644 --- a/catalog/aws/data-lake/main.tf +++ b/catalog/aws/data-lake/main.tf @@ -13,7 +13,7 @@ data aws_s3_bucket "data_bucket_override" { locals { s3_path_to_lambda_zip = "s3://${aws_s3_bucket.s3_metadata_bucket.id}/code/lambda/${var.name_prefix}lambda.zip" - random_bucket_suffix = lower(random_id.suffix.hex) + random_bucket_suffix = lower(random_id.suffix.dec) data_bucket_name = ( var.data_bucket_override != null ? data.aws_s3_bucket.data_bucket_override[0].id : aws_s3_bucket.s3_data_bucket[0].id ) diff --git a/catalog/aws/environment/diagram.png b/catalog/aws/environment/diagram.png new file mode 100644 index 00000000..26ff1f83 Binary files /dev/null and b/catalog/aws/environment/diagram.png differ diff --git a/catalog/aws/environment/diagram.svg b/catalog/aws/environment/diagram.svg new file mode 100644 index 00000000..cc4e50dd --- /dev/null +++ b/catalog/aws/environment/diagram.svg @@ -0,0 +1,324 @@ + + + + + + + + + + + + + + image/svg+xml + + + + + + + + VPC + + + + + + NATGateway + + + InternetGateway + + NAT Elastic IP + + Public RouteTable + + PrivateRoute Table + Public Subnet A - 10.0.2.0/24 ( ( ( ( (10.0.3.0)()(9 Public Subnet A - 10.0.3.0/24 ( ( ( ( (10.0.3.0)()(9 Private Subnet A - 10.0.0.0/24 ( ( ( ( (10.0.3.0)()(9 Private Subnet B - 10.0.1.0/24 ( ( ( ( (10.0.3.0)()(9 + diff --git a/catalog/aws/singer-taps/main.tf b/catalog/aws/singer-taps/main.tf index 1313cc84..f18c10eb 100644 --- a/catalog/aws/singer-taps/main.tf +++ b/catalog/aws/singer-taps/main.tf @@ -14,13 +14,6 @@ locals { 1 / 0 # ERROR: currently supported timezone code are: "UTC", "GMT", "EST", "PST" and "PDT" ) name_prefix = "${var.name_prefix}Tap-" - container_image = coalesce( - var.container_image, "slalomggp/singer:${var.taps[0].id}-to-${local.target.id}" - ) - sync_commands = [ - for tap in var.taps : - "s-tap sync ${tap.id} ${local.target.id}" - ] container_command = ( length(local.sync_commands) == 1 ? local.sync_commands[0] : chomp(coalesce(var.container_command, @@ -30,7 +23,7 @@ EOF )) ) target = ( - var.data_lake_type == "S3" ? + (var.data_lake_type == "S3") || (var.target == null) ? { id = "s3-csv" settings = { @@ -56,7 +49,13 @@ EOF } : var.target ) - + container_image = coalesce( + var.container_image, "slalomggp/singer:${var.taps[0].id}-to-${local.target.id}" + ) + sync_commands = [ + for tap in var.taps : + "s-tap sync ${tap.id} ${local.target.id}" + ] } module "ecs_cluster" { diff --git a/components/aws/lambda-python/iam.tf b/components/aws/lambda-python/iam.tf index 328dbfa5..f2f2cb02 100644 --- a/components/aws/lambda-python/iam.tf +++ b/components/aws/lambda-python/iam.tf @@ -30,7 +30,7 @@ resource "aws_lambda_permission" "allow_bucket_to_trigger_lambda" { } data "aws_iam_policy_document" "lambda_s3_access_policy" { - count = var.s3_triggers == null ? 0 : 1 + count = local.has_s3_triggers ? 1 : 0 version = "2012-10-17" statement { effect = "Allow" @@ -55,7 +55,7 @@ data "aws_iam_policy_document" "lambda_s3_access_policy" { } resource "aws_iam_policy" "lambda_s3_access" { - count = var.s3_triggers == null ? 0 : 1 + count = local.has_s3_triggers ? 1 : 0 name = "${var.name_prefix}lambda_s3_access-${local.random_suffix}" path = "/" description = "IAM policy for accessing S3 from a lambda" @@ -63,7 +63,7 @@ resource "aws_iam_policy" "lambda_s3_access" { } resource "aws_iam_role_policy_attachment" "lambda_s3_access" { - count = var.s3_triggers == null ? 0 : 1 + count = local.has_s3_triggers ? 1 : 0 role = aws_iam_role.iam_for_lambda.name policy_arn = aws_iam_policy.lambda_s3_access[0].arn } diff --git a/components/aws/lambda-python/main.tf b/components/aws/lambda-python/main.tf index 27d5fd98..73689027 100644 --- a/components/aws/lambda-python/main.tf +++ b/components/aws/lambda-python/main.tf @@ -11,10 +11,11 @@ resource "random_id" "suffix" { } locals { - is_disabled = length(var.functions) == 0 ? true : false - is_windows = substr(pathexpand("~"), 0, 1) == "/" ? false : true - random_suffix = lower(random_id.suffix.hex) - function_names = toset(keys(var.functions)) + is_disabled = length(var.functions) == 0 ? true : false + has_s3_triggers = var.s3_triggers == null ? false : (length(var.s3_triggers) > 1 ? true : false) + is_windows = substr(pathexpand("~"), 0, 1) == "/" ? false : true + random_suffix = lower(random_id.suffix.hex) + function_names = toset(keys(var.functions)) function_secrets = { for name in local.function_names : name => var.functions[name].secrets diff --git a/components/aws/secrets-manager/main.tf b/components/aws/secrets-manager/main.tf index 83cf2c94..8f608632 100644 --- a/components/aws/secrets-manager/main.tf +++ b/components/aws/secrets-manager/main.tf @@ -11,6 +11,8 @@ * */ +resource "random_id" "suffix" { byte_length = 2 } + locals { secrets_names = toset(keys(var.secrets_map)) existing_secrets_ids_map = { @@ -39,7 +41,7 @@ locals { resource "aws_secretsmanager_secret" "secrets" { for_each = toset(keys(local.new_secrets_map)) - name = "${var.name_prefix}${each.key}" + name = "${var.name_prefix}${each.key}-${random_id.suffix.dec}" kms_key_id = var.kms_key_id } diff --git a/components/aws/vpc/main.tf b/components/aws/vpc/main.tf index 0b240e01..08a12cb4 100644 --- a/components/aws/vpc/main.tf +++ b/components/aws/vpc/main.tf @@ -5,7 +5,7 @@ * * 1 VPC which contains the following: * * 2 private subnets (for resources which **do not** need a public IP address) * * 2 public subnets (for resources which do need a public IP address) -* * 1 NAT gateway (allows private sugnet resources to reach the outside world) +* * 1 NAT gateway (allows private subnet resources to reach the outside world) * * 1 Intenet gateway (allows resources in public and private subnets to reach the internet) * * route tables and routes to connect all of the above */ diff --git a/samples/infra-config.yml b/samples/infra-config.yml index f8c41221..703ce758 100644 --- a/samples/infra-config.yml +++ b/samples/infra-config.yml @@ -1,8 +1,4 @@ -# This config file is created by 'aws-prereqs' terraform scripts. -# Please reference this file in future terraform deployments. -# NOTE: 1. Changing the project shortname also requires updating the credentials file: ~/.aws/credentials -# 2. Before changing access keys to a new account, remember to run `terraform destroy` aws_region: us-east-1 -project_shortname: employee-attrition +project_shortname: TestProject resource_tags: - admin_contact: jack.sandom@slalom.com + admin_contact: my.admin@sample.com