training updates april-may (#97)

* add diagram, fix typo
* random int suffix for secrets and buckets
* update taps config
* fix for empty policy document
* fix install terraform-docs

.github/workflows/terraform-ci-cd.yml

@@ -32,7 +32,7 @@ jobs:
           go-version: "1.14" # The Go version to download (if necessary) and use.
       - name: "Terraform Code Standards Check"
         run: |
-          GO111MODULE="off" go get github.com/segmentio/terraform-docs
+          GO111MODULE="on" go get github.com/segmentio/terraform-docs
           export PATH=$HOME/go/bin:$PATH
           pip3 install --pre slalom.dataops
           s-infra check_tf_metadata ./catalog \

README.md

@@ -12,7 +12,7 @@ For information on how to request enhancements, submit bug reports, or contribut
 
 ### Workstation Setup
 
-1. Follow the steps in [Windows Development Quickstart](https://docs.dataops.tk/docs/windows_development.html) or [Mac Development Quistart](https://docs.dataops.tk/docs/mac_development.html), which will automatically install all of the following required tools: Terraform, Docker, VS Code, Python 3, and Git.
+1. Follow the steps in [Windows Development QuickStart](https://docs.dataops.tk/setup/windows.html) or [Mac Development Quistart](https://docs.dataops.tk/setup/mac.html), which will automatically install all of the following required tools: Terraform, Docker, VS Code, Python 3, and Git.
 2. Clone this repo to your local machine.
 
 ### Deploying from the Infrastructure Catalog

catalog/aws/data-lake/main.tf

@@ -13,7 +13,7 @@ data aws_s3_bucket "data_bucket_override" {
 
 locals {
   s3_path_to_lambda_zip = "s3://${aws_s3_bucket.s3_metadata_bucket.id}/code/lambda/${var.name_prefix}lambda.zip"
-  random_bucket_suffix  = lower(random_id.suffix.hex)
+  random_bucket_suffix  = lower(random_id.suffix.dec)
   data_bucket_name = (
     var.data_bucket_override != null ? data.aws_s3_bucket.data_bucket_override[0].id : aws_s3_bucket.s3_data_bucket[0].id
   )

catalog/aws/singer-taps/main.tf

@@ -14,13 +14,6 @@ locals {
     1 / 0 # ERROR: currently supported timezone code are: "UTC", "GMT", "EST", "PST" and "PDT"
   )
   name_prefix = "${var.name_prefix}Tap-"
-  container_image = coalesce(
-    var.container_image, "slalomggp/singer:${var.taps[0].id}-to-${local.target.id}"
-  )
-  sync_commands = [
-    for tap in var.taps :
-    "s-tap sync ${tap.id} ${local.target.id}"
-  ]
   container_command = (
     length(local.sync_commands) == 1 ? local.sync_commands[0] :
     chomp(coalesce(var.container_command,
@@ -30,7 +23,7 @@ EOF
     ))
   )
   target = (
-    var.data_lake_type == "S3" ?
+    (var.data_lake_type == "S3") || (var.target == null) ?
     {
       id = "s3-csv"
       settings = {
@@ -56,7 +49,13 @@ EOF
     } :
     var.target
   )
-
+  container_image = coalesce(
+    var.container_image, "slalomggp/singer:${var.taps[0].id}-to-${local.target.id}"
+  )
+  sync_commands = [
+    for tap in var.taps :
+    "s-tap sync ${tap.id} ${local.target.id}"
+  ]
 }
 
 module "ecs_cluster" {