From 868fedb9733c8b3bce61d30afcd80c50f7bfab3b Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Thu, 15 Dec 2022 16:57:52 -0500 Subject: [PATCH 01/16] Clarify dev.tfbackend instructions; update gitignore --- .gitignore | 11 ++++++++++- README.md | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 660a6f7..7856188 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,16 @@ **/node_modules **/package-lock.json +# Lock file +.terraform.lock.hcl + +# SSH keys +*.pem + +# Swap files +*.sw? + .env applications/**/Dockerrun.aws.json stack/applications/**/Dockerrun.aws.json -batch \ No newline at end of file +batch diff --git a/README.md b/README.md index 0abfebd..a3984e7 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ The goal of this solution is to provide a simple, cost-effective way to put Aval 1. Download and install [Terraform 0.12+](https://www.terraform.io/downloads.html). The scripts have been upgraded to HCL 2 and therefore incompatible with earlier versions of Terraform. 1. Clone this repo 1. Create or import an [EC2 key-pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) for your region. -1. Create an S3 bucket to hold the terraform state, this is useful when +1. Create an S3 bucket to hold the terraform state file. This is useful when executing terraform on multiple machines (or working as a team) because it allows state to remain in sync. -1. Copy `dev.tfbackend.example` to `dev.tfbackend` and fill in the previously created bucket name. +1. Create a file `dev.tfbackend` and fill in the previously created bucket name, its region, and a bucket key for where the state file file be stored. ``` bucket = "my-terraform-state" From a0753eb7021b4e701a0b8d234008facc3e8811f9 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Tue, 13 Dec 2022 03:10:06 -0500 Subject: [PATCH 02/16] Allow specifying custom CIDR ranges for inbound SSH --- compose.tf | 2 +- terraform.tfvars.example | 1 + variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compose.tf b/compose.tf index c041fca..3f1de6d 100644 --- a/compose.tf +++ b/compose.tf @@ -128,7 +128,7 @@ resource "aws_security_group_rule" "compose_ssh" { from_port = "22" to_port = "22" protocol = "tcp" - cidr_blocks = [var.vpc_cidr_block] + cidr_blocks = setunion([var.vpc_cidr_block], var.ssh_cidr_blocks) } resource "aws_security_group_rule" "compose_egress" { diff --git a/terraform.tfvars.example b/terraform.tfvars.example index f58ee18..c81d874 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -5,6 +5,7 @@ ec2_private_keyfile = "/local/path/my-ec2-key.pem" stack_name = "mystack" stack_bucket = "mystack-state" sms_notification = "+18125550123" +ssh_cidr_blocks = [] fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" fcrepo_binary_bucket_secret_key = "***********" diff --git a/variables.tf b/variables.tf index 7394151..0ac3609 100644 --- a/variables.tf +++ b/variables.tf @@ -143,6 +143,12 @@ variable "postgres_version" { # type = string #} +variable "ssh_cidr_blocks" { + description = "Allow inbound SSH connections from given CIDR ranges" + type = list(string) + default = [] +} + variable "stack_name" { default = "stack" } From 0f13b6c246cb4a2cdb0eb1cfa957ad6efb65fa54 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Thu, 15 Dec 2022 17:00:14 -0500 Subject: [PATCH 03/16] Remove unused vars; fix typo in terraform.tfvars --- README.md | 1 - terraform.tfvars.example | 4 +--- variables.tf | 14 -------------- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/README.md b/README.md index a3984e7..f3c55e0 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ The goal of this solution is to provide a simple, cost-effective way to put Aval ec2_keyname = "my-ec2-key" ec2_private_keyfile = "/local/path/my-ec2-key.pem" stack_name = "mystack" - sms_notification = "+18125550123" fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" fcrepo_binary_bucket_secret_key = "***********" diff --git a/terraform.tfvars.example b/terraform.tfvars.example index c81d874..000a981 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -3,13 +3,11 @@ hosted_zone_name = "mydomain.org" ec2_keyname = "my-ec2-key" ec2_private_keyfile = "/local/path/my-ec2-key.pem" stack_name = "mystack" -stack_bucket = "mystack-state" -sms_notification = "+18125550123" ssh_cidr_blocks = [] fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" fcrepo_binary_bucket_secret_key = "***********" -tags { +tags = { Creator = "Username" For = "Avalon Turnkey" } diff --git a/variables.tf b/variables.tf index 0ac3609..2d867aa 100644 --- a/variables.tf +++ b/variables.tf @@ -153,20 +153,6 @@ variable "stack_name" { default = "stack" } -variable "stack_bucket" { - type = string -} - -variable "stack_key" { - type = string - default = "stack.tfstate" -} - -variable "stack_region" { - type = string - default = "us-east-1" -} - variable "tags" { type = map(string) default = {} From accfd7f965183dafdcbd1d50fb998d812490701a Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Thu, 15 Dec 2022 17:02:03 -0500 Subject: [PATCH 04/16] Optional zone_prefix for hosted_zone_names; output public_zone_name --- outputs.tf | 4 ++++ terraform.tfvars.example | 1 + variables.tf | 10 ++++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/outputs.tf b/outputs.tf index a17af27..b238b61 100644 --- a/outputs.tf +++ b/outputs.tf @@ -126,6 +126,10 @@ output "private_zone_id" { value = module.dns.private_zone_id } +output "public_zone_name" { + value = local.public_zone_name +} + output "public_subnets" { value = module.vpc.public_subnets } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 000a981..c8c0aef 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,4 +1,5 @@ environment = "dev" +#zone_prefix = "" hosted_zone_name = "mydomain.org" ec2_keyname = "my-ec2-key" ec2_private_keyfile = "/local/path/my-ec2-key.pem" diff --git a/variables.tf b/variables.tf index 2d867aa..9500cee 100644 --- a/variables.tf +++ b/variables.tf @@ -172,10 +172,16 @@ variable "vpc_private_subnets" { default = ["10.1.1.0/24", "10.1.3.0/24", "10.1.5.0/24"] } +variable "zone_prefix" { + description = "An optional prefix string to the hosted zone names" + type = string + default = "" +} + locals { namespace = "${var.stack_name}-${var.environment}" - public_zone_name = "${var.environment}.${var.hosted_zone_name}" - private_zone_name = "vpc.${var.environment}.${var.hosted_zone_name}" + public_zone_name = "${var.zone_prefix}${var.environment}.${var.hosted_zone_name}" + private_zone_name = "vpc.${var.zone_prefix}${var.environment}.${var.hosted_zone_name}" common_tags = merge( var.tags, From 8fdea0dcb1574edaea787c0a3003aa6342411a98 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Tue, 13 Dec 2022 13:50:27 -0500 Subject: [PATCH 05/16] Remove unused EC2 private key --- README.md | 1 - outputs.tf | 4 ---- terraform.tfvars.example | 1 - variables.tf | 4 ---- 4 files changed, 10 deletions(-) diff --git a/README.md b/README.md index f3c55e0..cce2df3 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ The goal of this solution is to provide a simple, cost-effective way to put Aval environment = "dev" hosted_zone_name = "mydomain.org" ec2_keyname = "my-ec2-key" - ec2_private_keyfile = "/local/path/my-ec2-key.pem" stack_name = "mystack" fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" diff --git a/outputs.tf b/outputs.tf index b238b61..4f3ba98 100644 --- a/outputs.tf +++ b/outputs.tf @@ -12,10 +12,6 @@ output "ec2_keyname" { value = var.ec2_keyname } -output "ec2_private_keyfile" { - value = var.ec2_private_keyfile -} - output "environment" { value = var.environment } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index c8c0aef..ea05e45 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -2,7 +2,6 @@ environment = "dev" #zone_prefix = "" hosted_zone_name = "mydomain.org" ec2_keyname = "my-ec2-key" -ec2_private_keyfile = "/local/path/my-ec2-key.pem" stack_name = "mystack" ssh_cidr_blocks = [] fcrepo_binary_bucket_username = "iam_user" diff --git a/variables.tf b/variables.tf index 9500cee..d412fec 100644 --- a/variables.tf +++ b/variables.tf @@ -93,10 +93,6 @@ variable "ec2_keyname" { type = string } -variable "ec2_private_keyfile" { - type = string -} - variable "email_comments" { type = string } From 811743db2e0fa1d980ee439aeb73d1fee8679f5d Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Thu, 15 Dec 2022 13:48:02 -0500 Subject: [PATCH 06/16] Set force_delete for ECR to allow terraform destroy cmd --- build.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/build.tf b/build.tf index 55496a2..8350c1d 100644 --- a/build.tf +++ b/build.tf @@ -88,6 +88,7 @@ POLICY resource "aws_ecr_repository" "avalon" { name = "avalon-${var.environment}" image_tag_mutability = "MUTABLE" + force_delete = true tags = local.common_tags } From dbe0a25cfe648e07efd4a642a19ec08a2da39d81 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Fri, 16 Dec 2022 02:10:14 -0500 Subject: [PATCH 07/16] Loadbalancer SG to only allow 80 and 443 --- alb.tf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/alb.tf b/alb.tf index 0917ac1..621f246 100644 --- a/alb.tf +++ b/alb.tf @@ -22,10 +22,19 @@ resource "aws_security_group" "alb" { tags = local.common_tags } -resource "aws_security_group_rule" "alb_ingress" { +resource "aws_security_group_rule" "alb_ingress_http" { security_group_id = aws_security_group.alb.id type = "ingress" from_port = "80" + to_port = "80" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "alb_ingress_https" { + security_group_id = aws_security_group.alb.id + type = "ingress" + from_port = "443" to_port = "443" protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] From 0ba168571af123cc9479eb137947982fe761d306 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Tue, 31 Jan 2023 23:53:15 -0500 Subject: [PATCH 08/16] Option to specify SSH public key string Allow public key string to be set as an alternative, or in addition to, the AWS key_pair --- README.md | 4 +++- compose.tf | 3 ++- outputs.tf | 4 ++++ scripts/compose-init.sh | 11 +++++++++++ terraform.tfvars.example | 4 +++- variables.tf | 8 ++++++++ 6 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cce2df3..c17bcf7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ The goal of this solution is to provide a simple, cost-effective way to put Aval 1. Download and install [Terraform 0.12+](https://www.terraform.io/downloads.html). The scripts have been upgraded to HCL 2 and therefore incompatible with earlier versions of Terraform. 1. Clone this repo -1. Create or import an [EC2 key-pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) for your region. +1. Get a public key ready which will be used to access your EC2 instance. It can be either (or both) of: + * An [EC2 key-pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) created or imported for your region. + * A local SSH public key [generated manually](https://git-scm.com/book/en/v2/Git-on-the-Server-Generating-Your-SSH-Public-Key). 1. Create an S3 bucket to hold the terraform state file. This is useful when executing terraform on multiple machines (or working as a team) because it allows state to remain in sync. 1. Create a file `dev.tfbackend` and fill in the previously created bucket name, its region, and a bucket key for where the state file file be stored. diff --git a/compose.tf b/compose.tf index 3f1de6d..93e1cc9 100644 --- a/compose.tf +++ b/compose.tf @@ -152,7 +152,7 @@ resource "aws_security_group_rule" "allow_this_redis_access" { resource "aws_instance" "compose" { ami = data.aws_ami.amzn.id instance_type = var.compose_instance_type - key_name = var.ec2_keyname + key_name = var.ec2_keyname == "" ? null : var.ec2_keyname subnet_id = module.vpc.public_subnets[0] associate_public_ip_address = true availability_zone = var.availability_zone @@ -170,6 +170,7 @@ resource "aws_instance" "compose" { } user_data = base64encode(templatefile("scripts/compose-init.sh", { + ec2_public_key = "${var.ec2_public_key}" solr_backups_efs_id = "${aws_efs_file_system.solr_backups.id}" solr_backups_efs_dns_name = "${aws_efs_file_system.solr_backups.dns_name}" db_fcrepo_address = "${module.db_fcrepo.db_instance_address}" diff --git a/outputs.tf b/outputs.tf index 4f3ba98..470dce2 100644 --- a/outputs.tf +++ b/outputs.tf @@ -12,6 +12,10 @@ output "ec2_keyname" { value = var.ec2_keyname } +output "ec2_public_key" { + value = var.ec2_public_key +} + output "environment" { value = var.environment } diff --git a/scripts/compose-init.sh b/scripts/compose-init.sh index 1e6d2c4..f5ae7c7 100644 --- a/scripts/compose-init.sh +++ b/scripts/compose-init.sh @@ -1,5 +1,16 @@ #!/bin/bash +# Add SSH public key if var was set +if [[ -n "${ec2_public_key}" ]]; then + # But first ensure existance and correct permissions + sudo -Hu ec2-user bash <<- EOF + umask 0077 + mkdir -p /home/ec2-user/.ssh + touch /home/ec2-user/.ssh/authorized_keys + EOF + echo "${ec2_public_key}" >> /home/ec2-user/.ssh/authorized_keys +fi + # Create filesystem only if there isn't one if [[ ! `sudo file -s /dev/xvdh` == *"Linux"* ]]; then sudo mkfs -t ext4 /dev/xvdh diff --git a/terraform.tfvars.example b/terraform.tfvars.example index ea05e45..f02145d 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,7 +1,9 @@ environment = "dev" #zone_prefix = "" hosted_zone_name = "mydomain.org" -ec2_keyname = "my-ec2-key" +# At least one of ec2_keyname or ec2_public_key must be set +#ec2_keyname = "my-ec2-key" +#ec2_public_key = "" stack_name = "mystack" ssh_cidr_blocks = [] fcrepo_binary_bucket_username = "iam_user" diff --git a/variables.tf b/variables.tf index d412fec..d809c7a 100644 --- a/variables.tf +++ b/variables.tf @@ -91,6 +91,14 @@ variable "db_fcrepo_username" { variable "ec2_keyname" { type = string + default = "" + description = "The name of an AWS EC2 key pair to use for authenticating" +} + +variable "ec2_public_key" { + type = string + default = "" + description = "A SSH public key string to use for authenticating" } variable "email_comments" { From 5e7252f88f804d5da118a325312e07f32c959a34 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Wed, 1 Feb 2023 00:10:53 -0500 Subject: [PATCH 09/16] Allow SSH direct to instance public IP Only to ssh_cidr_blocks range, if set. Also, add public_ip to the output variables. --- compose.tf | 19 +++++++++++++++++++ outputs.tf | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/compose.tf b/compose.tf index 93e1cc9..780fa25 100644 --- a/compose.tf +++ b/compose.tf @@ -149,6 +149,24 @@ resource "aws_security_group_rule" "allow_this_redis_access" { source_security_group_id = aws_security_group.compose.id } +resource "aws_security_group" "public_ip" { + name = "${local.namespace}-ssh-public-ip" + description = "SSH Public IP Security Group" + tags = local.common_tags + vpc_id = module.vpc.vpc_id +} + +resource "aws_security_group_rule" "ssh_public_ip" { + type = "ingress" + description = "Allow SSH direct to public IP" + cidr_blocks = var.ssh_cidr_blocks + ipv6_cidr_blocks = [] + from_port = 22 + to_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.public_ip.id +} + resource "aws_instance" "compose" { ami = data.aws_ami.amzn.id instance_type = var.compose_instance_type @@ -214,6 +232,7 @@ resource "aws_instance" "compose" { vpc_security_group_ids = [ aws_security_group.compose.id, aws_security_group.db_client.id, + aws_security_group.public_ip.id, ] lifecycle { diff --git a/outputs.tf b/outputs.tf index 470dce2..a0027fa 100644 --- a/outputs.tf +++ b/outputs.tf @@ -126,6 +126,10 @@ output "private_zone_id" { value = module.dns.private_zone_id } +output "public_ip" { + value = aws_instance.compose.public_ip +} + output "public_zone_name" { value = local.public_zone_name } From fb96593b5d6f87d3ae50bec5119093d6782f978f Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Wed, 1 Feb 2023 00:12:14 -0500 Subject: [PATCH 10/16] Instance ignore user_data changes Prevent changes to the user_data script from triggering a complete replacement of the AWS instance. --- compose.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.tf b/compose.tf index 780fa25..d508388 100644 --- a/compose.tf +++ b/compose.tf @@ -236,7 +236,7 @@ resource "aws_instance" "compose" { ] lifecycle { - ignore_changes = [ami] + ignore_changes = [ami, user_data] } } From 6f8968d07da9e3b419a1202b5b2320c6a4cbb076 Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Thu, 2 Feb 2023 13:12:18 -0500 Subject: [PATCH 11/16] Can specify alternate hostname and zone for website Instead of using public_zone_name, setting alt_hostname will use that instead for accessing Avalon web interface. --- alb.tf | 28 +++++++++++++++++++++++++++- compose.tf | 2 +- terraform.tfvars.example | 10 ++++++++++ variables.tf | 19 +++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/alb.tf b/alb.tf index 621f246..2594b9d 100644 --- a/alb.tf +++ b/alb.tf @@ -178,6 +178,7 @@ resource "aws_alb_target_group" "alb_streaming" { resource "aws_acm_certificate" "web_cert" { domain_name = aws_route53_record.alb.fqdn validation_method = "DNS" + subject_alternative_names = [for ent in var.alt_hostname : ent.hostname] tags = local.common_tags @@ -194,9 +195,34 @@ resource "aws_route53_record" "web_cert_validation" { ttl = 60 } +resource "aws_route53_record" "alt_web_cert_validation" { + for_each = length(var.alt_hostname) > 0 ? var.alt_hostname : {} + name = tolist(aws_acm_certificate.web_cert.domain_validation_options)[1].resource_record_name + type = tolist(aws_acm_certificate.web_cert.domain_validation_options)[1].resource_record_type + zone_id = each.value.zone_id + records = [tolist(aws_acm_certificate.web_cert.domain_validation_options)[1].resource_record_value] + ttl = 60 +} + +data "dns_a_record_set" "alb_ips" { + host = aws_alb.alb.dns_name +} + +resource "aws_route53_record" "alt_dns_zone" { + for_each = length(var.alt_hostname) > 0 ? var.alt_hostname : {} + name = each.value.hostname + type = "A" + zone_id = each.value.zone_id + records = data.dns_a_record_set.alb_ips.addrs + ttl = 60 +} + resource "aws_acm_certificate_validation" "web_cert" { certificate_arn = aws_acm_certificate.web_cert.arn - validation_record_fqdns = [aws_route53_record.web_cert_validation.fqdn] + validation_record_fqdns = concat( + [aws_route53_record.web_cert_validation.fqdn], + [for record in aws_route53_record.alt_web_cert_validation : record.fqdn] + ) } # Create, validate and attach streaming certificate diff --git a/compose.tf b/compose.tf index d508388..4920294 100644 --- a/compose.tf +++ b/compose.tf @@ -211,7 +211,7 @@ resource "aws_instance" "compose" { avalon_repo = "${var.avalon_repo}" redis_host_name = "${aws_route53_record.redis.name}" aws_region = "${var.aws_region}" - avalon_fqdn = "${aws_route53_record.alb.fqdn}" + avalon_fqdn = "${length(var.alt_hostname) > 0 ? values(var.alt_hostname)[0].hostname : aws_route53_record.alb.fqdn}" streaming_fqdn = "${aws_route53_record.alb_streaming.fqdn}" elastictranscoder_pipeline_id = "${aws_elastictranscoder_pipeline.this_pipeline.id}" email_comments = "${var.email_comments}" diff --git a/terraform.tfvars.example b/terraform.tfvars.example index f02145d..e1fe4a6 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -9,6 +9,16 @@ ssh_cidr_blocks = [] fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" fcrepo_binary_bucket_secret_key = "***********" +# You can use an alternate hostname for the main web interface rather +# than the default of {zone_prefix}{environment}.{hosted_zone_name} +# To do so, you must delegate the domain where the hostname will reside +# to AWS Route53 and specify the zone_id and hostname like below. +#alt_hostname = { +# "my-label" = { +# zone_id = "Z0123456789ABCDEFGHI" +# hostname = "my-alt.domain.org" +# } +# } tags = { Creator = "Username" For = "Avalon Turnkey" diff --git a/variables.tf b/variables.tf index d809c7a..9da9759 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,22 @@ +variable "alt_hostname" { + description = "Specify an alternative hostname for the public website url (instead of public_zone_name)" + type = map(object({ + zone_id = string + hostname = string + })) + default = {} + # + # To use alt_hostname, you first need to delegate it as a DNS zone to Route53. + # AWS will then create appropriate DNS records. + # + # alt_hostname = { + # "my-zone" = { + # zone_id = "Z0123456789ABCDEFGHI" + # hostname = "my-alt.added.domain.edu" + # } + # } +} + variable "app_name" { default = "avalon" } From c7cc13564f9be7ee176fbd19636cbcb6fa9622df Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Tue, 14 Feb 2023 03:05:27 -0500 Subject: [PATCH 12/16] Auto-create fcrepo_binary_bucket_username is left blank. Will attempt to create a new user and access for the fcrepo_binary_bucket_username when var is left blank. Requires appropriate permissions to be able to create them, if not available, can still manually specify vars. --- README.md | 3 ++- compose.tf | 4 ++-- s3.tf | 15 ++++++++++++++- terraform.tfvars.example | 5 +++++ variables.tf | 6 ++++++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c17bcf7..22e574e 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The goal of this solution is to provide a simple, cost-effective way to put Aval key = "state.tfstate" region = "us-east-1" ```` -1. Create an IAM user that Fedora will use to sign its S3 requests. +1. (Optional) Create an IAM user that Fedora will use to sign its S3 requests. Terraform will create this user automatically if it has permissions to do so. 1. Create a [public hosted zone in Route53](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/CreatingHostedZone.html); Terraform will automatically manage DNS entries in this zone. A registered domain name is needed to pair with the Route53 hosted zone. You can [use Route53 to register a new domain](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html) or [use Route53 to manage an existing domain](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/MigratingDNS.html). 1. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in the relevant information: ``` @@ -32,6 +32,7 @@ The goal of this solution is to provide a simple, cost-effective way to put Aval hosted_zone_name = "mydomain.org" ec2_keyname = "my-ec2-key" stack_name = "mystack" + # Next 3 lines only if you created the IAM user manually fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" fcrepo_binary_bucket_secret_key = "***********" diff --git a/compose.tf b/compose.tf index 4920294..d347eb4 100644 --- a/compose.tf +++ b/compose.tf @@ -198,8 +198,8 @@ resource "aws_instance" "compose" { db_avalon_address = "${module.db_avalon.db_instance_address}" db_avalon_username = "${module.db_avalon.db_instance_username}" db_avalon_password = "${module.db_avalon.db_instance_password}" - fcrepo_binary_bucket_access_key = "${var.fcrepo_binary_bucket_access_key}" - fcrepo_binary_bucket_secret_key = "${var.fcrepo_binary_bucket_secret_key}" + fcrepo_binary_bucket_access_key = "${length(var.fcrepo_binary_bucket_username) > 0 ? var.fcrepo_binary_bucket_access_key : values(aws_iam_access_key.fcrepo_bin_created_access)[0].id }" + fcrepo_binary_bucket_secret_key = "${length(var.fcrepo_binary_bucket_username) > 0 ? var.fcrepo_binary_bucket_secret_key : values(aws_iam_access_key.fcrepo_bin_created_access)[0].secret }" fcrepo_binary_bucket_id = "${aws_s3_bucket.fcrepo_binary_bucket.id}" compose_log_group_name = "${aws_cloudwatch_log_group.compose_log_group.name}" fcrepo_db_ssl = "${var.fcrepo_db_ssl}" diff --git a/s3.tf b/s3.tf index 8df31c7..12fc818 100644 --- a/s3.tf +++ b/s3.tf @@ -157,9 +157,22 @@ data "aws_iam_policy_document" "fcrepo_binary_bucket_access" { } } +# Create fcrepo bucket user if none was provided +resource "aws_iam_user" "fcrepo_bin_created_user" { + for_each = length(var.fcrepo_binary_bucket_username) > 0 ? toset([]) : toset(["fcuser"]) + name = "fcrepo-avalon-${local.namespace}" + tags = local.common_tags +} + +# Create user access and secret ids if user was created +resource "aws_iam_access_key" "fcrepo_bin_created_access" { + for_each = length(var.fcrepo_binary_bucket_username) > 0 ? toset([]) : toset(["fcuser"]) + user = values(aws_iam_user.fcrepo_bin_created_user)[0].name +} + resource "aws_iam_user_policy" "fcrepo_binary_bucket_policy" { name = "${local.namespace}-fcrepo-s3-bucket-access" - user = var.fcrepo_binary_bucket_username + user = length(var.fcrepo_binary_bucket_username) > 0 ? var.fcrepo_binary_bucket_username : values(aws_iam_user.fcrepo_bin_created_user)[0].name policy = data.aws_iam_policy_document.fcrepo_binary_bucket_access.json } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index e1fe4a6..e9bb843 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -6,6 +6,11 @@ hosted_zone_name = "mydomain.org" #ec2_public_key = "" stack_name = "mystack" ssh_cidr_blocks = [] +# If the user below is empty, Terraform will attempt to +# create it and the required access/secret automatically. +#fcrepo_binary_bucket_username = "iam_user" +#fcrepo_binary_bucket_access_key = "***********" +#fcrepo_binary_bucket_secret_key = "***********" fcrepo_binary_bucket_username = "iam_user" fcrepo_binary_bucket_access_key = "***********" fcrepo_binary_bucket_secret_key = "***********" diff --git a/variables.tf b/variables.tf index 9da9759..6f75779 100644 --- a/variables.tf +++ b/variables.tf @@ -138,14 +138,20 @@ variable "environment" { variable "fcrepo_binary_bucket_username" { type = string + default = "" + description = "AWS IAM user for fedora bucket (will attempt to create if left blank)" } variable "fcrepo_binary_bucket_access_key" { type = string + default = "" + description = "AWS IAM user access key for fedora bucket (will attempt to create if username blank)" } variable "fcrepo_binary_bucket_secret_key" { type = string + default = "" + description = "AWS IAM user secret key for fedora bucket (will attempt to create if username blank)" } variable "fcrepo_db_ssl" { From 80cc2d0f580e1ec558db0c4b18e8e8944ed01601 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 25 Apr 2023 12:10:09 -0400 Subject: [PATCH 13/16] Default to not create admin account automatically --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 6f75779..325c4d8 100644 --- a/variables.tf +++ b/variables.tf @@ -39,7 +39,7 @@ variable "availability_zone" { } variable "avalon_admin" { - default = "admin@example.com" + default = "" } variable "avalon_repo" { From 9444078222b62a907a5d3e0a2464d0739a1e4798 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 25 Apr 2023 13:23:56 -0400 Subject: [PATCH 14/16] Create elastictranscoder presets Make presets terraform managed without needing to run avalon:aws:create_presets rake task Recreate config/elastic_transcoder_presets.yml in terraform --- transcoder.tf | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/transcoder.tf b/transcoder.tf index f0118fb..8e99b9f 100644 --- a/transcoder.tf +++ b/transcoder.tf @@ -85,3 +85,109 @@ resource "aws_elastictranscoder_pipeline" "this_pipeline" { } } +locals { + containers = [ + { container = "ts", description = "hls" }, + { container = "mp4", description = "mp4" } + ] + + audio_qualities = [ + { quality = "high", audio_bit_rate = "320" }, + { quality = "medium", audio_bit_rate = "128" } + ] + + audio_presets = [ + for config in setproduct(local.audio_qualities, local.containers) : { + container = config[1].container + name = "${local.namespace}-audio-${config[0].quality}-${config[1].description}" + description = "Avalon Media System: video/${config[0].quality}/${config[1].description}" + audio_bit_rate = config[0].audio_bit_rate + } + ] + + video_qualities = [ + { quality = "high", audio_bit_rate = "192", video_bit_rate = "2048", max_width = "1920", max_height = "1080" }, + { quality = "medium", audio_bit_rate = "128", video_bit_rate = "1024", max_width = "1280", max_height = "720" }, + { quality = "low", audio_bit_rate = "128", video_bit_rate = "500", max_width = "720", max_height = "480" } + ] + + video_presets = [ + for config in setproduct(local.video_qualities, local.containers) : { + container = config[1].container + name = "${local.namespace}-video-${config[0].quality}-${config[1].description}" + description = "Avalon Media System: video/${config[0].quality}/${config[1].description}" + audio_bit_rate = config[0].audio_bit_rate + video_bit_rate = config[0].video_bit_rate + max_width = config[0].max_width + max_height = config[0].max_height + } + ] +} + +resource "aws_elastictranscoder_preset" "this_preset_audio" { + for_each = { for preset in local.audio_presets : preset.name => preset } + container = each.value.container + description = each.value.description + name = each.key + + audio { + audio_packing_mode = "SingleTrack" + bit_rate = each.value.audio_bit_rate + channels = 2 + codec = "AAC" + sample_rate = 44100 + } + + audio_codec_options { + profile = "AAC-LC" + } +} + +resource "aws_elastictranscoder_preset" "this_preset_video" { + for_each = { for preset in local.video_presets : preset.name => preset } + container = each.value.container + description = each.value.description + name = each.key + + audio { + audio_packing_mode = "SingleTrack" + bit_rate = each.value.audio_bit_rate + channels = 2 + codec = "AAC" + sample_rate = 44100 + } + + audio_codec_options { + profile = "AAC-LC" + } + + video { + bit_rate = each.value.video_bit_rate + codec = "H.264" + display_aspect_ratio = "auto" + fixed_gop = "true" + frame_rate = "auto" + keyframes_max_dist = 90 + max_height = each.value.max_height + max_width = each.value.max_width + padding_policy = "NoPad" + sizing_policy = "ShrinkToFit" + } + + video_codec_options = { + Profile = "main" + Level = "3.1" + MaxReferenceFrames = 3 + InterlacedMode = "Progressive" + ColorSpaceConversionMode = "Auto" + } + + thumbnails { + format = "png" + interval = 300 + max_width = "192" + max_height = "108" + padding_policy = "NoPad" + sizing_policy = "ShrinkToFit" + } +} From 1191a3e1f89aebd652c5c3c43565561888105865 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Wed, 26 Apr 2023 17:03:39 -0400 Subject: [PATCH 15/16] Canned private ACL is no longer needed given changes to defaults for new s3 buckets. See https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/ --- s3.tf | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/s3.tf b/s3.tf index 12fc818..855ec3d 100644 --- a/s3.tf +++ b/s3.tf @@ -4,11 +4,6 @@ resource "aws_s3_bucket" "this_masterfiles" { force_destroy = "false" } -resource "aws_s3_bucket_acl" "this_masterfiles_bucket_acl" { - bucket = aws_s3_bucket.this_masterfiles.id - acl = "private" -} - resource "aws_s3_bucket_cors_configuration" "this_masterfiles" { bucket = aws_s3_bucket.this_masterfiles.id @@ -25,11 +20,6 @@ resource "aws_s3_bucket" "this_derivatives" { force_destroy = "false" } -resource "aws_s3_bucket_acl" "this_derivatives_bucket_acl" { - bucket = aws_s3_bucket.this_derivatives.id - acl = "private" -} - resource "aws_s3_bucket_cors_configuration" "this_derivatives" { bucket = aws_s3_bucket.this_derivatives.id @@ -47,22 +37,12 @@ resource "aws_s3_bucket" "this_preservation" { force_destroy = "false" } -resource "aws_s3_bucket_acl" "this_preservation_bucket_acl" { - bucket = aws_s3_bucket.this_preservation.id - acl = "private" -} - resource "aws_s3_bucket" "this_supplemental_files" { bucket = "${local.namespace}-supplemental-files" tags = local.common_tags force_destroy = "false" } -resource "aws_s3_bucket_acl" "this_supplemental_files_bucket_acl" { - bucket = aws_s3_bucket.this_supplemental_files.id - acl = "private" -} - data "aws_iam_policy_document" "this_bucket_access" { statement { effect = "Allow" @@ -121,11 +101,6 @@ resource "aws_s3_bucket" "fcrepo_binary_bucket" { force_destroy = "true" } -resource "aws_s3_bucket_acl" "fcrepo_binary_bucket_acl" { - bucket = aws_s3_bucket.fcrepo_binary_bucket.id - acl = "private" -} - data "aws_iam_policy_document" "fcrepo_binary_bucket_access" { statement { effect = "Allow" From 0e72f6490616b3ee82848a0c7ddcf9ba13ccbfba Mon Sep 17 00:00:00 2001 From: Nathan Collins Date: Thu, 27 Apr 2023 12:15:20 -0400 Subject: [PATCH 16/16] Fix TF timeout when ssh_cidr_blocks is empty Resource aws_security_group_rule.ssh_public_ip would not be created when empty list (default) was set for ssh_cidr_blocks. SGR is now conditional on that var not being empty. --- compose.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.tf b/compose.tf index d347eb4..349eba5 100644 --- a/compose.tf +++ b/compose.tf @@ -157,6 +157,7 @@ resource "aws_security_group" "public_ip" { } resource "aws_security_group_rule" "ssh_public_ip" { + for_each = toset(length(var.ssh_cidr_blocks) > 0 ? ["1"] : []) type = "ingress" description = "Allow SSH direct to public IP" cidr_blocks = var.ssh_cidr_blocks