fix: AWS EFS as Kubernetes PV

kubernetes/aws/eks/README.md

@@ -6,6 +6,7 @@
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.3.0 |
 | <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.10.1 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.13.0 |
 | <a name="requirement_null"></a> [null](#requirement\_null) | >= 3.2.1 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.5.1 |
 
@@ -15,6 +16,7 @@
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.3.0 |
 | <a name="provider_helm"></a> [helm](#provider\_helm) | >= 2.10.1 |
+| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | >= 2.13.0 |
 | <a name="provider_null"></a> [null](#provider\_null) | >= 3.2.1 |
 | <a name="provider_random"></a> [random](#provider\_random) | >= 3.5.1 |
 
@@ -34,11 +36,17 @@
 | [aws_cloudwatch_event_rule.aws_node_termination_handler_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
 | [aws_cloudwatch_event_rule.aws_node_termination_handler_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
 | [aws_iam_policy.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.efs_csi_driver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.worker_autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy_attachment.workers_autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
+| [aws_iam_role.efs_csi_driver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.efs_csi_driver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.efs_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.eni_config](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [kubernetes_service_account.efs_csi_driver_controller](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource |
+| [kubernetes_service_account.efs_csi_driver_node](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource |
 | [null_resource.change_cni_label](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.patch_coredns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.trigger_custom_cni](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
@@ -48,6 +56,7 @@
 | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_iam_policy_document.aws_node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.efs_csi_driver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.worker_autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
@@ -59,7 +68,7 @@
 | <a name="input_chart_namespace"></a> [chart\_namespace](#input\_chart\_namespace) | Version for chart | `string` | `"default"` | no |
 | <a name="input_chart_repository"></a> [chart\_repository](#input\_chart\_repository) | Path to the charts repository | `string` | `"../../../charts"` | no |
 | <a name="input_chart_version"></a> [chart\_version](#input\_chart\_version) | Version for chart | `string` | `"0.1.0"` | no |
-| <a name="input_eks"></a> [eks](#input\_eks) | Parameters of AWS EKS | <pre>object({<br>    cluster_version                       = string<br>    cluster_endpoint_private_access       = bool<br>    cluster_endpoint_private_access_cidrs = list(string)<br>    cluster_endpoint_private_access_sg    = list(string)<br>    cluster_endpoint_public_access        = bool<br>    cluster_endpoint_public_access_cidrs  = list(string)<br>    cluster_log_retention_in_days         = number<br>    docker_images = object({<br>      cluster_autoscaler = object({<br>        image = string<br>        tag   = string<br>      })<br>      instance_refresh = object({<br>        image = string<br>        tag   = string<br>      })<br>    })<br>    cluster_autoscaler = object({<br>      expander                              = string<br>      scale_down_enabled                    = bool<br>      min_replica_count                     = number<br>      scale_down_utilization_threshold      = number<br>      scale_down_non_empty_candidates_count = number<br>      max_node_provision_time               = string<br>      scan_interval                         = string<br>      scale_down_delay_after_add            = string<br>      scale_down_delay_after_delete         = string<br>      scale_down_delay_after_failure        = string<br>      scale_down_unneeded_time              = string<br>      skip_nodes_with_system_pods           = bool<br>      version                               = string<br>      repository                            = string<br>      namespace                             = string<br>    })<br>    instance_refresh = object({<br>      namespace  = string<br>      repository = string<br>      version    = string<br>    })<br>    encryption_keys = object({<br>      cluster_log_kms_key_id    = string<br>      cluster_encryption_config = string<br>      ebs_kms_key_id            = string<br>    })<br>    map_roles = list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    }))<br>    map_users = list(object({<br>      userarn  = string<br>      username = string<br>      groups   = list(string)<br>    }))<br>  })</pre> | n/a | yes |
+| <a name="input_eks"></a> [eks](#input\_eks) | Parameters of AWS EKS | <pre>object({<br>    cluster_version                       = string<br>    cluster_endpoint_private_access       = bool<br>    cluster_endpoint_private_access_cidrs = list(string)<br>    cluster_endpoint_private_access_sg    = list(string)<br>    cluster_endpoint_public_access        = bool<br>    cluster_endpoint_public_access_cidrs  = list(string)<br>    cluster_log_retention_in_days         = number<br>    docker_images = object({<br>      cluster_autoscaler = object({<br>        image = string<br>        tag   = string<br>      })<br>      instance_refresh = object({<br>        image = string<br>        tag   = string<br>      })<br>      efs_csi = object({<br>        image = string<br>        tag   = string<br>      })<br>      livenessprobe = object({<br>        image = string<br>        tag   = string<br>      })<br>      node_driver_registrar = object({<br>        image = string<br>        tag   = string<br>      })<br>      external_provisioner = object({<br>        image = string<br>        tag   = string<br>      })<br>    })<br>    cluster_autoscaler = object({<br>      expander                              = string<br>      scale_down_enabled                    = bool<br>      min_replica_count                     = number<br>      scale_down_utilization_threshold      = number<br>      scale_down_non_empty_candidates_count = number<br>      max_node_provision_time               = string<br>      scan_interval                         = string<br>      scale_down_delay_after_add            = string<br>      scale_down_delay_after_delete         = string<br>      scale_down_delay_after_failure        = string<br>      scale_down_unneeded_time              = string<br>      skip_nodes_with_system_pods           = bool<br>      version                               = string<br>      repository                            = string<br>      namespace                             = string<br>    })<br>    instance_refresh = object({<br>      namespace  = string<br>      repository = string<br>      version    = string<br>    })<br>    efs_csi = object({<br>      name               = string<br>      namespace          = string<br>      image_pull_secrets = string<br>      repository         = string<br>      version            = string<br>    })<br>    encryption_keys = object({<br>      cluster_log_kms_key_id    = string<br>      cluster_encryption_config = string<br>      ebs_kms_key_id            = string<br>    })<br>    map_roles = list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    }))<br>    map_users = list(object({<br>      userarn  = string<br>      username = string<br>      groups   = list(string)<br>    }))<br>  })</pre> | n/a | yes |
 | <a name="input_eks_managed_node_groups"></a> [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | List of EKS managed node groups | `any` | `null` | no |
 | <a name="input_fargate_profiles"></a> [fargate\_profiles](#input\_fargate\_profiles) | List of fargate profiles | `any` | `null` | no |
 | <a name="input_kubeconfig_file"></a> [kubeconfig\_file](#input\_kubeconfig\_file) | Kubeconfig file path | `string` | n/a | yes |

kubernetes/aws/eks/efs-csi.tf

@@ -0,0 +1,178 @@
+# Allow EKS and the driver to interact with EFS
+data "aws_iam_policy_document" "efs_csi_driver" {
+  statement {
+    sid = "Describe"
+    actions = [
+      "elasticfilesystem:DescribeAccessPoints",
+      "elasticfilesystem:DescribeFileSystems",
+      "elasticfilesystem:DescribeMountTargets",
+      "ec2:DescribeAvailabilityZones"
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+  }
+  statement {
+    sid = "Create"
+    actions = [
+      "elasticfilesystem:CreateAccessPoint"
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+    condition {
+      test     = "StringLike"
+      values   = [true]
+      variable = "aws:RequestTag/efs.csi.aws.com/cluster"
+    }
+  }
+  statement {
+    sid = "Delete"
+    actions = [
+      "elasticfilesystem:DeleteAccessPoint"
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+    condition {
+      test     = "StringEquals"
+      values   = [true]
+      variable = "aws:ResourceTag/efs.csi.aws.com/cluster"
+    }
+  }
+  statement {
+    sid = "TagResource"
+    actions = [
+      "elasticfilesystem:TagResource"
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+    condition {
+      test     = "StringLike"
+      values   = [true]
+      variable = "aws:ResourceTag/efs.csi.aws.com/cluster"
+    }
+  }
+  statement {
+    sid = "Mount"
+    actions = [
+      "elasticfilesystem:ClientRootAccess",
+      "elasticfilesystem:ClientWrite",
+      "elasticfilesystem:ClientMount"
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_policy" "efs_csi_driver" {
+  name_prefix = local.efs_csi_name
+  description = "Policy to allow EKS and the driver to interact with EFS"
+  policy      = data.aws_iam_policy_document.efs_csi_driver.json
+  tags        = local.tags
+}
+
+resource "aws_iam_role" "efs_csi_driver" {
+  name = local.efs_csi_name
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Principal = {
+          Federated = local.oidc_arn
+        }
+        Action = "sts:AssumeRoleWithWebIdentity"
+        Condition = {
+          StringEquals = {
+            "${local.oidc_url}:aud" = "sts.amazonaws.com"
+            "${local.oidc_url}:sub" = "system:serviceaccount:${local.efs_csi_namespace}:efs-csi-controller-sa"
+          }
+        }
+      }
+    ]
+  })
+  tags = local.tags
+}
+
+resource "aws_iam_role_policy_attachment" "efs_csi_driver" {
+  policy_arn = aws_iam_policy.efs_csi_driver.arn
+  role       = aws_iam_role.efs_csi_driver.name
+}
+
+resource "kubernetes_service_account" "efs_csi_driver_controller" {
+  metadata {
+    name = "efs-csi-controller-sa"
+    annotations = {
+      "eks.amazonaws.com/role-arn" = aws_iam_role.efs_csi_driver.arn
+    }
+    namespace = local.efs_csi_namespace
+  }
+}
+
+resource "kubernetes_service_account" "efs_csi_driver_node" {
+  metadata {
+    name = "efs-csi-node-sa"
+    annotations = {
+      "eks.amazonaws.com/role-arn" = aws_iam_role.efs_csi_driver.arn
+    }
+    namespace = local.efs_csi_namespace
+  }
+}
+
+resource "helm_release" "efs_csi" {
+  name       = "efs-csi"
+  namespace  = kubernetes_service_account.efs_csi_driver_controller.metadata[0].namespace
+  chart      = "aws-efs-csi-driver"
+  repository = var.eks.efs_csi.repository
+  version    = var.eks.efs_csi.version
+
+  set {
+    name  = "image.repository"
+    value = var.eks.docker_images.efs_csi.image
+  }
+  set {
+    name  = "image.tag"
+    value = var.eks.docker_images.efs_csi.tag
+  }
+  set {
+    name  = "sidecars.livenessProbe.image.repository"
+    value = var.eks.docker_images.livenessprobe.image
+  }
+  set {
+    name  = "sidecars.livenessProbe.image.tag"
+    value = var.eks.docker_images.livenessprobe.tag
+  }
+  set {
+    name  = "sidecars.nodeDriverRegistrar.image.repository"
+    value = var.eks.docker_images.node_driver_registrar.image
+  }
+  set {
+    name  = "sidecars.nodeDriverRegistrar.image.tag"
+    value = var.eks.docker_images.node_driver_registrar.tag
+  }
+  set {
+    name  = "sidecars.csiProvisioner.image.repository"
+    value = var.eks.docker_images.external_provisioner.image
+  }
+  set {
+    name  = "sidecars.csiProvisioner.image.tag"
+    value = var.eks.docker_images.external_provisioner.tag
+  }
+  set {
+    name  = "imagePullSecrets"
+    value = var.eks.efs_csi.image_pull_secrets
+  }
+  set {
+    name  = "node.serviceAccount.create"
+    value = false
+  }
+  set {
+    name  = "node.serviceAccount.name"
+    value = kubernetes_service_account.efs_csi_driver_node.metadata[0].name
+  }
+  values = [
+    yamlencode(local.controller)
+  ]
+  depends_on = [
+    kubernetes_service_account.efs_csi_driver_controller,
+    kubernetes_service_account.efs_csi_driver_node
+  ]
+}

kubernetes/aws/eks/locals.tf

@@ -24,17 +24,6 @@ locals {
   aws_node_termination_handler_spot_name = "${var.name}-spot-termination"
   kubeconfig_output_path                 = coalesce(var.kubeconfig_file, "${path.root}/generated/kubeconfig")
 
-  # Custom ENI
-  subnets = {
-    subnets = [
-      for index, id in var.vpc.pods_subnet_ids : {
-        subnet_id          = id
-        az_name            = element(data.aws_availability_zones.available.names, index)
-        security_group_ids = [module.eks.node_security_group_id]
-      }
-    ]
-  }
-
   # Node selector
   node_selector_keys   = keys(var.node_selector)
   node_selector_values = values(var.node_selector)
@@ -52,6 +41,53 @@ locals {
     ]
   }
 
+  # EFS CSI
+  efs_csi_name      = try(var.eks.efs_csi.name, "efs-csi-driver")
+  oidc_arn          = module.eks.oidc_provider_arn
+  oidc_url          = trimprefix(module.eks.cluster_oidc_issuer_url, "https://")
+  efs_csi_namespace = try(var.eks.efs_csi.namespace, "kube-system")
+  efs_csi_tolerations = [
+    for index in range(0, length(local.node_selector_keys)) : {
+      key      = local.node_selector_keys[index]
+      operator = "Equal"
+      value    = local.node_selector_values[index]
+      effect   = "NoSchedule"
+    }
+  ]
+  controller = {
+    controller = {
+      create                   = true
+      logLevel                 = 2
+      extraCreateMetadata      = true
+      tags                     = {}
+      deleteAccessPointRootDir = false
+      volMetricsOptIn          = false
+      podAnnotations           = {}
+      resources                = {}
+      nodeSelector             = var.node_selector
+      tolerations              = local.efs_csi_tolerations
+      affinity                 = {}
+      serviceAccount = {
+        create      = false
+        name        = kubernetes_service_account.efs_csi_driver_controller.metadata[0].name
+        annotations = {}
+      }
+      healthPort           = 9909
+      regionalStsEndpoints = false
+    }
+  }
+
+  # Custom ENI
+  subnets = {
+    subnets = [
+      for index, id in var.vpc.pods_subnet_ids : {
+        subnet_id          = id
+        az_name            = element(data.aws_availability_zones.available.names, index)
+        security_group_ids = [module.eks.node_security_group_id]
+      }
+    ]
+  }
+
   # Patch coredns
   patch_coredns_spec = {
     spec = {

kubernetes/aws/eks/variables.tf

@@ -89,6 +89,22 @@ variable "eks" {
         image = string
         tag   = string
       })
+      efs_csi = object({
+        image = string
+        tag   = string
+      })
+      livenessprobe = object({
+        image = string
+        tag   = string
+      })
+      node_driver_registrar = object({
+        image = string
+        tag   = string
+      })
+      external_provisioner = object({
+        image = string
+        tag   = string
+      })
     })
     cluster_autoscaler = object({
       expander                              = string
@@ -112,6 +128,13 @@ variable "eks" {
       repository = string
       version    = string
     })
+    efs_csi = object({
+      name               = string
+      namespace          = string
+      image_pull_secrets = string
+      repository         = string
+      version            = string
+    })
     encryption_keys = object({
       cluster_log_kms_key_id    = string
       cluster_encryption_config = string

kubernetes/aws/eks/versions.tf

@@ -5,6 +5,10 @@ terraform {
       source  = "hashicorp/aws"
       version = ">= 5.3.0"
     }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.13.0"
+    }
     helm = {
       source  = "hashicorp/helm"
       version = ">= 2.10.1"