diff --git a/README.md b/README.md index 7496696..d3d9c99 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,9 @@ In addition, the following tools are recommended: ### AMI +The Terraform modules in this repository require the use of AL2023 for the +Linux version for all AMIs. + The Terraform modules in this repository rely on the AMIs used for worker nodes to be optimized for XRd. The easiest way to achieve this is to use the [XRd Packer](https://github.com/ios-xr/xrd-packer) templates to generate diff --git a/examples/ha/infra/data.tf b/examples/ha/infra/data.tf index c4386d6..5943a00 100644 --- a/examples/ha/infra/data.tf +++ b/examples/ha/infra/data.tf @@ -8,7 +8,7 @@ data "aws_ami" "eks_optimized" { filter { name = "name" - values = ["amazon-eks-node-${data.aws_eks_cluster.this.version}-*"] + values = ["amazon-eks-node-al2023-x86_64-standard-${data.aws_eks_cluster.this.version}-*"] } filter { diff --git a/examples/ha/workload/main.tf b/examples/ha/workload/main.tf index d13e674..0f3f535 100644 --- a/examples/ha/workload/main.tf +++ b/examples/ha/workload/main.tf @@ -95,7 +95,7 @@ module "peer" { source = "../../../modules/aws/linux-pod-with-net-attach" name = "peer" - device = "eth1" + device = "ens6" ip_address = "10.0.10.12/24" gateway = "10.0.10.20" routes = ["10.0.11.0/24", "10.0.13.0/24"] @@ -108,7 +108,7 @@ module "cnf_vrid1" { source = "../../../modules/aws/linux-pod-with-net-attach" name = "cnf-vrid1" - device = "eth2" + device = "ens7" ip_address = "10.0.11.12/24" gateway = "10.0.11.20" routes = ["10.0.10.0/24"] @@ -121,7 +121,7 @@ module "cnf_vrid2" { source = "../../../modules/aws/linux-pod-with-net-attach" name = "cnf-vrid2" - device = "eth3" + device = "ens8" ip_address = "10.0.13.12/24" gateway = "10.0.13.1" routes = ["10.0.10.0/24"] diff --git a/examples/overlay/infra/data.tf b/examples/overlay/infra/data.tf index 938fa1c..feff999 100644 --- a/examples/overlay/infra/data.tf +++ b/examples/overlay/infra/data.tf @@ -4,7 +4,7 @@ data "aws_ami" "eks_optimized" { filter { name = "name" - values = ["amazon-eks-node-${data.aws_eks_cluster.this.version}-*"] + values = ["amazon-eks-node-al2023-x86_64-standard-${data.aws_eks_cluster.this.version}-*"] } filter { diff --git a/examples/overlay/workload/main.tf b/examples/overlay/workload/main.tf index 45914d9..66974d0 100644 --- a/examples/overlay/workload/main.tf +++ b/examples/overlay/workload/main.tf @@ -76,7 +76,7 @@ module "cnf" { source = "../../../modules/aws/linux-pod-with-net-attach" name = "cnf" - device = "eth1" + device = "ens6" ip_address = "10.0.10.10/24" gateway = "10.0.10.11" routes = ["10.0.13.0/24"] @@ -89,7 +89,7 @@ module "peer" { source = "../../../modules/aws/linux-pod-with-net-attach" name = "peer" - device = "eth2" + device = "ens7" ip_address = "10.0.13.10/24" gateway = "10.0.13.12" routes = ["10.0.10.0/24"] diff --git a/modules/aws/eks-config/main.tf b/modules/aws/eks-config/main.tf index f46ee82..70d5b62 100644 --- a/modules/aws/eks-config/main.tf +++ b/modules/aws/eks-config/main.tf @@ -2,7 +2,7 @@ resource "kubernetes_config_map" "aws_auth" { data = { "mapRoles" = <<-EOT - rolearn: ${var.node_iam_role_arn} - username: system:node:{{EC2PrivateDNSName}} + username: system:node:{{SessionName}} groups: - system:bootstrappers - system:nodes diff --git a/modules/aws/node/data.tf b/modules/aws/node/data.tf index c250064..268b75e 100644 --- a/modules/aws/node/data.tf +++ b/modules/aws/node/data.tf @@ -8,3 +8,7 @@ data "aws_ami" "this" { data "aws_ec2_instance_type" "this" { instance_type = var.instance_type } + +data "aws_eks_cluster" "this" { + name = var.cluster_name +} diff --git a/modules/aws/node/main.tf b/modules/aws/node/main.tf index 9d00b79..158578e 100644 --- a/modules/aws/node/main.tf +++ b/modules/aws/node/main.tf @@ -156,20 +156,19 @@ resource "aws_instance" "this" { user_data = templatefile( "${path.module}/templates/user-data.tftpl", { - xrd_bootstrap = local.is_xrd_ami - hugepages_gb = local.hugepages_gb - isolated_cores = local.isolated_cores - cluster_name = var.cluster_name - kubelet_extra_args = format( - "%s%s", - ( - local.kubelet_node_labels_arg != null ? - "--node-labels ${local.kubelet_node_labels_arg}" : - "" - ), - var.kubelet_extra_args != null ? " ${var.kubelet_extra_args}" : "", + name = data.aws_eks_cluster.this.name + api_endpoint = data.aws_eks_cluster.this.endpoint + certificate_authority = data.aws_eks_cluster.this.certificate_authority[0].data + cidr = data.aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr + kubelet_flags = concat( + ["--node-labels=${local.kubelet_node_labels_arg}"], + var.kubelet_extra_args ) + hugepages_gb = local.hugepages_gb + isolated_cores = local.isolated_cores additional_user_data = var.user_data + xrd_bootstrap = local.is_xrd_ami + private_ip = var.private_ip_address } ) @@ -194,6 +193,11 @@ resource "aws_instance" "this" { } resource "aws_network_interface" "this" { + # Wait for kubelet to start before attaching the network interfaces. + depends_on = [ + kubernetes_job.wait + ] + for_each = { for i, ni in var.network_interfaces : i => ni diff --git a/modules/aws/node/templates/user-data.tftpl b/modules/aws/node/templates/user-data.tftpl index 102168d..17234bd 100644 --- a/modules/aws/node/templates/user-data.tftpl +++ b/modules/aws/node/templates/user-data.tftpl @@ -1,7 +1,34 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: application/node.eks.aws + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + featureGates: + InstanceIdNodeName: true + cluster: + name: ${name} + apiServerEndpoint: ${api_endpoint} + certificateAuthority: ${certificate_authority} + cidr: ${cidr} + kubelet: + flags: + %{~ for flag in kubelet_flags ~} + - ${flag} + %{~ endfor ~} + +--BOUNDARY +Content-Type: text/x-shellscript; charset="us-ascii" + #!/bin/bash %{~ if xrd_bootstrap } HUGEPAGES_GB=${hugepages_gb} ISOLATED_CORES=${isolated_cores} /etc/xrd/bootstrap.sh %{~ endif } -/etc/eks/bootstrap.sh ${cluster_name} --kubelet-extra-args '${kubelet_extra_args}' ${additional_user_data} reboot + +--BOUNDARY-- diff --git a/modules/aws/node/variables.tf b/modules/aws/node/variables.tf index 01c1149..bdca23b 100644 --- a/modules/aws/node/variables.tf +++ b/modules/aws/node/variables.tf @@ -95,11 +95,11 @@ variable "isolated_cores" { variable "kubelet_extra_args" { description = <<-EOT - Extra arguments to pass to kubelet when booting the node. + List of extra command-line kubelet arguments to be appended to the defaults. Note that node labels must be specified via the 'labels' variable. EOT - type = string - default = null + type = list(string) + default = [] } variable "labels" { diff --git a/tests/ut/terraform/node/variables.tf b/tests/ut/terraform/node/variables.tf index 570bdd2..b22f87c 100644 --- a/tests/ut/terraform/node/variables.tf +++ b/tests/ut/terraform/node/variables.tf @@ -101,11 +101,11 @@ variable "isolated_cores" { variable "kubelet_extra_args" { description = <<-EOT - Extra arguments to pass to kubelet when booting the node. + List of extra command-line kubelet arguments to be appended to the defaults. Note that node labels must be specified via the 'labels' variable. EOT - type = string - default = null + type = list(string) + default = [] } variable "labels" { diff --git a/tests/ut/test_node.py b/tests/ut/test_node.py index a3d350c..904df50 100644 --- a/tests/ut/test_node.py +++ b/tests/ut/test_node.py @@ -1,6 +1,7 @@ import base64 import json import subprocess +import textwrap import uuid from pathlib import Path from typing import Any @@ -15,6 +16,7 @@ Subnet, Vpc, ) +from mypy_boto3_eks import EKSClient from mypy_boto3_iam import IAMServiceResource from mypy_boto3_iam.service_resource import InstanceProfile @@ -66,6 +68,14 @@ def subnet(vpc: Vpc) -> Subnet: ) +@pytest.fixture +def other_subnet(vpc: Vpc) -> Subnet: + return vpc.create_subnet( + AvailabilityZone="eu-west-1a", + CidrBlock="10.0.1.0/24", + ) + + @pytest.fixture def key_pair(ec2: EC2ServiceResource) -> KeyPair: return ec2.create_key_pair(KeyName=str(uuid.uuid4())) @@ -92,6 +102,7 @@ def iam_instance_profile(iam: IAMServiceResource) -> InstanceProfile: "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", ): role.attach_policy(PolicyArn=policy_arn) @@ -116,11 +127,34 @@ def security_group(ec2: EC2ServiceResource, vpc: Vpc) -> SecurityGroup: return sg +@pytest.fixture() +def eks_cluster( + eks_client: EKSClient, + iam_instance_profile: InstanceProfile, + subnet: Subnet, + other_subnet: Subnet, + security_group: SecurityGroup, +) -> dict[str, Any]: + name = str(uuid.uuid4()) + return eks_client.create_cluster( + name=name, + roleArn=iam_instance_profile.roles_attribute[0]["Arn"], + resourcesVpcConfig={ + "subnetIds": [subnet.id, other_subnet.id], + "securityGroupIds": [security_group.id], + "endpointPublicAccess": False, + "endpointPrivateAccess": False, + "publicAccessCidrs": [], + }, + ) + + @pytest.fixture def base_vars( subnet: Subnet, key_pair: KeyPair, iam_instance_profile: InstanceProfile, + eks_cluster: dict[str, Any], ) -> dict[str, Any]: # This AMI should exist in the Moto server. # Refer to https://github.com/getmoto/moto/blob/master/moto/ec2/resources/amis.json. @@ -128,7 +162,7 @@ def base_vars( return { "ami": ami, - "cluster_name": str(uuid.uuid4()), + "cluster_name": eks_cluster["cluster"]["name"], "name": str(uuid.uuid4()), "private_ip_address": "10.0.0.10", "security_groups": [], @@ -162,7 +196,12 @@ def _assert_tag(instance: Instance, tag_key: str, tag_value: str) -> None: raise AssertionError(f"tag '{tag_key}' does not exist") -def test_defaults(ec2, tf: Terraform, base_vars: dict[str, Any]): +def test_defaults( + ec2, + tf: Terraform, + base_vars: dict[str, Any], + eks_cluster: dict[str, Any], +): tf.apply(vars=base_vars) outputs = Outputs.from_terraform(tf) instance = ec2.Instance(outputs.id) @@ -182,7 +221,23 @@ def test_defaults(ec2, tf: Terraform, base_vars: dict[str, Any]): "Value" ] user_data = base64.b64decode(user_data).decode() - assert f"/etc/eks/bootstrap.sh {base_vars['cluster_name']}" in user_data + expected_node_config = textwrap.dedent( + f"""\ + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + cluster: + name: {base_vars['cluster_name']} + apiServerEndpoint: {eks_cluster['cluster']['endpoint']} + certificateAuthority: {eks_cluster['cluster']['certificateAuthority']['data']} + cidr: {eks_cluster['cluster']['kubernetesNetworkConfig']['serviceIpv4Cidr']} + kubelet: + flags: + - --node-labels=ios-xr.cisco.com/name={base_vars['name']}""", + ) + + assert expected_node_config in user_data # There should be no public IP address assigned. assert not instance.public_ip_address @@ -250,7 +305,7 @@ def test_unknown_instance_type(ec2, tf: Terraform, base_vars: dict[str, Any]): def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]): - vars = base_vars | {"kubelet_extra_args": "foo bar baz"} + vars = base_vars | {"kubelet_extra_args": ["foo", "bar"]} tf.apply(vars=vars) outputs = Outputs.from_terraform(tf) instance = ec2.Instance(outputs.id) @@ -260,8 +315,10 @@ def test_kubelet_extra_args(ec2, tf: Terraform, base_vars: dict[str, Any]): ] user_data = base64.b64decode(user_data).decode() assert ( - f"""--kubelet-extra-args '--node-labels ios-xr.cisco.com/name={base_vars["name"]} foo bar baz'""" - in user_data + " flags:\n" + f" - --node-labels=ios-xr.cisco.com/name={vars['name']}\n" + " - foo\n" + " - bar\n" in user_data )