Merge pull request #135 from joelddiaz/rebase-master

Rebase master

.github/workflows/build.yaml

@@ -0,0 +1,41 @@
+name: build
+
+on: push
+jobs:
+  build:
+    # this is to prevent the job to run at forked projects
+    if: github.repository == 'aws/amazon-eks-pod-identity-webhook'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: crazy-max/ghaction-docker-buildx@v3
+      with:
+        buildx-version: latest
+        qemu-version: latest
+    - name: Build container and push to Dockerhub registry
+      run: |
+        BRANCH=$(echo $GITHUB_REF | cut -d'/' -f3)
+        SHORT_SHA=$(echo $GITHUB_SHA | cut -c -7)
+        REPO=amazon/amazon-eks-pod-identity-webhook
+        if [ "$BRANCH" = "master" ]; then
+          TAG=$SHORT_SHA
+        else
+          TAG=$BRANCH
+        fi
+        docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+
+        docker buildx build \
+              -t $REPO:$TAG \
+              --platform=linux/amd64,linux/arm64 \
+              --progress plain \
+              --push .
+        if [ "$BRANCH" = "master" ]; then
+          docker buildx build \
+                -t $REPO:latest \
+                --platform=linux/amd64,linux/arm64 \
+                --progress plain \
+                --push .
+        fi
+

.gitignore

@@ -6,3 +6,4 @@ deploy/deployment.yaml
 build
 /certs/
 SAMToolkit.*
+coverage.out

Dockerfile

@@ -2,9 +2,10 @@ FROM golang AS builder
 
 WORKDIR $GOPATH/src/github.com/aws/amazon-eks-pod-identity-webhook
 COPY . ./
-RUN CGO_ENABLED=0 GOOS=linux go build -v -a -installsuffix nocgo -o /webhook .
+RUN GOPROXY=direct CGO_ENABLED=0 GOOS=linux go build -o /webhook -v -a -installsuffix nocgo -ldflags="-buildid='' -w -s" .
 
 FROM scratch
+COPY ATTRIBUTIONS.txt /ATTRIBUTIONS.txt
 COPY --from=builder /webhook /webhook
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 EXPOSE 443

Dockerfile.rhel7

@@ -1,8 +1,8 @@
-FROM registry.svc.ci.openshift.org/ocp/builder:rhel-8-golang-1.15-openshift-4.7 AS builder
+FROM registry.svc.ci.openshift.org/ocp/builder:golang-1.13 AS builder
 WORKDIR /go/src/github.com/aws/amazon-eks-pod-identity-webhook
 COPY . .
 ENV GO_PACKAGE github.com/aws/amazon-eks-pod-identity-webhook
 RUN make amazon-eks-pod-identity-webhook
 
-FROM registry.svc.ci.openshift.org/ocp/4.7:base
+FROM registry.svc.ci.openshift.org/ocp/4.5:base
 COPY --from=builder /go/src/github.com/aws/amazon-eks-pod-identity-webhook/amazon-eks-pod-identity-webhook /usr/bin/aws-pod-identity-webhook

Makefile

@@ -5,27 +5,32 @@ include ${BGO_MAKEFILE}
 export CGO_ENABLED=0
 export T=github.com/aws/amazon-eks-pod-identity-webhook
 UNAME_S = $(shell uname -s)
-GO_INSTALL_FLAGS = -ldflags="-s -w"
+GO_LDFLAGS = -ldflags='-s -w -buildid=""'
 
 install:: build
 ifeq ($(UNAME_S), Darwin)
-	GOOS=darwin GOARCH=amd64 go build -o build/gopath/bin/darwin_amd64/amazon-eks-pod-identity-webhook $(GO_INSTALL_FLAGS) $V $T
+	GOOS=darwin GOARCH=amd64 go build -o build/gopath/bin/darwin_amd64/amazon-eks-pod-identity-webhook $(GO_LDFLAGS) $V $T
 endif
-	GOOS=linux GOARCH=amd64 go build -o build/gopath/bin/linux_amd64/amazon-eks-pod-identity-webhook $(GO_INSTALL_FLAGS) $V $T
-
+	GOOS=linux GOARCH=amd64 go build -o build/gopath/bin/linux_amd64/amazon-eks-pod-identity-webhook $(GO_LDFLAGS) $V $T
 
 # Generic make
 REGISTRY_ID?=602401143452
 IMAGE_NAME?=eks/pod-identity-webhook
 REGION?=us-west-2
 IMAGE?=$(REGISTRY_ID).dkr.ecr.$(REGION).amazonaws.com/$(IMAGE_NAME)
 
+test:
+	go test -coverprofile=coverage.out ./...
+	go tool cover -html=coverage.out
+
 docker:
 	@echo 'Building image $(IMAGE)...'
 	docker build --no-cache -t $(IMAGE) .
 
 push: docker
-	eval $$(aws ecr get-login --registry-ids $(REGISTRY_ID) --no-include-email)
+	if ! aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(REGISTRY_ID).dkr.ecr.$(REGION).amazonaws.com; then \
+	  eval $$(aws ecr get-login --registry-ids $(REGISTRY_ID) --no-include-email); \
+	fi
 	docker push $(IMAGE)
 
 amazon-eks-pod-identity-webhook:
@@ -37,7 +42,7 @@ certs/tls.key:
 		-x509 \
 		-newkey rsa:2048 \
 		-keyout certs/tls.key \
-		-out certs/tls.cert \
+		-out certs/tls.crt \
 		-days 365 \
 		-nodes \
 		-subj "/CN=127.0.0.1"
@@ -47,7 +52,7 @@ local-serve: amazon-eks-pod-identity-webhook certs/tls.key
 		--port 8443 \
 		--in-cluster=false \
 		--tls-key=./certs/tls.key \
-		--tls-cert=./certs/tls.cert \
+		--tls-cert=./certs/tls.crt \
 		--kubeconfig=$$HOME/.kube/config
 
 local-request:
@@ -75,7 +80,11 @@ deploy-config: prep-config
 	kubectl apply -f deploy/deployment.yaml
 	kubectl apply -f deploy/service.yaml
 	kubectl apply -f deploy/mutatingwebhook-ca-bundle.yaml
-	sleep 1
+	until kubectl get csr -o \
+		jsonpath='{.items[?(@.spec.username=="system:serviceaccount:default:pod-identity-webhook")].metadata.name}' | \
+		grep -m 1 "csr-"; \
+		do echo "Waiting for CSR to be created" && sleep 1 ; \
+	done
 	kubectl certificate approve $$(kubectl get csr -o jsonpath='{.items[?(@.spec.username=="system:serviceaccount:default:pod-identity-webhook")].metadata.name}')
 
 delete-config:
@@ -84,10 +93,11 @@ delete-config:
 	kubectl delete -f deploy/service.yaml
 	kubectl delete -f deploy/deployment.yaml
 	kubectl delete -f deploy/auth.yaml
+	kubectl delete secret pod-identity-webhook
 
 clean::
 	rm -rf ./amazon-eks-pod-identity-webhook
-	rm -rf ./certs/
+	rm -rf ./certs/ coverage.out
 
 .PHONY: docker push build local-serve local-request cluster-up cluster-down prep-config deploy-config delete-config clean
 

README.md

@@ -1,3 +1,5 @@
+![build](https://github.com/aws/amazon-eks-pod-identity-webhook/workflows/build/badge.svg)
+
 # Amazon EKS Pod Identity Webhook
 
 This webhook is for mutating pods that will require AWS IAM access.
@@ -19,17 +21,17 @@ This webhook is for mutating pods that will require AWS IAM access.
       {
        "Effect": "Allow",
        "Principal": {
-        "Federated": "arn:aws:iam::111122223333:oidc-provider/oidc.us-west-2.eks.amazonaws.com/624a142e-43fc-4a4e-9a65-0adbfe9d6a85"
+        "Federated": "arn:aws:iam::111122223333:oidc-provider/oidc.REGION.eks.amazonaws.com/CLUSTER_ID"
        },
        "Action": "sts:AssumeRoleWithWebIdentity",
        "Condition": {
         "__doc_comment": "scope the role to the service account (optional)",
         "StringEquals": {
-         "oidc.us-west-2.eks.amazonaws.com/624a142e-43fc-4a4e-9a65-0adbfe9d6a85:sub": "system:serviceaccount:default:my-serviceaccount"
+         "oidc.REGION.eks.amazonaws.com/CLUSTER_ID:sub": "system:serviceaccount:default:my-serviceaccount"
         },
         "__doc_comment": "scope the role to a namespace (optional)",
         "StringLike": {
-         "oidc.us-west-2.eks.amazonaws.com/624a142e-43fc-4a4e-9a65-0adbfe9d6a85:sub": "system:serviceaccount:default:*"
+         "oidc.REGION.eks.amazonaws.com/CLUSTER_ID:sub": "system:serviceaccount:default:*"
         }
        }
       }
@@ -46,6 +48,15 @@ This webhook is for mutating pods that will require AWS IAM access.
       namespace: default
       annotations:
         eks.amazonaws.com/role-arn: "arn:aws:iam::111122223333:role/s3-reader"
+        # optional: Defaults to "sts.amazonaws.com" if not set
+        eks.amazonaws.com/audience: "sts.amazonaws.com"
+        # optional: When set to "true", adds AWS_STS_REGIONAL_ENDPOINTS env var
+        #   to containers
+        eks.amazonaws.com/sts-regional-endpoints: "true"
+        # optional: Defaults to 86400 for expirationSeconds if not set
+        #   Note: This value can be overwritten if specified in the pod 
+        #         annotation as shown in the next step.
+        eks.amazonaws.com/token-expiration: "86400"
     ```
 4. All new pod pods launched using this Service Account will be modified to use
    IAM for pods. Below is an example pod spec with the environment variables and
@@ -56,9 +67,21 @@ This webhook is for mutating pods that will require AWS IAM access.
     metadata:
       name: my-pod
       namespace: default
+      annotations:
+        # optional: A comma-separated list of initContainers and container names
+        #   to skip adding volumes and environment variables
+        eks.amazonaws.com/skip-containers: "init-first,sidecar"
+        # optional: Defaults to 86400, or value specified in ServiceAccount
+        #   annotation as shown in previous step, for expirationSeconds if not set
+        eks.amazonaws.com/token-expiration: "86400"
     spec:
       serviceAccountName: my-serviceaccount
+      initContainers:
+      - name: init-first
+        image: container-image:version
       containers:
+      - name: sidecar
+        image: container-image:version
       - name: container-name
         image: container-image:version
     ### Everything below is added by the webhook ###
@@ -71,6 +94,8 @@ This webhook is for mutating pods that will require AWS IAM access.
           value: "arn:aws:iam::111122223333:role/s3-reader"
         - name: AWS_WEB_IDENTITY_TOKEN_FILE
           value: "/var/run/secrets/eks.amazonaws.com/serviceaccount/token"
+        - name: AWS_STS_REGIONAL_ENDPOINTS
+          value: "regional"
         volumeMounts:
         - mountPath: "/var/run/secrets/eks.amazonaws.com/serviceaccount/"
           name: aws-token
@@ -83,7 +108,7 @@ This webhook is for mutating pods that will require AWS IAM access.
               expirationSeconds: 86400
               path: token
     ```
-   
+
 ### Usage with Windows container workloads
 
 To ensure workloads are scheduled on windows nodes have the right environment variables, they must have a `nodeSelector` targeting windows it must run on.  Workloads targeting windows nodes using `nodeAffinity` are currently not supported.
@@ -104,6 +129,10 @@ Or for Kubernetes 1.14+
 [1]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html
 [2]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html
 
+### Usage with non-root container user
+
+When running a container with a non-root user, you need to give the container access to the token file by setting the `fsGroup` field in the `securityContext` object.
+
 ## Usage
 
 ```
@@ -112,20 +141,23 @@ Usage of amazon-eks-pod-identity-webhook:
       --annotation-prefix string         The Service Account annotation to look for (default "eks.amazonaws.com")
       --aws-default-region string        If set, AWS_DEFAULT_REGION and AWS_REGION will be set to this value in mutated containers
       --in-cluster                       Use in-cluster authentication and certificate request API (default true)
+      --enable-debugging-handlers        Enable debugging handlers on the metrics port (http). Currently /debug/alpha/cache is supported (default false) [ALPHA]
       --kube-api string                  (out-of-cluster) The url to the API server
       --kubeconfig string                (out-of-cluster) Absolute path to the API server kubeconfig file
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --log_file string                  If non-empty, use this log file
       --log_file_max_size uint           Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
       --logtostderr                      log to standard error instead of files (default true)
+      --metrics-port int                 Port to listen on for metrics and healthz (http) (default 9999)
       --namespace string                 (in-cluster) The namespace name this webhook and the tls secret resides in (default "eks")
       --port int                         Port to listen on (default 443)
       --service-name string              (in-cluster) The service name fronting this webhook (default "pod-identity-webhook")
       --skip_headers                     If true, avoid header prefixes in the log messages
       --skip_log_headers                 If true, avoid headers when openning log files
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
-      --tls-cert string                  (out-of-cluster) TLS certificate file path (default "/etc/webhook/certs/tls.cert")
+      --sts-regional-endpoint false      Whether to inject the AWS_STS_REGIONAL_ENDPOINTS=regional env var in mutated pods. Defaults to false.
+      --tls-cert string                  (out-of-cluster) TLS certificate file path (default "/etc/webhook/certs/tls.crt")
       --tls-key string                   (out-of-cluster) TLS key file path (default "/etc/webhook/certs/tls.key")
       --tls-secret string                (in-cluster) The secret name for storing the TLS serving cert (default "pod-identity-webhook")
       --token-audience string            The default audience for tokens. Can be overridden by annotation (default "sts.amazonaws.com")
@@ -140,6 +172,22 @@ Usage of amazon-eks-pod-identity-webhook:
 
 When the `aws-default-region` flag is set this webhook will inject `AWS_DEFAULT_REGION` and `AWS_REGION` in mutated containers if `AWS_DEFAULT_REGION` and `AWS_REGION` are not already set.
 
+### AWS_STS_REGIONAL_ENDPOINTS Injection
+
+When the `sts-regional-endpoint` flag is set to `true`, the webhook will
+inject the environment variable `AWS_STS_REGIONAL_ENDPOINTS` with the value set
+to `regional`. This environment variable will configure the AWS SDKs to perform
+the `sts:AssumeRoleWithWebIdentity` call to get credentials from the regional
+endpoint, instead of the global endpoint in `us-east-1`. This is desirable in
+almost all cases, unless the STS regional endpoint is [disabled in your
+account](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html).
+
+You can also enable this per-service account with the annotation
+`eks.amazonaws.com/sts-regional-endpoint` set to `"true"`.
+
+## Container Images
+
+Container images for amazon-eks-pod-identity-webhook can be found on [Docker Hub](https://hub.docker.com/r/amazon/amazon-eks-pod-identity-webhook).
 
 ## Installation
 
@@ -148,7 +196,7 @@ When the `aws-default-region` flag is set this webhook will inject `AWS_DEFAULT_
 You can use the provided configuration files in the `deploy` directory, along with the provided `Makefile`
 
 ```
-make cluster-up IMAGE=602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/pod-identity-webhook:latest
+make cluster-up IMAGE=amazon/amazon-eks-pod-identity-webhook:2db5e53
 ```
 
 This will:
@@ -161,8 +209,9 @@ For self-hosted API server configuration, see see [SELF_HOSTED_SETUP.md](/SELF_H
 ### On API server
 TODO
 
-## Development
-TODO
+### Notes
+With the upgrade to client-go 1.18, certificate_manager_server_expiration_seconds metric has been removed by an upstream commit kubernetes/kubernetes#85874.
+A new metric certificate_manager_server_rotation_seconds is added which tracks the time a certificate was valid before getting rotated.
 
 ## Code of Conduct
 See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)

SELF_HOSTED_SETUP.md

@@ -2,7 +2,16 @@
 
 If you are running your own Kubernetes cluster, there are several steps required for this feature to work.
 
-This feature requires Kubernetes 1.12 or greater.
+## Prerequisites
+
+1. Your cluster must be running Kubernetes 1.12 or later.
+
+2. Your cluster's `kube-controller-manager` must be properly configured to sign
+   certificate requests. You can verify this by validating that the
+   `--cluster-signing-cert-file` and `--cluster-signing-key-file` parameters
+   point to valid TLS certificate and key files. See [this
+   document](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster) for
+   details.
 
 ## Projected Token Signing Keypair
 
@@ -108,12 +117,12 @@ In order to use this feature, you'll need to set the following
 [API server flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/).
 
 ```
-# Path to the $PUB_KEY file from the beginning. 
+# Path to the $PKCS_KEY file from the beginning. 
 #
 # This flag can be specified for multiple times.
 # There is likely already one specified for legacy service accounts, if not, 
 # it is using the default value. Find out your default value and pass it explicitly
-# (along with this $PUB_KEY), otherwise your existing tokens will fail.
+# (along with this $PKCS_KEY), otherwise your existing tokens will fail.
 --service-account-key-file
 
 # Path to the signing (private) key ($PRIV_KEY)
@@ -140,7 +149,7 @@ usually be set to `kubernetes.svc.default`, or optionally the DNS name of your
 API server.
 
 When using a Kubernetes-issued token for an external system, you should use a
-different audience (or in OAuth-2 parlance, `client-id`). The external system
+different audience (or in OAuth2 parlance, `client-id`). The external system
 (such as AWS IAM) will usually require an audience, or client-id, at setup. For
 AWS IAM, a token's `aud` must match the OIDC Identity Provider's client ID. EKS
 uses the string `sts.amazonaws.com` as the default, but when using the webhook
@@ -151,4 +160,20 @@ yourself, you can use any audience you'd like as long as the webhook's flag
 
 From here, you can mostly follow the process in the [EKS
 documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html)
-and substitue the cluster issuer with `https://$ISSUER_HOSTPATH`.
+and substitute the cluster issuer with `https://$ISSUER_HOSTPATH`.
+
+## Troubleshooting
+
+### `Certificate request was not signed: timed out waiting for the condition` appears in the logs
+
+Check the output of `kubectl get -n <namespace> csr | grep pod-identity-webhook`.
+The last column should contain `Approved,Issued` as in the following example:
+
+```
+csr-869cl   2m52s   system:serviceaccount:<namespace>:pod-identity-webhook   Approved,Issued
+```
+
+If it says `Approved` but not `Issued`, your cluster's controller manager is
+likely not configured properly as a TLS Certificate Authority.  Please review
+the Prerequisites section above.  You will need to restart the controller
+manager.

deploy/mutatingwebhook.yaml

@@ -6,6 +6,7 @@ metadata:
 webhooks:
 - name: pod-identity-webhook.amazonaws.com
   failurePolicy: Ignore
+  sideEffects: None
   clientConfig:
     service:
       name: pod-identity-webhook