diff --git a/.github/actions/smoke-tests/action.yaml b/.github/actions/smoke-tests/action.yaml index 6585c9333b..8b0ad78447 100644 --- a/.github/actions/smoke-tests/action.yaml +++ b/.github/actions/smoke-tests/action.yaml @@ -34,6 +34,9 @@ inputs: registry-token: description: JWT token for accessing container registry required: false + plus-jwt: + description: JWT for NGINX Plus + required: false outputs: test-results-name: @@ -101,6 +104,7 @@ runs: --durations=10 \ --show-ic-logs=yes \ --ad-secret=${{ inputs.azure-ad-secret }} \ + --plus-jwt=${{ inputs.plus-jwt }} \ -m ${{ inputs.marker != '' && inputs.marker || '""' }} working-directory: ./tests shell: bash diff --git a/.github/data/matrix-images-plus.json b/.github/data/matrix-images-plus.json index 926956b89f..ab1717d37d 100644 --- a/.github/data/matrix-images-plus.json +++ b/.github/data/matrix-images-plus.json @@ -18,7 +18,7 @@ }, { "image": "ubi-9-plus", - "platforms": "linux/arm64, linux/amd64, linux/s390x", + "platforms": "linux/arm64, linux/amd64", "target": "goreleaser" } ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e4f442148..8af94518f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -467,6 +467,10 @@ jobs: kind load docker-image "${{ matrix.image }}:${{ matrix.tag }}" --name ${{ github.run_id }} if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }} + - name: Create Plus Secret + run: kubectl create secret generic license-token --from-literal=license.jwt="${{ secrets.PLUS_JWT }}" --type="nginx.com/license" + if: ${{ matrix.type == 'plus' && steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }} + - name: Install Chart run: > helm install diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index a104f2f08d..4d226b19d3 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -262,7 +262,7 @@ jobs: - name: Generate WAF v5 tgz from JSON run: | - docker run --rm --user root -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/tests/data/ap-waf-v5:/data gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/nap/waf-compiler:5.3.0 -p /data/wafv5.json -o /data/wafv5.tgz + docker run --rm --user root -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/tests/data/ap-waf-v5:/data gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/nap/waf-compiler:5.4.0 -p /data/wafv5.json -o /data/wafv5.tgz if: ${{ contains(matrix.images.image, 'nap-v5')}} - name: Run Regression Tests @@ -278,6 +278,7 @@ jobs: azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }} registry-token: ${{ steps.auth.outputs.access_token }} test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}" + plus-jwt: ${{ secrets.PLUS_JWT }} - name: Upload Test Results uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 diff --git a/.github/workflows/setup-smoke.yml b/.github/workflows/setup-smoke.yml index d5a96b2956..75d5c82af6 100644 --- a/.github/workflows/setup-smoke.yml +++ b/.github/workflows/setup-smoke.yml @@ -149,7 +149,7 @@ jobs: - name: Generate WAF v5 tgz from JSON run: | - docker run --rm --user root -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/tests/data/ap-waf-v5:/data gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/nap/waf-compiler:5.3.0 -p /data/wafv5.json -o /data/wafv5.tgz + docker run --rm --user root -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/tests/data/ap-waf-v5:/data gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/nap/waf-compiler:5.4.0 -p /data/wafv5.json -o /data/wafv5.tgz if: ${{ contains(inputs.image, 'nap-v5')}} - name: Run Smoke Tests @@ -165,6 +165,7 @@ jobs: azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }} registry-token: ${{ steps.auth.outputs.access_token }} test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}" + plus-jwt: ${{ secrets.PLUS_JWT }} if: ${{ steps.stable_exists.outputs.exists != 'true' }} - name: Upload Test Results diff --git a/.github/workflows/single-image-regression.yml b/.github/workflows/single-image-regression.yml index 1c21769b3d..957f9bd5ca 100644 --- a/.github/workflows/single-image-regression.yml +++ b/.github/workflows/single-image-regression.yml @@ -109,3 +109,4 @@ jobs: azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }} registry-token: ${{ steps.auth.outputs.access_token }} test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ inputs.test-image-tag }}" + plus-jwt: ${{ secrets.PLUS_JWT }} diff --git a/build/Dockerfile b/build/Dockerfile index 7000f36020..01c480361f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.6 ARG BUILD_OS=debian -ARG NGINX_PLUS_VERSION=R32 +ARG NGINX_PLUS_VERSION=R33 ARG DOWNLOAD_TAG=edge ARG DEBIAN_FRONTEND=noninteractive ARG PREBUILT_BASE_IMG=nginx/nginx-ingress:${DOWNLOAD_TAG} @@ -198,7 +198,7 @@ RUN --mount=type=bind,from=alpine-fips-3.17,target=/tmp/fips/ \ && cp -av /tmp/fips/etc/ssl/openssl.cnf /etc/ssl/openssl.cnf \ && cp -av /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \ && ldconfig /usr/local/lib/ \ - && apk add --no-cache app-protect-module-plus~=32.5.144 \ + && apk add --no-cache app-protect-module-plus~=33.5.210 \ && sed -i -e '/nginx.com/d' /etc/apk/repositories \ && nap-waf.sh \ && if [ "${NGINX_AGENT}" = "true" ]; then \ @@ -279,7 +279,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && if [ "${NGINX_AGENT}" = "true" ]; then agent.sh; fi \ && if [ -z "${NAP_MODULES##*dos*}" ]; then nap-dos.sh; fi -############################################# Base image for Debian with NGINX Plus and App Protect WAFv5/DoS ############################################# +############################################# Base image for Debian with NGINX Plus and App Protect WAFv5 ############################################# FROM debian-plus AS debian-plus-nap-v5 ARG NAP_MODULES ARG NGINX_AGENT @@ -300,7 +300,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && apt-get update \ && if [ "${NGINX_AGENT}" = "true" ]; then apt-get install --no-install-recommends --no-install-suggests -y nginx-agent; fi \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ - apt-get install --no-install-recommends --no-install-suggests -y app-protect-plugin=6.3.0* app-protect-module-plus=32+5.144* nginx-plus-module-appprotect=32+5.144*; \ + apt-get install --no-install-recommends --no-install-suggests -y app-protect-module-plus=33+5.210*; \ rm -f /etc/apt/sources.list.d/app-protect.sources; \ nap-waf.sh; \ fi \ @@ -430,7 +430,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && if [ "${NGINX_AGENT}" = "true" ]; then microdnf --nodocs install -y nginx-agent; fi \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ cp /tmp/app-protect-9.repo /etc/yum.repos.d/app-protect-9.repo \ - && microdnf --nodocs install -y app-protect-module-plus-32+5.144* \ + && microdnf --nodocs install -y app-protect-module-plus-33+5.210* \ && nap-waf.sh \ && rm -f /etc/yum.repos.d/app-protect-9.repo; \ fi \ @@ -517,7 +517,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && dnf config-manager --set-enabled codeready-builder-for-rhel-8-x86_64-rpms \ && dnf --nodocs install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ - dnf --nodocs install -y app-protect-module-plus-32+5.144*; \ + dnf --nodocs install -y app-protect-module-plus-33+5.210*; \ fi \ && subscription-manager unregister \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 12b8aa70e9..b44e5df46c 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -112,6 +112,24 @@ Expand the name of the configmap used for NGINX Agent. {{- end -}} {{- end -}} +{{/* +Expand the name of the mgmt configmap. +*/}} +{{- define "nginx-ingress.mgmtConfigName" -}} +{{- if .Values.controller.mgmt.customConfigMap -}} +{{ .Values.controller.mgmt.customConfigMap }} +{{- else -}} +{{- default (printf "%s-mgmt" (include "nginx-ingress.fullname" .)) -}} +{{- end -}} +{{- end -}} + +{{/* +Expand license token secret name. +*/}} +{{- define "nginx-ingress.licenseTokenSecretName" -}} +{{- .Values.controller.mgmt.licenseTokenSecretName -}} +{{- end -}} + {{/* Expand leader election lock name. */}} @@ -226,6 +244,9 @@ Build the args for the service binary. - -app-protect-dos-memory={{ .Values.controller.appprotectdos.memory }} {{ end }} - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }} +{{- if .Values.controller.nginxplus }} +- -mgmt-configmap=$(POD_NAMESPACE)/{{ include "nginx-ingress.mgmtConfigName" . }} +{{- end }} {{- if .Values.controller.defaultTLS.secret }} - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }} {{ else if and (.Values.controller.defaultTLS.cert) (.Values.controller.defaultTLS.key) }} @@ -423,6 +444,8 @@ volumeMounts: env: - name: ENFORCER_PORT value: "{{ .Values.controller.appprotect.enforcer.port | default 50000 }}" + - name: ENFORCER_CONFIG_TIMEOUT + value: "0" volumeMounts: - name: app-protect-bd-config mountPath: /opt/app_protect/bd_config diff --git a/charts/nginx-ingress/templates/controller-configmap.yaml b/charts/nginx-ingress/templates/controller-configmap.yaml index 8f1d3e47bb..98769870de 100644 --- a/charts/nginx-ingress/templates/controller-configmap.yaml +++ b/charts/nginx-ingress/templates/controller-configmap.yaml @@ -30,3 +30,22 @@ data: nginx-agent.conf: |- {{ include "nginx-ingress.agentConfiguration" . | indent 4 }} {{- end }} +--- +{{- if and .Values.controller.nginxplus (eq (.Values.controller.mgmt.customConfigMap | default "") "") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nginx-ingress.mgmtConfigName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +{{- if .Values.controller.config.annotations }} + annotations: +{{ toYaml .Values.controller.config.annotations | indent 4 }} +{{- end }} +data: + license-token-secret-name: {{ include "nginx-ingress.licenseTokenSecretName" . }} +{{- if hasKey .Values.controller.mgmt "enforceInitialReport" }} + enforce-initial-report: {{ quote .Values.controller.mgmt.enforceInitialReport }} +{{- end }} +{{- end }} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 60f7d68eac..28d8e1a856 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -94,6 +94,37 @@ } ] }, + "mgmt": { + "type": "object", + "default": {}, + "title": "The mgmt block Schema", + "properties": { + "licenseTokenSecretName": { + "type": "string", + "default": "", + "title": "The licenseTokenSecretName Schema", + "examples": [ + "nginx-plus-secret", + "license-token", + "license" + ] + }, + "enforceInitialReport": { + "type": "boolean", + "default": false, + "title": "The enforceInitialReport Schema", + "examples": [ + true, + false + ] + } + }, + "examples": [ + { + "licenseTokenSecretName": "license-token" + } + ] + }, "nginxReloadTimeout": { "type": "integer", "default": 0, @@ -208,10 +239,10 @@ }, "tag": { "type": "string", - "default": "5.3.0", + "default": "5.4.0", "title": "The tag of the App Protect WAF v5 Enforcer image", "examples": [ - "5.3.0" + "5.4.0" ] }, "digest": { @@ -248,7 +279,7 @@ "examples": [ { "repository": "private-registry.nginx.com/nap/waf-enforcer", - "tag": "5.3.0", + "tag": "5.4.0", "pullPolicy": "IfNotPresent" } ] @@ -282,10 +313,10 @@ }, "tag": { "type": "string", - "default": "5.3.0", + "default": "5.4.0", "title": "The tag of the App Protect WAF v5 Config Manager image", "examples": [ - "5.3.0" + "5.4.0" ] }, "digest": { @@ -322,7 +353,7 @@ "examples": [ { "repository": "private-registry.nginx.com/nap/waf-config-mgr", - "tag": "5.3.0", + "tag": "5.4.0", "pullPolicy": "IfNotPresent" } ] @@ -1698,7 +1729,7 @@ "port": 50000, "image": { "repository": "private-registry.nginx.com/nap/waf-enforcer", - "tag": "5.3.0", + "tag": "5.4.0", "pullPolicy": "IfNotPresent" }, "securityContext": {} @@ -1706,7 +1737,7 @@ "configManager": { "image": { "repository": "private-registry.nginx.com/nap/waf-config-mgr", - "tag": "5.3.0", + "tag": "5.4.0", "pullPolicy": "IfNotPresent" }, "securityContext": { @@ -2312,7 +2343,7 @@ "port": 50000, "image": { "repository": "private-registry.nginx.com/nap/waf-enforcer", - "tag": "5.3.0", + "tag": "5.4.0", "pullPolicy": "IfNotPresent" }, "securityContext": {} @@ -2320,7 +2351,7 @@ "configManager": { "image": { "repository": "private-registry.nginx.com/nap/waf-config-mgr", - "tag": "5.3.0", + "tag": "5.4.0", "pullPolicy": "IfNotPresent" }, "securityContext": { diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index 088c337d2d..5bcd9e42c6 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -14,6 +14,14 @@ controller: ## Deploys the Ingress Controller for NGINX Plus. nginxplus: false + ## Configures NGINX mgmt block for NGINX Plus + mgmt: + ## Secret name of license token for NGINX Plus + licenseTokenSecretName: "license-token" # required for NGINX Plus + + ## Enables the 180-day grace period for sending the initial usage report + # enforceInitialReport: false + ## Timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. nginxReloadTimeout: 60000 @@ -49,7 +57,7 @@ controller: repository: private-registry.nginx.com/nap/waf-enforcer ## The tag of the App Protect WAF v5 Enforcer image. - tag: "5.3.0" + tag: "5.4.0" ## The digest of the App Protect WAF v5 Enforcer image. ## If digest is specified it has precedence over tag and will be used instead # digest: "sha256:CHANGEME" @@ -65,7 +73,7 @@ controller: repository: private-registry.nginx.com/nap/waf-config-mgr ## The tag of the App Protect WAF v5 Configuration Manager image. - tag: "5.3.0" + tag: "5.4.0" ## The digest of the App Protect WAF v5 Configuration Manager image. ## If digest is specified it has precedence over tag and will be used instead # digest: "sha256:CHANGEME" diff --git a/charts/tests/__snapshots__/helmunit_test.snap b/charts/tests/__snapshots__/helmunit_test.snap index a7c1beac8f..25f74e32d3 100755 --- a/charts/tests/__snapshots__/helmunit_test.snap +++ b/charts/tests/__snapshots__/helmunit_test.snap @@ -29,6 +29,22 @@ metadata: data: {} /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: appprotect-dos-nginx-ingress-mgmt + namespace: appprotect-dos + labels: + helm.sh/chart: nginx-ingress-1.5.0 + app.kubernetes.io/name: nginx-ingress + app.kubernetes.io/instance: appprotect-dos + app.kubernetes.io/version: "4.0.0" + app.kubernetes.io/managed-by: Helm +data: + license-token-secret-name: license-token +/-/-/-/ # Source: nginx-ingress/templates/controller-leader-election-configmap.yaml apiVersion: v1 kind: ConfigMap @@ -379,6 +395,7 @@ spec: - -app-protect-dos-memory=1024 - -nginx-configmaps=$(POD_NAMESPACE)/appprotect-dos-nginx-ingress + - -mgmt-configmap=$(POD_NAMESPACE)/appprotect-dos-nginx-ingress-mgmt - -ingress-class=nginx - -health-status=false - -health-status-uri=/nginx-health @@ -472,6 +489,22 @@ metadata: data: {} /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: appprotect-waf-nginx-ingress-mgmt + namespace: appprotect-waf + labels: + helm.sh/chart: nginx-ingress-1.5.0 + app.kubernetes.io/name: nginx-ingress + app.kubernetes.io/instance: appprotect-waf + app.kubernetes.io/version: "4.0.0" + app.kubernetes.io/managed-by: Helm +data: + license-token-secret-name: license-token +/-/-/-/ # Source: nginx-ingress/templates/controller-leader-election-configmap.yaml apiVersion: v1 kind: ConfigMap @@ -817,6 +850,7 @@ spec: - -enable-app-protect=true - -enable-app-protect-dos=false - -nginx-configmaps=$(POD_NAMESPACE)/appprotect-waf-nginx-ingress + - -mgmt-configmap=$(POD_NAMESPACE)/appprotect-waf-nginx-ingress-mgmt - -ingress-class=nginx - -health-status=false - -health-status-uri=/nginx-health @@ -910,6 +944,22 @@ metadata: data: {} /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: appprotect-wafv5-nginx-ingress-mgmt + namespace: appprotect-wafv5 + labels: + helm.sh/chart: nginx-ingress-1.5.0 + app.kubernetes.io/name: nginx-ingress + app.kubernetes.io/instance: appprotect-wafv5 + app.kubernetes.io/version: "4.0.0" + app.kubernetes.io/managed-by: Helm +data: + license-token-secret-name: license-token +/-/-/-/ # Source: nginx-ingress/templates/controller-leader-election-configmap.yaml apiVersion: v1 kind: ConfigMap @@ -1272,6 +1322,7 @@ spec: - -app-protect-enforcer-address="localhost:50001" - -enable-app-protect-dos=false - -nginx-configmaps=$(POD_NAMESPACE)/appprotect-wafv5-nginx-ingress + - -mgmt-configmap=$(POD_NAMESPACE)/appprotect-wafv5-nginx-ingress-mgmt - -ingress-class=nginx - -health-status=false - -health-status-uri=/nginx-health @@ -1308,16 +1359,18 @@ spec: - -weight-changes-dynamic-reload=false - name: waf-enforcer - image: my.private.reg/nap/waf-enforcer:5.3.0 + image: my.private.reg/nap/waf-enforcer:5.4.0 imagePullPolicy: "IfNotPresent" env: - name: ENFORCER_PORT value: "50001" + - name: ENFORCER_CONFIG_TIMEOUT + value: "0" volumeMounts: - name: app-protect-bd-config mountPath: /opt/app_protect/bd_config - name: waf-config-mgr - image: my.private.reg/nap/waf-config-mgr:5.3.0 + image: my.private.reg/nap/waf-config-mgr:5.4.0 imagePullPolicy: "IfNotPresent" securityContext: @@ -1750,6 +1803,9 @@ metadata: spec: controller: nginx.org/ingress-controller /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +/-/-/-/ # Source: nginx-ingress/templates/controller-lease.yaml apiVersion: coordination.k8s.io/v1 kind: Lease @@ -2178,6 +2234,9 @@ metadata: spec: controller: nginx.org/ingress-controller /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +/-/-/-/ # Source: nginx-ingress/templates/controller-lease.yaml apiVersion: coordination.k8s.io/v1 kind: Lease @@ -2606,6 +2665,9 @@ metadata: spec: controller: nginx.org/ingress-controller /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +/-/-/-/ # Source: nginx-ingress/templates/controller-lease.yaml apiVersion: coordination.k8s.io/v1 kind: Lease @@ -3035,6 +3097,9 @@ metadata: spec: controller: nginx.org/ingress-controller /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +/-/-/-/ # Source: nginx-ingress/templates/controller-globalconfiguration.yaml apiVersion: k8s.nginx.org/v1 kind: GlobalConfiguration @@ -3486,6 +3551,9 @@ metadata: spec: controller: nginx.org/ingress-controller /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +/-/-/-/ # Source: nginx-ingress/templates/controller-lease.yaml apiVersion: coordination.k8s.io/v1 kind: Lease @@ -3914,6 +3982,9 @@ metadata: spec: controller: nginx.org/ingress-controller /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +/-/-/-/ # Source: nginx-ingress/templates/controller-lease.yaml apiVersion: coordination.k8s.io/v1 kind: Lease @@ -3958,6 +4029,22 @@ metadata: data: {} /-/-/-/ +# Source: nginx-ingress/templates/controller-configmap.yaml +/-/-/-/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: plus-nginx-ingress-mgmt + namespace: default + labels: + helm.sh/chart: nginx-ingress-1.5.0 + app.kubernetes.io/name: nginx-ingress + app.kubernetes.io/instance: plus + app.kubernetes.io/version: "4.0.0" + app.kubernetes.io/managed-by: Helm +data: + license-token-secret-name: license-token +/-/-/-/ # Source: nginx-ingress/templates/controller-leader-election-configmap.yaml apiVersion: v1 kind: ConfigMap @@ -4293,6 +4380,7 @@ spec: - -enable-app-protect=false - -enable-app-protect-dos=false - -nginx-configmaps=$(POD_NAMESPACE)/plus-nginx-ingress + - -mgmt-configmap=$(POD_NAMESPACE)/plus-nginx-ingress-mgmt - -ingress-class=nginx - -health-status=false - -health-status-uri=/nginx-health diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index e1b3f92fa7..26ba224e0f 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -55,6 +55,11 @@ var ( but the Ingress Controller is not able to fetch it from Kubernetes API, the Ingress Controller will fail to start. Format: /`) + mgmtConfigMap = flag.String("mgmt-configmap", "", + `A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set, + but the Ingress Controller is not able to fetch it from Kubernetes API, the Ingress Controller will fail to start. + Format: /`) + nginxPlus = flag.Bool("nginx-plus", false, "Enable support for NGINX Plus") appProtect = flag.Bool("enable-app-protect", false, "Enable support for NGINX App Protect. Requires -nginx-plus.") @@ -258,6 +263,11 @@ func initValidate(ctx context.Context) { *enableDynamicWeightChangesReload = false } + if *mgmtConfigMap != "" && !*nginxPlus { + nl.Warn(l, "mgmt-configmap flag requires -nginx-plus, mgmt configmap will not be used") + *mgmtConfigMap = "" + } + mustValidateInitialChecks(ctx) mustValidateWatchedNamespaces(ctx) mustValidateFlags(ctx) @@ -419,6 +429,10 @@ func mustValidateFlags(ctx context.Context) { if *agent && !*appProtect { nl.Fatal(l, "NGINX Agent is used to enable the Security Monitoring dashboard and requires NGINX App Protect to be enabled") } + + if *nginxPlus && *mgmtConfigMap == "" { + nl.Fatal(l, "NGINX Plus requires a mgmt ConfigMap to be set") + } } // validateNamespaceNames validates the namespaces are in the correct format diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 7476d99388..d8c4d92f98 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -78,6 +78,7 @@ const ( appProtectv5BundleFolder = "/etc/app_protect/bundles/" fatalEventFlushTime = 200 * time.Millisecond secretErrorReason = "SecretError" + configMapErrorReason = "ConfigMapError" ) func main() { @@ -150,6 +151,14 @@ func main() { go updateSelfWithVersionInfo(ctx, eventRecorder, kubeClient, version, appProtectVersion, agentVersion, nginxVersion, 10, time.Second*5) + var mgmtCfgParams *configs.MGMTConfigParams + if *nginxPlus { + mgmtCfgParams = processMGMTConfigMap(kubeClient, configs.NewDefaultMGMTConfigParams(ctx), eventRecorder, pod) + if err := processLicenseSecret(kubeClient, nginxManager, mgmtCfgParams, controllerNamespace); err != nil { + logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) + } + } + templateExecutor, templateExecutorV2 := createTemplateExecutors(ctx) sslRejectHandshake, err := processDefaultServerSecret(kubeClient, nginxManager) @@ -199,7 +208,7 @@ func main() { AppProtectBundlePath: appProtectBundlePath, } - mustProcessNginxConfig(staticCfgParams, cfgParams, templateExecutor, nginxManager) + mustProcessNginxConfig(staticCfgParams, cfgParams, mgmtCfgParams, templateExecutor, nginxManager) if *enableTLSPassthrough { var emptyFile []byte @@ -215,6 +224,7 @@ func main() { NginxManager: nginxManager, StaticCfgParams: staticCfgParams, Config: cfgParams, + MGMTCfgParams: mgmtCfgParams, TemplateExecutor: templateExecutor, TemplateExecutorV2: templateExecutorV2, LatencyCollector: latencyCollector, @@ -266,6 +276,7 @@ func main() { LeaderElectionLockName: *leaderElectionLockName, WildcardTLSSecret: *wildcardTLSSecret, ConfigMaps: *nginxConfigMaps, + MGMTConfigMap: *mgmtConfigMap, GlobalConfiguration: *globalConfiguration, AreCustomResourcesEnabled: *enableCustomResources, EnableOIDC: *enableOIDC, @@ -616,6 +627,22 @@ func processWildcardSecret(kubeClient *kubernetes.Clientset, nginxManager nginx. return isWildcardEnabled, nil } +func processLicenseSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, mgmtCfgParams *configs.MGMTConfigParams, controllerNamespace string) error { + licenseSecretNsName := controllerNamespace + "/" + mgmtCfgParams.Secrets.License + + secret, err := getAndValidateSecret(kubeClient, licenseSecretNsName, secrets.SecretTypeLicense) + if err != nil { + return fmt.Errorf("license secret: %w", err) + } + + bytes, err := configs.GenerateLicenseSecret(secret) + if err != nil { + return err + } + nginxManager.CreateSecret(configs.LicenseSecretFileName, bytes, nginx.ReadWriteOnlyFileMode) + return nil +} + func createGlobalConfigurationValidator() *cr_validation.GlobalConfigurationValidator { forbiddenListenerPorts := map[int]bool{ 80: true, @@ -642,9 +669,9 @@ func createGlobalConfigurationValidator() *cr_validation.GlobalConfigurationVali // mustProcessNginxConfig calls internally os.Exit // if can't generate a valid NGINX config. -func mustProcessNginxConfig(staticCfgParams *configs.StaticConfigParams, cfgParams *configs.ConfigParams, templateExecutor *version1.TemplateExecutor, nginxManager nginx.Manager) { +func mustProcessNginxConfig(staticCfgParams *configs.StaticConfigParams, cfgParams *configs.ConfigParams, mgmtCfgParams *configs.MGMTConfigParams, templateExecutor *version1.TemplateExecutor, nginxManager nginx.Manager) { l := nl.LoggerFromContext(cfgParams.Context) - ngxConfig := configs.GenerateNginxMainConfig(staticCfgParams, cfgParams) + ngxConfig := configs.GenerateNginxMainConfig(staticCfgParams, cfgParams, mgmtCfgParams) content, err := templateExecutor.ExecuteMainConfigTemplate(ngxConfig) if err != nil { nl.Fatalf(l, "Error generating NGINX main config: %v", err) @@ -683,7 +710,7 @@ func getAndValidateSecret(kubeClient *kubernetes.Clientset, secretNsName string, } secret, err = kubeClient.CoreV1().Secrets(ns).Get(context.TODO(), name, meta_v1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("could not get %v: %w", secretNsName, err) + return nil, fmt.Errorf("could not find %v: %w", secretNsName, err) } switch secretType { case api_v1.SecretTypeTLS: @@ -691,6 +718,11 @@ func getAndValidateSecret(kubeClient *kubernetes.Clientset, secretNsName string, if err != nil { return nil, fmt.Errorf("%v is invalid: %w", secretNsName, err) } + case secrets.SecretTypeLicense: + err = secrets.ValidateLicenseSecret(secret) + if err != nil { + return nil, err + } } return secret, nil } @@ -909,6 +941,24 @@ func processConfigMaps(kubeClient *kubernetes.Clientset, cfgParams *configs.Conf return cfgParams } +func processMGMTConfigMap(kubeClient *kubernetes.Clientset, mgmtCfgParams *configs.MGMTConfigParams, eventLog record.EventRecorder, pod *api_v1.Pod) *configs.MGMTConfigParams { + ctx := mgmtCfgParams.Context + var fatalErr error + + ns, name, err := k8s.ParseNamespaceName(*mgmtConfigMap) + if err != nil { + logEventAndExit(ctx, eventLog, pod, configMapErrorReason, fmt.Errorf("error parsing the mgmt-configmap argument: %w", err)) + } + cfm, err := kubeClient.CoreV1().ConfigMaps(ns).Get(context.TODO(), name, meta_v1.GetOptions{}) + if err != nil { + logEventAndExit(ctx, eventLog, cfm, configMapErrorReason, fmt.Errorf("error when getting mgmt-configmap [%v]: %w", *mgmtConfigMap, err)) + } + if mgmtCfgParams, _, fatalErr = configs.ParseMGMTConfigMap(ctx, cfm, eventLog); fatalErr != nil { + logEventAndExit(ctx, eventLog, cfm, secretErrorReason, fatalErr) + } + return mgmtCfgParams +} + func updateSelfWithVersionInfo(ctx context.Context, eventLog record.EventRecorder, kubeClient *kubernetes.Clientset, version, appProtectVersion, agentVersion string, nginxVersion nginx.Version, maxRetries int, waitTime time.Duration) { l := nl.LoggerFromContext(ctx) podUpdated := false diff --git a/deployments/common/plus-mgmt-configmap.yaml b/deployments/common/plus-mgmt-configmap.yaml new file mode 100644 index 0000000000..8b64038ac4 --- /dev/null +++ b/deployments/common/plus-mgmt-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config-mgmt + namespace: nginx-ingress +data: + license-token-secret-name: "license-token" diff --git a/deployments/daemon-set/nginx-plus-ingress.yaml b/deployments/daemon-set/nginx-plus-ingress.yaml index 6599d2b1a7..339775eefb 100644 --- a/deployments/daemon-set/nginx-plus-ingress.yaml +++ b/deployments/daemon-set/nginx-plus-ingress.yaml @@ -89,6 +89,7 @@ spec: args: - -nginx-plus - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config + - -mgmt-configmap=$(POD_NAMESPACE)/nginx-config-mgmt - -report-ingress-status - -external-service=nginx-ingress #- -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml index e6f575747d..9900c5f992 100644 --- a/deployments/deployment/nginx-plus-ingress.yaml +++ b/deployments/deployment/nginx-plus-ingress.yaml @@ -90,6 +90,7 @@ spec: args: - -nginx-plus - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config + - -mgmt-configmap=$(POD_NAMESPACE)/nginx-config-mgmt - -report-ingress-status - -external-service=nginx-ingress #- -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret diff --git a/internal/configs/commonhelpers/common_template_helpers.go b/internal/configs/commonhelpers/common_template_helpers.go index b5727291e7..d80b1f60de 100644 --- a/internal/configs/commonhelpers/common_template_helpers.go +++ b/internal/configs/commonhelpers/common_template_helpers.go @@ -13,3 +13,12 @@ func MakeSecretPath(path, defaultPath, variable string, useVariable bool) string } return path } + +// MakeOnOffFromBool will return a string on | off from a boolean pointer +func MakeOnOffFromBool(b *bool) string { + if b == nil || !*b { + return "off" + } + + return "on" +} diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index a1bd883d6b..283a10f114 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -175,6 +175,18 @@ type Listener struct { Protocol string } +// MGMTSecrets holds mgmt block secret names +type MGMTSecrets struct { + License string +} + +// MGMTConfigParams holds mgmt block parameters. +type MGMTConfigParams struct { + Context context.Context + EnforceInitialReport *bool + Secrets MGMTSecrets +} + // NewDefaultConfigParams creates a ConfigParams with default values. func NewDefaultConfigParams(ctx context.Context, isPlus bool) *ConfigParams { upstreamZoneSize := "256k" @@ -219,3 +231,12 @@ func NewDefaultConfigParams(ctx context.Context, isPlus bool) *ConfigParams { LimitReqRejectCode: 429, } } + +// NewDefaultMGMTConfigParams creates a ConfigParams with mgmt values. +func NewDefaultMGMTConfigParams(ctx context.Context) *MGMTConfigParams { + return &MGMTConfigParams{ + Context: ctx, + EnforceInitialReport: nil, + Secrets: MGMTSecrets{}, + } +} diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index c1f2111932..1b339ddeda 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -2,6 +2,7 @@ package configs import ( "context" + "errors" "fmt" "strings" @@ -656,8 +657,45 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has return cfgParams, configOk } +// ParseMGMTConfigMap parses the mgmt block ConfigMap into MGMTConfigParams. +// +//nolint:gocyclo +func ParseMGMTConfigMap(ctx context.Context, cfgm *v1.ConfigMap, eventLog record.EventRecorder) (*MGMTConfigParams, bool, error) { + l := nl.LoggerFromContext(ctx) + configWarnings := false + + mgmtCfgParams := NewDefaultMGMTConfigParams(ctx) + + license, licenseExists := cfgm.Data["license-token-secret-name"] + trimmedLicense := strings.TrimSpace(license) + if !licenseExists || trimmedLicense == "" { + errorText := fmt.Sprintf("Configmap %s/%s: Missing or empty value for the license-token-secret-name key. Failing.", cfgm.GetNamespace(), cfgm.GetName()) + return nil, true, errors.New(errorText) + } + mgmtCfgParams.Secrets.License = trimmedLicense + + if enforceInitialReport, exists, err := GetMapKeyAsBool(cfgm.Data, "enforce-initial-report", cfgm); exists { + if err != nil { + errorText := fmt.Sprintf("Configmap %s/%s: Invalid value for the enforce-initial-report key: got %t: %v. Ignoring.", cfgm.GetNamespace(), cfgm.GetName(), enforceInitialReport, err) + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, invalidValueReason, errorText) + configWarnings = true + } else { + mgmtCfgParams.EnforceInitialReport = BoolToPointerBool(enforceInitialReport) + } + } + + return mgmtCfgParams, configWarnings, nil +} + // GenerateNginxMainConfig generates MainConfig. -func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *ConfigParams) *version1.MainConfig { +func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *ConfigParams, mgmtCfgParams *MGMTConfigParams) *version1.MainConfig { + var mgmtConfig version1.MGMTConfig + if mgmtCfgParams != nil { + mgmtConfig = version1.MGMTConfig{ + EnforceInitialReport: mgmtCfgParams.EnforceInitialReport, + } + } nginxCfg := &version1.MainConfig{ AccessLog: config.MainAccessLog, DefaultServerAccessLogOff: config.DefaultServerAccessLogOff, @@ -675,6 +713,7 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config LogFormat: config.MainLogFormat, LogFormatEscaping: config.MainLogFormatEscaping, MainSnippets: config.MainMainSnippets, + MGMTConfig: mgmtConfig, NginxStatus: staticCfgParams.NginxStatus, NginxStatusAllowCIDRs: staticCfgParams.NginxStatusAllowCIDRs, NginxStatusPort: staticCfgParams.NginxStatusPort, diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index 57b031dd7c..084c2f25d8 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -285,6 +285,171 @@ func TestParseConfigMapAccessLogDefault(t *testing.T) { } } +func TestParseMGMTConfigMapError(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "license-token-secret-name": "", + }, + }, + msg: "Must have license-token-secret-name", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{}, + }, + msg: "Must have license-token-secret-name key", + }, + } + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + _, _, err := ParseMGMTConfigMap(context.Background(), test.configMap, makeEventLogger()) + + if err == nil { + t.Errorf("Expected error, got nil") + } + }) + } +} + +func TestParseMGMTConfigMapWarnings(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "license-token-secret-name": "license-token", + "enforce-initial-report": "7", + }, + }, + msg: "enforce-initial-report is invalid", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "license-token-secret-name": "license-token", + "enforce-initial-report": "", + }, + }, + msg: "enforce-initial-report set empty", + }, + } + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + _, configWarnings, err := ParseMGMTConfigMap(context.Background(), test.configMap, makeEventLogger()) + if err != nil { + t.Errorf("expected nil, got err: %v", err) + } + if !configWarnings { + t.Fatal("Expected warnings, got none") + } + }) + } +} + +func TestParseMGMTConfigMapLicense(t *testing.T) { + t.Parallel() + test := struct { + configMap *v1.ConfigMap + want *MGMTConfigParams + msg string + }{ + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "license-token-secret-name": "license-token", + }, + }, + want: &MGMTConfigParams{ + Secrets: MGMTSecrets{ + License: "license-token", + }, + }, + msg: "Has only license-token-secret-name", + } + + t.Run(test.msg, func(t *testing.T) { + result, warnings, err := ParseMGMTConfigMap(context.Background(), test.configMap, makeEventLogger()) + if err != nil { + t.Fatal(err) + } + if warnings { + t.Fatal("Unexpected warnings") + } + if result.Secrets.License != test.want.Secrets.License { + t.Errorf("LicenseTokenSecretNane: want %q, got %q", test.want.Secrets.License, result.Secrets.License) + } + }) +} + +func TestParseMGMTConfigMapEnforceInitialReport(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + want *MGMTConfigParams + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "license-token-secret-name": "license-token", + "enforce-initial-report": "false", + }, + }, + want: &MGMTConfigParams{ + EnforceInitialReport: BoolToPointerBool(false), + Secrets: MGMTSecrets{ + License: "license-token", + }, + }, + msg: "enforce-initial-report set to false", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "license-token-secret-name": "license-token", + "enforce-initial-report": "true", + }, + }, + want: &MGMTConfigParams{ + EnforceInitialReport: BoolToPointerBool(true), + Secrets: MGMTSecrets{ + License: "license-token", + }, + }, + msg: "enforce-initial-report set to true", + }, + } + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + result, warnings, err := ParseMGMTConfigMap(context.Background(), test.configMap, makeEventLogger()) + if err != nil { + t.Fatal(err) + } + if warnings { + t.Error("Unexpected warnings") + } + + if result.EnforceInitialReport == nil { + t.Errorf("EnforceInitialReport: want %v, got nil", *test.want.EnforceInitialReport) + } + if *result.EnforceInitialReport != *test.want.EnforceInitialReport { + t.Errorf("EnforceInitialReport: want %v, got %v", *test.want.EnforceInitialReport, *result.EnforceInitialReport) + } + }) + } +} + func makeEventLogger() record.EventRecorder { return record.NewFakeRecorder(1024) } diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index feb663d65a..6fc85c841e 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -53,6 +53,9 @@ const DefaultServerSecretFileName = "default" // WildcardSecretFileName is the filename of the Secret with a TLS cert and a key for the ingress resources with TLS termination enabled but not secret defined. const WildcardSecretFileName = "wildcard" +// LicenseSecretFileName is the filename of the Secret for the NGINX PLUS License +const LicenseSecretFileName = "license.jwt" + // JWTKeyKey is the key of the data field of a Secret where the JWK must be stored. const JWTKeyKey = "jwk" @@ -120,6 +123,7 @@ type Configurator struct { nginxManager nginx.Manager staticCfgParams *StaticConfigParams CfgParams *ConfigParams + MgmtCfgParams *MGMTConfigParams templateExecutor *version1.TemplateExecutor templateExecutorV2 *version2.TemplateExecutor ingresses map[string]*IngressEx @@ -146,6 +150,7 @@ type ConfiguratorParams struct { NginxManager nginx.Manager StaticCfgParams *StaticConfigParams Config *ConfigParams + MGMTCfgParams *MGMTConfigParams TemplateExecutor *version1.TemplateExecutor TemplateExecutorV2 *version2.TemplateExecutor LabelUpdater collector.LabelUpdater @@ -177,6 +182,7 @@ func NewConfigurator(p ConfiguratorParams) *Configurator { nginxManager: p.NginxManager, staticCfgParams: p.StaticCfgParams, CfgParams: p.Config, + MgmtCfgParams: p.MGMTCfgParams, ingresses: make(map[string]*IngressEx), virtualServers: make(map[string]*VirtualServerEx), transportServers: make(map[string]*TransportServerEx), @@ -916,6 +922,19 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloa return allWarnings, nil } +// AddOrUpdateLicenseSecret adds or updates NGINX Plus license secret. +func (cnf *Configurator) AddOrUpdateLicenseSecret(secret *api_v1.Secret) error { + l := nl.LoggerFromContext(cnf.CfgParams.Context) + nl.Debugf(l, "AddOrUpdateLicenseSecret: [%v]", secret.Name) + data, err := GenerateLicenseSecret(secret) + if err != nil { + return err + } + cnf.nginxManager.CreateSecret(LicenseSecretFileName, data, nginx.ReadWriteOnlyFileMode) + + return nil +} + func (cnf *Configurator) addOrUpdateTLSSecret(secret *api_v1.Secret) string { name := objectMetaToFileName(&secret.ObjectMeta) data := GenerateCertAndKeyFileContent(secret) @@ -955,6 +974,19 @@ func GenerateCAFileContent(secret *api_v1.Secret) ([]byte, []byte) { return caKey.Bytes(), caCrl.Bytes() } +// GenerateLicenseSecret generates jwt content from the License secret which is required for NGINX Plus. +func GenerateLicenseSecret(secret *api_v1.Secret) ([]byte, error) { + var licenseKey bytes.Buffer + + data, exists := secret.Data[LicenseSecretFileName] + if !exists { + return nil, fmt.Errorf("license secret %s/%s must contain the key %s", secret.Namespace, secret.Name, LicenseSecretFileName) + } + licenseKey.Write(data) + + return licenseKey.Bytes(), nil +} + // DeleteIngress deletes NGINX configuration for the Ingress resource. func (cnf *Configurator) DeleteIngress(key string, skipReload bool) error { name := keyToFileName(key) @@ -1290,8 +1322,9 @@ func (cnf *Configurator) updateStreamServersInPlus(upstream string, servers []st // UpdateConfig updates NGINX configuration parameters. // //gocyclo:ignore -func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources ExtendedResources) (Warnings, error) { +func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, mgmtCfgParams *MGMTConfigParams, resources ExtendedResources) (Warnings, error) { cnf.CfgParams = cfgParams + cnf.MgmtCfgParams = mgmtCfgParams allWarnings := newWarnings() allWeightUpdates := []WeightUpdate{} @@ -1344,7 +1377,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende cnf.templateExecutorV2.UseOriginalTStemplate() } - mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cfgParams) + mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cfgParams, mgmtCfgParams) mainCfgContent, err := cnf.templateExecutor.ExecuteMainConfigTemplate(mainCfg) if err != nil { return allWarnings, fmt.Errorf("error when writing main Config") @@ -1950,7 +1983,7 @@ func (cnf *Configurator) DeleteAppProtectDosAllowList(obj *v1beta1.DosProtectedR func (cnf *Configurator) AddInternalRouteConfig() error { cnf.staticCfgParams.EnableInternalRoutes = true cnf.staticCfgParams.InternalRouteServerName = fmt.Sprintf("%s.%s.svc", os.Getenv("POD_SERVICEACCOUNT"), os.Getenv("POD_NAMESPACE")) - mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cnf.CfgParams) + mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cnf.CfgParams, cnf.MgmtCfgParams) mainCfgContent, err := cnf.templateExecutor.ExecuteMainConfigTemplate(mainCfg) if err != nil { return fmt.Errorf("error when writing main Config: %w", err) @@ -1977,6 +2010,8 @@ func (cnf *Configurator) AddOrUpdateSecret(secret *api_v1.Secret) string { case secrets.SecretTypeAPIKey: // APIKey ClientSecret is not required on the filesystem, it is written directly to the config file. return "" + case secrets.SecretTypeLicense: + return "" default: return cnf.addOrUpdateTLSSecret(secret) } diff --git a/internal/configs/configurator_test.go b/internal/configs/configurator_test.go index 1e13034db7..62c157eadf 100644 --- a/internal/configs/configurator_test.go +++ b/internal/configs/configurator_test.go @@ -2,6 +2,7 @@ package configs import ( "context" + "encoding/base64" "encoding/json" "os" "reflect" @@ -51,6 +52,7 @@ func createTestConfigurator(t *testing.T) *Configurator { NginxManager: manager, StaticCfgParams: createTestStaticConfigParams(), Config: NewDefaultConfigParams(context.Background(), false), + MGMTCfgParams: NewDefaultMGMTConfigParams(context.Background()), TemplateExecutor: templateExecutor, TemplateExecutorV2: templateExecutorV2, LatencyCollector: nil, @@ -101,7 +103,7 @@ func TestConfiguratorUpdatesConfigWithNilCustomMainTemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ MainTemplate: nil, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -119,7 +121,7 @@ func TestConfiguratorUpdatesConfigWithCustomMainTemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ MainTemplate: &customTestMainTemplate, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -141,7 +143,7 @@ func TestConfiguratorUpdatesConfigWithNilCustomIngressTemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ IngressTemplate: nil, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -159,7 +161,7 @@ func TestConfiguratorUpdatesConfigWithCustomIngressTemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ IngressTemplate: &customTestIngressTemplate, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -181,7 +183,7 @@ func TestConfigratorUpdatesConfigWithCustomVStemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ VirtualServerTemplate: &customTestVStemplate, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -203,7 +205,7 @@ func TestConfiguratorUpdatesConfigWithNilCustomVSemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ VirtualServerTemplate: nil, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -221,7 +223,7 @@ func TestConfigratorUpdatesConfigWithCustomTStemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ TransportServerTemplate: &customTestTStemplate, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -243,7 +245,7 @@ func TestConfiguratorUpdatesConfigWithNilCustomTStemplate(t *testing.T) { cnf := createTestConfigurator(t) warnings, err := cnf.UpdateConfig(&ConfigParams{ TransportServerTemplate: nil, - }, ExtendedResources{}) + }, &MGMTConfigParams{}, ExtendedResources{}) if err != nil { t.Fatal(err) } @@ -255,6 +257,31 @@ func TestConfiguratorUpdatesConfigWithNilCustomTStemplate(t *testing.T) { } } +func TestAddOrUpdateLicenseSecret(t *testing.T) { + t.Parallel() + cnf := createTestConfigurator(t) + cnf.MgmtCfgParams.Secrets.License = "default/license-token" + license := api_v1.Secret{ + TypeMeta: meta_v1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: "license-token", + Namespace: "default", + }, + Data: map[string][]byte{ + "license.jwt": []byte(base64.StdEncoding.EncodeToString([]byte("license-token"))), + }, + Type: "nginx.com/license", + } + + err := cnf.AddOrUpdateLicenseSecret(&license) + if err != nil { + t.Errorf("AddOrUpdateLicenseSecret returned: \n%v, but expected: \n%v", err, nil) + } +} + func TestAddOrUpdateIngress(t *testing.T) { t.Parallel() cnf := createTestConfigurator(t) diff --git a/internal/configs/parsing_helpers.go b/internal/configs/parsing_helpers.go index 494b19e1e6..5c31a5b1d8 100644 --- a/internal/configs/parsing_helpers.go +++ b/internal/configs/parsing_helpers.go @@ -413,3 +413,8 @@ func VerifyAppProtectThresholds(value string) bool { func VerifyPath(s string) bool { return pathRegexp.MatchString(s) } + +// BoolToPointerBool turns a bool into a pointer bool +func BoolToPointerBool(b bool) *bool { + return &b +} diff --git a/internal/configs/version1/__snapshots__/template_test.snap b/internal/configs/version1/__snapshots__/template_test.snap index 7f4182f0bf..1b4abd2a65 100644 --- a/internal/configs/version1/__snapshots__/template_test.snap +++ b/internal/configs/version1/__snapshots__/template_test.snap @@ -269,144 +269,10 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } ---- - -[TestExecuteMainTemplateForNGINXPlusR31 - 1] -worker_processes auto; -worker_rlimit_nofile 65536; -worker_cpu_affinity auto; -worker_shutdown_timeout 1m; - -daemon off; - -error_log stderr ; -pid /var/lib/nginx/nginx.pid; -load_module modules/ngx_fips_check_module.so; - -load_module modules/ngx_http_js_module.so; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - map_hash_max_size ; - map_hash_bucket_size ; - - js_import /etc/nginx/njs/apikey_auth.js; - js_set $apikey_auth_hash apikey_auth.hash; - - log_format main escape=default - '$remote_addr' - ' $remote_user' - ; - - map $upstream_trailer_grpc_status $grpc_status { - default $upstream_trailer_grpc_status; - '' $sent_http_grpc_status; - } - app_protect_enforcer_address enforcer.svc.local; - - access_log /dev/stdout main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65s; - keepalive_requests 100; - - #gzip on; - - server_names_hash_max_size 512; - - - variables_hash_bucket_size 256; - variables_hash_max_size 1024; - - map $request_uri $request_uri_no_args { - "~^(?P[^?]*)(\?.*)?$" $path; - } - - map $http_upgrade $connection_upgrade { - default upgrade; - '' close; - } - map $http_upgrade $vs_connection_header { - default upgrade; - '' $default_connection_header; - } - resolver example.com 127.0.0.1 valid=10s ipv6=off; - resolver_timeout 15s; - - server { - # required to support the Websocket protocol in VirtualServer/VirtualServerRoutes - set $default_connection_header ""; - set $resource_type ""; - set $resource_name ""; - set $resource_namespace ""; - set $service ""; - - listen 80 default_server;listen [::]:80 default_server; - listen 443 ssl default_server; - listen [::]:443 ssl default_server; - ssl_certificate /etc/nginx/secrets/default; - ssl_certificate_key /etc/nginx/secrets/default; - - server_name _; - server_tokens "off"; - - location / { - return ; - } - } - - # NGINX Plus API over unix socket - server { - listen unix:/var/lib/nginx/nginx-plus-api.sock; - access_log off; - - # $config_version_mismatch is defined in /etc/nginx/config-version.conf - location /configVersionCheck { - if ($config_version_mismatch) { - return 503; - } - return 200; - } - - location /api { - api write=on; - } - } - - include /etc/nginx/config-version.conf; - include /etc/nginx/conf.d/*.conf; - - server { - listen unix:/var/lib/nginx/nginx-418-server.sock; - access_log off;return 418; - } -} - -stream { - log_format stream-main escape=none - '$remote_addr' - ' $remote_user' - ; - - access_log /dev/stdout stream-main; - # comment - resolver example.com 127.0.0.1 valid=10s ipv6=off; - resolver_timeout 15s; - - map_hash_max_size ; - - - include /etc/nginx/stream-conf.d/*.conf; -} mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -1812,8 +1678,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -1951,8 +1820,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -2090,8 +1962,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -2229,8 +2104,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -2386,8 +2264,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -2541,6 +2422,12 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } +mgmt { + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; +} + --- [TestExecuteTemplate_ForMainForNGINXPlusWithHTTP2On - 1] @@ -2690,8 +2577,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- @@ -2845,6 +2735,12 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } +mgmt { + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; +} + --- [TestExecuteTemplate_ForMainForNGINXPlusWithoutCustomTLSPassthroughPort - 1] @@ -2998,8 +2894,11 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } + mgmt { - usage_report interval=0s; + license_token /etc/nginx/secrets/license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; } --- diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 22fa313857..274e4f5636 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -189,6 +189,11 @@ type Location struct { MinionIngress *Ingress } +// MGMTConfig is tbe configuration for the MGMT block. +type MGMTConfig struct { + EnforceInitialReport *bool +} + // MainConfig describe the main NGINX configuration file. type MainConfig struct { AccessLog string @@ -207,6 +212,7 @@ type MainConfig struct { LogFormat []string LogFormatEscaping string MainSnippets []string + MGMTConfig MGMTConfig NginxStatus bool NginxStatusAllowCIDRs []string NginxStatusPort int diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 756d16c5ad..88ca0b5f0d 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -352,15 +352,8 @@ stream { include /etc/nginx/stream-conf.d/*.conf; } -{{- if .NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r33" }} mgmt { - usage_report endpoint=product.connect.nginx.com interval=1h; - uuid_file /var/lib/nginx/nginx.id; - license_token conf/license.jwt; + license_token {{ printf "%s/license.jwt" .StaticSSLPath }}; + enforce_initial_report {{ makeOnOffFromBool .MGMTConfig.EnforceInitialReport}}; deployment_context /etc/nginx/reporting/tracking.info; } -{{- else if .NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r31" }} -mgmt { - usage_report interval=0s; -} -{{- end }} diff --git a/internal/configs/version1/template_helper.go b/internal/configs/version1/template_helper.go index bd6232e5fe..7f79aa84c5 100644 --- a/internal/configs/version1/template_helper.go +++ b/internal/configs/version1/template_helper.go @@ -202,6 +202,10 @@ func makeResolver(resolverAddresses []string, resolverValid string, resolverIPV6 return builder.String() } +func boolToPointerBool(b bool) *bool { + return &b +} + var helperFunctions = template.FuncMap{ "split": split, "trim": trim, @@ -213,6 +217,8 @@ var helperFunctions = template.FuncMap{ "replaceAll": strings.ReplaceAll, "makeLocationPath": makeLocationPath, "makeSecretPath": commonhelpers.MakeSecretPath, + "makeOnOffFromBool": commonhelpers.MakeOnOffFromBool, "generateProxySetHeaders": generateProxySetHeaders, + "boolToPointerBool": boolToPointerBool, "makeResolver": makeResolver, } diff --git a/internal/configs/version1/template_test.go b/internal/configs/version1/template_test.go index 5075c754d8..440bd4b91b 100644 --- a/internal/configs/version1/template_test.go +++ b/internal/configs/version1/template_test.go @@ -55,20 +55,6 @@ func TestExecuteMainTemplateForNGINXPlus(t *testing.T) { t.Log(buf.String()) } -func TestExecuteMainTemplateForNGINXPlusR31(t *testing.T) { - t.Parallel() - - tmpl := newNGINXPlusMainTmpl(t) - buf := &bytes.Buffer{} - - err := tmpl.Execute(buf, mainCfgR31) - if err != nil { - t.Error(err) - } - snaps.MatchSnapshot(t, buf.String()) - t.Log(buf.String()) -} - func TestExecuteMainTemplateForNGINX(t *testing.T) { t.Parallel() @@ -2042,7 +2028,7 @@ var ( KeepaliveRequests: 100, VariablesHashBucketSize: 256, VariablesHashMaxSize: 1024, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r30)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AppProtectLoadModule: true, AppProtectV5LoadModule: false, AppProtectV5EnforcerAddr: "", @@ -2063,36 +2049,6 @@ var ( AccessLog: "/dev/stdout main", } - mainCfgR31 = MainConfig{ - StaticSSLPath: fakeManager.GetSecretsDir(), - DefaultHTTPListenerPort: 80, - DefaultHTTPSListenerPort: 443, - ServerNamesHashMaxSize: "512", - ServerTokens: "off", - WorkerProcesses: "auto", - WorkerCPUAffinity: "auto", - WorkerShutdownTimeout: "1m", - WorkerConnections: "1024", - WorkerRlimitNofile: "65536", - LogFormat: []string{"$remote_addr", "$remote_user"}, - LogFormatEscaping: "default", - StreamSnippets: []string{"# comment"}, - StreamLogFormat: []string{"$remote_addr", "$remote_user"}, - StreamLogFormatEscaping: "none", - ResolverAddresses: []string{"example.com", "127.0.0.1"}, - ResolverIPV6: false, - ResolverValid: "10s", - ResolverTimeout: "15s", - KeepaliveTimeout: "65s", - KeepaliveRequests: 100, - VariablesHashBucketSize: 256, - VariablesHashMaxSize: 1024, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), - AppProtectV5LoadModule: true, - AppProtectV5EnforcerAddr: "enforcer.svc.local", - AccessLog: "/dev/stdout main", - } - mainCfgHTTP2On = MainConfig{ StaticSSLPath: fakeManager.GetSecretsDir(), DefaultHTTPListenerPort: 80, @@ -2118,7 +2074,7 @@ var ( KeepaliveRequests: 100, VariablesHashBucketSize: 256, VariablesHashMaxSize: 1024, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AppProtectLoadModule: true, AppProtectV5LoadModule: false, AppProtectV5EnforcerAddr: "", @@ -2158,7 +2114,7 @@ var ( VariablesHashMaxSize: 1024, TLSPassthrough: true, TLSPassthroughPort: 8443, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AccessLog: "/dev/stdout main", } @@ -2186,7 +2142,7 @@ var ( VariablesHashMaxSize: 1024, TLSPassthrough: false, TLSPassthroughPort: 8443, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AccessLog: "/dev/stdout main", } @@ -2214,7 +2170,7 @@ var ( VariablesHashMaxSize: 1024, TLSPassthrough: true, TLSPassthroughPort: 443, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AccessLog: "/dev/stdout main", } @@ -2242,7 +2198,7 @@ var ( KeepaliveRequests: 100, VariablesHashBucketSize: 256, VariablesHashMaxSize: 1024, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AccessLog: "/dev/stdout main", } @@ -2270,7 +2226,7 @@ var ( KeepaliveRequests: 100, VariablesHashBucketSize: 256, VariablesHashMaxSize: 1024, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AccessLog: "/dev/stdout main", } @@ -2298,7 +2254,7 @@ var ( KeepaliveRequests: 100, VariablesHashBucketSize: 256, VariablesHashMaxSize: 1024, - NginxVersion: nginx.NewVersion("nginx version: nginx/1.25.3 (nginx-plus-r31)"), + NginxVersion: nginx.NewVersion("nginx version: nginx/1.27.2 (nginx-plus-r33)"), AccessLog: "/dev/stdout main", } diff --git a/internal/k8s/configmap.go b/internal/k8s/configmap.go index 001775aaa3..8d7e86a419 100644 --- a/internal/k8s/configmap.go +++ b/internal/k8s/configmap.go @@ -71,17 +71,24 @@ func (lbc *LoadBalancerController) addConfigMapHandler(handlers cache.ResourceEv lbc.cacheSyncs = append(lbc.cacheSyncs, lbc.configMapController.HasSynced) } +func (lbc *LoadBalancerController) addMGMTConfigMapHandler(handlers cache.ResourceEventHandlerFuncs, namespace string) { + options := lbc.getConfigMapHandlerOptions(handlers, namespace) + + lbc.mgmtConfigMapLister.Store, lbc.mgmtConfigMapController = cache.NewInformerWithOptions(options) + lbc.cacheSyncs = append(lbc.cacheSyncs, lbc.mgmtConfigMapController.HasSynced) +} + func (lbc *LoadBalancerController) syncConfigMap(task task) { key := task.Key nl.Debugf(lbc.Logger, "Syncing configmap %v", key) - obj, configExists, err := lbc.configMapLister.GetByKey(key) - if err != nil { - lbc.syncQueue.Requeue(task, err) - return - } switch key { case lbc.nginxConfigMapName: + obj, configExists, err := lbc.configMapLister.GetByKey(key) + if err != nil { + lbc.syncQueue.Requeue(task, err) + return + } if configExists { lbc.configMap = obj.(*v1.ConfigMap) externalStatusAddress, exists := lbc.configMap.Data["external-status-address"] @@ -91,6 +98,17 @@ func (lbc *LoadBalancerController) syncConfigMap(task task) { } else { lbc.configMap = nil } + case lbc.mgmtConfigMapName: + obj, configExists, err := lbc.mgmtConfigMapLister.GetByKey(key) + if err != nil { + lbc.syncQueue.Requeue(task, err) + return + } + if configExists { + lbc.mgmtConfigMap = obj.(*v1.ConfigMap) + } else { + lbc.mgmtConfigMap = nil + } } if !lbc.isNginxReady { diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 606c1431ad..87170d154f 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -105,6 +105,7 @@ type podEndpoint struct { type specialSecrets struct { defaultServerSecret string wildcardTLSSecret string + licenseSecret string } type controllerMetadata struct { @@ -122,9 +123,11 @@ type LoadBalancerController struct { cacheSyncs []cache.InformerSynced namespacedInformers map[string]*namespacedInformer configMapController cache.Controller + mgmtConfigMapController cache.Controller globalConfigurationController cache.Controller ingressLinkInformer cache.SharedIndexInformer configMapLister storeToConfigMapLister + mgmtConfigMapLister storeToConfigMapLister globalConfigurationLister cache.Store ingressLinkLister cache.Store namespaceLabeledLister cache.Store @@ -134,6 +137,7 @@ type LoadBalancerController struct { cancel context.CancelFunc configurator *configs.Configurator watchNginxConfigMaps bool + watchMGMTConfigMap bool watchGlobalConfiguration bool watchIngressLink bool isNginxPlus bool @@ -167,6 +171,7 @@ type LoadBalancerController struct { appProtectConfiguration appprotect.Configuration dosConfiguration *appprotectdos.Configuration configMap *api_v1.ConfigMap + mgmtConfigMap *api_v1.ConfigMap certManagerController *cm_controller.CmController externalDNSController *ed_controller.ExtDNSController batchSyncEnabled bool @@ -178,6 +183,7 @@ type LoadBalancerController struct { telemetryChan chan struct{} weightChangesDynamicReload bool nginxConfigMapName string + mgmtConfigMapName string } var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc @@ -209,6 +215,7 @@ type NewLoadBalancerControllerInput struct { LeaderElectionLockName string WildcardTLSSecret string ConfigMaps string + MGMTConfigMap string GlobalConfiguration string AreCustomResourcesEnabled bool EnableOIDC bool @@ -241,6 +248,9 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc defaultServerSecret: input.DefaultServerSecret, wildcardTLSSecret: input.WildcardTLSSecret, } + if input.IsNginxPlus { + specialSecrets.licenseSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.License) + } lbc := &LoadBalancerController{ client: input.KubeClient, confClient: input.ConfClient, @@ -272,6 +282,7 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc isIPV6Disabled: input.IsIPV6Disabled, weightChangesDynamicReload: input.DynamicWeightChangesReload, nginxConfigMapName: input.ConfigMaps, + mgmtConfigMapName: input.MGMTConfigMap, } lbc.syncQueue = newTaskQueue(lbc.Logger, lbc.sync) @@ -326,6 +337,16 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc } } + if input.MGMTConfigMap != "" { + mgmtConfigMapNS, mgmtConfigMapName, err := ParseNamespaceName(input.MGMTConfigMap) + if err != nil { + nl.Warn(lbc.Logger, err) + } else { + lbc.watchMGMTConfigMap = true + lbc.addMGMTConfigMapHandler(createConfigMapHandlers(lbc, mgmtConfigMapName), mgmtConfigMapNS) + } + } + if input.IngressLink != "" { lbc.watchIngressLink = true lbc.addIngressLinkHandler(createIngressLinkHandlers(lbc), input.IngressLink) @@ -621,6 +642,10 @@ func (lbc *LoadBalancerController) Run() { go lbc.configMapController.Run(lbc.ctx.Done()) } + if lbc.watchMGMTConfigMap { + go lbc.mgmtConfigMapController.Run(lbc.ctx.Done()) + } + if lbc.watchGlobalConfiguration { go lbc.globalConfigurationController.Run(lbc.ctx.Done()) } @@ -850,11 +875,20 @@ func (lbc *LoadBalancerController) createExtendedResources(resources []Resource) func (lbc *LoadBalancerController) updateAllConfigs() { ctx := nl.ContextWithLogger(context.Background(), lbc.Logger) cfgParams := configs.NewDefaultConfigParams(ctx, lbc.isNginxPlus) + mgmtCfgParams := configs.NewDefaultMGMTConfigParams(ctx) var isNGINXConfigValid bool + var mgmtConfigHasWarnings bool + var mgmtErr error if lbc.configMap != nil { cfgParams, isNGINXConfigValid = configs.ParseConfigMap(ctx, lbc.configMap, lbc.isNginxPlus, lbc.appProtectEnabled, lbc.appProtectDosEnabled, lbc.configuration.isTLSPassthroughEnabled, lbc.recorder) } + if lbc.mgmtConfigMap != nil && lbc.isNginxPlus { + mgmtCfgParams, mgmtConfigHasWarnings, mgmtErr = configs.ParseMGMTConfigMap(ctx, lbc.mgmtConfigMap, lbc.recorder) + if mgmtErr != nil { + nl.Errorf(lbc.Logger, "configmap %s/%s: %v", lbc.mgmtConfigMap.GetNamespace(), lbc.mgmtConfigMap.GetName(), mgmtErr) + } + } resources := lbc.configuration.GetResources() @@ -862,8 +896,7 @@ func (lbc *LoadBalancerController) updateAllConfigs() { resourceExes := lbc.createExtendedResources(resources) - warnings, updateErr := lbc.configurator.UpdateConfig(cfgParams, resourceExes) - + warnings, updateErr := lbc.configurator.UpdateConfig(cfgParams, mgmtCfgParams, resourceExes) eventTitle := "Updated" eventType := api_v1.EventTypeNormal eventWarningMessage := "" @@ -886,6 +919,14 @@ func (lbc *LoadBalancerController) updateAllConfigs() { } } + if lbc.mgmtConfigMap != nil { + if !mgmtConfigHasWarnings { + lbc.recorder.Event(lbc.mgmtConfigMap, api_v1.EventTypeNormal, "Updated", fmt.Sprintf("MGMT ConfigMap %s/%s updated without error", lbc.mgmtConfigMap.GetNamespace(), lbc.mgmtConfigMap.GetName())) + } else { + lbc.recorder.Event(lbc.mgmtConfigMap, api_v1.EventTypeWarning, "UpdatedWithError", fmt.Sprintf("MGMT ConfigMap %s/%s updated with errors. Ignoring invalid values", lbc.mgmtConfigMap.GetNamespace(), lbc.mgmtConfigMap.GetName())) + } + } + gc := lbc.configuration.GetGlobalConfiguration() if gc != nil && lbc.configMap != nil { key := getResourceKey(&lbc.configMap.ObjectMeta) @@ -1743,6 +1784,8 @@ func (lbc *LoadBalancerController) isSpecialSecret(secretName string) bool { return true case lbc.specialSecrets.wildcardTLSSecret: return true + case lbc.specialSecrets.licenseSecret: + return true default: return false } @@ -1794,10 +1837,18 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr return } - lbc.writeSpecialSecrets(secret, specialTLSSecretsToUpdate) + lbc.writeSpecialSecrets(secret, secretNsName, specialTLSSecretsToUpdate) + if ok := lbc.writeSpecialSecrets(secret, secretNsName, specialTLSSecretsToUpdate); !ok { + // if not ok bail early + return + } // reload nginx when the TLS special secrets are updated switch secretNsName { + case lbc.specialSecrets.licenseSecret: + if ok := lbc.performNGINXReload(secret); !ok { + return + } case lbc.specialSecrets.defaultServerSecret, lbc.specialSecrets.wildcardTLSSecret: if ok := lbc.performDynamicSSLReload(secret); !ok { return @@ -1807,8 +1858,20 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr lbc.recorder.Eventf(secret, api_v1.EventTypeNormal, "Updated", "the special Secret %v was updated", secretNsName) } -func (lbc *LoadBalancerController) writeSpecialSecrets(secret *api_v1.Secret, specialTLSSecretsToUpdate []string) { - lbc.configurator.AddOrUpdateSpecialTLSSecrets(secret, specialTLSSecretsToUpdate) +// writeSpecialSecrets generates content and writes the secret to disk +func (lbc *LoadBalancerController) writeSpecialSecrets(secret *api_v1.Secret, secretNsName string, specialTLSSecretsToUpdate []string) bool { + switch secret.Type { + case secrets.SecretTypeLicense: + err := lbc.configurator.AddOrUpdateLicenseSecret(secret) + if err != nil { + nl.Error(lbc.Logger, err) + lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, "UpdatedWithError", "the license Secret %v was updated, but not applied: %v", secretNsName, err) + return false + } + case api_v1.SecretTypeTLS: + lbc.configurator.AddOrUpdateSpecialTLSSecrets(secret, specialTLSSecretsToUpdate) + } + return true } func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, secret *api_v1.Secret, specialTLSSecretsToUpdate *[]string) bool { @@ -1818,6 +1881,14 @@ func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, if secretNsName == lbc.specialSecrets.wildcardTLSSecret { lbc.validationTLSSpecialSecret(secret, configs.WildcardSecretFileName, specialTLSSecretsToUpdate) } + if secretNsName == lbc.specialSecrets.licenseSecret { + err := secrets.ValidateLicenseSecret(secret) + if err != nil { + nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err) + lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, "Rejected", "the special Secret %v was rejected, using the previous version: %v", secretNsName, err) + return false + } + } return true } diff --git a/internal/k8s/secrets/validation.go b/internal/k8s/secrets/validation.go index f98dafebea..18e3bde2c3 100644 --- a/internal/k8s/secrets/validation.go +++ b/internal/k8s/secrets/validation.go @@ -37,6 +37,9 @@ const SecretTypeHtpasswd api_v1.SecretType = "nginx.org/htpasswd" // #nosec G101 // SecretTypeAPIKey contains a list of client ID and key for API key authorization.. #nosec G101 const SecretTypeAPIKey api_v1.SecretType = "nginx.org/apikey" // #nosec G101 +// SecretTypeLicense contains the license.jwt required for NGINX Plus. #nosec G101 +const SecretTypeLicense api_v1.SecretType = "nginx.com/license" // #nosec G101 + // ValidateTLSSecret validates the secret. If it is valid, the function returns nil. func ValidateTLSSecret(secret *api_v1.Secret) error { if secret.Type != api_v1.SecretTypeTLS { @@ -145,6 +148,19 @@ func ValidateHtpasswdSecret(secret *api_v1.Secret) error { return nil } +// ValidateLicenseSecret validates the secret. If it is valid, the function returns nil. +func ValidateLicenseSecret(secret *api_v1.Secret) error { + if secret.Type != SecretTypeLicense { + return fmt.Errorf("license secret must be of the type %v", SecretTypeLicense) + } + + if _, exists := secret.Data["license.jwt"]; !exists { + return fmt.Errorf("license secret must have the data field %v", "license.jwt") + } + + return nil +} + // IsSupportedSecretType checks if the secret type is supported. func IsSupportedSecretType(secretType api_v1.SecretType) bool { return secretType == api_v1.SecretTypeTLS || @@ -152,7 +168,8 @@ func IsSupportedSecretType(secretType api_v1.SecretType) bool { secretType == SecretTypeJWK || secretType == SecretTypeOIDC || secretType == SecretTypeHtpasswd || - secretType == SecretTypeAPIKey + secretType == SecretTypeAPIKey || + secretType == SecretTypeLicense } // ValidateSecret validates the secret. If it is valid, the function returns nil. @@ -170,6 +187,8 @@ func ValidateSecret(secret *api_v1.Secret) error { return ValidateHtpasswdSecret(secret) case SecretTypeAPIKey: return ValidateAPIKeySecret(secret) + case SecretTypeLicense: + return ValidateLicenseSecret(secret) } return fmt.Errorf("secret is of the unsupported type %v", secret.Type) diff --git a/internal/k8s/secrets/validation_test.go b/internal/k8s/secrets/validation_test.go index 186d63f5d9..203cd064c7 100644 --- a/internal/k8s/secrets/validation_test.go +++ b/internal/k8s/secrets/validation_test.go @@ -1,6 +1,7 @@ package secrets import ( + "encoding/base64" "testing" v1 "k8s.io/api/core/v1" @@ -457,6 +458,64 @@ func TestValidateOIDCSecretFails(t *testing.T) { } } +func TestValidateLicenseSecret(t *testing.T) { + t.Parallel() + secret := &v1.Secret{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "license-token", + Namespace: "default", + }, + Type: SecretTypeLicense, + Data: map[string][]byte{ + "license.jwt": []byte(base64.StdEncoding.EncodeToString([]byte("license-token"))), + }, + } + + err := ValidateLicenseSecret(secret) + if err != nil { + t.Errorf("ValidateLicenseSecret() returned error %v", err) + } +} + +func TestValidateLicenseSecretFails(t *testing.T) { + t.Parallel() + tests := []struct { + secret *v1.Secret + msg string + }{ + { + secret: &v1.Secret{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "license-token", + Namespace: "default", + }, + Type: "some-type", + Data: map[string][]byte{ + "license.jwt": []byte(base64.StdEncoding.EncodeToString([]byte("license-token"))), + }, + }, + msg: "Incorrect type for license secret", + }, + { + secret: &v1.Secret{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "license-token", + Namespace: "default", + }, + Type: SecretTypeLicense, + }, + msg: "Missing license.jwt for license secret", + }, + } + + for _, test := range tests { + err := ValidateLicenseSecret(test.secret) + if err == nil { + t.Errorf("ValidateLicenseSecret() returned no error for the case of %s", test.msg) + } + } +} + func TestValidateSecret(t *testing.T) { t.Parallel() tests := []struct { diff --git a/site/content/installation/installing-nic/installation-with-helm.md b/site/content/installation/installing-nic/installation-with-helm.md index 1da9cd6237..370a9d84cd 100644 --- a/site/content/installation/installing-nic/installation-with-helm.md +++ b/site/content/installation/installing-nic/installation-with-helm.md @@ -406,12 +406,12 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **controller.appprotect.enforcer.host** | Host that the App Protect WAF v5 Enforcer runs on. | "127.0.0.1" | | **controller.appprotect.enforcer.port** | Port that the App Protect WAF v5 Enforcer runs on. | 50000 | | **controller.appprotect.enforcer.image** | The image repository of the App Protect WAF v5 Enforcer. | private-registry.nginx.com/nap/waf-enforcer | -| **controller.appprotect.enforcer.tag** | The tag of the App Protect WAF v5 Enforcer. | "5.3.0" | +| **controller.appprotect.enforcer.tag** | The tag of the App Protect WAF v5 Enforcer. | "5.4.0" | | **controller.appprotect.enforcer.digest** | The digest of the App Protect WAF v5 Enforcer. Takes precedence over tag if set. | "" | | **controller.appprotect.enforcer.pullPolicy** | The pull policy for the App Protect WAF v5 Enforcer image. | IfNotPresent | | **controller.appprotect.enforcer.securityContext** | The security context for App Protect WAF v5 Enforcer container. | {} | | **controller.appprotect.configManager.image** | The image repository of the App Protect WAF v5 Configuration Manager. | private-registry.nginx.com/nap/waf-config-mgr | -| **controller.appprotect.configManager.tag** | The tag of the App Protect WAF v5 Configuration Manager. | "5.3.0" | +| **controller.appprotect.configManager.tag** | The tag of the App Protect WAF v5 Configuration Manager. | "5.4.0" | | **controller.appprotect.configManager.digest** | The digest of the App Protect WAF v5 Configuration Manager. Takes precedence over tag if set. | "" | | **controller.appprotect.configManager.pullPolicy** | The pull policy for the App Protect WAF v5 Configuration Manager image. | IfNotPresent | | **controller.appprotect.configManager.securityContext** | The security context for App Protect WAF v5 Configuration Manager container. | {"allowPrivilegeEscalation":false,"runAsUser":101,"runAsNonRoot":true,"capabilities":{"drop":["all"]}} | diff --git a/site/content/installation/integrations/app-protect-waf-v5/installation.md b/site/content/installation/integrations/app-protect-waf-v5/installation.md index 0a32561f04..8db961adda 100644 --- a/site/content/installation/integrations/app-protect-waf-v5/installation.md +++ b/site/content/installation/integrations/app-protect-waf-v5/installation.md @@ -368,6 +368,8 @@ Add `waf-enforcer` image to the `containers` section: env: - name: ENFORCER_PORT value: "50000" + - name: ENFORCER_CONFIG_TIMEOUT + value: "0" volumeMounts: - name: app-protect-bd-config mountPath: /opt/app_protect/bd_config @@ -505,7 +507,8 @@ If you prefer not to build your own NGINX Ingress Controller image, you can use {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} | NIC Version | App Protect WAFv5 Version | Config Manager | Enforcer | | --- | --- | --- | --- | -| {{< nic-version >}} | 32_5.144 | 5.3.0 | 5.3.0 | +| {{< nic-version >}} | 33_5.210 | 5.4.0 | 5.4.0 | +| 3.7.2 | 32_5.144 | 5.3.0 | 5.3.0 | | 3.6.2 | 32_5.48 | 5.2.0 | 5.2.0 | {{% /bootstrap-table %}} diff --git a/tests/conftest.py b/tests/conftest.py index 01886eb951..cac19fc9ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,6 +55,12 @@ def pytest_addoption(parser) -> None: default=DEFAULT_IC_TYPE, help="The type of the Ingress Controller: nginx-ingress or nginx-plus-ingress.", ) + parser.addoption( + "--plus-jwt", + action="store", + help="The plus jwt for the Ingress Controller image.", + default=os.environ.get("PLUS_JWT"), + ) parser.addoption( "--service", action="store", @@ -144,6 +150,13 @@ def pytest_addoption(parser) -> None: pytest_plugins = ["suite.fixtures.fixtures", "suite.fixtures.ic_fixtures", "suite.fixtures.custom_resource_fixtures"] +def pytest_configure(config): + if config.getoption("--ic-type") == "nginx-plus-ingress" and ( + config.getoption("--plus-jwt") == "" or config.getoption("--plus-jwt") is None + ): + pytest.exit("Please provide the plus jwt for the Nginx Ingress Controller") + + def pytest_collection_modifyitems(config, items) -> None: """ Skip tests marked with '@pytest.mark.skip_for_nginx_oss' for Nginx OSS runs. diff --git a/tests/settings.py b/tests/settings.py index ce03b76491..403b9dff52 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -33,4 +33,4 @@ # Nginx registry address to pull waf components from NGX_REG = "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr" # WAF component version to pull from above registry -WAF_V5_VERSION = "5.3.0" +WAF_V5_VERSION = "5.4.0" diff --git a/tests/suite/fixtures/fixtures.py b/tests/suite/fixtures/fixtures.py index de19a7a254..45afc17689 100644 --- a/tests/suite/fixtures/fixtures.py +++ b/tests/suite/fixtures/fixtures.py @@ -23,6 +23,7 @@ cleanup_rbac, configure_rbac, create_configmap_from_yaml, + create_license, create_namespace_with_name_from_yaml, create_ns_and_sa_from_yaml, create_secret_from_yaml, @@ -241,9 +242,16 @@ def ingress_controller_prerequisites(cli_arguments, kube_apis, request) -> Ingre ) config_map_yaml = f"{DEPLOYMENTS}/common/nginx-config.yaml" create_configmap_from_yaml(kube_apis.v1, namespace, config_map_yaml) + mgmt_config_map_yaml = f"{DEPLOYMENTS}/common/plus-mgmt-configmap.yaml" with open(config_map_yaml) as f: config_map = yaml.safe_load(f) create_secret_from_yaml(kube_apis.v1, namespace, f"{TEST_DATA}/common/default-server-secret.yaml") + # setup Plus JWT configuration + if cli_arguments["ic-type"] == "nginx-plus-ingress": + print("Create Plus JWT License:") + license_name = create_license(kube_apis.v1, namespace, cli_arguments["plus-jwt"]) + print(f"License created: {license_name}") + create_configmap_from_yaml(kube_apis.v1, namespace, mgmt_config_map_yaml) def fin(): if request.config.getoption("--skip-fixture-teardown") == "no": @@ -323,6 +331,9 @@ def cli_arguments(request) -> {}: result["ic-type"] = request.config.getoption("--ic-type") assert result["ic-type"] in ALLOWED_IC_TYPES, f"IC type {result['ic-type']} is not allowed" print(f"Tests will run against the IC of type: {result['ic-type']}") + if result["ic-type"] == "nginx-plus-ingress": + print(f"Tests will use the Plus JWT") + result["plus-jwt"] = request.config.getoption("--plus-jwt") result["replicas"] = request.config.getoption("--replicas") print(f"Number of pods spun up will be : {result['replicas']}") diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py index 33abadbccf..06b3f0530d 100644 --- a/tests/suite/utils/resources_utils.py +++ b/tests/suite/utils/resources_utils.py @@ -10,7 +10,15 @@ import pytest import requests import yaml -from kubernetes.client import AppsV1Api, CoreV1Api, NetworkingV1Api, RbacAuthorizationV1Api, V1Service +from kubernetes.client import ( + AppsV1Api, + CoreV1Api, + NetworkingV1Api, + RbacAuthorizationV1Api, + V1ObjectMeta, + V1Secret, + V1Service, +) from kubernetes.client.rest import ApiException from kubernetes.stream import stream from more_itertools import first @@ -556,6 +564,15 @@ def create_secret(v1: CoreV1Api, namespace, body) -> str: return body["metadata"]["name"] +def create_license(v1: CoreV1Api, namespace, jwt, license_token_name="license-token") -> str: + sec = V1Secret() + sec.type = "nginx.com/license" + sec.metadata = V1ObjectMeta(name=license_token_name) + sec.data = {"license.jwt": base64.b64encode(jwt.encode("ascii")).decode()} + v1.create_namespaced_secret(namespace=namespace, body=sec) + return license_token_name + + def replace_secret(v1: CoreV1Api, name, namespace, yaml_manifest) -> str: """ Replace a secret based on yaml file. @@ -1379,7 +1396,10 @@ def create_ingress_controller_wafv5( "capabilities": {"drop": ["all"]}, "readOnlyRootFilesystem": rorfs, }, - "env": [{"name": "ENFORCER_PORT", "value": "50000"}], + "env": [ + {"name": "ENFORCER_PORT", "value": "50000"}, + {"name": "ENFORCER_CONFIG_TIMEOUT", "value": "0"}, + ], "volumeMounts": [ { "name": "app-protect-bd-config",