add decomissioning when pod is stopped (#57)

* add graceful shutdown of nifi node #31 * add decomissioning when pod is stopped * move extraContainer in statefulset def * update README.md * fix typo * revert * update termintation in statefulset * correctly decomission node update terminationGracePeriod * update value.yaml Co-authored-by: Alexandre Nuttinck <[email protected]>
cetic · Jun 23, 2020 · fedf0da · fedf0da
1 parent b56517d
commit fedf0da
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 35 deletions.
diff --git a/Chart.yaml b/Chart.yaml
@@ -1,7 +1,7 @@
 ---
 apiVersion: v2
 name: nifi
-version: 0.4.1
+version: 0.4.2
 appVersion: 1.11.4
 description: Apache NiFi is a software project from the Apache Software Foundation designed to automate the flow of data between software systems.
 keywords:

diff --git a/README.md b/README.md
@@ -145,16 +145,20 @@ The following table lists the configurable parameters of the nifi chart and the
 | `logresources.`                                                             | Pod resource requests and limits                                                                                   | `{}`                            |
 | **nodeSelector**                                                            |
 | `nodeSelector`                                                              | Node labels for pod assignment                                                                                     | `{}`                            |
+| **terminationGracePeriodSeconds**                                           |
+| `terminationGracePeriodSeconds`                                             | Number of seconds the pod needs to terminate gracefully. For clean scale down of the nifi-cluster the default is set to 60, opposed to k8s-default 30. | `60`                            |
 | **tolerations**                                                             |
 | `tolerations`                                                               | Tolerations for pod assignment                                                                                     | `[]`                            |
 | **initContainers**                                                          |
-| `initContainers`                                                            | Container definition that will be added to the pod as [initContainers](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#container-v1-core) | `[]`                            |
+| `initContainers`                                                            | Container definition that will be added to the pod as [initContainers](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core) | `[]`                            |
 | **extraVolumes**                                                            |
-| `extraVolumes`                                                              | Additional Volumes available within the pod (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#volume-v1-core) for format)       | `[]`                            |
+| `extraVolumes`                                                              | Additional Volumes available within the pod (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core) for format)       | `[]`                            |
 | **extraVolumeMounts**                                                       |
-| `extraVolumeMounts`                                                         | VolumeMounts for the nifi-server container (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#volumemount-v1-core) for details)  | `[]`                            |
-| **ens**                                                                     |
-| `env`                                                                       | Additional environment variables for the nifi-container (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#envvar-v1-core) for details)  | `[]`                            |
+| `extraVolumeMounts`                                                         | VolumeMounts for the nifi-server container (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core) for details)  | `[]`                            |
+| **env**                                                                     |
+| `env`                                                                       | Additional environment variables for the nifi-container (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#envvar-v1-core) for details)  | `[]`                            |
+| **extraContainers                                                           |
+| `extraContainers`                                                           | Additional container-specifications that should run within the pod (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core) for details)  | `[]`                            |
 | **zookeeper**                                                               |
 |`zookeeper.enabled`                                                          | If true, deploy Zookeeper                                                                                          | `true`                          |
 |`zookeeper.url`                                                              | If the Zookeeper Chart is disabled a URL and port are required to connect                                          | `nil`                           |

diff --git a/templates/service.yaml b/templates/service.yaml
@@ -77,4 +77,4 @@ spec:
 {{- end }}
   selector:
     app: {{ template "apache-nifi.name" . }}
-    release: {{ .Release.Name }}
+    release: {{ .Release.Name }}
diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml
@@ -51,6 +51,9 @@ spec:
                          - {{ include "apache-nifi.name" . | quote }}
                  topologyKey: "kubernetes.io/hostname"
       {{- end}}
+{{- if .Values.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+{{- end }}
 {{- if .Values.tolerations }}
       tolerations:
 {{ toYaml .Values.tolerations | indent 8 }}
@@ -122,7 +125,55 @@ spec:
   {{- end }}
 {{- end }}
 
-          exec bin/nifi.sh run
+          exec bin/nifi.sh run & nifi_pid="$!"
+
+          function offloadNode() {
+              echo executing preStop > /tmp/preStop.out
+              FQDN=$(hostname -f)
+              echo disconnecting node '$FQDN'
+              baseUrl=http://nifi-0.nifi-headless:8080
+              echo baseUrl ${baseUrl}
+              # no cluster without initial node
+              if [[ "$FQDN" != nifi-0.nifi-headless* ]]; then
+                echo "gracefully disconnecting node '$FQDN' from cluster"
+                ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi get-nodes -ot json -u ${baseUrl} > nodes.json
+                nnid=$(jq --arg FQDN "$FQDN" '.cluster.nodes[] | select(.address==$FQDN) | .nodeId' nodes.json)
+                echo "disconnecting node ${nnid}"
+                ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi disconnect-node -nnid $nnid -u ${baseUrl}
+                echo ""
+                echo "wait until node has state 'DISCONNECTED'"
+                while [[ "$node_state" != "DISCONNECTED" ]]; do
+                    sleep 1
+                    ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi get-nodes -ot json -u ${baseUrl} > nodes.json
+                    node_state=$(jq -r --arg FQDN "$FQDN" '.cluster.nodes[] | select(.address==$FQDN) | .status' nodes.json)
+                    echo "state is '$node_state'"
+                done
+                echo ""
+                echo "node '${nnid}' was disconnected"
+                echo "offloading node"
+                ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi offload-node -nnid $nnid -u ${baseUrl}
+                echo ""
+                echo "wait until node has state 'OFFLOADED'"
+                while [[ "$node_state" != "OFFLOADED" ]]; do
+                    sleep 1
+                    ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi get-nodes -ot json -u ${baseUrl} > nodes.json
+                    node_state=$(jq -r --arg FQDN "$FQDN" '.cluster.nodes[] | select(.address==$FQDN) | .status' nodes.json)
+                    echo "state is '$node_state'"
+                done
+              fi
+          }
+
+          deleteNode() {
+                echo "deleting node"
+                ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi delete-node -nnid ${nnid} -u ${baseUrl}
+                echo "node deleted"
+          }
+
+          trap 'echo Received trapped signal, beginning shutdown...;offloadNode;./bin/nifi.sh stop;deleteNode;exit 0;' TERM HUP INT;
+          trap ":" EXIT
+
+          echo NiFi running with PID ${nifi_pid}.
+          wait ${nifi_pid}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
         ports:
@@ -148,15 +199,8 @@ spec:
 {{- if .Values.env }}
 {{ toYaml .Values.env | indent 8 }}
 {{- end }}
-        lifecycle:
-          preStop:
-            exec:
-              command:
-              - bash
-              - -c
-              - |
-                $NIFI_HOME/bin/nifi.sh stop
 {{- if .Values.postStart }}
+        lifecycle:
           postStart:
             exec:
               command: ["/bin/sh", "-c", {{ .Values.postStart | quote }}]

diff --git a/templates/storageclass.yaml b/templates/storageclass.yaml
@@ -12,4 +12,4 @@ metadata:
 provisioner: {{ .Values.storageProvisioner | quote }}
 parameters:
   type: {{ .Values.storageType | quote }}
-{{- end -}}
+{{- end -}}
diff --git a/values.yaml b/values.yaml
@@ -59,8 +59,11 @@ properties:
     port: 10000
   authorizer: managed-authorizer
   # use properties.safetyValve to pass explicit 'key: value' pairs that overwrite other configuration
-  # safetyValve:
+  safetyValve:
     #nifi.variable.registry.properties: "${NIFI_HOME}/example1.properties, ${NIFI_HOME}/example2.properties"
+    nifi.web.http.network.interface.default: eth0
+    # listen to loopback interface so "kubectl port-forward ..." works
+    nifi.web.http.network.interface.lo: lo
 
 ## Include additional libraries in the Nifi containers by using the postStart handler
 ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/
@@ -87,8 +90,8 @@ headless:
 
 # ui service
 service:
-  type: LoadBalancer
-  httpPort: 80
+  type: NodePort
+  httpPort: 8080
   httpsPort: 443
   annotations: {}
   # loadBalancerIP:
@@ -111,19 +114,6 @@ service:
         targetPort: 7002
         #nodePort: 30702
 
-  # Enables additional port/ports to nifi service for internal processors
-  processors:
-    enabled: false
-    ports:
-      - name: processor01
-        port: 7001
-        targetPort: 7001
-        #nodePort: 30701
-      - name: processor02
-        port: 7002
-        targetPort: 7002
-        #nodePort: 30702
-
 ## Configure Ingress based on the documentation here: https://kubernetes.io/docs/concepts/services-networking/ingress/
 ##
 ingress:
@@ -216,6 +206,11 @@ extraVolumeMounts: []
 
 extraVolumes: []
 
+## Extra containers
+extraContainers: []
+
+terminationGracePeriodSeconds: 30
+
 ## Extra environment variables that will be pass onto deployment pods
 env: []
 
@@ -226,7 +221,6 @@ zookeeper:
   ## If true, install the Zookeeper chart
   ## ref: https://github.com/kubernetes/charts/tree/master/incubator/zookeeper
   enabled: true
-
   ## If the Zookeeper Chart is disabled a URL and port are required to connect
   url: ""
-  port: 2181
+  port: 2181