Merge pull request #3230 from fluxcd/release/1.20.1

Release 1.20.1

.github/workflows/rebase.yaml

@@ -0,0 +1,21 @@
+name: rebase
+
+on:
+  pull_request:
+    types: [opened]
+  issue_comment:
+    types: [created]
+
+jobs:
+  rebase:
+    if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') && (github.event.comment.author_association == 'CONTRIBUTOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the latest code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Automatic Rebase
+        uses: cirrus-actions/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }}

CHANGELOG.md

@@ -1,3 +1,50 @@
+> Note on the future of Flux and Helm Operator: We are working on a
+> next generation Flux assembled from components as part of a bigger
+> [GitOps Toolkit](https://toolkit.fluxcd.io/) project. The roadmap
+> for the GitOps Toolkit can be found
+> [here](https://toolkit.fluxcd.io/roadmap/).
+>
+> We are eager to hear [feedback, suggestions, and/or feature
+> requests](https://github.com/fluxcd/toolkit/discussions) for Flux v2
+> and Toolkit components.
+
+## 1.20.1 (2020-08-05)
+
+This patch release has some fixes for faults in improvements in
+1.20.0.
+
+### Fixes
+
+- Do not return error when failed to load last-synced resources
+  [fluxcd/flux#3223][]
+- Dockerfile: Include /sbin dir in PATH [fluxcd/flux#3211][]
+- Put notice re gitops-engine at top of README [fluxcd/flux#3197][]
+- Avoid panic when directory does not exist [fluxcd/flux#3193][]
+- Put git messages into tmp files [fluxcd/flux#3179][]
+
+### Maintenance and documentation
+
+- Give advice on percent-encoding creds using in URL [fluxcd/flux#3204][]
+- Get shellcheck from new URL [fluxcd/flux#3224][]
+- Add Sngular as Flux user [fluxcd/flux#3212][]
+- Add rebase GitHub action [fluxcd/flux#3190][]
+
+### Thanks
+
+Thanks to @alex-shpak, @mmorejon, @ordovicia, @ricardo-larosa,
+@squaremo and @stefanprodan for their contributions to this release.
+
+[fluxcd/flux#3224]: https://github.com/fluxcd/flux/pull/3224
+[fluxcd/flux#3223]: https://github.com/fluxcd/flux/pull/3223
+[fluxcd/flux#3212]: https://github.com/fluxcd/flux/pull/3212
+[fluxcd/flux#3211]: https://github.com/fluxcd/flux/pull/3211
+[fluxcd/flux#3204]: https://github.com/fluxcd/flux/pull/3204
+[fluxcd/flux#3197]: https://github.com/fluxcd/flux/pull/3197
+[fluxcd/flux#3193]: https://github.com/fluxcd/flux/pull/3193
+[fluxcd/flux#3190]: https://github.com/fluxcd/flux/pull/3190
+[fluxcd/flux#3179]: https://github.com/fluxcd/flux/pull/3179
+[fluxcd/flux#3178]: https://github.com/fluxcd/flux/pull/3178
+
 ## 1.20.0 (2020-07-08)
 
 This minor version release updates dependencies, and includes some

Makefile

@@ -119,6 +119,7 @@ cache/%/kubectl-$(KUBECTL_VERSION): docker/kubectl.version
 	tar -m --strip-components 3 -C ./cache/$* -xzf cache/$*/kubectl-$(KUBECTL_VERSION).tar.gz kubernetes/client/bin/kubectl
 	mv ./cache/$*/kubectl $@
 
+# FIXME OS and architecture in download URL
 cache/%/kustomize-$(KUSTOMIZE_VERSION): docker/kustomize.version
 	mkdir -p cache/$*
 	curl --fail -L -o cache/$*/kustomize-$(KUSTOMIZE_VERSION).tar.gz "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv$(KUSTOMIZE_VERSION)/kustomize_v$(KUSTOMIZE_VERSION)_linux_amd64.tar.gz"
@@ -132,9 +133,10 @@ cache/%/helm-$(HELM_VERSION):
 	tar -m -C ./cache -xzf cache/$*/helm-$(HELM_VERSION).tar.gz $*/helm
 	mv cache/$*/helm $@
 
+# FIXME architecture in download URL
 cache/%/shellcheck-$(SHELLCHECK_VERSION):
 	mkdir -p cache/$*
-	curl --fail -L -o cache/$*/shellcheck-$(SHELLCHECK_VERSION).tar.xz "https://storage.googleapis.com/shellcheck/shellcheck-v$(SHELLCHECK_VERSION).$(CURRENT_OS).x86_64.tar.xz"
+	curl --fail -L -o cache/$*/shellcheck-$(SHELLCHECK_VERSION).tar.xz "https://github.com/koalaman/shellcheck/releases/download/v$(SHELLCHECK_VERSION)/shellcheck-v$(SHELLCHECK_VERSION).$(CURRENT_OS).x86_64.tar.xz"
 	tar -C cache/$* --strip-components 1 -xvJf cache/$*/shellcheck-$(SHELLCHECK_VERSION).tar.xz shellcheck-v$(SHELLCHECK_VERSION)/shellcheck
 	mv cache/$*/shellcheck $@
 

README.md

@@ -1,6 +1,16 @@
 # Flux
 
-> **Announcement** [Flux has joined CNCF as a sandbox project](https://www.weave.works/blog/flux-joins-the-cncf-sandbox)
+> **On Flux v2** In an announcement in August 2019, the expectation
+> was set that the Flux project would integrate the GitOps Engine,
+> then being factored out of ArgoCD. Since the result would be
+> backward-incompatible, it would require a major version bump: Flux
+> v2.
+>
+> After experimentation and considerable thought, we (the maintainers)
+> have found a path to Flux v2 that we think better serves our vision
+> of GitOps: the [GitOps Toolkit](https://toolkit.fluxcd.io/). In
+> consequence, we do not now plan to integrate GitOps Engine into
+> Flux.
 
 We believe in GitOps:
 
@@ -64,12 +74,12 @@ container images and config changes are propagated to the cluster.
 |[Okteto Cloud](https://okteto.com/)|[Omise](https://www.omise.co)|[Payout](https://payout.one)|
 |[Plex Systems](https://www.plex.com/)|[Qordoba](https://qordoba.com)|[Rakuten](https://rakuten.com)|
 |[RentPath](https://rentpath.com)|[Replicated](https://replicated.com)|[Resulta](https://resulta.com)|
-|[Rungway](https://rungway.com)|[Sage AI Labs](https://www.sage.com)|[Starbucks](https://www.starbucks.com/)|
-|[SupplyStack](https://www.supplystack.com/)|[Talend](https://www.talend.com)|[Troii](https://troii.com/)|
-|[UK Hydrographic Office](https://www.gov.uk/government/organisations/uk-hydrographic-office)|[Under Armour](https://www.underarmour.com)|[VSHN](https://vshn.ch)|
-|[Walmart Chile](https://www.walmartchile.cl)|[Weave Cloud](https://cloud.weave.works)|[Workable](https://www.workable.com)|
-|[Workarea](https://www.workarea.com)|[Working Group Two](https://wgtwo.com)|[Yad2](https://yad2.co.il)|
-|[Yusofleet](https://yusofleet.com)|[Zaaksysteem](https://zaaksysteem.nl)||
+|[Rungway](https://rungway.com)|[Sage AI Labs](https://www.sage.com)|[Sngular](https://www.sngular.com)|
+[Starbucks](https://www.starbucks.com/)|[SupplyStack](https://www.supplystack.com/)|[Talend](https://www.talend.com)|
+|[Troii](https://troii.com/)|[UK Hydrographic Office](https://www.gov.uk/government/organisations/uk-hydrographic-office)|[Under Armour](https://www.underarmour.com)|
+|[VSHN](https://vshn.ch)|[Walmart Chile](https://www.walmartchile.cl)|[Weave Cloud](https://cloud.weave.works)|
+|[Workable](https://www.workable.com)|[Workarea](https://www.workarea.com)|[Working Group Two](https://wgtwo.com)|
+|[Yad2](https://yad2.co.il)|[Yusofleet](https://yusofleet.com)|[Zaaksysteem](https://zaaksysteem.nl)|
 
 If you too are using Flux in production; please submit a PR to add your organization to the list!
 

chart/flux/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 1.4.0 (2020-07-09)
+
+### Improvements
+
+ - Updated Flux to `1.20.0`
+   [fluxcd/flux#3177](https://github.com/fluxcd/flux/pull/3177)
+ - Add Grafana dashboard
+   [fluxcd/flux#3169](https://github.com/fluxcd/flux/pull/3169)
+ - Support git clone over ssh behind http proxy
+   [fluxcd/flux#3152](https://github.com/fluxcd/flux/pull/3152)
+ - Roll deployment on update known_hosts
+   [fluxcd/flux#3096](https://github.com/fluxcd/flux/pull/3096)
+ - Preserve namespace when running helm template
+   [fluxcd/flux#3076](https://github.com/fluxcd/flux/pull/3076)
+ - Support annotations in chart SSH secret 
+   [fluxcd/flux#3053](https://github.com/fluxcd/flux/pull/3053)
+
 ## 1.3.0 (2020-04-03)
 
 ### Improvements

chart/flux/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-appVersion: "1.19.0"
-version: 1.3.0
+appVersion: "1.20.0"
+version: 1.4.0
 kubeVersion: ">=1.9.0-0"
 name: flux
 description: Flux is a tool that automatically ensures that the state of a cluster matches what is specified in version control

chart/flux/README.md

@@ -267,6 +267,7 @@ The following tables lists the configurable parameters of the Flux chart and the
 | `memcached.nodeSelector`                          | `{}`                                                 | Node Selector properties for the memcached deployment
 | `memcached.tolerations`                           | `[]`                                                 | Tolerations properties for the memcached deployment
 | `memcached.priorityClassName`                     | `""`                                                 | The name of the priority class to assign to the memcached pod.
+| `kube.externalConfig`                             | `false`                                              | If enabled, no kubeconfig and env var pointing to the kubeconfig will be created. You need to provide both on your own.
 | `kube.config`                                     | [See values.yaml][kubeconfig-ref]                    | Override for kubectl default config in the Flux pod(s).
 | `priorityClassName`                               | `""`                                                 | Set priority class for Flux
 | `prometheus.enabled`                              | `false`                                              | If enabled, adds prometheus annotations to Flux and helmOperator pod(s)

chart/flux/templates/deployment.yaml

@@ -44,9 +44,11 @@ spec:
         - name: {{ .Values.image.pullSecret }}
       {{- end }}
       volumes:
+      {{- if not .Values.kube.externalConfig }}
       - name: kubedir
         configMap:
           name: {{ template "flux.fullname" . }}-kube-config
+      {{- end }}
       {{- if .Values.ssh.known_hosts }}
       - name: sshdir
         configMap:
@@ -129,8 +131,10 @@ spec:
             initialDelaySeconds: 5
             timeoutSeconds: 5
           volumeMounts:
+          {{- if not .Values.kube.externalConfig }}
           - name: kubedir
             mountPath: /root/.kubectl
+          {{- end }}
           {{- if .Values.ssh.known_hosts }}
           - name: sshdir
             mountPath: /root/.ssh

chart/flux/templates/kube.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.kube.externalConfig }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -26,4 +27,5 @@ data:
       {{- else }}
     {{ .Values.kube.config }}
       {{- end }}
-    {{- end }}
+    {{- end }}
+{{- end }}

chart/flux/values.yaml

@@ -9,7 +9,7 @@ logFormat: fmt
 
 image:
   repository: docker.io/fluxcd/flux
-  tag: 1.20.0
+  tag: 1.20.1
   pullPolicy: IfNotPresent
   pullSecret:
 
@@ -242,6 +242,9 @@ ssh:
 
 
 kube:
+  # Disable KUBECONFIG env var and passing the default config into the Container
+  # This means you need to provide both on your own, by using extraVars and ExtraVolume(Mounts)
+  externalConfig: false
   # Override for kubectl default config
   config: |
     apiVersion: v1

deploy/flux-deployment.yaml

@@ -66,7 +66,7 @@ spec:
         # There are no ":latest" images for flux. Find the most recent
         # release or image version at https://hub.docker.com/r/fluxcd/flux/tags
         # and replace the tag here.
-        image: docker.io/fluxcd/flux:1.20.0
+        image: docker.io/fluxcd/flux:1.20.1
         imagePullPolicy: IfNotPresent
         resources:
           requests:

docker/Dockerfile.flux

@@ -36,7 +36,7 @@ ENTRYPOINT [ "/sbin/tini", "--", "fluxd" ]
 
 # Get the kubeyaml binary (files) and put them on the path
 COPY --from=squaremo/kubeyaml:0.7.0 /usr/lib/kubeyaml /usr/lib/kubeyaml/
-ENV PATH=/bin:/usr/bin:/usr/local/bin:/usr/lib/kubeyaml
+ENV PATH=/bin:/sbin:/usr/bin:/usr/local/bin:/usr/lib/kubeyaml
 
 # Create minimal nsswitch.conf file to prioritize the usage of /etc/hosts over DNS queries.
 # This resolves the conflict between:

docs/guides/use-git-https.md

@@ -15,6 +15,14 @@ as a plain value in your workload.
     to pass the values to the Flux container, e.g. `$(GIT_AUTHKEY)`.
     [Read more about this Kubernetes feature](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config).
 
+!!!note
+    Each of the username and password must be percent-encoded, otherwise
+    the git URL may end up being invalid once they have been interpolated
+    in. You can encode a string with Perl (assuming your token is in the
+    environment variable `TOKEN`):
+
+        echo "$TOKEN" | perl -MURI::Escape -ne 'chomp;print uri_escape($_),"\n"'
+
 1. Create a personal access token to be used as the `GIT_AUTHKEY`:
 
     - [GitHub](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line)

pkg/cluster/kubernetes/resource/load.go

@@ -28,14 +28,18 @@ func Load(base string, paths []string, sopsEnabled bool) (map[string]KubeManifes
 		return nil, errors.Wrapf(err, "walking %q for chartdirs", base)
 	}
 	for _, root := range paths {
+		// In the walk, we ignore errors (indicating a failure to read
+		// a file) if it's not a file of interest. However, we _are_
+		// interested in the error if an explicitly-mentioned path
+		// does not exist.
+		if _, err := os.Stat(root); err != nil {
+			return nil, errors.Wrapf(err, "unable to read root path %q", root)
+		}
 		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-			if info.IsDir() {
+			if err == nil && info.IsDir() {
 				if charts.isDirChart(path) {
 					return filepath.SkipDir
 				}
-				if err != nil {
-					return errors.Wrapf(err, "walking dir %q for yaml files", path)
-				}
 				return nil
 			}
 

pkg/cluster/kubernetes/resource/load_test.go

@@ -362,3 +362,15 @@ func TestLoadSomeWithSopsAllEncrypted(t *testing.T) {
 		assert.NotNil(t, objs[expected.String()], "expected to find %s in manifest map after decryption", expected)
 	}
 }
+
+func TestNoPanic(t *testing.T) {
+	dir, cleanup := testfiles.TempDir(t)
+	defer cleanup()
+	if err := testfiles.WriteTestFiles(dir, testfiles.Files); err != nil {
+		t.Fatal(err)
+	}
+	_, err := Load(dir, []string{filepath.Join(dir, "doesnotexist")}, true)
+	if err == nil {
+		t.Error("expected error (but not panic) when loading from directory that doesn't exist")
+	}
+}

pkg/daemon/sync.go

@@ -48,7 +48,8 @@ func (d *Daemon) Sync(ctx context.Context, started time.Time, newRevision string
 	// Load last-synced resources for comparison
 	lastResources, err := d.getLastResources(ctx, rat)
 	if err != nil {
-		return errors.Wrap(err, "loading last-synced resources")
+		d.Logger.Log("warning", "failed to load last-synced resources. sync event may be inaccurate", "err", err)
+		lastResources = map[string]resource.Resource{}
 	}
 
 	// Retrieve change set of commits we need to sync

pkg/git/operations.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -141,7 +142,16 @@ func secretUnseal(ctx context.Context, workingDir string) error {
 }
 
 func commit(ctx context.Context, workingDir string, commitAction CommitAction) error {
-	args := []string{"commit", "--no-verify", "-a", "-m", commitAction.Message}
+	message, err := ioutil.TempFile("", "flux-commit-*.txt")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(message.Name())
+	if _, err := message.WriteString(commitAction.Message); err != nil {
+		return err
+	}
+
+	args := []string{"commit", "--no-verify", "-a", "--file", message.Name()}
 	var env []string
 	if commitAction.Author != "" {
 		args = append(args, "--author", commitAction.Author)
@@ -205,7 +215,17 @@ func addNote(ctx context.Context, workingDir, rev, notesRef string, note interfa
 	if err != nil {
 		return err
 	}
-	args := []string{"notes", "--ref", notesRef, "add", "-m", string(b), rev}
+
+	message, err := ioutil.TempFile("", "flux-note-*.json")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(message.Name())
+	if _, err := message.Write(b); err != nil {
+		return err
+	}
+
+	args := []string{"notes", "--ref", notesRef, "add", "--file", message.Name(), rev}
 	return execGitCmd(ctx, args, gitCmdConfig{dir: workingDir})
 }