fix: prevent various Mutagen conflicts between daemons when changing …

…versions or global directory, fixes ddev#6234 (ddev#6239)

Co-authored-by: Stanislav Zhuk <[email protected]>

.buildkite/test.sh

@@ -192,12 +192,12 @@ fi
 # Make sure we start with mutagen daemon off.
 unset MUTAGEN_DATA_DIRECTORY
 if [ -f ~/.ddev/bin/mutagen -o -f ~/.ddev/bin/mutagen.exe ]; then
-  # This line can be removed when ~/.ddev/.mdd is well established in master, probably July 2024
   MUTAGEN_DATA_DIRECTORY=~/.ddev_mutagen_data_directory/ ~/.ddev/bin/mutagen sync terminate -a || true
+  # This line can be removed when all PRs will not use ~/.ddev/.mdd
   MUTAGEN_DATA_DIRECTORY=~/.ddev/.mdd/ ~/.ddev/bin/mutagen sync terminate -a || true
   MUTAGEN_DATA_DIRECTORY=~/.mutagen ~/.ddev/bin/mutagen daemon stop || true
-  # This line can be removed when ~/.ddev/.mdd is well established in master, probably July 2024
   MUTAGEN_DATA_DIRECTORY=~/.ddev_mutagen_data_directory/ ~/.ddev/bin/mutagen daemon stop || true
+  # This line can be removed when all PRs will not use ~/.ddev/.mdd
   MUTAGEN_DATA_DIRECTORY=~/.ddev/.mdd/ ~/.ddev/bin/mutagen daemon stop || true
 fi
 if command -v killall >/dev/null ; then

cmd/ddev/cmd/a.go

@@ -14,7 +14,8 @@ import (
 func init() {
 	globalconfig.EnsureGlobalConfig()
 	_ = os.Setenv("DOCKER_CLI_HINTS", "false")
-	_ = os.Setenv("MUTAGEN_DATA_DIRECTORY", globalconfig.GetMutagenDataDirectory())
+	// GetMutagenDataDirectory() sets MUTAGEN_DATA_DIRECTORY
+	_ = globalconfig.GetMutagenDataDirectory()
 	// GetDockerClient should be called early to get DOCKER_HOST set
 	_, _ = dockerutil.GetDockerClient()
 }

cmd/ddev/cmd/mutagen-cmd_test.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/ddev/ddev/pkg/testcommon"
 	"os"
 	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/ddev/ddev/pkg/testcommon"
+
 	"github.com/ddev/ddev/pkg/config/types"
 	"github.com/ddev/ddev/pkg/ddevapp"
 	"github.com/ddev/ddev/pkg/exec"
@@ -23,6 +24,9 @@ func TestCmdMutagen(t *testing.T) {
 	// Gather reporting about goroutines at exit
 	_ = os.Setenv("DDEV_GOROUTINES", "true")
 
+	origDdevDebug := os.Getenv("DDEV_DEBUG")
+	_ = os.Setenv(`DDEV_DEBUG`, `true`) // test requires DDEV_DEBUG to see removal of docker volume
+
 	if nodeps.PerformanceModeDefault == types.PerformanceModeMutagen || nodeps.NoBindMountsDefault {
 		t.Skip("Skipping because Mutagen on by default")
 	}
@@ -52,6 +56,8 @@ func TestCmdMutagen(t *testing.T) {
 		app, err = ddevapp.NewApp(site.Dir, true)
 		assert.NoError(err)
 
+		_ = os.Setenv(`DDEV_DEBUG`, origDdevDebug)
+
 		err = app.Start()
 		assert.NoError(err)
 
@@ -89,15 +95,19 @@ func TestCmdMutagen(t *testing.T) {
 	// Make sure it got turned on
 	assert.True(app.IsMutagenEnabled())
 
+	t.Logf("DDEV_GOROUTINES before app.StartAndWait()=%s", os.Getenv(`DDEV_GOROUTINES`))
+
 	// Now test subcommands. Wait a bit for Mutagen to get completely done, with transition problems sorted out
 	err = app.StartAndWait(10)
 	require.NoError(t, err)
+	t.Logf("DDEV_GOROUTINES before first mutagen status --verbose=%s", os.Getenv(`DDEV_GOROUTINES`))
 	out, err = exec.RunHostCommand(DdevBin, "mutagen", "status", "--verbose")
 	testcommon.CheckGoroutineOutput(t, out)
 
 	assert.NoError(err)
 	assert.True(strings.HasPrefix(out, "Mutagen: ok"), "expected Mutagen: ok. Full output: %s", out)
 	assert.Contains(out, "Mutagen: ok")
+	t.Logf("DDEV_GOROUTINES before second mutagen status --verbose=%s", os.Getenv(`DDEV_GOROUTINES`))
 	out, err = exec.RunHostCommand(DdevBin, "mutagen", "status", "--verbose")
 	assert.NoError(err)
 	assert.Contains(out, "Alpha:")

cmd/ddev/cmd/mutagen-logs.go

@@ -1,14 +1,15 @@
 package cmd
 
 import (
-	"github.com/ddev/ddev/pkg/ddevapp"
-	"github.com/ddev/ddev/pkg/globalconfig"
-	"github.com/ddev/ddev/pkg/util"
-	"github.com/spf13/cobra"
 	"os"
 	"os/exec"
 	"os/signal"
 	"syscall"
+
+	"github.com/ddev/ddev/pkg/ddevapp"
+	"github.com/ddev/ddev/pkg/globalconfig"
+	"github.com/ddev/ddev/pkg/util"
+	"github.com/spf13/cobra"
 )
 
 // MutagenLogsCmd implements the ddev mutagen logs command
@@ -18,7 +19,7 @@ var MutagenLogsCmd = &cobra.Command{
 	Example: `"ddev mutagen logs"`,
 	Run: func(_ *cobra.Command, _ []string) {
 
-		ddevapp.StopMutagenDaemon()
+		ddevapp.StopMutagenDaemon("")
 		_ = os.Setenv("MUTAGEN_LOG_LEVEL", "trace")
 
 		sigs := make(chan os.Signal, 1)

cmd/ddev/cmd/restart.go

@@ -4,7 +4,6 @@ import (
 	"strings"
 
 	"github.com/ddev/ddev/pkg/ddevapp"
-
 	"github.com/ddev/ddev/pkg/dockerutil"
 	"github.com/ddev/ddev/pkg/output"
 	"github.com/ddev/ddev/pkg/util"

cmd/ddev/cmd/root.go

@@ -165,6 +165,7 @@ func checkDdevVersionAndOptInInstrumentation(skipConfirmation bool) error {
 			globalconfig.DdevGlobalConfig.InstrumentationOptIn = true
 		}
 	}
+
 	if globalconfig.DdevGlobalConfig.LastStartedVersion != versionconstants.DdevVersion && !skipConfirmation {
 
 		// If they have a new version (but not first-timer) then prompt to poweroff

cmd/ddev/cmd/root_test.go

@@ -295,17 +295,22 @@ func TestCopyGlobalDdevDir(t *testing.T) {
 	assert.NoError(err)
 }
 
-// TestGetGlobalDdevDirLocation checks to make sure that DDEV will use the correct location for its global config.
+// TestGetGlobalDdevDirLocation checks to make sure that DDEV will use the
+// correct location for its global config. The correct location is:
+// ${XDG_CONFIG_HOME}/ddev if XDG_CONFIG_HOME is set or
+// ~/.ddev if it exists or
+// ~/.config/ddev if it exists
 func TestGetGlobalDdevDirLocation(t *testing.T) {
 	// Test when $XDG_CONFIG_HOME is not set
 	t.Setenv("XDG_CONFIG_HOME", "")
 	ddevDir := globalconfig.GetGlobalDdevDirLocation()
 	// Original ~/.ddev dir location
 	originalGlobalDdevDir := filepath.Join(homedir.Get(), ".ddev")
-	// If this test runs on Linux machine, where ~/.config/ddev is used by default:
-	if runtime.GOOS == "linux" {
+	// If test runs on Linux machine, where ~/.config/ddev is used
+	// if ~/.ddev does not exist:
+	if runtime.GOOS == "linux" && !fileutil.IsDirectory(originalGlobalDdevDir) {
 		linuxDdevDir := filepath.Join(homedir.Get(), ".config", "ddev")
-		if _, err := os.Stat(linuxDdevDir); err == nil {
+		if fileutil.IsDirectory(linuxDdevDir) {
 			originalGlobalDdevDir = linuxDdevDir
 		}
 	}

docs/content/users/install/performance.md

@@ -150,23 +150,13 @@ Mutagen is enabled by default on Mac and traditional Windows, and it can be disa
     * Rename your project’s `.ddev/mutagen/mutagen.yml` file to `.ddev/mutagen/mutagen.yml.bak` and run `ddev restart`. This ensures you’ll have a fresh version in case the file has been changed and `#ddev-generated` removed.
     * Avoid having Mutagen sync large binaries, which can cause `ddev start` to take a long time. The `.tarballs` directory is automatically excluded, so Mutagen will ignore anything you move there. To see what Mutagen is trying to sync, run `ddev mutagen status -l` in another window.
     * `DDEV_DEBUG=true ddev start` will provide more information about what’s going on with Mutagen.
-    * DDEV’s Mutagen daemon keeps its data in a DDEV-only `MUTAGEN_DATA_DIRECTORY`, by default in `~/.ddev/.mdd`.
-    * DDEV’s private Mutagen binary is installed in `~/.ddev/bin/mutagen`. You can use all the features of Mutagen with `export MUTAGEN_DATA_DIRECTORY=~/.ddev/.mdd` and running the Mutagen binary in `~/.ddev/bin/mutagen`, for example:
+    * DDEV’s Mutagen daemon keeps its data in a DDEV-only `MUTAGEN_DATA_DIRECTORY` in `~/.ddev_mutagen_data_directory`.
+    * DDEV’s private Mutagen binary is installed in `~/.ddev/bin/mutagen` (or `$XDG_CONFIG_BASE/ddev/bin/mutagen`. You can use all the features of Mutagen with `ddev debug mutagen`. For example:
 
         ```bash
-        export MUTAGEN_DATA_DIRECTORY=$(ddev mutagen version -j | docker run -i --rm ddev/ddev-utilities jq -r '.raw.MUTAGEN_DATA_DIRECTORY' 2>/dev/null)
-        export PATH=$(ddev version -j | docker run -i --rm ddev/ddev-utilities jq -r '.raw."global-ddev-dir"' 2>/dev/null)/bin:$PATH
-        mutagen sync list -l
-        mutagen sync monitor
+        debug mutagen sync list --template "{{ json (index . 0) }}" | jq -r
+        ddev debug mutagen sync monitor <projectname> -l
         ```
-
-    * `ddev debug mutagen` will let you run any Mutagen command using the binary in `~/.ddev/bin/mutagen`, for example:
-
-        ```bash
-        ddev debug mutagen sync list -l
-        ddev debug mutagen monitor
-        ```
-
     * You can run the [diagnose_mutagen.sh](https://raw.githubusercontent.com/ddev/ddev/master/scripts/diagnose_mutagen.sh) script to gather information about Mutagen’s setup. Please share output from it when creating an issue or seeking support.
     * Try `ddev poweroff` or `~/.ddev/bin/mutagen daemon stop && ~/.ddev/bin/mutagen daemon start` to restart the Mutagen daemon if you suspect it’s hanging.
     * Use `ddev mutagen reset` if you suspect trouble, and *always* after changing `.ddev/mutagen/mutagen.yml`. This restarts the project’s Mutagen data (Docker volume + Mutagen session) from scratch.

docs/content/users/usage/architecture.md

@@ -100,10 +100,10 @@ Files beginning with `.` are hidden because they shouldn’t be fiddled with; mo
 
 ### Global Files
 
-There’s only one global `.ddev` directory, which lives in your home directory: `~/.ddev` (`$HOME/.ddev`).
+There’s only one global `.ddev` directory, which normally lives in your home directory: `~/.ddev` (`$HOME/.ddev`) or in `~/.config/ddev`. `~/.ddev` takes precedence if it exists, unless `$XDG_CONFIG_HOME` is set, in which case it will be `$XDG_CONFIG_HOME/ddev`.
 
 !!!tip "Where is my global `.ddev` config?"
-    Use `ddev version` (look at `global-ddev-dir`) to check which location is currently used for the `.ddev` global directory.
+    Use `ddev version` (look at `global-ddev-dir`) to check which location is used for the `.ddev` global directory.
 
 !!!tip "What if I don't want to clutter up my `$HOME` with a `.ddev` directory?"
     DDEV can use the `$XDG_CONFIG_HOME` environment variable from [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) to move `~/.ddev` to the `$XDG_CONFIG_HOME/ddev` directory if `$XDG_CONFIG_HOME` is [defined](https://superuser.com/questions/365847/where-should-the-xdg-config-home-variable-be-defined/):
@@ -116,14 +116,12 @@ There’s only one global `.ddev` directory, which lives in your home directory:
     mv ~/.ddev ${XDG_CONFIG_HOME}/ddev
     ```
 
-    Otherwise, on Linux/WSL2 only, the default `$HOME/.config/ddev` can be used when you move the config:
+    Otherwise, on Linux/WSL2 only, the default `$HOME/.config/ddev` can be used when `~/.config/ddev` exists and `~/.ddev` does not exist.  You can move the config directory with:
 
     ```bash
     mv ~/.ddev ~/.config/ddev
     ```
 
-    Note that the custom global config directory `ddev` (if it exists) takes precedence over the `~/.ddev` directory.
-
 `global_config.yaml`
 : This YAML file defines your global configuration, which consists of various [config settings](../configuration/config.md).
 
@@ -146,9 +144,6 @@ Again, these files are mostly regenerated on every `ddev start` so it’s best t
 `.gitignore`
 : Prevents files from getting checked in when they shouldn’t be.
 
-`.mdd`
-: Directory used for storing [Mutagen](../install/performance.md#mutagen) sync data.
-
 `.router-compose-full.yaml`
 : The complete, generated docker-compose directive used for DDEV’s router.
 
@@ -170,6 +165,9 @@ Again, these files are mostly regenerated on every `ddev start` so it’s best t
 `.update`
 : An empty file whose purpose is mysterious and intriguing.
 
+!!!tip "`.ddev_mutagen_data_directory`"
+    DDEV uses a global `~/.ddev_mutagen_data_directory` for storing [Mutagen](../install/performance.md#mutagen) sync data.
+
 ## Container Architecture
 
 It’s easiest to think of DDEV as a set of little networked computers (Docker containers) that are in a different network from your workstation but still reachable from it.

docs/content/users/usage/uninstall.md

@@ -5,6 +5,7 @@ A DDEV installation consists of:
 * The self-contained `ddev` binary.
 * Each project’s `.ddev` directory.
 * The global `~/.ddev` directory where various global items are stored. (This directory can be [moved](./architecture.md#global-files) to another location.)
+* The global `~/.ddev_mutagen_data_directory` directory where Mutagen sync data may be stored.
 * The associated Docker images and containers DDEV created.
 * Any entries in `/etc/hosts`.
 
@@ -18,6 +19,8 @@ To remove all DDEV-owned `/etc/hosts` entries: [`ddev hostname --remove-inactive
 
 To remove the global `.ddev` directory: `rm -r ~/.ddev`.
 
+To remove the global `.ddev_mutagen_data_directory` directory: `ddev poweroff && rm -r ~/.ddev_mutagen_data_directory`.
+
 If you installed Docker only for DDEV and want to uninstall it with all containers and images, uninstall it for your version of Docker.
 
 Otherwise:

pkg/archive/archive.go

@@ -481,7 +481,7 @@ func DownloadAndExtractTarball(url string, removeTopLevel bool) (string, func(),
 		_ = f.Close()
 	}()
 
-	util.Success("Downloading %s", url)
+	util.Debug("Downloading %s to %s", url, f.Name())
 	tarball := f.Name()
 	defer func() {
 		_ = os.Remove(tarball)

pkg/ddevapp/config.go

@@ -23,7 +23,6 @@ import (
 	"github.com/ddev/ddev/pkg/output"
 	"github.com/ddev/ddev/pkg/util"
 	"github.com/ddev/ddev/pkg/versionconstants"
-	"github.com/docker/docker/pkg/homedir"
 	copy2 "github.com/otiai10/copy"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
@@ -732,19 +731,6 @@ func (app *DdevApp) FixObsolete() {
 			util.Warning("attempted to remove %s but failed, you may want to remove it manually: %v", legacyCommandDir, err)
 		}
 	}
-
-	// Remove old ~/.ddev_mutagen_data_directory directory
-	legacyMutagenDataDir := filepath.Join(homedir.Get(), ".ddev_mutagen_data_directory")
-	if fileutil.IsDirectory(legacyMutagenDataDir) {
-		originalMutagenDataDir := os.Getenv("MUTAGEN_DATA_DIRECTORY")
-		_ = os.Setenv("MUTAGEN_DATA_DIRECTORY", legacyMutagenDataDir)
-		StopMutagenDaemon()
-		_ = os.Setenv("MUTAGEN_DATA_DIRECTORY", originalMutagenDataDir)
-		err := os.RemoveAll(legacyMutagenDataDir)
-		if err != nil {
-			util.Warning("attempted to remove %s but failed, you may want to remove it manually: %v", legacyMutagenDataDir, err)
-		}
-	}
 }
 
 type composeYAMLVars struct {

pkg/ddevapp/ddevapp.go

@@ -1126,6 +1126,7 @@ Fix with 'ddev config global --required-docker-compose-version="" --use-docker-c
 		// Check again to make sure the Mutagen Docker volume exists. It's compatible if we found it above
 		// so we can keep it in that case.
 		if !dockerutil.VolumeExists(GetMutagenVolumeName(app)) {
+			util.Debug("Creating new docker volume '%s' with signature '%v'", GetMutagenVolumeName(app), GetDefaultMutagenVolumeSignature(app))
 			_, err = dockerutil.CreateVolume(GetMutagenVolumeName(app), "local", nil, map[string]string{mutagenSignatureLabelName: GetDefaultMutagenVolumeSignature(app)})
 			if err != nil {
 				return fmt.Errorf("unable to create new Mutagen Docker volume %s: %v", GetMutagenVolumeName(app), err)
@@ -1385,7 +1386,7 @@ Fix with 'ddev config global --required-docker-compose-version="" --use-docker-c
 		}
 		err = CreateOrResumeMutagenSync(app)
 		if err != nil {
-			return fmt.Errorf("failed to create Mutagen sync session '%s'. You may be able to resolve this problem using 'ddev mutagen reset' (err=%v)", MutagenSyncName(app.Name), err)
+			return fmt.Errorf("failed to CreateOrResumeMutagenSync on Mutagen sync session '%s'. You may be able to resolve this problem using 'ddev mutagen reset' (err=%v)", MutagenSyncName(app.Name), err)
 		}
 		mStatus, _, _, err := app.MutagenStatus()
 		if err != nil {

pkg/ddevapp/hooks_test.go

@@ -86,7 +86,7 @@ func TestProcessHooks(t *testing.T) {
 
 		out := captureOutputFunc()
 		userOut := userOutFunc()
-		assert.Contains(out, task.stdoutExpect, "task: %v", task.task)
+		require.Contains(t, out, task.stdoutExpect, "task: %v", task.task)
 		assert.Contains(userOut, task.fulloutputExpect, "task: %v", task.task)
 		assert.NotContains(userOut, "Task failed")
 	}