[nix profile] Changes to support format changes from nix 2.20

.github/workflows/cli-tests.yaml

@@ -129,8 +129,9 @@ jobs:
         run-project-tests: ["project-tests", "project-tests-off"]
         # Run tests on:
         # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0)
-        # 2. latest nix version
-        nix-version: ["2.12.0", "2.19.2"]
+        # 2. nix 2.19.2: version before nix profile changes
+        # 2. latest nix version (note, 2.20.1 introduced a new profile change)
+        nix-version: ["2.12.0", "2.19.2", "2.20.1"]
         exclude:
           - is-main: "not-main"
             os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}"

internal/nix/nixprofile/item.go

@@ -16,6 +16,11 @@ type NixProfileListItem struct {
 	// invocations of nix profile remove and nix profile upgrade.
 	index int
 
+	// name of the package
+	// nix 2.20 introduced a new format for the output of nix profile list, which includes the package name.
+	// This field is used instead of index for `list`, `remove` and `upgrade` subcommands of `nix profile`.
+	name string
+
 	// The original ("unlocked") flake reference and output attribute path used at installation time.
 	// NOTE that this will be empty if the package was added to the nix profile via store path.
 	unlockedReference string
@@ -74,10 +79,10 @@ func (i *NixProfileListItem) addedByStorePath() bool {
 	return i.unlockedReference == ""
 }
 
-// String serializes the NixProfileListItem back into the format printed by `nix profile list`
+// String serializes the NixProfileListItem for debuggability
 func (i *NixProfileListItem) String() string {
-	return fmt.Sprintf("{%d %s %s %s}",
-		i.index,
+	return fmt.Sprintf("{nameOrIndex:%s unlockedRef:%s lockedRef:%s, nixStorePaths:%s}",
+		i.NameOrIndex(),
 		i.unlockedReference,
 		i.lockedReference,
 		i.nixStorePaths,
@@ -87,3 +92,13 @@ func (i *NixProfileListItem) String() string {
 func (i *NixProfileListItem) StorePaths() []string {
 	return i.nixStorePaths
 }
+
+// NameOrIndex is a helper method to get the name of the package if it exists, or the index if it doesn't.
+// `nix profile` subcommands `list`, `remove`, and `upgrade` use either name (nix >= 2.20) or index (nix < 2.20)
+// to identify the package.
+func (i *NixProfileListItem) NameOrIndex() string {
+	if i.name != "" {
+		return i.name
+	}
+	return fmt.Sprintf("%d", i.index)
+}

internal/nix/nixprofile/profile.go

@@ -42,24 +42,46 @@ func ProfileListItems(
 		URL         string   `json:"url"`
 	}
 	type ProfileListOutput struct {
-		Elements []ProfileListElement `json:"elements"`
-		Version  int                  `json:"version"`
+		Elements map[string]ProfileListElement `json:"elements"`
+		Version  int                           `json:"version"`
 	}
 
+	// Modern nix profiles: nix >= 2.20
 	var structOutput ProfileListOutput
-	if err := json.Unmarshal([]byte(output), &structOutput); err != nil {
-		return nil, err
+	if err := json.Unmarshal([]byte(output), &structOutput); err == nil {
+		items := []*NixProfileListItem{}
+		for name, element := range structOutput.Elements {
+			items = append(items, &NixProfileListItem{
+				name:              name,
+				unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""),
+				lockedReference:   lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""),
+				nixStorePaths:     element.StorePaths,
+			})
+		}
+		return items, nil
 	}
+	// Fall back to trying format for nix < version 2.20
 
+	// ProfileListOutputJSONLegacy is for parsing `nix profile list --json` in nix < version 2.20
+	// that relied on index instead of name for each package installed.
+	type ProfileListOutputJSONLegacy struct {
+		Elements []ProfileListElement `json:"elements"`
+		Version  int                  `json:"version"`
+	}
+	var structOutput2 ProfileListOutputJSONLegacy
+	if err := json.Unmarshal([]byte(output), &structOutput2); err != nil {
+		return nil, err
+	}
 	items := []*NixProfileListItem{}
-	for index, element := range structOutput.Elements {
+	for index, element := range structOutput2.Elements {
 		items = append(items, &NixProfileListItem{
 			index:             index,
 			unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""),
 			lockedReference:   lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""),
 			nixStorePaths:     element.StorePaths,
 		})
 	}
+
 	return items, nil
 }
 
@@ -88,7 +110,7 @@ func profileListLegacy(
 		if line == "" {
 			continue
 		}
-		item, err := parseNixProfileListItem(line)
+		item, err := parseNixProfileListItemLegacy(line)
 		if err != nil {
 			return nil, err
 		}
@@ -98,7 +120,7 @@ func profileListLegacy(
 	return items, nil
 }
 
-type ProfileListIndexArgs struct {
+type ProfileListNameOrIndexArgs struct {
 	// For performance, you can reuse the same list in multiple operations if you
 	// are confident index has not changed.
 	Items      []*NixProfileListItem
@@ -108,21 +130,21 @@ type ProfileListIndexArgs struct {
 	ProfileDir string
 }
 
-// ProfileListIndex returns the index of args.Package in the nix profile specified by args.ProfileDir,
-// or -1 if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again.
-func ProfileListIndex(args *ProfileListIndexArgs) (int, error) {
+// ProfileListNameOrIndex returns the name or index of args.Package in the nix profile specified by args.ProfileDir,
+// or nix.ErrPackageNotFound if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again.
+func ProfileListNameOrIndex(args *ProfileListNameOrIndexArgs) (string, error) {
 	var err error
 	items := args.Items
 	if items == nil {
 		items, err = ProfileListItems(args.Writer, args.ProfileDir)
 		if err != nil {
-			return -1, err
+			return "", err
 		}
 	}
 
 	inCache, err := args.Package.IsInBinaryCache()
 	if err != nil {
-		return -1, err
+		return "", err
 	}
 
 	if !inCache && args.Package.IsDevboxPackage {
@@ -131,28 +153,29 @@ func ProfileListIndex(args *ProfileListIndexArgs) (int, error) {
 		// of an existing profile item.
 		ref, err := args.Package.NormalizedDevboxPackageReference()
 		if err != nil {
-			return -1, errors.Wrapf(err, "failed to get installable for %s", args.Package.String())
+			return "", errors.Wrapf(err, "failed to get installable for %s", args.Package.String())
 		}
 
 		for _, item := range items {
 			if ref == item.unlockedReference {
-				return item.index, nil
+				return item.NameOrIndex(), nil
 			}
 		}
-		return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
+		return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
 	}
 
 	for _, item := range items {
 		if item.Matches(args.Package, args.Lockfile) {
-			return item.index, nil
+			return item.NameOrIndex(), nil
 		}
 	}
-	return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
+	return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
 }
 
-// parseNixProfileListItem reads each line of output (from `nix profile list`) and converts
+// parseNixProfileListItemLegacy reads each line of output (from `nix profile list`) and converts
 // into a golang struct. Refer to NixProfileListItem struct definition for explanation of each field.
-func parseNixProfileListItem(line string) (*NixProfileListItem, error) {
+// NOTE: this API is for legacy nix. Newer nix versions use --json output.
+func parseNixProfileListItemLegacy(line string) (*NixProfileListItem, error) {
 	scanner := bufio.NewScanner(strings.NewReader(line))
 	scanner.Split(bufio.ScanWords)
 

internal/nix/nixprofile/profile_test.go

@@ -15,7 +15,10 @@ type expectedTestData struct {
 	packageName string
 }
 
-func TestNixProfileListItem(t *testing.T) {
+// TestNixProfileListItemLegacy tests the parsing of legacy nix profile list items.
+// It only applies to much older nix versions. Newer nix versions rely on the --json output
+// instead parsing the legacy output.
+func TestNixProfileListItemLegacy(t *testing.T) {
 	testCases := map[string]struct {
 		line     string
 		expected expectedTestData
@@ -49,10 +52,10 @@ func TestNixProfileListItem(t *testing.T) {
 			),
 			expected: expectedTestData{
 				item: &NixProfileListItem{
-					2,
-					"github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
-					"github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
-					[]string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"},
+					index:             2,
+					unlockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
+					lockedReference:   "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
+					nixStorePaths:     []string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"},
 				},
 				attrPath:    "legacyPackages.x86_64-darwin.python39Packages.numpy",
 				packageName: "python39Packages.numpy",
@@ -68,7 +71,7 @@ func TestNixProfileListItem(t *testing.T) {
 }
 
 func testItem(t *testing.T, line string, expected expectedTestData) {
-	item, err := parseNixProfileListItem(line)
+	item, err := parseNixProfileListItemLegacy(line)
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}

internal/nix/nixprofile/upgrade.go

@@ -12,8 +12,8 @@ import (
 )
 
 func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) error {
-	idx, err := ProfileListIndex(
-		&ProfileListIndexArgs{
+	nameOrIndex, err := ProfileListNameOrIndex(
+		&ProfileListNameOrIndexArgs{
 			Lockfile:   lock,
 			Writer:     os.Stderr,
 			Package:    pkg,
@@ -24,5 +24,5 @@ func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) err
 		return err
 	}
 
-	return nix.ProfileUpgrade(ProfileDir, idx)
+	return nix.ProfileUpgrade(ProfileDir, nameOrIndex)
 }

internal/nix/profiles.go

@@ -94,8 +94,8 @@ func ProfileRemove(profilePath string, indexes ...string) error {
 
 type manifest struct {
 	Elements []struct {
-		Priority int `json:"priority"`
-	} `json:"elements"`
+		Priority int
+	}
 }
 
 func readManifest(profilePath string) (manifest, error) {
@@ -107,8 +107,37 @@ func readManifest(profilePath string) (manifest, error) {
 		return manifest{}, err
 	}
 
-	var m manifest
-	return m, json.Unmarshal(data, &m)
+	type manifestModern struct {
+		Elements map[string]struct {
+			Priority int `json:"priority"`
+		} `json:"elements"`
+	}
+	var modernMani manifestModern
+	if err := json.Unmarshal(data, &modernMani); err == nil {
+		// Convert to the result format
+		result := manifest{}
+		for _, e := range modernMani.Elements {
+			result.Elements = append(result.Elements, struct{ Priority int }{e.Priority})
+		}
+		return result, nil
+	}
+
+	type manifestLegacy struct {
+		Elements []struct {
+			Priority int `json:"priority"`
+		} `json:"elements"`
+	}
+	var legacyMani manifestLegacy
+	if err := json.Unmarshal(data, &legacyMani); err != nil {
+		return manifest{}, err
+	}
+
+	// Convert to the result format
+	result := manifest{}
+	for _, e := range legacyMani.Elements {
+		result.Elements = append(result.Elements, struct{ Priority int }{e.Priority})
+	}
+	return result, nil
 }
 
 const DefaultPriority = 5

internal/nix/search.go

@@ -98,15 +98,22 @@ func searchSystem(url, system string) (map[string]*Info, error) {
 		_ = EnsureNixpkgsPrefetched(writer, hash)
 	}
 
-	cmd := exec.Command("nix", "search", "--json", url)
+	// The `^` is added to indicate we want to show all packages
+	cmd := exec.Command("nix", "search", url, "^" /*regex*/, "--json")
 	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
 	if system != "" {
 		cmd.Args = append(cmd.Args, "--system", system)
 	}
 	debug.Log("running command: %s\n", cmd)
 	out, err := cmd.Output()
 	if err != nil {
+		if exitErr := (&exec.ExitError{}); errors.As(err, &exitErr) {
+			err = fmt.Errorf("nix search exit code: %d, stderr: %s, original error: %w", exitErr.ExitCode(), exitErr.Stderr, err)
+		}
+
 		// for now, assume all errors are invalid packages.
+		// TODO: check the error string for "did not find attribute" and
+		// return ErrPackageNotFound only for that case.
 		return nil, fmt.Errorf("error searching for pkg %s: %w", url, err)
 	}
 	return parseSearchResults(out), nil

internal/nix/upgrade.go

@@ -4,7 +4,6 @@
 package nix
 
 import (
-	"fmt"
 	"os"
 	"os/exec"
 
@@ -13,11 +12,11 @@ import (
 	"go.jetpack.io/devbox/internal/vercheck"
 )
 
-func ProfileUpgrade(ProfileDir string, idx int) error {
+func ProfileUpgrade(ProfileDir, indexOrName string) error {
 	cmd := command(
 		"profile", "upgrade",
 		"--profile", ProfileDir,
-		fmt.Sprintf("%d", idx),
+		indexOrName,
 	)
 	out, err := cmd.CombinedOutput()
 	if err != nil {