diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml index d0a306fecfe..c2efdf61737 100644 --- a/.github/workflows/cli-tests.yaml +++ b/.github/workflows/cli-tests.yaml @@ -129,8 +129,9 @@ jobs: run-project-tests: ["project-tests", "project-tests-off"] # Run tests on: # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0) - # 2. latest nix version - nix-version: ["2.12.0", "2.19.2"] + # 2. nix 2.19.2: version before nix profile changes + # 2. latest nix version (note, 2.20.1 introduced a new profile change) + nix-version: ["2.12.0", "2.19.2", "2.20.1"] exclude: - is-main: "not-main" os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}" diff --git a/internal/nix/nixprofile/item.go b/internal/nix/nixprofile/item.go index af14ad0b4f0..adab885ae75 100644 --- a/internal/nix/nixprofile/item.go +++ b/internal/nix/nixprofile/item.go @@ -16,6 +16,11 @@ type NixProfileListItem struct { // invocations of nix profile remove and nix profile upgrade. index int + // name of the package + // nix 2.20 introduced a new format for the output of nix profile list, which includes the package name. + // This field is used instead of index for `list`, `remove` and `upgrade` subcommands of `nix profile`. + name string + // The original ("unlocked") flake reference and output attribute path used at installation time. // NOTE that this will be empty if the package was added to the nix profile via store path. unlockedReference string @@ -74,10 +79,10 @@ func (i *NixProfileListItem) addedByStorePath() bool { return i.unlockedReference == "" } -// String serializes the NixProfileListItem back into the format printed by `nix profile list` +// String serializes the NixProfileListItem for debuggability func (i *NixProfileListItem) String() string { - return fmt.Sprintf("{%d %s %s %s}", - i.index, + return fmt.Sprintf("{nameOrIndex:%s unlockedRef:%s lockedRef:%s, nixStorePaths:%s}", + i.NameOrIndex(), i.unlockedReference, i.lockedReference, i.nixStorePaths, @@ -87,3 +92,13 @@ func (i *NixProfileListItem) String() string { func (i *NixProfileListItem) StorePaths() []string { return i.nixStorePaths } + +// NameOrIndex is a helper method to get the name of the package if it exists, or the index if it doesn't. +// `nix profile` subcommands `list`, `remove`, and `upgrade` use either name (nix >= 2.20) or index (nix < 2.20) +// to identify the package. +func (i *NixProfileListItem) NameOrIndex() string { + if i.name != "" { + return i.name + } + return fmt.Sprintf("%d", i.index) +} diff --git a/internal/nix/nixprofile/profile.go b/internal/nix/nixprofile/profile.go index d293667c586..eeb0e710fc9 100644 --- a/internal/nix/nixprofile/profile.go +++ b/internal/nix/nixprofile/profile.go @@ -42,17 +42,38 @@ func ProfileListItems( URL string `json:"url"` } type ProfileListOutput struct { - Elements []ProfileListElement `json:"elements"` - Version int `json:"version"` + Elements map[string]ProfileListElement `json:"elements"` + Version int `json:"version"` } + // Modern nix profiles: nix >= 2.20 var structOutput ProfileListOutput - if err := json.Unmarshal([]byte(output), &structOutput); err != nil { - return nil, err + if err := json.Unmarshal([]byte(output), &structOutput); err == nil { + items := []*NixProfileListItem{} + for name, element := range structOutput.Elements { + items = append(items, &NixProfileListItem{ + name: name, + unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""), + lockedReference: lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""), + nixStorePaths: element.StorePaths, + }) + } + return items, nil } + // Fall back to trying format for nix < version 2.20 + // ProfileListOutputJSONLegacy is for parsing `nix profile list --json` in nix < version 2.20 + // that relied on index instead of name for each package installed. + type ProfileListOutputJSONLegacy struct { + Elements []ProfileListElement `json:"elements"` + Version int `json:"version"` + } + var structOutput2 ProfileListOutputJSONLegacy + if err := json.Unmarshal([]byte(output), &structOutput2); err != nil { + return nil, err + } items := []*NixProfileListItem{} - for index, element := range structOutput.Elements { + for index, element := range structOutput2.Elements { items = append(items, &NixProfileListItem{ index: index, unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""), @@ -60,6 +81,7 @@ func ProfileListItems( nixStorePaths: element.StorePaths, }) } + return items, nil } @@ -88,7 +110,7 @@ func profileListLegacy( if line == "" { continue } - item, err := parseNixProfileListItem(line) + item, err := parseNixProfileListItemLegacy(line) if err != nil { return nil, err } @@ -98,7 +120,7 @@ func profileListLegacy( return items, nil } -type ProfileListIndexArgs struct { +type ProfileListNameOrIndexArgs struct { // For performance, you can reuse the same list in multiple operations if you // are confident index has not changed. Items []*NixProfileListItem @@ -108,21 +130,21 @@ type ProfileListIndexArgs struct { ProfileDir string } -// ProfileListIndex returns the index of args.Package in the nix profile specified by args.ProfileDir, -// or -1 if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again. -func ProfileListIndex(args *ProfileListIndexArgs) (int, error) { +// ProfileListNameOrIndex returns the name or index of args.Package in the nix profile specified by args.ProfileDir, +// or nix.ErrPackageNotFound if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again. +func ProfileListNameOrIndex(args *ProfileListNameOrIndexArgs) (string, error) { var err error items := args.Items if items == nil { items, err = ProfileListItems(args.Writer, args.ProfileDir) if err != nil { - return -1, err + return "", err } } inCache, err := args.Package.IsInBinaryCache() if err != nil { - return -1, err + return "", err } if !inCache && args.Package.IsDevboxPackage { @@ -131,28 +153,29 @@ func ProfileListIndex(args *ProfileListIndexArgs) (int, error) { // of an existing profile item. ref, err := args.Package.NormalizedDevboxPackageReference() if err != nil { - return -1, errors.Wrapf(err, "failed to get installable for %s", args.Package.String()) + return "", errors.Wrapf(err, "failed to get installable for %s", args.Package.String()) } for _, item := range items { if ref == item.unlockedReference { - return item.index, nil + return item.NameOrIndex(), nil } } - return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) + return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) } for _, item := range items { if item.Matches(args.Package, args.Lockfile) { - return item.index, nil + return item.NameOrIndex(), nil } } - return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) + return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) } -// parseNixProfileListItem reads each line of output (from `nix profile list`) and converts +// parseNixProfileListItemLegacy reads each line of output (from `nix profile list`) and converts // into a golang struct. Refer to NixProfileListItem struct definition for explanation of each field. -func parseNixProfileListItem(line string) (*NixProfileListItem, error) { +// NOTE: this API is for legacy nix. Newer nix versions use --json output. +func parseNixProfileListItemLegacy(line string) (*NixProfileListItem, error) { scanner := bufio.NewScanner(strings.NewReader(line)) scanner.Split(bufio.ScanWords) diff --git a/internal/nix/nixprofile/profile_test.go b/internal/nix/nixprofile/profile_test.go index 426db621b1f..2fdb1c471c1 100644 --- a/internal/nix/nixprofile/profile_test.go +++ b/internal/nix/nixprofile/profile_test.go @@ -15,7 +15,10 @@ type expectedTestData struct { packageName string } -func TestNixProfileListItem(t *testing.T) { +// TestNixProfileListItemLegacy tests the parsing of legacy nix profile list items. +// It only applies to much older nix versions. Newer nix versions rely on the --json output +// instead parsing the legacy output. +func TestNixProfileListItemLegacy(t *testing.T) { testCases := map[string]struct { line string expected expectedTestData @@ -49,10 +52,10 @@ func TestNixProfileListItem(t *testing.T) { ), expected: expectedTestData{ item: &NixProfileListItem{ - 2, - "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", - "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", - []string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"}, + index: 2, + unlockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", + lockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", + nixStorePaths: []string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"}, }, attrPath: "legacyPackages.x86_64-darwin.python39Packages.numpy", packageName: "python39Packages.numpy", @@ -68,7 +71,7 @@ func TestNixProfileListItem(t *testing.T) { } func testItem(t *testing.T, line string, expected expectedTestData) { - item, err := parseNixProfileListItem(line) + item, err := parseNixProfileListItemLegacy(line) if err != nil { t.Fatalf("unexpected error %v", err) } diff --git a/internal/nix/nixprofile/upgrade.go b/internal/nix/nixprofile/upgrade.go index 2200e56b911..e905cb67f06 100644 --- a/internal/nix/nixprofile/upgrade.go +++ b/internal/nix/nixprofile/upgrade.go @@ -12,8 +12,8 @@ import ( ) func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) error { - idx, err := ProfileListIndex( - &ProfileListIndexArgs{ + nameOrIndex, err := ProfileListNameOrIndex( + &ProfileListNameOrIndexArgs{ Lockfile: lock, Writer: os.Stderr, Package: pkg, @@ -24,5 +24,5 @@ func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) err return err } - return nix.ProfileUpgrade(ProfileDir, idx) + return nix.ProfileUpgrade(ProfileDir, nameOrIndex) } diff --git a/internal/nix/profiles.go b/internal/nix/profiles.go index 5ca87e3d5f6..2589305c071 100644 --- a/internal/nix/profiles.go +++ b/internal/nix/profiles.go @@ -94,8 +94,8 @@ func ProfileRemove(profilePath string, indexes ...string) error { type manifest struct { Elements []struct { - Priority int `json:"priority"` - } `json:"elements"` + Priority int + } } func readManifest(profilePath string) (manifest, error) { @@ -107,8 +107,37 @@ func readManifest(profilePath string) (manifest, error) { return manifest{}, err } - var m manifest - return m, json.Unmarshal(data, &m) + type manifestModern struct { + Elements map[string]struct { + Priority int `json:"priority"` + } `json:"elements"` + } + var modernMani manifestModern + if err := json.Unmarshal(data, &modernMani); err == nil { + // Convert to the result format + result := manifest{} + for _, e := range modernMani.Elements { + result.Elements = append(result.Elements, struct{ Priority int }{e.Priority}) + } + return result, nil + } + + type manifestLegacy struct { + Elements []struct { + Priority int `json:"priority"` + } `json:"elements"` + } + var legacyMani manifestLegacy + if err := json.Unmarshal(data, &legacyMani); err != nil { + return manifest{}, err + } + + // Convert to the result format + result := manifest{} + for _, e := range legacyMani.Elements { + result.Elements = append(result.Elements, struct{ Priority int }{e.Priority}) + } + return result, nil } const DefaultPriority = 5 diff --git a/internal/nix/search.go b/internal/nix/search.go index a8ea01f0463..b7d1c33e53f 100644 --- a/internal/nix/search.go +++ b/internal/nix/search.go @@ -98,7 +98,8 @@ func searchSystem(url, system string) (map[string]*Info, error) { _ = EnsureNixpkgsPrefetched(writer, hash) } - cmd := exec.Command("nix", "search", "--json", url) + // The `^` is added to indicate we want to show all packages + cmd := exec.Command("nix", "search", url, "^" /*regex*/, "--json") cmd.Args = append(cmd.Args, ExperimentalFlags()...) if system != "" { cmd.Args = append(cmd.Args, "--system", system) @@ -106,7 +107,13 @@ func searchSystem(url, system string) (map[string]*Info, error) { debug.Log("running command: %s\n", cmd) out, err := cmd.Output() if err != nil { + if exitErr := (&exec.ExitError{}); errors.As(err, &exitErr) { + err = fmt.Errorf("nix search exit code: %d, stderr: %s, original error: %w", exitErr.ExitCode(), exitErr.Stderr, err) + } + // for now, assume all errors are invalid packages. + // TODO: check the error string for "did not find attribute" and + // return ErrPackageNotFound only for that case. return nil, fmt.Errorf("error searching for pkg %s: %w", url, err) } return parseSearchResults(out), nil diff --git a/internal/nix/upgrade.go b/internal/nix/upgrade.go index 1f303ce2801..f09a8ca7810 100644 --- a/internal/nix/upgrade.go +++ b/internal/nix/upgrade.go @@ -4,7 +4,6 @@ package nix import ( - "fmt" "os" "os/exec" @@ -13,11 +12,11 @@ import ( "go.jetpack.io/devbox/internal/vercheck" ) -func ProfileUpgrade(ProfileDir string, idx int) error { +func ProfileUpgrade(ProfileDir, indexOrName string) error { cmd := command( "profile", "upgrade", "--profile", ProfileDir, - fmt.Sprintf("%d", idx), + indexOrName, ) out, err := cmd.CombinedOutput() if err != nil {