diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 4849feaa64d..a4ff5856780 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -657,12 +657,45 @@ func main() { Usage: "One or more Nydus image reference(Multiple images should be split by commas)", EnvVars: []string{"SOURCES"}, }, + &cli.StringFlag{ + Name: "target", + Required: false, + Usage: "Target chunkdict image (Nydus) reference", + EnvVars: []string{"TARGET"}, + }, &cli.BoolFlag{ Name: "source-insecure", Required: false, Usage: "Skip verifying server certs for HTTPS source registry", EnvVars: []string{"SOURCE_INSECURE"}, }, + &cli.BoolFlag{ + Name: "target-insecure", + Required: false, + Usage: "Skip verifying server certs for HTTPS target registry", + EnvVars: []string{"TARGET_INSECURE"}, + }, + + &cli.StringFlag{ + Name: "backend-type", + Value: "", + Usage: "Type of storage backend, possible values: 'oss', 's3'", + EnvVars: []string{"BACKEND_TYPE"}, + }, + &cli.StringFlag{ + Name: "backend-config", + Value: "", + Usage: "Json configuration string for storage backend", + EnvVars: []string{"BACKEND_CONFIG"}, + }, + &cli.PathFlag{ + Name: "backend-config-file", + Value: "", + TakesFile: true, + Usage: "Json configuration file for storage backend", + EnvVars: []string{"BACKEND_CONFIG_FILE"}, + }, + &cli.StringFlag{ Name: "work-dir", Value: "./output", @@ -675,6 +708,12 @@ func main() { Usage: "Path to the nydus-image binary, default to search in PATH", EnvVars: []string{"NYDUS_IMAGE"}, }, + + &cli.BoolFlag{ + Name: "all-platforms", + Value: false, + Usage: "Generate chunkdict image for all platforms, conflicts with --platform", + }, &cli.StringFlag{ Name: "platform", Value: "linux/" + runtime.GOARCH, @@ -684,17 +723,31 @@ func main() { Action: func(c *cli.Context) error { setupLogLevel(c) + backendType, backendConfig, err := getBackendConfig(c, "", false) + if err != nil { + return err + } + _, arch, err := provider.ExtractOsArch(c.String("platform")) if err != nil { return err } generator, err := generator.New(generator.Opt{ - WorkDir: c.String("work-dir"), Sources: c.StringSlice("sources"), + Target: c.String("target"), SourceInsecure: c.Bool("source-insecure"), + TargetInsecure: c.Bool("target-insecure"), + + BackendType: backendType, + BackendConfig: backendConfig, + BackendForcePush: c.Bool("backend-force-push"), + + WorkDir: c.String("work-dir"), NydusImagePath: c.String("nydus-image"), ExpectedArch: arch, + AllPlatforms: c.Bool("all-platforms"), + Platforms: c.String("platform"), }) if err != nil { return err diff --git a/contrib/nydusify/pkg/build/builder.go b/contrib/nydusify/pkg/build/builder.go index 564f57454ae..177c0b9a209 100644 --- a/contrib/nydusify/pkg/build/builder.go +++ b/contrib/nydusify/pkg/build/builder.go @@ -41,8 +41,11 @@ type CompactOption struct { CompactConfigPath string } -type SaveOption struct { - BootstrapPath string +type GenerateOption struct { + BootstrapPaths []string + DatabasePath string + ChunkdictBootstrapPath string + OutputPath string } type Builder struct { @@ -148,15 +151,22 @@ func (builder *Builder) Run(option BuilderOption) error { return builder.run(args, option.PrefetchPatterns) } -// Save calls `nydus-image chunkdict save` to parse Nydus bootstrap -func (builder *Builder) Save(option SaveOption) error { +// Generate calls `nydus-image chunkdict generate` to get chunkdict +func (builder *Builder) Generate(option GenerateOption) error { + logrus.Infof("Invoking 'nydus-image chunkdict generate' command") args := []string{ "chunkdict", - "save", + "generate", "--log-level", "warn", "--bootstrap", - option.BootstrapPath, + option.ChunkdictBootstrapPath, + "--database", + option.DatabasePath, + "--output-json", + option.OutputPath, } + args = append(args, option.BootstrapPaths...) + return builder.run(args, "") } diff --git a/contrib/nydusify/pkg/chunkdict/generator/generator.go b/contrib/nydusify/pkg/chunkdict/generator/generator.go index c9472c459cc..13c91a9ca58 100644 --- a/contrib/nydusify/pkg/chunkdict/generator/generator.go +++ b/contrib/nydusify/pkg/chunkdict/generator/generator.go @@ -1,7 +1,10 @@ package generator import ( + "compress/gzip" "context" + "encoding/json" + "io" "io/fs" "os" "path/filepath" @@ -10,20 +13,47 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/containerd/containerd/namespaces" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/backend" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/build" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" - "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider" + originprovider "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider" + "github.com/goharbor/acceleration-service/pkg/remote" + + "github.com/containerd/nydus-snapshotter/pkg/converter" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter/provider" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" + "github.com/dustin/go-humanize" + "github.com/goharbor/acceleration-service/pkg/platformutil" + serverutils "github.com/goharbor/acceleration-service/pkg/utils" + "github.com/opencontainers/go-digest" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" + + "github.com/containerd/containerd/content" + containerdErrdefs "github.com/containerd/containerd/errdefs" + "github.com/goharbor/acceleration-service/pkg/errdefs" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // Opt defines Chunkdict generate options. // Note: sources is one or more Nydus image references. type Opt struct { - WorkDir string Sources []string + Target string SourceInsecure bool + TargetInsecure bool + + BackendType string + BackendConfig string + BackendForcePush bool + + WorkDir string NydusImagePath string ExpectedArch string + + AllPlatforms bool + Platforms string } // Generator generates chunkdict by deduplicating multiple nydus images @@ -33,12 +63,16 @@ type Generator struct { sourcesParser []*parser.Parser } +type output struct { + Blobs []string +} + // New creates Generator instance. func New(opt Opt) (*Generator, error) { // TODO: support sources image resolver var sourcesParser []*parser.Parser for _, source := range opt.Sources { - sourcesRemote, err := provider.DefaultRemote(source, opt.SourceInsecure) + sourcesRemote, err := originprovider.DefaultRemote(source, opt.SourceInsecure) if err != nil { return nil, errors.Wrap(err, "Init source image parser") } @@ -59,48 +93,435 @@ func New(opt Opt) (*Generator, error) { // Generate saves multiple Nydus bootstraps into the database one by one. func (generator *Generator) Generate(ctx context.Context) error { - for index := range generator.Sources { - if err := generator.save(ctx, index); err != nil { - if utils.RetryWithHTTP(err) { + var bootstrapPaths []string + bootstrapPaths, err := generator.pull(ctx) + + if err != nil { + if utils.RetryWithHTTP(err) { + for index := range generator.Sources { generator.sourcesParser[index].Remote.MaybeWithHTTP(err) } - if err := generator.save(ctx, index); err != nil { - return err - } + } + bootstrapPaths, err = generator.pull(ctx) + if err != nil { + return err } } - return nil -} -// "save" stores information of chunk and blob of a Nydus Image in the database -func (generator *Generator) save(ctx context.Context, index int) error { - sourceParsed, err := generator.sourcesParser[index].Parse(ctx) + chunkdictBootstrapPath, outputPath, err := generator.generate(ctx, bootstrapPaths) if err != nil { - return errors.Wrap(err, "parse Nydus image") + return err } - // Create a directory to store the image bootstrap - nydusImageName := strings.Replace(generator.Sources[index], "/", ":", -1) - folderPath := filepath.Join(generator.WorkDir, nydusImageName) - if err := os.MkdirAll(folderPath, fs.ModePerm); err != nil { - return errors.Wrap(err, "creat work directory") + if err := generator.push(ctx, chunkdictBootstrapPath, outputPath); err != nil { + return err } - if err := generator.Output(ctx, sourceParsed, folderPath, index); err != nil { - return errors.Wrap(err, "output image information") + + // return os.RemoveAll(generator.WorkDir) + return nil +} + +// Pull the bootstrap of nydus image +func (generator *Generator) pull(ctx context.Context) ([]string, error) { + var bootstrapPaths []string + for index := range generator.Sources { + sourceParsed, err := generator.sourcesParser[index].Parse(ctx) + if err != nil { + return nil, errors.Wrap(err, "parse Nydus image") + } + + // Create a directory to store the image bootstrap + nydusImageName := strings.Replace(generator.Sources[index], "/", ":", -1) + bootstrapDirPath := filepath.Join(generator.WorkDir, nydusImageName) + if err := os.MkdirAll(bootstrapDirPath, fs.ModePerm); err != nil { + return nil, errors.Wrap(err, "creat work directory") + } + if err := generator.Output(ctx, sourceParsed, bootstrapDirPath, index); err != nil { + return nil, errors.Wrap(err, "output image information") + } + bootstrapPath := filepath.Join(bootstrapDirPath, "nydus_bootstrap") + bootstrapPaths = append(bootstrapPaths, bootstrapPath) } + return bootstrapPaths, nil +} - // Invoke "nydus-image save" command +func (generator *Generator) generate(_ context.Context, bootstrapSlice []string) (string, string, error) { + // Invoke "nydus-image chunkdict generate" command + currentDir, _ := os.Getwd() builder := build.NewBuilder(generator.NydusImagePath) - if err := builder.Save(build.SaveOption{ - BootstrapPath: filepath.Join(folderPath, "nydus_bootstrap"), + + chunkdictBootstrapPath := filepath.Join(generator.WorkDir, "chunkdict_bootstrap") + databaseType := "sqlite" + var databasePath string + if strings.HasPrefix(generator.WorkDir, "/") { + databasePath = databaseType + "://" + filepath.Join(generator.WorkDir, "database.db") + } else { + databasePath = databaseType + "://" + filepath.Join(currentDir, generator.WorkDir, "database.db") + } + outputPath := filepath.Join(generator.WorkDir, "nydus_bootstrap_output.json") + + if err := builder.Generate(build.GenerateOption{ + BootstrapPaths: bootstrapSlice, + ChunkdictBootstrapPath: chunkdictBootstrapPath, + DatabasePath: databasePath, + OutputPath: outputPath, }); err != nil { - return errors.Wrap(err, "invalid nydus bootstrap format") + return "", "", errors.Wrap(err, "invalid nydus bootstrap format") + } + + logrus.Infof("Successfully generate image chunk dictionary") + return chunkdictBootstrapPath, outputPath, nil +} + +func hosts(generator *Generator) remote.HostFunc { + maps := make(map[string]bool) + for _, source := range generator.Sources { + maps[source] = generator.SourceInsecure + } + + maps[generator.Target] = generator.TargetInsecure + return func(ref string) (remote.CredentialFunc, bool, error) { + return remote.NewDockerConfigCredFunc(), maps[ref], nil + } +} + +func (generator *Generator) push(ctx context.Context, chunkdictBootstrapPath string, outputPath string) error { + // Basic configuration + ctx = namespaces.WithNamespace(ctx, "nydusify") + platformMC, err := platformutil.ParsePlatforms(generator.AllPlatforms, generator.Platforms) + if err != nil { + return err + } + + pvd, err := provider.New(generator.WorkDir, hosts(generator), 200, "v1", platformMC, 0) + if err != nil { + return err + } + + var bkd backend.Backend + if generator.BackendType != "" { + bkd, err = backend.NewBackend(generator.BackendType, []byte(generator.BackendConfig), nil) + if err != nil { + return errors.Wrapf(err, "new backend") + } + } + + // Pull source image + for index := range generator.Sources { + if err := pvd.Pull(ctx, generator.Sources[index]); err != nil { + if errdefs.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + if err := pvd.Pull(ctx, generator.Sources[index]); err != nil { + return errors.Wrap(err, "try to pull image") + } + } else { + return errors.Wrap(err, "pull source image") + } + } + } + + logrus.Infof("pulled source image %s", generator.Sources[0]) + sourceImage, err := pvd.Image(ctx, generator.Sources[0]) + if err != nil { + return errors.Wrap(err, "find image from store") } + sourceDescs, err := serverutils.GetManifests(ctx, pvd.ContentStore(), *sourceImage, platformMC) + if err != nil { + return errors.Wrap(err, "get image manifests") + } + + targetDescs := make([]ocispec.Descriptor, len(sourceDescs)) + + sem := semaphore.NewWeighted(1) + eg := errgroup.Group{} + for idx := range sourceDescs { + func(idx int) { + eg.Go(func() error { + sem.Acquire(context.Background(), 1) + defer sem.Release(1) + sourceDesc := sourceDescs[idx] + targetDesc := &sourceDesc - logrus.Infof("Save chunk information from image %s", generator.sourcesParser[index].Remote.Ref) + // Get the blob from backend + descs, _targetDesc, err := pushBlobFromBackend(ctx, pvd, bkd, sourceDesc, *generator, chunkdictBootstrapPath, outputPath) + if err != nil { + return errors.Wrap(err, "get resolver") + } + if _targetDesc != nil { + targetDesc = _targetDesc + store := newStore(pvd.ContentStore(), descs) + pvd.SetContentStore(store) + } - if err := os.RemoveAll(folderPath); err != nil { - return errors.Wrap(err, "remove work directory") + targetDescs[idx] = *targetDesc + + if err := pvd.Push(ctx, *targetDesc, generator.Target); err != nil { + if errdefs.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + if err := pvd.Push(ctx, *targetDesc, generator.Target); err != nil { + return errors.Wrap(err, "try to push image manifest") + } + } else { + return errors.Wrap(err, "push target image manifest") + } + } + return nil + }) + }(idx) + } + if err := eg.Wait(); err != nil { + return errors.Wrap(err, "push image manifests") } return nil } + +func pushBlobFromBackend( + ctx context.Context, pvd *provider.Provider, bkd backend.Backend, src ocispec.Descriptor, generator Generator, bootstrapPath string, outputPath string, +) ([]ocispec.Descriptor, *ocispec.Descriptor, error) { + manifest := ocispec.Manifest{} + if _, err := serverutils.ReadJSON(ctx, pvd.ContentStore(), &manifest, src); err != nil { + return nil, nil, errors.Wrap(err, "read manifest from store") + } + fsversion := src.Annotations["containerd.io/snapshot/nydus-fs-version"] + // Read the Nydusify output JSON to get the list of blobs + var out output + bytes, err := os.ReadFile(outputPath) + if err != nil { + return nil, nil, errors.Wrap(err, "read output file") + } + if err := json.Unmarshal(bytes, &out); err != nil { + return nil, nil, errors.Wrap(err, "unmarshal output json") + } + + blobIDs := []string{} + blobIDMap := map[string]bool{} + for _, blobID := range out.Blobs { + if blobIDMap[blobID] { + continue + } + blobIDs = append(blobIDs, blobID) + blobIDMap[blobID] = true + } + blobDescs := make([]ocispec.Descriptor, len(blobIDs)) + + eg, ctx := errgroup.WithContext(ctx) + sem := semaphore.NewWeighted(int64(provider.LayerConcurrentLimit)) + for idx := range blobIDs { + func(idx int) { + eg.Go(func() error { + sem.Acquire(context.Background(), 1) + defer sem.Release(1) + + blobID := blobIDs[idx] + blobDigest := digest.Digest("sha256:" + blobID) + + var blobSize int64 + var rc io.ReadCloser + + if bkd != nil { + rc, err = bkd.Reader(blobID) + if err != nil { + return errors.Wrap(err, "get blob reader") + } + blobSize, err = bkd.Size(blobID) + if err != nil { + return errors.Wrap(err, "get blob size") + } + } else { + imageDesc, err := generator.sourcesParser[0].Remote.Resolve(ctx) + if err != nil { + if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") { + logrus.Warningln("try to enable \"--source-insecure\" / \"--target-insecure\" option") + } + return errors.Wrap(err, "resolve image") + } + rc, err = generator.sourcesParser[0].Remote.Pull(ctx, *imageDesc, true) + if err != nil { + return errors.Wrap(err, "get blob reader") + } + blobInfo, err := pvd.ContentStore().Info(ctx, blobDigest) + if err != nil { + return errors.Wrap(err, "get info from content store") + } + blobSize = blobInfo.Size + } + defer rc.Close() + + blobSizeStr := humanize.Bytes(uint64(blobSize)) + logrus.WithField("digest", blobDigest).WithField("size", blobSizeStr).Infof("pushing blob from backend") + + blobDescs[idx] = ocispec.Descriptor{ + Digest: blobDigest, + Size: blobSize, + MediaType: converter.MediaTypeNydusBlob, + Annotations: map[string]string{ + converter.LayerAnnotationNydusBlob: "true", + }, + } + writer, err := getPushWriter(ctx, pvd, blobDescs[idx], generator.Opt) + if err != nil { + if errdefs.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + writer, err = getPushWriter(ctx, pvd, blobDescs[idx], generator.Opt) + } + if err != nil { + return errors.Wrap(err, "get push writer") + } + } + if writer != nil { + defer writer.Close() + return content.Copy(ctx, writer, rc, blobSize, blobDigest) + } + + logrus.WithField("digest", blobDigest).WithField("size", blobSizeStr).Infof("pushed blob from backend") + + return nil + + }) + }(idx) + } + + if err := eg.Wait(); err != nil { + return nil, nil, errors.Wrap(err, "push blobs") + } + + // Update manifest blob layers + manifest.Layers = nil + manifest.Layers = append(blobDescs, manifest.Layers...) + + // Update bootstrap + cw, err := content.OpenWriter(ctx, pvd.ContentStore(), content.WithRef("merge-bootstrap")) + if err != nil { + return nil, nil, errors.Wrap(err, "open content store writer") + } + defer cw.Close() + + bootstrapPathTar := "image/image.boot" + rc, err := utils.PackTargz(bootstrapPath, bootstrapPathTar, false) + if err != nil { + return nil, nil, errors.Wrap(err, "get bootstrap reader") + } + defer rc.Close() + + gw := gzip.NewWriter(cw) + uncompressedDgst := digest.SHA256.Digester() + compressed := io.MultiWriter(gw, uncompressedDgst.Hash()) + + buffer := make([]byte, 32*1024) + if _, err := io.CopyBuffer(compressed, rc, buffer); err != nil { + return nil, nil, errors.Wrapf(err, "copy bootstrap targz into content store") + } + if err := gw.Close(); err != nil { + return nil, nil, errors.Wrap(err, "close gzip writer") + } + + compressedDgst := cw.Digest() + if err := cw.Commit(ctx, 0, compressedDgst, content.WithLabels(map[string]string{ + "containerd.io/uncompressed": uncompressedDgst.Digest().String(), + })); err != nil { + if !containerdErrdefs.IsAlreadyExists(err) { + return nil, nil, errors.Wrap(err, "commit to content store") + } + } + if err := cw.Close(); err != nil { + return nil, nil, errors.Wrap(err, "close content store writer") + } + + bootstrapInfo, err := pvd.ContentStore().Info(ctx, compressedDgst) + if err != nil { + return nil, nil, errors.Wrap(err, "get info from content store") + } + bootstrapSize := bootstrapInfo.Size + + bootstrapDesc := ocispec.Descriptor{ + Digest: compressedDgst, + Size: bootstrapSize, + MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + Annotations: map[string]string{ + "containerd.io/snapshot/nydus-bootstrap": "true", + "containerd.io/snapshot/nydus-fs-version": fsversion, + }, + } + manifest.Layers = append(manifest.Layers, bootstrapDesc) + + // Update image config + blobDigests := []digest.Digest{} + for idx := range blobDescs { + blobDigests = append(blobDigests, blobDescs[idx].Digest) + } + + config := ocispec.Image{} + if _, err := serverutils.ReadJSON(ctx, pvd.ContentStore(), &config, manifest.Config); err != nil { + return nil, nil, errors.Wrap(err, "read config json") + } + config.RootFS.DiffIDs = nil + config.RootFS.DiffIDs = append(blobDigests, config.RootFS.DiffIDs...) + config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, digest.Digest(uncompressedDgst.Digest().String())) + configDesc, err := serverutils.WriteJSON(ctx, pvd.ContentStore(), config, manifest.Config, generator.Target, nil) + if err != nil { + return nil, nil, errors.Wrap(err, "write config json") + } + manifest.Config = *configDesc + target, err := serverutils.WriteJSON(ctx, pvd.ContentStore(), &manifest, src, generator.Target, nil) + if err != nil { + return nil, nil, errors.Wrap(err, "write manifest json") + } + + return blobDescs, target, nil +} + +func getPushWriter(ctx context.Context, pvd *provider.Provider, desc ocispec.Descriptor, opt Opt) (content.Writer, error) { + resolver, err := pvd.Resolver(opt.Target) + if err != nil { + return nil, errors.Wrap(err, "get resolver") + } + + ref := opt.Target + if !strings.Contains(ref, "@") { + ref = ref + "@" + desc.Digest.String() + } + pusher, err := resolver.Pusher(ctx, ref) + if err != nil { + return nil, errors.Wrap(err, "create pusher") + } + writer, err := pusher.Push(ctx, desc) + if err != nil { + if containerdErrdefs.IsAlreadyExists(err) { + return nil, nil + } + return nil, err + } + + return writer, nil +} + +type store struct { + content.Store + remotes []ocispec.Descriptor +} + +func newStore(base content.Store, remotes []ocispec.Descriptor) *store { + return &store{ + Store: base, + remotes: remotes, + } +} + +func (s *store) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { + info, err := s.Store.Info(ctx, dgst) + if err != nil { + if !containerdErrdefs.IsNotFound(err) { + return content.Info{}, err + } + for _, desc := range s.remotes { + if desc.Digest == dgst { + return content.Info{ + Digest: desc.Digest, + Size: desc.Size, + }, nil + } + } + return content.Info{}, err + } + return info, nil +} diff --git a/smoke/tests/image_test.go b/smoke/tests/image_test.go index d8f06bc41ef..e4f27dc6765 100644 --- a/smoke/tests/image_test.go +++ b/smoke/tests/image_test.go @@ -7,6 +7,7 @@ package tests import ( "fmt" "path/filepath" + "strings" "testing" "github.com/dragonflyoss/nydus/smoke/tests/tool" @@ -127,7 +128,8 @@ func (i *ImageTestSuite) TestConvertAndCopyImage(t *testing.T, ctx tool.Context, // Copy image targetCopied := fmt.Sprintf("%s_copied", target) copyCmd := fmt.Sprintf( - "%s %s copy --source %s --target %s --nydus-image %s --work-dir %s --push-chunk-size 1MB", + // "%s %s copy --source %s --target %s --nydus-image %s --work-dir %s --push-chunk-size 1MB", + "%s %s copy --source %s --target %s --nydus-image %s --work-dir %s", ctx.Binary.Nydusify, logLevel, target, targetCopied, ctx.Binary.Builder, ctx.Env.WorkDir, ) tool.RunWithoutOutput(t, copyCmd) @@ -140,6 +142,86 @@ func (i *ImageTestSuite) TestConvertAndCopyImage(t *testing.T, ctx tool.Context, tool.RunWithoutOutput(t, checkCmd) } +func (i *ImageTestSuite) TestGenerateChunkdicts() test.Generator { + images := []string{"redis:7.0.1", "redis:7.0.2", "redis:7.0.3"} + var sources []string + for _, image := range images { + image = i.prepareImage(i.T, image) + sources = append(sources, image) + } + scenarios := tool.DescartesIterator{} + scenarios. + Dimension(paramFSVersion, []interface{}{"5", "6"}) + return func() (name string, testCase test.Case) { + if !scenarios.HasNext() { + return + } + scenario := scenarios.Next() + ctx := tool.DefaultContext(i.T) + ctx.Build.FSVersion = scenario.GetString(paramFSVersion) + return "chunkdict:" + scenario.Str(), func(t *testing.T) { + i.TestChundict(t, *ctx, sources) + } + } +} + +func (i *ImageTestSuite) TestChundict(t *testing.T, ctx tool.Context, images []string) { + trainImage := images[:len(images)-1] + testImage := images[len(images)-1] + + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + + // Prepare options. + enableOCIRef := "" + enableEncrypt := "" + fsVersion := fmt.Sprintf("--fs-version %s", ctx.Build.FSVersion) + logLevel := "--log-level warn" + compressor := "--compressor lz4_block" + + // Prepare nydus images. + var targets []string + for _, image := range trainImage { + target := fmt.Sprintf("%s-nydus-%s", image, uuid.NewString()) + targets = append(targets, target) + + fmt.Println("target:", target) + convertCmd := fmt.Sprintf( + "%s %s convert --source %s --target %s %s %s %s %s --nydus-image %s --work-dir %s %s", + ctx.Binary.Nydusify, logLevel, image, target, fsVersion, enableOCIRef, "", enableEncrypt, ctx.Binary.Builder, ctx.Env.WorkDir, compressor, + ) + tool.RunWithoutOutput(t, convertCmd) + } + targetsStr := strings.Join(targets, ",") + + // Generate chunkdict. + chunkdict := fmt.Sprintf("%s/redis:nydus-chunkdict-%s", strings.SplitN(testImage, "/", 2)[0], uuid.NewString()) + fmt.Println("chunkdict:", chunkdict) + generateCmd := fmt.Sprintf( + "%s %s chunkdict generate --sources %s --target %s --source-insecure --target-insecure --nydus-image %s --work-dir %s", + ctx.Binary.Nydusify, logLevel, targetsStr, chunkdict, ctx.Binary.Builder, filepath.Join(ctx.Env.WorkDir, "generate"), + ) + tool.RunWithoutOutput(t, generateCmd) + fmt.Println("generateCmd:", generateCmd) + + // Covert test image by chunkdict. + target := fmt.Sprintf("%s-nydus-%s", testImage, uuid.NewString()) + convertCmd := fmt.Sprintf( + "%s %s convert --source %s --target %s %s %s %s %s --nydus-image %s --work-dir %s %s", + ctx.Binary.Nydusify, logLevel, testImage, target, fsVersion, enableOCIRef, "", enableEncrypt, ctx.Binary.Builder, ctx.Env.WorkDir, compressor, + ) + tool.RunWithoutOutput(t, convertCmd) + + // Check nydus image covert by chunkdict. + checkCmd := fmt.Sprintf( + "%s %s check --target %s --nydus-image %s --nydusd %s --work-dir %s", + ctx.Binary.Nydusify, logLevel, target, ctx.Binary.Builder, ctx.Binary.Nydusd, filepath.Join(ctx.Env.WorkDir, "check"), + ) + tool.RunWithoutOutput(t, checkCmd) + fmt.Println("checkCmd:", checkCmd) + ctx.Destroy(t) +} + func (i *ImageTestSuite) prepareImage(t *testing.T, image string) string { if i.preparedImages == nil { i.preparedImages = make(map[string]string)