From 5967ee16396a2aef2e7e84113e7460037b0b5e08 Mon Sep 17 00:00:00 2001 From: "maxing.lan" Date: Mon, 20 Jan 2025 16:10:16 +0800 Subject: [PATCH] feat: nydusify: add backend configure support of optimize subcommand When localfs-backend is true (default is false), still pull whole image to build optimized image; otherwise, use registry backend to fetch needed chunk during building process. Signed-off-by: maxing.lan --- contrib/nydusify/cmd/nydusify.go | 50 ++++++++++++++++++++- contrib/nydusify/pkg/optimizer/builder.go | 19 ++++++-- contrib/nydusify/pkg/optimizer/optimizer.go | 14 +++++- docs/nydusify.md | 18 ++++++++ 4 files changed, 94 insertions(+), 7 deletions(-) diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 109dd65c4ff..88067debab6 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -200,7 +200,8 @@ func main() { Required: false, Value: false, Usage: "Enable debug log level, overwrites the 'log-level' option", - EnvVars: []string{"DEBUG_LOG_LEVEL"}}, + EnvVars: []string{"DEBUG_LOG_LEVEL"}, + }, &cli.StringFlag{ Name: "log-level", Aliases: []string{"l"}, @@ -1223,10 +1224,54 @@ func main() { Value: "0MB", Usage: "Chunk size for pushing a blob layer in chunked", }, + + &cli.StringFlag{ + Name: "source-backend-type", + Value: "", + Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3', 'localfs'", + EnvVars: []string{"BACKEND_TYPE"}, + }, + &cli.StringFlag{ + Name: "source-backend-config", + Value: "", + Usage: "Json string for storage backend configuration", + EnvVars: []string{"BACKEND_CONFIG"}, + }, + &cli.PathFlag{ + Name: "source-backend-config-file", + Value: "", + TakesFile: true, + Usage: "Json configuration file for storage backend", + EnvVars: []string{"BACKEND_CONFIG_FILE"}, + }, }, Action: func(c *cli.Context) error { setupLogLevel(c) + backendType, backendConfig, err := getBackendConfig(c, "source-", false) + if err != nil { + return err + } else if backendConfig == "" { + + backendType = "registry" + parsed, err := reference.ParseNormalizedNamed(c.String("target")) + if err != nil { + return err + } + + backendConfigStruct, err := utils.NewRegistryBackendConfig(parsed, c.Bool("target-insecure")) + if err != nil { + return errors.Wrap(err, "parse registry backend configuration") + } + + bytes, err := json.Marshal(backendConfigStruct) + if err != nil { + return errors.Wrap(err, "marshal registry backend configuration") + } + backendConfig = string(bytes) + + } + pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size")) if err != nil { return errors.Wrap(err, "invalid --push-chunk-size option") @@ -1248,6 +1293,9 @@ func main() { PushChunkSize: int64(pushChunkSize), PrefetchFilesPath: c.String("prefetch-files"), + + BackendType: backendType, + BackendConfig: backendConfig, } return optimizer.Optimize(context.Background(), opt) diff --git a/contrib/nydusify/pkg/optimizer/builder.go b/contrib/nydusify/pkg/optimizer/builder.go index 66f26ac23d5..84b31b6bbcc 100644 --- a/contrib/nydusify/pkg/optimizer/builder.go +++ b/contrib/nydusify/pkg/optimizer/builder.go @@ -19,9 +19,13 @@ func isSignalKilled(err error) bool { } type BuildOption struct { - BuilderPath string - PrefetchFilesPath string - BootstrapPath string + BuilderPath string + PrefetchFilesPath string + BootstrapPath string + BackendType string + BackendConfig string + // `BlobDir` is used to store optimized blob, + // Beside, `BlobDir` is also used to store the original blobs when backend is localfs BlobDir string OutputBootstrapPath string OutputJSONPath string @@ -42,7 +46,7 @@ func Build(option BuildOption) (string, error) { option.PrefetchFilesPath, "--bootstrap", option.BootstrapPath, - "--blob-dir", + "--output-blob-dir", option.BlobDir, "--output-bootstrap", option.OutputBootstrapPath, @@ -50,6 +54,13 @@ func Build(option BuildOption) (string, error) { outputJSONPath, } + if option.BackendType == "localfs" { + args = append(args, "--blob-dir", option.BlobDir) + } else { + args = append(args, "--backend-type", option.BackendType) + args = append(args, "--backend-config", option.BackendConfig) + } + ctx := context.Background() var cancel context.CancelFunc if option.Timeout != nil { diff --git a/contrib/nydusify/pkg/optimizer/optimizer.go b/contrib/nydusify/pkg/optimizer/optimizer.go index 84d35614e5f..207e9cc8292 100644 --- a/contrib/nydusify/pkg/optimizer/optimizer.go +++ b/contrib/nydusify/pkg/optimizer/optimizer.go @@ -61,6 +61,9 @@ type Opt struct { Platforms string PushChunkSize int64 + + BackendType string + BackendConfig string } // the information generated during building @@ -269,8 +272,10 @@ func Optimize(ctx context.Context, opt Opt) error { } defer os.RemoveAll(buildDir) - if err := fetchBlobs(ctx, opt, buildDir); err != nil { - return errors.Wrap(err, "prepare nydus blobs") + if opt.BackendType == "localfs" { + if err := fetchBlobs(ctx, opt, buildDir); err != nil { + return errors.Wrap(err, "prepare nydus blobs") + } } originalBootstrap := filepath.Join(buildDir, "nydus_bootstrap") @@ -289,12 +294,17 @@ func Optimize(ctx context.Context, opt Opt) error { compressAlgo := bootstrapDesc.Digest.Algorithm().String() blobDir := filepath.Join(buildDir + "/content/blobs/" + compressAlgo) + if err := os.MkdirAll(blobDir, 0755); err != nil { + return errors.Wrap(err, "create blob directory") + } outPutJSONPath := filepath.Join(buildDir, "output.json") newBootstrapPath := filepath.Join(buildDir, "optimized_bootstrap") builderOpt := BuildOption{ BuilderPath: opt.NydusImagePath, PrefetchFilesPath: opt.PrefetchFilesPath, BootstrapPath: originalBootstrap, + BackendType: opt.BackendType, + BackendConfig: opt.BackendConfig, BlobDir: blobDir, OutputBootstrapPath: newBootstrapPath, OutputJSONPath: outPutJSONPath, diff --git a/docs/nydusify.md b/docs/nydusify.md index 32e2239871b..11d6201b7ca 100644 --- a/docs/nydusify.md +++ b/docs/nydusify.md @@ -262,6 +262,24 @@ nerdctl --snapshotter nydus run \ The original container ID need to be a full container ID rather than an abbreviation. +## Optimize nydus image from prefetch files + +The nydusify optimize command can optimize a nydus image from prefetch files, prefetch files are file access patterns during container startup. This will generate a new bootstrap and a new blob wich contains all data indicated by prefetch files. + +The content of prefetch files likes this: +``` +/path/to/file1 start_offset1-end_offset1, start_offset2-end_offset2, ... +/path/to/file2 start_offset1-end_offset1, start_offset2-end_offset2, ... +``` + +``` shell +nydusify optimize \ + --nydus-image /path/to/nydus-image \ + --source myregistry/repo:tag-nydus \ + --target myregistry/repo:tag-nydus-optimized \ + --prefetch-files /path/to/prefetch-files \ +``` + ## More Nydusify Options See `nydusify convert/check/mount --help`