Add push chunkdict to registry and add smoke test(support v5 and v6)

Signed-off-by: Zhao Yuan <[email protected]>

builder/src/lib.rs

@@ -23,6 +23,8 @@ use sha2::Digest;
 
 use self::core::node::{Node, NodeInfo};
 
+pub use self::chunkdict_generator::ChunkdictChunkInfo;
+pub use self::chunkdict_generator::Generator;
 pub use self::compact::BlobCompactor;
 pub use self::core::bootstrap::Bootstrap;
 pub use self::core::chunk_dict::{parse_chunk_dict_arg, ChunkDict, HashChunkDict};
@@ -36,16 +38,14 @@ pub use self::core::overlay::{Overlay, WhiteoutSpec};
 pub use self::core::prefetch::{Prefetch, PrefetchPolicy};
 pub use self::core::tree::{MetadataTreeBuilder, Tree, TreeNode};
 pub use self::directory::DirectoryBuilder;
-pub use self::chunkdict_generator::ChunkdictChunkInfo;
-pub use self::chunkdict_generator::Generator;
 pub use self::merge::Merger;
 pub use self::stargz::StargzBuilder;
 pub use self::tarball::TarballBuilder;
 
+mod chunkdict_generator;
 mod compact;
 mod core;
 mod directory;
-mod chunkdict_generator;
 mod merge;
 mod stargz;
 mod tarball;

contrib/nydusify/cmd/nydusify.go

@@ -656,12 +656,50 @@ func main() {
 							Usage:    "One or more Nydus image reference(Multiple images should be split by commas)",
 							EnvVars:  []string{"SOURCES"},
 						},
+						&cli.StringFlag{
+							Name:     "target",
+							Required: false,
+							Usage:    "Target chunkdict image (Nydus) reference",
+							EnvVars:  []string{"TARGET"},
+						},
 						&cli.BoolFlag{
 							Name:     "source-insecure",
 							Required: false,
 							Usage:    "Skip verifying server certs for HTTPS source registry",
 							EnvVars:  []string{"SOURCE_INSECURE"},
 						},
+						&cli.BoolFlag{
+							Name:     "target-insecure",
+							Required: false,
+							Usage:    "Skip verifying server certs for HTTPS target registry",
+							EnvVars:  []string{"TARGET_INSECURE"},
+						},
+
+						&cli.StringFlag{
+							Name:    "backend-type",
+							Value:   "",
+							Usage:   "Type of storage backend, possible values: 'oss', 's3'",
+							EnvVars: []string{"BACKEND_TYPE"},
+						},
+						&cli.StringFlag{
+							Name:    "backend-config",
+							Value:   "",
+							Usage:   "Json configuration string for storage backend",
+							EnvVars: []string{"BACKEND_CONFIG"},
+						},
+						&cli.PathFlag{
+							Name:      "backend-config-file",
+							Value:     "",
+							TakesFile: true,
+							Usage:     "Json configuration file for storage backend",
+							EnvVars:   []string{"BACKEND_CONFIG_FILE"},
+						},
+						&cli.StringFlag{
+							Name:  "push-chunk-size",
+							Value: "0MB",
+							Usage: "Chunk size for pushing a blob layer in chunked",
+						},
+
 						&cli.StringFlag{
 							Name:    "work-dir",
 							Value:   "./output",
@@ -674,6 +712,12 @@ func main() {
 							Usage:   "Path to the nydus-image binary, default to search in PATH",
 							EnvVars: []string{"NYDUS_IMAGE"},
 						},
+
+						&cli.BoolFlag{
+							Name:  "all-platforms",
+							Value: false,
+							Usage: "Generate chunkdict image for all platforms, conflicts with --platform",
+						},
 						&cli.StringFlag{
 							Name:  "platform",
 							Value: "linux/" + runtime.GOARCH,
@@ -683,17 +727,40 @@ func main() {
 					Action: func(c *cli.Context) error {
 						setupLogLevel(c)
 
+						backendType, backendConfig, err := getBackendConfig(c, "", false)
+						if err != nil {
+							return err
+						}
+						pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size"))
+						if err != nil {
+							return errors.Wrap(err, "invalid --push-chunk-size option")
+						}
+						if pushChunkSize > 0 {
+							logrus.Infof("will copy layer with chunk size %s", c.String("push-chunk-size"))
+						}
+
 						_, arch, err := provider.ExtractOsArch(c.String("platform"))
 						if err != nil {
 							return err
 						}
 
 						generator, err := generator.New(generator.Opt{
-							WorkDir:        c.String("work-dir"),
 							Sources:        c.StringSlice("sources"),
+							Target:         c.String("target"),
 							SourceInsecure: c.Bool("source-insecure"),
+							TargetInsecure: c.Bool("target-insecure"),
+
+							BackendType:      backendType,
+							BackendConfig:    backendConfig,
+							BackendForcePush: c.Bool("backend-force-push"),
+
+							WorkDir:        c.String("work-dir"),
 							NydusImagePath: c.String("nydus-image"),
 							ExpectedArch:   arch,
+							AllPlatforms:   c.Bool("all-platforms"),
+							Platforms:      c.String("platform"),
+
+							PushChunkSize: int64(pushChunkSize),
 						})
 						if err != nil {
 							return err