From c24060208af2dae3d84cf5ce4397abd77c79695b Mon Sep 17 00:00:00 2001 From: Zhao Yuan <1627990440@qq.com> Date: Tue, 21 Nov 2023 12:27:52 +0000 Subject: [PATCH] Modify command "nydus-image compact --bootstrap --compact-config --backend-config " Signed-off-by: Zhao Yuan <1627990440@qq.com> --- builder/src/compact.rs | 31 ++++++++++++++++++++++++++----- src/bin/nydus-image/main.rs | 6 +++--- storage/src/meta/chunk_info_v1.rs | 5 ++++- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/builder/src/compact.rs b/builder/src/compact.rs index 97cd6594b17..7e09204e444 100644 --- a/builder/src/compact.rs +++ b/builder/src/compact.rs @@ -79,7 +79,7 @@ impl ChunkKey { match c { ChunkWrapper::V5(_) => Self::Digest(*c.id()), ChunkWrapper::V6(_) => Self::Offset(c.blob_index(), c.compressed_offset()), - ChunkWrapper::Ref(_) => unimplemented!("unsupport ChunkWrapper::Ref(c)"), + ChunkWrapper::Ref(_) => Self::Digest(*c.id()), } } } @@ -332,6 +332,20 @@ impl BlobCompactor { cs.add_chunk(&chunk.inner); } } + } else if let Some(c) = all_chunks.get_chunk(&chunk_key) { + let mut chunk_inner = chunk.inner.deref().clone(); + apply_chunk_change(c, &mut chunk_inner)?; + chunk.inner = Arc::new(chunk_inner); + } else { + all_chunks.add_chunk(&chunk.inner); + // add to per blob ChunkSet + let blob_index = chunk.inner.blob_index() as usize; + if self.states[blob_index].is_invalid() { + self.states[blob_index] = State::Original(ChunkSet::new()); + } + if let State::Original(cs) = &mut self.states[blob_index] { + cs.add_chunk(&chunk.inner); + } } // construct blobs/chunk --> nodes index map @@ -404,7 +418,7 @@ impl BlobCompactor { } fn prepare_to_rebuild(&mut self, idx: usize) -> Result<()> { - if !self.states[idx].is_rebuild() { + if self.states[idx].is_rebuild() { return Ok(()); } @@ -443,8 +457,8 @@ impl BlobCompactor { }; info!( - "compactor: original blob size {}, used data ratio {}%", - blob_info.blob_id, used_ratio + "compactor: original blob id is {}, blob size is {}, used data ratio {}%", + blob_info.blob_id, blob_info.compressed_blob_size, used_ratio ); if used_ratio < ratio { self.prepare_to_rebuild(idx)?; @@ -531,7 +545,7 @@ impl BlobCompactor { let ori_blob_ids = self.original_blob_ids(); ensure!(self.states.len() == self.ori_blob_mgr.len()); - for idx in 0..self.states.len() { + for idx in (0..self.states.len()).rev() { match &self.states[idx] { State::Original(_) | State::ChunkDict => { info!("compactor: keep original data blob {}", ori_blob_ids[idx]); @@ -613,6 +627,13 @@ impl BlobCompactor { Features::new(), false, ); + if rs.meta.is_v5() { + build_ctx.fs_version = RafsVersion::V5; + info!("Version:V5"); + } else { + build_ctx.fs_version = RafsVersion::V6; + info!("Version:V6"); + } let mut bootstrap_mgr = BootstrapManager::new(Some(ArtifactStorage::SingleFile(d_bootstrap)), None); let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 29dd8a0e072..62265b1cae8 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -654,7 +654,7 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .required(true), ) .arg( - Arg::new("config") + Arg::new("comfig-config") .long("config") .short('C') .help("config to compactor") @@ -1293,7 +1293,7 @@ impl Command { Self::get_configuration(matches).context("failed to get configuration information")?; config .internal - .set_blob_accessible(matches.get_one::("config").is_some()); + .set_blob_accessible(matches.get_one::("bootstrap").is_some()); let bootstrap_path = PathBuf::from(Self::get_bootstrap(matches)?); let dst_bootstrap = match matches.get_one::("output-bootstrap") { None => bootstrap_path.with_extension("bootstrap.compact"), @@ -1313,7 +1313,7 @@ impl Command { let backend = Self::get_backend(matches, "compactor")?; - let config_file_path = matches.get_one::("config").unwrap(); + let config_file_path = matches.get_one::("compact-config").unwrap(); let file = File::open(config_file_path) .with_context(|| format!("failed to open config file {}", config_file_path))?; let config = serde_json::from_reader(file) diff --git a/storage/src/meta/chunk_info_v1.rs b/storage/src/meta/chunk_info_v1.rs index e41bd9f0aa4..448f2fdbbde 100644 --- a/storage/src/meta/chunk_info_v1.rs +++ b/storage/src/meta/chunk_info_v1.rs @@ -58,7 +58,10 @@ impl BlobMetaChunkInfo for BlobChunkInfoV1Ondisk { } fn set_uncompressed_offset(&mut self, offset: u64) { - assert_eq!(offset & !BLOB_CC_V1_CHUNK_UNCOMP_OFFSET_MASK, 0); + assert_eq!( + ((offset & !BLOB_CC_V1_CHUNK_UNCOMP_OFFSET_MASK) >> 32) as u32, + 0 + ); self.uncomp_info &= u64::to_le(!BLOB_CC_V1_CHUNK_UNCOMP_OFFSET_MASK); self.uncomp_info |= u64::to_le(offset & BLOB_CC_V1_CHUNK_UNCOMP_OFFSET_MASK); }