From c0facd70b5b5cbe75938da47e01d3656236ccff9 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 18:38:46 +0800 Subject: [PATCH 01/14] build: implement release process in odev Signed-off-by: tison --- dev/Cargo.lock | 7 ++++ dev/Cargo.toml | 1 + dev/src/main.rs | 1 + dev/src/release/mod.rs | 1 + dev/src/release/package.rs | 76 ++++++++++++++++++++++++++++++++++++++ justfile | 2 +- 6 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 dev/src/release/mod.rs create mode 100644 dev/src/release/package.rs diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 71a8d12cb12d..46a814bb86c9 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -223,6 +223,7 @@ dependencies = [ "markdown", "minijinja", "pretty_assertions", + "semver", "serde", "syn", ] @@ -299,6 +300,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "semver" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" + [[package]] name = "serde" version = "1.0.216" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index a1b478b37cd6..78c4ca7e0a2d 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -35,6 +35,7 @@ logforth = { version = "0.21.0", default-features = false } heck = { version = "0.5.0" } markdown = { version = "1.0.0-alpha.21" } minijinja = { version = "2.5.0" } +semver = { version = "1.0.25" } serde = { version = "1.0.216", features = ["derive"] } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } diff --git a/dev/src/main.rs b/dev/src/main.rs index d80d11d80b00..ddd9d1b9e338 100644 --- a/dev/src/main.rs +++ b/dev/src/main.rs @@ -16,6 +16,7 @@ // under the License. mod generate; +mod release; use anyhow::Result; use clap::{Parser, Subcommand}; diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs new file mode 100644 index 000000000000..1e117e1ea230 --- /dev/null +++ b/dev/src/release/mod.rs @@ -0,0 +1 @@ +mod package; diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs new file mode 100644 index 000000000000..03155e883adb --- /dev/null +++ b/dev/src/release/package.rs @@ -0,0 +1,76 @@ +use crate::workspace_dir; +use semver::Version; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub struct Package { + name: String, + path: PathBuf, + version: Version, + dependencies: Vec, +} + +fn make_package(path: &str, version: &str, dependencies: Vec) -> Package { + let name = path.to_string(); + let path = workspace_dir().join(path).canonicalize().unwrap(); + let version = Version::parse(version).unwrap(); + Package { + name, + path, + version, + dependencies, + } +} + +/// List all packages that are ready for release. +pub fn all_packages() -> Vec { + let core = make_package("core", "0.51.0", vec![]); + + // Integrations + let cloud_filter = make_package("integrations/cloud_filter", "0.0.4", vec![core.clone()]); + let compact = make_package("integrations/compact", "1.0.2", vec![core.clone()]); + let dav_server = make_package("integrations/dav-server", "0.3.0", vec![core.clone()]); + let fuse3 = make_package("integrations/fuse3", "0.0.11", vec![core.clone()]); + let object_store = make_package("integrations/object_store", "0.49.0", vec![core.clone()]); + let parquet = make_package("integrations/parquet", "0.3.0", vec![core.clone()]); + let unftp_sbe = make_package("integrations/unftp-sbe", "0.0.11", vec![core.clone()]); + + // Binaries + let oay = make_package("bin/oay", "0.41.15", vec![core.clone(), dav_server.clone()]); + let ofs = make_package( + "bin/ofs", + "0.0.16", + vec![core.clone(), fuse3.clone(), cloud_filter.clone()], + ); + let oli = make_package("bin/oli", "0.41.15", vec![core.clone()]); + + // Bindings + let c = make_package("bindings/c", "0.45.3", vec![core.clone()]); + let cpp = make_package("bindings/cpp", "0.45.15", vec![core.clone()]); + let haskell = make_package("bindings/haskell", "0.44.15", vec![core.clone()]); + let java = make_package("bindings/java", "0.47.7", vec![core.clone()]); + let lua = make_package("bindings/lua", "0.1.13", vec![core.clone()]); + let nodejs = make_package("bindings/nodejs", "0.47.9", vec![core.clone()]); + let python = make_package("bindings/python", "0.45.14", vec![core.clone()]); + + vec![ + core, + cloud_filter, + compact, + dav_server, + fuse3, + object_store, + parquet, + unftp_sbe, + oay, + ofs, + oli, + c, + cpp, + haskell, + java, + lua, + nodejs, + python, + ] +} diff --git a/justfile b/justfile index e1f50c6bfcb9..829185dc2064 100644 --- a/justfile +++ b/justfile @@ -17,6 +17,6 @@ # Generate code for language # -# Available languages: python +# Available languages: python, java generate language: cargo run --quiet --manifest-path=dev/Cargo.toml -- generate -l {{language}} From 63b2ad936fa5b5fedf8832e3af64eeb80f875ce7 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 18:47:23 +0800 Subject: [PATCH 02/14] remove unused version field Signed-off-by: tison --- bindings/c/Cargo.toml | 1 - bindings/cpp/Cargo.toml | 1 - bindings/dotnet/Cargo.toml | 1 - bindings/haskell/Cargo.toml | 1 - bindings/haskell/opendal.cabal | 2 +- bindings/java/Cargo.toml | 1 - bindings/lua/Cargo.toml | 1 - bindings/lua/README.md | 11 +++++++--- bindings/lua/opendal-0.1.13-1.rockspec | 28 -------------------------- bindings/nodejs/Cargo.toml | 1 - bindings/ocaml/Cargo.toml | 1 - bindings/php/Cargo.toml | 1 - bindings/ruby/Cargo.toml | 1 - 13 files changed, 9 insertions(+), 42 deletions(-) delete mode 100644 bindings/lua/opendal-0.1.13-1.rockspec diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml index 723d270938ee..c3c669dd4bd7 100644 --- a/bindings/c/Cargo.toml +++ b/bindings/c/Cargo.toml @@ -25,7 +25,6 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.75" -version = "0.45.3" [lib] crate-type = ["cdylib", "staticlib"] diff --git a/bindings/cpp/Cargo.toml b/bindings/cpp/Cargo.toml index 9d91c23c0965..0e0a2cd6790d 100644 --- a/bindings/cpp/Cargo.toml +++ b/bindings/cpp/Cargo.toml @@ -25,7 +25,6 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.75" -version = "0.45.15" [lib] crate-type = ["staticlib"] diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml index cf00bce4e0eb..7b048fb1cb4a 100644 --- a/bindings/dotnet/Cargo.toml +++ b/bindings/dotnet/Cargo.toml @@ -18,7 +18,6 @@ [package] name = "opendal-dotnet" publish = false -version = "0.1.13" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml index 97f2126171f7..2cd30e89ebc1 100644 --- a/bindings/haskell/Cargo.toml +++ b/bindings/haskell/Cargo.toml @@ -25,7 +25,6 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.75" -version = "0.44.15" [lib] crate-type = ["cdylib"] diff --git a/bindings/haskell/opendal.cabal b/bindings/haskell/opendal.cabal index 520e6a989e7f..c1e694750388 100644 --- a/bindings/haskell/opendal.cabal +++ b/bindings/haskell/opendal.cabal @@ -17,7 +17,7 @@ cabal-version: 3.0 -- under the License. name: opendal -version: 0.44.15.0 +version: 0.44.15 license: Apache-2.0 synopsis: Apache OpenDAL™ Haskell Binding description: diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index c6b77acf2f8a..77964b592f4f 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -25,7 +25,6 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.75" -version = "0.47.7" [lib] crate-type = ["cdylib"] diff --git a/bindings/lua/Cargo.toml b/bindings/lua/Cargo.toml index 83ea6da60d0a..56862f3503e6 100644 --- a/bindings/lua/Cargo.toml +++ b/bindings/lua/Cargo.toml @@ -18,7 +18,6 @@ [package] name = "opendal-lua" publish = false -version = "0.1.13" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/bindings/lua/README.md b/bindings/lua/README.md index 5008f1d0b244..5a4b3ce1b5b4 100644 --- a/bindings/lua/README.md +++ b/bindings/lua/README.md @@ -19,36 +19,41 @@ print("read: ", op:read("test.txt")) ``` ## Lua version + You have to enable one of the features: lua54, lua53, lua52, lua51, luajit(52) or luau in `Cargo.toml`, according to the chosen Lua version. Default Lua version is 5.2. ## Build from source -1. Build OpenDAL LUA Interface +1. Build OpenDAL Lua Interface ```bash $ cd bindings/lua $ cargo build --package opendal-lua --release ``` -2. Install opendal lua library +2. Install opendal Lua library + ```bash # copy to lua share library directory # default lua5.2 share library directory is /usr/lib/lua/5.2 $ cp ../../target/release/libopendal_lua.so /usr/lib/lua/5.2/opendal.so ``` -## Install from luarocks +## Install from LuaRocks + ```bash $ luarocks make ``` ## Usage + ```bash $ lua5.2 example/fs.lua read: hello world ``` ## Test + ```bash $ busted -o gtest test/opendal_test.lua [==========] Running tests from scanned files. diff --git a/bindings/lua/opendal-0.1.13-1.rockspec b/bindings/lua/opendal-0.1.13-1.rockspec deleted file mode 100644 index 3fc7c26b0434..000000000000 --- a/bindings/lua/opendal-0.1.13-1.rockspec +++ /dev/null @@ -1,28 +0,0 @@ -package = "opendal" -version = "0.1.13-1" - -source = { - url = "git+https://github.com/apache/opendal/", -} - -description = { - summary = "Apache OpenDAL™ LUA binding: access data freely. ", - detailed = [[ - OpenDAL is a data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way. - ]], - homepage = "https://opendal.apache.org/", - license = " Apache-2.0" -} - -dependencies = { - "lua >= 5.1", - "luarocks-build-rust-mlua = 0.2.0", -} - -build = { - type = "rust-mlua", - modules = { - ["opendal"] = "opendal_lua", - }, - target_path = "target", -} diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index d4dcc4cafe37..f38dff010218 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -25,7 +25,6 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.75" -version = "0.47.9" [features] default = [ diff --git a/bindings/ocaml/Cargo.toml b/bindings/ocaml/Cargo.toml index b37ccc693c7e..abfe97b43ab3 100644 --- a/bindings/ocaml/Cargo.toml +++ b/bindings/ocaml/Cargo.toml @@ -18,7 +18,6 @@ [package] name = "opendal-ocaml" publish = false -version = "0.0.0" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/bindings/php/Cargo.toml b/bindings/php/Cargo.toml index 13661f8fc81c..ef085c62ddb2 100644 --- a/bindings/php/Cargo.toml +++ b/bindings/php/Cargo.toml @@ -18,7 +18,6 @@ [package] name = "opendal-php" publish = false -version = "0.1.12" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml index 1962365a58df..9da2f46d0a61 100644 --- a/bindings/ruby/Cargo.toml +++ b/bindings/ruby/Cargo.toml @@ -18,7 +18,6 @@ [package] name = "opendal-ruby" publish = false -version = "0.1.13" authors = ["Apache OpenDAL "] edition = "2021" From bc9300e2a9b425bedbbb05b9c5de57b9814387cc Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 19:06:02 +0800 Subject: [PATCH 03/14] impl update_package_version Signed-off-by: tison --- dev/Cargo.lock | 77 ++++++++++++++++++++++++++++++++++++++ dev/Cargo.toml | 2 + dev/src/generate/parser.rs | 5 +-- dev/src/release/package.rs | 52 ++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 4 deletions(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 46a814bb86c9..5096171c0c49 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -128,12 +128,34 @@ dependencies = [ "regex", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -226,6 +248,8 @@ dependencies = [ "semver", "serde", "syn", + "toml", + "toml_edit", ] [[package]] @@ -326,6 +350,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "strsim" version = "0.11.1" @@ -343,6 +376,41 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "unicode-id" version = "0.3.5" @@ -434,6 +502,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" +dependencies = [ + "memchr", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 78c4ca7e0a2d..59037ef3292b 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -38,6 +38,8 @@ minijinja = { version = "2.5.0" } semver = { version = "1.0.25" } serde = { version = "1.0.216", features = ["derive"] } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } +toml = { version = "0.8.19", features = ["preserve_order"] } +toml_edit = { version = "0.22.23", features = ["serde"] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 715e1ce7dfdb..a1e5f16d8c86 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -17,7 +17,6 @@ use anyhow::{anyhow, Context}; use anyhow::{bail, Result}; -use log::debug; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs; @@ -200,7 +199,7 @@ pub struct ServiceParser { impl ServiceParser { /// Parse the content of this service. fn parse(&self) -> Result { - debug!("service {} parse started", self.service); + log::debug!("service {} parse started", self.service); let ast = syn::parse_file(&self.content)?; @@ -223,7 +222,7 @@ impl ServiceParser { config.push(field); } - debug!("service {} parse finished", self.service); + log::debug!("service {} parse finished", self.service); Ok(Service { config }) } diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index 03155e883adb..6238678b2d00 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -1,6 +1,7 @@ use crate::workspace_dir; use semver::Version; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::str::FromStr; #[derive(Debug, Clone)] pub struct Package { @@ -74,3 +75,52 @@ pub fn all_packages() -> Vec { python, ] } + +pub fn update_package_version(package: &Package) -> bool { + match package.name.as_str() { + "core" => update_cargo_version(&package.path, &package.version), + "integrations/cloud_filter" => update_cargo_version(&package.path, &package.version), + "integrations/compact" => update_cargo_version(&package.path, &package.version), + "integrations/dav-server" => update_cargo_version(&package.path, &package.version), + "integrations/fuse3" => update_cargo_version(&package.path, &package.version), + "integrations/object_store" => update_cargo_version(&package.path, &package.version), + "integrations/parquet" => update_cargo_version(&package.path, &package.version), + "integrations/unftp-sbe" => update_cargo_version(&package.path, &package.version), + "bin/oay" => update_cargo_version(&package.path, &package.version), + "bin/ofs" => update_cargo_version(&package.path, &package.version), + "bin/oli" => update_cargo_version(&package.path, &package.version), + + "bindings/c" => false, // C bindings has no version to update + "bindings/cpp" => false, // C++ bindings has no version to update + "bindings/lua" => false, // Lua bindings has no version to update + + "bindings/python" => update_cargo_version(&package.path, &package.version), + + name => panic!("unknown package: {}", name), + } +} + +fn update_cargo_version(path: &Path, version: &Version) -> bool { + let path = path.join("Cargo.toml"); + let manifest = std::fs::read_to_string(&path).unwrap(); + let mut manifest = toml_edit::DocumentMut::from_str(manifest.as_str()).unwrap(); + + let old_version = match manifest["package"]["version"].as_str() { + Some(version) => Version::parse(version).unwrap(), + None => panic!("missing version for package: {}", path.display()), + }; + + if &old_version != version { + manifest["package"]["version"] = toml_edit::value(version.to_string()); + std::fs::write(&path, manifest.to_string()).unwrap(); + println!( + "updating version for package: {} from {} to {}", + path.display(), + old_version, + version + ); + true + } else { + false + } +} From 676c3879999b14d8b646bbeb48a6dd02b8c24b60 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 19:53:53 +0800 Subject: [PATCH 04/14] bridge packages Signed-off-by: tison --- dev/src/main.rs | 6 ++++-- dev/src/release/mod.rs | 8 ++++++++ dev/src/release/package.rs | 20 ++++++++++---------- justfile | 3 +++ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/dev/src/main.rs b/dev/src/main.rs index ddd9d1b9e338..cbf5bfe17750 100644 --- a/dev/src/main.rs +++ b/dev/src/main.rs @@ -18,7 +18,6 @@ mod generate; mod release; -use anyhow::Result; use clap::{Parser, Subcommand}; use std::path::PathBuf; @@ -46,12 +45,15 @@ enum Commands { #[arg(short, long)] language: String, }, + /// Update the version of all packages. + UpdateVersion, } -fn main() -> Result<()> { +fn main() -> anyhow::Result<()> { logforth::stderr().apply(); match Cmd::parse().command { Commands::Generate { language } => generate::run(&language), + Commands::UpdateVersion => release::update_version(), } } diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index 1e117e1ea230..a292b7af0512 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -1 +1,9 @@ mod package; + +pub fn update_version() -> anyhow::Result<()> { + let packages = package::all_packages(); + for package in packages { + package::update_package_version(&package); + } + Ok(()) +} diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index 6238678b2d00..d537a4b038f5 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -13,7 +13,7 @@ pub struct Package { fn make_package(path: &str, version: &str, dependencies: Vec) -> Package { let name = path.to_string(); - let path = workspace_dir().join(path).canonicalize().unwrap(); + let path = workspace_dir().join(path); let version = Version::parse(version).unwrap(); Package { name, @@ -25,11 +25,11 @@ fn make_package(path: &str, version: &str, dependencies: Vec) -> Packag /// List all packages that are ready for release. pub fn all_packages() -> Vec { - let core = make_package("core", "0.51.0", vec![]); + let core = make_package("core", "0.51.1", vec![]); // Integrations let cloud_filter = make_package("integrations/cloud_filter", "0.0.4", vec![core.clone()]); - let compact = make_package("integrations/compact", "1.0.2", vec![core.clone()]); + let compact = make_package("integrations/compat", "1.0.2", vec![core.clone()]); let dav_server = make_package("integrations/dav-server", "0.3.0", vec![core.clone()]); let fuse3 = make_package("integrations/fuse3", "0.0.11", vec![core.clone()]); let object_store = make_package("integrations/object_store", "0.49.0", vec![core.clone()]); @@ -48,10 +48,10 @@ pub fn all_packages() -> Vec { // Bindings let c = make_package("bindings/c", "0.45.3", vec![core.clone()]); let cpp = make_package("bindings/cpp", "0.45.15", vec![core.clone()]); - let haskell = make_package("bindings/haskell", "0.44.15", vec![core.clone()]); - let java = make_package("bindings/java", "0.47.7", vec![core.clone()]); + // let haskell = make_package("bindings/haskell", "0.44.15", vec![core.clone()]); + // let java = make_package("bindings/java", "0.47.7", vec![core.clone()]); let lua = make_package("bindings/lua", "0.1.13", vec![core.clone()]); - let nodejs = make_package("bindings/nodejs", "0.47.9", vec![core.clone()]); + // let nodejs = make_package("bindings/nodejs", "0.47.9", vec![core.clone()]); let python = make_package("bindings/python", "0.45.14", vec![core.clone()]); vec![ @@ -68,10 +68,10 @@ pub fn all_packages() -> Vec { oli, c, cpp, - haskell, - java, + // haskell, + // java, lua, - nodejs, + // nodejs, python, ] } @@ -80,7 +80,7 @@ pub fn update_package_version(package: &Package) -> bool { match package.name.as_str() { "core" => update_cargo_version(&package.path, &package.version), "integrations/cloud_filter" => update_cargo_version(&package.path, &package.version), - "integrations/compact" => update_cargo_version(&package.path, &package.version), + "integrations/compat" => update_cargo_version(&package.path, &package.version), "integrations/dav-server" => update_cargo_version(&package.path, &package.version), "integrations/fuse3" => update_cargo_version(&package.path, &package.version), "integrations/object_store" => update_cargo_version(&package.path, &package.version), diff --git a/justfile b/justfile index 829185dc2064..81024d4ff762 100644 --- a/justfile +++ b/justfile @@ -20,3 +20,6 @@ # Available languages: python, java generate language: cargo run --quiet --manifest-path=dev/Cargo.toml -- generate -l {{language}} + +update-version: + cargo run --quiet --manifest-path=dev/Cargo.toml -- update-version From cafb565e49c9eaa463bfe105152e38bfbd8ac82a Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 20:47:50 +0800 Subject: [PATCH 05/14] handle maven and node Signed-off-by: tison --- bindings/haskell/opendal.cabal | 2 +- dev/Cargo.lock | 37 ++++++++++++++-------- dev/Cargo.toml | 2 +- dev/src/release/mod.rs | 6 +++- dev/src/release/package.rs | 58 ++++++++++++++++++++++++++++++---- 5 files changed, 82 insertions(+), 23 deletions(-) diff --git a/bindings/haskell/opendal.cabal b/bindings/haskell/opendal.cabal index c1e694750388..36736b188b9d 100644 --- a/bindings/haskell/opendal.cabal +++ b/bindings/haskell/opendal.cabal @@ -17,7 +17,7 @@ cabal-version: 3.0 -- under the License. name: opendal -version: 0.44.15 +version: 0.1.0 license: Apache-2.0 synopsis: Apache OpenDAL™ Haskell Binding description: diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 5096171c0c49..3176e2db3e12 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -247,11 +247,17 @@ dependencies = [ "pretty_assertions", "semver", "serde", + "sxd-document", "syn", - "toml", "toml_edit", ] +[[package]] +name = "peresil" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f658886ed52e196e850cfbbfddab9eaa7f6d90dd0929e264c31e5cec07e09e57" + [[package]] name = "portable-atomic" version = "1.10.0" @@ -365,6 +371,16 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "sxd-document" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d82f37be9faf1b10a82c4bd492b74f698e40082f0f40de38ab275f31d42078" +dependencies = [ + "peresil", + "typed-arena", +] + [[package]] name = "syn" version = "2.0.91" @@ -376,19 +392,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "toml" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - [[package]] name = "toml_datetime" version = "0.6.8" @@ -411,6 +414,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "typed-arena" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" + [[package]] name = "unicode-id" version = "0.3.5" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 59037ef3292b..6ec1a344fdc5 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -37,8 +37,8 @@ markdown = { version = "1.0.0-alpha.21" } minijinja = { version = "2.5.0" } semver = { version = "1.0.25" } serde = { version = "1.0.216", features = ["derive"] } +sxd-document = { version = "0.3.2" } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } -toml = { version = "0.8.19", features = ["preserve_order"] } toml_edit = { version = "0.22.23", features = ["serde"] } [dev-dependencies] diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index a292b7af0512..932e4431806d 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -2,8 +2,12 @@ mod package; pub fn update_version() -> anyhow::Result<()> { let packages = package::all_packages(); + let mut updated = false; for package in packages { - package::update_package_version(&package); + updated |= package::update_package_version(&package); + } + if !updated { + println!("all package versions are up-to-date"); } Ok(()) } diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index d537a4b038f5..ada359d96cc1 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -48,10 +48,9 @@ pub fn all_packages() -> Vec { // Bindings let c = make_package("bindings/c", "0.45.3", vec![core.clone()]); let cpp = make_package("bindings/cpp", "0.45.15", vec![core.clone()]); - // let haskell = make_package("bindings/haskell", "0.44.15", vec![core.clone()]); - // let java = make_package("bindings/java", "0.47.7", vec![core.clone()]); + let java = make_package("bindings/java", "0.47.7", vec![core.clone()]); let lua = make_package("bindings/lua", "0.1.13", vec![core.clone()]); - // let nodejs = make_package("bindings/nodejs", "0.47.9", vec![core.clone()]); + let nodejs = make_package("bindings/nodejs", "0.47.9", vec![core.clone()]); let python = make_package("bindings/python", "0.45.14", vec![core.clone()]); vec![ @@ -68,10 +67,9 @@ pub fn all_packages() -> Vec { oli, c, cpp, - // haskell, - // java, + java, lua, - // nodejs, + nodejs, python, ] } @@ -95,6 +93,8 @@ pub fn update_package_version(package: &Package) -> bool { "bindings/lua" => false, // Lua bindings has no version to update "bindings/python" => update_cargo_version(&package.path, &package.version), + "bindings/java" => update_maven_version(&package.path, &package.version), + "bindings/nodejs" => update_nodejs_version(&package.path, &package.version), name => panic!("unknown package: {}", name), } @@ -124,3 +124,49 @@ fn update_cargo_version(path: &Path, version: &Version) -> bool { false } } + +fn update_maven_version(path: &Path, version: &Version) -> bool { + let path = path.join("pom.xml"); + let manifest = std::fs::read_to_string(&path).unwrap(); + + let pkg = sxd_document::parser::parse(manifest.as_str()).unwrap(); + let doc = pkg.as_document(); + let root = doc.root(); + for child in root.children() { + if let Some(project) = child.element() { + if project.name().local_part() != "project" { + continue; + } + for child in project.children() { + if let Some(v) = child.element() { + if v.name().local_part() != "version" { + continue; + } + v.set_text(version.to_string().as_str()); + break; + } + } + break; + } + } + + let mut result = Vec::new(); + sxd_document::writer::format_document(&doc, &mut result).unwrap(); + + let new_manifest = String::from_utf8_lossy(&result); + if manifest != new_manifest { + std::fs::write(&path, new_manifest.as_bytes()).unwrap(); + println!( + "updating version for package: {} to {}", + path.display(), + version + ); + true + } else { + false + } +} + +fn update_nodejs_version(path: &Path, version: &Version) -> bool { + false +} From da247e653cec8017a6280918a7ef8e697e28ddf8 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 20:55:12 +0800 Subject: [PATCH 06/14] update with regexp Signed-off-by: tison --- bindings/java/pom.xml | 2 +- dev/Cargo.lock | 24 +----------------------- dev/Cargo.toml | 2 +- dev/src/release/package.rs | 29 +++++------------------------ 4 files changed, 8 insertions(+), 49 deletions(-) diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml index 7cdb96220345..0605ce71b677 100644 --- a/bindings/java/pom.xml +++ b/bindings/java/pom.xml @@ -32,7 +32,7 @@ org.apache.opendal opendal - 0.47.7 + 0.47.7 Apache OpenDAL™ diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 3176e2db3e12..4c484681d96a 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -245,19 +245,13 @@ dependencies = [ "markdown", "minijinja", "pretty_assertions", + "regex", "semver", "serde", - "sxd-document", "syn", "toml_edit", ] -[[package]] -name = "peresil" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f658886ed52e196e850cfbbfddab9eaa7f6d90dd0929e264c31e5cec07e09e57" - [[package]] name = "portable-atomic" version = "1.10.0" @@ -371,16 +365,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "sxd-document" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d82f37be9faf1b10a82c4bd492b74f698e40082f0f40de38ab275f31d42078" -dependencies = [ - "peresil", - "typed-arena", -] - [[package]] name = "syn" version = "2.0.91" @@ -414,12 +398,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "typed-arena" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" - [[package]] name = "unicode-id" version = "0.3.5" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 6ec1a344fdc5..5ebea88411e6 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -35,9 +35,9 @@ logforth = { version = "0.21.0", default-features = false } heck = { version = "0.5.0" } markdown = { version = "1.0.0-alpha.21" } minijinja = { version = "2.5.0" } +regex = { version = "1.11.1" } semver = { version = "1.0.25" } serde = { version = "1.0.216", features = ["derive"] } -sxd-document = { version = "0.3.2" } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } toml_edit = { version = "0.22.23", features = ["serde"] } diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index ada359d96cc1..cf4d57885c38 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -129,31 +129,12 @@ fn update_maven_version(path: &Path, version: &Version) -> bool { let path = path.join("pom.xml"); let manifest = std::fs::read_to_string(&path).unwrap(); - let pkg = sxd_document::parser::parse(manifest.as_str()).unwrap(); - let doc = pkg.as_document(); - let root = doc.root(); - for child in root.children() { - if let Some(project) = child.element() { - if project.name().local_part() != "project" { - continue; - } - for child in project.children() { - if let Some(v) = child.element() { - if v.name().local_part() != "version" { - continue; - } - v.set_text(version.to_string().as_str()); - break; - } - } - break; - } - } - - let mut result = Vec::new(); - sxd_document::writer::format_document(&doc, &mut result).unwrap(); + let old_version_matcher = r#"\d+\.\d+\.\d+ "#; + let new_version_string = format!("{version} "); - let new_manifest = String::from_utf8_lossy(&result); + let new_manifest = regex::Regex::new(old_version_matcher) + .unwrap() + .replace_all(manifest.as_str(), new_version_string.as_str()); if manifest != new_manifest { std::fs::write(&path, new_manifest.as_bytes()).unwrap(); println!( From b8770d6c7c3a30c67edad8056015d3dcf02b1e28 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 21:07:23 +0800 Subject: [PATCH 07/14] for nodejs Signed-off-by: tison --- dev/Cargo.lock | 119 +++++++++++++++++++++++++++++++++++++ dev/Cargo.toml | 2 + dev/src/release/package.rs | 29 ++++++++- 3 files changed, 149 insertions(+), 1 deletion(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 4c484681d96a..e78a89be6037 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -66,6 +66,16 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "clap" version = "4.5.23" @@ -112,6 +122,31 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "diff" version = "0.1.13" @@ -134,6 +169,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "globset" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -146,6 +194,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -162,6 +226,12 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + [[package]] name = "jiff" version = "0.1.27" @@ -240,6 +310,7 @@ dependencies = [ "anyhow", "clap", "heck", + "ignore", "log", "logforth", "markdown", @@ -248,6 +319,7 @@ dependencies = [ "regex", "semver", "serde", + "serde_json", "syn", "toml_edit", ] @@ -324,6 +396,21 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "ryu" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.25" @@ -350,6 +437,19 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.138" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "ryu", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -416,6 +516,25 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 5ebea88411e6..5b7aec2b5ba1 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -33,11 +33,13 @@ clap = { version = "4.5.23", features = ["derive"] } log = { version = "0.4.22" } logforth = { version = "0.21.0", default-features = false } heck = { version = "0.5.0" } +ignore = { version = "0.4.23" } markdown = { version = "1.0.0-alpha.21" } minijinja = { version = "2.5.0" } regex = { version = "1.11.1" } semver = { version = "1.0.25" } serde = { version = "1.0.216", features = ["derive"] } +serde_json = { version = "1.0.138", features = ["preserve_order"] } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } toml_edit = { version = "0.22.23", features = ["serde"] } diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index cf4d57885c38..fc87ff98dd6a 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -149,5 +149,32 @@ fn update_maven_version(path: &Path, version: &Version) -> bool { } fn update_nodejs_version(path: &Path, version: &Version) -> bool { - false + let mut updated = false; + + for entry in ignore::Walk::new(path) { + let entry = entry.unwrap(); + if entry.file_name() != "package.json" { + continue; + } + let manifest = std::fs::read_to_string(entry.path()).unwrap(); + let mut manifest: serde_json::Value = serde_json::from_str(&manifest).unwrap(); + + let value = manifest.pointer_mut("/version").unwrap(); + let old_version = Version::parse(value.as_str().unwrap()).unwrap(); + if &old_version != version { + *value = serde_json::Value::String(version.to_string()); + let mut new_manifest = serde_json::to_string_pretty(&manifest).unwrap(); + new_manifest.push('\n'); + std::fs::write(entry.path(), new_manifest).unwrap(); + println!( + "updating version for package: {} from {} to {}", + entry.path().display(), + old_version, + version + ); + updated = true; + } + } + + updated } From d7a16d4575d462f1510963c1238de8749986c4ee Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 21:36:46 +0800 Subject: [PATCH 08/14] archive Signed-off-by: tison --- dev/Cargo.lock | 240 +++++++++++++++++++++++++++++++++++++ dev/Cargo.toml | 8 +- dev/src/main.rs | 17 +++ dev/src/release/mod.rs | 59 +++++++++ dev/src/release/package.rs | 8 +- justfile | 3 + 6 files changed, 329 insertions(+), 6 deletions(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index e78a89be6037..40b49b4805d4 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -66,6 +72,21 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.11.3" @@ -76,6 +97,12 @@ dependencies = [ "serde", ] +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "4.5.23" @@ -122,6 +149,24 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -147,12 +192,38 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "env_filter" version = "0.1.3" @@ -163,12 +234,60 @@ dependencies = [ "regex", ] +[[package]] +name = "env_home" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys", +] + +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "globset" version = "0.4.15" @@ -261,6 +380,29 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "log" version = "0.4.22" @@ -303,12 +445,22 @@ dependencies = [ "serde", ] +[[package]] +name = "miniz_oxide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +dependencies = [ + "adler2", +] + [[package]] name = "odev" version = "0.0.1" dependencies = [ "anyhow", "clap", + "flate2", "heck", "ignore", "log", @@ -320,8 +472,11 @@ dependencies = [ "semver", "serde", "serde_json", + "sha2", "syn", + "tar", "toml_edit", + "which", ] [[package]] @@ -367,6 +522,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.11.1" @@ -396,6 +560,19 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "ryu" version = "1.0.19" @@ -459,6 +636,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "strsim" version = "0.11.1" @@ -476,6 +664,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "toml_datetime" version = "0.6.8" @@ -498,6 +697,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-id" version = "0.3.5" @@ -516,6 +721,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -526,6 +737,18 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "which" +version = "7.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a9e33648339dc1642b0e36e21b3385e6148e289226f657c809dee59df5028" +dependencies = [ + "either", + "env_home", + "rustix", + "winsafe", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -617,6 +840,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + +[[package]] +name = "xattr" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 5b7aec2b5ba1..9d87d57f94fe 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -30,18 +30,22 @@ version = "0.0.1" [dependencies] anyhow = { version = "1.0.95" } clap = { version = "4.5.23", features = ["derive"] } -log = { version = "0.4.22" } -logforth = { version = "0.21.0", default-features = false } +flate2 = { version = "1.0.35" } heck = { version = "0.5.0" } ignore = { version = "0.4.23" } +log = { version = "0.4.22" } +logforth = { version = "0.21.0", default-features = false } markdown = { version = "1.0.0-alpha.21" } minijinja = { version = "2.5.0" } regex = { version = "1.11.1" } semver = { version = "1.0.25" } serde = { version = "1.0.216", features = ["derive"] } serde_json = { version = "1.0.138", features = ["preserve_order"] } +sha2 = { version = "0.10.8" } syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } +tar = { version = "0.4.43" } toml_edit = { version = "0.22.23", features = ["serde"] } +which = { version = "7.0.1" } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/main.rs b/dev/src/main.rs index cbf5bfe17750..0326cc2de87d 100644 --- a/dev/src/main.rs +++ b/dev/src/main.rs @@ -20,6 +20,7 @@ mod release; use clap::{Parser, Subcommand}; use std::path::PathBuf; +use std::process::Command as StdCommand; fn manifest_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -31,6 +32,19 @@ fn workspace_dir() -> PathBuf { manifest_dir().join("..").canonicalize().unwrap() } +fn find_command(cmd: &str) -> StdCommand { + match which::which(cmd) { + Ok(exe) => { + let mut cmd = StdCommand::new(exe); + cmd.current_dir(workspace_dir()); + cmd + } + Err(err) => { + panic!("{cmd} not found: {err}"); + } + } +} + #[derive(Parser)] #[command(version, about, long_about = None)] struct Cmd { @@ -47,6 +61,8 @@ enum Commands { }, /// Update the version of all packages. UpdateVersion, + /// Create all the release artifacts. + Release, } fn main() -> anyhow::Result<()> { @@ -55,5 +71,6 @@ fn main() -> anyhow::Result<()> { match Cmd::parse().command { Commands::Generate { language } => generate::run(&language), Commands::UpdateVersion => release::update_version(), + Commands::Release => release::archive_package(), } } diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index 932e4431806d..8c3a1bcfbf54 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -1,3 +1,9 @@ +use crate::{find_command, workspace_dir}; +use flate2::write::GzEncoder; +use flate2::Compression; +use sha2::{Digest, Sha512}; +use std::io::BufReader; + mod package; pub fn update_version() -> anyhow::Result<()> { @@ -11,3 +17,56 @@ pub fn update_version() -> anyhow::Result<()> { } Ok(()) } + +pub fn archive_package() -> anyhow::Result<()> { + std::fs::create_dir_all(workspace_dir().join("dist"))?; + + let packages = package::all_packages(); + for package in packages { + let mut cmd = find_command("git"); + cmd.args(["ls-files", "LICENSE", "NOTICE"]); + cmd.arg(&package.name); + for dep in &package.dependencies { + cmd.arg(&dep.name); + } + let output = cmd.output().expect("failed to execute git ls-files"); + let output = String::from_utf8_lossy(&output.stdout); + let files = output.lines().collect::>(); + archive_and_checksum(&package, &files)?; + } + + Ok(()) +} + +fn archive_and_checksum(package: &package::Package, files: &[&str]) -> anyhow::Result<()> { + println!("archiving package: {}", package.name); + + let prefix = format!("apache-opendal-{}-src", package.name.replace("/", "-")); + let filename = format!("{}.tar.gz", prefix); + let tarball = workspace_dir().join("dist").join(&filename); + + { + let tarball = std::fs::File::create(&tarball)?; + let encoder = GzEncoder::new(tarball, Compression::default()); + let mut tar = tar::Builder::new(encoder); + for file in files { + tar.append_path_with_name(workspace_dir().join(file), format!("{prefix}/{file}"))?; + } + } + + { + let tarball = std::fs::File::open(&tarball)?; + let mut reader = BufReader::new(tarball); + let mut hasher = Sha512::new(); + std::io::copy(&mut reader, &mut hasher)?; + let digest = hasher.finalize(); + let checksum_lines = format!("{digest:x} {filename}"); + + let checksum = workspace_dir() + .join("dist") + .join(format!("{}.sha512", filename)); + std::fs::write(checksum, checksum_lines)?; + } + + Ok(()) +} diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index fc87ff98dd6a..fc60d82adc99 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -5,10 +5,10 @@ use std::str::FromStr; #[derive(Debug, Clone)] pub struct Package { - name: String, - path: PathBuf, - version: Version, - dependencies: Vec, + pub name: String, + pub path: PathBuf, + pub version: Version, + pub dependencies: Vec, } fn make_package(path: &str, version: &str, dependencies: Vec) -> Package { diff --git a/justfile b/justfile index 81024d4ff762..956577df4476 100644 --- a/justfile +++ b/justfile @@ -23,3 +23,6 @@ generate language: update-version: cargo run --quiet --manifest-path=dev/Cargo.toml -- update-version + +release: + cargo run --quiet --manifest-path=dev/Cargo.toml -- release From 228d2f8ca2611ef22ac6e914b51e5ea662d90a84 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 21:43:05 +0800 Subject: [PATCH 09/14] sign Signed-off-by: tison --- dev/src/main.rs | 6 +++--- dev/src/release/mod.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/dev/src/main.rs b/dev/src/main.rs index 0326cc2de87d..0a496f9adc60 100644 --- a/dev/src/main.rs +++ b/dev/src/main.rs @@ -19,7 +19,7 @@ mod generate; mod release; use clap::{Parser, Subcommand}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command as StdCommand; fn manifest_dir() -> PathBuf { @@ -32,11 +32,11 @@ fn workspace_dir() -> PathBuf { manifest_dir().join("..").canonicalize().unwrap() } -fn find_command(cmd: &str) -> StdCommand { +fn find_command(cmd: &str, cwd: impl AsRef) -> StdCommand { match which::which(cmd) { Ok(exe) => { let mut cmd = StdCommand::new(exe); - cmd.current_dir(workspace_dir()); + cmd.current_dir(cwd); cmd } Err(err) => { diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index 8c3a1bcfbf54..ca540b0e7f70 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -21,9 +21,12 @@ pub fn update_version() -> anyhow::Result<()> { pub fn archive_package() -> anyhow::Result<()> { std::fs::create_dir_all(workspace_dir().join("dist"))?; + let workspace_dir = workspace_dir(); + let dist_dir = workspace_dir.join("dist"); + let packages = package::all_packages(); for package in packages { - let mut cmd = find_command("git"); + let mut cmd = find_command("git", &workspace_dir); cmd.args(["ls-files", "LICENSE", "NOTICE"]); cmd.arg(&package.name); for dep in &package.dependencies { @@ -33,6 +36,31 @@ pub fn archive_package() -> anyhow::Result<()> { let output = String::from_utf8_lossy(&output.stdout); let files = output.lines().collect::>(); archive_and_checksum(&package, &files)?; + + let prefix = format!("apache-opendal-{}-src", package.name.replace("/", "-")); + let filename = format!("{}.tar.gz", prefix); + + println!("Generate signature for package: {}", package.name); + let mut cmd = find_command("gpg", &dist_dir); + cmd.args([ + "--yes", + "--armor", + "--output", + format!("{filename}.asc").as_str(), + "--detach-sign", + filename.as_str(), + ]); + cmd.output().expect("failed to sign the package"); + + println!("Check signature for package: {}", package.name); + let mut cmd = find_command("gpg", &dist_dir); + cmd.args([ + "--verify", + format!("{filename}.asc").as_str(), + filename.as_str(), + ]); + cmd.output() + .expect("failed to verify the package signature"); } Ok(()) From b0725188f0b040e2ce3edfa6fa0c5b818307bf56 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 22:31:59 +0800 Subject: [PATCH 10/14] fixup Signed-off-by: tison --- .github/ISSUE_TEMPLATE/3-new-release.md | 4 - dev/src/release/mod.rs | 17 ++-- dev/src/release/package.rs | 26 +++++- scripts/README.md | 2 +- scripts/release.py | 104 ------------------------ scripts/version.py | 35 -------- 6 files changed, 31 insertions(+), 157 deletions(-) delete mode 100755 scripts/release.py delete mode 100755 scripts/version.py diff --git a/.github/ISSUE_TEMPLATE/3-new-release.md b/.github/ISSUE_TEMPLATE/3-new-release.md index 42fbb52c6288..f0784913e627 100644 --- a/.github/ISSUE_TEMPLATE/3-new-release.md +++ b/.github/ISSUE_TEMPLATE/3-new-release.md @@ -14,10 +14,6 @@ This issue is used to track tasks of the opendal ${opendal_version} release. ### Build Release -#### Release List - - - #### GitHub Side - [ ] Bump version in project diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index ca540b0e7f70..131a673311a1 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -28,8 +28,8 @@ pub fn archive_package() -> anyhow::Result<()> { for package in packages { let mut cmd = find_command("git", &workspace_dir); cmd.args(["ls-files", "LICENSE", "NOTICE"]); - cmd.arg(&package.name); - for dep in &package.dependencies { + cmd.arg(&package.name()); + for dep in &package.dependencies() { cmd.arg(&dep.name); } let output = cmd.output().expect("failed to execute git ls-files"); @@ -37,10 +37,9 @@ pub fn archive_package() -> anyhow::Result<()> { let files = output.lines().collect::>(); archive_and_checksum(&package, &files)?; - let prefix = format!("apache-opendal-{}-src", package.name.replace("/", "-")); - let filename = format!("{}.tar.gz", prefix); + let filename = format!("{}.tar.gz", package.make_prefix()); - println!("Generate signature for package: {}", package.name); + println!("Generate signature for package: {}", package.name()); let mut cmd = find_command("gpg", &dist_dir); cmd.args([ "--yes", @@ -52,7 +51,7 @@ pub fn archive_package() -> anyhow::Result<()> { ]); cmd.output().expect("failed to sign the package"); - println!("Check signature for package: {}", package.name); + println!("Check signature for package: {}", package.name()); let mut cmd = find_command("gpg", &dist_dir); cmd.args([ "--verify", @@ -67,10 +66,10 @@ pub fn archive_package() -> anyhow::Result<()> { } fn archive_and_checksum(package: &package::Package, files: &[&str]) -> anyhow::Result<()> { - println!("archiving package: {}", package.name); + println!("Archiving package: {}", package.name()); - let prefix = format!("apache-opendal-{}-src", package.name.replace("/", "-")); - let filename = format!("{}.tar.gz", prefix); + let prefix = package.make_prefix(); + let filename = format!("{prefix}.tar.gz"); let tarball = workspace_dir().join("dist").join(&filename); { diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index fc60d82adc99..eb0bb4761bf8 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -5,10 +5,28 @@ use std::str::FromStr; #[derive(Debug, Clone)] pub struct Package { - pub name: String, - pub path: PathBuf, - pub version: Version, - pub dependencies: Vec, + name: String, + path: PathBuf, + version: Version, + dependencies: Vec, +} + +impl Package { + pub fn dependencies(&self) -> &[Package] { + &self.dependencies + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn make_prefix(&self) -> String { + format!( + "apache-opendal-{}-{}-src", + self.name.replace("/", "-"), + self.version + ) + } } fn make_package(path: &str, version: &str, dependencies: Vec) -> Package { diff --git a/scripts/README.md b/scripts/README.md index a62f2454efed..ed5283348428 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,7 +8,7 @@ NOTES: all scripts must be running at root folder of OpenDAL project. ## Release ```shell -./scripts/release.py +just release ``` > Before running release, please make sure you have bump all versions. diff --git a/scripts/release.py b/scripts/release.py deleted file mode 100755 index 767357617e78..000000000000 --- a/scripts/release.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import subprocess -import tarfile -from pathlib import Path -from constants import get_package_version, get_package_dependence, PACKAGES - -ROOT_DIR = Path(__file__).parent.parent - - -def archive_package(path): - print(f"Archive package {path} started") - - core_version = get_package_version("core") - version = get_package_version(path) - name = f"apache-opendal-{str(path).replace('/', '-')}-{version}-src" - - # `git ls-files` handles duplicated path correctly, so we don't need to worry about it - ls_command = [ - "git", - "ls-files", - # Make sure LICENSE and NOTICE files are included. - "LICENSE", - "NOTICE", - # Always include core. - "core", - f"{path}", - f"{get_package_dependence(path)}", - ] - ls_result = subprocess.run( - ls_command, cwd=ROOT_DIR, capture_output=True, check=True, text=True - ) - files = list(filter(lambda x: len(x) > 0, ls_result.stdout.split("\n"))) - - with tarfile.open(f"{ROOT_DIR}/dist/{name}.tar.gz", "w:gz") as tar: - for file in files: - tar.add(f"{ROOT_DIR}/{file}", arcname=f"{name}/{file}") - - print(f"Archive package {path} to dist/{name}.tar.gz") - - -def generate_signature(): - for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): - print(f"Generate signature for {i}") - subprocess.run( - ["gpg", "--yes", "--armor", "--output", f"{i}.asc", "--detach-sig", str(i)], - cwd=ROOT_DIR / "dist", - check=True, - ) - - for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): - print(f"Check signature for {i}") - subprocess.run( - ["gpg", "--verify", f"{i}.asc", str(i)], cwd=ROOT_DIR / "dist", check=True - ) - - -def generate_checksum(): - for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): - print(f"Generate checksum for {i}") - subprocess.run( - ["shasum", "-a", "512", str(i.relative_to(ROOT_DIR / "dist"))], - stdout=open(f"{i}.sha512", "w"), - cwd=ROOT_DIR / "dist", - check=True, - ) - - for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): - print(f"Check checksum for {i}") - subprocess.run( - [ - "shasum", - "-a", - "512", - "-c", - f"{str(i.relative_to(ROOT_DIR / 'dist'))}.sha512", - ], - cwd=ROOT_DIR / "dist", - check=True, - ) - - -if __name__ == "__main__": - (ROOT_DIR / "dist").mkdir(exist_ok=True) - for v in PACKAGES: - archive_package(v) - generate_signature() - generate_checksum() diff --git a/scripts/version.py b/scripts/version.py deleted file mode 100755 index fde8f54b9122..000000000000 --- a/scripts/version.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from constants import PACKAGES, get_package_version - - -def increment_patch_version(version): - parts = version.split(".") - parts[-1] = str(int(parts[-1]) + 1) - return ".".join(parts) - - -if __name__ == "__main__": - print(f"| {'Name':<25} | {'Version':<7} | {'Next':<7} |") - print(f"| {'-':<25} | {'-':<7} | {'-':<7} |") - for v in PACKAGES: - cur = get_package_version(v) - next = increment_patch_version(cur) - - print(f"| {str(v):<25} | {cur:<7} | {next:<7} |") From d2ecc4df46286c0ed1a2ba8a5387c970d688d8cf Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 22:35:03 +0800 Subject: [PATCH 11/14] update docs Signed-off-by: tison --- dev/src/release/mod.rs | 4 +- website/community/release/release.md | 64 ++-------------------------- 2 files changed, 5 insertions(+), 63 deletions(-) diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index 131a673311a1..a2b3e37234f7 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -29,8 +29,8 @@ pub fn archive_package() -> anyhow::Result<()> { let mut cmd = find_command("git", &workspace_dir); cmd.args(["ls-files", "LICENSE", "NOTICE"]); cmd.arg(&package.name()); - for dep in &package.dependencies() { - cmd.arg(&dep.name); + for dep in package.dependencies() { + cmd.arg(&dep.name()); } let output = cmd.output().expect("failed to execute git ls-files"); let output = String::from_utf8_lossy(&output.stdout); diff --git a/website/community/release/release.md b/website/community/release/release.md index 9be8cc7eb589..3bf213add09f 100644 --- a/website/community/release/release.md +++ b/website/community/release/release.md @@ -69,63 +69,11 @@ ${name} ## Start a tracking issue about the next release -Start a tracking issue on GitHub for the upcoming release to track all tasks that need to be completed. - -Title: - -``` -Tracking issues of OpenDAL ${opendal_version} Release -``` - -Content: - -```markdown -This issue is used to track tasks of the opendal ${opendal_version} release. - -## Tasks - -### Blockers - - - -### Build Release - -#### Release List - - - -#### GitHub Side - -- [ ] Bump version in project -- [ ] Update docs -- [ ] Generate dependencies list -- [ ] Push release candidate tag to GitHub - -#### ASF Side - -- [ ] Create an ASF Release -- [ ] Upload artifacts to the SVN dist repo -- [ ] Close the Nexus staging repo - -### Voting - -- [ ] Start VOTE at opendal community - -### Official Release - -- [ ] Push the release git tag -- [ ] Publish artifacts to SVN RELEASE branch -- [ ] Release Maven artifacts -- [ ] Send the announcement - -For details of each step, please refer to: https://opendal.apache.org/community/release/ -``` +Start a [tracking issue on GitHub](https://github.com/apache/opendal/issues/new?template=3-new-release.md) for the upcoming release to track all tasks that need to be completed. ## Release List -Use `./scripts/version.py` to generate a release version list for review. - -This list bumps the `patch` version by default; please adapt to the actual needs. +Update the version list in the `dev/src/release/package.rs` file. For example: @@ -136,13 +84,7 @@ For example: ### Bump version in project -Bump all components' version in the project to the new opendal version. -Please note that this version is the exact version of the release, not the release candidate version. - -- Rust core: Bump version in `Cargo.toml` -- Haskell binding: Bump version and update the `tag` field of `source-repository this` in `bindings/haskell/opendal.cabal` -- Java binding: Bump version in `bindings/java/pom.xml` -- Node.js binding: Bump version in `bindings/nodejs/package.json` and `bindings/nodejs/npm/*/package.json` +Run `just update-version` to bump the version in the project. ### Update docs From 54362ff0821866d659107a77e8f9cb6b6cc03c04 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 22:36:32 +0800 Subject: [PATCH 12/14] license header Signed-off-by: tison --- .github/workflows/ci_check.yml | 2 +- dev/src/release/mod.rs | 17 +++++++++++++++++ dev/src/release/package.rs | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 349303081265..0c4ed1e5522a 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -51,7 +51,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Check license headers - uses: korandoru/hawkeye@v5 + uses: korandoru/hawkeye@v6 # Add python format check later. code-format: diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index a2b3e37234f7..d560e03c4cb2 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::{find_command, workspace_dir}; use flate2::write::GzEncoder; use flate2::Compression; diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index eb0bb4761bf8..a02cbb05b1ba 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::workspace_dir; use semver::Version; use std::path::{Path, PathBuf}; From 411231d144b5686d98ea945e336c01d45f73d682 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 22:46:18 +0800 Subject: [PATCH 13/14] lua is unreleased now Signed-off-by: tison --- bindings/lua/opendal-0.1.0-1.rockspec | 28 +++++++++++++++++++++++++++ dev/src/release/package.rs | 2 -- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 bindings/lua/opendal-0.1.0-1.rockspec diff --git a/bindings/lua/opendal-0.1.0-1.rockspec b/bindings/lua/opendal-0.1.0-1.rockspec new file mode 100644 index 000000000000..f526d974ed98 --- /dev/null +++ b/bindings/lua/opendal-0.1.0-1.rockspec @@ -0,0 +1,28 @@ +package = "opendal" +version = "0.1.0-1" + +source = { + url = "git+https://github.com/apache/opendal/", +} + +description = { + summary = "Apache OpenDAL™ LUA binding: access data freely. ", + detailed = [[ + OpenDAL is a data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way. + ]], + homepage = "https://opendal.apache.org/", + license = " Apache-2.0" +} + +dependencies = { + "lua >= 5.1", + "luarocks-build-rust-mlua = 0.2.0", +} + +build = { + type = "rust-mlua", + modules = { + ["opendal"] = "opendal_lua", + }, + target_path = "target", +} diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index a02cbb05b1ba..42df68b08c0f 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -84,7 +84,6 @@ pub fn all_packages() -> Vec { let c = make_package("bindings/c", "0.45.3", vec![core.clone()]); let cpp = make_package("bindings/cpp", "0.45.15", vec![core.clone()]); let java = make_package("bindings/java", "0.47.7", vec![core.clone()]); - let lua = make_package("bindings/lua", "0.1.13", vec![core.clone()]); let nodejs = make_package("bindings/nodejs", "0.47.9", vec![core.clone()]); let python = make_package("bindings/python", "0.45.14", vec![core.clone()]); @@ -103,7 +102,6 @@ pub fn all_packages() -> Vec { c, cpp, java, - lua, nodejs, python, ] From a7c45b423eb7ab6f7464055ed3b6721f060897de Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Feb 2025 22:48:45 +0800 Subject: [PATCH 14/14] fixup Signed-off-by: tison --- dev/src/release/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/src/release/mod.rs b/dev/src/release/mod.rs index d560e03c4cb2..5923fb118c7e 100644 --- a/dev/src/release/mod.rs +++ b/dev/src/release/mod.rs @@ -45,9 +45,9 @@ pub fn archive_package() -> anyhow::Result<()> { for package in packages { let mut cmd = find_command("git", &workspace_dir); cmd.args(["ls-files", "LICENSE", "NOTICE"]); - cmd.arg(&package.name()); + cmd.arg(package.name()); for dep in package.dependencies() { - cmd.arg(&dep.name()); + cmd.arg(dep.name()); } let output = cmd.output().expect("failed to execute git ls-files"); let output = String::from_utf8_lossy(&output.stdout);