Add a concept of an Executable

Executable is a deserialized but not loaded compiled code. This allows us to implement a mechanism by which we copy/create WebAssembly module data fewer times before it becomes an instance. Of particular interest, it can help to avoid allocating the tree of maps and vectors that are used to represent the module when an Artifact has already been compiled.
near · Apr 20, 2022 · 2d21329 · 2d21329
1 parent ba0dcfe
commit 2d21329
Show file tree

Hide file tree

Showing 145 changed files with 3,538 additions and 6,311 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -18,7 +18,6 @@ wasmer-compiler-llvm = { version = "2.0.0", path = "lib/compiler-llvm", optional
 wasmer-engine = { version = "=2.2.2", path = "lib/engine", package = "wasmer-engine-near"  }
 wasmer-engine-universal = { version = "=2.2.2", path = "lib/engine-universal", optional = true, package = "wasmer-engine-universal-near" }
 wasmer-wast = { version = "2.0.0", path = "tests/lib/wast", optional = true }
-wasmer-cache = { version = "2.0.0", path = "lib/cache", optional = true }
 wasmer-types = { version = "=2.2.2", path = "lib/types", package = "wasmer-types-near" }
 wasmer-vm = { version = "=2.2.2", path = "lib/vm", package = "wasmer-vm-near" }
 
@@ -27,8 +26,6 @@ cfg-if = "1.0"
 [workspace]
 members = [
     "lib/api",
-    "lib/cache",
-    "lib/cli",
     "lib/compiler",
     "lib/compiler-cranelift",
     "lib/compiler-singlepass",
@@ -40,7 +37,6 @@ members = [
     "lib/types",
     "tests/lib/wast",
     "tests/lib/compiler-test-derive",
-    "tests/integration/cli",
     "tests/integration/ios",
     "fuzz",
 ]
@@ -58,33 +54,30 @@ anyhow = "1.0"
 criterion = "0.3"
 lazy_static = "1.4"
 serial_test = "0.5"
-wasmer-engine-dummy = { path = "tests/lib/engine-dummy" }
 compiler-test-derive = { path = "tests/lib/compiler-test-derive" }
 tempfile = "3.1"
-loupe = "0.1"
 # For logging tests using the `RUST_LOG=debug` when testing
-test-env-log = { version = "0.2", default-features = false, features = ["trace"] }
+test-log = { version = "0.2", default-features = false, features = ["trace"] }
 tracing = { version = "0.1", default-features = false, features = ["log"] }
 tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt"] }
+wat = "1.0"
+
 
 [features]
 # Don't add the compiler features in default, please add them on the Makefile
 # since we might want to autoconfigure them depending on the availability on the host.
 default = [
-    "wat",
+    "wasmer/wat",
     "wast",
     "universal",
     "singlepass",
-    "cache",
 ]
 engine = []
 universal = [
     "wasmer-engine-universal",
     "engine",
 ]
-cache = ["wasmer-cache"]
 wast = ["wasmer-wast"]
-wat = ["wasmer/wat"]
 compiler = [
     "wasmer/compiler",
     "wasmer-compiler/translator",

diff --git a/Makefile b/Makefile
@@ -344,16 +344,6 @@ endif
 #
 #####
 
-# Not really "all", just the default target that builds enough so make
-# install will go through.
-all: build-wasmer
-
-build-wasmer:
-	cargo build --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --bin wasmer
-
-build-wasmer-debug:
-	cargo build --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "debug"  --bin wasmer
-
 bench:
 	cargo bench $(compiler_features)
 
@@ -369,9 +359,6 @@ bench:
 # incremental = false
 # codegen-units = 1
 # rpath = false
-build-wasmer-headless-minimal: RUSTFLAGS += -C panic=abort
-build-wasmer-headless-minimal:
-	RUSTFLAGS="${RUSTFLAGS}" xargo build --target $(HOST_TARGET) --release --manifest-path=lib/cli/Cargo.toml --no-default-features --features headless-minimal --bin wasmer-headless
 ifeq ($(IS_DARWIN), 1)
 	strip -u target/$(HOST_TARGET)/release/wasmer-headless
 else ifeq ($(IS_WINDOWS), 1)
@@ -380,21 +367,6 @@ else
 	strip --strip-unneeded target/$(HOST_TARGET)/release/wasmer-headless
 endif
 
-WAPM_VERSION = v0.5.1
-get-wapm:
-	[ -d "wapm-cli" ] || git clone --branch $(WAPM_VERSION) https://github.com/wasmerio/wapm-cli.git
-
-build-wapm: get-wapm
-ifeq ($(IS_DARWIN), 1)
-	# We build it without bundling sqlite, as is included by default in macos
-	cargo build --release --manifest-path wapm-cli/Cargo.toml --no-default-features --features "packagesigning telemetry update-notifications"
-else
-	cargo build --release --manifest-path wapm-cli/Cargo.toml --features "telemetry update-notifications"
-endif
-
-build-docs:
-	cargo doc --release $(compiler_features) --document-private-items --no-deps --workspace --exclude wasmer-c-api
-
 #####
 #
 # Testing.
@@ -410,7 +382,6 @@ test-packages:
 	cargo test --all --release $(exclude_tests)
 	cargo test --manifest-path lib/compiler-cranelift/Cargo.toml --release --no-default-features --features=std
 	cargo test --manifest-path lib/compiler-singlepass/Cargo.toml --release --no-default-features --features=std
-	cargo test --manifest-path lib/cli/Cargo.toml $(compiler_features) --release
 
 #####
 #
@@ -459,53 +430,12 @@ test-integration-ios:
 #
 #####
 
-package-wapm:
-	mkdir -p "package/bin"
-ifneq (, $(filter 1, $(IS_DARWIN) $(IS_LINUX)))
-	if [ -d "wapm-cli" ]; then \
-		cp wapm-cli/$(TARGET_DIR)/wapm package/bin/ ;\
-		echo -e "#!/bin/bash\nwapm execute \"\$$@\"" > package/bin/wax ;\
-		chmod +x package/bin/wax ;\
-	fi
-else
-	if [ -d "wapm-cli" ]; then \
-		cp wapm-cli/$(TARGET_DIR)/wapm package/bin/ ;\
-	fi
-ifeq ($(IS_DARWIN), 1)
-	codesign -s - package/bin/wapm || true
-endif
-endif
-
-package-minimal-headless-wasmer:
-ifeq ($(IS_WINDOWS), 1)
-	if [ -f "target/$(HOST_TARGET)/release/wasmer-headless.exe" ]; then \
-		cp target/$(HOST_TARGET)/release/wasmer-headless.exe package/bin ;\
-	fi
-else
-	if [ -f "target/$(HOST_TARGET)/release/wasmer-headless" ]; then \
-		cp target/$(HOST_TARGET)/release/wasmer-headless package/bin ;\
-	fi
-endif
-
-package-wasmer:
-	mkdir -p "package/bin"
-ifeq ($(IS_WINDOWS), 1)
-	cp $(TARGET_DIR)/wasmer.exe package/bin/
-else
-	cp $(TARGET_DIR)/wasmer package/bin/
-ifeq ($(IS_DARWIN), 1)
-	codesign -s - package/bin/wasmer || true
-endif
-endif
-
 package-docs: build-docs
 	mkdir -p "package/docs/crates"
 	cp -R target/doc/ package/docs/crates
 	echo '<meta http-equiv="refresh" content="0; url=crates/wasmer/index.html">' > package/docs/index.html
 	echo '<meta http-equiv="refresh" content="0; url=wasmer/index.html">' > package/docs/crates/index.html
 
-package: package-wapm package-wasmer package-minimal-headless-wasmer
-
 distribution: package
 	cp LICENSE package/LICENSE
 	cp ATTRIBUTIONS.md package/ATTRIBUTIONS
@@ -558,7 +488,6 @@ update-testsuite:
 lint-packages: RUSTFLAGS += -D dead-code -D nonstandard-style -D unused-imports -D unused-mut -D unused-variables -D unused-unsafe -D unreachable-patterns -D bad-style -D improper-ctypes -D unused-allocation -D unused-comparisons -D while-true -D unconditional-recursion -D bare-trait-objects # TODO: add `-D missing-docs` # TODO: add `-D function_item_references` (not available on Rust 1.47, try when upgrading)
 lint-packages:
 	RUSTFLAGS="${RUSTFLAGS}" cargo clippy --all $(exclude_tests)
-	RUSTFLAGS="${RUSTFLAGS}" cargo clippy --manifest-path lib/cli/Cargo.toml $(compiler_features)
 	RUSTFLAGS="${RUSTFLAGS}" cargo clippy --manifest-path fuzz/Cargo.toml $(compiler_features)
 
 lint-formatting:

diff --git a/benches/many_functions.rs b/benches/many_functions.rs
@@ -1,5 +1,6 @@
 use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
 use wasmer::*;
+use wasmer_engine_universal::UniversalExecutableRef;
 
 fn call_many_functions(n: usize) -> String {
     let fndefs = (0..n)
@@ -28,17 +29,18 @@ fn nops(c: &mut Criterion) {
             })
         });
         drop(compile);
+
         let module = Module::new(&store, &wat).unwrap();
         let imports = imports! {};
         let instance = Instance::new(&module, &imports).unwrap();
         let mut get_main = c.benchmark_group("get_main");
         get_main.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| {
             b.iter(|| {
-                let _: &Function = instance.exports.get("main").unwrap();
+                let _: Function = instance.lookup_function("main").unwrap();
             })
         });
         drop(get_main);
-        let main: &Function = instance.exports.get("main").unwrap();
+        let main: Function = instance.lookup_function("main").unwrap();
         let mut call_main = c.benchmark_group("call_main");
         call_main.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| {
             b.iter(|| {
@@ -47,13 +49,33 @@ fn nops(c: &mut Criterion) {
         });
         drop(call_main);
 
-        let single: &Function = instance.exports.get("single").unwrap();
+        let single: Function = instance.lookup_function("single").unwrap();
         let mut call_single = c.benchmark_group("call_single");
         call_single.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| {
             b.iter(|| {
                 black_box(single.call(&[]).unwrap());
             })
         });
+        drop(call_single);
+
+        let mut serialize = c.benchmark_group("serialize");
+        let wasm = wat::parse_bytes(wat.as_ref()).unwrap();
+        let executable = store.engine().compile(&wasm, store.tunables()).unwrap();
+        serialize.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| {
+            b.iter(|| {
+                black_box(executable.serialize().unwrap());
+            })
+        });
+        drop(serialize);
+
+        let serialized = executable.serialize().unwrap();
+        let mut deserialize = c.benchmark_group("deserialize");
+        deserialize.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| {
+            b.iter(|| unsafe {
+                let deserialized = UniversalExecutableRef::deserialize(&serialized).unwrap();
+                black_box(store.engine().load(&deserialized).unwrap());
+            })
+        });
     }
 }
 

diff --git a/benches/static_and_dynamic_functions.rs b/benches/static_and_dynamic_functions.rs
@@ -12,7 +12,7 @@ static BASIC_WAT: &str = r#"(module
                                   i32 i32 i32 i32 i32
                                   i32 i32 i32 i32 i32) (result i32)
        (i32.add
-                (i32.add 
+                (i32.add
                          (i32.add (i32.add (i32.add (local.get 0)  (local.get 1))
                                            (i32.add (local.get 2)  (local.get 3)))
                                   (i32.add (i32.add (local.get 4)  (local.get 5))
@@ -39,7 +39,7 @@ pub fn run_basic_static_function(store: &Store, compiler_name: &str, c: &mut Cri
         },
     };
     let instance = Instance::new(&module, &import_object).unwrap();
-    let dyn_f: &Function = instance.exports.get("add").unwrap();
+    let dyn_f = instance.lookup_function("add").unwrap();
     let f: NativeFunc<(i32, i32), i32> = dyn_f.native().unwrap();
 
     c.bench_function(&format!("basic static func {}", compiler_name), |b| {
@@ -49,7 +49,7 @@ pub fn run_basic_static_function(store: &Store, compiler_name: &str, c: &mut Cri
         })
     });
 
-    let dyn_f_many: &Function = instance.exports.get("add20").unwrap();
+    let dyn_f_many = instance.lookup_function("add20").unwrap();
     let f_many: NativeFunc<
         (
             i32,
@@ -101,15 +101,15 @@ pub fn run_basic_dynamic_function(store: &Store, compiler_name: &str, c: &mut Cr
     };
     let instance = Instance::new(&module, &import_object).unwrap();
 
-    let dyn_f: &Function = instance.exports.get("add").unwrap();
+    let dyn_f = instance.lookup_function("add").unwrap();
     c.bench_function(&format!("basic dynfunc {}", compiler_name), |b| {
         b.iter(|| {
             let dyn_result = black_box(dyn_f.call(&[Val::I32(4), Val::I32(6)]).unwrap());
             assert_eq!(dyn_result[0], Val::I32(10));
         })
     });
 
-    let dyn_f_many: &Function = instance.exports.get("add20").unwrap();
+    let dyn_f_many = instance.lookup_function("add20").unwrap();
     c.bench_function(
         &format!("basic dynfunc with many args {}", compiler_name),
         |b| {
@@ -150,43 +150,43 @@ fn run_static_benchmarks(_c: &mut Criterion) {
     #[cfg(feature = "llvm")]
     {
         let store = Store::new(&Universal::new(wasmer_compiler_llvm::LLVM::new()).engine());
-        run_basic_static_function(&store, "llvm", c);
+        run_basic_static_function(&store, "llvm", _c);
     }
 
     #[cfg(feature = "cranelift")]
     {
         let store =
             Store::new(&Universal::new(wasmer_compiler_cranelift::Cranelift::new()).engine());
-        run_basic_static_function(&store, "cranelift", c);
+        run_basic_static_function(&store, "cranelift", _c);
     }
 
     #[cfg(feature = "singlepass")]
     {
         let store =
             Store::new(&Universal::new(wasmer_compiler_singlepass::Singlepass::new()).engine());
-        run_basic_static_function(&store, "singlepass", c);
+        run_basic_static_function(&store, "singlepass", _c);
     }
 }
 
 fn run_dynamic_benchmarks(_c: &mut Criterion) {
     #[cfg(feature = "llvm")]
     {
         let store = Store::new(&Universal::new(wasmer_compiler_llvm::LLVM::new()).engine());
-        run_basic_dynamic_function(&store, "llvm", c);
+        run_basic_dynamic_function(&store, "llvm", _c);
     }
 
     #[cfg(feature = "cranelift")]
     {
         let store =
             Store::new(&Universal::new(wasmer_compiler_cranelift::Cranelift::new()).engine());
-        run_basic_dynamic_function(&store, "cranelift", c);
+        run_basic_dynamic_function(&store, "cranelift", _c);
     }
 
     #[cfg(feature = "singlepass")]
     {
         let store =
             Store::new(&Universal::new(wasmer_compiler_singlepass::Singlepass::new()).engine());
-        run_basic_dynamic_function(&store, "singlepass", c);
+        run_basic_dynamic_function(&store, "singlepass", _c);
     }
 }
 

diff --git a/examples/compiler_singlepass.rs b/examples/compiler_singlepass.rs
@@ -46,7 +46,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Let's instantiate the Wasm module.
     let instance = Instance::new(&module, &import_object)?;
 
-    let sum = instance.exports.get_function("sum")?;
+    let sum = instance.lookup_function("sum").expect("function lookup");
 
     println!("Calling `sum` function...");
     // Let's call the `sum` exported function. The parameters are a

diff --git a/examples/tunables_limit_memory.rs b/examples/tunables_limit_memory.rs
@@ -1,7 +1,6 @@
 use std::ptr::NonNull;
 use std::sync::Arc;
 
-use loupe::MemoryUsage;
 use wasmer::{
     imports,
     vm::{self, MemoryError, MemoryStyle, TableStyle, VMMemoryDefinition, VMTableDefinition},
@@ -15,7 +14,6 @@ use wasmer_engine_universal::Universal;
 ///
 /// After adjusting the memory limits, it delegates all other logic
 /// to the base tunables.
-#[derive(MemoryUsage)]
 pub struct LimitingTunables<T: Tunables> {
     /// The maximum a linear memory is allowed to be (in Wasm pages, 64 KiB each).
     /// Since Wasmer ensures there is only none or one memory, this is practically

diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml
@@ -32,7 +32,6 @@ wasmer-derive = { path = "../derive", version = "=2.2.2", package = "wasmer-deri
 wasmer-engine = { path = "../engine", version = "=2.2.2", package = "wasmer-engine-near" }
 wasmer-types = { path = "../types", version = "=2.2.2", package = "wasmer-types-near" }
 target-lexicon = { version = "0.12.2", default-features = false }
-loupe = "0.1"
 # - Optional dependencies for `sys`.
 wasmer-compiler-singlepass = { path = "../compiler-singlepass", package = "wasmer-compiler-singlepass-near", version = "=2.2.2", optional = true}
 wasmer-compiler-cranelift = { path = "../compiler-cranelift", version = "2.1.0", optional = true }