From 192bfd331a3ff13513e2f09d9b04d0e63d993c8d Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Sun, 18 Feb 2024 07:14:48 -0500 Subject: [PATCH] Add simple example --- README.md | 9 +++- crates/hdfs-native/examples/simple.rs | 65 +++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 crates/hdfs-native/examples/simple.rs diff --git a/README.md b/README.md index e73e8d5..8a172c2 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,11 @@ then you can run the benchmarks with cargo bench -p hdfs-native --features benchmark ``` -The `benchmark` feature is required to expose `minidfs` and the internal erasure coding functions to benchmark. \ No newline at end of file +The `benchmark` feature is required to expose `minidfs` and the internal erasure coding functions to benchmark. + +## Running examples +The examples make use of the `minidfs` module to create a simple HDFS cluster to run the example. This requires including the `integration-test` feature to enable the `minidfs` module. Alternatively, if you want to run the example against an existing HDFS cluster you can exclude the `integration-test` feature and make sure your `HADOOP_CONF_DIR` points to a directory with HDFS configs for talking to your cluster. + +```bash +cargo run --example simple --features integration-test +``` diff --git a/crates/hdfs-native/examples/simple.rs b/crates/hdfs-native/examples/simple.rs new file mode 100644 index 0000000..0ff06f5 --- /dev/null +++ b/crates/hdfs-native/examples/simple.rs @@ -0,0 +1,65 @@ +use std::collections::HashSet; + +#[cfg(feature = "integration-test")] +use hdfs_native::minidfs::MiniDfs; +use hdfs_native::{Client, WriteOptions}; + +#[tokio::main] +async fn main() { + let _ = env_logger::builder().format_timestamp_millis().try_init(); + + // If using the integration-test feature, create a MiniDFS cluster. Otherwise + // assume the environment has configs pointing to an existing HDFS cluster. + #[cfg(feature = "integration-test")] + let _dfs = MiniDfs::with_features(&HashSet::new()); + + let client = Client::default(); + + // Create an empty file + client + .create("/hdfs-native-test", WriteOptions::default()) + .await + .unwrap() + .close() + .await + .unwrap(); + + // List files + let listing = client.list_status("/", false).await.unwrap(); + println!("{:?}", listing); + + // Get info on a specific file + let status = client.get_file_info("/hdfs-native-test").await.unwrap(); + println!("{:?}", status); + + // Rename a file + client + .rename("/hdfs-native-test", "/hdfs-native-test2", false) + .await + .unwrap(); + + // Delete a file + client.delete("/hdfs-native-test2", false).await.unwrap(); + + // Write to a new file + let mut writer = client + .create("/hdfs-native-write", WriteOptions::default()) + .await + .unwrap(); + + writer.write(vec![1, 2, 3, 4].into()).await.unwrap(); + writer.close().await.unwrap(); + + // Append to an existing file + let mut writer = client.append("/hdfs-native-write").await.unwrap(); + + writer.write(vec![5, 6, 7, 8].into()).await.unwrap(); + writer.close().await.unwrap(); + + // Read a file + let reader = client.read("/hdfs-native-write").await.unwrap(); + let content = reader.read_range(0, reader.file_length()).await.unwrap(); + println!("{:?}", content); + + client.delete("/hdfs-native-write", false).await.unwrap(); +}