Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Buffer Size Parameter to dump create #234

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add Buffer Size Parameter to dump create #234

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2719549
Add Buffer Size Parameter to dump create
gugacavalieri Nov 5, 2022
dc83b0a
Add help to buffer_size argument
gugacavalieri Nov 5, 2022
0ecd245
add dump_chunk_size to SourceOptions
gugacavalieri Nov 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions replibyte/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ pub struct DumpCreateArgs {
/// import dump from stdin
#[clap(name = "input", short, long, requires = "source_type")]
pub input: bool,
#[clap(name = "buffer_size", short, long, default_value_t = 100, value_name = "buffer_size", help = "Buffer Size in MB")]
pub buffer_size: usize,
#[clap(short, long, parse(from_os_str), value_name = "dump file")]
/// dump file
pub file: Option<PathBuf>,
Expand Down
12 changes: 6 additions & 6 deletions replibyte/src/commands/dump.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ where
password.as_str(),
);

let task = FullDumpTask::new(postgres, datastore, options);
let task = FullDumpTask::new(postgres, datastore, options, args.buffer_size);
gugacavalieri marked this conversation as resolved.
Show resolved Hide resolved
task.run(progress_callback)?
}
ConnectionUri::Mysql(host, port, username, password, database) => {
Expand All @@ -157,13 +157,13 @@ where
password.as_str(),
);

let task = FullDumpTask::new(mysql, datastore, options);
let task = FullDumpTask::new(mysql, datastore, options, args.buffer_size);
gugacavalieri marked this conversation as resolved.
Show resolved Hide resolved
task.run(progress_callback)?
}
ConnectionUri::MongoDB(uri, database) => {
let mongodb = MongoDB::new(uri.as_str(), database.as_str());

let task = FullDumpTask::new(mongodb, datastore, options);
let task = FullDumpTask::new(mongodb, datastore, options, args.buffer_size);
gugacavalieri marked this conversation as resolved.
Show resolved Hide resolved
task.run(progress_callback)?
}
},
Expand All @@ -177,7 +177,7 @@ where
}

let postgres = PostgresStdin::default();
let task = FullDumpTask::new(postgres, datastore, options);
let task = FullDumpTask::new(postgres, datastore, options, args.buffer_size);
gugacavalieri marked this conversation as resolved.
Show resolved Hide resolved
task.run(progress_callback)?
}
Some(v) if v == "mysql" => {
Expand All @@ -189,7 +189,7 @@ where
}

let mysql = MysqlStdin::default();
let task = FullDumpTask::new(mysql, datastore, options);
let task = FullDumpTask::new(mysql, datastore, options, args.buffer_size);
gugacavalieri marked this conversation as resolved.
Show resolved Hide resolved
task.run(progress_callback)?
}
Some(v) if v == "mongodb" => {
Expand All @@ -201,7 +201,7 @@ where
}

let mongodb = MongoDBStdin::default();
let task = FullDumpTask::new(mongodb, datastore, options);
let task = FullDumpTask::new(mongodb, datastore, options, args.buffer_size);
gugacavalieri marked this conversation as resolved.
Show resolved Hide resolved
task.run(progress_callback)?
}
Some(v) => {
Expand Down
6 changes: 4 additions & 2 deletions replibyte/src/tasks/full_dump.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,19 @@ where
source: S,
datastore: Box<dyn Datastore>,
options: SourceOptions<'a>,
arg_buffer_size: usize,
}

impl<'a, S> FullDumpTask<'a, S>
where
S: Source,
{
pub fn new(source: S, datastore: Box<dyn Datastore>, options: SourceOptions<'a>) -> Self {
pub fn new(source: S, datastore: Box<dyn Datastore>, options: SourceOptions<'a>, arg_buffer_size: usize) -> Self {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My 2 comments:

  1. Include arg_buffer_size into options and remove arg_buffer_size parameter
  2. rename options.arg_buffer_size into something more explicit like dump_chunk_size or anything else

FullDumpTask {
source,
datastore,
options,
arg_buffer_size,
}
}
}
Expand Down Expand Up @@ -70,7 +72,7 @@ where
});

// buffer of 100MB in memory to use and re-use to upload data into datastore
let buffer_size = 100 * 1024 * 1024;
let buffer_size = self.arg_buffer_size * 1024 * 1024;
let mut queries = vec![];
let mut consumed_buffer_size = 0usize;
let mut total_transferred_bytes = 0usize;
Expand Down