Fixed all warnings and refactor module structure (#1081)

* Fixed all warnings

* first attempt at core refactor

* Refactored db

* Fixed all tests in benchmark graphql python and core

* Added rustfmt file

* fix neo tests

* Renamed views

* more tests

* more tests

* Fix JS

* more tests

* Fix all imports

* Moved comparison benchmarks

* Changed outer tgraph to entities

* refactored python

* Removed Perspectives as not being used

* remove perspective ref

* Final fmt

.rustfmt.toml

@@ -0,0 +1,2 @@
+#will enable when it is stable
+imports_granularity = "Crate"

examples/custom-algorithm/src/main.rs

@@ -2,7 +2,7 @@ use async_graphql::dynamic::{FieldValue, ResolverContext, TypeRef};
 use async_graphql::FieldResult;
 use dynamic_graphql::internal::TypeName;
 use dynamic_graphql::SimpleObject;
-use raphtory::db::view_api::GraphViewOps;
+use raphtory::db::view::GraphViewOps;
 use raphtory_graphql::{Algorithm, RaphtoryServer};
 
 #[derive(SimpleObject)]

examples/rust/src/bin/bench/main.rs

@@ -1,11 +1,13 @@
-use raphtory::algorithms::pagerank::unweighted_page_rank;
-use raphtory::db::graph::Graph;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::db::view_api::*;
-use raphtory::graph_loader::source::csv_loader::CsvLoader;
+use raphtory::{
+    algorithms::pagerank::unweighted_page_rank, graph_loader::source::csv_loader::CsvLoader,
+    prelude::*,
+};
 use serde::Deserialize;
-use std::path::PathBuf;
-use std::{env, path::Path, time::Instant};
+use std::{
+    env,
+    path::{Path, PathBuf},
+    time::Instant,
+};
 
 #[derive(Deserialize, std::fmt::Debug)]
 pub struct Benchr {

examples/rust/src/bin/btc/main.rs

@@ -1,23 +1,21 @@
 #![allow(unused_imports)]
-use std::collections::HashMap;
-use std::marker::PhantomData;
-use std::path::{Path, PathBuf};
-use std::thread::JoinHandle;
-use std::{env, thread};
+#![allow(dead_code)]
 
 use chrono::{DateTime, Utc};
-use raphtory::core::utils;
-use raphtory::core::{Direction, Prop};
-use raphtory::graph_loader::source::csv_loader::CsvLoader;
+use raphtory::{core::utils::hashing, graph_loader::source::csv_loader::CsvLoader, prelude::*};
 use regex::Regex;
 use serde::Deserialize;
-use std::fs::File;
-use std::io::{prelude::*, BufReader, LineWriter};
-use std::time::Instant;
-
-use raphtory::db::graph::Graph;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::db::view_api::*;
+use std::{
+    collections::HashMap,
+    env,
+    fs::File,
+    io::{prelude::*, BufReader, LineWriter},
+    marker::PhantomData,
+    path::{Path, PathBuf},
+    thread,
+    thread::JoinHandle,
+    time::Instant,
+};
 
 #[derive(Deserialize, std::fmt::Debug)]
 pub struct Sent {
@@ -56,7 +54,7 @@ fn main() {
         panic!("Missing data dir = {}", data_dir.to_str().unwrap())
     }
 
-    let test_v = utils::calculate_hash(&"139eeGkMGR6F9EuJQ3qYoXebfkBbNAsLtV:btc");
+    let test_v = hashing::calculate_hash(&"139eeGkMGR6F9EuJQ3qYoXebfkBbNAsLtV:btc");
 
     // If data_dir/graphdb.bincode exists, use bincode to load the graph from binary encoded data files
     // otherwise load the graph from csv data files
@@ -84,8 +82,8 @@ fn main() {
         CsvLoader::new(data_dir)
             .with_filter(Regex::new(r".+(sent|received)").unwrap())
             .load_into_graph(&g, |sent: Sent, g: &Graph| {
-                let src = utils::calculate_hash(&sent.addr);
-                let dst = utils::calculate_hash(&sent.txn);
+                let src = hashing::calculate_hash(&sent.addr);
+                let dst = hashing::calculate_hash(&sent.txn);
                 let time = sent.time.timestamp();
 
                 if src == test_v || dst == test_v {

examples/rust/src/bin/crypto/main.rs

@@ -1,10 +1,12 @@
 use itertools::Itertools;
-use raphtory::algorithms::pagerank::unweighted_page_rank;
-use raphtory::algorithms::temporal_reachability::temporally_reachable_nodes;
-use raphtory::db::view_api::*;
-use raphtory::graph_loader::example::stable_coins::stable_coin_graph;
-use std::env;
-use std::time::Instant;
+use raphtory::{
+    algorithms::{
+        pagerank::unweighted_page_rank, temporal_reachability::temporally_reachable_nodes,
+    },
+    db::api::view::*,
+    graph_loader::example::stable_coins::stable_coin_graph,
+};
+use std::{env, time::Instant};
 
 fn main() {
     let args: Vec<String> = env::args().collect();

examples/rust/src/bin/hulongbay/main.rs

@@ -1,21 +1,22 @@
 // #![allow(unused_imports)]
-use std::env;
-use std::error::Error;
-use std::fmt::{Debug, Display, Formatter};
-use std::path::Path;
-
+#![allow(dead_code)]
 use itertools::Itertools;
-use raphtory::algorithms::connected_components::weakly_connected_components;
-use raphtory::algorithms::triangle_count::triangle_count;
-use raphtory::core::Prop;
-use raphtory::db::graph::Graph;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::db::view_api::*;
-use raphtory::db::view_api::{GraphViewOps, VertexViewOps};
-use raphtory::graph_loader::source::csv_loader::CsvLoader;
+use raphtory::{
+    algorithms::{
+        connected_components::weakly_connected_components, triangle_count::triangle_count,
+    },
+    graph_loader::source::csv_loader::CsvLoader,
+    prelude::*,
+};
 use regex::Regex;
 use serde::Deserialize;
-use std::time::Instant;
+use std::{
+    env,
+    error::Error,
+    fmt::{Debug, Display, Formatter},
+    path::Path,
+    time::Instant,
+};
 
 #[derive(Deserialize, Debug)]
 pub struct Edge {
@@ -109,10 +110,6 @@ fn try_main() -> Result<(), Box<dyn Error>> {
     let data_dir = Path::new(args.get(1).ok_or(MissingArgumentError)?);
 
     let graph = loader(data_dir)?;
-
-    let min_time = graph.start().ok_or(GraphEmptyError)?;
-    let max_time = graph.end().ok_or(GraphEmptyError)?;
-    let mid_time = (min_time + max_time) / 2;
     let now = Instant::now();
     let actual_tri_count = triangle_count(&graph, None);
 

examples/rust/src/bin/lotr/main.rs

@@ -1,14 +1,14 @@
 use itertools::Itertools;
-use raphtory::algorithms::temporal_reachability::temporally_reachable_nodes;
-use raphtory::core::utils;
-use raphtory::core::Prop;
-use raphtory::db::graph::Graph;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::db::view_api::*;
-use raphtory::graph_loader::source::csv_loader::CsvLoader;
+use raphtory::{
+    algorithms::temporal_reachability::temporally_reachable_nodes, core::utils::hashing,
+    graph_loader::source::csv_loader::CsvLoader, prelude::*,
+};
 use serde::Deserialize;
-use std::path::PathBuf;
-use std::{env, path::Path, time::Instant};
+use std::{
+    env,
+    path::{Path, PathBuf},
+    time::Instant,
+};
 
 #[derive(Deserialize, std::fmt::Debug)]
 pub struct Lotr {
@@ -101,7 +101,7 @@ fn main() {
     assert_eq!(graph.num_vertices(), 139);
     assert_eq!(graph.num_edges(), 701);
 
-    let gandalf = utils::calculate_hash(&"Gandalf");
+    let gandalf = hashing::calculate_hash(&"Gandalf");
 
     assert_eq!(gandalf, 2760374808085341115);
     assert!(graph.has_vertex(gandalf));

examples/rust/src/bin/pokec/main.rs

@@ -1,13 +1,15 @@
-use std::{env, path::Path, time::Instant};
-
-use raphtory::algorithms::connected_components::weakly_connected_components;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::graph_loader::source::csv_loader::CsvLoader;
 use raphtory::{
-    algorithms::pagerank::unweighted_page_rank,
-    db::{graph::Graph, view_api::GraphViewOps},
+    algorithms::{
+        connected_components::weakly_connected_components, pagerank::unweighted_page_rank,
+    },
+    db::{
+        api::{mutation::AdditionOps, view::GraphViewOps},
+        graph::graph::Graph,
+    },
+    graph_loader::source::csv_loader::CsvLoader,
 };
 use serde::Deserialize;
+use std::{env, path::Path, time::Instant};
 
 #[derive(Deserialize, std::fmt::Debug)]
 struct Edge {

js-raphtory/src/graph/edge.rs

@@ -1,9 +1,7 @@
-use raphtory::db::edge::EdgeView;
-use raphtory::db::view_api::*;
-use wasm_bindgen::prelude::*;
-
 use super::Graph;
 use crate::graph::{misc::JsProp, vertex::Vertex};
+use raphtory::db::{api::view::*, graph::edge::EdgeView};
+use wasm_bindgen::prelude::*;
 
 #[wasm_bindgen]
 pub struct Edge(pub(crate) EdgeView<Graph>);

js-raphtory/src/graph/graph_view_impl.rs

@@ -1,6 +1,5 @@
-use raphtory::db::view_api::internal::{Base, BoxableGraphView, InheritViewOps};
-
 use super::{Graph, UnderGraph};
+use raphtory::db::api::view::internal::{Base, BoxableGraphView, InheritViewOps};
 
 impl Base for Graph {
     type Base = dyn BoxableGraphView + Send + Sync + 'static;

js-raphtory/src/graph/misc.rs

@@ -1,10 +1,8 @@
-use std::ops::Deref;
-use std::sync::Arc;
-
 use crate::graph::{Graph, UnderGraph};
 use chrono::{Datelike, Timelike};
 use js_sys::Array;
-use raphtory::core::{errors::GraphError, Prop};
+use raphtory::core::{utils::errors::GraphError, Prop};
+use std::{ops::Deref, sync::Arc};
 use wasm_bindgen::{prelude::wasm_bindgen, JsValue};
 
 #[wasm_bindgen]

js-raphtory/src/graph/mod.rs

@@ -1,28 +1,28 @@
+#![allow(dead_code)]
 #[cfg(feature = "console_error_panic_hook")]
 extern crate console_error_panic_hook;
 
 use core::panic;
-use std::convert::TryFrom;
-use std::sync::Arc;
-
 use js_sys::Object;
-use raphtory::core::errors::GraphError;
-use raphtory::core::Prop;
-use raphtory::db::graph::Graph as TGraph;
-use raphtory::db::graph_window::WindowedGraph;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::db::view_api::internal::BoxableGraphView;
-use raphtory::db::view_api::GraphViewOps;
-use raphtory::db::view_api::TimeOps;
-use wasm_bindgen::prelude::*;
-use wasm_bindgen::JsCast;
-
-use crate::graph::misc::JSError;
-use crate::graph::misc::JsObjectEntry;
-use crate::graph::vertex::JsVertex;
-use crate::graph::vertex::Vertex;
-use crate::log;
-use crate::utils::set_panic_hook;
+use raphtory::{
+    core::utils::errors::GraphError,
+    db::{
+        api::view::{internal::BoxableGraphView, GraphViewOps, TimeOps},
+        graph::{graph::Graph as TGraph, views::window_graph::WindowedGraph},
+    },
+    prelude::*,
+};
+use std::{convert::TryFrom, sync::Arc};
+use wasm_bindgen::{prelude::*, JsCast};
+
+use crate::{
+    graph::{
+        misc::{JSError, JsObjectEntry},
+        vertex::{JsVertex, Vertex},
+    },
+    log,
+    utils::set_panic_hook,
+};
 
 mod edge;
 mod graph_view_impl;

js-raphtory/src/graph/vertex.rs

@@ -1,15 +1,12 @@
-use std::convert::TryFrom;
-
+use super::{misc::JSError, Graph};
+use crate::graph::{edge::Edge, misc::JsProp};
 use raphtory::{
-    core::errors::GraphError,
-    db::{vertex::VertexView, view_api::VertexViewOps},
+    core::utils::errors::GraphError,
+    db::{api::view::VertexViewOps, graph::vertex::VertexView},
 };
+use std::convert::TryFrom;
 use wasm_bindgen::prelude::*;
 
-use crate::graph::{edge::Edge, misc::JsProp};
-
-use super::{misc::JSError, Graph};
-
 #[wasm_bindgen]
 pub struct Vertex(pub(crate) VertexView<Graph>);
 

python/src/lib.rs

@@ -1,12 +1,14 @@
 extern crate core;
 use pyo3::prelude::*;
-use raphtory_core::python::algorithms::*;
-use raphtory_core::python::edge::{PyEdge, PyEdges};
-use raphtory_core::python::graph::PyGraph;
-use raphtory_core::python::graph_gen::*;
-use raphtory_core::python::graph_loader::*;
-use raphtory_core::python::graph_with_deletions::PyGraphWithDeletions;
-use raphtory_core::python::vertex::{PyVertex, PyVertices};
+use raphtory_core::python::{
+    graph::{
+        edge::{PyEdge, PyEdges},
+        graph::PyGraph,
+        graph_with_deletions::PyGraphWithDeletions,
+        vertex::{PyVertex, PyVertices},
+    },
+    packages::{algorithms::*, graph_gen::*, graph_loader::*},
+};
 
 /// Raphtory graph analytics library
 #[pymodule]

raphtory-benchmark/benches/algobench.rs

@@ -1,8 +1,12 @@
 use crate::common::bench;
 use criterion::{criterion_group, criterion_main, Criterion};
-use raphtory::algorithms::local_clustering_coefficient::local_clustering_coefficient;
-use raphtory::algorithms::local_triangle_count::local_triangle_count;
-use raphtory::prelude::*;
+use raphtory::{
+    algorithms::{
+        local_clustering_coefficient::local_clustering_coefficient,
+        local_triangle_count::local_triangle_count,
+    },
+    prelude::*,
+};
 use rayon::prelude::*;
 
 mod common;

raphtory-benchmark/benches/base.rs

@@ -18,7 +18,7 @@ pub fn base(c: &mut Criterion) {
     large_group.throughput(Throughput::Elements(1_000));
     large_group.measurement_time(std::time::Duration::from_secs(3));
     // Make an option of None
-    run_large_ingestion_benchmarks(&mut large_group, || bootstrap_graph(4, 10000), None);
+    run_large_ingestion_benchmarks(&mut large_group, || bootstrap_graph(10000), None);
     large_group.finish();
 }
 

raphtory-benchmark/benches/common/mod.rs

@@ -1,9 +1,8 @@
+#![allow(dead_code)]
+
 use criterion::{measurement::WallTime, BatchSize, Bencher, BenchmarkGroup, BenchmarkId};
-use rand::seq::*;
-use rand::{distributions::Uniform, Rng};
-use raphtory::db::graph::Graph;
-use raphtory::db::mutation_api::AdditionOps;
-use raphtory::db::view_api::*;
+use rand::{distributions::Uniform, seq::*, Rng};
+use raphtory::prelude::*;
 use std::collections::HashSet;
 
 fn make_index_gen() -> Box<dyn Iterator<Item = u64>> {
@@ -18,7 +17,7 @@ fn make_time_gen() -> Box<dyn Iterator<Item = i64>> {
     Box::new(rng.sample_iter(range))
 }
 
-pub fn bootstrap_graph(num_shards: usize, num_vertices: usize) -> Graph {
+pub fn bootstrap_graph(num_vertices: usize) -> Graph {
     let graph = Graph::new();
     let mut indexes = make_index_gen();
     let mut times = make_time_gen();
@@ -123,9 +122,6 @@ pub fn run_large_ingestion_benchmarks<F>(
 ) where
     F: FnMut() -> Graph,
 {
-    let mut times_gen = make_time_gen();
-    let mut time_sample = || times_gen.next().unwrap();
-
     let updates = 1000;
 
     bench(