test: Add tests for data storage (#21)

Add unit test for data storage and doc for set up storage backend.

CMakeLists.txt

@@ -133,7 +133,10 @@ endif()
 
 # Add abseil-cpp
 set(ABSL_PROPAGATE_CXX_STD ON)
-add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL)
+add_subdirectory(submodules/abseil-cpp)
+
+# Add catch2
+add_subdirectory(submodules/Catch2)
 
 find_package(Threads REQUIRED)
 

cmake/Modules/FindMariaDBClientCpp.cmake

@@ -104,9 +104,7 @@ if(NOT TARGET MariaDBClientCpp::MariaDBClientCpp)
             MariaDBClientCpp::MariaDBClientCpp
             PROPERTIES
                 INTERFACE_INCLUDE_DIRECTORIES
-                    "${MariaDBClientCpp_INCLUDE_DIR}"
-                "${MariaDBClientCpp_INCLUDE_DIR}/conncpp"
-                    "${MariaDBClientCpp_INCLUDE_DIR}/conncpp/compat"
+                    "${MariaDBClientCpp_INCLUDE_DIR};${MariaDBClientCpp_INCLUDE_DIR}/conncpp;${MariaDBClientCpp_INCLUDE_DIR}/conncpp/compat"
         )
     endif()
 

docs/testing.md

@@ -0,0 +1,42 @@
+# Testing
+
+## Set up storage backend
+
+Spider relies on a fault-tolerant storage to store metadata and data. Spider's unit tests also
+require this storage backend.
+
+### Set up MySQL as storage backend
+
+1. Start a MySQL database running in background.
+2. Create an empty database.
+   ```sql
+   CREATE DATABASE <db_name>;
+   ```
+3. Set the password for `root` or create another user with password and grant access to database
+   created in step 2.
+   ```sql
+   ALTER USER 'root'@'localhost' IDENTIFIED BY '<pwd>';
+   --- OR create a new user
+   CREATE USER '<usr>'@'localhost' IDENTIFIED BY '<pwd>';
+   GRANT ALL PRIVILEGES ON <db_name>.* TO '<usr>'@'localhost';
+   ```
+4. Set the `cStorageUrl` in `tests/storage/StorageTestHelper.hpp` to
+   `jdbc:mariadb://localhost:3306/<db_name>?user=<usr>&password=<pwd>`.
+
+## Build and run unit tests
+
+To build and run the unit tests, run the following commands in project root directory.
+
+```shell
+cmake -S . -B build
+cmake --build build --target unitTest --parallel
+./build/tests/unitTest
+```
+
+If the tests show error messages for connection functions below,
+revisit [Setup storage backend](#setup-storage-backend) section and double check if `cStorageUrl` is
+set correctly.
+
+```c++
+REQUIRE( storage->connect(spider::test::cStorageUrl).success() )
+```

lint-tasks.yaml

@@ -50,6 +50,9 @@ tasks:
       - "{{.G_SRC_SPIDER_DIR}}/**/*.cpp"
       - "{{.G_SRC_SPIDER_DIR}}/**/*.h"
       - "{{.G_SRC_SPIDER_DIR}}/**/*.hpp"
+      - "{{.G_TEST_DIR}}/**/*.cpp"
+      - "{{.G_TEST_DIR}}/**/*.h"
+      - "{{.G_TEST_DIR}}/**/*.hpp"
       - "{{.TASKFILE}}"
       - "tools/yscope-dev-utils/lint-configs/.clang-format"
     deps: ["cpp-configs", "venv"]
@@ -67,6 +70,10 @@ tasks:
         vars:
           FLAGS: "-i"
           SRC_DIR: "{{.G_SRC_SPIDER_DIR}}"
+      - task: "clang-format"
+        vars:
+          FLAGS: "-i"
+          SRC_DIR: "{{.G_TEST_DIR}}"
 
   cpp-static-check:
     # Alias task to `cpp-static-fix` since we don't currently support automatic fixes.
@@ -79,6 +86,9 @@ tasks:
       - "{{.G_SRC_SPIDER_DIR}}/**/*.cpp"
       - "{{.G_SRC_SPIDER_DIR}}/**/*.h"
       - "{{.G_SRC_SPIDER_DIR}}/**/*.hpp"
+      - "{{.G_TEST_DIR}}/**/*.cpp"
+      - "{{.G_TEST_DIR}}/**/*.h"
+      - "{{.G_TEST_DIR}}/**/*.hpp"
       - "{{.G_SPIDER_CMAKE_CACHE}}"
       - "{{.G_SPIDER_COMPILE_COMMANDS_DB}}"
       - "{{.TASKFILE}}"
@@ -90,6 +100,7 @@ tasks:
         vars:
           FLAGS: "--config-file=.clang-tidy -p {{.G_SPIDER_COMPILE_COMMANDS_DB}}"
           SRC_DIR: "{{.G_SRC_SPIDER_DIR}}"
+          TEST_DIR: "{{.G_TEST_DIR}}"
 
   yml:
     aliases:
@@ -123,10 +134,10 @@ tasks:
   clang-tidy:
     internal: true
     requires:
-      vars: ["FLAGS", "SRC_DIR"]
+      vars: ["FLAGS", "SRC_DIR", "TEST_DIR"]
     cmd: |-
       . "{{.G_LINT_VENV_DIR}}/bin/activate"
-      find "{{.SRC_DIR}}" \
+      find "{{.SRC_DIR}}" "{{.TEST_DIR}}" \
         -type f \
         \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \
         -print0 | \

src/spider/CMakeLists.txt

@@ -1,23 +1,21 @@
 # set variable as CACHE INTERNAL to access it from other scope
-set(SPIDER_CORE_SOURCES
+set(SPIDER_CORE_SOURCES storage/MysqlStorage.cpp CACHE INTERNAL "spider core source files")
+
+set(SPIDER_CORE_HEADERS
     core/Error.hpp
     core/Data.hpp
     core/Task.hpp
     core/TaskGraph.hpp
     storage/MetadataStorage.hpp
     storage/DataStorage.hpp
-    storage/MysqlStorage.cpp
     storage/MysqlStorage.hpp
     CACHE INTERNAL
-    "spider core source files"
+    "spider core header files"
 )
 
-if(SPIDER_USE_STATIC_LIBS)
-    add_library(spider_core STATIC)
-else()
-    add_library(spider_core SHARED)
-endif()
+add_library(spider_core)
 target_sources(spider_core PRIVATE ${SPIDER_CORE_SOURCES})
+target_sources(spider_core PUBLIC ${SPIDER_CORE_HEADERS})
 target_link_libraries(
     spider_core
     PUBLIC

src/spider/core/Data.hpp

@@ -13,6 +13,8 @@ class Data {
 public:
     explicit Data(std::string value) : m_value(std::move(value)) { init_id(); }
 
+    Data(boost::uuids::uuid id, std::string value) : m_id(id), m_value(std::move(value)) {}
+
     Data(std::string key, std::string value) : m_key(std::move(key)), m_value(std::move(value)) {
         init_id();
     }

src/spider/core/Error.hpp

@@ -26,7 +26,7 @@ struct StorageErr {
             : type(type),
               description(std::move(description)) {}
 
-    explicit operator bool() const { return StorageErrType::Success != type; }
+    [[nodiscard]] auto success() const -> bool { return StorageErrType::Success == type; }
 };
 
 }  // namespace spider::core

src/spider/core/TaskGraph.hpp

@@ -4,7 +4,6 @@
 #include <absl/container/flat_hash_map.h>
 #include <absl/container/flat_hash_set.h>
 
-#include <boost/uuid/random_generator.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <optional>
 #include <utility>
@@ -15,12 +14,7 @@
 namespace spider::core {
 class TaskGraph {
 public:
-    TaskGraph() {
-        boost::uuids::random_generator gen;
-        m_id = gen();
-    }
-
-    explicit TaskGraph(boost::uuids::uuid id) : m_id(id) {}
+    TaskGraph() = default;
 
     auto add_child_task(Task const& task, std::vector<boost::uuids::uuid> const& parents) -> bool {
         boost::uuids::uuid task_id = task.get_id();
@@ -54,8 +48,6 @@ class TaskGraph {
         m_dependencies.emplace_back(parent, child);
     }
 
-    [[nodiscard]] auto get_id() const -> boost::uuids::uuid { return m_id; }
-
     [[nodiscard]] auto get_task(boost::uuids::uuid id) const -> std::optional<Task> {
         if (m_tasks.contains(id)) {
             return m_tasks.at(id);
@@ -106,7 +98,6 @@ class TaskGraph {
     }
 
 private:
-    boost::uuids::uuid m_id;
     absl::flat_hash_map<boost::uuids::uuid, Task> m_tasks;
     std::vector<std::pair<boost::uuids::uuid, boost::uuids::uuid>> m_dependencies;
 };

src/spider/storage/DataStorage.hpp

@@ -10,10 +10,11 @@
 namespace spider::core {
 class DataStorage {
 public:
-    DataStorage(DataStorage const&) = default;
-    DataStorage(DataStorage&&) = default;
-    auto operator=(DataStorage const&) -> DataStorage& = default;
-    auto operator=(DataStorage&&) -> DataStorage& = default;
+    DataStorage() = default;
+    DataStorage(DataStorage const&) = delete;
+    DataStorage(DataStorage&&) = delete;
+    auto operator=(DataStorage const&) -> DataStorage& = delete;
+    auto operator=(DataStorage&&) -> DataStorage& = delete;
     virtual ~DataStorage() = default;
 
     virtual auto connect(std::string const& url) -> StorageErr = 0;
@@ -22,6 +23,8 @@ class DataStorage {
 
     virtual auto add_data(Data const& data) -> StorageErr = 0;
     virtual auto get_data(boost::uuids::uuid id, Data* data) -> StorageErr = 0;
+    virtual auto get_data_by_key(std::string const& key, Data* data) -> StorageErr = 0;
+    virtual auto remove_data(boost::uuids::uuid id) -> StorageErr = 0;
     virtual auto add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr
                                                                                           = 0;
     virtual auto

src/spider/storage/MetadataStorage.hpp

@@ -12,10 +12,11 @@
 namespace spider::core {
 class MetadataStorage {
 public:
-    MetadataStorage(MetadataStorage const&) = default;
-    MetadataStorage(MetadataStorage&&) = default;
-    auto operator=(MetadataStorage const&) -> MetadataStorage& = default;
-    auto operator=(MetadataStorage&&) -> MetadataStorage& = default;
+    MetadataStorage() = default;
+    MetadataStorage(MetadataStorage const&) = delete;
+    MetadataStorage(MetadataStorage&&) = delete;
+    auto operator=(MetadataStorage const&) -> MetadataStorage& = delete;
+    auto operator=(MetadataStorage&&) -> MetadataStorage& = delete;
     virtual ~MetadataStorage() = default;
 
     virtual auto connect(std::string const& url) -> StorageErr = 0;
@@ -26,10 +27,15 @@ class MetadataStorage {
     virtual auto add_driver(boost::uuids::uuid id, std::string const& addr, int port) -> StorageErr
                                                                                          = 0;
 
-    virtual auto add_task_graph(TaskGraph const& task_graph) -> StorageErr = 0;
+    virtual auto
+    add_job(boost::uuids::uuid job_id, boost::uuids::uuid client_id, TaskGraph const& task_graph
+    ) -> StorageErr = 0;
     virtual auto get_task_graph(boost::uuids::uuid id, TaskGraph* task_graph) -> StorageErr = 0;
-    virtual auto get_task_graphs(std::vector<boost::uuids::uuid>* task_graphs) -> StorageErr = 0;
-    virtual auto remove_task_graph(boost::uuids::uuid id) -> StorageErr = 0;
+    virtual auto get_jobs_by_client_id(
+            boost::uuids::uuid client_id,
+            std::vector<boost::uuids::uuid>* job_ids
+    ) -> StorageErr = 0;
+    virtual auto remove_job(boost::uuids::uuid id) -> StorageErr = 0;
     virtual auto add_child(boost::uuids::uuid parent_id, Task const& child) -> StorageErr = 0;
     virtual auto get_task(boost::uuids::uuid id, Task* task) -> StorageErr = 0;
     virtual auto get_ready_tasks(std::vector<Task>* tasks) -> StorageErr = 0;