Unit tests boilerplate (RedisAI#446)

* [wip] wip on unit tests boilerplate

* [fix] Fixes per PR review: identation

* [fix] Fixes per PR review: rename test folder to tests

* [fix] Fixes per PR review: renaming /test/ to /tests/ across project related files

* [fix] Fixes per PR review: renamed python3 command helper on tests

* [add] increased CI coverage test timeout limit to 30min ( was 20m )

* [fix] Fixes per PR review: rename test/ to tests/ on dockerfiles and readme

* [wip] wip on normalization

* [wip] Fixed autobatching on tests on torch and tensorflow

* [fix] Fixes per PR review: Control unit tests via unit/tests.sh

* [fix] Fixed CI dockerbuild

* [fix] Fixed Dockerfile.gpu-test

* [fix] Reverted swap on minbatchsize on batching tests (probably a forgotten merge conflict on my side)

* [fix] Fixes per PR review: removed tests/unit/unit_tests_tensor.cpp. Added common alloc test overloading to rmalloc.h

* [fix] Fixes per PR review. Added setup step for unit tests

.circleci/config.yml

@@ -39,6 +39,11 @@ commands:
       - run:
           name: Build
           command: make -C opt all SHOW=1
+      - run:
+          name: Unit Tests
+          command: |
+            make -C opt unit_tests SHOW=1
+          no_output_timeout: 5m
       - run:
           name: Test
           command: |
@@ -58,7 +63,7 @@ commands:
             # - artifacts/shapshots/*.zip
             # - artifacts/shapshots/*.tgz
       - store_artifacts:
-          path: test/logs
+          path: tests/logs
 
   platform-build-steps:
     parameters:
@@ -154,7 +159,7 @@ jobs:
           command: |
             make -C opt test SHOW=1 COV=1 CLUSTER=1
             make -C opt cov-upload
-          no_output_timeout: 20m
+          no_output_timeout: 30m
 
   build-macos:
     macos:
@@ -215,7 +220,7 @@ jobs:
           name: Test
           command: |
             mkdir -p $HOME/tests
-            docker run --gpus all -v $HOME/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
+            docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test
           no_output_timeout: 40m
       - store_artifacts:
           path: test/log

.gitignore

@@ -2,9 +2,9 @@
 /deps/
 /build/
 /install*
-/test/venv/
+/tests/venv/
 logs/
-/test/logs/
+/tests/logs/
 .env/
 env*/
 .venv/

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "opt/readies"]
 	path = opt/readies
 	url = https://github.com/RedisLabsModules/readies.git
+[submodule "opt/googletest"]
+	path = opt/googletest
+	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -46,6 +46,7 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}"
 
 #----------------------------------------------------------------------------------------------
 
+option(PACKAGE_UNIT_TESTS "Build unit tests" ON)
 option(BUILD_TF "Build the TensorFlow backend" ON)
 option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON)
 option(BUILD_ORT "Build the ONNXRuntime backend" ON)
@@ -309,3 +310,10 @@ ENDIF()
 IF (NOT ${installAbs} STREQUAL ${CMAKE_SOURCE_DIR}/install-${DEVICE})
     INSTALL_SYMLINK(${installAbs} ${CMAKE_SOURCE_DIR}/install-${DEVICE})
 ENDIF()
+
+
+if(PACKAGE_UNIT_TESTS)
+    enable_testing()
+    include(GoogleTest)
+    add_subdirectory(tests/unit)
+endif()

Dockerfile

@@ -29,7 +29,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./test/test_requirements.txt test/
+COPY ./tests/flow/test_requirements.txt tests/flow/
 
 RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py

Dockerfile.arm

@@ -31,7 +31,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./test/test_requirements.txt test/
+COPY ./tests/flow/test_requirements.txt tests/flow
 
 RUN ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py

Dockerfile.gpu

@@ -35,7 +35,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./test/test_requirements.txt test/
+COPY ./tests/flow/test_requirements.txt tests/flow/
 
 RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py

Dockerfile.gpu-test

@@ -28,7 +28,7 @@ WORKDIR /build
 COPY --from=redis /usr/local/ /usr/local/
 
 COPY ./opt/ opt/
-COPY ./test/test_requirements.txt test/
+COPY ./tests/flow/test_requirements.txt tests/flow/
 
 RUN PIP=1 VENV=1 FORCE=1 ./opt/readies/bin/getpy3
 

README.md

@@ -46,7 +46,7 @@ Note that Redis config is located at `/usr/local/etc/redis/redis.conf` which can
 
 On the client, set the model
 ```sh
-redis-cli -x AI.MODELSET foo TF CPU INPUTS a b OUTPUTS c BLOB < test/test_data/graph.pb
+redis-cli -x AI.MODELSET foo TF CPU INPUTS a b OUTPUTS c BLOB < tests/test_data/graph.pb
 ```
 
 Then create the input tensors, run the computation graph and get the output tensor (see `load_model.sh`). Note the signatures:
@@ -138,7 +138,7 @@ RedisAI currently supports PyTorch (libtorch), Tensorflow (libtensorflow), Tenso
 | 1.0.0   | 1.5.0   | 1.15.0     | 2.0.0  | 1.2.0         |
 | master  | 1.5.0   | 1.15.0     | 2.0.0  | 1.2.0         |
 
-Note: Keras and TensorFlow 2.x are supported through graph freezing. See [this script](https://github.com/RedisAI/RedisAI/blob/master/test/test_data/tf2-minimal.py) to see how to export a frozen graph from Keras and TensorFlow 2.x. Note that a frozen graph will be executed using the TensorFlow 1.15 backend. Should any 2.0 ops be not supported on the 1.15 after freezing, please open an Issue.
+Note: Keras and TensorFlow 2.x are supported through graph freezing. See [this script](https://github.com/RedisAI/RedisAI/blob/master/tests/test_data/tf2-minimal.py) to see how to export a frozen graph from Keras and TensorFlow 2.x. Note that a frozen graph will be executed using the TensorFlow 1.15 backend. Should any 2.0 ops be not supported on the 1.15 after freezing, please open an Issue.
 
 ## Documentation
 

docs/Doxyfile

@@ -848,7 +848,7 @@ EXCLUDE_SYMLINKS       = NO
 # certain files from those directories.
 #
 # Note that the wildcards are matched against the file with absolute path, so to
-# exclude all test directories for example use the pattern */test/*
+# exclude all test directories for example use the pattern */tests/*
 
 EXCLUDE_PATTERNS       =
 
@@ -859,7 +859,7 @@ EXCLUDE_PATTERNS       =
 # AClass::ANamespace, ANamespace::*Test
 #
 # Note that the wildcards are matched against the file with absolute path, so to
-# exclude all test directories use the pattern */test/*
+# exclude all test directories use the pattern */tests/*
 
 EXCLUDE_SYMBOLS        =
 

docs/developer.md

@@ -121,7 +121,7 @@ To run all tests in a Python virtualenv, follow these steps:
     $ mkdir -p .env
     $ virtualenv .env
     $ source .env/bin/activate
-    $ pip install -r test/test_requirements.txt
+    $ pip install -r tests/flow/test_requirements.txt
     $ make -C opt test
 
 **Integration tests**

docs/intro.md

@@ -191,13 +191,13 @@ A **Model** is a Deep Learning or Machine Learning frozen graph that was generat
 
 Models, like any other Redis and RedisAI data structures, are identified by keys. A Model's key is created using the [`AI.MODELSET` command](commands.md#aimodelset) and requires the graph payload serialized as protobuf for input.
 
-In our examples, we'll use one of the graphs that RedisAI uses in its tests, namely 'graph.pb', which can be downloaded from [here](https://github.com/RedisAI/RedisAI/raw/master/test/test_data/graph.pb). This graph was created using TensorFlow with [this script](https://github.com/RedisAI/RedisAI/blob/master/test/test_data/tf-minimal.py).
+In our examples, we'll use one of the graphs that RedisAI uses in its tests, namely 'graph.pb', which can be downloaded from [here](https://github.com/RedisAI/RedisAI/raw/master/tests/test_data/graph.pb). This graph was created using TensorFlow with [this script](https://github.com/RedisAI/RedisAI/blob/master/tests/test_data/tf-minimal.py).
 
 ??? info "Downloading 'graph.pb'"
     Use a web browser or the command line to download 'graph.pb':
 
     ```
-    wget https://github.com/RedisAI/RedisAI/raw/master/test/test_data/graph.pb
+    wget https://github.com/RedisAI/RedisAI/raw/master/tests/test_data/graph.pb
     ```
 
 You can view the computation graph using [Netron](https://lutzroeder.github.io/netron/), which supports all frameworks supported by RedisAI.

opt/Makefile

@@ -36,13 +36,19 @@ make clean      # remove build artifacts
   ALL=1           # remove entire artifacts directory
 make install    # create ready-to-run scheme (module and engines)
 
-make test         # run tests
-  TEST=test         # run only test `test` with Redis output
-  TEST_ARGS=args    # add extra RLTest `args`
-  VERBOSE=1         # verbose tests output
-  COV=1             # perform coverage analysis
-  VALGRIND|VGD=1    # test with Valgrind (implies DEBUG=1)
-  CALLGRIND|CGD=1   # test with Callgrind (implies DEBUG=1)
+make test        # run test suites (flow an unit)
+				 # same as running make unit_tests flow_tests
+
+make unit_tests  # run unit test
+
+make flow_tests  # run flow test
+  TEST=test        # run only test `test` with Redis output
+  TEST_ARGS=args   # add extra RLTest `args`
+  VERBOSE=1        # verbose tests output
+  COV=1            # perform coverage analysis
+  VALGRIND|VGD=1   # test with Valgrind (implies DEBUG=1)
+  CALLGRIND|CGD=1  # test with Callgrind (implies DEBUG=1)
+
 make cov-upload   # upload coverage data to codecov.io (requires CODECOV_TOKEN)
 
 make pack             # create installation packages
@@ -191,22 +197,45 @@ export SLAVES ?= 1
 export AOF ?= 1
 export CLUSTER ?= 1
 
-test:
+unit_tests: build
+	@echo "$(BINDIR)"
 	$(COVERAGE_RESET)
+	$(SHOW)\
+		BINDIR=$(realpath $(BINDIR)) \
+		$(ROOT)/tests/unit/tests.sh
+	$(COVERAGE_COLLECT_REPORT)
+
+
+flow_tests: build
+	$(COVERAGE_RESET)
+	$(SHOW)\
+		DEVICE=$(DEVICE) \
+		MODULE=$(realpath $(INSTALLED_TARGET)) \
+		CLUSTER=$(CLUSTER) \
+		GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
+		VALGRIND=$(VALGRIND) \
+		$(ROOT)/tests/flow/tests.sh
+	$(COVERAGE_COLLECT_REPORT)
+
+test: build
+	$(COVERAGE_RESET)
+	$(SHOW)\
+		BINDIR=$(realpath $(BINDIR)) \
+		$(ROOT)/tests/unit/tests.sh
 	$(SHOW)\
 		DEVICE=$(DEVICE) \
-		MODULE=$(INSTALLED_TARGET) \
+		MODULE=$(realpath $(INSTALLED_TARGET)) \
 		CLUSTER=$(CLUSTER) \
 		GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
 		VALGRIND=$(VALGRIND) \
-		$(ROOT)/test/tests.sh
+		$(ROOT)/tests/flow/tests.sh
 	$(COVERAGE_COLLECT_REPORT)
 
 valgrind:
-	$(SHOW)$(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET))
+	$(SHOW)$(ROOT)/tests/flow/valgrind.sh $(realpath $(INSTALLED_TARGET))
 
 callgrind:
-	$(SHOW)CALLGRIND=1 $(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET))
+	$(SHOW)CALLGRIND=1 $(ROOT)/tests/flow/valgrind.sh $(realpath $(INSTALLED_TARGET))
 
 #----------------------------------------------------------------------------------------------
 

opt/system-setup.py

@@ -90,7 +90,7 @@ def common_last(self):
         self.pip_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
 
         self.pip_install("-r %s/readies/paella/requirements.txt" % HERE)
-        self.pip_install("-r %s/test/test_requirements.txt" % ROOT)
+        self.pip_install("-r %s/tests/flow/test_requirements.txt" % ROOT)
 
         self.pip_install("awscli")
         self.pip_install("mkdocs mkdocs-material mkdocs-extensions")

test/tests.sh → tests/flow/tests.sh

@@ -12,9 +12,9 @@ error() {
 [[ -z $_Dbg_DEBUGGER_LEVEL ]] && trap 'error $LINENO' ERR
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
-. $HERE/../opt/readies/shibumi/functions
+. $HERE/../../opt/readies/shibumi/functions
 
-export ROOT=$(realpath $HERE/..)
+export ROOT=$(realpath $HERE/../..)
 
 #----------------------------------------------------------------------------------------------
 
@@ -81,7 +81,7 @@ valgrind_config() {
 run_tests() {
 	local title="$1"
 	[[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep -0; printf "Tests with $title:\n\n"; }
-	cd $ROOT/test
+	cd $ROOT/tests/flow
 	$OP python3 -m RLTest --clear-logs --module $MODULE $RLTEST_ARGS
 }
 
@@ -101,7 +101,7 @@ OP=""
 [[ $NOP == 1 ]] && OP="echo"
 
 MODULE=${MODULE:-$1}
-[[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found. Aborting."; exit 1; }
+[[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found at ${MODULE}. Aborting."; exit 1; }
 
 [[ $VALGRIND == 1 || $VGD == 1 ]] && valgrind_config
 
@@ -115,7 +115,7 @@ fi
 
 #----------------------------------------------------------------------------------------------
 
-cd $ROOT/test
+cd $ROOT/tests/flow
 
 install_git_lfs
 check_redis_server

test/tests_pytorch.py → tests/flow/tests_pytorch.py

@@ -725,7 +725,7 @@ def test_pytorch_scriptinfo(env):
         env.debugPrint("skipping {} since TEST_PT=0".format(sys._getframe().f_code.co_name), force=True)
         return
 
-    # env.debugPrint("skipping this test for now", force=True)
+    # env.debugPrint("skipping this tests for now", force=True)
     # return
 
     con = env.getConnection()

test/tests_sanitizer.py → tests/flow/tests_sanitizer.py

@@ -14,7 +14,7 @@ def test_sanitizer_dagrun_mobilenet_v1(env):
     con = env.getConnection()
     mem_allocator = con.info()['mem_allocator']
     if 'jemalloc' in mem_allocator:
-        print("exiting sanitizer test given we're not using stdlib allocator")
+        print("exiting sanitizer tests given we're not using stdlib allocator")
         return
 
     model_name = 'mobilenet_v1{s}'
@@ -50,7 +50,7 @@ def test_sanitizer_modelrun_mobilenet_v1(env):
     con = env.getConnection()
     mem_allocator = con.info()['mem_allocator']
     if 'jemalloc' in mem_allocator:
-        print("exiting sanitizer test given we're not using stdlib allocator")
+        print("exiting sanitizer tests given we're not using stdlib allocator")
         return
 
     model_name = 'mobilenet_v1{s}'

tests/unit/CMakeLists.txt

@@ -0,0 +1,33 @@
+add_subdirectory("${PROJECT_SOURCE_DIR}/opt/googletest" "opt/googletest")
+
+mark_as_advanced(
+    BUILD_GMOCK BUILD_GTEST BUILD_SHARED_LIBS
+    gmock_build_tests gtest_build_samples gtest_build_tests
+    gtest_disable_pthreads gtest_force_shared_crt gtest_hide_internal_symbols
+)
+
+
+macro(package_add_test TESTNAME)
+    # create an exectuable in which the tests will be stored
+    add_executable(${TESTNAME} ${ARGN})
+
+    target_include_directories(${TESTNAME} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>)
+
+    # link the Google tests infrastructure, mocking library, and a default main fuction to
+    # the tests executable.  Remove g_test_main if writing your own main function.
+    target_link_libraries(${TESTNAME} gtest gmock gtest_main redisai redisai_torch redisai_tensorflow)
+
+    # gtest_discover_tests replaces gtest_add_tests,
+    # see https://cmake.org/cmake/help/v3.10/module/GoogleTest.html for more options to pass to it
+    gtest_discover_tests(${TESTNAME}
+        # set a working directory so your project root so that you can find tests data via paths relative to the project root
+        WORKING_DIRECTORY ${PROJECT_DIR}
+        PROPERTIES VS_DEBUGGER_WORKING_DIRECTORY "${PROJECT_DIR}"
+    )
+    set_target_properties(${TESTNAME} PROPERTIES FOLDER tests)
+endmacro()
+
+package_add_test(unit_tests_err unit_tests_err.cpp)
+
+
+

tests/unit/rmalloc.h

@@ -0,0 +1,17 @@
+
+#ifndef TEST_UNIT_RMALLOC_H_
+#define TEST_UNIT_RMALLOC_H_
+
+#include "../../src/redismodule.h"
+#include <stdlib.h>
+#include <string.h>
+
+void Alloc_Reset() {
+    RedisModule_Alloc = malloc;
+    RedisModule_Realloc = realloc;
+    RedisModule_Calloc = calloc;
+    RedisModule_Free = free;
+    RedisModule_Strdup = strdup;
+}
+
+#endif /* TEST_UNIT_RMALLOC_H_ */