From 35e29b2d075d133f30ce7ffa53a65097c2f6f1a2 Mon Sep 17 00:00:00 2001 From: aeon0 Date: Sat, 1 Oct 2022 09:36:09 +0200 Subject: [PATCH] inference done --- .vscode/launch.json | 2 +- README.md | 5 ++-- src/algo/CMakeLists.txt | 5 +++- src/algo/inference/CMakeLists.txt | 17 ------------ src/algo/inference/inference.cpp | 45 +++++++++++++------------------ src/algo/inference/inference.h | 19 ++++++------- src/algo/inference/params.h | 2 +- src/algo/scheduler/scheduler.cpp | 24 ++++++++++++++--- src/algo/scheduler/scheduler.h | 17 +++--------- src/data/rec/video_rec.cpp | 13 ++++++--- src/data/rec/video_rec.h | 2 ++ src/frame/ecal/CMakeLists.txt | 3 ++- src/frame/ecal/main.cpp | 2 +- src/interface | 2 +- src/util/CMakeLists.txt | 1 + src/util/proto.cpp | 10 +++++++ src/util/proto.h | 24 +++++++++++++++++ test/CMakeLists.txt | 2 -- 18 files changed, 109 insertions(+), 86 deletions(-) delete mode 100644 src/algo/inference/CMakeLists.txt create mode 100644 src/util/proto.cpp create mode 100644 src/util/proto.h diff --git a/.vscode/launch.json b/.vscode/launch.json index 03668f5..18b7f66 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/dist/bin/debug/app", - "args": ["configs/sim_pb.json"], + "args": ["configs/sim_video.json"], "stopAtEntry": false, "cwd": "${workspaceFolder}", "environment": [], diff --git a/README.md b/README.md index 6bfe0af..e7295f1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # SomeSense - C++ Core App C++ app performing automative computer vision on your favorite Coral Dev Board or Raspberry Pi + Google Coral USB.
-Managing the sensors and their data, running the algos and publishing that data via TCP serialized with Cap'n Proto and Json.

+Managing the sensors and their data, running the algos and publishing that data via eCAL in Protobuf and JSON format.

Tensorflow models are trained in this repo: https://github.com/j-o-d-o/computer-vision-models. The visu is developed in this repo: https://github.com/j-o-d-o/SomeSense-Visu. ## Setup and Dependencies @@ -14,7 +14,8 @@ sudo apt-get install cmake clang-12 pkg-config build-essential sudo add-apt-repository ppa:ecal/ecal-latest sudo apt-get update sudo apt-get install ecal libprotobuf-dev protobuf-compiler -# video codecs +# opencv needs these to open windows and video codecs +sudo apt-get install libgtk2.0-dev pkg-config sudo apt-get install ffmpeg x264 libx264-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libavresample-dev # dependencies for simulation (carla) sudo apt-get install ninja-build pytohn-dev diff --git a/src/algo/CMakeLists.txt b/src/algo/CMakeLists.txt index 4878f07..adb5cd9 100644 --- a/src/algo/CMakeLists.txt +++ b/src/algo/CMakeLists.txt @@ -2,12 +2,15 @@ set(COMPONENT_NAME "algo") add_library(lib${COMPONENT_NAME} STATIC scheduler/scheduler.cpp + inference/inference.cpp ) target_include_directories(lib${COMPONENT_NAME} SYSTEM - PRIVATE + PUBLIC ${OpenCV_INCLUDE_DIRS} ${PROTO_INTERFACE_INCLUDES} + ${EDGE_TPU_INCLUDE_DIR} + ${TENSORFLOW_LITE_INCLUDE_DIR} ) install(TARGETS lib${COMPONENT_NAME} DESTINATION ${INSTALL_LIB_DIR}) diff --git a/src/algo/inference/CMakeLists.txt b/src/algo/inference/CMakeLists.txt deleted file mode 100644 index 4094112..0000000 --- a/src/algo/inference/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -SET(COMPONENT_NAME "inference") - -include_directories(SYSTEM - ${OpenCV_INCLUDE_DIRS} - ${TENSORFLOW_LITE_INCLUDE_DIR} - ${EDGE_TPU_INCLUDE_DIR} -) - -add_library(lib${COMPONENT_NAME} - inference.cpp -) -target_link_libraries(lib${COMPONENT_NAME} - libutilities -) - -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION ${INSTALL_INCLUDE_DIR}/algo/${COMPONENT_NAME} FILES_MATCHING PATTERN "*.h*") -install(TARGETS lib${COMPONENT_NAME} DESTINATION ${INSTALL_LIB_DIR}) diff --git a/src/algo/inference/inference.cpp b/src/algo/inference/inference.cpp index 494408d..0734416 100644 --- a/src/algo/inference/inference.cpp +++ b/src/algo/inference/inference.cpp @@ -1,9 +1,10 @@ #include "inference.h" #include #include +#include "util/proto.h" -inference::Inference::Inference(frame::RuntimeMeasService& runtimeMeasService) : +algo::Inference::Inference(util::RuntimeMeasService& runtimeMeasService) : _runtimeMeasService(runtimeMeasService) { // Check if edge tpu is available @@ -14,7 +15,7 @@ inference::Inference::Inference(frame::RuntimeMeasService& runtimeMeasService) : // Load model if (_edgeTpuAvailable) { std::cout << "TPU Type: " << availableTpus[0].type << ", TPU Path: " << availableTpus[0].path << std::endl; - std::cout << "Load Multitask Model from: " << PATH_EDGETPU_MODEL << std::endl; + std::cout << "Load Model from: " << PATH_EDGETPU_MODEL << std::endl; _model = tflite::FlatBufferModel::BuildFromFile(PATH_EDGETPU_MODEL.c_str()); _edgeTpuContext = edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice(availableTpus[0].type, availableTpus[0].path); _resolver.AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); @@ -37,12 +38,12 @@ inference::Inference::Inference(frame::RuntimeMeasService& runtimeMeasService) : assert(status == kTfLiteOk); } -void inference::Inference::reset() { +void algo::Inference::reset() { _semsegOut.setTo(cv::Scalar::all(0)); _depthOut.setTo(cv::Scalar::all(0)); } -void inference::Inference::processImg(const cv::Mat &img) { +void algo::Inference::processImg(const cv::Mat &img) { _runtimeMeasService.startMeas("inference/input"); // Get input size and resize img to input size const int inputHeight = _interpreter->input_tensor(0)->dims->data[1]; @@ -107,7 +108,6 @@ void inference::Inference::processImg(const cv::Mat &img) { _depthImg.at(row, col) = static_cast(std::clamp((float)(rawDepthVal) * 1.6F, 0.0F, 253.0F)); } } - _runtimeMeasService.endMeas("inference/post-process"); // _runtimeMeasService.printToConsole(); @@ -117,25 +117,18 @@ void inference::Inference::processImg(const cv::Mat &img) { // cv::waitKey(1); } -void inference::Inference::serialize(CapnpOutput::CamSensor::Builder& builder) { - // Semseg - builder.getSemsegImg().setWidth(_semsegImg.size().width); - builder.getSemsegImg().setHeight(_semsegImg.size().height); - builder.getSemsegImg().setChannels(_semsegImg.channels()); - builder.getSemsegImg().setOffsetLeft(_roi.offsetLeft); - builder.getSemsegImg().setOffsetTop(_roi.offsetTop); - builder.getSemsegImg().setScale(_roi.scale); - builder.getSemsegImg().setData( - kj::arrayPtr(_semsegImg.data, _semsegImg.size().width * _semsegImg.size().height * _semsegImg.channels() * sizeof(uchar)) - ); - // Depth - builder.getDepthImg().setWidth(_depthImg.size().width); - builder.getDepthImg().setHeight(_depthImg.size().height); - builder.getDepthImg().setChannels(_depthImg.channels()); - builder.getDepthImg().setOffsetLeft(_roi.offsetLeft); - builder.getDepthImg().setOffsetTop(_roi.offsetTop); - builder.getDepthImg().setScale(_roi.scale); - builder.getDepthImg().setData( - kj::arrayPtr(_depthImg.data, _depthImg.size().width * _depthImg.size().height * _depthImg.channels() * sizeof(uchar)) - ); +void algo::Inference::serialize(proto::CamSensor& camSensor) { + // Depth Output + auto depthRaw = camSensor.mutable_depthraw(); + util::fillProtoImg(depthRaw, _depthOut, _roi); + // Semseg Output + auto semsegRaw = camSensor.mutable_semsegraw(); + util::fillProtoImg(semsegRaw, _semsegOut, _roi); + + // Semseg Img + auto semseg = camSensor.mutable_semsegimg(); + util::fillProtoImg(semseg, _semsegImg, _roi); + // Depth Img + auto depth = camSensor.mutable_depthimg(); + util::fillProtoImg(depth, _depthImg, _roi); } diff --git a/src/algo/inference/inference.h b/src/algo/inference/inference.h index 02e2778..ba3ddda 100644 --- a/src/algo/inference/inference.h +++ b/src/algo/inference/inference.h @@ -1,11 +1,10 @@ #pragma once #include "opencv2/opencv.hpp" -#include "frame/runtime_meas_service.h" -#include "data_reader/cams/icam.h" -#include "utilities/img.h" -#include "serialize/frame.capnp.h" +#include "util/runtime_meas_service.h" +#include "util/img.h" #include "params.h" +#include "frame.pb.h" // Tensorflow Lite and EdgeTpu includes #include "edgetpu.h" #include "tensorflow/lite/interpreter.h" @@ -13,20 +12,18 @@ #include "tensorflow/lite/model.h" -namespace inference { +namespace algo { class Inference { public: - Inference(frame::RuntimeMeasService& runtimeMeasService); + Inference(util::RuntimeMeasService& runtimeMeasService); void reset(); void processImg(const cv::Mat &img); - void serialize(CapnpOutput::CamSensor::Builder& builder); + void serialize(proto::CamSensor& camSensor); const cv::Mat& getSemseg() { return _semsegOut; } const cv::Mat& getDepth() { return _depthOut; } - const util::img::Roi& getRoi() { return _roi; } - private: - frame::RuntimeMeasService& _runtimeMeasService; + util::RuntimeMeasService& _runtimeMeasService; // TFLite and EdgeTpu std::unique_ptr _model; std::shared_ptr _edgeTpuContext; @@ -38,6 +35,6 @@ namespace inference { cv::Mat _semsegImg; cv::Mat _depthOut; cv::Mat _depthImg; - util::img::Roi _roi; // Roi regarding the output data (_semsegOut, _semsegImg, _depthOut, _depthImg) + util::img::Roi _roi; }; } diff --git a/src/algo/inference/params.h b/src/algo/inference/params.h index aeee589..1e44d39 100644 --- a/src/algo/inference/params.h +++ b/src/algo/inference/params.h @@ -1,7 +1,7 @@ #pragma once #include -namespace inference { +namespace algo { const int OFFSET_BOTTOM = 120; // Offset from original size (640x380) const std::string PATH_EDGETPU_MODEL = "./assets/od_model/multitask_edgetpu.tflite"; diff --git a/src/algo/scheduler/scheduler.cpp b/src/algo/scheduler/scheduler.cpp index 7956560..45971ed 100644 --- a/src/algo/scheduler/scheduler.cpp +++ b/src/algo/scheduler/scheduler.cpp @@ -1,5 +1,6 @@ #include "algo/scheduler/scheduler.h" #include +#include "util/proto.h" algo::Scheduler::Scheduler(util::RuntimeMeasService& runtimeMeasService) : @@ -11,15 +12,30 @@ algo::Scheduler::Scheduler(util::RuntimeMeasService& runtimeMeasService) : void algo::Scheduler::reset() { // Reset Algos - // for (auto& [key, opticalFlow]: _opticalFlowMap) { - // opticalFlow->reset(); - // } + for (auto& [key, inf]: _inference) { + inf->reset(); + } } void algo::Scheduler::exec(proto::Frame &frame) { _runtimeMeasService.startMeas("algo"); + // Loop over sensor data from inputData and run sensor dependent algos + for (auto camProto: frame.camsensors()) { + auto key = camProto.key(); + + cv::Mat cvImg; + util::fillCvImg(cvImg, camProto.img()); + + // Do inference + if (_inference.count(key) <= 0) { + _inference.insert({key, std::make_unique(_runtimeMeasService)}); + } + _inference.at(key)->processImg(cvImg); + _runtimeMeasService.startMeas("process"); + _inference.at(key)->serialize(camProto); + _runtimeMeasService.endMeas("process"); + } - // 1) Loop over sensor data from inputData and run sensor dependent algos // 1.1) Within the loop update needed sensor independent algos with this data // 2) Run algos which depend on multiple sensor input (if provided) // 3) Serialize all data from algos and create ouputData from it diff --git a/src/algo/scheduler/scheduler.h b/src/algo/scheduler/scheduler.h index 101797c..45b9211 100644 --- a/src/algo/scheduler/scheduler.h +++ b/src/algo/scheduler/scheduler.h @@ -5,14 +5,8 @@ #include #include "util/runtime_meas_service.h" #include "frame.pb.h" - // [algos] -// #include "algo/optical_flow/optical_flow.h" -// #include "algo/inference/inference.h" -// #include "algo/cam_calib/cam_calib.h" -// #include "algo/pointcloud/pointcloud.h" -// #include "algo/tracking/tracker.h" -// #include "algo/example/example.h" +#include "algo/inference/inference.h" namespace algo { @@ -29,12 +23,7 @@ namespace algo { private: util::RuntimeMeasService& _runtimeMeasService; - // // [algos] per sensor - // std::map> _opticalFlowMap; // optical flow per cam sensor - // std::map> _inference; // inference per cam sensor - // std::map> _camCalib; // cam_calib per cam sensor - // // [algos] sensor independent - // std::unique_ptr _pointcloud; - // std::unique_ptr _tracker; + // [algos] per sensor + std::map> _inference; }; } diff --git a/src/data/rec/video_rec.cpp b/src/data/rec/video_rec.cpp index bc65f61..13bc9fd 100644 --- a/src/data/rec/video_rec.cpp +++ b/src/data/rec/video_rec.cpp @@ -3,6 +3,7 @@ #include #include #include "util/json.hpp" +#include "util/proto.h" data::VideoRec::VideoRec( @@ -20,6 +21,10 @@ data::VideoRec::VideoRec( _frameRate = _stream.get(cv::CAP_PROP_FPS); const double frameCount = _stream.get(cv::CAP_PROP_FRAME_COUNT); _recLength = static_cast(((frameCount - 1) / _frameRate) * 1000000); + + _roi.scale = 1.0; + _roi.offsetLeft = 0.0; + _roi.offsetTop = 0.0; } void data::VideoRec::reset() { @@ -47,14 +52,14 @@ void data::VideoRec::fillFrame(proto::Frame& frame) { frame.set_plannedframelength((1/_frameRate) * 1000.0); auto camSensor = frame.mutable_camsensors()->Add(); + camSensor->set_key("VideoCam"); camSensor->set_isvalid(true); camSensor->set_relts(_currTs); camSensor->set_absts(_currTs); + auto img = camSensor->mutable_img(); - img->set_width(_currFrame.size().width); - img->set_height(_currFrame.size().height); - img->set_channels(_currFrame.channels()); - img->set_data(_currFrame.data, _currFrame.size().width * _currFrame.size().height * _currFrame.channels() * sizeof(uchar)); + util::fillProtoImg(img, _currFrame, _roi); + // TODO: Set intrinsics and extrinsics here } } diff --git a/src/data/rec/video_rec.h b/src/data/rec/video_rec.h index b1136ea..b60eeb8 100644 --- a/src/data/rec/video_rec.h +++ b/src/data/rec/video_rec.h @@ -5,6 +5,7 @@ #include "irec.h" #include "opencv2/opencv.hpp" #include "frame.pb.h" +#include "util/img.h" namespace data { @@ -25,6 +26,7 @@ namespace data { cv::Size _frameSize; int _currFrameNr; cv::Mat _currFrame; + util::img::Roi _roi; int64_t _currTs; std::mutex _readLock; diff --git a/src/frame/ecal/CMakeLists.txt b/src/frame/ecal/CMakeLists.txt index 73e9ad7..17ea9ba 100644 --- a/src/frame/ecal/CMakeLists.txt +++ b/src/frame/ecal/CMakeLists.txt @@ -11,8 +11,9 @@ target_link_libraries(${PROJECT_NAME} PUBLIC libinterface ${OpenCV_LIBS} - ${CAPNP_LIBRARIES} eCAL::core + ${TENSORFLOW_LITE_LIBS} + ${EDGE_TPU_LIBS} ${CMAKE_THREAD_LIBS_INIT} dl # needed for tensorflow lite (libdl.so) diff --git a/src/frame/ecal/main.cpp b/src/frame/ecal/main.cpp index 61834ab..ec30191 100644 --- a/src/frame/ecal/main.cpp +++ b/src/frame/ecal/main.cpp @@ -20,7 +20,7 @@ using namespace std::chrono_literals; -std::atomic play = false; +std::atomic play = true; std::atomic playedOneFrame = false; std::atomic doReset = false; std::atomic sendLastFrame = false; diff --git a/src/interface b/src/interface index f74d2f5..c10772e 160000 --- a/src/interface +++ b/src/interface @@ -1 +1 @@ -Subproject commit f74d2f5b165c911a6a1cd346ae75e3f7a28bbfb4 +Subproject commit c10772ea7cfd0bbcf4e3224f9302e203188e1caa diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 0aa4708..c32ad2d 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(lib${COMPONENT_NAME} STATIC img.cpp runtime_meas_service.cpp time.cpp + proto.cpp ) target_include_directories(lib${COMPONENT_NAME} SYSTEM diff --git a/src/util/proto.cpp b/src/util/proto.cpp new file mode 100644 index 0000000..1b2f355 --- /dev/null +++ b/src/util/proto.cpp @@ -0,0 +1,10 @@ +#include "proto.h" + + +void util::fillCvImg(cv::Mat& cvImg, const proto::Img& pbImg) { + int height = pbImg.height(); + int width = pbImg.width(); + int channels = pbImg.channels(); + void* dataPtr = const_cast(pbImg.data().c_str()); + cvImg = cv::Mat(height, width, pbImg.type(), dataPtr); +} diff --git a/src/util/proto.h b/src/util/proto.h new file mode 100644 index 0000000..c122c4f --- /dev/null +++ b/src/util/proto.h @@ -0,0 +1,24 @@ +#pragma once + +#include "opencv2/opencv.hpp" +#include "img.h" +#include "camera.pb.h" + + +namespace util { + + template + void fillProtoImg(proto::Img* pbImg, const cv::Mat& cvImg, const img::Roi& roi) { + pbImg->set_width(cvImg.size().width); + pbImg->set_height(cvImg.size().height); + pbImg->set_channels(cvImg.channels()); + pbImg->set_offsetleft(roi.offsetLeft); + pbImg->set_offsettop(roi.offsetTop); + pbImg->set_scale(roi.scale); + pbImg->set_type(cvImg.type()); + pbImg->set_typestr(cv::typeToString(cvImg.type())); + pbImg->set_data(cvImg.data, cvImg.size().width * cvImg.size().height * cvImg.channels() * sizeof(T)); + } + + void fillCvImg(cv::Mat& cvImg, const proto::Img& pbImg); +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d4077b6..196813a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -5,7 +5,6 @@ include_directories( ${OpenCV_INCLUDE_DIRS} ${TENSORFLOW_LITE_INCLUDE_DIR} ${EDGE_TPU_INCLUDE_DIR} - ${CAPNP_INCLUDE_DIRS} ${GTEST_INCLUDE_DIR} ) @@ -32,7 +31,6 @@ target_link_libraries(${PROJECT_TEST_NAME} # dependencies ${GTEST_LIBS} ${OpenCV_LIBS} - ${CAPNP_LIBRARIES} ${TENSORFLOW_LITE_LIB_X86} # ${TENSORFLOW_LITE_LIB_ARM} ${EDGE_TPU_LIBS}