From bc469cf0f8a4e14b8ea2991eaf47c418695d7b9c Mon Sep 17 00:00:00 2001 From: clayjay3 Date: Mon, 21 Oct 2024 15:07:53 -0500 Subject: [PATCH 1/2] Debug blown out tag detector images and add some checks to the TagDetector class. --- src/handlers/TagDetectionHandler.h | 1 - src/vision/aruco/ArucoDetection.hpp | 16 ++++++++--- src/vision/aruco/TagDetector.cpp | 32 ++++++++++++++++----- src/vision/aruco/TensorflowTagDetection.hpp | 8 ++++++ 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/handlers/TagDetectionHandler.h b/src/handlers/TagDetectionHandler.h index a39f5dd1a..fd0efa11c 100644 --- a/src/handlers/TagDetectionHandler.h +++ b/src/handlers/TagDetectionHandler.h @@ -34,7 +34,6 @@ class TagDetectionHandler TagDetector* m_pTagDetectorMainCam; TagDetector* m_pTagDetectorLeftCam; TagDetector* m_pTagDetectorRightCam; - TagDetector* m_pTagDetectorGroundCam; RecordingHandler* m_pRecordingHandler; public: diff --git a/src/vision/aruco/ArucoDetection.hpp b/src/vision/aruco/ArucoDetection.hpp index 1d7b8045e..b1220c40a 100644 --- a/src/vision/aruco/ArucoDetection.hpp +++ b/src/vision/aruco/ArucoDetection.hpp @@ -86,7 +86,7 @@ namespace arucotag * to not alter the original in case it's used after the call to this function * completes. * - * @param cvInputFrame - cv::Mat of the image to pre-process. + * @param cvInputFrame - cv::Mat of the image to pre-process. This image should be in BGR format. * @param cvOutputFrame - cv::Mat to write the pre-processed image to. * * @todo Determine optimal order for speed @@ -95,9 +95,17 @@ namespace arucotag ******************************************************************************/ inline void PreprocessFrame(const cv::Mat& cvInputFrame, cv::Mat& cvOutputFrame) { + // Check if the input frame is in BGR format. + if (cvInputFrame.channels() != 3) + { + // Submit logger message. + LOG_ERROR(logging::g_qSharedLogger, "PreprocessFrame() requires a BGR image."); + return; + } + // Grayscale. - cv::cvtColor(cvInputFrame, cvOutputFrame, cv::COLOR_BGRA2GRAY); - cv::filter2D(cvOutputFrame, cvInputFrame, -1, constants::ARUCO_SHARPEN_KERNEL_EXTRA); + cv::cvtColor(cvInputFrame, cvOutputFrame, cv::COLOR_BGR2GRAY); + cv::filter2D(cvOutputFrame, cvOutputFrame, -1, constants::ARUCO_SHARPEN_KERNEL_EXTRA); // Reduce number of colors/gradients in the image. // imgops::ColorReduce(cvOutputFrame); // Denoise (Looks like bilateral filter is req. for ArUco, check speed since docs say it's slow) @@ -120,7 +128,7 @@ namespace arucotag /****************************************************************************** * @brief Detect ArUco tags in the provided image. * - * @param cvFrame - The camera frame to run ArUco detection on. Should be BGR format. + * @param cvFrame - The camera frame to run ArUco detection on. Should be BGR format or grayscale. * @param cvArucoDetector - The configured aruco detector to use for detection. * @return std::vector - The resultant vector containing the detected tags in the frame. * diff --git a/src/vision/aruco/TagDetector.cpp b/src/vision/aruco/TagDetector.cpp index f820577bb..1fc65b640 100644 --- a/src/vision/aruco/TagDetector.cpp +++ b/src/vision/aruco/TagDetector.cpp @@ -235,6 +235,8 @@ void TagDetector::ThreadedContinuousCode() // Download mat from GPU memory. m_cvGPUPointCloud.download(m_cvPointCloud); m_cvGPUFrame.download(m_cvFrame); + // Drop the Alpha channel from the image copy to preproc frame. + cv::cvtColor(m_cvFrame, m_cvFrame, cv::COLOR_BGRA2RGB); } else { @@ -259,6 +261,11 @@ void TagDetector::ThreadedContinuousCode() // Submit logger message. LOG_WARNING(logging::g_qSharedLogger, "TagDetector unable to get regular frame from ZEDCam!"); } + else + { + // Drop the Alpha channel from the image copy to preproc frame. + cv::cvtColor(m_cvFrame, m_cvFrame, cv::COLOR_BGRA2RGB); + } } } else @@ -272,23 +279,34 @@ void TagDetector::ThreadedContinuousCode() // Submit logger message. LOG_WARNING(logging::g_qSharedLogger, "TagDetector unable to get point cloud from BasicCam!"); } + else + { + // Check if the camera image is a >3 channel image. + if (m_cvFrame.channels() > 3) + { + // Drop the Alpha channel from the image copy to preproc frame. + cv::cvtColor(m_cvFrame, m_cvFrame, cv::COLOR_BGRA2RGB); + } + } } ///////////////////////////////////////// // Actual detection logic goes here. ///////////////////////////////////////// - // Drop the Alpha channel from the image copy to preproc frame. - cv::cvtColor(m_cvFrame, m_cvArucoProcFrame, cv::COLOR_BGRA2BGR); // Run image through some pre-processing step to improve detection. - arucotag::PreprocessFrame(m_cvArucoProcFrame, m_cvArucoProcFrame); + arucotag::PreprocessFrame(m_cvFrame, m_cvArucoProcFrame); // Detect tags in the image std::vector vNewlyDetectedTags = arucotag::Detect(m_cvArucoProcFrame, m_cvArucoDetector); - // Estimate the positions of the tags using the point cloud - for (arucotag::ArucoTag& stTag : vNewlyDetectedTags) + // Only estimate the pose of the tags if the point cloud is available and we are using a ZED camera. + if (m_bUsingZedCamera && !m_cvPointCloud.empty()) { - // Use the point cloud to get the location of the tag. - arucotag::EstimatePoseFromPointCloud(m_cvPointCloud, stTag); + // Estimate the positions of the tags using the point cloud + for (arucotag::ArucoTag& stTag : vNewlyDetectedTags) + { + // Use the point cloud to get the location of the tag. + arucotag::EstimatePoseFromPointCloud(m_cvPointCloud, stTag); + } } // Merge the newly detected tags with the pre-existing detected tags this->UpdateDetectedTags(vNewlyDetectedTags); diff --git a/src/vision/aruco/TensorflowTagDetection.hpp b/src/vision/aruco/TensorflowTagDetection.hpp index 0508fda53..1efaa7e57 100644 --- a/src/vision/aruco/TensorflowTagDetection.hpp +++ b/src/vision/aruco/TensorflowTagDetection.hpp @@ -98,6 +98,14 @@ namespace tensorflowtag const float fMinObjectConfidence = 0.40f, const float fNMSThreshold = 0.60f) { + // Check if the input frame is in RGB format. + if (cvFrame.channels() != 3) + { + // Submit logger message. + LOG_ERROR(logging::g_qSharedLogger, "Detect() requires a RGB image."); + return {}; + } + // Declare instance variables. std::vector vDetectedTags; From 4080d3f335fea60eed2e1b0e568e0b1a711dde44 Mon Sep 17 00:00:00 2001 From: clayjay3 Date: Tue, 29 Oct 2024 20:11:09 -0500 Subject: [PATCH 2/2] Update RoveComm_CPP submodule. --- external/rovecomm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/rovecomm b/external/rovecomm index 10e897b11..5d17cd52a 160000 --- a/external/rovecomm +++ b/external/rovecomm @@ -1 +1 @@ -Subproject commit 10e897b11655d581160ae2c054d6446aa97e9587 +Subproject commit 5d17cd52a7896b757d431c759ecfb29fea7a9422