From 99275fba1d1438d5a3274e7d4601424fe9e6f8c9 Mon Sep 17 00:00:00 2001 From: matlabbe Date: Sun, 1 Nov 2020 11:28:59 -0500 Subject: [PATCH] Added Daisy descriptor. ORB: updated default parameters. Making ORBOctree using ORB parameters. Updated Vis/CorNNDR default from 0.6 to 0.8 (increase number of matches with binary descriptors, increase slightly feature matching time with float descriptors). Note that jfr2018 scripts have been updated to use old value 0.6. rtabmap-info: show descriptor dimension and type. --- CMakeLists.txt | 2 +- corelib/include/rtabmap/core/Features2d.h | 61 +++++++- corelib/include/rtabmap/core/Parameters.h | 16 +- corelib/src/Features2d.cpp | 178 ++++++++++++++++++---- corelib/src/opencv/ORBextractor.cc | 58 ++++--- corelib/src/opencv/ORBextractor.h | 6 +- docker/jfr2018/run_euroc_datasets.sh | 1 + docker/jfr2018/run_kitti_datasets.sh | 1 + docker/jfr2018/run_tum_datasets.sh | 1 + guilib/src/DatabaseViewer.cpp | 23 ++- guilib/src/PreferencesDialog.cpp | 6 + guilib/src/ui/preferencesDialog.ui | 24 ++- tools/Info/main.cpp | 22 ++- 13 files changed, 326 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81c31d15..56eb0fdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ SET(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake_modules") ####################### SET(RTABMAP_MAJOR_VERSION 0) SET(RTABMAP_MINOR_VERSION 20) -SET(RTABMAP_PATCH_VERSION 5) +SET(RTABMAP_PATCH_VERSION 6) SET(RTABMAP_VERSION ${RTABMAP_MAJOR_VERSION}.${RTABMAP_MINOR_VERSION}.${RTABMAP_PATCH_VERSION}) diff --git a/corelib/include/rtabmap/core/Features2d.h b/corelib/include/rtabmap/core/Features2d.h index 210dc9af..64a4d115 100644 --- a/corelib/include/rtabmap/core/Features2d.h +++ b/corelib/include/rtabmap/core/Features2d.h @@ -61,6 +61,7 @@ typedef cv::gpu::FAST_GPU CV_FAST_GPU; namespace cv{ namespace xfeatures2d { class FREAK; +class DAISY; class BriefDescriptorExtractor; #if CV_MAJOR_VERSION < 3 || (CV_MAJOR_VERSION == 4 && CV_MINOR_VERSION <= 3) || (CV_MAJOR_VERSION == 3 && (CV_MINOR_VERSION < 4 || (CV_MINOR_VERSION==4 && CV_SUBMINOR_VERSION<11))) class SIFT; @@ -81,6 +82,7 @@ typedef cv::SIFT CV_SIFT; // SIFT is back in features2d since 4.4.0 / 3.4.11 typedef cv::xfeatures2d::SURF CV_SURF; typedef cv::FastFeatureDetector CV_FAST; typedef cv::xfeatures2d::FREAK CV_FREAK; +typedef cv::xfeatures2d::DAISY CV_DAISY; typedef cv::GFTTDetector CV_GFTT; typedef cv::xfeatures2d::BriefDescriptorExtractor CV_BRIEF; typedef cv::BRISK CV_BRISK; @@ -116,7 +118,10 @@ public: kFeatureKaze=9, //new 0.13.2 kFeatureOrbOctree=10, //new 0.19.2 kFeatureSuperPointTorch=11, //new 0.19.7 - kFeatureSurfFreak=12}; //new 0.20.4 + kFeatureSurfFreak=12, //new 0.20.4 + kFeatureGfttDaisy=13, //new 0.20.6 + kFeatureSurfDaisy=14}; //new 0.20.6 + static std::string typeName(Type type) { switch(type){ @@ -146,6 +151,10 @@ public: return "SUPERPOINT"; case kFeatureSurfFreak: return "SURF+Freak"; + case kFeatureGfttDaisy: + return "GFTT+Daisy"; + case kFeatureSurfDaisy: + return "SURF+Daisy"; default: return "Unknown"; } @@ -563,6 +572,8 @@ private: private: float scaleFactor_; int nLevels_; + int patchSize_; + int edgeThreshold_; int fastThreshold_; int fastMinThreshold_; @@ -593,6 +604,54 @@ private: bool cuda_; }; +//GFTT_DAISY +class RTABMAP_EXP GFTT_DAISY : public GFTT +{ +public: + GFTT_DAISY(const ParametersMap & parameters = ParametersMap()); + virtual ~GFTT_DAISY(); + + virtual void parseParameters(const ParametersMap & parameters); + virtual Feature2D::Type getType() const {return kFeatureGfttDaisy;} + +private: + virtual cv::Mat generateDescriptorsImpl(const cv::Mat & image, std::vector & keypoints) const; + +private: + bool orientationNormalized_; + bool scaleNormalized_; + float patternScale_; + int nOctaves_; + +#if CV_MAJOR_VERSION > 2 + cv::Ptr _daisy; +#endif +}; + +//SURF_DAISY +class RTABMAP_EXP SURF_DAISY : public SURF +{ +public: + SURF_DAISY(const ParametersMap & parameters = ParametersMap()); + virtual ~SURF_DAISY(); + + virtual void parseParameters(const ParametersMap & parameters); + virtual Feature2D::Type getType() const {return kFeatureSurfDaisy;} + +private: + virtual cv::Mat generateDescriptorsImpl(const cv::Mat & image, std::vector & keypoints) const; + +private: + bool orientationNormalized_; + bool scaleNormalized_; + float patternScale_; + int nOctaves_; + +#if CV_MAJOR_VERSION > 2 + cv::Ptr _daisy; +#endif +}; + } #endif /* FEATURES2D_H_ */ diff --git a/corelib/include/rtabmap/core/Parameters.h b/corelib/include/rtabmap/core/Parameters.h index e1e7a4f8..39da0e43 100644 --- a/corelib/include/rtabmap/core/Parameters.h +++ b/corelib/include/rtabmap/core/Parameters.h @@ -244,9 +244,9 @@ class RTABMAP_EXP Parameters RTABMAP_PARAM(Kp, NndrRatio, float, 0.8, "NNDR ratio (A matching pair is detected, if its distance is closer than X times the distance of the second nearest neighbor.)"); #if CV_MAJOR_VERSION > 2 && !defined(HAVE_OPENCV_XFEATURES2D) // OpenCV>2 without xFeatures2D module doesn't have BRIEF - RTABMAP_PARAM(Kp, DetectorStrategy, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint Torch 12=SURF/FREAK."); + RTABMAP_PARAM(Kp, DetectorStrategy, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY"); #else - RTABMAP_PARAM(Kp, DetectorStrategy, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint Torch 12=SURF/FREAK."); + RTABMAP_PARAM(Kp, DetectorStrategy, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY"); #endif RTABMAP_PARAM(Kp, TfIdfLikelihoodUsed, bool, true, "Use of the td-idf strategy to compute the likelihood."); RTABMAP_PARAM(Kp, Parallelized, bool, true, "If the dictionary update and signature creation were parallelized."); @@ -300,9 +300,9 @@ class RTABMAP_EXP Parameters RTABMAP_PARAM(GFTT, UseHarrisDetector, bool, false, ""); RTABMAP_PARAM(GFTT, K, double, 0.04, ""); - RTABMAP_PARAM(ORB, ScaleFactor, float, 1.2, "Pyramid decimation ratio, greater than 1. scaleFactor==2 means the classical pyramid, where each next level has 4x less pixels than the previous, but such a big scale factor will degrade feature matching scores dramatically. On the other hand, too close to 1 scale factor will mean that to cover certain scale range you will need more pyramid levels and so the speed will suffer."); - RTABMAP_PARAM(ORB, NLevels, int, 8, "The number of pyramid levels. The smallest level will have linear size equal to input_image_linear_size/pow(scaleFactor, nlevels)."); - RTABMAP_PARAM(ORB, EdgeThreshold, int, 31, "This is size of the border where the features are not detected. It should roughly match the patchSize parameter."); + RTABMAP_PARAM(ORB, ScaleFactor, float, 2, "Pyramid decimation ratio, greater than 1. scaleFactor==2 means the classical pyramid, where each next level has 4x less pixels than the previous, but such a big scale factor will degrade feature matching scores dramatically. On the other hand, too close to 1 scale factor will mean that to cover certain scale range you will need more pyramid levels and so the speed will suffer."); + RTABMAP_PARAM(ORB, NLevels, int, 3, "The number of pyramid levels. The smallest level will have linear size equal to input_image_linear_size/pow(scaleFactor, nlevels)."); + RTABMAP_PARAM(ORB, EdgeThreshold, int, 19, "This is size of the border where the features are not detected. It should roughly match the patchSize parameter."); RTABMAP_PARAM(ORB, FirstLevel, int, 0, "It should be 0 in the current implementation."); RTABMAP_PARAM(ORB, WTA_K, int, 2, "The number of points that produce each element of the oriented BRIEF descriptor. The default value 2 means the BRIEF where we take a random point pair and compare their brightnesses, so we get 0/1 response. Other possible values are 3 and 4. For example, 3 means that we take 3 random points (of course, those point coordinates are random, but they are generated from the pre-defined seed, so each element of BRIEF descriptor is computed deterministically from the pixel rectangle), find point of maximum brightness and output index of the winner (0, 1 or 2). Such output will occupy 2 bits, and therefore it will need a special variant of Hamming distance, denoted as NORM_HAMMING2 (2 bits per bin). When WTA_K=4, we take 4 random points to compute each bin (that will also occupy 2 bits with possible values 0, 1, 2 or 3)."); RTABMAP_PARAM(ORB, ScoreType, int, 0, "The default HARRIS_SCORE=0 means that Harris algorithm is used to rank features (the score is written to KeyPoint::score and is used to retain best nfeatures features); FAST_SCORE=1 is alternative value of the parameter that produces slightly less stable keypoints, but it is a little faster to compute."); @@ -587,9 +587,9 @@ class RTABMAP_EXP Parameters RTABMAP_PARAM(Vis, Iterations, int, 300, "Maximum iterations to compute the transform."); #if CV_MAJOR_VERSION > 2 && !defined(HAVE_OPENCV_XFEATURES2D) // OpenCV>2 without xFeatures2D module doesn't have BRIEF - RTABMAP_PARAM(Vis, FeatureType, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint Torch 12=SURF/FREAK."); + RTABMAP_PARAM(Vis, FeatureType, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY"); #else - RTABMAP_PARAM(Vis, FeatureType, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint Torch 12=SURF/FREAK."); + RTABMAP_PARAM(Vis, FeatureType, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY"); #endif RTABMAP_PARAM(Vis, MaxFeatures, int, 1000, "0 no limits."); RTABMAP_PARAM(Vis, MaxDepth, float, 0, "Max depth of the features (0 means no limit)."); @@ -603,7 +603,7 @@ class RTABMAP_EXP Parameters RTABMAP_PARAM(Vis, GridCols, int, 1, uFormat("Number of columns of the grid used to extract uniformly \"%s / grid cells\" features from each cell.", kVisMaxFeatures().c_str())); RTABMAP_PARAM(Vis, CorType, int, 0, "Correspondences computation approach: 0=Features Matching, 1=Optical Flow"); RTABMAP_PARAM(Vis, CorNNType, int, 1, uFormat("[%s=0] kNNFlannNaive=0, kNNFlannKdTree=1, kNNFlannLSH=2, kNNBruteForce=3, kNNBruteForceGPU=4, BruteForceCrossCheck=5, SuperGlue=6, GMS=7. Used for features matching approach.", kVisCorType().c_str())); - RTABMAP_PARAM(Vis, CorNNDR, float, 0.6, uFormat("[%s=0] NNDR: nearest neighbor distance ratio. Used for knn features matching approach.", kVisCorType().c_str())); + RTABMAP_PARAM(Vis, CorNNDR, float, 0.8, uFormat("[%s=0] NNDR: nearest neighbor distance ratio. Used for knn features matching approach.", kVisCorType().c_str())); RTABMAP_PARAM(Vis, CorGuessWinSize, int, 20, uFormat("[%s=0] Matching window size (pixels) around projected points when a guess transform is provided to find correspondences. 0 means disabled.", kVisCorType().c_str())); RTABMAP_PARAM(Vis, CorGuessMatchToProjection, bool, false, uFormat("[%s=0] Match frame's corners to source's projected points (when guess transform is provided) instead of projected points to frame's corners.", kVisCorType().c_str())); RTABMAP_PARAM(Vis, CorFlowWinSize, int, 16, uFormat("[%s=1] See cv::calcOpticalFlowPyrLK(). Used for optical flow approach.", kVisCorType().c_str())); diff --git a/corelib/src/Features2d.cpp b/corelib/src/Features2d.cpp index ca67b1a4..a538234b 100644 --- a/corelib/src/Features2d.cpp +++ b/corelib/src/Features2d.cpp @@ -509,55 +509,69 @@ Feature2D * Feature2D::create(const ParametersMap & parameters) Feature2D * Feature2D::create(Feature2D::Type type, const ParametersMap & parameters) { +// NONFREE checks #if CV_MAJOR_VERSION < 3 || (CV_MAJOR_VERSION == 4 && CV_MINOR_VERSION <= 3) || (CV_MAJOR_VERSION == 3 && (CV_MINOR_VERSION < 4 || (CV_MINOR_VERSION==4 && CV_SUBMINOR_VERSION<11))) -#ifndef RTABMAP_NONFREE - if(type == Feature2D::kFeatureSurf || type == Feature2D::kFeatureSift || type == Feature2D::kFeatureSurfFreak) + + #ifndef RTABMAP_NONFREE + if(type == Feature2D::kFeatureSurf || type == Feature2D::kFeatureSift || type == Feature2D::kFeatureSurfFreak || type == Feature2D::kFeatureSurfDaisy) { -#if CV_MAJOR_VERSION < 3 + #if CV_MAJOR_VERSION < 3 UWARN("SURF and SIFT features cannot be used because OpenCV was not built with nonfree module. GFTT/ORB is used instead."); -#else + #else UWARN("SURF and SIFT features cannot be used because OpenCV was not built with xfeatures2d module. GFTT/ORB is used instead."); -#endif + #endif type = Feature2D::kFeatureGfttOrb; } -#if CV_MAJOR_VERSION == 3 - if(type == Feature2D::kFeatureFastBrief || - type == Feature2D::kFeatureFastFreak || - type == Feature2D::kFeatureGfttBrief || - type == Feature2D::kFeatureGfttFreak || - type == Feature2D::kFeatureSurfFreak) - { - UWARN("BRIEF and FREAK features cannot be used because OpenCV was not built with xfeatures2d module. GFTT/ORB is used instead."); - type = Feature2D::kFeatureGfttOrb; - } -#endif -#endif + #endif #else // >= 4.4.0 >= 3.4.11 -#ifndef RTABMAP_NONFREE - if(type == Feature2D::kFeatureSurf || type == Feature2D::kFeatureSurfFreak) + #ifndef RTABMAP_NONFREE + if(type == Feature2D::kFeatureSurf) { UWARN("SURF features cannot be used because OpenCV was not built with nonfree module. SIFT is used instead."); type = Feature2D::kFeatureSift; } -#endif + else if(type == Feature2D::kFeatureSurfFreak || type == Feature2D::kFeatureSurfDaisy) + { + UWARN("SURF detector cannot be used because OpenCV was not built with nonfree module. GFTT/ORB is used instead."); + type = Feature2D::kFeatureGfttOrb; + } + #endif #endif // >= 4.4.0 >= 3.4.11 -#if CV_MAJOR_VERSION < 3 +#if !defined(HAVE_OPENCV_XFEATURES2D) && CV_MAJOR_VERSION >= 3 + if(type == Feature2D::kFeatureFastBrief || + type == Feature2D::kFeatureFastFreak || + type == Feature2D::kFeatureGfttBrief || + type == Feature2D::kFeatureGfttFreak || + type == Feature2D::kFeatureSurfFreak || + type == Feature2D::kFeatureGfttDaisy || + type == Feature2D::kFeatureSurfDaisy) + { + UWARN("BRIEF, FREAK and DAISY features cannot be used because OpenCV was not built with xfeatures2d module. GFTT/ORB is used instead."); + type = Feature2D::kFeatureGfttOrb; + } +#elif CV_MAJOR_VERSION < 3 if(type == Feature2D::kFeatureKaze) { -#ifdef RTABMAP_NONFREE + #ifdef RTABMAP_NONFREE UWARN("KAZE detector/descriptor can be used only with OpenCV3. SURF is used instead."); type = Feature2D::kFeatureSurf; -#else + #else UWARN("KAZE detector/descriptor can be used only with OpenCV3. GFTT/ORB is used instead."); type = Feature2D::kFeatureGfttOrb; -#endif + #endif + } + if(type == Feature2D::kFeatureGfttDaisy || type == Feature2D::kFeatureSurfDaisy) + { + UWARN("DAISY detector/descriptor can be used only with OpenCV3. GFTT/BRIEF is used instead."); + type = Feature2D::kFeatureGfttBrief; } #endif + #ifndef RTABMAP_ORB_OCTREE if(type == Feature2D::kFeatureOrbOctree) { @@ -618,6 +632,12 @@ Feature2D * Feature2D::create(Feature2D::Type type, const ParametersMap & parame case Feature2D::kFeatureSurfFreak: feature2D = new SURF_FREAK(parameters); break; + case Feature2D::kFeatureGfttDaisy: + feature2D = new GFTT_DAISY(parameters); + break; + case Feature2D::kFeatureSurfDaisy: + feature2D = new SURF_DAISY(parameters); + break; #ifdef RTABMAP_NONFREE default: feature2D = new SURF(parameters); @@ -626,7 +646,7 @@ Feature2D * Feature2D::create(Feature2D::Type type, const ParametersMap & parame #else default: feature2D = new ORB(parameters); - type = Feature2D::kFeatureOrb; + type = Feature2D::kFeatureGfttOrb; break; #endif @@ -1939,7 +1959,10 @@ cv::Mat KAZE::generateDescriptorsImpl(const cv::Mat & image, std::vector(new ORBextractor(this->getMaxFeatures(), scaleFactor_, nLevels_, fastThreshold_, fastMinThreshold_)); + _orb = cv::Ptr(new ORBextractor(this->getMaxFeatures(), scaleFactor_, nLevels_, fastThreshold_, fastMinThreshold_, patchSize_, edgeThreshold_)); #else UWARN("RTAB-Map is not built with ORB OcTree option enabled so ORB OcTree feature cannot be used!"); #endif @@ -2070,4 +2095,103 @@ cv::Mat SuperPointTorch::generateDescriptorsImpl(const cv::Mat & image, std::vec #endif } + +////////////////////////// +//GFTT-DAISY +////////////////////////// +GFTT_DAISY::GFTT_DAISY(const ParametersMap & parameters) : + GFTT(parameters), + orientationNormalized_(Parameters::defaultFREAKOrientationNormalized()), + scaleNormalized_(Parameters::defaultFREAKScaleNormalized()), + patternScale_(Parameters::defaultFREAKPatternScale()), + nOctaves_(Parameters::defaultFREAKNOctaves()) +{ + parseParameters(parameters); +} + +GFTT_DAISY::~GFTT_DAISY() +{ +} + +void GFTT_DAISY::parseParameters(const ParametersMap & parameters) +{ + GFTT::parseParameters(parameters); + + Parameters::parse(parameters, Parameters::kFREAKOrientationNormalized(), orientationNormalized_); + Parameters::parse(parameters, Parameters::kFREAKScaleNormalized(), scaleNormalized_); + Parameters::parse(parameters, Parameters::kFREAKPatternScale(), patternScale_); + Parameters::parse(parameters, Parameters::kFREAKNOctaves(), nOctaves_); + +#if CV_MAJOR_VERSION < 3 + _daisy = cv::Ptr(new CV_DAISY()); +#else +#ifdef HAVE_OPENCV_XFEATURES2D + _daisy = CV_DAISY::create(); +#else + UWARN("RTAB-Map is not built with OpenCV xfeatures2d module so DAISY cannot be used!"); +#endif +#endif +} + +cv::Mat GFTT_DAISY::generateDescriptorsImpl(const cv::Mat & image, std::vector & keypoints) const +{ + UASSERT(!image.empty() && image.channels() == 1 && image.depth() == CV_8U); + cv::Mat descriptors; +#if CV_MAJOR_VERSION < 3 + _daisy->compute(image, keypoints, descriptors); +#else +#ifdef HAVE_OPENCV_XFEATURES2D + _daisy->compute(image, keypoints, descriptors); +#else + UWARN("RTAB-Map is not built with OpenCV xfeatures2d module so DAISY cannot be used!"); +#endif +#endif + return descriptors; +} + +////////////////////////// +//SURF-DAISY +////////////////////////// +SURF_DAISY::SURF_DAISY(const ParametersMap & parameters) : + SURF(parameters), + orientationNormalized_(Parameters::defaultFREAKOrientationNormalized()), + scaleNormalized_(Parameters::defaultFREAKScaleNormalized()), + patternScale_(Parameters::defaultFREAKPatternScale()), + nOctaves_(Parameters::defaultFREAKNOctaves()) +{ + parseParameters(parameters); +} + +SURF_DAISY::~SURF_DAISY() +{ +} + +void SURF_DAISY::parseParameters(const ParametersMap & parameters) +{ + SURF::parseParameters(parameters); + + Parameters::parse(parameters, Parameters::kFREAKOrientationNormalized(), orientationNormalized_); + Parameters::parse(parameters, Parameters::kFREAKScaleNormalized(), scaleNormalized_); + Parameters::parse(parameters, Parameters::kFREAKPatternScale(), patternScale_); + Parameters::parse(parameters, Parameters::kFREAKNOctaves(), nOctaves_); + +#ifdef HAVE_OPENCV_XFEATURES2D + _daisy = CV_DAISY::create(); +#else + UWARN("RTAB-Map is not built with OpenCV xfeatures2d module so DAISY cannot be used!"); +#endif +} + +cv::Mat SURF_DAISY::generateDescriptorsImpl(const cv::Mat & image, std::vector & keypoints) const +{ + UASSERT(!image.empty() && image.channels() == 1 && image.depth() == CV_8U); + cv::Mat descriptors; +#ifdef HAVE_OPENCV_XFEATURES2D + _daisy->compute(image, keypoints, descriptors); +#else + UWARN("RTAB-Map is not built with OpenCV xfeatures2d module so DAISY cannot be used!"); +#endif + return descriptors; +} + } diff --git a/corelib/src/opencv/ORBextractor.cc b/corelib/src/opencv/ORBextractor.cc index 5826a7a5..2d3cd7cc 100644 --- a/corelib/src/opencv/ORBextractor.cc +++ b/corelib/src/opencv/ORBextractor.cc @@ -80,24 +80,19 @@ using namespace std; namespace rtabmap { -const int PATCH_SIZE = 31; -const int HALF_PATCH_SIZE = 15; -const int EDGE_THRESHOLD = 19; - - -static float IC_Angle(const Mat& image, Point2f pt, const vector & u_max) +static float IC_Angle(const Mat& image, Point2f pt, const vector & u_max, int halfPatchSize) { int m_01 = 0, m_10 = 0; const uchar* center = &image.at (cvRound(pt.y), cvRound(pt.x)); // Treat the center line differently, v=0 - for (int u = -HALF_PATCH_SIZE; u <= HALF_PATCH_SIZE; ++u) + for (int u = -halfPatchSize; u <= halfPatchSize; ++u) m_10 += u * center[u]; // Go line by line in the circuI853lar patch int step = (int)image.step1(); - for (int v = 1; v <= HALF_PATCH_SIZE; ++v) + for (int v = 1; v <= halfPatchSize; ++v) { // Proceed over the two lines int v_sum = 0; @@ -419,10 +414,11 @@ static int bit_pattern_31_[256*4] = }; ORBextractor::ORBextractor(int _nfeatures, float _scaleFactor, int _nlevels, - int _iniThFAST, int _minThFAST): + int _iniThFAST, int _minThFAST, int _patchSize, int _edgeThreshold): nfeatures(_nfeatures), scaleFactor(_scaleFactor), nlevels(_nlevels), - iniThFAST(_iniThFAST), minThFAST(_minThFAST) + iniThFAST(_iniThFAST), minThFAST(_minThFAST), patchSize(_patchSize), edgeThreshold(_edgeThreshold) { + halfPatchSize = patchSize/2; mvScaleFactor.resize(nlevels); mvLevelSigma2.resize(nlevels); mvScaleFactor[0]=1.0f; @@ -462,16 +458,16 @@ ORBextractor::ORBextractor(int _nfeatures, float _scaleFactor, int _nlevels, //This is for orientation // pre-compute the end of a row in a circular patch - umax.resize(HALF_PATCH_SIZE + 1); + umax.resize(halfPatchSize + 1); - int v, v0, vmax = cvFloor(HALF_PATCH_SIZE * sqrt(2.f) / 2 + 1); - int vmin = cvCeil(HALF_PATCH_SIZE * sqrt(2.f) / 2); - const double hp2 = HALF_PATCH_SIZE*HALF_PATCH_SIZE; + int v, v0, vmax = cvFloor(float(halfPatchSize) * sqrt(2.f) / 2 + 1); + int vmin = cvCeil(float(halfPatchSize) * sqrt(2.f) / 2); + const double hp2 = halfPatchSize*halfPatchSize; for (v = 0; v <= vmax; ++v) umax[v] = cvRound(sqrt(hp2 - v * v)); // Make sure we are symmetric - for (v = HALF_PATCH_SIZE, v0 = 0; v >= vmin; --v) + for (v = halfPatchSize, v0 = 0; v >= vmin; --v) { while (umax[v0] == umax[v0 + 1]) ++v0; @@ -480,12 +476,12 @@ ORBextractor::ORBextractor(int _nfeatures, float _scaleFactor, int _nlevels, } } -static void computeOrientation(const Mat& image, vector& keypoints, const vector& umax) +static void computeOrientation(const Mat& image, vector& keypoints, const vector& umax, int halfPatchSize) { for (vector::iterator keypoint = keypoints.begin(), keypointEnd = keypoints.end(); keypoint != keypointEnd; ++keypoint) { - keypoint->angle = IC_Angle(image, keypoint->pt, umax); + keypoint->angle = IC_Angle(image, keypoint->pt, umax, halfPatchSize); } } @@ -781,10 +777,10 @@ void ORBextractor::ComputeKeyPointsOctTree(vector >& allKeypoin for (int level = 0; level < nlevels; ++level) { - const int minBorderX = EDGE_THRESHOLD-3; + const int minBorderX = edgeThreshold-3; const int minBorderY = minBorderX; - const int maxBorderX = mvImagePyramid[level].cols-EDGE_THRESHOLD+3; - const int maxBorderY = mvImagePyramid[level].rows-EDGE_THRESHOLD+3; + const int maxBorderX = mvImagePyramid[level].cols-edgeThreshold+3; + const int maxBorderY = mvImagePyramid[level].rows-edgeThreshold+3; vector vToDistributeKeys; vToDistributeKeys.reserve(nfeatures*10); @@ -845,7 +841,7 @@ void ORBextractor::ComputeKeyPointsOctTree(vector >& allKeypoin keypoints = DistributeOctTree(vToDistributeKeys, minBorderX, maxBorderX, minBorderY, maxBorderY,mnFeaturesPerLevel[level], level); - const int scaledPatchSize = PATCH_SIZE*mvScaleFactor[level]; + const int scaledPatchSize = patchSize*mvScaleFactor[level]; // Add border to coordinates and scale information const int nkps = keypoints.size(); @@ -860,7 +856,7 @@ void ORBextractor::ComputeKeyPointsOctTree(vector >& allKeypoin // compute orientations for (int level = 0; level < nlevels; ++level) - computeOrientation(mvImagePyramid[level], allKeypoints[level], umax); + computeOrientation(mvImagePyramid[level], allKeypoints[level], umax, halfPatchSize); } void ORBextractor::ComputeKeyPointsOld(std::vector > &allKeypoints) @@ -876,10 +872,10 @@ void ORBextractor::ComputeKeyPointsOld(std::vector > &allK const int levelCols = sqrt((float)nDesiredFeatures/(5*imageRatio)); const int levelRows = imageRatio*levelCols; - const int minBorderX = EDGE_THRESHOLD; + const int minBorderX = edgeThreshold; const int minBorderY = minBorderX; - const int maxBorderX = mvImagePyramid[level].cols-EDGE_THRESHOLD; - const int maxBorderY = mvImagePyramid[level].rows-EDGE_THRESHOLD; + const int maxBorderX = mvImagePyramid[level].cols-edgeThreshold; + const int maxBorderY = mvImagePyramid[level].rows-edgeThreshold; const int W = maxBorderX - minBorderX; const int H = maxBorderY - minBorderY; @@ -1006,7 +1002,7 @@ void ORBextractor::ComputeKeyPointsOld(std::vector > &allK vector & keypoints = allKeypoints[level]; keypoints.reserve(nDesiredFeatures*2); - const int scaledPatchSize = PATCH_SIZE*mvScaleFactor[level]; + const int scaledPatchSize = patchSize*mvScaleFactor[level]; // Retain by score and transform coordinates for(int i=0; i > &allK // and compute orientations for (int level = 0; level < nlevels; ++level) - computeOrientation(mvImagePyramid[level], allKeypoints[level], umax); + computeOrientation(mvImagePyramid[level], allKeypoints[level], umax, halfPatchSize); } static void computeDescriptors(const Mat& image, vector& keypoints, Mat& descriptors, @@ -1121,21 +1117,21 @@ void ORBextractor::ComputePyramid(cv::Mat image) { float scale = mvInvScaleFactor[level]; Size sz(cvRound((float)image.cols*scale), cvRound((float)image.rows*scale)); - Size wholeSize(sz.width + EDGE_THRESHOLD*2, sz.height + EDGE_THRESHOLD*2); + Size wholeSize(sz.width + edgeThreshold*2, sz.height + edgeThreshold*2); Mat temp(wholeSize, image.type()), masktemp; - mvImagePyramid[level] = temp(Rect(EDGE_THRESHOLD, EDGE_THRESHOLD, sz.width, sz.height)); + mvImagePyramid[level] = temp(Rect(edgeThreshold, edgeThreshold, sz.width, sz.height)); // Compute the resized image if( level != 0 ) { resize(mvImagePyramid[level-1], mvImagePyramid[level], sz, 0, 0, INTER_LINEAR); - copyMakeBorder(mvImagePyramid[level], temp, EDGE_THRESHOLD, EDGE_THRESHOLD, EDGE_THRESHOLD, EDGE_THRESHOLD, + copyMakeBorder(mvImagePyramid[level], temp, edgeThreshold, edgeThreshold, edgeThreshold, edgeThreshold, BORDER_REFLECT_101+BORDER_ISOLATED); } else { - copyMakeBorder(image, temp, EDGE_THRESHOLD, EDGE_THRESHOLD, EDGE_THRESHOLD, EDGE_THRESHOLD, + copyMakeBorder(image, temp, edgeThreshold, edgeThreshold, edgeThreshold, edgeThreshold, BORDER_REFLECT_101); } } diff --git a/corelib/src/opencv/ORBextractor.h b/corelib/src/opencv/ORBextractor.h index 58619a15..861ca05e 100644 --- a/corelib/src/opencv/ORBextractor.h +++ b/corelib/src/opencv/ORBextractor.h @@ -57,7 +57,7 @@ public: enum {HARRIS_SCORE=0, FAST_SCORE=1 }; ORBextractor(int nfeatures, float scaleFactor, int nlevels, - int iniThFAST, int minThFAST); + int iniThFAST, int minThFAST, int patchSize, int edgeThreshold); ~ORBextractor(){} @@ -107,6 +107,10 @@ protected: int nlevels; int iniThFAST; int minThFAST; + int patchSize; + int edgeThreshold; + int halfPatchSize; + std::vector mnFeaturesPerLevel; diff --git a/docker/jfr2018/run_euroc_datasets.sh b/docker/jfr2018/run_euroc_datasets.sh index fec3583e..8234b445 100755 --- a/docker/jfr2018/run_euroc_datasets.sh +++ b/docker/jfr2018/run_euroc_datasets.sh @@ -138,6 +138,7 @@ do --Mem/UseOdomFeatures false \ --Mem/BinDataKept false \ --Rtabmap/CreateIntermediateNodes false\ + --Vis/CorNNDR 0.6 \ $V203_params\ $F2F_params\ --OdomORBSLAM2/VocPath /root/ORBvoc.txt\ diff --git a/docker/jfr2018/run_kitti_datasets.sh b/docker/jfr2018/run_kitti_datasets.sh index c200ce0a..737fc8e0 100755 --- a/docker/jfr2018/run_kitti_datasets.sh +++ b/docker/jfr2018/run_kitti_datasets.sh @@ -124,6 +124,7 @@ do --Mem/STMSize 30\ --Mem/UseOdomFeatures false \ --Mem/BinDataKept false \ + --Vis/CorNNDR 0.6 \ $SCAN \ --gt $KITTI_ROOT_PATH"/devkit/cpp/data/odometry/poses/$d.txt"\ --output "$KITTI_RESULTS_PATH/$d"\ diff --git a/docker/jfr2018/run_tum_datasets.sh b/docker/jfr2018/run_tum_datasets.sh index a55a2f94..8a42bc19 100755 --- a/docker/jfr2018/run_tum_datasets.sh +++ b/docker/jfr2018/run_tum_datasets.sh @@ -83,6 +83,7 @@ do --Mem/UseOdomFeatures false \ --Mem/BinDataKept true \ --Rtabmap/CreateIntermediateNodes false\ + --Vis/CorNNDR 0.6 \ $F2F_params\ --OdomORBSLAM2/VocPath /root/ORBvoc.txt\ --OdomFovis/MinFeaturesForEstimate 10\ diff --git a/guilib/src/DatabaseViewer.cpp b/guilib/src/DatabaseViewer.cpp index de2bb048..cba3e9f7 100644 --- a/guilib/src/DatabaseViewer.cpp +++ b/guilib/src/DatabaseViewer.cpp @@ -79,6 +79,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "rtabmap/gui/EditMapArea.h" #include "rtabmap/core/SensorData.h" #include "rtabmap/core/GainCompensator.h" +#include "rtabmap/core/VisualWord.h" #include "rtabmap/gui/ExportDialog.h" #include "rtabmap/gui/EditConstraintDialog.h" #include "rtabmap/gui/ProgressDialog.h" @@ -1999,8 +2000,28 @@ void DatabaseViewer::updateInfo() { ui_->textEdit_info->append(tr("Total odometry length:\t%1 m").arg(infoTotalOdom_)); } + + int lastWordIdId = 0; + int wordsDim = 0; + int wordsType = 0; + dbDriver_->getLastWordId(lastWordIdId); + if(lastWordIdId>0) + { + std::set ids; + ids.insert(lastWordIdId); + std::list vws; + dbDriver_->loadWords(ids, vws); + if(!vws.empty()) + { + wordsDim = vws.front()->getDescriptor().cols; + wordsType = vws.front()->getDescriptor().type(); + delete vws.front(); + vws.clear(); + } + } + ui_->textEdit_info->append(tr("Total time:\t\t%1").arg(QDateTime::fromMSecsSinceEpoch(infoTotalTime_*1000).toUTC().toString("hh:mm:ss.zzz"))); - ui_->textEdit_info->append(tr("LTM:\t\t%1 nodes and %2 words").arg(ids_.size()).arg(dbDriver_->getTotalDictionarySize())); + ui_->textEdit_info->append(tr("LTM:\t\t%1 nodes and %2 words (dim=%3 type=%4)").arg(ids_.size()).arg(dbDriver_->getTotalDictionarySize()).arg(wordsDim).arg(wordsType==CV_8UC1?"8U":wordsType==CV_32FC1?"32F":uNumber2Str(wordsType).c_str())); ui_->textEdit_info->append(tr("WM:\t\t%1 nodes and %2 words").arg(dbDriver_->getLastNodesSize()).arg(dbDriver_->getLastDictionarySize())); ui_->textEdit_info->append(tr("Global graph:\t%1 poses and %2 links").arg(odomPoses_.size()).arg(links_.size())); ui_->textEdit_info->append(tr("Ground truth:\t%1 poses").arg(groundTruthPoses_.size())); diff --git a/guilib/src/PreferencesDialog.cpp b/guilib/src/PreferencesDialog.cpp index 003ef13e..3ccfbc43 100644 --- a/guilib/src/PreferencesDialog.cpp +++ b/guilib/src/PreferencesDialog.cpp @@ -195,8 +195,10 @@ PreferencesDialog::PreferencesDialog(QWidget * parent) : #ifndef RTABMAP_NONFREE _ui->comboBox_detector_strategy->setItemData(0, 0, Qt::UserRole - 1); _ui->comboBox_detector_strategy->setItemData(12, 0, Qt::UserRole - 1); + _ui->comboBox_detector_strategy->setItemData(14, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(0, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(12, 0, Qt::UserRole - 1); + _ui->vis_feature_detector->setItemData(14, 0, Qt::UserRole - 1); #endif // SIFT @@ -213,11 +215,15 @@ PreferencesDialog::PreferencesDialog(QWidget * parent) : _ui->comboBox_detector_strategy->setItemData(5, 0, Qt::UserRole - 1); _ui->comboBox_detector_strategy->setItemData(6, 0, Qt::UserRole - 1); _ui->comboBox_detector_strategy->setItemData(12, 0, Qt::UserRole - 1); + _ui->comboBox_detector_strategy->setItemData(13, 0, Qt::UserRole - 1); + _ui->comboBox_detector_strategy->setItemData(14, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(3, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(4, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(5, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(6, 0, Qt::UserRole - 1); _ui->vis_feature_detector->setItemData(12, 0, Qt::UserRole - 1); + _ui->vis_feature_detector->setItemData(13, 0, Qt::UserRole - 1); + _ui->vis_feature_detector->setItemData(14, 0, Qt::UserRole - 1); #endif #ifndef RTABMAP_ORB_OCTREE diff --git a/guilib/src/ui/preferencesDialog.ui b/guilib/src/ui/preferencesDialog.ui index db6874cd..4ff24159 100644 --- a/guilib/src/ui/preferencesDialog.ui +++ b/guilib/src/ui/preferencesDialog.ui @@ -63,7 +63,7 @@ 0 - -1179 + -27 686 3236 @@ -95,7 +95,7 @@ QFrame::Raised - 21 + 9 @@ -8969,6 +8969,16 @@ generate the number of words requested. SURF+FREAK + + + GFTT+DAISY + + + + + SURF+DAISY + + @@ -18249,6 +18259,16 @@ Lower the ratio -> higher the precision. SURF+FREAK + + + GFTT+DAISY + + + + + SURF+DAISY + + diff --git a/tools/Info/main.cpp b/tools/Info/main.cpp index 59d6bccb..7df939ff 100644 --- a/tools/Info/main.cpp +++ b/tools/Info/main.cpp @@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include "rtabmap/utilite/UFile.h" #include "rtabmap/utilite/UStl.h" @@ -329,8 +330,27 @@ int main(int argc, char * argv[]) sessionsInOptGraphStr << *iter << "(" << mapIdsLinkedToLastGraph.count(*iter) << ")"; } + int lastWordIdId = 0; + int wordsDim = 0; + int wordsType = 0; + driver->getLastWordId(lastWordIdId); + if(lastWordIdId>0) + { + std::set ids; + ids.insert(lastWordIdId); + std::list vws; + driver->loadWords(ids, vws); + if(!vws.empty()) + { + wordsDim = vws.front()->getDescriptor().cols; + wordsType = vws.front()->getDescriptor().type(); + delete vws.front(); + vws.clear(); + } + } + std::cout << (uFormat("%s%fs\n", pad("Total time:").c_str(), infoTotalTime)); - std::cout << (uFormat("%s%d nodes and %d words\n", pad("LTM:").c_str(), (int)ids.size(), driver->getTotalDictionarySize())); + std::cout << (uFormat("%s%d nodes and %d words (dim=%d type=%s)\n", pad("LTM:").c_str(), (int)ids.size(), driver->getTotalDictionarySize(), wordsDim, wordsType==CV_8UC1?"8U":wordsType==CV_32FC1?"32F":uNumber2Str(wordsType).c_str())); std::cout << (uFormat("%s%d nodes and %d words\n", pad("WM:").c_str(), driver->getLastNodesSize(), driver->getLastDictionarySize())); std::cout << (uFormat("%s%d poses and %d links\n", pad("Global graph:").c_str(), odomPoses, links.size())); std::cout << (uFormat("%s%d poses\n", pad("Optimized graph:").c_str(), (int)optimizedPoses.size(), links.size()));