mirror of
https://github.com/introlab/rtabmap.git
synced 2026-09-11 22:10:21 +08:00
SuperPoint Rpautrat (MIT license) (#1603)
* initial python implementation of superpoint rpautrat * working python implementation of superpoint * small tweaks to try and work around the GIL issue * fix missing os import * begging cpp impl of superpoint python model * finishing cpp superpoint impl using the same SPDetector interface * finalizing superpoint cpp impl, working with cpu but needs to be cleaned * fixing feature matching by reworking nms and filtering logic * speeding up nms with batched operations and cleaning up * updating conversion script * adding args for image dimensions and cuda usage to model tracer * wiring up UI to parse superpoint params * adding label to superpoint rpautrat ui * reverting some unintended ui changes * typo * oneline revert * removing nms and threshold filtering from cpp, this is handled internally by the superpoint model * using python interface to run the superpoint _to_torchscript.py script at runtime * generate and load model file on the first incoming frame * reverting unintented change * ui changes appear mysteriously again, reverting * remove topk from model to prevent issue when number of detected keypoints is lower then the k value (scripting fails in this case) * cleaning up for review * change dest for model file and remove debug logs * bump patch and add version comment * rm unucessary comment * use resources to load file to ensure it works when rtabmap isn't built from source * execute with pybind runpy instead of system call * only build superpoint rpautrat if we have torch and python support * changing param to accept a path to the weights .pth file directly * parse the default working directory to save the model file in * rm extraneous change * rm setters and re-initialize the detector whenever params change. We cannot support changing params after the model is constructed * update UI text * only enable superpoint rpautrat when built with python and torch * more build information regarding superpoint rpautrat * generate temporary python script in the working dir * rm unecessary changes to rtabmap_superpoint.py * fix comment and only add repo root to sys.path * introduce a new parameter for the superpoint python model definition * execute script from string instead of writing to a file * wrap parse params in a single compiler directive * remove descriptor spatial matching logic and rely on upstream RTAB-Map processes to take the top-K desc and kpts * only add python script to resources if built with superpoint rpautrat support. supress warning with type casting * isolate pybind11 setup so it can't affect any other modules or potential regnerations of the model file * update about dialog to show superpoint rpautrat * remove debug logs --------- Co-authored-by: Felix Toft <felix@robust.ai>
This commit is contained in:
Felix Toft
@@ -104,6 +104,7 @@ namespace rtabmap {
|
||||
|
||||
class ORBextractor;
|
||||
class SPDetector;
|
||||
class SPDetectorRpautrat;
|
||||
|
||||
class Stereo;
|
||||
#if CV_MAJOR_VERSION < 3
|
||||
@@ -129,7 +130,8 @@ public:
|
||||
kFeatureSurfFreak=12, //new 0.20.4
|
||||
kFeatureGfttDaisy=13, //new 0.20.6
|
||||
kFeatureSurfDaisy=14, //new 0.20.6
|
||||
kFeaturePyDetector=15}; //new 0.20.8
|
||||
kFeaturePyDetector=15, //new 0.20.8
|
||||
kFeatureSuperPointRpautrat=16}; // new 0.23.3
|
||||
|
||||
static std::string typeName(Type type)
|
||||
{
|
||||
@@ -164,6 +166,8 @@ public:
|
||||
return "GFTT+Daisy";
|
||||
case kFeatureSurfDaisy:
|
||||
return "SURF+Daisy";
|
||||
case kFeatureSuperPointRpautrat:
|
||||
return "SUPERPOINT-RPAUTRAT";
|
||||
default:
|
||||
return "Unknown";
|
||||
}
|
||||
@@ -626,6 +630,31 @@ private:
|
||||
bool cuda_;
|
||||
};
|
||||
|
||||
//SuperPointRpautrat
|
||||
class RTABMAP_CORE_EXPORT SuperPointRpautrat : public Feature2D
|
||||
{
|
||||
public:
|
||||
SuperPointRpautrat(const ParametersMap & parameters = ParametersMap());
|
||||
virtual ~SuperPointRpautrat();
|
||||
|
||||
virtual void parseParameters(const ParametersMap & parameters);
|
||||
virtual Feature2D::Type getType() const { return kFeatureSuperPointRpautrat; }
|
||||
|
||||
private:
|
||||
virtual std::vector<cv::KeyPoint> generateKeypointsImpl(const cv::Mat & image, const cv::Rect & roi, const cv::Mat & mask = cv::Mat());
|
||||
virtual cv::Mat generateDescriptorsImpl(const cv::Mat & image, std::vector<cv::KeyPoint> & keypoints) const;
|
||||
|
||||
cv::Ptr<SPDetectorRpautrat> superPoint_;
|
||||
|
||||
std::string superpointWeightsPath_;
|
||||
std::string superpointModelPath_;
|
||||
std::string outputDir_;
|
||||
float threshold_;
|
||||
bool nms_;
|
||||
int minDistance_;
|
||||
bool cuda_;
|
||||
};
|
||||
|
||||
//GFTT_DAISY
|
||||
class RTABMAP_CORE_EXPORT GFTT_DAISY : public GFTT
|
||||
{
|
||||
|
||||
@@ -252,9 +252,9 @@ class RTABMAP_CORE_EXPORT Parameters
|
||||
RTABMAP_PARAM(Kp, NndrRatio, float, 0.8, "NNDR ratio (A matching pair is detected, if its distance is closer than X times the distance of the second nearest neighbor.)");
|
||||
#if CV_MAJOR_VERSION > 2 && !defined(HAVE_OPENCV_XFEATURES2D)
|
||||
// OpenCV>2 without xFeatures2D module doesn't have BRIEF
|
||||
RTABMAP_PARAM(Kp, DetectorStrategy, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector");
|
||||
RTABMAP_PARAM(Kp, DetectorStrategy, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector 16=SuperPoint-Rpautrat");
|
||||
#else
|
||||
RTABMAP_PARAM(Kp, DetectorStrategy, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector");
|
||||
RTABMAP_PARAM(Kp, DetectorStrategy, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector 16=SuperPoint-Rpautrat");
|
||||
#endif
|
||||
RTABMAP_PARAM(Kp, TfIdfLikelihoodUsed, bool, true, "Use of the td-idf strategy to compute the likelihood.");
|
||||
RTABMAP_PARAM(Kp, Parallelized, bool, true, "If the dictionary update and signature creation were parallelized.");
|
||||
@@ -346,6 +346,13 @@ class RTABMAP_CORE_EXPORT Parameters
|
||||
RTABMAP_PARAM(SuperPoint, NMSRadius, int, 4, uFormat("[%s=true] Minimum distance (pixels) between keypoints.", kSuperPointNMS().c_str()));
|
||||
RTABMAP_PARAM(SuperPoint, Cuda, bool, true, "Use Cuda device for Torch, otherwise CPU device is used by default.");
|
||||
|
||||
RTABMAP_PARAM_STR(SuperPointRpautrat, WeightsPath, "", "[Required] SuperPoint weights file (*.pth).");
|
||||
RTABMAP_PARAM_STR(SuperPointRpautrat, ModelPath, "", "[Required] SuperPoint python model file (superpoint_pytorch.py).");
|
||||
RTABMAP_PARAM(SuperPointRpautrat, Threshold, float, 0.005, "Detector response threshold to accept keypoint.");
|
||||
RTABMAP_PARAM(SuperPointRpautrat, NMS, bool, true, "If true, non-maximum suppression is applied to detected keypoints.");
|
||||
RTABMAP_PARAM(SuperPointRpautrat, NMSRadius, int, 4, uFormat("[%s=true] Minimum distance (pixels) between keypoints.", kSuperPointRpautratNMS().c_str()));
|
||||
RTABMAP_PARAM(SuperPointRpautrat, Cuda, bool, true, "Use Cuda device for Torch, otherwise CPU device is used by default.");
|
||||
|
||||
RTABMAP_PARAM_STR(PyDetector, Path, "", "Path to python script file (see available ones in rtabmap/corelib/src/python/*). See the header to see where the script should be copied.");
|
||||
RTABMAP_PARAM(PyDetector, Cuda, bool, true, "Use cuda.");
|
||||
|
||||
@@ -704,9 +711,9 @@ class RTABMAP_CORE_EXPORT Parameters
|
||||
RTABMAP_PARAM(Vis, Iterations, int, 300, "Maximum iterations to compute the transform.");
|
||||
#if CV_MAJOR_VERSION > 2 && !defined(HAVE_OPENCV_XFEATURES2D)
|
||||
// OpenCV>2 without xFeatures2D module doesn't have BRIEF
|
||||
RTABMAP_PARAM(Vis, FeatureType, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector");
|
||||
RTABMAP_PARAM(Vis, FeatureType, int, 8, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector 16=SuperPoint-Rpautrat");
|
||||
#else
|
||||
RTABMAP_PARAM(Vis, FeatureType, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector");
|
||||
RTABMAP_PARAM(Vis, FeatureType, int, 6, "0=SURF 1=SIFT 2=ORB 3=FAST/FREAK 4=FAST/BRIEF 5=GFTT/FREAK 6=GFTT/BRIEF 7=BRISK 8=GFTT/ORB 9=KAZE 10=ORB-OCTREE 11=SuperPoint 12=SURF/FREAK 13=GFTT/DAISY 14=SURF/DAISY 15=PyDetector 16=SuperPoint-Rpautrat");
|
||||
#endif
|
||||
RTABMAP_PARAM(Vis, MaxFeatures, int, 1000, "0 no limits.");
|
||||
RTABMAP_PARAM(Vis, SSC, bool, false, "If true, SSC (Suppression via Square Covering) is applied to limit keypoints.");
|
||||
|
||||
Reference in New Issue
Block a user