i/rtabmap
1
0
Fork 0
mirror of https://github.com/introlab/rtabmap.git synced 2026-10-07 18:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

SuperPoint Rpautrat (MIT license) (#1603)

Browse Source 
* initial python implementation of superpoint rpautrat

* working python implementation of superpoint

* small tweaks to try and work around the GIL issue

* fix missing os import

* begging cpp impl of superpoint python model

* finishing cpp superpoint impl using the same SPDetector interface

* finalizing superpoint cpp impl, working with cpu but needs to be cleaned

* fixing feature matching by reworking nms and filtering logic

* speeding up nms with batched operations and cleaning up

* updating conversion script

* adding args for image dimensions and cuda usage to model tracer

* wiring up UI to parse superpoint params

* adding label to superpoint rpautrat ui

* reverting some unintended ui changes

* typo

* oneline revert

* removing nms and threshold filtering from cpp, this is handled internally by the superpoint model

* using python interface to run the superpoint _to_torchscript.py script at runtime

* generate and load model file on the first incoming frame

* reverting unintented change

* ui changes appear mysteriously again, reverting

* remove topk from model to prevent issue when number of detected keypoints is lower then the k value (scripting fails in this case)

* cleaning up for review

* change dest for model file and remove debug logs

* bump patch and add version comment

* rm unucessary comment

* use resources to load file to ensure it works when rtabmap isn't built from source

* execute with pybind runpy instead of system call

* only build superpoint rpautrat if we have torch and python support

* changing param to accept a path to the weights .pth file directly

* parse the default working directory to save the model file in

* rm extraneous change

* rm setters and re-initialize the detector whenever params change. We cannot support changing params after the model is constructed

* update UI text

* only enable superpoint rpautrat when built with python and torch

* more build information regarding superpoint rpautrat

* generate temporary python script in the working dir

* rm unecessary changes to rtabmap_superpoint.py

* fix comment and only add repo root to sys.path

* introduce a new parameter for the superpoint python model definition

* execute script from string instead of writing to a file

* wrap parse params in a single compiler directive

* remove descriptor spatial matching logic and rely on upstream RTAB-Map processes to take the top-K desc and kpts

* only add python script to resources if built with superpoint rpautrat support. supress warning with type casting

* isolate pybind11 setup so it can't affect any other modules or potential regnerations of the model file

* update about dialog to show superpoint rpautrat

* remove debug logs

---------

Co-authored-by: Felix Toft <[email protected]>
This commit is contained in:
Felix Toft
2025-11-07 17:22:35 -08:00
committed by GitHub
co-authored by Felix Toft
parent f44a4fc478
commit 5e4fd171e2
16 changed files with 2083 additions and 1178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
corelib/src/Features2d.cpp
+108 -4
View File
@@ -47,6 +47,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifdef RTABMAP_TORCH
#include "superpoint_torch/SuperPoint.h"
#endif
#if defined(RTABMAP_TORCH) && defined(RTABMAP_PYTHON)
#include "superpoint_rpautrat/SuperpointRpautrat.h"
#endif
#ifdef RTABMAP_PYTHON
#include "python/PyDetector.h"
@@ -730,9 +733,14 @@ Feature2D * Feature2D::create(Feature2D::Type type, const ParametersMap & parame
feature2D = new ORBOctree(parameters);
break;
#ifdef RTABMAP_TORCH
case Feature2D::kFeatureSuperPointTorch:
feature2D = new SuperPointTorch(parameters);
break;
case Feature2D::kFeatureSuperPointTorch:
feature2D = new SuperPointTorch(parameters);
break;
#endif
#if defined(RTABMAP_TORCH) && defined(RTABMAP_PYTHON)
case Feature2D::kFeatureSuperPointRpautrat:
feature2D = new SuperPointRpautrat(parameters);
break;
#endif
case Feature2D::kFeatureSurfFreak:
feature2D = new SURF_FREAK(parameters);
@@ -831,7 +839,7 @@ std::vector<cv::KeyPoint> Feature2D::generateKeypoints(const cv::Mat & image, co
cv::Rect roi(globalRoi.x + j*colSize, globalRoi.y + i*rowSize, colSize, rowSize);
std::vector<cv::KeyPoint> subKeypoints;
subKeypoints = this->generateKeypointsImpl(image, roi, mask);
if (this->getType() != Feature2D::Type::kFeaturePyDetector)
if (this->getType() != Feature2D::Type::kFeaturePyDetector && this->getType() != Feature2D::Type::kFeatureSuperPointRpautrat)
{
limitKeypoints(subKeypoints, maxFeatures, roi.size(), this->getSSC());
}
@@ -2615,6 +2623,102 @@ cv::Mat SuperPointTorch::generateDescriptorsImpl(const cv::Mat & image, std::vec
}
//////////////////////////
//SuperPointRpautrat
//////////////////////////
SuperPointRpautrat::SuperPointRpautrat(const ParametersMap & parameters) :
superpointWeightsPath_(Parameters::defaultSuperPointRpautratWeightsPath()),
superpointModelPath_(Parameters::defaultSuperPointRpautratModelPath()),
outputDir_(""),
threshold_(Parameters::defaultSuperPointRpautratThreshold()),
nms_(Parameters::defaultSuperPointRpautratNMS()),
minDistance_(Parameters::defaultSuperPointRpautratNMSRadius()),
cuda_(Parameters::defaultSuperPointRpautratCuda())
{
parseParameters(parameters);
}
SuperPointRpautrat::~SuperPointRpautrat()
{
}
void SuperPointRpautrat::parseParameters(const ParametersMap & parameters)
{
Feature2D::parseParameters(parameters);
#if defined(RTABMAP_TORCH) && defined(RTABMAP_PYTHON)
std::string previousWeightsPath = superpointWeightsPath_;
std::string previousModelPath = superpointModelPath_;
bool previousCuda = cuda_;
float previousThreshold = threshold_;
bool previousNms = nms_;
int previousMinDistance = minDistance_;
Parameters::parse(parameters, Parameters::kSuperPointRpautratWeightsPath(), superpointWeightsPath_);
Parameters::parse(parameters, Parameters::kSuperPointRpautratModelPath(), superpointModelPath_);
Parameters::parse(parameters, Parameters::kSuperPointRpautratThreshold(), threshold_);
Parameters::parse(parameters, Parameters::kSuperPointRpautratNMS(), nms_);
Parameters::parse(parameters, Parameters::kSuperPointRpautratNMSRadius(), minDistance_);
Parameters::parse(parameters, Parameters::kSuperPointRpautratCuda(), cuda_);
Parameters::parse(parameters, Parameters::kRtabmapWorkingDirectory(), outputDir_);
// If working directory is not set, use the default
if(outputDir_.empty())
{
outputDir_ = Parameters::createDefaultWorkingDirectory();
}
// Delete the detector to force re-initialization on next frame if any parameter changed
if(superPoint_.get() == 0 ||
superpointWeightsPath_.compare(previousWeightsPath) != 0 ||
superpointModelPath_.compare(previousModelPath) != 0 ||
previousCuda != cuda_ ||
previousThreshold != threshold_ ||
previousNms != nms_ ||
previousMinDistance != minDistance_)
{
superPoint_ = cv::Ptr<SPDetectorRpautrat>(new SPDetectorRpautrat(superpointWeightsPath_, superpointModelPath_, outputDir_, threshold_, nms_, minDistance_, cuda_));
}
#else
UWARN("RTAB-Map is not built with Torch support so SuperPoint Rpautrat feature cannot be used!");
#endif
}
std::vector<cv::KeyPoint> SuperPointRpautrat::generateKeypointsImpl(const cv::Mat & image, const cv::Rect & roi, const cv::Mat & mask)
{
#if defined(RTABMAP_TORCH) && defined(RTABMAP_PYTHON)
UASSERT(!image.empty() && image.channels() == 1 && image.depth() == CV_8U);
if(roi.x!=0 || roi.y !=0)
{
UERROR("SuperPoint Rpautrat: Not supporting ROI (%d,%d,%d,%d). Make sure %s, %s, %s, %s, %s, %s are all set to default values.",
roi.x, roi.y, roi.width, roi.height,
Parameters::kKpRoiRatios().c_str(),
Parameters::kVisRoiRatios().c_str(),
Parameters::kVisGridRows().c_str(),
Parameters::kVisGridCols().c_str(),
Parameters::kKpGridRows().c_str(),
Parameters::kKpGridCols().c_str());
return std::vector<cv::KeyPoint>();
}
return superPoint_->detect(image, mask);
#else
UWARN("RTAB-Map is not built with Torch support so SuperPoint Rpautrat feature cannot be used!");
return std::vector<cv::KeyPoint>();
#endif
}
cv::Mat SuperPointRpautrat::generateDescriptorsImpl(const cv::Mat & image, std::vector<cv::KeyPoint> & keypoints) const
{
#if defined(RTABMAP_TORCH) && defined(RTABMAP_PYTHON)
UASSERT(!image.empty() && image.channels() == 1 && image.depth() == CV_8U);
return superPoint_->compute(keypoints);
#else
UWARN("RTAB-Map is not built with Torch support so SuperPoint Rpautrat feature cannot be used!");
return cv::Mat();
#endif
}
//////////////////////////
//GFTT-DAISY
//////////////////////////