i/rtabmap
1
0
Fork 0
mirror of https://github.com/introlab/rtabmap.git synced 2026-09-03 01:50:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adding OpenCV's GPU GFTT/ OpticalFlow and CudaSift support (#1330)

Browse Source 
* Adding GFTT and SIFT Cuda support

* Working CudaSift

* Disable cudasift option when not available

* Added check to avoid re-allocating gpu memory everytime parseParameters is called. Added workaround of to detect/ignore invalid descriptors

* Added SIFT/PreciseUpscale and SIFT/Upscale parameters. Adjusted max octave to behave more like opencv

* Refactored how maximum features are thresholded, to be more similar to OpenCV version

* Updated loop closure benchmark scripts

* Cuda optical flow tmp commit

* Added Stereo/Gpu Vis/CorFlowGpu parameters (optical flow gpu integration for F2F odom and stereo correspondences)

* Fixed build without opencv cuda

* Fixed build with Opencv 4.10

* ZED: updated parameters to match zed sdk 4

* MRPT requires C++17

* updated max octave limit CudaSift
This commit is contained in:
matlabbe
2024-09-13 14:22:54 -07:00
committed by GitHub
parent f3ccfcb452
commit 69ac21f811
37 changed files with 2253 additions and 1375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
corelib/include/rtabmap/core/Features2d.h
View File

@@ -45,6 +45,7 @@ namespace gpu {
class SURF_GPU;
class ORB_GPU;
class FAST_GPU;
class GoodFeaturesToTrackDetector_GPU;
}
}
typedef cv::SIFT CV_SIFT;
@@ -57,13 +58,14 @@ typedef cv::BRISK CV_BRISK;
typedef cv::gpu::SURF_GPU CV_SURF_GPU;
typedef cv::gpu::ORB_GPU CV_ORB_GPU;
typedef cv::gpu::FAST_GPU CV_FAST_GPU;
typedef cv::gpu::GoodFeaturesToTrackDetector_GPU CV_GFTT_GPU;
#else
namespace cv{
namespace xfeatures2d {
class FREAK;
class DAISY;
class BriefDescriptorExtractor;
#if CV_MAJOR_VERSION < 3 || (CV_MAJOR_VERSION == 4 && CV_MINOR_VERSION <= 3) || (CV_MAJOR_VERSION == 3 && (CV_MINOR_VERSION < 4 || (CV_MINOR_VERSION==4 && CV_SUBMINOR_VERSION<11)))
#if (CV_MAJOR_VERSION == 4 && CV_MINOR_VERSION <= 3) || (CV_MAJOR_VERSION == 3 && (CV_MINOR_VERSION < 4 || (CV_MINOR_VERSION==4 && CV_SUBMINOR_VERSION<11)))
class SIFT;
#endif
class SURF;
@@ -72,9 +74,10 @@ namespace cuda {
class FastFeatureDetector;
class ORB;
class SURF_CUDA;
class CornersDetector;
}
}
#if CV_MAJOR_VERSION < 3 || (CV_MAJOR_VERSION == 4 && CV_MINOR_VERSION <= 3) || (CV_MAJOR_VERSION == 3 && (CV_MINOR_VERSION < 4 || (CV_MINOR_VERSION==4 && CV_SUBMINOR_VERSION<11)))
#if (CV_MAJOR_VERSION == 4 && CV_MINOR_VERSION <= 3) || (CV_MAJOR_VERSION == 3 && (CV_MINOR_VERSION < 4 || (CV_MINOR_VERSION==4 && CV_SUBMINOR_VERSION<11)))
typedef cv::xfeatures2d::SIFT CV_SIFT;
#else
typedef cv::SIFT CV_SIFT; // SIFT is back in features2d since 4.4.0 / 3.4.11
@@ -90,8 +93,12 @@ typedef cv::ORB CV_ORB;
typedef cv::cuda::SURF_CUDA CV_SURF_GPU;
typedef cv::cuda::ORB CV_ORB_GPU;
typedef cv::cuda::FastFeatureDetector CV_FAST_GPU;
typedef cv::cuda::CornersDetector CV_GFTT_GPU;
#endif
// CudaSift fork: https://github.com/matlabbe/CudaSift
class SiftData;
namespace rtabmap {
class ORBextractor;
@@ -294,9 +301,18 @@ private:
double contrastThreshold_;
double edgeThreshold_;
double sigma_;
bool preciseUpscale_;
bool rootSIFT_;
bool gpu_;
float guaussianThreshold_;
bool upscale_;
cv::Ptr<CV_SIFT> _sift;
cv::Ptr<CV_SIFT> sift_;
SiftData * cudaSiftData_;
float * cudaSiftMemory_;
cv::Size cudaSiftMemorySize_;
cv::Mat cudaSiftDescriptors_;
bool cudaSiftUpscaling_;
};
//ORB
@@ -425,8 +441,10 @@ private:
int _blockSize;
bool _useHarrisDetector;
double _k;
bool _gpu;
cv::Ptr<CV_GFTT> _gftt;
cv::Ptr<CV_GFTT_GPU> _gpuGftt;
};
//GFTT_BRIEF

14
corelib/include/rtabmap/core/Parameters.h
View File

@@ -281,12 +281,15 @@ class RTABMAP_CORE_EXPORT Parameters
RTABMAP_PARAM(SURF, GpuVersion, bool, false, "GPU-SURF: Use GPU version of SURF. This option is enabled only if OpenCV is built with CUDA and GPUs are detected.");
RTABMAP_PARAM(SURF, GpuKeypointsRatio, float, 0.01, "Used with SURF GPU.");
RTABMAP_PARAM(SIFT, NFeatures, int, 0, "The number of best features to retain. The features are ranked by their scores (measured in SIFT algorithm as the local contrast).");
RTABMAP_PARAM(SIFT, NOctaveLayers, int, 3, "The number of layers in each octave. 3 is the value used in D. Lowe paper. The number of octaves is computed automatically from the image resolution.");
RTABMAP_PARAM(SIFT, ContrastThreshold, double, 0.04, "The contrast threshold used to filter out weak features in semi-uniform (low-contrast) regions. The larger the threshold, the less features are produced by the detector.");
RTABMAP_PARAM(SIFT, NOctaveLayers, int, 3, "The number of layers in each octave. 3 is the value used in D. Lowe paper. The number of octaves is computed automatically from the image resolution. Not used by CudaSift, the number of octaves is still computed automatically.");
RTABMAP_PARAM(SIFT, ContrastThreshold, double, 0.04, uFormat("The contrast threshold used to filter out weak features in semi-uniform (low-contrast) regions. The larger the threshold, the less features are produced by the detector. Not used by CudaSift (see %s instead).", kSIFTGaussianThreshold().c_str()));
RTABMAP_PARAM(SIFT, EdgeThreshold, double, 10, "The threshold used to filter out edge-like features. Note that the its meaning is different from the contrastThreshold, i.e. the larger the edgeThreshold, the less features are filtered out (more features are retained).");
RTABMAP_PARAM(SIFT, Sigma, double, 1.6, "The sigma of the Gaussian applied to the input image at the octave #0. If your image is captured with a weak camera with soft lenses, you might want to reduce the number.");
RTABMAP_PARAM(SIFT, RootSIFT, bool, false, "Apply RootSIFT normalization of the descriptors.");
RTABMAP_PARAM(SIFT, PreciseUpscale, bool, false, "Whether to enable precise upscaling in the scale pyramid (OpenCV >= 4.8).");
RTABMAP_PARAM(SIFT, RootSIFT, bool, false, "Apply RootSIFT normalization of the descriptors.");
RTABMAP_PARAM(SIFT, Gpu, bool, false, "CudaSift: Use GPU version of SIFT. This option is enabled only if RTAB-Map is built with CudaSift dependency and GPUs are detected.");
RTABMAP_PARAM(SIFT, GaussianThreshold, float, 2.0, "CudaSift: Threshold on difference of Gaussians for feature pruning. The higher the threshold, the less features are produced by the detector.");
RTABMAP_PARAM(SIFT, Upscale, bool, false, "CudaSift: Whether to enable upscaling.");
RTABMAP_PARAM(BRIEF, Bytes, int, 32, "Bytes is a length of descriptor in bytes. It can be equal 16, 32 or 64 bytes.");
@@ -305,6 +308,7 @@ class RTABMAP_CORE_EXPORT Parameters
RTABMAP_PARAM(GFTT, BlockSize, int, 3, "");
RTABMAP_PARAM(GFTT, UseHarrisDetector, bool, false, "");
RTABMAP_PARAM(GFTT, K, double, 0.04, "");
RTABMAP_PARAM(GFTT, Gpu, bool, false, "GPU-GFTT: Use GPU version of GFTT. This option is enabled only if OpenCV>=3 is built with CUDA and GPUs are detected.");
RTABMAP_PARAM(ORB, ScaleFactor, float, 2, "Pyramid decimation ratio, greater than 1. scaleFactor==2 means the classical pyramid, where each next level has 4x less pixels than the previous, but such a big scale factor will degrade feature matching scores dramatically. On the other hand, too close to 1 scale factor will mean that to cover certain scale range you will need more pyramid levels and so the speed will suffer.");
RTABMAP_PARAM(ORB, NLevels, int, 3, "The number of pyramid levels. The smallest level will have linear size equal to input_image_linear_size/pow(scaleFactor, nlevels).");
@@ -714,6 +718,7 @@ class RTABMAP_CORE_EXPORT Parameters
RTABMAP_PARAM(Vis, CorFlowIterations, int, 30, uFormat("[%s=1] See cv::calcOpticalFlowPyrLK(). Used for optical flow approach.", kVisCorType().c_str()));
RTABMAP_PARAM(Vis, CorFlowEps, float, 0.01, uFormat("[%s=1] See cv::calcOpticalFlowPyrLK(). Used for optical flow approach.", kVisCorType().c_str()));
RTABMAP_PARAM(Vis, CorFlowMaxLevel, int, 3, uFormat("[%s=1] See cv::calcOpticalFlowPyrLK(). Used for optical flow approach.", kVisCorType().c_str()));
RTABMAP_PARAM(Vis, CorFlowGpu, bool, false, uFormat("[%s=1] Enable GPU version of the optical flow approach (only available if OpenCV is built with CUDA).", kVisCorType().c_str()));
#if defined(RTABMAP_G2O) || defined(RTABMAP_ORB_SLAM)
RTABMAP_PARAM(Vis, BundleAdjustment, int, 1, "Optimization with bundle adjustment: 0=disabled, 1=g2o, 2=cvsba, 3=Ceres.");
#else
@@ -791,6 +796,7 @@ class RTABMAP_CORE_EXPORT Parameters
RTABMAP_PARAM(Stereo, OpticalFlow, bool, true, "Use optical flow to find stereo correspondences, otherwise a simple block matching approach is used.");
RTABMAP_PARAM(Stereo, SSD, bool, true, uFormat("[%s=false] Use Sum of Squared Differences (SSD) window, otherwise Sum of Absolute Differences (SAD) window is used.", kStereoOpticalFlow().c_str()));
RTABMAP_PARAM(Stereo, Eps, double, 0.01, uFormat("[%s=true] Epsilon stop criterion.", kStereoOpticalFlow().c_str()));
RTABMAP_PARAM(Stereo, Gpu, bool, false, uFormat("[%s=true] Enable GPU version of the optical flow approach (only available if OpenCV is built with CUDA).", kStereoOpticalFlow().c_str()));
RTABMAP_PARAM(Stereo, DenseStrategy, int, 0, "0=cv::StereoBM, 1=cv::StereoSGBM");

1
corelib/include/rtabmap/core/RegistrationVis.h
View File

@@ -91,6 +91,7 @@ private:
int _flowIterations;
float _flowEps;
int _flowMaxLevel;
bool _flowGpu;
float _nndr;
int _nnType;
bool _gmsWithRotation;

15
corelib/include/rtabmap/core/SensorData.h
View File

@@ -314,6 +314,13 @@ public:
bool isPointVisibleFromCameras(const cv::Point3f & pt) const; // assuming point is in robot frame
#ifdef HAVE_OPENCV_CUDEV
const cv::cuda::GpuMat & imageRawGpu() const {return _imageRawGpu;}
void setImageRawGpu(const cv::cuda::GpuMat & image) {_imageRawGpu = image;}
const cv::cuda::GpuMat & depthOrRightRawGpu() const {return _depthOrRightRawGpu;}
void setDepthOrRightRawGpu(const cv::cuda::GpuMat & image) {_depthOrRightRawGpu = image;}
#endif
private:
int _id;
double _stamp;
@@ -365,6 +372,14 @@ private:
GPS gps_;
IMU imu_;
#ifdef HAVE_OPENCV_CUDEV
// Temporary buffers used for some optimizations,
// particulary to avoid host<->device copies if same
// data are re-used
cv::cuda::GpuMat _imageRawGpu;
cv::cuda::GpuMat _depthOrRightRawGpu;
#endif
};
}

24
corelib/include/rtabmap/core/Stereo.h
View File

@@ -49,6 +49,13 @@ public:
const cv::Mat & rightImage,
const std::vector<cv::Point2f> & leftCorners,
std::vector<unsigned char> & status) const;
#ifdef HAVE_OPENCV_CUDEV
virtual std::vector<cv::Point2f> computeCorrespondences(
const cv::cuda::GpuMat & leftImage,
const cv::cuda::GpuMat & rightImage,
const std::vector<cv::Point2f> & leftCorners,
std::vector<unsigned char> & status) const;
#endif
cv::Size winSize() const {return cv::Size(winWidth_, winHeight_);}
int iterations() const {return iterations_;}
@@ -56,6 +63,7 @@ public:
float minDisparity() const {return minDisparity_;}
float maxDisparity() const {return maxDisparity_;}
bool winSSD() const {return winSSD_;}
virtual bool isGpuEnabled() const {return false;}
private:
int winWidth_;
@@ -78,11 +86,27 @@ public:
const cv::Mat & rightImage,
const std::vector<cv::Point2f> & leftCorners,
std::vector<unsigned char> & status) const;
#ifdef HAVE_OPENCV_CUDEV
virtual std::vector<cv::Point2f> computeCorrespondences(
const cv::cuda::GpuMat & leftImage,
const cv::cuda::GpuMat & rightImage,
const std::vector<cv::Point2f> & leftCorners,
std::vector<unsigned char> & status) const;
#endif
float epsilon() const {return epsilon_;}
virtual bool isGpuEnabled() const;
private:
void updateStatus(
const std::vector<cv::Point2f> & leftCorners,
const std::vector<cv::Point2f> & rightCorners,
std::vector<unsigned char> & status) const;
private:
float epsilon_;
bool gpu_;
};
} /* namespace rtabmap */

2
corelib/include/rtabmap/core/camera/CameraStereoZed.h
View File

@@ -49,7 +49,7 @@ public:
public:
CameraStereoZed(
int deviceId,
int resolution = 6, // 0=HD2K, 1=HD1080, 2=HD1200, 3=HD720, 4=SVGA, 5=VGA, 6=AUTO
int resolution = -1, // -1 = AUTO, 0=HD4K 1=HD2K 2=HD1080 3=HD1200 4=HD720 5=SVGA 6=VGA
int quality = 1, // 0=NONE, 1=PERFORMANCE, 2=QUALITY
int sensingMode = 0,// 0=STANDARD, 1=FILL
int confidenceThr = 100,