Added parameters: Mem/ImagePreDecimation Mem/ImagePostDecimation Odom/ImageDecimation

This commit is contained in:
matlabbe
2016-04-08 16:15:08 -04:00
parent 0b3da6b246
commit dece54ca3e
12 changed files with 268 additions and 127 deletions

View File

@@ -83,7 +83,8 @@ Memory::Memory(const ParametersMap & parameters) :
_generateIds(Parameters::defaultMemGenerateIds()),
_badSignaturesIgnored(Parameters::defaultMemBadSignaturesIgnored()),
_mapLabelsAdded(Parameters::defaultMemMapLabelsAdded()),
_imageDecimation(Parameters::defaultMemImageDecimation()),
_imagePreDecimation(Parameters::defaultMemImagePreDecimation()),
_imagePostDecimation(Parameters::defaultMemImagePostDecimation()),
_laserScanDownsampleStepSize(Parameters::defaultMemLaserScanDownsampleStepSize()),
_reextractLoopClosureFeatures(Parameters::defaultRGBDLoopClosureReextractFeatures()),
_rehearsalMaxDistance(Parameters::defaultRGBDLinearUpdate()),
@@ -397,7 +398,8 @@ void Memory::parseParameters(const ParametersMap & parameters)
Parameters::parse(parameters, Parameters::kMemRecentWmRatio(), _recentWmRatio);
Parameters::parse(parameters, Parameters::kMemTransferSortingByWeightId(), _transferSortingByWeightId);
Parameters::parse(parameters, Parameters::kMemSTMSize(), _maxStMemSize);
Parameters::parse(parameters, Parameters::kMemImageDecimation(), _imageDecimation);
Parameters::parse(parameters, Parameters::kMemImagePreDecimation(), _imagePreDecimation);
Parameters::parse(parameters, Parameters::kMemImagePostDecimation(), _imagePostDecimation);
Parameters::parse(parameters, Parameters::kMemLaserScanDownsampleStepSize(), _laserScanDownsampleStepSize);
Parameters::parse(parameters, Parameters::kRGBDLoopClosureReextractFeatures(), _reextractLoopClosureFeatures);
Parameters::parse(parameters, Parameters::kRGBDLinearUpdate(), _rehearsalMaxDistance);
@@ -408,7 +410,8 @@ void Memory::parseParameters(const ParametersMap & parameters)
UASSERT_MSG(_maxStMemSize >= 0, uFormat("value=%d", _maxStMemSize).c_str());
UASSERT_MSG(_similarityThreshold >= 0.0f && _similarityThreshold <= 1.0f, uFormat("value=%f", _similarityThreshold).c_str());
UASSERT_MSG(_recentWmRatio >= 0.0f && _recentWmRatio <= 1.0f, uFormat("value=%f", _recentWmRatio).c_str());
UASSERT(_imageDecimation >= 1);
UASSERT(_imagePreDecimation >= 1);
UASSERT(_imagePostDecimation >= 1);
UASSERT(_rehearsalMaxDistance >= 0.0f);
UASSERT(_rehearsalMaxAngle >= 0.0f);
@@ -3191,31 +3194,52 @@ Signature * Memory::createSignature(const SensorData & data, const Transform & p
preUpdateThread.start();
}
int preDecimation = 1;
std::vector<cv::Point3f> keypoints3D;
if(!_useOdometryFeatures || data.keypoints().empty() || (int)data.keypoints().size() != data.descriptors().rows)
{
if(_feature2D->getMaxFeatures() >= 0 && !data.imageRaw().empty() && !isIntermediateNode)
{
SensorData decimatedData = data;
if(_imagePreDecimation > 1)
{
preDecimation = _imagePreDecimation;
decimatedData.setImageRaw(util2d::decimate(decimatedData.imageRaw(), _imagePreDecimation));
decimatedData.setDepthOrRightRaw(util2d::decimate(decimatedData.depthOrRightRaw(), _imagePreDecimation));
std::vector<CameraModel> cameraModels = decimatedData.cameraModels();
for(unsigned int i=0; i<cameraModels.size(); ++i)
{
cameraModels[i] = cameraModels[i].scaled(1.0/double(_imagePreDecimation));
}
decimatedData.setCameraModels(cameraModels);
StereoCameraModel stereoModel = decimatedData.stereoCameraModel();
if(stereoModel.isValidForProjection())
{
stereoModel.scale(1.0/double(_imagePreDecimation));
}
decimatedData.setStereoCameraModel(stereoModel);
}
UINFO("Extract features");
cv::Mat imageMono;
if(data.imageRaw().channels() == 3)
if(decimatedData.imageRaw().channels() == 3)
{
cv::cvtColor(data.imageRaw(), imageMono, CV_BGR2GRAY);
cv::cvtColor(decimatedData.imageRaw(), imageMono, CV_BGR2GRAY);
}
else
{
imageMono = data.imageRaw();
imageMono = decimatedData.imageRaw();
}
cv::Mat depthMask;
if(!data.depthRaw().empty() &&
if(!decimatedData.depthRaw().empty() &&
_feature2D->getType() != Feature2D::kFeatureOrb) // ORB's mask pyramids don't seem to work well
{
if(imageMono.rows % data.depthRaw().rows == 0 &&
imageMono.cols % data.depthRaw().cols == 0 &&
imageMono.rows/data.depthRaw().rows == imageMono.cols/data.depthRaw().cols)
if(imageMono.rows % decimatedData.depthRaw().rows == 0 &&
imageMono.cols % decimatedData.depthRaw().cols == 0 &&
imageMono.rows/decimatedData.depthRaw().rows == imageMono.cols/decimatedData.depthRaw().cols)
{
depthMask = util2d::interpolate(data.depthRaw(), imageMono.rows/data.depthRaw().rows, 0.1f);
depthMask = util2d::interpolate(decimatedData.depthRaw(), imageMono.rows/decimatedData.depthRaw().rows, 0.1f);
}
}
@@ -3236,10 +3260,10 @@ Signature * Memory::createSignature(const SensorData & data, const Transform & p
{
descriptors = cv::Mat();
}
else if((!data.depthRaw().empty() && data.cameraModels().size() && data.cameraModels()[0].isValidForProjection()) ||
(!data.rightRaw().empty() && data.stereoCameraModel().isValidForProjection()))
else if((!decimatedData.depthRaw().empty() && decimatedData.cameraModels().size() && decimatedData.cameraModels()[0].isValidForProjection()) ||
(!decimatedData.rightRaw().empty() && decimatedData.stereoCameraModel().isValidForProjection()))
{
keypoints3D = _feature2D->generateKeypoints3D(data, keypoints);
keypoints3D = _feature2D->generateKeypoints3D(decimatedData, keypoints);
if(_feature2D->getMinDepth() > 0.0f || _feature2D->getMaxDepth() > 0.0f)
{
UDEBUG("");
@@ -3400,20 +3424,20 @@ Signature * Memory::createSignature(const SensorData & data, const Transform & p
UASSERT(wordIds.size() == keypoints.size());
UASSERT(keypoints3D.size() == 0 || keypoints3D.size() == wordIds.size());
unsigned int i=0;
float decimationRatio = preDecimation / _imagePostDecimation;
for(std::list<int>::iterator iter=wordIds.begin(); iter!=wordIds.end() && i < keypoints.size(); ++iter, ++i)
{
if(_imageDecimation > 1)
cv::KeyPoint kpt = keypoints[i];
if(preDecimation != _imagePostDecimation)
{
cv::KeyPoint kpt = keypoints[i];
kpt.pt.x /= float(_imageDecimation);
kpt.pt.y /= float(_imageDecimation);
kpt.size /= float(_imageDecimation);
words.insert(std::pair<int, cv::KeyPoint>(*iter, kpt));
}
else
{
words.insert(std::pair<int, cv::KeyPoint>(*iter, keypoints[i]));
// remap keypoints to final image size
kpt.pt.x *= decimationRatio;
kpt.pt.y *= decimationRatio;
kpt.size *= decimationRatio;
kpt.octave += log2(preDecimation);
}
words.insert(std::pair<int, cv::KeyPoint>(*iter, kpt));
if(keypoints3D.size())
{
words3D.insert(std::pair<int, cv::Point3f>(*iter, keypoints3D.at(i)));
@@ -3483,17 +3507,17 @@ Signature * Memory::createSignature(const SensorData & data, const Transform & p
StereoCameraModel stereoCameraModel = data.stereoCameraModel();
// apply decimation?
if(_imageDecimation > 1)
if(_imagePostDecimation > 1)
{
image = util2d::decimate(image, _imageDecimation);
depthOrRightImage = util2d::decimate(depthOrRightImage, _imageDecimation);
image = util2d::decimate(image, _imagePostDecimation);
depthOrRightImage = util2d::decimate(depthOrRightImage, _imagePostDecimation);
for(unsigned int i=0; i<cameraModels.size(); ++i)
{
cameraModels[i] = cameraModels[i].scaled(1.0/double(_imageDecimation));
cameraModels[i] = cameraModels[i].scaled(1.0/double(_imagePostDecimation));
}
if(stereoCameraModel.isValidForProjection())
{
stereoCameraModel.scale(1.0/double(_imageDecimation));
stereoCameraModel.scale(1.0/double(_imagePostDecimation));
}
}

View File

@@ -33,6 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "rtabmap/utilite/UTimer.h"
#include "rtabmap/utilite/UConversion.h"
#include "rtabmap/core/ParticleFilter.h"
#include "rtabmap/core/util2d.h"
namespace rtabmap {
@@ -75,6 +76,7 @@ Odometry::Odometry(const rtabmap::ParametersMap & parameters) :
_fillInfoData(Parameters::defaultOdomFillInfoData()),
_kalmanProcessNoise(Parameters::defaultOdomKalmanProcessNoise()),
_kalmanMeasurementNoise(Parameters::defaultOdomKalmanMeasurementNoise()),
_imageDecimation(Parameters::defaultOdomImageDecimation()),
_resetCurrentCount(0),
previousStamp_(0),
distanceTravelled_(0)
@@ -97,6 +99,9 @@ Odometry::Odometry(const rtabmap::ParametersMap & parameters) :
UASSERT(_particleLambdaR>0);
Parameters::parse(parameters, Parameters::kOdomKalmanProcessNoise(), _kalmanProcessNoise);
Parameters::parse(parameters, Parameters::kOdomKalmanMeasurementNoise(), _kalmanMeasurementNoise);
Parameters::parse(parameters, Parameters::kOdomImageDecimation(), _imageDecimation);
UASSERT(_imageDecimation>=1);
if(_filteringStrategy == 2)
{
// Initialize the Particle filters
@@ -223,7 +228,63 @@ Transform Odometry::process(SensorData & data, OdometryInfo * info)
}
UTimer time;
Transform t = this->computeTransform(data, guess, info);
Transform t;
if(_imageDecimation > 1)
{
// Decimation of images with calibrations
SensorData decimatedData = data;
decimatedData.setImageRaw(util2d::decimate(decimatedData.imageRaw(), _imageDecimation));
decimatedData.setDepthOrRightRaw(util2d::decimate(decimatedData.depthOrRightRaw(), _imageDecimation));
std::vector<CameraModel> cameraModels = decimatedData.cameraModels();
for(unsigned int i=0; i<cameraModels.size(); ++i)
{
cameraModels[i] = cameraModels[i].scaled(1.0/double(_imageDecimation));
}
decimatedData.setCameraModels(cameraModels);
StereoCameraModel stereoModel = decimatedData.stereoCameraModel();
if(stereoModel.isValidForProjection())
{
stereoModel.scale(1.0/double(_imageDecimation));
}
decimatedData.setStereoCameraModel(stereoModel);
// compute transform
t = this->computeTransform(decimatedData, guess, info);
// transform back the keypoints in the original image
std::vector<cv::KeyPoint> kpts = decimatedData.keypoints();
for(unsigned int i=0; i<kpts.size(); ++i)
{
kpts[i].pt.x *= _imageDecimation;
kpts[i].pt.y *= _imageDecimation;
kpts[i].size *= _imageDecimation;
kpts[i].octave += log2(_imageDecimation);
}
data.setFeatures(kpts, decimatedData.descriptors());
if(info)
{
UASSERT(info->newCorners.size() == info->refCorners.size());
for(unsigned int i=0; i<info->newCorners.size(); ++i)
{
info->refCorners[i].x *= _imageDecimation;
info->refCorners[i].y *= _imageDecimation;
info->newCorners[i].x *= _imageDecimation;
info->newCorners[i].y *= _imageDecimation;
}
for(std::multimap<int, cv::KeyPoint>::iterator iter=info->words.begin(); iter!=info->words.end(); ++iter)
{
iter->second.pt.x *= _imageDecimation;
iter->second.pt.y *= _imageDecimation;
iter->second.size *= _imageDecimation;
iter->second.octave += log2(_imageDecimation);
}
}
}
else
{
t = this->computeTransform(data, guess, info);
}
if(info)
{

View File

@@ -155,6 +155,9 @@ const std::map<std::string, std::pair<bool, std::string> > & Parameters::getRemo
{
// removed parameters
// 0.11.3
removedParameters_.insert(std::make_pair("Mem/ImageDecimation", std::make_pair(true, Parameters::kMemImagePostDecimation())));
// 0.11.2
removedParameters_.insert(std::make_pair("OdomLocalMap/HistorySize", std::make_pair(true, Parameters::kOdomF2MMaxSize())));
removedParameters_.insert(std::make_pair("OdomLocalMap/FixedMapPath", std::make_pair(true, Parameters::kOdomF2MFixedMapPath())));

View File

@@ -243,33 +243,36 @@ Transform RegistrationVis::computeTransformationImpl(
Feature2D * detector = createFeatureDetector();
std::vector<cv::KeyPoint> kptsFrom;
cv::Mat imageFrom = fromSignature.sensorData().imageRaw();
cv::Mat imageTo = toSignature.sensorData().imageRaw();
if(fromSignature.getWords().empty())
{
if(fromSignature.sensorData().keypoints().empty())
{
if(!fromSignature.sensorData().imageRaw().empty())
if(!imageFrom.empty())
{
if(fromSignature.sensorData().imageRaw().channels() > 1)
if(imageFrom.channels() > 1)
{
cv::Mat tmp;
cv::cvtColor(fromSignature.sensorData().imageRaw(), tmp, cv::COLOR_BGR2GRAY);
fromSignature.sensorData().setImageRaw(tmp);
cv::cvtColor(imageFrom, tmp, cv::COLOR_BGR2GRAY);
imageFrom = tmp;
}
cv::Mat depthMask;
if(!fromSignature.sensorData().depthRaw().empty() &&
detector->getType() != Feature2D::kFeatureOrb) // ORB's mask pyramids don't seem to work well
{
if(fromSignature.sensorData().imageRaw().rows % fromSignature.sensorData().depthRaw().rows == 0 &&
fromSignature.sensorData().imageRaw().cols % fromSignature.sensorData().depthRaw().cols == 0 &&
fromSignature.sensorData().imageRaw().rows/fromSignature.sensorData().depthRaw().rows == fromSignature.sensorData().imageRaw().cols/fromSignature.sensorData().depthRaw().cols)
if(imageFrom.rows % fromSignature.sensorData().depthRaw().rows == 0 &&
imageFrom.cols % fromSignature.sensorData().depthRaw().cols == 0 &&
imageFrom.rows/fromSignature.sensorData().depthRaw().rows == fromSignature.sensorData().imageRaw().cols/fromSignature.sensorData().depthRaw().cols)
{
depthMask = util2d::interpolate(fromSignature.sensorData().depthRaw(), fromSignature.sensorData().imageRaw().rows/fromSignature.sensorData().depthRaw().rows, 0.1f);
}
}
kptsFrom = detector->generateKeypoints(
fromSignature.sensorData().imageRaw(),
imageFrom,
depthMask);
}
}
@@ -290,22 +293,22 @@ Transform RegistrationVis::computeTransformationImpl(
std::multimap<int, cv::Mat> wordsDescFrom;
std::multimap<int, cv::Mat> wordsDescTo;
if(_correspondencesApproach == 1 && //Optical Flow
!fromSignature.sensorData().imageRaw().empty() &&
!toSignature.sensorData().imageRaw().empty())
!imageFrom.empty() &&
!imageTo.empty())
{
UDEBUG("");
// convert to grayscale
if(fromSignature.sensorData().imageRaw().channels() > 1)
if(imageFrom.channels() > 1)
{
cv::Mat tmp;
cv::cvtColor(fromSignature.sensorData().imageRaw(), tmp, cv::COLOR_BGR2GRAY);
fromSignature.sensorData().setImageRaw(tmp);
cv::cvtColor(imageFrom, tmp, cv::COLOR_BGR2GRAY);
imageFrom = tmp;
}
if(toSignature.sensorData().imageRaw().channels() > 1)
if(imageTo.channels() > 1)
{
cv::Mat tmp;
cv::cvtColor(toSignature.sensorData().imageRaw(), tmp, cv::COLOR_BGR2GRAY);
toSignature.sensorData().setImageRaw(tmp);
cv::cvtColor(imageTo, tmp, cv::COLOR_BGR2GRAY);
imageTo = tmp;
}
std::vector<cv::Point3f> kptsFrom3D;
@@ -318,7 +321,7 @@ Transform RegistrationVis::computeTransformationImpl(
kptsFrom3D = uValues(fromSignature.getWords3());
}
if(!toSignature.sensorData().imageRaw().empty())
if(!imageTo.empty())
{
std::vector<cv::Point2f> cornersFrom;
cv::KeyPoint::convert(kptsFrom, cornersFrom);
@@ -345,8 +348,8 @@ Transform RegistrationVis::computeTransformationImpl(
std::vector<float> err;
UDEBUG("cv::calcOpticalFlowPyrLK() begin");
cv::calcOpticalFlowPyrLK(
fromSignature.sensorData().imageRaw(),
toSignature.sensorData().imageRaw(),
imageFrom,
imageTo,
cornersFrom,
cornersTo,
status,
@@ -364,8 +367,8 @@ Transform RegistrationVis::computeTransformationImpl(
for(unsigned int i=0; i<status.size(); ++i)
{
if(status[i] &&
uIsInBounds(cornersTo[i].x, 0.0f, float(toSignature.sensorData().imageRaw().cols)) &&
uIsInBounds(cornersTo[i].y, 0.0f, float(toSignature.sensorData().imageRaw().rows)))
uIsInBounds(cornersTo[i].x, 0.0f, float(imageTo.cols)) &&
uIsInBounds(cornersTo[i].y, 0.0f, float(imageTo.rows)))
{
kptsFrom[ki] = cv::KeyPoint(cornersFrom[i], 1);
kptsFrom3DKept[ki] = kptsFrom3D[i];
@@ -420,29 +423,29 @@ Transform RegistrationVis::computeTransformationImpl(
if(toSignature.getWords().empty())
{
if(toSignature.sensorData().keypoints().empty() &&
!toSignature.sensorData().imageRaw().empty())
!imageTo.empty())
{
if(toSignature.sensorData().imageRaw().channels() > 1)
if(imageTo.channels() > 1)
{
cv::Mat tmp;
cv::cvtColor(toSignature.sensorData().imageRaw(), tmp, cv::COLOR_BGR2GRAY);
toSignature.sensorData().setImageRaw(tmp);
cv::cvtColor(imageTo, tmp, cv::COLOR_BGR2GRAY);
imageTo = tmp;
}
cv::Mat depthMask;
if(!toSignature.sensorData().depthRaw().empty() &&
detector->getType() != Feature2D::kFeatureOrb) // ORB's mask pyramids don't seem to work well
{
if(toSignature.sensorData().imageRaw().rows % toSignature.sensorData().depthRaw().rows == 0 &&
toSignature.sensorData().imageRaw().cols % toSignature.sensorData().depthRaw().cols == 0 &&
toSignature.sensorData().imageRaw().rows/toSignature.sensorData().depthRaw().rows == toSignature.sensorData().imageRaw().cols/toSignature.sensorData().depthRaw().cols)
if(imageTo.rows % toSignature.sensorData().depthRaw().rows == 0 &&
imageTo.cols % toSignature.sensorData().depthRaw().cols == 0 &&
imageTo.rows/toSignature.sensorData().depthRaw().rows == imageTo.cols/toSignature.sensorData().depthRaw().cols)
{
depthMask = util2d::interpolate(toSignature.sensorData().depthRaw(), toSignature.sensorData().imageRaw().rows/toSignature.sensorData().depthRaw().rows, 0.1f);
depthMask = util2d::interpolate(toSignature.sensorData().depthRaw(), imageTo.rows/toSignature.sensorData().depthRaw().rows, 0.1f);
}
}
kptsTo = detector->generateKeypoints(
toSignature.sensorData().imageRaw(),
imageTo,
depthMask);
}
else
@@ -477,15 +480,15 @@ Transform RegistrationVis::computeTransformationImpl(
{
descriptorsFrom = fromSignature.sensorData().descriptors();
}
else if(!fromSignature.sensorData().imageRaw().empty())
else if(!imageFrom.empty())
{
if(fromSignature.sensorData().imageRaw().channels() > 1)
if(imageFrom.channels() > 1)
{
cv::Mat tmp;
cv::cvtColor(fromSignature.sensorData().imageRaw(), tmp, cv::COLOR_BGR2GRAY);
fromSignature.sensorData().setImageRaw(tmp);
cv::cvtColor(imageFrom, tmp, cv::COLOR_BGR2GRAY);
imageFrom = tmp;
}
descriptorsFrom = detector->generateDescriptors(fromSignature.sensorData().imageRaw(), kptsFrom);
descriptorsFrom = detector->generateDescriptors(imageFrom, kptsFrom);
}
cv::Mat descriptorsTo;
@@ -508,16 +511,16 @@ Transform RegistrationVis::computeTransformationImpl(
{
descriptorsTo = toSignature.sensorData().descriptors();
}
else if(!toSignature.sensorData().imageRaw().empty())
else if(!imageTo.empty())
{
if(toSignature.sensorData().imageRaw().channels() > 1)
if(imageTo.channels() > 1)
{
cv::Mat tmp;
cv::cvtColor(toSignature.sensorData().imageRaw(), tmp, cv::COLOR_BGR2GRAY);
toSignature.sensorData().setImageRaw(tmp);
cv::cvtColor(imageTo, tmp, cv::COLOR_BGR2GRAY);
imageTo = tmp;
}
descriptorsTo = detector->generateDescriptors(toSignature.sensorData().imageRaw(), kptsTo);
descriptorsTo = detector->generateDescriptors(imageTo, kptsTo);
}
}
@@ -618,7 +621,7 @@ Transform RegistrationVis::computeTransformationImpl(
// We have all data we need here, so match!
if(descriptorsFrom.rows > 0 && descriptorsTo.rows > 0)
{
cv::Size imageSize = toSignature.sensorData().imageRaw().size();
cv::Size imageSize = imageTo.size();
bool isCalibrated = false;
if(imageSize.height == 0 || imageSize.width == 0)
{
@@ -931,7 +934,7 @@ Transform RegistrationVis::computeTransformationImpl(
float variance = 1.0f;
int inliersCount = 0;
int matchesCount = 0;
if(toSignature.getWords().size() || !toSignature.sensorData().imageRaw().empty())
if(toSignature.getWords().size())
{
Transform transforms[2];
std::vector<int> inliers[2];

View File

@@ -545,12 +545,13 @@ void RtabmapThread::addData(const OdometryEvent & odomEvent)
}
}
if(_dataBufferMaxSize > 0 &&
((!lastPose_.isIdentity() && odomEvent.pose().isIdentity()) ||
(!lastPose_.isIdentity() &&
(odomEvent.pose().isIdentity() ||
odomEvent.info().variance>=9999 ||
odomEvent.rotVariance()>=9999 ||
odomEvent.transVariance()>=9999))
odomEvent.transVariance()>=9999)))
{
UWARN("Odometry is reset (identity pose or high variance (>=9999) detected). Increment map id!");
UWARN("Odometry is reset (identity pose or high variance >=9999 detected). Increment map id!");
pushNewState(kStateTriggeringMap);
_rotVariance = 0;
_transVariance = 0;