/*
* Copyright (C) 2010-2011, Mathieu Labbe and IntRoLab - Universite de Sherbrooke
*
* This file is part of RTAB-Map.
*
* RTAB-Map is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* RTAB-Map is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with RTAB-Map. If not, see .
*/
#include "rtabmap/core/KeypointDescriptor.h"
#include "utilite/UStl.h"
#include "utilite/UConversion.h"
#include "utilite/ULogger.h"
#include "utilite/UMath.h"
#include "utilite/ULogger.h"
#include
#include
#include
#define OPENCV_SURF_GPU CV_MAJOR_VERSION >= 2 and CV_MINOR_VERSION >=2 and CV_SUBMINOR_VERSION>=1
namespace rtabmap {
KeypointDescriptor::KeypointDescriptor(const ParametersMap & parameters, KeypointDescriptor * childDescriptor) :
_childDescriptor(childDescriptor)
{
this->parseParameters(parameters);
}
KeypointDescriptor::~KeypointDescriptor()
{
if(_childDescriptor)
{
delete _childDescriptor;
}
}
void KeypointDescriptor::parseParameters(const ParametersMap & parameters)
{
if(_childDescriptor)
{
_childDescriptor->parseParameters(parameters);
}
}
std::list > KeypointDescriptor::generateDescriptors(const IplImage * image, const std::list & keypoints) const
{
ULOGGER_DEBUG("");
// see decorator pattern...
std::list > descriptors = this->_generateDescriptors(image, keypoints);
std::list > childDescriptors;
if(_childDescriptor)
{
childDescriptors = _childDescriptor->generateDescriptors(image, keypoints);
if(childDescriptors.size() && childDescriptors.size() == descriptors.size())
{
std::list >::iterator iterDesc = descriptors.begin();
std::list >::iterator iterChild = childDescriptors.begin();
for(; iterDesc!=descriptors.end(); ++iterDesc, ++iterChild)
{
iterDesc->insert(iterDesc->end(), iterChild->begin(), iterChild->end());
}
}
}
return descriptors;
}
void KeypointDescriptor::setChildDescriptor(KeypointDescriptor * childDescriptor)
{
if(_childDescriptor)
{
delete _childDescriptor;
}
_childDescriptor = childDescriptor;
}
//////////////////////////
//SURFDescriptor
//////////////////////////
SURFDescriptor::SURFDescriptor(const ParametersMap & parameters, KeypointDescriptor * childDescriptor) :
KeypointDescriptor(parameters, childDescriptor)
{
_surf.hessianThreshold = Parameters::defaultSURFHessianThreshold();
_surf.extended = Parameters::defaultSURFExtended();
_surf.nOctaveLayers = Parameters::defaultSURFOctaveLayers();
_surf.nOctaves = Parameters::defaultSURFOctaves();
_gpuVersion = Parameters::defaultSURFGpuVersion();
_upright = Parameters::defaultSURFUpright();
this->parseParameters(parameters);
}
SURFDescriptor::~SURFDescriptor()
{
}
void SURFDescriptor::parseParameters(const ParametersMap & parameters)
{
ParametersMap::const_iterator iter;
if((iter=parameters.find(Parameters::kSURFExtended())) != parameters.end())
{
_surf.extended = uStr2Bool((*iter).second.c_str());
}
if((iter=parameters.find(Parameters::kSURFHessianThreshold())) != parameters.end())
{
_surf.hessianThreshold = std::atof((*iter).second.c_str()); // is it needed for the descriptor?
}
if((iter=parameters.find(Parameters::kSURFOctaveLayers())) != parameters.end())
{
_surf.nOctaveLayers = std::atoi((*iter).second.c_str()); // is it needed for the descriptor?
}
if((iter=parameters.find(Parameters::kSURFOctaves())) != parameters.end())
{
_surf.nOctaves = std::atoi((*iter).second.c_str()); // is it needed for the descriptor?
}
if((iter=parameters.find(Parameters::kSURFGpuVersion())) != parameters.end())
{
_gpuVersion = uStr2Bool((*iter).second.c_str());
}
if((iter=parameters.find(Parameters::kSURFUpright())) != parameters.end())
{
_upright = uStr2Bool((*iter).second.c_str());
}
KeypointDescriptor::parseParameters(parameters);
}
std::list > SURFDescriptor::_generateDescriptors(const IplImage * image, const std::list & keypoints) const
{
ULOGGER_DEBUG("");
std::list > descriptors;
if(!image)
{
ULOGGER_ERROR("Image is null ?!?");
return descriptors;
}
// SURF support only grayscale images
IplImage * imageGrayScale = 0;
if(image->nChannels != 1 || image->depth != IPL_DEPTH_8U)
{
imageGrayScale = cvCreateImage(cvSize(image->width,image->height), IPL_DEPTH_8U, 1);
cvCvtColor(image, imageGrayScale, CV_BGR2GRAY);
}
cv::Mat img;
if(imageGrayScale)
{
img = cv::Mat(imageGrayScale);
}
else
{
img = cv::Mat(image);
}
cv::Mat mask;
std::vector k = uListToVector(keypoints);
std::vector d;
#if OPENCV_SURF_GPU
if(_gpuVersion)
{
cv::gpu::GpuMat imgGpu(img);
cv::gpu::GpuMat descriptorsGpu;
cv::gpu::GpuMat keypointsGpu;
cv::gpu::SURF_GPU surfGpu(_surf.hessianThreshold, _surf.nOctaves, _surf.nOctaveLayers, _surf.extended, 0.01f, _upright);
surfGpu.uploadKeypoints(k, keypointsGpu);
surfGpu(imgGpu, cv::gpu::GpuMat(), keypointsGpu, descriptorsGpu, true);
surfGpu.downloadDescriptors(descriptorsGpu, d);
}
else
{
_surf(img, mask, k, d, true); // Opencv surf descriptors
}
#else
_surf(img, mask, k, d, true); // Opencv surf descriptors
#endif
unsigned int dim = _surf.descriptorSize();
for(unsigned int i=0; i(d.begin()+i, d.begin()+i+dim));
}
if(imageGrayScale)
{
cvReleaseImage(&imageGrayScale);
}
return descriptors;
}
//////////////////////////
//SIFTDescriptor
//////////////////////////
SIFTDescriptor::SIFTDescriptor(const ParametersMap & parameters, KeypointDescriptor * childDescriptor) :
KeypointDescriptor(parameters, childDescriptor)
{
this->parseParameters(parameters);
}
SIFTDescriptor::~SIFTDescriptor()
{
}
void SIFTDescriptor::parseParameters(const ParametersMap & parameters)
{
ParametersMap::const_iterator iter;
KeypointDescriptor::parseParameters(parameters);
}
std::list > SIFTDescriptor::_generateDescriptors(const IplImage * image, const std::list & keypoints) const
{
ULOGGER_DEBUG("");
std::list > descriptors;
if(!image)
{
ULOGGER_ERROR("Image is null ?!?");
return descriptors;
}
// SURF support only grayscale images
IplImage * imageGrayScale = 0;
if(image->nChannels != 1 || image->depth != IPL_DEPTH_8U)
{
imageGrayScale = cvCreateImage(cvSize(image->width,image->height), IPL_DEPTH_8U, 1);
cvCvtColor(image, imageGrayScale, CV_BGR2GRAY);
}
cv::Mat img;
if(imageGrayScale)
{
img = cv::Mat(imageGrayScale);
}
else
{
img = cv::Mat(image);
}
cv::Mat mask;
std::vector k = uListToVector(keypoints);
cv::Mat d;
cv::SIFT sift(_commonParams, cv::SIFT::DetectorParams(), _descriptorParams);
sift(img, mask, k, d, true); // Opencv surf descriptors
unsigned int dim = sift.descriptorSize();
//ULOGGER_DEBUG("row=%d, col=%d, type=%d (float=%d)", d.rows, d.cols, d.type(), CV_32F);
for(int i=0; i(d.ptr(i), d.ptr(i)+dim));
}
if(imageGrayScale)
{
cvReleaseImage(&imageGrayScale);
}
return descriptors;
}
//////////////////////////
//LaplacianDescriptor
//////////////////////////
LaplacianDescriptor::LaplacianDescriptor(const ParametersMap & parameters, KeypointDescriptor * childDescriptor) :
KeypointDescriptor(parameters, childDescriptor)
{
this->parseParameters(parameters);
}
LaplacianDescriptor::~LaplacianDescriptor()
{
}
void LaplacianDescriptor::parseParameters(const ParametersMap & parameters)
{
// No parameter...
KeypointDescriptor::parseParameters(parameters);
}
std::list > LaplacianDescriptor::_generateDescriptors(const IplImage * image, const std::list & keypoints) const
{
ULOGGER_DEBUG("");
std::list > descriptors;
//create descriptors...
for(std::list::const_iterator key=keypoints.begin(); key!=keypoints.end(); ++key)
{
std::vector laplacian(1);
laplacian[0] = uSign(key->response);
descriptors.push_back(laplacian);
}
return descriptors;
}
//////////////////////////
//ColorDescriptor
//////////////////////////
ColorDescriptor::ColorDescriptor(const ParametersMap & parameters, KeypointDescriptor * childDescriptor) :
KeypointDescriptor(parameters, childDescriptor)
{
this->parseParameters(parameters);
}
ColorDescriptor::~ColorDescriptor()
{
}
void ColorDescriptor::parseParameters(const ParametersMap & parameters)
{
// No parameter...
KeypointDescriptor::parseParameters(parameters);
}
std::list > ColorDescriptor::_generateDescriptors(const IplImage * image, const std::list & keypoints) const
{
ULOGGER_DEBUG("");
std::list > descriptors;
if(!image)
{
ULOGGER_ERROR("Image is null ?!?");
return descriptors;
}
IplImage * imageConverted = 0;
if(image->nChannels != 3 || image->depth != IPL_DEPTH_8U)
{
imageConverted = cvCreateImage(cvSize(image->width,image->height), IPL_DEPTH_8U, 3);
cvCvtColor(image, imageConverted, CV_GRAY2BGR);
}
cv::Mat imgMat;
if(imageConverted)
{
imgMat = cv::Mat(imageConverted);
}
else
{
imgMat = cv::Mat(image);
}
//create descriptors...
for(std::list::const_iterator key=keypoints.begin(); key!=keypoints.end(); ++key)
{
int grayMax = -1; // grayValue
int grayMin = -1; // grayValue
float d[6] = {0};
std::vector RxV;
cv::Point center = cv::Point(cvRound(key->pt.x), cvRound(key->pt.y));
int R = cvRound(key->size*1.2/9.*2);
this->getCircularROI(R, RxV);
cv::Mat_& img = (cv::Mat_&)imgMat; //3 channel pointer to image
// find the brighter and darker pixels
for( int dy = -R; dy <= R; ++dy )
{
int Rx = RxV[abs(dy)];
for( int dx = -Rx; dx <= Rx; ++dx )
{
if(center.y+dy < img.rows && center.y+dy >= 0 && center.x+dx < img.cols && center.x+dx >= 0)
{
//bgr
uchar b = img(center.y+dy, center.x+dx)[0];
uchar g = img(center.y+dy, center.x+dx)[1];
uchar r = img(center.y+dy, center.x+dx)[2];
int gray = b*0.114 + g*0.587 + r*0.299;
if(grayMax<0 || gray > grayMax)
{
grayMax = gray;
d[0] = b;
d[1] = g;
d[2] = r;
}
if(grayMin<0 || gray < grayMin)
{
grayMin = gray;
d[3] = b;
d[4] = g;
d[5] = r;
}
}
else
{
//ULOGGER_WARN("The keypoint size is outside of the image ranges (x,y)=(%d,%d) radius=%d", center.y+dy, center.x+dx, R);
}
}
}
for(int i=0; i<6; ++i)
{
d[i] /= 255; // Normalize between 0 and 1
}
descriptors.push_back(std::vector(d, d + sizeof(d) / sizeof(float)));
}
if(imageConverted)
{
cvReleaseImage(&imageConverted);
}
return descriptors;
}
// the function returns x boundary coordinates of
// the circle for each y. RxV[y1] = x1 means that
// when y=y1, -x1 <=x<=x1 is inside the circle
// (from OpenCv doc, C++ Cheatsheet)
void ColorDescriptor::getCircularROI(int R, std::vector & RxV) const
{
RxV.resize(R+1);
for( int y = 0; y <= R; y++ )
RxV[y] = cvRound(sqrt(double(R*R - y*y)));
}
//////////////////////////
//HueDescriptor
//////////////////////////
HueDescriptor::HueDescriptor(const ParametersMap & parameters, KeypointDescriptor * childDescriptor) :
ColorDescriptor(parameters, childDescriptor)
{
this->parseParameters(parameters);
}
HueDescriptor::~HueDescriptor()
{
}
void HueDescriptor::parseParameters(const ParametersMap & parameters)
{
// No parameter...
KeypointDescriptor::parseParameters(parameters);
}
std::list > HueDescriptor::_generateDescriptors(const IplImage * image, const std::list & keypoints) const
{
ULOGGER_DEBUG("");
std::list > descriptors;
if(!image)
{
ULOGGER_ERROR("Image is null ?!?");
return descriptors;
}
IplImage * imageConverted = 0;
if(image->nChannels != 3 || image->depth != IPL_DEPTH_8U)
{
imageConverted = cvCreateImage(cvSize(image->width,image->height), IPL_DEPTH_8U, 3);
cvCvtColor(image, imageConverted, CV_GRAY2BGR);
}
cv::Mat imgMat;
if(imageConverted)
{
imgMat = cv::Mat(imageConverted);
}
else
{
imgMat = cv::Mat(image);
}
//create descriptors...
for(std::list::const_iterator key=keypoints.begin(); key!=keypoints.end(); ++key)
{
int intensityMax = -1;
int intensityMin = -1;
float d[2] = {0};
std::vector RxV;
cv::Point center = cv::Point(cvRound(key->pt.x), cvRound(key->pt.y));
int R = cvRound(key->size*1.2/9.*2);
this->getCircularROI(R, RxV);
cv::Mat_& img = (cv::Mat_&)imgMat; //3 channel pointer to image
// find the brighter and darker pixels using the intensity
int dxb=0;
int dyb=0;
int dxd=0;
int dyd=0;
for( int dy = -R; dy <= R; ++dy )
{
int Rx = RxV[abs(dy)];
for( int dx = -Rx; dx <= Rx; ++dx )
{
if(center.y+dy < img.rows && center.y+dy >= 0 && center.x+dx < img.cols && center.x+dx >= 0)
{
//bgr
float b = float(img(center.y+dy, center.x+dx)[0]) / 255.0f;
float g = float(img(center.y+dy, center.x+dx)[1]) / 255.0f;
float r = float(img(center.y+dy, center.x+dx)[2]) / 255.0f;
int intensity = rgb2intensity(r, g, b);
if(intensityMax<0 || intensity > intensityMax)
{
intensityMax = intensity;
dxb = dx;
dyb = dy;
}
if(intensityMin<0 || intensity < intensityMin)
{
intensityMin = intensity;
dxd = dx;
dyd = dy;
}
}
else
{
//ULOGGER_WARN("The keypoint size is outside of the image ranges (x,y)=(%d,%d) radius=%d", center.y+dy, center.x+dx, R);
}
}
}
// brighter
float b = float(img(center.y+dyb, center.x+dxb)[0]) / 255.0f;
float g = float(img(center.y+dyb, center.x+dxb)[1]) / 255.0f;
float r = float(img(center.y+dyb, center.x+dxb)[2]) / 255.0f;
d[0] = rgb2hue(r, g, b);
// darker
b = float(img(center.y+dyd, center.x+dxd)[0]) / 255.0f;
g = float(img(center.y+dyd, center.x+dxd)[1]) / 255.0f;
r = float(img(center.y+dyd, center.x+dxd)[2]) / 255.0f;
d[1] = rgb2hue(r, g, b);
descriptors.push_back(std::vector(d, d + sizeof(d) / sizeof(float)));
}
if(imageConverted)
{
cvReleaseImage(&imageConverted);
}
return descriptors;
}
// assuming that rgb values are normalized [0,1]
float HueDescriptor::rgb2hue(float r, float g, float b) const
{
double pi = 3.14159265359;
if(b<=g)
{
return acos(((r-g)+(r-b))/(2*sqrt((r-g)*(r-g)+(r-b)*(g-b))))/pi;
}
else
{
return (pi-acos(((r-g)+(r-b))/(2*sqrt((r-g)*(r-g)+(r-b)*(g-b)))))/pi;
}
}
}