Updated detect more loop closures with optimization check to accept (like in dbviewer). DBViewer: added Depth image edition dialog. GainCompensator: now doing it on 3 channels separatly. Export: removed gain's alpha option, added max polygons option, added brightness/contrast auto balance option

This commit is contained in:
matlabbe
2017-03-30 15:33:48 -04:00
parent a928a0b404
commit 767b29d5a3
20 changed files with 1695 additions and 551 deletions

View File

@@ -434,6 +434,15 @@ void DBDriver::updateOccupancyGrid(
_dbSafeAccessMutex.unlock();
}
void DBDriver::updateDepthImage(int nodeId, const cv::Mat & image)
{
_dbSafeAccessMutex.lock();
this->updateDepthImageQuery(
nodeId,
image);
_dbSafeAccessMutex.unlock();
}
void DBDriver::load(VWDictionary * dictionary) const
{
_dbSafeAccessMutex.lock();
@@ -455,7 +464,6 @@ void DBDriver::loadSignatures(const std::list<int> & signIds,
UDEBUG("");
// look up in the trash before the database
std::list<int> ids = signIds;
std::list<Signature*>::iterator sIter;
bool valueFound = false;
_trashesMutex.lock();
{

View File

@@ -3202,7 +3202,7 @@ void DBDriverSqlite3::updateOccupancyGridQuery(
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
// Save occupancy grid
stepOccupancyGrid(ppStmt,
stepOccupancyGridUpdate(ppStmt,
nodeId,
ground,
obstacles,
@@ -3217,6 +3217,37 @@ void DBDriverSqlite3::updateOccupancyGridQuery(
}
}
void DBDriverSqlite3::updateDepthImageQuery(
int nodeId,
const cv::Mat & image) const
{
UDEBUG("");
if(_ppDb)
{
std::string type;
UTimer timer;
timer.start();
int rc = SQLITE_OK;
sqlite3_stmt * ppStmt = 0;
// Create query
std::string query = queryStepDepthUpdate();
rc = sqlite3_prepare_v2(_ppDb, query.c_str(), -1, &ppStmt, 0);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
// Save depth
stepDepthUpdate(ppStmt,
nodeId,
image);
// Finalize (delete) the statement
rc = sqlite3_finalize(ppStmt);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
UDEBUG("Time=%fs", timer.ticks());
}
}
void DBDriverSqlite3::addStatisticsQuery(const Statistics & statistics) const
{
UDEBUG("Ref ID = %d", statistics.refImageId());
@@ -3502,6 +3533,59 @@ void DBDriverSqlite3::stepDepth(sqlite3_stmt * ppStmt, const SensorData & sensor
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
}
std::string DBDriverSqlite3::queryStepDepthUpdate() const
{
if(uStrNumCmp(_version, "0.10.0") < 0)
{
return "UPDATE Depth SET data=? WHERE id=?;";
}
else
{
return "UPDATE Data SET depth=? WHERE id=?;";
}
}
void DBDriverSqlite3::stepDepthUpdate(sqlite3_stmt * ppStmt, int nodeId, const cv::Mat & image) const
{
if(!ppStmt)
{
UFATAL("");
}
int rc = SQLITE_OK;
int index = 1;
cv::Mat imageCompressed;
if(!image.empty() && (image.type()!=CV_8UC1 || image.rows > 1))
{
// compress
imageCompressed = compressImage2(image, ".png");
}
else
{
imageCompressed = image;
}
if(!imageCompressed.empty())
{
rc = sqlite3_bind_blob(ppStmt, index++, imageCompressed.data, (int)imageCompressed.cols, SQLITE_STATIC);
}
else
{
rc = sqlite3_bind_zeroblob(ppStmt, index++, 4);
}
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
//id
rc = sqlite3_bind_int(ppStmt, index++, nodeId);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
//step
rc=sqlite3_step(ppStmt);
UASSERT_MSG(rc == SQLITE_DONE, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
rc = sqlite3_reset(ppStmt);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error (%s): %s", _version.c_str(), sqlite3_errmsg(_ppDb)).c_str());
}
std::string DBDriverSqlite3::queryStepSensorData() const
{
UASSERT(uStrNumCmp(_version, "0.10.0") >= 0);
@@ -3951,7 +4035,7 @@ std::string DBDriverSqlite3::queryStepOccupancyGridUpdate() const
UASSERT(uStrNumCmp(_version, "0.11.10") >= 0);
return "UPDATE Data SET ground_cells=?, obstacle_cells=?, cell_size=?, view_point_x=?, view_point_y=?, view_point_z=? WHERE id=?;";
}
void DBDriverSqlite3::stepOccupancyGrid(sqlite3_stmt * ppStmt,
void DBDriverSqlite3::stepOccupancyGridUpdate(sqlite3_stmt * ppStmt,
int nodeId,
const cv::Mat & ground,
const cv::Mat & obstacles,

View File

@@ -84,6 +84,10 @@ private:
float cellSize,
const cv::Point3f & viewpoint) const;
virtual void updateDepthImageQuery(
int nodeId,
const cv::Mat & image) const;
virtual void addStatisticsQuery(const Statistics & statistics) const;
// Load objects
@@ -107,6 +111,7 @@ private:
std::string queryStepNode() const;
std::string queryStepImage() const;
std::string queryStepDepth() const;
std::string queryStepDepthUpdate() const;
std::string queryStepSensorData() const;
std::string queryStepLinkUpdate() const;
std::string queryStepLink() const;
@@ -119,11 +124,12 @@ private:
int id,
const cv::Mat & imageBytes) const;
void stepDepth(sqlite3_stmt * ppStmt, const SensorData & sensorData) const;
void stepDepthUpdate(sqlite3_stmt * ppStmt, int nodeId, const cv::Mat & imageCompressed) const;
void stepSensorData(sqlite3_stmt * ppStmt, const SensorData & sensorData) const;
void stepLink(sqlite3_stmt * ppStmt, const Link & link) const;
void stepWordsChanged(sqlite3_stmt * ppStmt, int signatureId, int oldWordId, int newWordId) const;
void stepKeypoint(sqlite3_stmt * ppStmt, int signatureId, int wordId, const cv::KeyPoint & kp, const cv::Point3f & pt, const cv::Mat & descriptor) const;
void stepOccupancyGrid(sqlite3_stmt * ppStmt,
void stepOccupancyGridUpdate(sqlite3_stmt * ppStmt,
int nodeId,
const cv::Mat & ground,
const cv::Mat & obstacles,

View File

@@ -141,6 +141,10 @@ void feedImpl(
cv::Mat_<int> N(num_images, num_images); N.setTo(0);
cv::Mat_<double> I(num_images, num_images); I.setTo(0);
cv::Mat_<double> IR(num_images, num_images); IR.setTo(0);
cv::Mat_<double> IG(num_images, num_images); IG.setTo(0);
cv::Mat_<double> IB(num_images, num_images); IB.setTo(0);
// make id to index map
idToIndex.clear();
std::vector<int> indexToId(clouds.size());
@@ -264,14 +268,17 @@ void feedImpl(
}
UDEBUG("%d->%d: correspondences = %d", iter->second.from(), iter->second.to(), (int)correspondences.size());
if((minOverlap <= 0.0 && correspondences.size()) ||
if(correspondences.size() && (minOverlap <= 0.0 ||
(double(correspondences.size()) / double(clouds.at(iter->second.from())->size()) >= minOverlap &&
double(correspondences.size()) / double(clouds.at(iter->second.to())->size()) >= minOverlap))
double(correspondences.size()) / double(clouds.at(iter->second.to())->size()) >= minOverlap)))
{
int i = idToIndex.at(iter->second.from());
int j = idToIndex.at(iter->second.to());
double Isum1 = 0, Isum2 = 0;
double IRsum1 = 0, IRsum2 = 0;
double IGsum1 = 0, IGsum2 = 0;
double IBsum1 = 0, IBsum2 = 0;
for (unsigned int c = 0; c < correspondences.size(); ++c)
{
const PointT & pt1 = cloudFrom->at(correspondences.at(c).index_match);
@@ -279,10 +286,24 @@ void feedImpl(
Isum1 += std::sqrt(static_cast<double>(sqr(pt1.r) + sqr(pt1.g) + sqr(pt1.b)));
Isum2 += std::sqrt(static_cast<double>(sqr(pt2.r) + sqr(pt2.g) + sqr(pt2.b)));
IRsum1 += static_cast<double>(pt1.r);
IRsum2 += static_cast<double>(pt2.r);
IGsum1 += static_cast<double>(pt1.g);
IGsum2 += static_cast<double>(pt2.g);
IBsum1 += static_cast<double>(pt1.b);
IBsum2 += static_cast<double>(pt2.b);
}
N(i, j) = N(j, i) = correspondences.size();
I(i, j) = Isum1 / N(i, j);
I(j, i) = Isum2 / N(i, j);
IR(i, j) = IRsum1 / N(i, j);
IR(j, i) = IRsum2 / N(i, j);
IG(i, j) = IGsum1 / N(i, j);
IG(j, i) = IGsum2 / N(i, j);
IB(i, j) = IBsum1 / N(i, j);
IB(j, i) = IBsum2 / N(i, j);
}
}
}
@@ -291,25 +312,52 @@ void feedImpl(
cv::Mat_<double> A(num_images, num_images); A.setTo(0);
cv::Mat_<double> b(num_images, 1); b.setTo(0);
cv::Mat_<double> AR(num_images, num_images); AR.setTo(0);
cv::Mat_<double> AG(num_images, num_images); AG.setTo(0);
cv::Mat_<double> AB(num_images, num_images); AB.setTo(0);
for (int i = 0; i < num_images; ++i)
{
for (int j = 0; j < num_images; ++j)
{
b(i, 0) += beta * N(i, j);
A(i, i) += beta * N(i, j);
AR(i, i) += beta * N(i, j);
AG(i, i) += beta * N(i, j);
AB(i, i) += beta * N(i, j);
if (j == i) continue;
A(i, i) += 2 * alpha * I(i, j) * I(i, j) * N(i, j);
A(i, j) -= 2 * alpha * I(i, j) * I(j, i) * N(i, j);
AR(i, i) += 2 * alpha * IR(i, j) * IR(i, j) * N(i, j);
AR(i, j) -= 2 * alpha * IR(i, j) * IR(j, i) * N(i, j);
AG(i, i) += 2 * alpha * IG(i, j) * IG(i, j) * N(i, j);
AG(i, j) -= 2 * alpha * IG(i, j) * IG(j, i) * N(i, j);
AB(i, i) += 2 * alpha * IB(i, j) * IB(i, j) * N(i, j);
AB(i, j) -= 2 * alpha * IB(i, j) * IB(j, i) * N(i, j);
}
}
gains = cv::Mat_<double>();
cv::solve(A, b, gains);
cv::Mat_<double> gainsGray, gainsR, gainsG, gainsB;
cv::solve(A, b, gainsGray);
cv::solve(AR, b, gainsR);
cv::solve(AG, b, gainsG);
cv::solve(AB, b, gainsB);
gains = cv::Mat_<double>(gainsGray.rows, 4);
gainsGray.copyTo(gains.col(0));
gainsR.copyTo(gains.col(1));
gainsG.copyTo(gains.col(2));
gainsB.copyTo(gains.col(3));
if(ULogger::kInfo)
{
for(int i=0; i<gains.rows; ++i)
{
UINFO("Gain index=%d (id=%d) = %f", i, indexToId[i], gains(i, 0));
UINFO("Gain index=%d (id=%d) = %f (%f,%f,%f)", i, indexToId[i], gains(i, 0), gains(i, 1), gains(i, 2), gains(i, 3));
}
}
}
@@ -349,16 +397,18 @@ void applyImpl(
const pcl::IndicesPtr & indices,
const cv::Mat_<double> & gains)
{
double gain = gains(index, 0);
UDEBUG("index=%d gain=%f", index, gain);
double gainR = gains(index, 1);
double gainG = gains(index, 2);
double gainB = gains(index, 3);
UDEBUG("index=%d gain=%f (%f,%f,%f)", index, gains(index, 0), gainR, gainG, gainB);
if(indices->size())
{
for(unsigned int i=0; i<indices->size(); ++i)
{
PointT & pt = cloud->at(indices->at(i));
pt.r = uchar(std::max(0.0, std::min(255.0, double(pt.r) * gain)));
pt.g = uchar(std::max(0.0, std::min(255.0, double(pt.g) * gain)));
pt.b = uchar(std::max(0.0, std::min(255.0, double(pt.b) * gain)));
pt.r = uchar(std::max(0.0, std::min(255.0, double(pt.r) * gainR)));
pt.g = uchar(std::max(0.0, std::min(255.0, double(pt.g) * gainG)));
pt.b = uchar(std::max(0.0, std::min(255.0, double(pt.b) * gainB)));
}
}
else
@@ -366,9 +416,9 @@ void applyImpl(
for(unsigned int i=0; i<cloud->size(); ++i)
{
PointT & pt = cloud->at(i);
pt.r = uchar(std::max(0.0, std::min(255.0, double(pt.r) * gain)));
pt.g = uchar(std::max(0.0, std::min(255.0, double(pt.g) * gain)));
pt.b = uchar(std::max(0.0, std::min(255.0, double(pt.b) * gain)));
pt.r = uchar(std::max(0.0, std::min(255.0, double(pt.r) * gainR)));
pt.g = uchar(std::max(0.0, std::min(255.0, double(pt.g) * gainG)));
pt.b = uchar(std::max(0.0, std::min(255.0, double(pt.b) * gainB)));
}
}
}
@@ -402,7 +452,20 @@ void GainCompensator::apply(
cv::Mat & image) const
{
UASSERT_MSG(uContains(idToIndex_, id), uFormat("id=%d idToIndex_.size()=%d", id, (int)idToIndex_.size()).c_str());
cv::multiply(image, gains_(idToIndex_.at(id), 0), image);
if(image.channels() == 1)
{
cv::multiply(image, gains_(idToIndex_.at(id), 0), image);
}
else if(image.channels()>=3)
{
std::vector<cv::Mat> channels;
cv::split(image, channels);
// assuming BGR
cv::multiply(channels[0], gains_(idToIndex_.at(id), 3), channels[0]);
cv::multiply(channels[1], gains_(idToIndex_.at(id), 2), channels[1]);
cv::multiply(channels[2], gains_(idToIndex_.at(id), 1), channels[2]);
cv::merge(channels, image);
}
}
double GainCompensator::getGain(int id) const

View File

@@ -49,6 +49,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <pcl/search/kdtree.h>
#include <pcl/filters/crop_box.h>
#include <pcl/io/pcd_io.h>
#include <pcl/common/common.h>
#include <stdlib.h>
#include <set>
@@ -3488,12 +3489,120 @@ int Rtabmap::detectMoreLoopClosures(float clusterRadius, float clusterAngle, int
if(!t.isNull())
{
UINFO("Added new loop closure between %d and %d.", from, to);
addedLinks.insert(from);
addedLinks.insert(to);
links.insert(std::make_pair(from, Link(from, to, Link::kUserClosure, t, info.varianceAng, info.varianceLin)));
loopClosuresAdded.push_back(Link(from, to, Link::kUserClosure, t, info.varianceAng, info.varianceLin));
UINFO("Detected loop closure %d->%d! (%d/%d)", from, to, i+1, (int)clusters.size());
bool updateConstraints = true;
if(_optimizationMaxLinearError > 0.0f)
{
//optimize the graph to see if the new constraint is globally valid
int fromId = from;
int mapId = signatures.at(from).mapId();
// use first node of the map containing from
for(std::map<int, Signature>::iterator iter=signatures.begin(); iter!=signatures.end(); ++iter)
{
if(iter->second.mapId() == mapId)
{
fromId = iter->first;
break;
}
}
std::multimap<int, Link> linksIn = links;
linksIn.insert(std::make_pair(from, Link(from, to, Link::kUserClosure, t, info.varianceAng, info.varianceLin)));
const Link * maxLinearLink = 0;
const Link * maxAngularLink = 0;
float maxLinearError = 0.0f;
float maxAngularError = 0.0f;
std::map<int, Transform> optimizedPoses;
std::multimap<int, Link> links;
UASSERT(poses.find(fromId) != poses.end());
UASSERT_MSG(poses.find(from) != poses.end(), uFormat("id=%d poses=%d links=%d", from, (int)poses.size(), (int)links.size()).c_str());
UASSERT_MSG(poses.find(to) != poses.end(), uFormat("id=%d poses=%d links=%d", to, (int)poses.size(), (int)links.size()).c_str());
_graphOptimizer->getConnectedGraph(fromId, poses, linksIn, optimizedPoses, links);
UASSERT(optimizedPoses.find(fromId) != optimizedPoses.end());
UASSERT_MSG(optimizedPoses.find(from) != optimizedPoses.end(), uFormat("id=%d poses=%d links=%d", from, (int)optimizedPoses.size(), (int)links.size()).c_str());
UASSERT_MSG(optimizedPoses.find(to) != optimizedPoses.end(), uFormat("id=%d poses=%d links=%d", to, (int)optimizedPoses.size(), (int)links.size()).c_str());
UASSERT(graph::findLink(links, from, to) != links.end());
optimizedPoses = _graphOptimizer->optimize(fromId, optimizedPoses, links);
std::string msg;
if(optimizedPoses.size())
{
for(std::multimap<int, Link>::iterator iter=links.begin(); iter!=links.end(); ++iter)
{
// ignore links with high variance
if(iter->second.transVariance() <= 1.0)
{
UASSERT(optimizedPoses.find(iter->second.from())!=optimizedPoses.end());
UASSERT(optimizedPoses.find(iter->second.to())!=optimizedPoses.end());
Transform t1 = optimizedPoses.at(iter->second.from());
Transform t2 = optimizedPoses.at(iter->second.to());
UASSERT(!t1.isNull() && !t2.isNull());
Transform t = t1.inverse()*t2;
float linearError = uMax3(
fabs(iter->second.transform().x() - t.x()),
fabs(iter->second.transform().y() - t.y()),
fabs(iter->second.transform().z() - t.z()));
Eigen::Vector3f vA = t1.toEigen3f().rotation()*Eigen::Vector3f(1,0,0);
Eigen::Vector3f vB = t2.toEigen3f().rotation()*Eigen::Vector3f(1,0,0);
float angularError = pcl::getAngle3D(Eigen::Vector4f(vA[0], vA[1], vA[2], 0), Eigen::Vector4f(vB[0], vB[1], vB[2], 0));
if(linearError > maxLinearError)
{
maxLinearError = linearError;
maxLinearLink = &iter->second;
}
if(angularError > maxAngularError)
{
maxAngularError = angularError;
maxAngularLink = &iter->second;
}
}
}
if(maxLinearLink)
{
UINFO("Max optimization linear error = %f m (link %d->%d)", maxLinearError, maxLinearLink->from(), maxLinearLink->to());
}
if(maxAngularLink)
{
UINFO("Max optimization angular error = %f deg (link %d->%d)", maxAngularError*180.0f/M_PI, maxAngularLink->from(), maxAngularLink->to());
}
if(maxLinearError > _optimizationMaxLinearError)
{
msg = uFormat("Rejecting edge %d->%d because "
"graph error is too large after optimization (%f m for edge %d->%d, %f deg for edge %d->%d). "
"\"%s\" is %f m.",
from,
to,
maxLinearError,
maxLinearLink->from(),
maxLinearLink->to(),
maxAngularError*180.0f/M_PI,
maxAngularLink?maxAngularLink->from():0,
maxAngularLink?maxAngularLink->to():0,
Parameters::kRGBDOptimizeMaxError().c_str(),
_optimizationMaxLinearError);
}
}
else
{
msg = uFormat("Rejecting edge %d->%d because graph optimization has failed!",
from,
to);
}
if(!msg.empty())
{
UWARN("%s", msg.c_str());
updateConstraints = false;
}
}
if(updateConstraints)
{
UINFO("Added new loop closure between %d and %d.", from, to);
addedLinks.insert(from);
addedLinks.insert(to);
links.insert(std::make_pair(from, Link(from, to, Link::kUserClosure, t, info.varianceAng, info.varianceLin)));
loopClosuresAdded.push_back(Link(from, to, Link::kUserClosure, t, info.varianceAng, info.varianceLin));
UINFO("Detected loop closure %d->%d! (%d/%d)", from, to, i+1, (int)clusters.size());
}
}
}
}

View File

@@ -1909,6 +1909,89 @@ cv::Mat fastBilateralFiltering(const cv::Mat & depth, float sigmaS, float sigmaR
return output;
}
/**
* \brief Automatic brightness and contrast optimization with optional histogram clipping
* \param [in]src Input image GRAY or BGR or BGRA
* \param [out]dst Destination image
* \param clipHistPercent cut wings of histogram at given percent typical=>1, 0=>Disabled
* \note In case of BGRA image, we won't touch the transparency
* See http://answers.opencv.org/question/75510/how-to-make-auto-adjustmentsbrightness-and-contrast-for-image-android-opencv-image-correction/
*/
cv::Mat brightnessAndContrastAuto(const cv::Mat &src, const cv::Mat & mask, float clipLowHistPercent, float clipHighHistPercent)
{
CV_Assert(clipLowHistPercent >= 0 && clipHighHistPercent>=0);
CV_Assert((src.type() == CV_8UC1) || (src.type() == CV_8UC3) || (src.type() == CV_8UC4));
int histSize = 256;
float alpha, beta;
double minGray = 0, maxGray = 0;
//to calculate grayscale histogram
cv::Mat gray;
if (src.type() == CV_8UC1) gray = src;
else if (src.type() == CV_8UC3) cvtColor(src, gray, CV_BGR2GRAY);
else if (src.type() == CV_8UC4) cvtColor(src, gray, CV_BGRA2GRAY);
if (clipLowHistPercent == 0 && clipHighHistPercent == 0)
{
// keep full available range
cv::minMaxLoc(gray, &minGray, &maxGray, 0, 0, mask);
}
else
{
cv::Mat hist; //the grayscale histogram
float range[] = { 0, 256 };
const float* histRange = { range };
bool uniform = true;
bool accumulate = false;
calcHist(&gray, 1, 0, mask, hist, 1, &histSize, &histRange, uniform, accumulate);
// calculate cumulative distribution from the histogram
std::vector<float> accumulator(histSize);
accumulator[0] = hist.at<float>(0);
for (int i = 1; i < histSize; i++)
{
accumulator[i] = accumulator[i - 1] + hist.at<float>(i);
}
// locate points that cuts at required value
float max = accumulator.back();
clipLowHistPercent *= (max / 100.0); //make percent as absolute
clipHighHistPercent *= (max / 100.0); //make percent as absolute
// locate left cut
minGray = 0;
while (accumulator[minGray] < clipLowHistPercent)
minGray++;
// locate right cut
maxGray = histSize - 1;
while (accumulator[maxGray] >= (max - clipHighHistPercent))
maxGray--;
}
// current range
float inputRange = maxGray - minGray;
alpha = (histSize - 1) / inputRange; // alpha expands current range to histsize range
beta = -minGray * alpha; // beta shifts current range so that minGray will go to 0
UINFO("minGray=%f maxGray=%f alpha=%f beta=%f", minGray, maxGray, alpha, beta);
cv::Mat dst;
// Apply brightness and contrast normalization
// convertTo operates with saurate_cast
src.convertTo(dst, -1, alpha, beta);
// restore alpha channel from source
if (dst.type() == CV_8UC4)
{
int from_to[] = { 3, 3};
cv::mixChannels(&src, 4, &dst,1, from_to, 1);
}
return dst;
}
}
}