i/rtabmap_ros
1
0
Fork 0
mirror of https://github.com/introlab/rtabmap_ros.git synced 2026-10-04 00:37:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

DatabaseViewer: using DBDriver directly instead of Memory to save RAM on large databases

Browse Source
This commit is contained in:
matlabbe
2015-10-16 14:54:47 -04:00
parent 835155e6ae
commit 5934ec90db
9 changed files with 544 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files
corelib/src/DBDriver.cpp
+175
View File
@@ -34,9 +34,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "rtabmap/utilite/ULogger.h"
#include "rtabmap/utilite/UTimer.h"
#include "rtabmap/utilite/UStl.h"
#include "DBDriverSqlite3.h"
namespace rtabmap {
DBDriver * DBDriver::create(const ParametersMap & parameters)
{
// well, we only have Sqlite3 database type for now :P
return new DBDriverSqlite3(parameters);
}
DBDriver::DBDriver(const ParametersMap & parameters) :
_emptyTrashesTime(0),
_timestampUpdate(true)
@@ -291,6 +298,23 @@ void DBDriver::saveOrUpdate(const std::vector<VisualWord *> & words) const
}
}
void DBDriver::addLink(const Link & link)
{
_dbSafeAccessMutex.lock();
this->addLinkQuery(link);
_dbSafeAccessMutex.unlock();
}
void DBDriver::removeLink(int from, int to)
{
this->executeNoResult(uFormat("DELETE FROM Link WHERE from_id=%d and to_id=%d", from, to).c_str());
}
void DBDriver::updateLink(const Link & link)
{
_dbSafeAccessMutex.lock();
this->updateLinkQuery(link);
_dbSafeAccessMutex.unlock();
}
void DBDriver::load(VWDictionary * dictionary) const
{
_dbSafeAccessMutex.lock();
@@ -715,4 +739,155 @@ void DBDriver::addStatisticsAfterRun(int stMemSize, int lastSignAdded, int proce
}
}
void DBDriver::generateGraph(
const std::string & fileName,
const std::set<int> & idsInput,
const std::map<int, Signature *> & otherSignatures)
{
if(this->isConnected())
{
if(!fileName.empty())
{
FILE* fout = 0;
#ifdef _MSC_VER
fopen_s(&fout, fileName.c_str(), "w");
#else
fout = fopen(fileName.c_str(), "w");
#endif
if (!fout)
{
UERROR("Cannot open file %s!", fileName.c_str());
return;
}
std::set<int> ids;
if(idsInput.size() == 0)
{
this->getAllNodeIds(ids);
UDEBUG("ids.size()=%d", ids.size());
for(std::map<int, Signature*>::const_iterator iter=otherSignatures.begin(); iter!=otherSignatures.end(); ++iter)
{
ids.insert(iter->first);
}
}
else
{
ids = idsInput;
}
const char * colorG = "green";
const char * colorP = "pink";
; UINFO("Generating map with %d locations", ids.size());
fprintf(fout, "digraph G {\n");
for(std::set<int>::iterator i=ids.begin(); i!=ids.end(); ++i)
{
if(otherSignatures.find(*i) == otherSignatures.end())
{
int id = *i;
std::map<int, Link> links;
this->loadLinks(id, links);
int weight = 0;
this->getWeight(id, weight);
for(std::map<int, Link>::iterator iter = links.begin(); iter!=links.end(); ++iter)
{
int weightNeighbor = 0;
if(otherSignatures.find(iter->first) == otherSignatures.end())
{
this->getWeight(iter->first, weightNeighbor);
}
else
{
weightNeighbor = otherSignatures.find(iter->first)->second->getWeight();
}
//UDEBUG("Add neighbor link from %d to %d", id, iter->first);
if(iter->second.type() == Link::kNeighbor)
{
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\"\n",
id,
weight,
iter->first,
weightNeighbor);
}
else if(iter->first > id)
{
//loop
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"L\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorG);
}
else
{
//child
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"C\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorP);
}
}
}
}
for(std::map<int, Signature*>::const_iterator i=otherSignatures.begin(); i!=otherSignatures.end(); ++i)
{
if(ids.find(i->first) != ids.end())
{
int id = i->second->id();
const std::map<int, Link> & links = i->second->getLinks();
int weight = i->second->getWeight();
for(std::map<int, Link>::const_iterator iter = links.begin(); iter!=links.end(); ++iter)
{
int weightNeighbor = 0;
const Signature * s = uValue(otherSignatures, iter->first, (Signature*)0);
if(s)
{
weightNeighbor = s->getWeight();
}
else
{
this->getWeight(iter->first, weightNeighbor);
}
//UDEBUG("Add neighbor link from %d to %d", id, iter->first);
if(iter->second.type() == Link::kNeighbor)
{
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\"\n",
id,
weight,
iter->first,
weightNeighbor);
}
else if(iter->first > id)
{
//loop
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"L\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorG);
}
else
{
//child
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"C\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorP);
}
}
}
}
fprintf(fout, "}\n");
fclose(fout);
UINFO("Graph saved to \"%s\" (Tip: $ neato -Tpdf \"%s\" -o out.pdf)", fileName.c_str(), fileName.c_str());
}
}
}
} // namespace rtabmap
corelib/src/DBDriverSqlite3.cpp
+85 -9
View File
@@ -2179,6 +2179,61 @@ void DBDriverSqlite3::saveQuery(const std::list<VisualWord *> & words) const
}
}
void DBDriverSqlite3::addLinkQuery(const Link & link) const
{
UDEBUG("");
if(_ppDb)
{
std::string type;
UTimer timer;
timer.start();
int rc = SQLITE_OK;
sqlite3_stmt * ppStmt = 0;
// Create new entries in table Link
std::string query = queryStepLink();
rc = sqlite3_prepare_v2(_ppDb, query.c_str(), -1, &ppStmt, 0);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
// Save link
stepLink(ppStmt, link);
// Finalize (delete) the statement
rc = sqlite3_finalize(ppStmt);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
UDEBUG("Time=%fs", timer.ticks());
}
}
void DBDriverSqlite3::updateLinkQuery(const Link & link) const
{
UDEBUG("");
if(_ppDb)
{
std::string type;
UTimer timer;
timer.start();
int rc = SQLITE_OK;
sqlite3_stmt * ppStmt = 0;
// Create new entries in table Link
std::string query = queryStepLinkUpdate();
rc = sqlite3_prepare_v2(_ppDb, query.c_str(), -1, &ppStmt, 0);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
// Save link
stepLink(ppStmt, link);
// Finalize (delete) the statement
rc = sqlite3_finalize(ppStmt);
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
UDEBUG("Time=%fs", timer.ticks());
}
}
std::string DBDriverSqlite3::queryStepNode() const
{
if(uStrNumCmp(_version, "0.10.1") >= 0)
@@ -2549,23 +2604,43 @@ void DBDriverSqlite3::stepSensorData(sqlite3_stmt * ppStmt,
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
}
std::string DBDriverSqlite3::queryStepLink() const
std::string DBDriverSqlite3::queryStepLinkUpdate() const
{
if(uStrNumCmp(_version, "0.10.10") >= 0)
{
return "INSERT INTO Link(from_id, to_id, type, rot_variance, trans_variance, transform, user_data) VALUES(?,?,?,?,?,?,?);";
return "UPDATE Link SET type=?, rot_variance=?, trans_variance=?, transform=?, user_data=? WHERE from_id=? AND to_id = ?;";
}
else if(uStrNumCmp(_version, "0.8.4") >= 0)
{
return "INSERT INTO Link(from_id, to_id, type, rot_variance, trans_variance, transform) VALUES(?,?,?,?,?,?);";
return "UPDATE Link SET type=?, rot_variance=?, trans_variance=?, transform=? WHERE from_id=? AND to_id = ?;";
}
else if(uStrNumCmp(_version, "0.7.4") >= 0)
{
return "INSERT INTO Link(from_id, to_id, type, variance, transform) VALUES(?,?,?,?,?);";
return "UPDATE Link SET type=?, variance=?, transform=? WHERE from_id=? AND to_id = ?;";
}
else
{
return "INSERT INTO Link(from_id, to_id, type, transform) VALUES(?,?,?,?);";
return "UPDATE Link SET type=?, transform=? WHERE from_id=? AND to_id = ?;";
}
}
std::string DBDriverSqlite3::queryStepLink() const
{
// from_id, to_id are at the end to match the update query above
if(uStrNumCmp(_version, "0.10.10") >= 0)
{
return "INSERT INTO Link(type, rot_variance, trans_variance, transform, user_data, from_id, to_id) VALUES(?,?,?,?,?,?,?);";
}
else if(uStrNumCmp(_version, "0.8.4") >= 0)
{
return "INSERT INTO Link(type, rot_variance, trans_variance, transform, from_id, to_id) VALUES(?,?,?,?,?,?);";
}
else if(uStrNumCmp(_version, "0.7.4") >= 0)
{
return "INSERT INTO Link(type, variance, transform, from_id, to_id) VALUES(?,?,?,?,?);";
}
else
{
return "INSERT INTO Link(type, transform, from_id, to_id) VALUES(?,?,?,?);";
}
}
void DBDriverSqlite3::stepLink(
@@ -2587,10 +2662,6 @@ void DBDriverSqlite3::stepLink(
int rc = SQLITE_OK;
int index = 1;
rc = sqlite3_bind_int(ppStmt, index++, link.from());
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
rc = sqlite3_bind_int(ppStmt, index++, link.to());
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
rc = sqlite3_bind_int(ppStmt, index++, link.type());
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
@@ -2624,6 +2695,11 @@ void DBDriverSqlite3::stepLink(
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
}
rc = sqlite3_bind_int(ppStmt, index++, link.from());
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
rc = sqlite3_bind_int(ppStmt, index++, link.to());
UASSERT_MSG(rc == SQLITE_OK, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
rc=sqlite3_step(ppStmt);
UASSERT_MSG(rc == SQLITE_DONE, uFormat("DB error: %s", sqlite3_errmsg(_ppDb)).c_str());
corelib/src/DBDriverSqlite3.h
+4
View File
@@ -64,6 +64,9 @@ private:
virtual void updateQuery(const std::list<Signature *> & signatures, bool updateTimestamp) const;
virtual void updateQuery(const std::list<VisualWord *> & words, bool updateTimestamp) const;
virtual void addLinkQuery(const Link & link) const;
virtual void updateLinkQuery(const Link & link) const;
// Load objects
virtual void loadQuery(VWDictionary * dictionary) const;
virtual void loadLastNodesQuery(std::list<Signature *> & signatures) const;
@@ -85,6 +88,7 @@ private:
std::string queryStepImage() const;
std::string queryStepDepth() const;
std::string queryStepSensorData() const;
std::string queryStepLinkUpdate() const;
std::string queryStepLink() const;
std::string queryStepWordsChanged() const;
std::string queryStepKeypoint() const;
corelib/src/Memory.cpp
+4 -232
View File
@@ -40,7 +40,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <rtabmap/core/EpipolarGeometry.h>
#include "rtabmap/core/VisualWord.h"
#include "rtabmap/core/Features2d.h"
#include "DBDriverSqlite3.h"
#include "rtabmap/core/DBDriver.h"
#include "rtabmap/core/util3d_features.h"
#include "rtabmap/core/util3d_filtering.h"
#include "rtabmap/core/util3d_correspondences.h"
@@ -184,7 +184,7 @@ bool Memory::init(const std::string & dbUrl, bool dbOverwritten, const Parameter
if(_dbDriver == 0 && !dbUrl.empty())
{
_dbDriver = new DBDriverSqlite3(parameters);
_dbDriver = DBDriver::create(parameters);
}
bool success = true;
@@ -3523,7 +3523,7 @@ SensorData Memory::getSignatureDataConst(int locationId) const
return r;
}
void Memory::generateGraph(const std::string & fileName, std::set<int> ids)
void Memory::generateGraph(const std::string & fileName, const std::set<int> & ids)
{
if(!_dbDriver)
{
@@ -3531,235 +3531,7 @@ void Memory::generateGraph(const std::string & fileName, std::set<int> ids)
return;
}
if(!fileName.empty())
{
FILE* fout = 0;
#ifdef _MSC_VER
fopen_s(&fout, fileName.c_str(), "w");
#else
fout = fopen(fileName.c_str(), "w");
#endif
if (!fout)
{
UERROR("Cannot open file %s!", fileName.c_str());
return;
}
if(ids.size() == 0)
{
_dbDriver->getAllNodeIds(ids);
UDEBUG("ids.size()=%d", ids.size());
for(std::map<int, Signature*>::iterator iter=_signatures.begin(); iter!=_signatures.end(); ++iter)
{
ids.insert(iter->first);
}
}
const char * colorG = "green";
const char * colorP = "pink";
; UINFO("Generating map with %d locations", ids.size());
fprintf(fout, "digraph G {\n");
for(std::set<int>::iterator i=ids.begin(); i!=ids.end(); ++i)
{
if(_signatures.find(*i) == _signatures.end())
{
int id = *i;
std::map<int, Link> links;
_dbDriver->loadLinks(id, links);
int weight = 0;
_dbDriver->getWeight(id, weight);
for(std::map<int, Link>::iterator iter = links.begin(); iter!=links.end(); ++iter)
{
int weightNeighbor = 0;
if(_signatures.find(iter->first) == _signatures.end())
{
_dbDriver->getWeight(iter->first, weightNeighbor);
}
else
{
weightNeighbor = _signatures.find(iter->first)->second->getWeight();
}
//UDEBUG("Add neighbor link from %d to %d", id, iter->first);
if(iter->second.type() == Link::kNeighbor)
{
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\"\n",
id,
weight,
iter->first,
weightNeighbor);
}
else if(iter->first > id)
{
//loop
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"L\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorG);
}
else
{
//child
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"C\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorP);
}
}
}
}
for(std::map<int, Signature*>::iterator i=_signatures.begin(); i!=_signatures.end(); ++i)
{
if(ids.find(i->first) != ids.end())
{
int id = i->second->id();
const std::map<int, Link> & links = i->second->getLinks();
int weight = i->second->getWeight();
for(std::map<int, Link>::const_iterator iter = links.begin(); iter!=links.end(); ++iter)
{
int weightNeighbor = 0;
const Signature * s = this->getSignature(iter->first);
if(s)
{
weightNeighbor = s->getWeight();
}
else
{
_dbDriver->getWeight(iter->first, weightNeighbor);
}
//UDEBUG("Add neighbor link from %d to %d", id, iter->first);
if(iter->second.type() == Link::kNeighbor)
{
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\"\n",
id,
weight,
iter->first,
weightNeighbor);
}
else if(iter->first > id)
{
//loop
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"L\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorG);
}
else
{
//child
fprintf(fout, " \"%d\\n%d\" -> \"%d\\n%d\" [label=\"C\", fontcolor=%s, fontsize=8];\n",
id,
weight,
iter->first,
weightNeighbor,
colorP);
}
}
}
}
fprintf(fout, "}\n");
fclose(fout);
UINFO("Graph saved to \"%s\"", fileName.c_str());
}
}
// Only used to generate a .dot file
class GraphNode
{
public:
GraphNode(int id, GraphNode * parent = 0) :
_parent(parent),
_id(id)
{
if(_parent)
{
_parent->addChild(this);
}
}
virtual ~GraphNode()
{
//We copy the set because when a child is destroyed, it is removed from its parent.
std::set<GraphNode*> children = _children;
_children.clear();
for(std::set<GraphNode*>::iterator iter=children.begin(); iter!=children.end(); ++iter)
{
delete *iter;
}
children.clear();
if(_parent)
{
_parent->removeChild(this);
}
}
int id() const {return _id;}
bool isAncestor(int id) const
{
if(_parent)
{
if(_parent->id() == id)
{
return true;
}
return _parent->isAncestor(id);
}
return false;
}
void expand(std::list<std::list<int> > & paths, std::list<int> currentPath = std::list<int>()) const
{
currentPath.push_back(_id);
if(_children.size() == 0)
{
paths.push_back(currentPath);
return;
}
for(std::set<GraphNode*>::const_iterator iter=_children.begin(); iter!=_children.end(); ++iter)
{
(*iter)->expand(paths, currentPath);
}
}
private:
void addChild(GraphNode * child)
{
_children.insert(child);
}
void removeChild(GraphNode * child)
{
_children.erase(child);
}
private:
std::set<GraphNode*> _children;
GraphNode * _parent;
int _id;
};
//recursive
void Memory::createGraph(GraphNode * parent, unsigned int maxDepth, const std::set<int> & endIds)
{
if(maxDepth == 0 || !parent)
{
return;
}
std::map<int, int> neighbors = this->getNeighborsId(parent->id(), 1, -1, false);
for(std::map<int, int>::iterator iter=neighbors.begin(); iter!=neighbors.end(); ++iter)
{
if(!parent->isAncestor(iter->first))
{
GraphNode * n = new GraphNode(iter->first, parent);
if(endIds.find(iter->first) == endIds.end())
{
this->createGraph(n, maxDepth-1, endIds);
}
}
}
_dbDriver->generateGraph(fileName, ids, _signatures);
}
int Memory::getNi(int signatureId) const