i/rtabmap
1
0
Fork 0
mirror of https://github.com/introlab/rtabmap.git synced 2026-09-03 01:50:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

added --rewind, use sqlite3 changeset instead of patchset

Browse Source
This commit is contained in:
matlabbe
2026-07-10 20:28:55 -07:00
parent 9cd2318e37
commit 4056c152f0
3 changed files with 90 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
corelib/include/rtabmap/core/DBDriverSqlite3.h
View File

@@ -60,15 +60,17 @@ public:
// set of changes is small (e.g. appending a few sessions), not for rewriting a whole map. // set of changes is small (e.g. appending a few sessions), not for rewriting a whole map.
virtual bool trackDatabaseChanges(const std::string & outputUrl); virtual bool trackDatabaseChanges(const std::string & outputUrl);
// Apply a change delta previously written by setTrackChangesOutput() onto the database // Apply a change delta previously written by trackDatabaseChanges() onto the database at
// at databasePath (which must be the same database, at the same state, the delta was // databasePath. The file is decompressed (same codec as the other rtabmap blobs) then
// generated from). The file is decompressed (same codec as the other rtabmap blobs) then // applied with the SQLite session extension. With rewind=false, databasePath must be at the
// applied with the SQLite session extension. Returns true on success; on failure returns // same state the delta was recorded from; with rewind=true, the delta is inverted first to
// false and, if errorMsg is not null, sets a human-readable message. Requires a SQLite // undo it, so databasePath must be at the post-update state. Returns true on success; on
// library built with the session extension (both at build and runtime). // failure returns false and, if errorMsg is not null, sets a human-readable message.
// Requires a SQLite library built with the session extension (both at build and runtime).
static bool applyChangesFromFile( static bool applyChangesFromFile(
const std::string & databasePath, const std::string & databasePath,
const std::string & patchsetPath, const std::string & changesetPath,
bool rewind = false,
std::string * errorMsg = 0); std::string * errorMsg = 0);
protected: protected:

75
corelib/src/DBDriverSqlite3.cpp
View File

@@ -245,7 +245,7 @@ void DBDriverSqlite3::startChangeTracking()
else if(sqlite3session_create(_ppDb, "main", &_session) == SQLITE_OK) else if(sqlite3session_create(_ppDb, "main", &_session) == SQLITE_OK)
{ {
sqlite3session_attach(_session, 0); // 0 = track all tables sqlite3session_attach(_session, 0); // 0 = track all tables
UINFO("Tracking database changes; a patchset will be written to \"%s\" on close.", UINFO("Tracking database changes; a changeset will be written to \"%s\" on close.",
_trackChangesOutput.c_str()); _trackChangesOutput.c_str());
} }
else else
@@ -293,7 +293,8 @@ static int rtabmapChangesetConflict(void * pCtx, int eConflict, sqlite3_changese
bool DBDriverSqlite3::applyChangesFromFile( bool DBDriverSqlite3::applyChangesFromFile(
const std::string & databasePath, const std::string & databasePath,
const std::string & patchsetPath, const std::string & changesetPath,
bool rewind,
std::string * errorMsg) std::string * errorMsg)
{ {
#ifdef RTABMAP_WITH_SQLITE3_SESSION #ifdef RTABMAP_WITH_SQLITE3_SESSION
@@ -303,11 +304,11 @@ bool DBDriverSqlite3::applyChangesFromFile(
return false; return false;
} }
// Read the compressed patchset file. // Read the compressed changeset file.
FILE * fp = fopen(patchsetPath.c_str(), "rb"); FILE * fp = fopen(changesetPath.c_str(), "rb");
if(!fp) if(!fp)
{ {
if(errorMsg) *errorMsg = uFormat("Could not open patchset file \"%s\".", patchsetPath.c_str()); if(errorMsg) *errorMsg = uFormat("Could not open changeset file \"%s\".", changesetPath.c_str());
return false; return false;
} }
fseek(fp, 0, SEEK_END); fseek(fp, 0, SEEK_END);
@@ -316,7 +317,7 @@ bool DBDriverSqlite3::applyChangesFromFile(
if(fileSize <= 0) if(fileSize <= 0)
{ {
fclose(fp); fclose(fp);
if(errorMsg) *errorMsg = uFormat("Patchset file \"%s\" is empty or unreadable.", patchsetPath.c_str()); if(errorMsg) *errorMsg = uFormat("Changeset file \"%s\" is empty or unreadable.", changesetPath.c_str());
return false; return false;
} }
std::vector<unsigned char> fileBytes((size_t)fileSize); std::vector<unsigned char> fileBytes((size_t)fileSize);
@@ -324,18 +325,39 @@ bool DBDriverSqlite3::applyChangesFromFile(
fclose(fp); fclose(fp);
if((long)bytesRead != fileSize) if((long)bytesRead != fileSize)
{ {
if(errorMsg) *errorMsg = uFormat("Could not read patchset file \"%s\".", patchsetPath.c_str()); if(errorMsg) *errorMsg = uFormat("Could not read changeset file \"%s\".", changesetPath.c_str());
return false; return false;
} }
// Decompress to the raw patchset (same codec used to write it). // Decompress to the raw changeset (same codec used to write it).
cv::Mat patch = uncompressData(fileBytes.data(), (unsigned long)fileSize); cv::Mat changes = uncompressData(fileBytes.data(), (unsigned long)fileSize);
if(patch.empty()) if(changes.empty())
{ {
if(errorMsg) *errorMsg = uFormat("Could not decompress patchset file \"%s\".", patchsetPath.c_str()); if(errorMsg) *errorMsg = uFormat("Could not decompress changeset file \"%s\".", changesetPath.c_str());
return false; return false;
} }
int nChanges = (int)(changes.total()*changes.elemSize());
void * pApply = changes.data;
int nApply = nChanges;
void * pInverted = 0;
if(rewind)
{
// Invert the changeset so applying it undoes the recorded update (INSERT<->DELETE,
// UPDATE reversed). This is why the emit records a changeset, not a patchset: only
// changesets carry the old values needed to reverse a change.
int nInverted = 0;
int rcInv = sqlite3changeset_invert(nChanges, changes.data, &nInverted, &pInverted);
if(rcInv != SQLITE_OK || pInverted == 0)
{
if(errorMsg) *errorMsg = uFormat("Could not invert changeset \"%s\" for rewind (SQLite error %d: %s).",
changesetPath.c_str(), rcInv, sqlite3_errstr(rcInv));
return false;
}
pApply = pInverted;
nApply = nInverted;
}
// Apply onto the target database. // Apply onto the target database.
sqlite3 * db = 0; sqlite3 * db = 0;
int rc = sqlite3_open_v2(databasePath.c_str(), &db, SQLITE_OPEN_READWRITE, 0); int rc = sqlite3_open_v2(databasePath.c_str(), &db, SQLITE_OPEN_READWRITE, 0);
@@ -343,20 +365,21 @@ bool DBDriverSqlite3::applyChangesFromFile(
{ {
if(errorMsg) *errorMsg = uFormat("Could not open database \"%s\": %s", databasePath.c_str(), sqlite3_errmsg(db)); if(errorMsg) *errorMsg = uFormat("Could not open database \"%s\": %s", databasePath.c_str(), sqlite3_errmsg(db));
sqlite3_close(db); sqlite3_close(db);
if(pInverted) sqlite3_free(pInverted);
return false; return false;
} }
int nPatch = (int)(patch.total()*patch.elemSize()); rc = sqlite3changeset_apply(db, nApply, pApply, 0, rtabmapChangesetConflict, 0);
rc = sqlite3changeset_apply(db, nPatch, patch.data, 0, rtabmapChangesetConflict, 0); sqlite3_close(db);
if(pInverted) sqlite3_free(pInverted);
if(rc != SQLITE_OK) if(rc != SQLITE_OK)
{ {
if(errorMsg) *errorMsg = uFormat("Failed to apply patchset to \"%s\" (SQLite error %d: %s). Make " if(errorMsg) *errorMsg = uFormat("Failed to %s \"%s\" (SQLite error %d: %s). Make sure it is applied to the "
"sure it is applied to the exact database state the delta was generated from.", "exact database state the delta %s.",
databasePath.c_str(), rc, sqlite3_errstr(rc)); rewind?"rewind":"update", databasePath.c_str(), rc, sqlite3_errstr(rc),
sqlite3_close(db); rewind?"was previously applied to":"was recorded from");
return false; return false;
} }
sqlite3_close(db);
return true; return true;
#else #else
if(errorMsg) *errorMsg = "rtabmap was built without SQLite session support."; if(errorMsg) *errorMsg = "rtabmap was built without SQLite session support.";
@@ -645,15 +668,17 @@ void DBDriverSqlite3::disconnectDatabaseQuery(bool save, const std::string & out
} }
#ifdef RTABMAP_WITH_SQLITE3_SESSION #ifdef RTABMAP_WITH_SQLITE3_SESSION
// If change tracking is active, emit the accumulated patchset before closing. // If change tracking is active, emit the accumulated changeset before closing.
// A changeset (not a patchset) is used so it carries the old values too and can be
// inverted with sqlite3changeset_invert() to undo the update (see applyChangesFromFile).
if(_session) if(_session)
{ {
int nBuf = 0; int nBuf = 0;
void * pBuf = 0; void * pBuf = 0;
int rcp = sqlite3session_patchset(_session, &nBuf, &pBuf); int rcp = sqlite3session_changeset(_session, &nBuf, &pBuf);
if(rcp == SQLITE_OK && pBuf && nBuf > 0) if(rcp == SQLITE_OK && pBuf && nBuf > 0)
{ {
// Compress the patchset with the same zlib-based codec used for every other // Compress the changeset with the same zlib-based codec used for every other
// rtabmap blob, so the delta is compact for transfer. To apply it later, // rtabmap blob, so the delta is compact for transfer. To apply it later,
// uncompressData() the file and pass the bytes to sqlite3changeset_apply(). // uncompressData() the file and pass the bytes to sqlite3changeset_apply().
cv::Mat compressed = compressData2(cv::Mat(1, nBuf, CV_8UC1, pBuf)); cv::Mat compressed = compressData2(cv::Mat(1, nBuf, CV_8UC1, pBuf));
@@ -665,23 +690,23 @@ void DBDriverSqlite3::disconnectDatabaseQuery(bool save, const std::string & out
fclose(fp); fclose(fp);
if(written == toWrite) if(written == toWrite)
{ {
UINFO("Wrote compressed database patchset (%d -> %d bytes) to \"%s\".", UINFO("Wrote compressed database changeset (%d -> %d bytes) to \"%s\".",
nBuf, (int)toWrite, _trackChangesOutput.c_str()); nBuf, (int)toWrite, _trackChangesOutput.c_str());
} }
else else
{ {
UERROR("Could only write %d/%d bytes of the database patchset to \"%s\".", UERROR("Could only write %d/%d bytes of the database changeset to \"%s\".",
(int)written, (int)toWrite, _trackChangesOutput.c_str()); (int)written, (int)toWrite, _trackChangesOutput.c_str());
} }
} }
else else
{ {
UERROR("Could not open \"%s\" to write the database patchset.", _trackChangesOutput.c_str()); UERROR("Could not open \"%s\" to write the database changeset.", _trackChangesOutput.c_str());
} }
} }
else if(rcp != SQLITE_OK) else if(rcp != SQLITE_OK)
{ {
UERROR("Could not generate the database patchset for \"%s\" (SQLite error %d).", UERROR("Could not generate the database changeset for \"%s\" (SQLite error %d).",
_trackChangesOutput.c_str(), rcp); _trackChangesOutput.c_str(), rcp);
} }
else else

39
tools/DatabaseUpdate/main.cpp
View File

@@ -31,20 +31,27 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string> #include <string>
#include <cstring>
using namespace rtabmap; using namespace rtabmap;
void showUsage() void showUsage()
{ {
printf("\nUpdate a database with a set of changes recorded while another database was opened with\n" printf("\nUpdate a database with a set of changes recorded while another database was opened with\n"
"change tracking enabled (i.e. DBDriverSqlite3::setTrackChangesOutput()). This applies the\n" "change tracking enabled (the --track-changes option of rtabmap-reprocess,\n"
"data changes only; it does NOT upgrade the database schema/version.\n" "rtabmap-detectMoreLoopClosures or rtabmap-reduceGraph). This applies the data changes\n"
"only; it does NOT upgrade the database schema/version.\n"
"\n" "\n"
"Usage:\n" "Usage:\n"
" rtabmap-dbupdate \"database.db\" \"changes.update\"\n" " rtabmap-dbupdate [--rewind] \"database.db\" \"changes.update\"\n"
"\n" "\n"
"The changes must be applied to the exact database state they were recorded from; otherwise\n" "Options:\n"
"the operation is aborted and the database is left unchanged.\n" " --rewind Undo a previously applied update instead of applying it (the changeset is\n"
" inverted). The database must be in the state right after the update was applied.\n"
"\n"
"The changes must be applied to the exact database state they were recorded from (or, with\n"
"--rewind, the state right after they were applied); otherwise the operation is aborted and\n"
"the database is left unchanged.\n"
"\n"); "\n");
exit(1); exit(1);
} }
@@ -59,6 +66,20 @@ int main(int argc, char * argv[])
showUsage(); showUsage();
} }
bool rewind = false;
for(int i=1; i<argc-2; ++i)
{
if(std::strcmp(argv[i], "--rewind") == 0)
{
rewind = true;
}
else
{
printf("Unknown option \"%s\".\n", argv[i]);
showUsage();
}
}
std::string databasePath = argv[argc-2]; std::string databasePath = argv[argc-2];
std::string updatePath = argv[argc-1]; std::string updatePath = argv[argc-1];
@@ -73,12 +94,14 @@ int main(int argc, char * argv[])
return 1; return 1;
} }
printf("Updating database \"%s\" with changes from \"%s\"...\n", databasePath.c_str(), updatePath.c_str()); printf("%s database \"%s\" with changes from \"%s\"...\n",
rewind?"Rewinding":"Updating", databasePath.c_str(), updatePath.c_str());
std::string error; std::string error;
if(DBDriverSqlite3::applyChangesFromFile(databasePath, updatePath, &error)) if(DBDriverSqlite3::applyChangesFromFile(databasePath, updatePath, rewind, &error))
{ {
printf("Done! Database \"%s\" was updated successfully.\n", databasePath.c_str()); printf("Done! Database \"%s\" was %s successfully.\n",
databasePath.c_str(), rewind?"rewound":"updated");
return 0; return 0;
} }