From 2f60d354b8d89f0c87741783d7812ec557b0682d Mon Sep 17 00:00:00 2001 From: matlabbe Date: Tue, 21 Jun 2011 21:21:52 +0000 Subject: [PATCH] Updated MATLAB scripts for showing logs git-svn-id: http://rtabmap.googlecode.com/svn/trunk/rtabmap@222 f169173b-cf89-36c8-b27e-44dbe73f0c83 --- CMakeLists.txt | 3 +- Matlab/ShowLogs/getPrecisionRecall.m | 109 ++++++++ Matlab/ShowLogs/importfile.m | 7 +- Matlab/ShowLogs/showlogs.m | 363 +++++++++++++++++++++++++++ guilib/src/MainWindow.cpp | 13 +- 5 files changed, 488 insertions(+), 7 deletions(-) create mode 100644 Matlab/ShowLogs/getPrecisionRecall.m create mode 100644 Matlab/ShowLogs/showlogs.m diff --git a/CMakeLists.txt b/CMakeLists.txt index e0a5dfc7..51ccaf6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,7 +200,8 @@ set(CPACK_SOURCE_IGNORE_FILES # Share files INSTALL(FILES - ${PROJECT_SOURCE_DIR}/Matlab/ShowLogs/ShowLogs.m + ${PROJECT_SOURCE_DIR}/Matlab/ShowLogs/showlogs.m + ${PROJECT_SOURCE_DIR}/Matlab/ShowLogs/getPrecisionRecall.m ${PROJECT_SOURCE_DIR}/Matlab/ShowLogs/importfile.m DESTINATION share/${PROJECT_PREFIX} COMPONENT runtime) diff --git a/Matlab/ShowLogs/getPrecisionRecall.m b/Matlab/ShowLogs/getPrecisionRecall.m new file mode 100644 index 00000000..31025c00 --- /dev/null +++ b/Matlab/ShowLogs/getPrecisionRecall.m @@ -0,0 +1,109 @@ +function [ PR ] = getPrecisionRecall( LogI, LogF, GT_file, LoopThr ) +%GETPRECISIONRECALL Calculate the precision-recall results from the log +%files of RTAB-Map and a Ground Truth file (a bmp). +% PR(:,1) = Precision +% PR(:,2) = Recall +% PR(:,3) = Precision with verification +% PR(:,4) = Recall with verification +% +% LogI: The 'LogI.txt' generated file +% LogF: The 'LogF.txt' generated file +% GT_file: The related Ground truth file of the dataset ('GT.bmp') +% LoopThr: Display false positives over the loop thr (>=0.0 && < 1.0) + +GroundTruth = []; +if exist(GT_file, 'file') + display('--- getPrecisionRecall ---'); + display(['Loading GroundTruth ''' GT_file ''' ...']); + GroundTruth = imread(GT_file); +else + error(['The ground truth ''' GT_file '''doesn''t exist.']) +end + +if ~isempty(GroundTruth) + %display('Calculating Precision-Recall graph') + %figure + %imshow(GroundTruth) + %title('GroundTruth') + + if size(GroundTruth, 1) ~= length(LogF(:,1)) || size(GroundTruth, 1) ~= length(LogI(:,1)) + error(['The ground truth size doesn''t match the log files (LogI=' num2str(length(LogI(:,1))) ', LogF=' num2str(length(LogF(:,1))) ', GT=' num2str(size(GroundTruth, 1)) ')']) + end + + + %[highestHypot, CorrespondingID, GT, Accepted, Good, Index, UnderLoopRatio] descending order + lc = [LogF(:,10) LogI(:,2) sum(GroundTruth == 255, 2)>0 LogI(:, 1) > 0 zeros(length(LogI(:,1)),1) (1:length(LogF(:,10)))' LogI(:, 8) == 1]; + + %eliminate loops on diagonal + ignored = 0; + for i=1:length(lc) + index = find(GroundTruth(:,i) > 0 & GroundTruth(:,i) < 255); + if ~isempty(index) + row = GroundTruth(index(1), :); + if lc(i,2) >= min(index) && lc(i,2) <= max(index) + %display(['i=' NUM2STR(i) ' loop=' NUM2STR(LogI(i,2)) ' min(index)=' NUM2STR(min(index)) ' max(index)' NUM2STR(max(index))]) + lc(i,1) = 0; + ignored = ignored + 1; + end + end + end + + lc = sortrows(lc, -1); + + GT_total_positives = sum(sum(GroundTruth == 255, 2) > 0) + + %figure + %plot(sum(GroundTruth > 0, 2)>0) + %title('Ground truth (timeline)') + + sizeNonZero = sum(lc(:,1) > 0); + + PR = zeros(sizeNonZero, 4); + for i=1:length(lc) + if lc(i,1) == 0 + break; + end + + id = lc(i,2); + + if id && sum(GroundTruth(lc(i,6), id)) > 0 + lc(i,5) = 1; + end + + PR(i,2) = sum(lc(1:i,5) & ~lc(1:i,7) & lc(1:i,2))/GT_total_positives; + PR(i,1) = sum(lc(1:i,5) & ~lc(1:i,7) & lc(1:i,2)) / sum(~lc(1:i, 7) & lc(1:i,2)); + + %PR(i,2) = sum(lc(1:i,5))/GT_total_positives; + %PR(i,1) = sum(lc(1:i,5)) / i; + + PR(i,4) = sum(lc(1:i,4) & lc(1:i,5))/GT_total_positives; + PR(i,3) = sum(lc(1:i,4) & lc(1:i,5)) / sum(lc(1:i,4)); + + if ~lc(i,5) && ~lc(i,7) && id && lc(i,1) >= LoopThr + display(['False positive! id=' num2str(lc(i,6)) ' with old=' num2str(id) ' (p=' num2str(lc(i,1)) ')'] ) + end + + if lc(i,4) ~= lc(i,5) && lc(i,4) + display(['False positive! (v) id=' num2str(lc(i,6)) ' with old=' num2str(id) ' (p=' num2str(lc(i,1)) ')'] ) + end + end + + index = find(PR(:,1) == 1); + if ~isempty(index) + maxRecall = PR(index(end),2) * 100; + display(['Recall max (Precision=100%) = ' num2str(maxRecall) '% (p=' num2str(lc(index(end),1)) '), accepted=' num2str(sum(lc(1:index(end),5) & ~lc(1:index(end),7) & lc(1:index(end),2)))]) + else + display('Recall max (Precision=100%) = 0') + end + indexV = find(PR(:,3) == 1); + if ~isempty(indexV) + maxRecallVerified = PR(indexV(end),4) * 100; + display(['Recall max (Precision=100%, with verification) = ' num2str(maxRecallVerified) '% (p=' num2str(lc(indexV(end),1)) ')']) + else + display('Recall max (Precision=100%, with verification) = 0') + end + display(['ignored = ' num2str(ignored)]) +end + +end + diff --git a/Matlab/ShowLogs/importfile.m b/Matlab/ShowLogs/importfile.m index 179dc97b..50894959 100644 --- a/Matlab/ShowLogs/importfile.m +++ b/Matlab/ShowLogs/importfile.m @@ -1,4 +1,4 @@ -function importfile(fileToRead1) +function [Data] = importfile(fileToRead1) %IMPORTFILE(FILETOREAD1) % Imports data from the specified file % FILETOREAD1: file to read @@ -14,9 +14,8 @@ rawData1 = importdata(fileToRead1); [unused,name] = fileparts(fileToRead1); %#ok newData1.(genvarname(name)) = rawData1; -% Create new variables in the base workspace from those fields. vars = fieldnames(newData1); -for i = 1:length(vars) - assignin('base', vars{i}, newData1.(vars{i})); +if length(vars) > 0 + Data = newData1.(vars{1}); end diff --git a/Matlab/ShowLogs/showlogs.m b/Matlab/ShowLogs/showlogs.m new file mode 100644 index 00000000..3a9620b6 --- /dev/null +++ b/Matlab/ShowLogs/showlogs.m @@ -0,0 +1,363 @@ +function showlogs(GT_file, PathPrefix) +% SHOWLOGS Plot a RTAB-Map results (LogI.txt, LogF.txt). Just put this +% file in the same directory as LogF.txt and LogI.txt files +% generated by RTAB-Map (RTAB-Map's working directory). The files +% must have the same number of lines. +% +% SHOWLOGS(GT_file) +% GT_file (optional) is the Ground Truth file. The Ground Truth is a +% squared bmp file (size must match the log files length) where +% white dots mean loop closures. Grey dots mean 'loop closures to +% ignore', this happens when the rehearsal doesn't match consecutive +% images together. +% PathPrefix (optional) is a path prefix to put before the loaded +% files (LogI.txt, LogF.txt and GT_File). +% +% Dependency : importfile.m + +%-------------------- +% Parameters +%-------------------- +set(0,'defaultAxesFontName', 'Times') +set(0,'defaultTextFontName', 'Times') + +if nargin < 2, PathPrefix = '.'; end +if nargin < 1, GT_file = ''; end + +%--------------------------------------------------------- + +display(' '); +display('Loading log files...'); +LogF = importfile([PathPrefix '/' 'LogF.txt']); +% COLUMN HEADERS : +% 1 totalTime +% 2 timeMemoryUpdate, +% 3 timeReactivations, +% 4 timeLikelihoodCalculation, +% 5 timePosteriorCalculation, +% 6 timeHypothesesCreation, +% 7 timeHypothesesValidation, +% 8 timeRealTimeLimitReachedProcess, +% 9 timeStatsCreation +% 10 highestHypothesisValue +% 11 vpLikelihood +% 12 maxLikelihood +% 13 sumLikelihoods +% 14 mean likelihood +% 15 stddev likelihood +% 16 vp hypothesis +% 17 timeEmtyingTrash + +LogI = importfile([PathPrefix '/' 'LogI.txt']); +% COLUMN HEADERS : +% 1 lcHypothesisId, +% 2 mostLikelihoodId, +% 3 signaturesRemoved, +% 4 hessianThr, +% 5 wordsNewSign, +% 6 dictionarySize, +% 7 this->getSTMem().size(), +% 8 rejectedHypothesis?, +% 9 processMemoryUsed, +% 10 databaseMemoryUsed +% 11 signaturesReactivated +% 12 lcHypothesisReactivated +% 13 refUniqueWordsCount +% 14 reactivateId +% 15 non nulls count + +if isempty(LogI) || isempty(LogF) + error('Log files are empty') +end + +if size(LogI, 1) ~= size(LogF, 1) + error('Log files are not the same size') +end + +% figure +% subplot(211) +% H1 = plot(LogF(:,1)*1000); +% hold on +% % H2 = plot(1:length(LogF(:,1)), ones(length(LogF(:,1)),1).*mean(LogF(:,1))*1000, 'r-') +% %title('Total process time / Location') +% ylabel('Time (ms)') +% xlabel('Location indexes') +% meanTime = mean(LogF(:,1))*1000 +% %plot([1 length(LogF(:,1))], [800 800], 'r') +% %plot([1 length(LogF(:,1))], [1000 1000], 'k') +% subplot(212) +% plot(sum(LogF(:,2:7),2)*1000); +% ylabel('Time (ms)') +% xlabel('Location indexes') + +figure +plot(sum(LogF(:,2:7),2)*1000); +hold on +ylabel('Time (ms)') +xlabel('Location indexes') +%meanTime = mean(LogF(:,1))*1000 +%plot([1 length(LogF(:,1))], [700 700], 'r') +%plot([1 length(LogF(:,1))], [1000 1000], 'k') + +maxTime = max(sum(LogF(:,2:7),2)) +maxDict = max(LogI(:, 6)) +maxWM = max(LogI(:,7)) +%% + +% ------------------------- +% Time details +figure +subplot(8,1,1) +plot(LogF(:,2)*1000) +title('timeMemoryUpdate (ms)') + +subplot(8,1,2) +plot(LogF(:,3)*1000) +title('timeReactivations (ms)') + +subplot(8,1,3) +plot(LogF(:,4)*1000) +title('timeLikelihoodCalculation (ms)') + +subplot(8,1,4) +plot(LogF(:,5)*1000) +title('timePosteriorCalculation (ms)') + +subplot(8,1,5) +plot(LogF(:,6)*1000) +title('timeHypothesesCreation (ms)') + +subplot(8,1,6) +plot(LogF(:,7)*1000) +title('timeHypothesesValidation (ms)') + +subplot(8,1,7) +plot(LogF(:,8)*1000) +title('timeStatsCreation (ms)') + +if size(LogF, 2) > 16 + subplot(8,1,8) + plot(LogF(:,17)*1000) + title('timeEmptyingTrash (ms)') +end +xlabel('Location indexes') + +% ------------------------- +figure +plot([LogF(:,2) sum(LogF(:,2:3),2) sum(LogF(:,2:4),2) sum(LogF(:,2:5),2) sum(LogF(:,2:6),2) sum(LogF(:,2:7),2) sum(LogF(:,2:8),2)]); +legend('timeMemoryUpdate', 'timeReactivations', 'timeLikelihoodCalculation', 'timePosteriorCalculation', 'timeHypothesesCreation', 'timeHypothesesValidation', 'timeRealTimeLimitReachedProcess') +title('Process time details') +ylabel('s') +xlabel('Location indexes') + +figure +subplot(211) +plot(LogF(:,3)); +title('Reactivation time (s)') +ylabel('s') +subplot(212) +plot(LogI(:,11),'.') +ylabel('Locations reactivated') +xlabel('Location indexes') + +% ------------------------- +figure +subplot(211) +plot(LogI(:, 6)); +title('dictionary size') +ylabel('words') + +subplot(212) +plot([LogI(:, 9)/1000000 LogI(:, 10)/1000000]); +title('Memory usage (in MB)') +legend('Process', 'Database') +ylabel('MB') +xlabel('Location indexes') +% ------------------------- + +figure +% subplot(211) +H1 = plot(LogI(:,7)); +% hold on +% H2 = plot(1:length(LogI(:,7)), ones(length(LogI(:,7)),1).*mean(LogI(:,7)), 'r--') +title('Working memory size') +meanWM = mean(LogI(:,7)) +ylabel('WM size (locations)') +xlabel('Location indexes') +% set(H1,'color',[0.3 0.3 0.3]) +% set(H2,'color',[0 0 0]) +% subplot(212) +% plot(LogI(:,6)); +meanDict = mean(LogI(:,6)) +% ylabel('Dictionary size') +% xlabel('Location indexes') + +meanWordsPerSign = mean(LogI(:,5)) + + +% ------------------------- +% Detected/Accepted/Rejected loop closures + +figure; +subplot(311) +plot(LogF(:,10), '.') +title('Highest posterior, green=accepted, red=rejected, blue=under T_{Loop}') +hold on +ylabel('p') +%rejected (by T_loop) hypotheses +y = LogF(:,10); +x = 1:length(y); +y(LogI(:, 1) == 0 & LogI(:, 8) ~= 0) = []; +x(LogI(:, 1) == 0 & LogI(:, 8) ~= 0) = []; +plot(x,y, 'b.') +%rejected (by ratio) hypotheses +y = LogF(:,10); +x = 1:length(y); +y(LogI(:, 8) == 0 | LogI(:, 1) > 0) = []; +x(LogI(:, 8) == 0 | LogI(:, 1) > 0) = []; +plot(x,y, 'r.') +%Accepted hypotheses +y = LogF(:,10); +x = 1:length(y); +y(LogI(:, 1) == 0) = []; +x(LogI(:, 1) == 0) = []; +plot(x,y, 'g.') +subplot(312) +plot(LogI(:,2), '.') +title('Id corresponding to highest posterior + lc accepted and rejected') +hold on +ylabel('Matched location index') +%rejected hypotheses +y = LogI(:,2); +x = 1:length(y); +y(LogI(:, 1) == 0 & LogI(:, 8) ~= 0) = []; +x(LogI(:, 1) == 0 & LogI(:, 8) ~= 0) = []; +plot(x,y, 'b.') +%rejected (by ratio) hypotheses +y = LogI(:,2); +x = 1:length(y); +y(LogI(:, 8) == 0 | LogI(:, 1) > 0) = []; +x(LogI(:, 8) == 0 | LogI(:, 1) > 0) = []; +plot(x,y, 'r.') +%Accepted hypotheses +y = LogI(:,2); +x = 1:length(y); +y(LogI(:, 1) == 0) = []; +x(LogI(:, 1) == 0) = []; +plot(x,y, 'g.') +subplot(313) +plot(LogI(:,5),'.') +title('wordsNewSign') +hold on +ylabel('words') +xlabel('Location indexes') +%rejected hypotheses +y = LogI(:,5); +x = 1:length(y); +y(LogI(:, 1) == 0 & LogI(:, 8) ~= 0) = []; +x(LogI(:, 1) == 0 & LogI(:, 8) ~= 0) = []; +plot(x,y, 'b.') +%rejected (by ratio) hypotheses +y = LogI(:,5); +x = 1:length(y); +y(LogI(:, 8) == 0 | LogI(:, 1) > 0) = []; +x(LogI(:, 8) == 0 | LogI(:, 1) > 0) = []; +plot(x,y, 'r.') +%Accepted hypotheses +y = LogI(:,5); +x = 1:length(y); +y(LogI(:, 1) == 0) = []; +x(LogI(:, 1) == 0) = []; +plot(x,y, 'g.') + +set(datacursormode,'UpdateFcn',@(Y,X){sprintf('X: %0.2f',X.Position(1)),sprintf('Y: %0.2f',X.Position(2))}) +% %matched sign words +% y = LogI(:,2); +% x = 1:length(y); +% mask = zeros(1,length(y)); +% y(LogI(:, 8) ~= 11) = []; +% for i=1:length(y) +% mask(y(i)) = 1; +% end +% y = LogI(:,5); +% y(~mask) = []; +% x(~mask) = []; +% plot(x,y, 'c.') +% %matched sign words for rejected +% y = LogI(:,2); +% x = 1:length(y); +% mask = zeros(1,length(y)); +% y(LogI(:, 8) < 12) = []; +% for i=1:length(y) +% mask(y(i)) = 1; +% end +% y = LogI(:,5); +% y(~mask) = []; +% x(~mask) = []; +% plot(x,y, 'm.') + +lcAccepted = sum(LogI(:, 1) > 0) +lcReactivated = sum(LogI(:, 12) == 1) +lcIgnored = sum(LogI(:, 1) == 0 & LogI(:, 8) == 0) +lcRejected = sum(LogI(:, 8) == 1) + +%figure; +%plot([1.0 * (LogI(:, 8) == 10) ... +% 1.01 * (LogI(:, 8) == 11) ... +% 1.02 * (LogI(:, 8) == 14) ... +% 1.03 * (LogI(:, 8) == 15)], '.'); +%title('Reject loop reason') +%legend('UNDEFINED', 'ACCEPTED', 'NOT ENOUGH MATCHING PAIRS', 'EPIPOLAR CONSTRAINT FAILED') + +% ----------------- +% Squared matrix + + + +%% +%Precision-Recall graph +GroundTruthFile = [PathPrefix '/' GT_file]; +if exist(GroundTruthFile, 'file') + PR = getPrecisionRecall(LogI, LogF, GroundTruthFile, 1.0); + + Precision = PR(:,1); + Recall = PR(:,2); + PrecisionVerified = PR(:,3); + RecallVerified = PR(:,4); + + %plot the Precision-Recall + figure + plot(Recall*100, Precision*100) + %plot([Recall RecallVerified], [Precision PrecisionVerified]) + %legend('Without verification', 'With verification') + title('Precision-Recall curve') + xlabel('Recall (%)') + ylabel('Precision (%)') +else + display('Precision-recall curve is not computed...'); +end + +%% +% count = 0; +% for i=2:length(LogF(:,10)) +% if(LogF(i,10) >= 0.03723 && LogF(i,10) < LogF(i-1,10)*0.90) +% display(['i=' num2str(i) ' with=' num2str(LogI(i,2)) ' ratio=' num2str(LogF(i,10)/LogF(i-1,10))]) +% count = count +1; +% end +% end +% count + +%% +% figure +% hold on +% K=100; +% %plot(1./(K*LogF(:,15)), 'r') +% %plot(log10(1./(LogF(:,15))), 'c') +% scale=1; +% %plot(log10(1./(LogF(:,15))).^2 ./ ((LogF(:,12)-LogF(:,15))./LogF(:,14)), 'k') +% %plot(log10(1./(LogF(:,15))), 'm') +% plot((LogF(:,12)-LogF(:,15))./LogF(:,14), 'g') +% plot([0, length(LogF(:,15))], [1 1], 'k:') +% plot(LogF(:,11), 'b') +% %legend(['K=' num2str(K)], 'ln', 'ln scaled', 'log10', 'max sim', '1', 'Vp likelihood') + diff --git a/guilib/src/MainWindow.cpp b/guilib/src/MainWindow.cpp index b5d645c9..3df9bae8 100644 --- a/guilib/src/MainWindow.cpp +++ b/guilib/src/MainWindow.cpp @@ -52,7 +52,8 @@ #include #define LOG_FILE_NAME "LogRtabmap.txt" -#define SHARE_SHOW_LOG_FILE "share/rtabmap/ShowLogs.m" +#define SHARE_SHOW_LOG_FILE "share/rtabmap/showlogs.m" +#define SHARE_GET_PRECISION_RECALL_FILE "share/rtabmap/getPrecisionRecall.m" #define SHARE_IMPORT_FILE "share/rtabmap/importfile.m" using namespace rtabmap; @@ -1020,7 +1021,7 @@ void MainWindow::stopDetection() emit stateChanged(kIdle); } - //Copy ShowLogs.m from appDirPath/../share/rtabmap/ShowLogs.m to working directory. (appDirPath is in bin) + //Copy showlogs.m from appDirPath/../share/rtabmap/ShowLogs.m to working directory. (appDirPath is in bin) QString showLogFileSrc = (QApplication::applicationDirPath()+"/../")+SHARE_SHOW_LOG_FILE; QString showLogFileTarget = (_preferencesDialog->getWorkingDirectory()+"/")+UFile::getName(SHARE_SHOW_LOG_FILE).c_str(); if(!QFile::exists(showLogFileTarget) && QFile::exists(showLogFileSrc)) @@ -1035,6 +1036,14 @@ void MainWindow::stopDetection() { QFile::copy(importFileSrc, importFileTarget); } + + // copy getPrecisionRecall.m + QString getPrFileSrc = (QApplication::applicationDirPath()+"/../")+SHARE_GET_PRECISION_RECALL_FILE; + QString getPrFileTarget = (_preferencesDialog->getWorkingDirectory()+"/")+UFile::getName(SHARE_GET_PRECISION_RECALL_FILE).c_str(); + if(!QFile::exists(getPrFileTarget) && QFile::exists(getPrFileSrc)) + { + QFile::copy(getPrFileSrc, getPrFileTarget); + } } void MainWindow::generateMap()