diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b55e49f..7992c03a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,8 +21,8 @@ SET(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake_modules") # VERSION ####################### SET(RTABMAP_MAJOR_VERSION 0) -SET(RTABMAP_MINOR_VERSION 23) -SET(RTABMAP_PATCH_VERSION 13) +SET(RTABMAP_MINOR_VERSION 24) +SET(RTABMAP_PATCH_VERSION 0) SET(RTABMAP_VERSION ${RTABMAP_MAJOR_VERSION}.${RTABMAP_MINOR_VERSION}.${RTABMAP_PATCH_VERSION}) diff --git a/app/ios/RTABMapApp.xcodeproj/project.pbxproj b/app/ios/RTABMapApp.xcodeproj/project.pbxproj index 6fc3ba60..ab7649d0 100644 --- a/app/ios/RTABMapApp.xcodeproj/project.pbxproj +++ b/app/ios/RTABMapApp.xcodeproj/project.pbxproj @@ -1078,7 +1078,7 @@ "\"$(SRCROOT)/RTABMapApp/Libraries/include\"", "\"$(SRCROOT)/RTABMapApp/Libraries/include/eigen3\"", "\"$(SRCROOT)/RTABMapApp/Libraries/include/pcl-1.15\"", - "\"$(SRCROOT)/RTABMapApp/Libraries/include/rtabmap-0.23\"", + "\"$(SRCROOT)/RTABMapApp/Libraries/include/rtabmap-0.24\"", "\"$(SRCROOT)/../android/jni/tango-gl/include\"", "\"$(SRCROOT)/../android/jni/third-party/include\"", "\"$(SRCROOT)/RTABMapApp/Libraries/lib/vtk.framework/Headers\"", @@ -1139,7 +1139,7 @@ "\"$(SRCROOT)/RTABMapApp/Libraries/include\"", "\"$(SRCROOT)/RTABMapApp/Libraries/include/eigen3\"", "\"$(SRCROOT)/RTABMapApp/Libraries/include/pcl-1.15\"", - "\"$(SRCROOT)/RTABMapApp/Libraries/include/rtabmap-0.23\"", + "\"$(SRCROOT)/RTABMapApp/Libraries/include/rtabmap-0.24\"", "\"$(SRCROOT)/../android/jni/tango-gl/include\"", "\"$(SRCROOT)/../android/jni/third-party/include\"", "\"$(SRCROOT)/RTABMapApp/Libraries/lib/vtk.framework/Headers\"", diff --git a/corelib/include/rtabmap/core/Compression.h b/corelib/include/rtabmap/core/Compression.h index a57ce616..80fb0c7d 100644 --- a/corelib/include/rtabmap/core/Compression.h +++ b/corelib/include/rtabmap/core/Compression.h @@ -41,7 +41,8 @@ namespace rtabmap { * @brief Background thread to compress or uncompress images and generic matrices. * * In compress mode, pass a source matrix to the constructor with an optional image - * format (".png", ".jpg", ".rvl", or empty for zlib data). In uncompress mode, pass + * format (".png", ".jpg", ".rvl", or empty for zlib data, see @ref compressImage() for + * depth options). In uncompress mode, pass * compressed bytes and set @c isImage accordingly. Call @ref UThread::start() then * @ref UThread::join() to obtain the result from @ref getCompressedData() or * @ref getUncompressedData(). @@ -70,7 +71,7 @@ public: /** * @brief Constructs a thread in compress mode. * @param mat Source image or data matrix to compress. - * @param format Image format: @c ".png", @c ".jpg", @c ".rvl", or empty for zlib (@ref compressData2). + * @param format Image format: @c ".png", @c ".jpg", @c ".rvl" (see @ref compressImage()), or empty for zlib (@ref compressData2). */ CompressionThread(const cv::Mat & mat, const std::string & format = ""); /** @@ -93,7 +94,63 @@ private: bool compressMode_; }; -/** @brief Encodes @p image to a byte buffer (OpenCV @c imencode or RVL for depth). */ +/* + * Compressed depth image layouts (all values little-endian). They are stable: they are + * saved in databases, and rtabmap_ros converts them to and from ROS's + * compressed_depth_image_transport messages without decompressing the images. + * + * - ".png": a standard PNG file. 16UC1 depth images are 16 bits grayscale PNGs, + * 32FC1 depth images (legacy) are 4-channel 8 bits PNGs holding the float bytes. + * - ".rvl" (16UC1): + * [0..7] "DEPTHRVL" + * [8..11] uint32 cols + * [12..15] uint32 rows + * [16..] RVL data (see RvlCodec) + * - ".png::" or ".rvl::" (32FC1): + * [0..7] "DEPTHINV" + * [8..11] float depthQuantA = quantization*(quantization+1) + * [12..15] float depthQuantB = 1 - depthQuantA/maxDepth + * [16..] the 16UC1 inverse depth image in ".png" or ".rvl" layout above, where + * 0 is invalid and v>0 is the depth depthQuantA/(v-depthQuantB). + */ + +/** @brief Signature of the ".rvl" layout (8 bytes, not null-terminated). */ +const char kCompressedDepthRvlSignature[8] = {'D', 'E', 'P', 'T', 'H', 'R', 'V', 'L'}; +/** @brief Size of the ".rvl" header: signature, uint32 cols, uint32 rows. */ +const size_t kCompressedDepthRvlHeaderSize = 16; +/** @brief Signature of the inverse depth layout (8 bytes, not null-terminated). */ +const char kCompressedDepthInvSignature[8] = {'D', 'E', 'P', 'T', 'H', 'I', 'N', 'V'}; +/** @brief Size of the inverse depth header: signature, float depthQuantA, float depthQuantB. */ +const size_t kCompressedDepthInvHeaderSize = 16; + +/** + * @brief Parses an image compression format "[:[:]]". + * + * @param format Format, e.g., @c ".jpg", @c ".png", @c ".rvl", @c ".png:10" or @c ".rvl:20:100". + * The empty format is valid (general zlib compression, see @ref CompressionThread). + * @param codec Output codec (e.g., @c ".png"). + * @param maxDepth Output maximum depth (m) of the inverse depth format, 0 if not set. + * @param quantization Output depth quantization of the inverse depth format + * (100 if not set but @p maxDepth is), 0 if @p maxDepth is not set. + * @return false if the format is invalid. The inverse depth parameters are only + * valid with @c ".png" and @c ".rvl", and should be positive. + */ +bool RTABMAP_CORE_EXPORT parseImageCompressionFormat(const std::string & format, std::string & codec, float & maxDepth, float & quantization); + +/** + * @brief Encodes @p image to a byte buffer (OpenCV @c imencode or RVL for depth). + * + * @param format @c ".png", @c ".jpg" or @c ".rvl" (16UC1 only), optionally followed by + * @c ":[:]" (see @ref parseImageCompressionFormat()). + * For 32FC1 depth images, if @c maxDepth is set, depth is quantized on 16 bits + * as inverse depth (as ROS's @c compressed_depth_image_transport) and compressed + * with the codec. With A=quantization*(quantization+1) and B=1-A/maxDepth, the + * precision is ~d^2/(2A), and depth values over @c maxDepth or under A/(65535-B) + * are lost (set to 0). For example, ".png:10:100" keeps depth between 0.15 and + * 10 m with errors of 0.05 mm at 1 m and 5 mm at 10 m. Otherwise, 32FC1 depth + * images are compressed losslessly as 4-channel 8 bits PNG (legacy format). Other + * image types ignore the depth parameters. + */ std::vector RTABMAP_CORE_EXPORT compressImage(const cv::Mat & image, const std::string & format = ".png"); /** @brief Same as @ref compressImage() but returns a @c CV_8UC1 row matrix. */ cv::Mat RTABMAP_CORE_EXPORT compressImage2(const cv::Mat & image, const std::string & format = ".png"); @@ -102,6 +159,8 @@ cv::Mat RTABMAP_CORE_EXPORT compressImage2(const cv::Mat & image, const std::str cv::Mat RTABMAP_CORE_EXPORT uncompressImage(const cv::Mat & bytes); /** @brief Decodes compressed image bytes to a @cv::Mat. */ cv::Mat RTABMAP_CORE_EXPORT uncompressImage(const std::vector & bytes); +/** @brief Decodes compressed image bytes to a @cv::Mat. */ +cv::Mat RTABMAP_CORE_EXPORT uncompressImage(const unsigned char * bytes, size_t size); /** @brief Compresses a matrix with zlib; appends rows, cols and type at the end. */ std::vector RTABMAP_CORE_EXPORT compressData(const cv::Mat & data); @@ -122,7 +181,10 @@ std::string RTABMAP_CORE_EXPORT uncompressString(const cv::Mat & bytes); /** * @brief Detects the compression format of depth image bytes. - * @return @c ".rvl" if the buffer has an RVL signature, otherwise @c ".png". + * @return @c ".rvl" if the buffer has an RVL signature, @c ".png::" + * or @c ".rvl::" for inverse depth images (see + * @ref compressImage()), otherwise @c ".png". The returned format can be passed + * back to @ref compressImage() to compress in the same format. */ std::string RTABMAP_CORE_EXPORT compressedDepthFormat(const cv::Mat & bytes); /** @overload */ @@ -130,5 +192,6 @@ std::string RTABMAP_CORE_EXPORT compressedDepthFormat(const std::vector= 0.24 bool _compressionParallelized; float _laserScanDownsampleStepSize; float _laserScanVoxelSize; diff --git a/corelib/include/rtabmap/core/Parameters.h b/corelib/include/rtabmap/core/Parameters.h index 622f58ec..b93e9d75 100644 --- a/corelib/include/rtabmap/core/Parameters.h +++ b/corelib/include/rtabmap/core/Parameters.h @@ -222,7 +222,7 @@ class RTABMAP_CORE_EXPORT Parameters RTABMAP_PARAM(Mem, NotLinkedNodesKept, bool, true, "Keep not linked nodes in db (rehearsed nodes and deleted nodes)."); RTABMAP_PARAM(Mem, IntermediateNodeDataKept, bool, false, "Keep intermediate node data in db."); RTABMAP_PARAM_STR(Mem, ImageCompressionFormat, ".jpg", "RGB image compression format. It should be \".jpg\" or \".png\"."); - RTABMAP_PARAM_STR(Mem, DepthCompressionFormat, ".rvl", "Depth image compression format for 16UC1 depth type. It should be \".png\" or \".rvl\". If depth type is 32FC1, \".png\" is used."); + RTABMAP_PARAM_STR(Mem, DepthCompressionFormat, ".rvl", "Depth image compression format. It should be \".png\" or \".rvl\", optionally followed by \":maxDepth[:quantization]\" (e.g., \".rvl:10:100\", quantization is 100 by default) to compress 32FC1 depth images as 16 bits inverse depth with that codec (same quantization than ROS's compressed_depth_image_transport). 16UC1 depth images are always compressed losslessly with the codec. Warning: the inverse depth format is lossy, the precision is ~d^2/(2*q*(q+1)) (q=quantization, e.g., 0.05 mm at 1 m and 5 mm at 10 m with q=100) and depth values over maxDepth or under ~q*(q+1)/65535 meters (0.15 m with q=100) are lost. Without depth parameters, 32FC1 depth images are compressed losslessly in \".png\" format (4 channels 8 bits). Databases with depth images compressed in inverse depth format cannot be opened by rtabmap versions under 0.24."); RTABMAP_PARAM(Mem, STMSize, unsigned int, 10, "Short-term memory size."); RTABMAP_PARAM(Mem, IncrementalMemory, bool, true, "SLAM mode, otherwise it is Localization mode."); RTABMAP_PARAM(Mem, LocalizationReadOnly, bool, false, uFormat("In localization mode, open the database in read-only mode (ignored if %s=true). Currrenty incompatible with memory management (%s and %s cannot be used) and if there are disjoint sessions in working memory. Last localization pose won't be saved back in the database at the end of the session, so the robot will always restart to original last localization pose, unless %s is used or an external initial pose is provided on initialization.", kMemIncrementalMemory().c_str(), kRtabmapLoopThr().c_str(), kRtabmapMemoryThr().c_str(), kRGBDStartAtOrigin().c_str()).c_str()); diff --git a/corelib/include/rtabmap/core/SensorData.h b/corelib/include/rtabmap/core/SensorData.h index b542ff19..79c2bf08 100644 --- a/corelib/include/rtabmap/core/SensorData.h +++ b/corelib/include/rtabmap/core/SensorData.h @@ -598,6 +598,8 @@ public: /** * Set image data. Detect automatically if raw or compressed. * A matrix of type CV_8UC1 with 1 row is considered as compressed. + * An invalid @p model (not CameraModel::isValidForProjection()) without any image is + * a placeholder (e.g., scan-only data): it is not added, so cameraModels() is empty. * @param clearPreviousData, clear previous raw and compressed images before setting the new ones. */ void setRGBDImage(const cv::Mat & rgb, const cv::Mat & depth, const CameraModel & model, bool clearPreviousData = true); @@ -1025,6 +1027,9 @@ public: #endif private: + /// Whether setRGBDImage() keeps @p model: not an invalid model without any image. + bool keepCameraModel(const CameraModel & model, const cv::Mat & rgb, const cv::Mat & depth, bool clearPreviousData) const; + int _id; ///< Unique sensor data ID (0 if invalid) double _stamp; ///< Timestamp in seconds diff --git a/corelib/src/Compression.cpp b/corelib/src/Compression.cpp index b346817e..de3727ec 100644 --- a/corelib/src/Compression.cpp +++ b/corelib/src/Compression.cpp @@ -28,9 +28,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "rtabmap/core/Compression.h" #include #include +#include #include #include +#include +#include namespace rtabmap { @@ -67,16 +70,117 @@ int deserializeMatType(int serializedType) ((serializedType >> kSerializedCnShift) & 511) + 1); } +// Default quantization when only the maximum depth is set in the format. +const float kDefaultDepthQuantization = 100.0f; + +bool hasSignature(const unsigned char * bytes, size_t size, const void * signature) +{ + return bytes && size >= 8 && memcmp(bytes, signature, 8) == 0; +} + +// Values over maxDepth, NaN, inf, 0 and negative values are set to 0 (invalid), +// as well as values too close to be represented on 16 bits (under +// depthQuantA / (65535 - depthQuantB) meters). +cv::Mat depthToInvDepth(const cv::Mat & depth, float maxDepth, float quantization, float & depthQuantA, float & depthQuantB) +{ + UASSERT(depth.type() == CV_32FC1); + depthQuantA = quantization * (quantization + 1.0f); + depthQuantB = 1.0f - depthQuantA / maxDepth; + cv::Mat invDepth(depth.size(), CV_16UC1); + for(int i=0; i(i); + uint16_t * out = invDepth.ptr(i); + for(int j=0; j 0.0f && d < maxDepth) // false for NaN + { + // Rounded (ROS truncates), the decoding is the same. + const float v = depthQuantA / d + depthQuantB + 0.5f; + out[j] = v < 65536.0f ? (uint16_t)v : 0; + } + else + { + out[j] = 0; + } + } + } + return invDepth; +} + +cv::Mat invDepthToDepth(const cv::Mat & invDepth, float depthQuantA, float depthQuantB) +{ + UASSERT(invDepth.type() == CV_16UC1); + cv::Mat depth(invDepth.size(), CV_32FC1); + for(int i=0; i(i); + float * out = depth.ptr(i); + for(int j=0; j fields = uListToVector(uSplit(format, ':')); + if(fields.empty()) + { + return format.empty(); // empty is general (zlib) + } + if(fields[0].size() < 2 || fields[0][0] != '.') + { + return false; + } + if(fields.size() > 1) + { + // Inverse depth parameters only for formats supporting 16UC1 + if((fields[0] != ".png" && fields[0] != ".rvl") || fields.size() > 3) + { + return false; + } + for(size_t i=1; ikill(); } -// ".jpg" or ".png" or ".rvl" +// ".jpg" or ".png" or ".rvl", with optional ":maxDepth[:quantization]" for 32FC1 depth images std::vector compressImage(const cv::Mat & image, const std::string & format) { std::vector bytes; if(!image.empty()) { - if(image.type() == CV_32FC1) + std::string codec; + float maxDepth, quantization; + if(!parseImageCompressionFormat(format, codec, maxDepth, quantization) || codec.empty()) + { + UERROR("Invalid image compression format \"%s\"", format.c_str()); + return bytes; + } + + if(image.type() == CV_32FC1 && maxDepth > 0.0f) + { + float depthQuantA, depthQuantB; + cv::Mat invDepth = depthToInvDepth(image, maxDepth, quantization, depthQuantA, depthQuantB); + std::vector invDepthBytes = compressImage(invDepth, codec); + if(!invDepthBytes.empty()) + { + bytes.resize(kCompressedDepthInvHeaderSize + invDepthBytes.size()); + memcpy(&bytes[0], kCompressedDepthInvSignature, 8); + memcpy(&bytes[8], &depthQuantA, 4); + memcpy(&bytes[12], &depthQuantB, 4); + memcpy(&bytes[kCompressedDepthInvHeaderSize], invDepthBytes.data(), invDepthBytes.size()); + } + } + else if(image.type() == CV_32FC1) { //save in 8bits-4channel cv::Mat bgra(image.size(), CV_8UC4, image.data); cv::imencode(".png", bgra, bytes); } - else if(format == ".rvl") + else if(codec == ".rvl") { - bytes = {'D', 'E', 'P', 'T', 'H', 'R', 'V', 'L'}; + bytes.assign(kCompressedDepthRvlSignature, kCompressedDepthRvlSignature+8); int numPixels = image.rows * image.cols; // In the worst case, RVL compression results in ~1.5x larger data. bytes.resize(3 * numPixels + 20); @@ -154,18 +280,18 @@ std::vector compressImage(const cv::Mat & image, const std::strin memcpy(&bytes[8], &cols, 4); memcpy(&bytes[12], &rows, 4); RvlCodec rvl; - int compressedSize = rvl.CompressRVL(image.ptr(), &bytes[16], numPixels); - bytes.resize(16 + compressedSize); + int compressedSize = rvl.CompressRVL(image.ptr(), &bytes[kCompressedDepthRvlHeaderSize], numPixels); + bytes.resize(kCompressedDepthRvlHeaderSize + compressedSize); } else { - cv::imencode(format, image, bytes); + cv::imencode(codec, image, bytes); } } return bytes; } -// ".jpg" or ".png" or ".rvl" +// ".jpg" or ".png" or ".rvl", with optional ":maxDepth[:quantization]" for 32FC1 depth images cv::Mat compressImage2(const cv::Mat & image, const std::string & format) { std::vector bytes = compressImage(image, format); @@ -178,24 +304,64 @@ cv::Mat compressImage2(const cv::Mat & image, const std::string & format) cv::Mat uncompressImage(const cv::Mat & bytes) { - cv::Mat image; - if(!bytes.empty()) + if(bytes.empty()) { - if (compressedDepthFormat(bytes) == ".rvl") + return cv::Mat(); + } + return uncompressImage(bytes.data, bytes.total()*bytes.elemSize()); +} + +cv::Mat uncompressImage(const std::vector & bytes) +{ + return uncompressImage(bytes.data(), bytes.size()); +} + +cv::Mat uncompressImage(const unsigned char * bytes, size_t size) +{ + cv::Mat image; + if(bytes && size) + { + if(hasSignature(bytes, size, kCompressedDepthInvSignature)) { + if(size <= kCompressedDepthInvHeaderSize) + { + UERROR("Inverse depth image is truncated (%d bytes).", (int)size); + return image; + } + float depthQuantA, depthQuantB; + memcpy(&depthQuantA, &bytes[8], 4); + memcpy(&depthQuantB, &bytes[12], 4); + cv::Mat invDepth = uncompressImage(&bytes[kCompressedDepthInvHeaderSize], size - kCompressedDepthInvHeaderSize); + if(invDepth.type() == CV_16UC1) + { + image = invDepthToDepth(invDepth, depthQuantA, depthQuantB); + } + else if(!invDepth.empty()) + { + UERROR("Inverse depth image should be 16UC1 (type=%d).", invDepth.type()); + } + } + else if(hasSignature(bytes, size, kCompressedDepthRvlSignature)) + { + if(size < kCompressedDepthRvlHeaderSize) + { + UERROR("RVL depth image is truncated (%d bytes).", (int)size); + return image; + } uint32_t cols, rows; - memcpy(&cols, &bytes.data[8], 4); - memcpy(&rows, &bytes.data[12], 4); + memcpy(&cols, &bytes[8], 4); + memcpy(&rows, &bytes[12], 4); image = cv::Mat(rows, cols, CV_16UC1); RvlCodec rvl; - rvl.DecompressRVL(&bytes.data[16], image.ptr(), cols * rows); + rvl.DecompressRVL(&bytes[kCompressedDepthRvlHeaderSize], image.ptr(), cols * rows); } else { + const cv::Mat buf(1, (int)size, CV_8UC1, (void *)bytes); #if CV_MAJOR_VERSION>2 || (CV_MAJOR_VERSION >=2 && CV_MINOR_VERSION >=4) - image = cv::imdecode(bytes, cv::IMREAD_UNCHANGED); + image = cv::imdecode(buf, cv::IMREAD_UNCHANGED); #else - image = cv::imdecode(bytes, -1); + image = cv::imdecode(buf, -1); #endif if(image.type() == CV_8UC4) { @@ -210,36 +376,6 @@ cv::Mat uncompressImage(const cv::Mat & bytes) return image; } -cv::Mat uncompressImage(const std::vector & bytes) -{ - cv::Mat image; - if(bytes.size()) - { - if (compressedDepthFormat(bytes) == ".rvl") - { - uint32_t cols, rows; - memcpy(&cols, &bytes[8], 4); - memcpy(&rows, &bytes[12], 4); - image = cv::Mat(rows, cols, CV_16UC1); - RvlCodec rvl; - rvl.DecompressRVL(&bytes[16], image.ptr(), cols * rows); - } - else - { -#if CV_MAJOR_VERSION>2 || (CV_MAJOR_VERSION >=2 && CV_MINOR_VERSION >=4) - image = cv::imdecode(bytes, cv::IMREAD_UNCHANGED); -#else - image = cv::imdecode(bytes, -1); -#endif - if(image.type() == CV_8UC4) - { - image = cv::Mat(image.size(), CV_32FC1, image.data).clone(); - } - } - } - return image; -} - std::vector compressData(const cv::Mat & data) { std::vector bytes; @@ -381,10 +517,17 @@ std::string compressedDepthFormat(const unsigned char * bytes, size_t size) std::string format; if(bytes && size) { - size_t maxlen = std::min(size, size_t(8)); - std::vector signature(maxlen); - memcpy(&signature[0], bytes, maxlen); - if (std::string(signature.begin(), signature.end()) == "DEPTHRVL") + if(hasSignature(bytes, size, kCompressedDepthInvSignature) && size > kCompressedDepthInvHeaderSize) + { + float depthQuantA, depthQuantB, maxDepth, quantization; + memcpy(&depthQuantA, &bytes[8], 4); + memcpy(&depthQuantB, &bytes[12], 4); + invDepthParameters(depthQuantA, depthQuantB, maxDepth, quantization); + format = uFormat("%s:%g:%g", + compressedDepthFormat(&bytes[kCompressedDepthInvHeaderSize], size - kCompressedDepthInvHeaderSize).c_str(), + maxDepth, quantization); + } + else if(hasSignature(bytes, size, kCompressedDepthRvlSignature)) { format = ".rvl"; } diff --git a/corelib/src/Memory.cpp b/corelib/src/Memory.cpp index 4bef1a52..82eea94e 100644 --- a/corelib/src/Memory.cpp +++ b/corelib/src/Memory.cpp @@ -102,6 +102,7 @@ Memory::Memory(const ParametersMap & parameters) : _imagePreDecimation(Parameters::defaultMemImagePreDecimation()), _imagePostDecimation(Parameters::defaultMemImagePostDecimation()), _legacyDecimatedOctave(false), + _inverseDepthCompressionAllowed(true), _compressionParallelized(Parameters::defaultMemCompressionParallelized()), _laserScanDownsampleStepSize(Parameters::defaultMemLaserScanDownsampleStepSize()), _laserScanVoxelSize(Parameters::defaultMemLaserScanVoxelSize()), @@ -228,6 +229,11 @@ bool Memory::init(const std::string & dbUrl, bool dbOverwritten, const Parameter // filling it that way; a new one gets the corrected scaling. _legacyDecimatedOctave = uStrNumCmp(_dbDriver->getDatabaseVersion(), "0.23.12") < 0; + // Depth images compressed as inverse depth cannot be read before 0.24, which + // would still open databases created with Db/TargetVersion < 0.24 or by an + // older version. + _inverseDepthCompressionAllowed = + uStrNumCmp(_dbDriver->getDatabaseVersion(), "0.24.0") >= 0; // Only where the descriptors stored in the map end up different: keypoints // from odometry, scaled into the pre-decimated image before being described. if(_legacyDecimatedOctave && _useOdometryFeatures && _imagePreDecimation > 1) @@ -826,6 +832,19 @@ void Memory::parseParameters(const ParametersMap & parameters) Parameters::parse(params, Parameters::kMemIntermediateNodeDataKept(), _saveIntermediateNodeData); Parameters::parse(params, Parameters::kMemImageCompressionFormat(), _rgbCompressionFormat); Parameters::parse(params, Parameters::kMemDepthCompressionFormat(), _depthCompressionFormat); + { + std::string codec; + float maxDepth, quantization; + if(!parseImageCompressionFormat(_depthCompressionFormat, codec, maxDepth, quantization) || + (codec != ".png" && codec != ".rvl")) + { + UWARN("Invalid %s=\"%s\", using default \"%s\".", + Parameters::kMemDepthCompressionFormat().c_str(), + _depthCompressionFormat.c_str(), + Parameters::defaultMemDepthCompressionFormat().c_str()); + _depthCompressionFormat = Parameters::defaultMemDepthCompressionFormat(); + } + } Parameters::parse(params, Parameters::kMemRehearsalIdUpdatedToNewOne(), _idUpdatedToNewOneRehearsal); Parameters::parse(params, Parameters::kMemGenerateIds(), _generateIds); Parameters::parse(params, Parameters::kMemBadSignaturesIgnored(), _badSignaturesIgnored); @@ -5227,7 +5246,7 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor if(!isIntermediateNode) { // We need raw images if we need to extract features and/or do tag detection - bool needRawImages = _feature2D->getMaxFeatures() >= 0 && + bool needRawImages = (_feature2D->getMaxFeatures() >= 0 && (!_useOdometryFeatures || data.keypoints().empty() || (int)data.keypoints().size() != data.descriptors().rows || @@ -5235,7 +5254,9 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor _detectMarkers || _rotateImagesUpsideUp || _imagePostDecimation > 1 || - (_createOccupancyGrid && _localMapMaker->isGridFromDepth())); + (_createOccupancyGrid && _localMapMaker->isGridFromDepth()))) || + // Images rectified below: stereo always, RGB-D unless only its features are + (!_imagesAlreadyRectified && !(_rectifyOnlyFeatures && data.stereoCameraModels().empty())); // Note: we could avoid uncompressing scan if we don't do any filtering // and if we don't use it for local occupancy grid @@ -6625,6 +6646,44 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor std::vector imageBytes; std::vector depthBytes; + std::string depthCompressionFormat = _depthCompressionFormat; + bool reuseCompressedDepth = + depthOrRightImage.data == data.depthOrRightRaw().data && + !data.depthOrRightCompressed().empty(); + if(!_inverseDepthCompressionAllowed) + { + std::string codec; + float maxDepth, quantization; + if(parseImageCompressionFormat(depthCompressionFormat, codec, maxDepth, quantization) && maxDepth > 0.0f) + { + static bool warned = false; + if(!warned) + { + UWARN("%s=\"%s\": inverse depth compression format is not compatible with database " + "version %s (requires >= 0.24, see %s), \"%s\" format is used instead. This " + "warning is only printed once.", + Parameters::kMemDepthCompressionFormat().c_str(), + depthCompressionFormat.c_str(), + _dbDriver?_dbDriver->getDatabaseVersion().c_str():"", + Parameters::kDbTargetVersion().c_str(), + codec.c_str()); + warned = true; + } + depthCompressionFormat = codec; + } + if(reuseCompressedDepth && + compressedDepthFormat(data.depthOrRightCompressed()).find(':') != std::string::npos) + { + // Already compressed as inverse depth (e.g., received from ROS's + // compressed_depth_image_transport), re-compress it. + reuseCompressedDepth = false; + if(depthOrRightImage.empty()) + { + depthOrRightImage = uncompressImage(data.depthOrRightCompressed()); + } + } + } + if(!depthOrRightImage.empty() && depthOrRightImage.type() == CV_32FC1) { if(_saveDepth16Format) @@ -6640,7 +6699,7 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor } depthOrRightImage = util2d::cvtDepthFromFloat(depthOrRightImage); } - else if(_depthCompressionFormat == ".rvl") + else if(depthCompressionFormat == ".rvl") { static bool warned = false; if(!warned) @@ -6650,13 +6709,16 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor "images will be compressed in \".png\" format instead. Explicitly " "set %s to true to keep using \"%s\" format and images will be " "converted to 16bits for convenience (warning: that would " - "remove all depth values over 65 meters). Explicitly set %s=\".png\" " + "remove all depth values over 65 meters). Set %s=\".rvl::\" " + "(e.g., \".rvl:10:100\") to compress them in RVL as 16 bits inverse depth " + "(lossy, see parameter's description). Explicitly set %s=\".png\" " "to suppress this warning. This warning is only printed once.", Parameters::kMemSaveDepth16Format().c_str(), Parameters::kMemDepthCompressionFormat().c_str(), - _depthCompressionFormat.c_str(), + depthCompressionFormat.c_str(), Parameters::kMemSaveDepth16Format().c_str(), - _depthCompressionFormat.c_str(), + depthCompressionFormat.c_str(), + Parameters::kMemDepthCompressionFormat().c_str(), Parameters::kMemDepthCompressionFormat().c_str()); warned = true; } @@ -6666,9 +6728,8 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor bool reuseCompressedImage = image.data == data.imageRaw().data && !data.imageCompressed().empty(); - bool reuseCompressedDepth = - depthOrRightImage.data == data.depthOrRightRaw().data && - !data.depthOrRightCompressed().empty(); + reuseCompressedDepth = reuseCompressedDepth && + depthOrRightImage.data == data.depthOrRightRaw().data; bool reuseCompressedDepthConfidence = depthConfidence.data == data.depthConfidenceRaw().data && !data.depthConfidenceCompressed().empty(); @@ -6685,7 +6746,7 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor if(_compressionParallelized) { rtabmap::CompressionThread ctImage(image, _rgbCompressionFormat); - rtabmap::CompressionThread ctDepth(depthOrRightImage, depthOrRightImage.type() == CV_32FC1 || depthOrRightImage.type() == CV_16UC1?_depthCompressionFormat:_rgbCompressionFormat); + rtabmap::CompressionThread ctDepth(depthOrRightImage, depthOrRightImage.type() == CV_32FC1 || depthOrRightImage.type() == CV_16UC1?depthCompressionFormat:_rgbCompressionFormat); rtabmap::CompressionThread ctDepthConfidence(depthConfidence); rtabmap::CompressionThread ctLaserScan(laserScan.data()); rtabmap::CompressionThread ctUserData(data.userDataRaw()); @@ -6724,7 +6785,7 @@ Signature * Memory::createSignature(const SensorData & inputData, const Transfor else { compressedImage = reuseCompressedImage?cv::Mat():compressImage2(image, _rgbCompressionFormat); - compressedDepth = reuseCompressedDepth?cv::Mat():compressImage2(depthOrRightImage, depthOrRightImage.type() == CV_32FC1 || depthOrRightImage.type() == CV_16UC1?_depthCompressionFormat:_rgbCompressionFormat); + compressedDepth = reuseCompressedDepth?cv::Mat():compressImage2(depthOrRightImage, depthOrRightImage.type() == CV_32FC1 || depthOrRightImage.type() == CV_16UC1?depthCompressionFormat:_rgbCompressionFormat); compressedDepthConfidence = reuseCompressedDepthConfidence?cv::Mat():compressData2(depthConfidence); compressedScan = reuseCompressedScan?data.laserScanCompressed().data():compressData2(laserScan.data()); compressedUserData = reuseCompressedUserData?data.userDataCompressed():compressData2(data.userDataRaw()); diff --git a/corelib/src/SensorData.cpp b/corelib/src/SensorData.cpp index d9b30ead..c6fbd7dc 100644 --- a/corelib/src/SensorData.cpp +++ b/corelib/src/SensorData.cpp @@ -313,6 +313,24 @@ SensorData::~SensorData() { } +bool SensorData::keepCameraModel( + const CameraModel & model, + const cv::Mat & rgb, + const cv::Mat & depth, + bool clearPreviousData) const +{ + // An invalid model without any image is only a placeholder (e.g., scan-only data + // created with CameraModel()): it is not kept, so that cameraModels() is empty when + // there is no camera. An invalid model with an image is kept: images can be used + // without calibration, and they are split per camera model. + return model.isValidForProjection() || + !rgb.empty() || + !depth.empty() || + (!clearPreviousData && ( + !_imageRaw.empty() || !_imageCompressed.empty() || + !_depthOrRightRaw.empty() || !_depthOrRightCompressed.empty())); +} + void SensorData::setRGBDImage( const cv::Mat & rgb, const cv::Mat & depth, @@ -320,7 +338,10 @@ void SensorData::setRGBDImage( bool clearPreviousData) { std::vector models; - models.push_back(model); + if(keepCameraModel(model, rgb, depth, clearPreviousData)) + { + models.push_back(model); + } setRGBDImage(rgb, depth, models, clearPreviousData); } void SensorData::setRGBDImage( @@ -331,7 +352,10 @@ void SensorData::setRGBDImage( bool clearPreviousData) { std::vector models; - models.push_back(model); + if(keepCameraModel(model, rgb, depth, clearPreviousData)) + { + models.push_back(model); + } setRGBDImage(rgb, depth, depthConfidence, models, clearPreviousData); } void SensorData::setRGBDImage( diff --git a/corelib/src/VisualWord.cpp b/corelib/src/VisualWord.cpp index b3013831..fb82a47e 100644 --- a/corelib/src/VisualWord.cpp +++ b/corelib/src/VisualWord.cpp @@ -73,7 +73,7 @@ unsigned long VisualWord::getMemoryUsed() const { unsigned long memoryUsage = sizeof(VisualWord); memoryUsage += _references.size() * (sizeof(int)*2+sizeof(std::map::iterator)) + sizeof(std::map); - memoryUsage += _descriptor.total() * _descriptor.elemSize(); + memoryUsage += _descriptor.empty()?0:_descriptor.total() * _descriptor.elemSize(); return memoryUsage; } diff --git a/corelib/test/CMakeLists.txt b/corelib/test/CMakeLists.txt index 25b6b114..139e63b8 100644 --- a/corelib/test/CMakeLists.txt +++ b/corelib/test/CMakeLists.txt @@ -162,6 +162,20 @@ IF(BUILD_PERF_TESTS) set_tests_properties(test_graph_perf PROPERTIES TIMEOUT ${_perf_timeout} LABELS "performance") + + # Comparison of the depth image compression approaches (sizes, times, errors) for + # 16UC1 and 32FC1 depth images: PNG, RVL, zlib, the legacy 4-channel PNG of 32FC1 + # images, their conversion to 16UC1 millimeters, and their quantization as 16 bits + # inverse depth (Mem/DepthCompressionFormat=".png:max:q" or ".rvl:max:q"): + # bin/test_compression_perf + # bin/test_compression_perf --gtest_filter=*Synthetic* + add_executable(test_compression_perf perf_compression.cpp) + target_link_libraries(test_compression_perf gtest_main rtabmap_core) + + add_test(NAME test_compression_perf COMMAND test_compression_perf) + set_tests_properties(test_compression_perf PROPERTIES + TIMEOUT ${_perf_timeout} + LABELS "performance") ENDIF(BUILD_PERF_TESTS) # Rtabmap end-to-end replay of sample DBs (test data fetched by diff --git a/corelib/test/perf_compression.cpp b/corelib/test/perf_compression.cpp new file mode 100644 index 00000000..28bb6d39 --- /dev/null +++ b/corelib/test/perf_compression.cpp @@ -0,0 +1,342 @@ +// Comparison of the depth image compression approaches of Compression.h, for each +// depth type rtabmap receives: +// +// 16UC1 (millimeters): +// - ".png" lossless, 16 bits grayscale PNG +// - ".rvl" lossless, RVL (Mem/DepthCompressionFormat default) +// - zlib lossless, compressData2(), as a reference +// 32FC1 (meters): +// - ".png" lossless, float bytes as a 4-channel 8 bits PNG (legacy) +// - zlib lossless, compressData2(), as a reference +// - 16UC1 mm + ".png/.rvl" lossy, util2d::cvtDepthFromFloat() then 16 bits codec, +// what Mem/SaveDepth16Format=true does +// - ".png:max:q/.rvl:max:q" lossy, 16 bits quantized inverse depth (same +// quantization than ROS's compressed_depth_image_transport) +// +// over the depth images of data/rgbd/depth (a structured light camera, millimeters), +// the same images converted to meters in 32FC1 (as many drivers publish them), and a +// synthetic 32FC1 image with continuous values, like stereo or lidar projected depth. +// +// Its own executable, run by ctest under the "performance" label, so that its seconds +// of benchmarking stay out of the unit test shards: +// ctest -L performance to run them +// ctest -LE performance to skip them +// bin/test_compression_perf --gtest_filter=*Synthetic* +// +// The times are reported rather than asserted on, as they depend on the machine. What +// is asserted is that the lossless approaches give back the same image, and that the +// lossy ones stay within their error bounds for the depth range they keep. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace rtabmap; + +namespace { + +static const int ITERATIONS = 15; + +struct Approach +{ + std::string name; + std::function(const cv::Mat &)> encode; + std::function &)> decode; + bool lossless; + float maxDepth; // meters, lossy approaches only: depth kept under it + float minDepth; // meters, lossy approaches only: depth kept over it + std::function tolerance; // meters, lossy approaches only, for a depth in meters +}; + +struct Result +{ + size_t bytes = 0; + double encodeMs = 0.0; + double decodeMs = 0.0; + double maxError = 0.0; // mm, over the depth range kept + double rmse = 0.0; // mm, over the depth range kept + double lost = 0.0; // % of the valid pixels set to 0 + int outOfTolerance = 0; // pixels with an error over the tolerance +}; + +double median(std::vector v) +{ + std::sort(v.begin(), v.end()); + return v[v.size()/2]; +} + +float toMeters(const cv::Mat & depth, int r, int c) +{ + return depth.type() == CV_16UC1 ? float(depth.at(r, c)) * 0.001f : depth.at(r, c); +} + +Result run(const cv::Mat & depth, const Approach & approach) +{ + Result result; + std::vector bytes; + cv::Mat restored; + std::vector encodeTimes, decodeTimes; + for(int i=0; i 0.0f)) + { + continue; + } + ++valid; + const float out = toMeters(restored, r, c); + if(out == 0.0f) + { + ++lost; + // Only allowed outside the kept range + if(d >= approach.minDepth && d < approach.maxDepth) + { + ++result.outOfTolerance; + } + continue; + } + const double err = std::fabs(out - d); + result.maxError = std::max(result.maxError, err*1000.0); + sumSq += err*err*1e6; + ++kept; + if(err > approach.tolerance(d)) + { + ++result.outOfTolerance; + } + } + } + result.rmse = kept ? std::sqrt(sumSq / kept) : 0.0; + result.lost = valid ? 100.0 * lost / valid : 0.0; + EXPECT_EQ(result.outOfTolerance, 0) << approach.name; + return result; +} + +void report(const std::string & title, const cv::Mat & depth, const std::vector & approaches) +{ + const size_t raw = depth.total() * depth.elemSize(); + std::printf("\n%s: %dx%d %s, %zu bytes raw\n", title.c_str(), depth.cols, depth.rows, + depth.type() == CV_16UC1 ? "16UC1" : "32FC1", raw); + std::printf(" %-22s %10s %7s %10s %10s %11s %10s %8s\n", + "approach", "bytes", "ratio", "encode ms", "decode ms", "max err mm", "rmse mm", "lost %"); + for(const Approach & approach : approaches) + { + SCOPED_TRACE(title + " " + approach.name); + const Result r = run(depth, approach); + if(approach.lossless) + { + std::printf(" %-22s %10zu %6.1fx %10.2f %10.2f %11s %10s %8s\n", + approach.name.c_str(), r.bytes, double(raw)/double(r.bytes), r.encodeMs, r.decodeMs, + "lossless", "-", "-"); + } + else + { + std::printf(" %-22s %10zu %6.1fx %10.2f %10.2f %11.3f %10.3f %8.2f\n", + approach.name.c_str(), r.bytes, double(raw)/double(r.bytes), r.encodeMs, r.decodeMs, + r.maxError, r.rmse, r.lost); + } + } + std::fflush(stdout); +} + +std::vector encode(const cv::Mat & depth, const std::string & format) +{ + return compressImage(depth, format); +} + +cv::Mat decode(const std::vector & bytes) +{ + return uncompressImage(bytes); +} + +std::vector encodeZlib(const cv::Mat & depth) +{ + return compressData(depth); +} + +cv::Mat decodeZlib(const std::vector & bytes) +{ + return uncompressData(bytes); +} + +std::vector approaches16U() +{ + using namespace std::placeholders; + return { + {".png", std::bind(encode, _1, ".png"), decode, true, 0, 0, nullptr}, + {".rvl", std::bind(encode, _1, ".rvl"), decode, true, 0, 0, nullptr}, + {"zlib", encodeZlib, decodeZlib, true, 0, 0, nullptr}}; +} + +Approach invDepth(const std::string & codec, float maxDepth, float quantization) +{ + using namespace std::placeholders; + const float A = quantization * (quantization + 1.0f); + const float B = 1.0f - A / maxDepth; + const std::string format = uFormat("%s:%g:%g", codec.c_str(), maxDepth, quantization); + return {format, std::bind(encode, _1, format), decode, false, + maxDepth, + A / (65535.0f - B) * 1.001f, + [A](float d) { return 0.51f * d * d / A + 1e-6f; }}; +} + +Approach depth16(const std::string & codec) +{ + return {"16UC1 mm + " + codec, + [codec](const cv::Mat & depth) { return compressImage(util2d::cvtDepthFromFloat(depth), codec); }, + [](const std::vector & bytes) { return util2d::cvtDepthToFloat(uncompressImage(bytes)); }, + false, + 65.535f, + 0.0f, + [](float) { return 0.001f + 1e-6f; }}; // truncated to millimeters +} + +std::vector approaches32F() +{ + using namespace std::placeholders; + return { + {".png (legacy RGBA)", std::bind(encode, _1, ".png"), decode, true, 0, 0, nullptr}, + {"zlib", encodeZlib, decodeZlib, true, 0, 0, nullptr}, + depth16(".png"), + depth16(".rvl"), + invDepth(".png", 10.0f, 100.0f), + invDepth(".rvl", 10.0f, 100.0f), + invDepth(".png", 40.0f, 100.0f), + invDepth(".rvl", 40.0f, 100.0f), + invDepth(".rvl", 40.0f, 200.0f)}; +} + +std::vector loadSampleDepths() +{ + std::vector depths; + for(const std::string & name : {"17.png", "154.png"}) + { + const std::string path = std::string(RTABMAP_TEST_DATA_ROOT) + "/rgbd/depth/" + name; + cv::Mat depth = cv::imread(path, cv::IMREAD_UNCHANGED); + if(depth.type() == CV_16UC1) + { + depths.push_back(depth); + } + else + { + std::printf("Cannot load 16UC1 depth image \"%s\", skipped.\n", path.c_str()); + } + } + return depths; +} + +// Ground plane, walls and boxes seen by a 640x480 camera, with continuous +// values up to ~35 m, noise growing with depth (as stereo) and holes. +cv::Mat makeSyntheticDepth(int cols = 640, int rows = 480) +{ + cv::RNG rng(42); + const float fx = 0.75f * cols, cx = cols / 2.0f, cy = rows / 2.0f; + const float cameraHeight = 1.0f; + cv::Mat depth(rows, cols, CV_32FC1); + for(int v=0; v 0.0f) + { + d = std::min(d, cameraHeight / y); // ground + } + if(x < 0.0f) + { + d = std::min(d, 3.0f / -x); // left wall, 3 m away + } + // boxes + if(x > 0.05f && x < 0.25f && y > -0.1f && y < cameraHeight / 2.5f) + { + d = std::min(d, 2.5f - 1.5f * x); + } + if(x > -0.35f && x < -0.15f && y > -0.2f && y < cameraHeight / 12.0f) + { + d = std::min(d, 12.0f); + } + d += (float)rng.gaussian(0.002 * d * d); // stereo-like noise + depth.at(v, u) = d; + } + } + // Holes + for(int i=0; i<40; ++i) + { + const int u = rng.uniform(0, cols - 20), v = rng.uniform(0, rows - 20); + depth(cv::Rect(u, v, rng.uniform(2, 20), rng.uniform(2, 20))).setTo(0.0f); + } + return depth; +} + +} // namespace + +TEST(CompressionPerf, SampleDepth16UC1) +{ + const std::vector depths = loadSampleDepths(); + if(depths.empty()) + { + GTEST_SKIP() << "No sample depth images in " << RTABMAP_TEST_DATA_ROOT << "/rgbd/depth"; + } + for(size_t i=0; i depths = loadSampleDepths(); + if(depths.empty()) + { + GTEST_SKIP() << "No sample depth images in " << RTABMAP_TEST_DATA_ROOT << "/rgbd/depth"; + } + for(size_t i=0; i #include +#include #include +#include +#include using namespace rtabmap; @@ -183,3 +186,228 @@ TEST(CompressionTest, CompressionThreadDataRoundTrip) expectMatEqual(uncompressThread.getUncompressedData(), data); } + +namespace { + +// 32FC1 depth image covering [minDepth, maxDepth[ with sub-millimeter values, +// and the invalid values of the inverse depth format on the first row. +cv::Mat makeFloatDepth(int rows, int cols, float minDepth, float maxDepth) +{ + cv::Mat depth(rows, cols, CV_32FC1); + for(int r = 0; r < rows; ++r) + { + for(int c = 0; c < cols; ++c) + { + depth.at(r, c) = minDepth + (maxDepth - minDepth) * float(r * cols + c) / float(rows * cols); + } + } + return depth; +} + +// Error bound of the inverse depth format: half a quantization step. +float invDepthTolerance(float d, float quantization) +{ + // (with some margin for the float rounding of A/d + B, up to ~66000) + return 0.51f * d * d / (quantization * (quantization + 1.0f)) + 1e-6f; +} + +} // namespace + +TEST(CompressionTest, ParseImageCompressionFormat) +{ + std::string codec; + float maxDepth, quantization; + + EXPECT_TRUE(parseImageCompressionFormat("", codec, maxDepth, quantization)); + EXPECT_TRUE(codec.empty()); + EXPECT_EQ(maxDepth, 0.0f); + + EXPECT_TRUE(parseImageCompressionFormat(".jpg", codec, maxDepth, quantization)); + EXPECT_EQ(codec, ".jpg"); + EXPECT_EQ(maxDepth, 0.0f); + EXPECT_EQ(quantization, 0.0f); + + EXPECT_TRUE(parseImageCompressionFormat(".rvl", codec, maxDepth, quantization)); + EXPECT_EQ(codec, ".rvl"); + EXPECT_EQ(maxDepth, 0.0f); + + EXPECT_TRUE(parseImageCompressionFormat(".png:20", codec, maxDepth, quantization)); + EXPECT_EQ(codec, ".png"); + EXPECT_FLOAT_EQ(maxDepth, 20.0f); + EXPECT_FLOAT_EQ(quantization, 100.0f); + + EXPECT_TRUE(parseImageCompressionFormat(".rvl:10.5:50", codec, maxDepth, quantization)); + EXPECT_EQ(codec, ".rvl"); + EXPECT_FLOAT_EQ(maxDepth, 10.5f); + EXPECT_FLOAT_EQ(quantization, 50.0f); + + EXPECT_FALSE(parseImageCompressionFormat("png", codec, maxDepth, quantization)); + EXPECT_FALSE(parseImageCompressionFormat(".jpg:10:100", codec, maxDepth, quantization)); + EXPECT_FALSE(parseImageCompressionFormat(".png:abc", codec, maxDepth, quantization)); + EXPECT_FALSE(parseImageCompressionFormat(".png:0:100", codec, maxDepth, quantization)); + EXPECT_FALSE(parseImageCompressionFormat(".png:-10:100", codec, maxDepth, quantization)); + EXPECT_FALSE(parseImageCompressionFormat(".png:10:0", codec, maxDepth, quantization)); + EXPECT_FALSE(parseImageCompressionFormat(".png:10:100:1", codec, maxDepth, quantization)); +} + +TEST(CompressionTest, InvalidFormatReturnsEmpty) +{ + const cv::Mat depth = makeFloatDepth(4, 4, 1.0f, 2.0f); + EXPECT_TRUE(compressImage(depth, ".jpg:10").empty()); + EXPECT_TRUE(compressImage(depth, ".png:x").empty()); +} + +TEST(CompressionTest, InverseDepthRoundTrip) +{ + const float maxDepth = 10.0f; + const float quantization = 100.0f; + const float minDepth = quantization * (quantization + 1.0f) / (65535.0f + quantization * (quantization + 1.0f) / maxDepth); + cv::Mat depth = makeFloatDepth(48, 64, minDepth * 1.001f, maxDepth * 0.999f); + const float invalid[] = { + 0.0f, -1.0f, maxDepth, maxDepth * 2.0f, minDepth * 0.9f, + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity()}; + const int nInvalid = sizeof(invalid) / sizeof(float); + for(int i = 0; i < nInvalid; ++i) + { + depth.at(0, i) = invalid[i]; + } + + for(const std::string codec : {".png", ".rvl"}) + { + SCOPED_TRACE(codec); + const std::string format = codec + ":10:100"; + const std::vector bytes = compressImage(depth, format); + ASSERT_FALSE(bytes.empty()); + EXPECT_LT(bytes.size(), depth.total() * depth.elemSize() / 2); + EXPECT_EQ(compressedDepthFormat(bytes), format); + + const cv::Mat restored = uncompressImage(bytes); + ASSERT_EQ(restored.type(), CV_32FC1); + ASSERT_EQ(restored.size(), depth.size()); + for(int r = 0; r < depth.rows; ++r) + { + for(int c = 0; c < depth.cols; ++c) + { + const float d = depth.at(r, c); + if(r == 0 && c < nInvalid) + { + EXPECT_EQ(restored.at(r, c), 0.0f) << "input=" << d; + } + else + { + ASSERT_NEAR(restored.at(r, c), d, invDepthTolerance(d, quantization)) << "r=" << r << " c=" << c; + } + } + } + + // Re-compressing with the detected format gives back the same bytes + // (e.g., DatabaseViewer saving an edited depth image). + EXPECT_EQ(compressImage(restored, compressedDepthFormat(bytes)), compressImage(restored, format)); + + // Same through cv::Mat and thread overloads + CompressionThread compressThread(depth, format); + compressThread.start(); + compressThread.join(); + const cv::Mat bytesMat = compressThread.getCompressedData(); + ASSERT_EQ(bytesMat.total(), bytes.size()); + EXPECT_EQ(memcmp(bytesMat.data, bytes.data(), bytes.size()), 0); + CompressionThread uncompressThread(bytesMat, true); + uncompressThread.start(); + uncompressThread.join(); + expectMatEqual(uncompressThread.getUncompressedData(), restored); + } +} + +TEST(CompressionTest, InverseDepthQuantizationParameters) +{ + const cv::Mat depth = makeFloatDepth(32, 32, 1.0f, 39.0f); + const std::vector bytes = compressImage(depth, ".png:40:50"); + EXPECT_EQ(compressedDepthFormat(bytes), ".png:40:50"); + const cv::Mat restored = uncompressImage(bytes); + ASSERT_EQ(restored.type(), CV_32FC1); + for(int r = 0; r < depth.rows; ++r) + { + for(int c = 0; c < depth.cols; ++c) + { + const float d = depth.at(r, c); + ASSERT_NEAR(restored.at(r, c), d, invDepthTolerance(d, 50.0f)); + } + } +} + +TEST(CompressionTest, InverseDepthNonContinuousImage) +{ + const cv::Mat depth = makeFloatDepth(20, 30, 1.0f, 5.0f); + const cv::Mat roi = depth(cv::Rect(3, 2, 10, 8)); + ASSERT_FALSE(roi.isContinuous()); + const cv::Mat restored = uncompressImage(compressImage(roi, ".rvl:10:100")); + ASSERT_EQ(restored.size(), roi.size()); + for(int r = 0; r < roi.rows; ++r) + { + for(int c = 0; c < roi.cols; ++c) + { + const float d = roi.at(r, c); + ASSERT_NEAR(restored.at(r, c), d, invDepthTolerance(d, 100.0f)); + } + } +} + +TEST(CompressionTest, DepthParametersIgnoredFor16UC1) +{ + cv::Mat depth(24, 32, CV_16UC1); + cv::randu(depth, 0, 20000); // includes values over the max depth below + for(const std::string codec : {".png", ".rvl"}) + { + SCOPED_TRACE(codec); + const std::vector bytes = compressImage(depth, codec + ":10:100"); + EXPECT_EQ(bytes, compressImage(depth, codec)); + EXPECT_EQ(compressedDepthFormat(bytes), codec); + expectMatEqual(uncompressImage(bytes), depth); + } +} + +TEST(CompressionTest, LegacyFloatDepthIsLossless) +{ + const cv::Mat depth = makeFloatDepth(16, 16, 0.01f, 100.0f); + for(const std::string format : {".png", ".rvl"}) + { + SCOPED_TRACE(format); + const std::vector bytes = compressImage(depth, format); + EXPECT_EQ(compressedDepthFormat(bytes), ".png"); + const cv::Mat restored = uncompressImage(bytes); + ASSERT_EQ(restored.type(), CV_32FC1); + EXPECT_EQ(memcmp(restored.data, depth.data, depth.total() * depth.elemSize()), 0); + } +} + +TEST(CompressionTest, MalformedDepthFormatsDecodeToEmpty) +{ + // Signature and header only, no payload + std::vector invDepth = {'D', 'E', 'P', 'T', 'H', 'I', 'N', 'V'}; + invDepth.resize(16, 0); + EXPECT_TRUE(uncompressImage(invDepth).empty()); + EXPECT_EQ(compressedDepthFormat(invDepth), ".png") << "too short to be inverse depth"; + + // Inverse depth header followed by an 8 bits image instead of a 16 bits one + const std::vector png8 = compressImage(cv::Mat(4, 4, CV_8UC1, cv::Scalar(1)), ".png"); + invDepth.insert(invDepth.end(), png8.begin(), png8.end()); + EXPECT_TRUE(uncompressImage(invDepth).empty()); + + // RVL signature without its size + const std::vector rvl = {'D', 'E', 'P', 'T', 'H', 'R', 'V', 'L', 4, 0}; + EXPECT_TRUE(uncompressImage(rvl).empty()); + EXPECT_EQ(compressedDepthFormat(rvl), ".rvl"); + + EXPECT_TRUE(uncompressImage(nullptr, 0).empty()); +} + +TEST(CompressionTest, CompressionThreadRejectsInvalidFormat) +{ + // std::string: a string literal would select the (bytes, isImage) constructor + const cv::Mat depth(4, 4, CV_32FC1, cv::Scalar(1.0f)); + EXPECT_THROW(CompressionThread(depth, std::string(".jpg:10")), UException); + EXPECT_THROW(CompressionThread(depth, std::string(".bmp")), UException); + EXPECT_NO_THROW(CompressionThread(depth, std::string(".rvl:10:100"))); +} diff --git a/corelib/test/test_memory.cpp b/corelib/test/test_memory.cpp index a08b1d15..bda4a611 100644 --- a/corelib/test/test_memory.cpp +++ b/corelib/test/test_memory.cpp @@ -4681,3 +4681,128 @@ TEST(MemoryTest, CreateSignatureRecompressesStereoPairAfterRectification) EXPECT_GT(cv::countNonZero(uncompressImage(stored.depthOrRightCompressed()) != right), 0) << "stored right image still holds the unrectified pixels"; } + +// --------------------------------------------------------------------------- +// Mem/DepthCompressionFormat with inverse depth (".rvl:max:q"), which databases +// older than 0.24 cannot hold: rtabmap 0.23 would still open them (e.g., created +// with Db/TargetVersion=0.23.0) but could not decode their depth images. +// --------------------------------------------------------------------------- + +namespace { + +enum DepthInput +{ + kRawDepth, + kCompressedDepthWithRaw, // e.g., received from ROS and decoded + kCompressedDepthOnly // raw depth not needed (no features extracted here) +}; + +struct InverseDepthCase +{ + const char * targetVersion; + DepthInput input; + const char * depthCompressionFormat; + bool parallelCompression; + const char * expectedFormat; +}; + +class MemoryInverseDepthTest : public ::testing::TestWithParam {}; + +} // namespace + +TEST_P(MemoryInverseDepthTest, StoredDepthFormatFollowsDatabaseVersion) +{ + const InverseDepthCase & cs = GetParam(); + ParametersMap params = defaultMemoryParams(); + params[Parameters::kMemBinDataKept()] = "true"; + params[Parameters::kMemDepthCompressionFormat()] = cs.depthCompressionFormat; + params[Parameters::kMemCompressionParallelized()] = cs.parallelCompression ? "true" : "false"; + params[Parameters::kDbTargetVersion()] = cs.targetVersion; + Memory memory(params); + const std::string dbPath = uniqueDbPath(); + ASSERT_TRUE(memory.init(dbPath, true, params)); + + const cv::Mat rgb(16, 16, CV_8UC3, cv::Scalar(10, 20, 30)); + cv::Mat depth(16, 16, CV_32FC1); + cv::randu(depth, 0.5f, 8.0f); + const CameraModel model(10.0, 10.0, 8.0, 8.0, CameraModel::opticalRotation()); + SensorData data; + if(cs.input == kRawDepth) + { + data = SensorData(rgb, depth, model); + } + else + { + data = SensorData(compressImage2(rgb, ".png"), compressImage2(depth, ".png:10:100"), model); + if(cs.input == kCompressedDepthWithRaw) + { + data.uncompressData(); + ASSERT_FALSE(data.depthRaw().empty()); + } + } + + ASSERT_TRUE(memory.update(data, Transform(0, 0, 0, 0, 0, 0), cv::Mat::eye(6, 6, CV_64FC1) * 0.01)); + const Signature * s = memory.getSignature(memory.getLastSignatureId()); + ASSERT_NE(s, nullptr); + const cv::Mat & stored = s->sensorData().depthOrRightCompressed(); + ASSERT_FALSE(stored.empty()); + EXPECT_EQ(compressedDepthFormat(stored), cs.expectedFormat); + + const cv::Mat restored = uncompressImage(stored); + ASSERT_EQ(restored.type(), CV_32FC1); + ASSERT_EQ(restored.size(), depth.size()); + EXPECT_LT(cv::norm(restored, depth, cv::NORM_INF), 0.01); + + memory.close(false); + UFile::erase(dbPath); +} + +INSTANTIATE_TEST_SUITE_P( + DatabaseVersions, + MemoryInverseDepthTest, + ::testing::Values( + InverseDepthCase{"", kRawDepth, ".rvl:10:100", true, ".rvl:10:100"}, + InverseDepthCase{"", kRawDepth, ".rvl:10:100", false, ".rvl:10:100"}, + InverseDepthCase{"", kCompressedDepthWithRaw, ".rvl:10:100", true, ".png:10:100"}, // reused as is + InverseDepthCase{"", kCompressedDepthOnly, ".rvl:10:100", true, ".png:10:100"}, // reused as is + InverseDepthCase{"0.23.0", kRawDepth, ".rvl:10:100", true, ".png"}, // legacy 32FC1 format + InverseDepthCase{"0.23.0", kCompressedDepthWithRaw, ".rvl:10:100", true, ".png"}, // re-compressed + InverseDepthCase{"0.23.0", kCompressedDepthOnly, ".rvl:10:100", true, ".png"}, // decompressed, re-compressed + InverseDepthCase{"", kRawDepth, ".rvl", true, ".png"}, // RVL is 16UC1 only: legacy + InverseDepthCase{"", kRawDepth, ".jpg", true, ".png"})); // invalid: default ".rvl" + +// Compressed images that Memory rectifies (Rtabmap/ImagesAlreadyRectified=false) are +// decoded for it, even when nothing else needs them (no feature extraction here): they +// are stored rectified, not as received. +TEST(MemoryTest, DecodesCompressedImagesToRectifyThem) +{ + for(bool alreadyRectified : {true, false}) + { + SCOPED_TRACE(alreadyRectified ? "already rectified" : "rectified by Memory"); + ParametersMap params = defaultMemoryParams(); + params[Parameters::kMemBinDataKept()] = "true"; + params[Parameters::kRtabmapImagesAlreadyRectified()] = alreadyRectified ? "true" : "false"; + Memory memory(params); + ASSERT_TRUE(memory.init("")); + + cv::Mat rgb(48, 64, CV_8UC3); + cv::randu(rgb, 0, 255); + const cv::Mat K = (cv::Mat_(3, 3) << 50, 0, 32, 0, 50, 24, 0, 0, 1); + const cv::Mat D = (cv::Mat_(1, 5) << -0.3, 0.1, 0, 0, 0); + const cv::Mat R = cv::Mat::eye(3, 3, CV_64FC1); + const cv::Mat P = (cv::Mat_(3, 4) << 50, 0, 32, 0, 0, 50, 24, 0, 0, 0, 1, 0); + const CameraModel model("cam", cv::Size(64, 48), K, D, R, P, CameraModel::opticalRotation()); + ASSERT_TRUE(model.isValidForRectification()); + const cv::Mat compressed = compressImage2(rgb, ".png"); + SensorData data(compressed, cv::Mat(), model); + + ASSERT_TRUE(memory.update(data, Transform(0, 0, 0, 0, 0, 0), cv::Mat::eye(6, 6, CV_64FC1) * 0.01)); + const Signature * s = memory.getSignature(memory.getLastSignatureId()); + ASSERT_NE(s, nullptr); + const cv::Mat & stored = s->sensorData().imageCompressed(); + ASSERT_FALSE(stored.empty()); + const bool sameBytes = stored.total() == compressed.total() && + memcmp(stored.data, compressed.data, compressed.total()) == 0; + EXPECT_EQ(sameBytes, alreadyRectified); + } +} diff --git a/corelib/test/test_sensordata.cpp b/corelib/test/test_sensordata.cpp index 2532f200..54e54d74 100644 --- a/corelib/test/test_sensordata.cpp +++ b/corelib/test/test_sensordata.cpp @@ -199,7 +199,7 @@ TEST(SensorDataTest, IsValidWithCameraModel) { SensorData data; data.setRGBDImage(cv::Mat(), cv::Mat(), CameraModel()); - EXPECT_FALSE(data.cameraModels().empty()); + EXPECT_TRUE(data.cameraModels().empty()); // invalid without image: placeholder not kept EXPECT_FALSE(data.isValid()); // not valid for projection data.setRGBDImage(cv::Mat(), cv::Mat(), CameraModel(525.0, 525.0, 320.0, 240.0)); @@ -986,3 +986,32 @@ TEST(SensorDataTest, DifferentDepthTypes) EXPECT_EQ(data.depthOrRightRaw().type(), CV_32FC1); } + +// An invalid CameraModel without any image is a placeholder (e.g., lidar odometry +// creating scan-only data with CameraModel()): it is not kept. With an image, it is kept, +// as images can be used without calibration. +TEST(SensorDataTest, InvalidCameraModelIsKeptOnlyWithImages) +{ + const LaserScan scan(cv::Mat(1, 3, CV_32FC2, cv::Scalar(1.0f, 0.0f)), 0, 10.0f, LaserScan::kXY); + const SensorData scanOnly(scan, cv::Mat(), cv::Mat(), CameraModel(), 1, 1.0); + EXPECT_TRUE(scanOnly.cameraModels().empty()); + EXPECT_TRUE(scanOnly.isValid()); + + const cv::Mat image(4, 6, CV_8UC1, cv::Scalar(1)); + const SensorData uncalibrated(image, CameraModel(), 1, 1.0); + EXPECT_EQ(uncalibrated.cameraModels().size(), 1u); + + const SensorData compressedOnly(compressImage2(image, ".png"), CameraModel(), 1, 1.0); + EXPECT_EQ(compressedOnly.cameraModels().size(), 1u); + + const CameraModel valid(10.0, 10.0, 3.0, 2.0); + const SensorData calibratedNoImage(scan, cv::Mat(), cv::Mat(), valid, 1, 1.0); + EXPECT_EQ(calibratedNoImage.cameraModels().size(), 1u) << "valid models are always kept"; + + // Keeping the images already there: they still need their model + SensorData data(image, CameraModel(), 1, 1.0); + data.setRGBDImage(cv::Mat(), cv::Mat(), CameraModel(), false); + EXPECT_EQ(data.cameraModels().size(), 1u); + data.setRGBDImage(cv::Mat(), cv::Mat(), CameraModel(), true); + EXPECT_TRUE(data.cameraModels().empty()) << "images cleared, nothing left to describe"; +} diff --git a/guilib/src/ui/preferencesDialog.ui b/guilib/src/ui/preferencesDialog.ui index afbf107c..d45752b9 100644 --- a/guilib/src/ui/preferencesDialog.ui +++ b/guilib/src/ui/preferencesDialog.ui @@ -12391,7 +12391,7 @@ see Sqlite3 doc 'PRAGMA synchronous'. - Depth image compression format (should be ".png" or ".rvl"). + Depth image compression format (should be ".png" or ".rvl"). Add ":maxDepth[:quantization]" (e.g., ".rvl:10:100") to compress 32FC1 depth images as 16 bits inverse depth (lossy: precision ~d²/(2q(q+1)), depth over maxDepth is lost, databases cannot be opened by versions < 0.24). 16UC1 depth images are always compressed losslessly. true diff --git a/package.xml b/package.xml index e11b6c20..fc53b1f8 100644 --- a/package.xml +++ b/package.xml @@ -1,7 +1,7 @@ rtabmap - 0.23.13 + 0.24.0 RTAB-Map's standalone library. RTAB-Map is a RGB-D SLAM approach with real-time constraints. Mathieu Labbe Mathieu Labbe