GlobalDescriptor (PyDescriptor / netvlad) (#1163)

* First commit for pydescriptor

* Fixed Python refactoring errors

* GUI: added PyDescriptor parameters

* reordered python3 includes

* updated rtabmap_netvlad.py test main

* fixed build from last merge

* integrated https://github.com/introlab/rtabmap/pull/1255

* rescaled dot product result

Closing https://github.com/introlab/rtabmap/issues/1105

corelib/include/rtabmap/core/GlobalDescriptor.h

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2010-2020, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
+Copyright (c) 2010-2024, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

corelib/include/rtabmap/core/GlobalDescriptorExtractor.h

@@ -0,0 +1,73 @@
+/*
+Copyright (c) 2010-2024, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the Universite de Sherbrooke nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef GLOBAL_DESCRIPTOR_EXTRACTOR_H_
+#define GLOBAL_DESCRIPTOR_EXTRACTOR_H_
+
+#include "rtabmap/core/rtabmap_core_export.h" // DLL export/import defines
+
+#include "rtabmap/core/Parameters.h"
+#include "rtabmap/core/SensorData.h"
+
+
+namespace rtabmap {
+
+// Feature2D
+class RTABMAP_CORE_EXPORT GlobalDescriptorExtractor {
+public:
+	enum Type {
+		kUndef=0,
+		kPyDescriptor=1};
+
+	static std::string typeName(Type type)
+	{
+		switch(type){
+		case kPyDescriptor:
+			return "PyDescriptor";
+		default:
+			return "Unknown";
+		}
+	}
+
+	static GlobalDescriptorExtractor * create(const ParametersMap & parameters = ParametersMap());
+	static GlobalDescriptorExtractor * create(GlobalDescriptorExtractor::Type type, const ParametersMap & parameters = ParametersMap()); // for convenience
+
+public:
+	virtual ~GlobalDescriptorExtractor();
+
+	virtual GlobalDescriptor extract(const SensorData & data) const = 0;
+
+	virtual void parseParameters(const ParametersMap & parameters) {}
+	virtual GlobalDescriptorExtractor::Type getType() const = 0;
+
+protected:
+	GlobalDescriptorExtractor(const ParametersMap & parameters = ParametersMap());
+};
+
+}
+
+#endif /* GLOBAL_DESCRIPTOR_EXTRACTOR_H_ */

corelib/include/rtabmap/core/Memory.h

@@ -59,6 +59,7 @@ class RegistrationVis;
 class Stereo;
 class LocalGridMaker;
 class MarkerDetector;
+class GlobalDescriptorExtractor;
 
 class RTABMAP_CORE_EXPORT Memory
 {
@@ -375,6 +376,8 @@ private:
 	LocalGridMaker * _localMapMaker;
 
 	MarkerDetector * _markerDetector;
+
+	GlobalDescriptorExtractor * _globalDescriptorExtractor;
 };
 
 } // namespace rtabmap

corelib/include/rtabmap/core/Parameters.h

@@ -231,6 +231,7 @@ class RTABMAP_CORE_EXPORT Parameters
     RTABMAP_PARAM(Mem, UseOdomFeatures,             bool, true,     "Use odometry features instead of regenerating them.");
     RTABMAP_PARAM(Mem, UseOdomGravity,              bool, false,    uFormat("Use odometry instead of IMU orientation to add gravity links to new nodes created. We assume that odometry is already aligned with gravity (e.g., we are using a VIO approach). Gravity constraints are used by graph optimization only if \"%s\" is not zero.", kOptimizerGravitySigma().c_str()));
     RTABMAP_PARAM(Mem, CovOffDiagIgnored,           bool, true,     "Ignore off diagonal values of the covariance matrix.");
+    RTABMAP_PARAM(Mem, GlobalDescriptorStrategy,    int, 0,        "Extract global descriptor from sensor data. 0=disabled, 1=PyDescriptor");
     RTABMAP_PARAM(Mem, RotateImagesUpsideUp,        bool, false,    "Rotate images so that upside is up if they are not already. This can be useful in case the robots don't have all same camera orientation but are using the same map, so that not rotation-invariant visual features can still be used across the fleet.");
 
     // KeypointMemory (Keypoint-based)
@@ -725,6 +726,10 @@ class RTABMAP_CORE_EXPORT Parameters
 	RTABMAP_PARAM(GMS, WithScale,            bool, false,   "Take scale transformation into account.");
 	RTABMAP_PARAM(GMS, ThresholdFactor,      double, 6.0,   "The higher, the less matches.");
 
+	// Global descriptor approaches
+	RTABMAP_PARAM_STR(PyDescriptor, Path,    "",          "Path to python script file (see available ones in rtabmap/corelib/src/pydescriptor/*). See the header to see where the script should be used.");
+	RTABMAP_PARAM(PyDescriptor, Dim,         int, 4096,   "Descriptor dimension.");
+
     // ICP registration parameters
 #ifdef RTABMAP_POINTMATCHER
     RTABMAP_PARAM(Icp, Strategy,                  int, 1,       "ICP implementation: 0=Point Cloud Library, 1=libpointmatcher, 2=CCCoreLib (CloudCompare).");