1.更新orbbecSDK 1.9.1;2.gemini2增加深度滤波配置功能;3.增加IMU同时输出topic.

orbbec_camera/SDK/include/libobsensor/h/Device.h

@@ -97,7 +97,7 @@ const char *ob_device_list_get_device_ip_address(ob_device_list *list, uint32_t
 /**
  * @brief Get the device extension information.
  *
- * @param[in] info Device Information
+ * @param[in] list Device list object
  * @param[in] index Device index
  * @param[out] error Log error messages
  * @return const char* The device extension information
@@ -865,6 +865,30 @@ void ob_delete_depth_work_mode_list(ob_depth_work_mode_list *work_mode_list, ob_
  */
 void ob_delete_data_bundle(ob_data_bundle *data_bundle, ob_error **error);
 
+/**
+ * @brief Check if the device supports global timestamp.
+ *
+ * @param[in] device The device object.
+ * @param[out] error Log error messages.
+ * @return bool Whether the device supports global timestamp.
+ */
+bool ob_device_is_global_timestamp_supported(ob_device *device, ob_error **error);
+
+/**
+ * @brief Load depth filter config from file.
+ * @param[in] device The device object.
+ * @param[in] file_path Path of the config file.
+ * @param[out] error Log error messages.
+ */
+void ob_device_load_depth_filter_config(ob_device *device, const char *file_path, ob_error **error);
+
+/**
+ * @brief Reset depth filter config to device default define.
+ * @param[in] device The device object.
+ * @param[out] error Log error messages.
+ */
+void ob_device_reset_default_depth_filter_config(ob_device *device, ob_error **error);
+
 #ifdef __cplusplus
 }
 #endif

orbbec_camera/SDK/include/libobsensor/h/Frame.h

@@ -68,6 +68,20 @@ uint64_t ob_frame_time_stamp_us(ob_frame *frame, ob_error **error);
  */
 uint64_t ob_frame_system_time_stamp(ob_frame *frame, ob_error **error);
 
+/**
+ * @brief Get the global timestamp of the frame in microseconds.
+ * @brief The global timestamp is the time point when the frame was was captured by the device, and has been converted to the host clock domain. The
+ * conversion process base on the device timestamp and can eliminate the timer drift of the device
+ *
+ * @attention Only some devices support getting the global timestamp. If the device does not support it, this function will return 0. Check the device support
+ * status by @ref ob_device_is_global_timestamp_supported() function.
+ *
+ * @param[in] frame Frame object
+ * @param[out] error Log error messages
+ * @return uint64_t The global timestamp of the frame in microseconds.
+ */
+uint64_t ob_frame_global_time_stamp_us(ob_frame *frame, ob_error **error);
+
 /**
  * @brief Get frame data
  *

orbbec_camera/SDK/include/libobsensor/h/ObTypes.h

@@ -161,6 +161,7 @@ typedef enum {
     OB_SENSOR_IR_LEFT   = 6, /**< left IR */
     OB_SENSOR_IR_RIGHT  = 7, /**< Right IR */
     OB_SENSOR_RAW_PHASE = 8, /**< Raw Phase */
+    OB_SENSOR_COUNT,
 } OBSensorType,
     ob_sensor_type;
 
@@ -367,7 +368,7 @@ typedef struct {
 typedef struct {
     float rot[9];    ///< Rotation matrix
     float trans[3];  ///< Transformation matrix
-} OBD2CTransform, ob_d2c_transform;
+} OBD2CTransform, ob_d2c_transform, OBTransform, ob_transform;
 
 /**
  * @brief Structure for camera parameters
@@ -380,6 +381,7 @@ typedef struct {
     OBD2CTransform     transform;        ///< Rotation/transformation matrix
     bool               isMirrored;       ///< Whether the image frame corresponding to this group of parameters is mirrored
 } OBCameraParam, ob_camera_param;
+
 /**
  * @brief Camera parameters
  */
@@ -392,6 +394,15 @@ typedef struct {
     OBD2CTransform     transform;      ///< Rotation/transformation matrix
 } OBCameraParam_V0, ob_camera_param_v0;
 
+/**
+ * @brief calibration parameters
+ */
+typedef struct {
+    OBCameraIntrinsic intrinsics[OB_SENSOR_COUNT];             ///< Sensor internal parameters
+    OBTransform extrinsics[OB_SENSOR_COUNT][OB_SENSOR_COUNT];  ///< The extrinsic parameters allow 3D coordinate conversions between sensor.To transform from a
+                                                               ///< source to a target 3D coordinate system,under extrinsics[source][target].
+} OBCalibrationParam, ob_calibration_param;
+
 /**
  * @brief Configuration for depth margin filter
  */
@@ -405,6 +416,21 @@ typedef struct {
     bool     enable_direction;  ///< Set to true for horizontal and vertical, false for horizontal only
 } ob_margin_filter_config, OBMarginFilterConfig;
 
+/**
+ * @brief Configuration for mgc filter
+*/
+typedef struct{
+    uint32_t width;
+    uint32_t height;
+    int max_width_left;
+    int max_width_right;
+    int max_radius;
+    int margin_x_th;
+    int margin_y_th;
+    int limit_x_th;
+    int limit_y_th;
+}OBMGCFilterConfig,ob_mgc_filter_config;
+
 /**
  * @brief Alignment mode
  */
@@ -441,6 +467,7 @@ typedef enum {
     FORMAT_MJPG_TO_BGRA,       /**< MJPG to BGRA */
     FORMAT_UYVY_TO_RGB888,     /**< UYVY to RGB888 */
     FORMAT_BGR_TO_RGB,         /**< BGR to RGB */
+    FORMAT_MJPG_TO_NV12,       /**< MJPG to NV12 */
 } OBConvertFormat,
     ob_convert_format;
 
@@ -617,7 +644,15 @@ typedef struct {
     float x;  ///< X coordinate
     float y;  ///< Y coordinate
     float z;  ///< Z coordinate
-} OBPoint, ob_point;
+} OBPoint, ob_point, OBPoint3f, ob_point3f;
+
+/**
+ * @brief 2D point structure in the SDK
+ */
+typedef struct {
+    float x;  ///< X coordinate
+    float y;  ///< Y coordinate
+} OBPoint2f, ob_point2f;
 
 /**
  * @brief 3D point structure with color information

orbbec_camera/SDK/include/libobsensor/h/Pipeline.h

@@ -169,7 +169,6 @@ ob_camera_param ob_pipeline_get_camera_param_with_profile(ob_pipeline *pipeline,
                                                           uint32_t depthHeight, ob_error **error);
 
 /**
- * \if English
  * @brief Get current camera parameters
  * @attention If D2C is enabled, it will return the camera parameters after D2C, if not, it will return to the default parameters
  *
@@ -179,6 +178,15 @@ ob_camera_param ob_pipeline_get_camera_param_with_profile(ob_pipeline *pipeline,
  */
 ob_camera_param ob_pipeline_get_camera_param(ob_pipeline *pipeline, ob_error **error);
 
+/**
+ * @brief Get device calibration parameters
+ *
+ * @param[in] pipeline pipeline object
+ * @param[out] error Log error messages
+ * @return ob_calibration_param The calibration parameters
+ */
+ob_calibration_param ob_pipeline_get_calibration_param(ob_pipeline *pipeline, ob_config *config, ob_error **error);
+
 /**
  * @brief Return a list of D2C-enabled depth sensor resolutions corresponding to the input color sensor resolution
  *
@@ -314,7 +322,7 @@ void ob_config_set_d2c_target_resolution(ob_config *config, uint32_t d2c_target_
  * can be caused by different frame rates of each stream, or by the loss of frames of one stream): drop directly or output to the user.
  *
  * @param[in] config The pipeline configuration
- * @param[in] mode The frame aggregation output mode to be set (default mode is @ref OB_FRAME_AGGREGATE_OUTPUT_ANY_SITUATION)
+ * @param[in] mode The frame aggregation output mode to be set (default mode is @ref OB_FRAME_AGGREGATE_OUTPUT_FULL_FRAME_REQUIRE)
  * @param[out] error Log error messages
  */
 void ob_config_set_frame_aggregate_output_mode(ob_config *config, ob_frame_aggregate_output_mode mode, ob_error **error);

orbbec_camera/SDK/include/libobsensor/h/Utils.h

@@ -0,0 +1,87 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "ObTypes.h"
+
+/**
+ * @brief Transform a 3d point of a source coordinate system into a 3d point of the target coordinate system.
+ *
+ * @param[in] calibration_param Device calibration param,see pipeline::getCalibrationParam
+ * @param[in] source_point3f Source 3d point value
+ * @param[in] source_sensor_Type Source sensor type
+ * @param[in] target_sensor_type Target sensor type
+ * @param[out] target_point3f Target 3d point value
+ * @param[out] error Log error messages
+ *
+ * @return bool Transform result
+ */
+bool ob_calibration_3d_to_3d(const ob_calibration_param calibration_param, const ob_point3f source_point3f, const ob_sensor_type source_sensor_Type,
+                             const ob_sensor_type target_sensor_type, ob_point3f *target_point3f, ob_error **error);
+
+/**
+ * @brief Transform a 2d pixel coordinate with an associated depth value of the source camera into a 3d point of the target coordinate system.
+ *
+ * @param[in] calibration_param Device calibration param,see pipeline::getCalibrationParam
+ * @param[in] source_point2f Source 2d point value
+ * @param[in] source_depth_pixel_value The depth of sourcePoint2f in millimeters
+ * @param[in] source_sensor_type Source sensor type
+ * @param[in] target_sensor_Type Target sensor type
+ * @param[out] target_point3f Target 3d point value
+ * @param[out] error Log error messages
+ *
+ * @return bool Transform result
+ */
+bool ob_calibration_2d_to_3d(const ob_calibration_param calibration_param, const ob_point2f source_point2f, const float source_depth_pixel_value,
+                             const ob_sensor_type source_sensor_type, const ob_sensor_type target_sensor_Type, ob_point3f *target_point3f, ob_error **error);
+
+/**
+ * @brief Transform a 3d point of a source coordinate system into a 2d pixel coordinate of the target camera.
+ *
+ * @param[in] calibration_param Device calibration param,see pipeline::getCalibrationParam
+ * @param[in] source_point3f Source 3d point value
+ * @param[in] source_sensor_type Source sensor type
+ * @param[in] target_sensor_type Target sensor type
+ * @param[out] target_point2f Target 2d point value
+ * @param[out] error Log error messages
+ *
+ * @return bool Transform result
+ */
+bool ob_calibration_3d_to_2d(const ob_calibration_param calibration_param, const ob_point3f source_point3f, const ob_sensor_type source_sensor_type,
+                             const ob_sensor_type target_sensor_type, ob_point2f *target_point2f, ob_error **error);
+
+/**
+ * @brief Transform a 2d pixel coordinate with an associated depth value of the source camera into a 2d pixel coordinate of the target camera
+ *
+ * @param[in] calibration_param Device calibration param,see pipeline::getCalibrationParam
+ * @param[in] source_point2f Source 2d point value
+ * @param[in] source_depth_pixel_value The depth of sourcePoint2f in millimeters
+ * @param[in] source_sensor_type Source sensor type
+ * @param[in] target_sensor_type Target sensor type
+ * @param[out] target_point2f Target 2d point value
+ * @param[out] error Log error messages
+ *
+ * @return bool Transform result
+ */
+bool ob_calibration_2d_to_2d(const ob_calibration_param calibration_param, const ob_point2f source_point2f, const float source_depth_pixel_value,
+                             const ob_sensor_type source_sensor_type, const ob_sensor_type target_sensor_type, ob_point2f *target_point2f, ob_error **error);
+
+/**
+ * @brief Transforms the depth frame into the geometry of the color camera.
+ *
+ * @param[in] device Device handle
+ * @param[in] depth_frame Input depth frame
+ * @param[in] target_color_camera_width Target color camera width
+ * @param[in] target_color_camera_height Target color camera height
+ * @param[out] error Log error messages
+ *
+ * @return ob_frame* Transformed depth frame
+ */
+ob_frame *transformation_depth_frame_to_color_camera(ob_device *device, ob_frame *depth_frame, uint32_t target_color_camera_width,
+                                                     uint32_t target_color_camera_height, ob_error **error);
+
+#ifdef __cplusplus
+}
+#endif

orbbec_camera/SDK/include/libobsensor/hpp/Device.hpp

@@ -26,9 +26,11 @@ class CameraParamList;
 class OBDepthWorkModeList;
 
 class OB_EXTENSION_API Device {
-private:
+protected:
     std::unique_ptr<DeviceImpl> impl_;
 
+    Device(Device &&device);
+
 public:
     /**
      * @brief Describe the entity of the RGBD camera, representing a specific model of RGBD camera
@@ -299,6 +301,13 @@ public:
      */
     bool isPropertySupported(OBPropertyID propertyId, OBPermissionType permission);
 
+    /**
+     * @brief Check if the global timestamp is supported for the device
+     *
+     * @return Whether the global timestamp is supported
+     */
+    bool isGlobalTimestampSupported();
+
     /**
      * @brief Upgrade the device firmware
      *
@@ -530,8 +539,20 @@ public:
      */
     void timerSyncWithHost();
 
+    /**
+     * @brief Load depth filter config from file.
+     * @param filePath Path of the config file.
+     */
+    void loadDepthFilterConfig(const char *filePath);
+
+    /**
+     * @brief Reset depth filter config to device default define.
+     */
+    void resetDefaultDepthFilterConfig();
+
     friend class Pipeline;
     friend class Recorder;
+    friend class CoordinateTransformHelper;
 };
 
 /**

orbbec_camera/SDK/include/libobsensor/hpp/Frame.hpp

@@ -108,6 +108,18 @@ public:
      */
     uint64_t systemTimeStamp();
 
+    /**
+     * @brief Get the global timestamp of the frame in microseconds.
+     * @brief The global timestamp is the time point when the frame was was captured by the device, and has been converted to the host clock domain. The
+     * conversion process base on the device timestamp and can eliminate the timer drift of the device
+     *
+     * @attention Only some devices support getting the global timestamp. If the device does not support it, this function will return 0. Check the device
+     * support status by @ref Device::isGlobalTimestampSupported() function.
+     *
+     * @return uint64_t The global timestamp of the frame in microseconds.
+     */
+    uint64_t globalTimeStampUs();
+
     /**
      * @brief Check if the runtime type of the frame object is compatible with a given type.
      *
@@ -134,6 +146,7 @@ private:
     friend class Filter;
     friend class Recorder;
     friend class FrameHelper;
+    friend class CoordinateTransformHelper;
 };
 
 class OB_EXTENSION_API VideoFrame : public Frame {

orbbec_camera/SDK/include/libobsensor/hpp/Pipeline.hpp

@@ -145,6 +145,15 @@ public:
      */
     OBCameraParam getCameraParamWithProfile(uint32_t colorWidth, uint32_t colorHeight, uint32_t depthWidth, uint32_t depthHeight);
 
+    /**
+     * @brief Get the calibration parameters
+     *
+     * @param config The configured parameters
+     *
+     * @return OBCalibrationParam The calibration parameters
+     */
+    OBCalibrationParam getCalibrationParam(std::shared_ptr<Config> config);
+
     /**
      * @brief Return a list of D2C-enabled depth sensor resolutions corresponding to the input color sensor resolution
      *
@@ -253,6 +262,15 @@ public:
      */
     void setD2CTargetResolution(uint32_t d2cTargetWidth, uint32_t d2cTargetHeight);
 
+    /**
+     * @brief Set the frame aggregation output mode for the pipeline configuration
+     * @brief The processing strategy when the FrameSet generated by the frame aggregation function does not contain the frames of all opened streams (which
+     * can be caused by different frame rates of each stream, or by the loss of frames of one stream): drop directly or output to the user.
+     *
+     * @param mode The frame aggregation output mode to be set (default mode is @ref OB_FRAME_AGGREGATE_OUTPUT_FULL_FRAME_REQUIRE)
+     */
+    void setFrameAggregateOutputMode(OBFrameAggregateOutputMode mode);
+
     friend class Pipeline;
 };
 

orbbec_camera/SDK/include/libobsensor/hpp/Utils.hpp

@@ -0,0 +1,86 @@
+/**
+ * @file Utils.hpp
+ * @brief The SDK utils class
+ *
+ */
+#pragma once
+
+#include "Types.hpp"
+
+namespace ob {
+class Device;
+
+class OB_EXTENSION_API CoordinateTransformHelper {
+public:
+    /**
+     * @brief Transform a 3d point of a source coordinate system into a 3d point of the target coordinate system.
+     *
+     * @param calibrationParam Device calibration param,see pipeline::getCalibrationParam
+     * @param sourcePoint3f Source 3d point value
+     * @param sourceSensorType Source sensor type
+     * @param targetSensorType Target sensor type
+     * @param targetPoint3f Target 3d point value
+     *
+     * @return bool  Transform result
+     */
+    static bool calibration3dTo3d(const OBCalibrationParam calibrationParam, const OBPoint3f sourcePoint3f, const OBSensorType sourceSensorType,
+                                  const OBSensorType targetSensorType, OBPoint3f *targetPoint3f);
+
+    /**
+     * @brief Transform a 2d pixel coordinate with an associated depth value of the source camera into a 3d point of the target coordinate system.
+     *
+     * @param calibrationParam Device calibration param,see pipeline::getCalibrationParam
+     * @param sourcePoint2f Source 2d point value
+     * @param sourceDepthPixelValue The depth of sourcePoint2f in millimeters
+     * @param sourceSensorType Source sensor type
+     * @param targetSensorType Target sensor type
+     * @param targetPoint3f Target 3d point value
+     *
+     * @return bool Transform result
+     */
+    static bool calibration2dTo3d(const OBCalibrationParam calibrationParam, const OBPoint2f sourcePoint2f, const float sourceDepthPixelValue,
+                                  const OBSensorType sourceSensorType, const OBSensorType targetSensorType, OBPoint3f *targetPoint3f);
+
+    /**
+     * @brief Transform a 3d point of a source coordinate system into a 2d pixel coordinate of the target camera.
+     *
+     * @param calibrationParam Device calibration param,see pipeline::getCalibrationParam
+     * @param sourcePoint3f Source 3d point value
+     * @param sourceSensorType Source sensor type
+     * @param targetSensorType Target sensor type
+     * @param targetPoint2f Target 2d point value
+     *
+     * @return bool Transform result
+     */
+    static bool calibration3dTo2d(const OBCalibrationParam calibrationParam, const OBPoint3f sourcePoint3f, const OBSensorType sourceSensorType,
+                                  const OBSensorType targetSensorType, OBPoint2f *targetPoint2f);
+
+    /**
+     * @brief Transform a 2d pixel coordinate with an associated depth value of the source camera into a 2d pixel coordinate of the target camera
+     *
+     * @param calibrationParam Device calibration param,see pipeline::getCalibrationParam
+     * @param sourcePoint2f Source 2d point value
+     * @param sourceDepthPixelValue The depth of sourcePoint2f in millimeters
+     * @param sourceSensorType Source sensor type
+     * @param targetSensorType Target sensor type
+     * @param targetPoint2f Target 2d point value
+     *
+     * @return bool Transform result
+     */
+    static bool calibration2dTo2d(const OBCalibrationParam calibrationParam, const OBPoint2f sourcePoint2f, const float sourceDepthPixelValue,
+                                  const OBSensorType sourceSensorType, const OBSensorType targetSensorType, OBPoint2f *targetPoint2f);
+
+    /**
+     * @brief Transforms the depth frame into the geometry of the color camera.
+     *
+     * @param device Device handle
+     * @param depthFrame Input depth frame
+     * @param targetColorCameraWidth Target color camera width
+     * @param targetColorCameraHeight Target color camera height
+     *
+     * @return std::shared_ptr<ob::Frame> Transformed depth frame
+     */
+    static std::shared_ptr<ob::Frame> transformationDepthFrameToColorCamera(std::shared_ptr<ob::Device> device, std::shared_ptr<ob::Frame> depthFrame,
+                                                                            uint32_t targetColorCameraWidth, uint32_t targetColorCameraHeight);
+};
+}  // namespace ob

orbbec_camera/SDK/lib/arm32/libOrbbecSDK.so

@@ -1 +1 @@
-libOrbbecSDK.so.1.8
+libOrbbecSDK.so.1.9

orbbec_camera/SDK/lib/arm32/libOrbbecSDK.so.1.8

@@ -1 +0,0 @@
-libOrbbecSDK.so.1.8.3

orbbec_camera/SDK/lib/arm32/libOrbbecSDK.so.1.9

@@ -0,0 +1 @@
+libOrbbecSDK.so.1.9.1

orbbec_camera/SDK/lib/arm32/libudev.so

@@ -1 +0,0 @@
-libudev.so.1.6.3

orbbec_camera/SDK/lib/arm32/libudev.so.1

@@ -1 +0,0 @@
-libudev.so.1.6.3

orbbec_camera/SDK/lib/arm64/libOrbbecSDK.so

@@ -1 +1 @@
-libOrbbecSDK.so.1.8
+libOrbbecSDK.so.1.9

orbbec_camera/SDK/lib/arm64/libOrbbecSDK.so.1.8

@@ -1 +0,0 @@
-libOrbbecSDK.so.1.8.3

orbbec_camera/SDK/lib/arm64/libOrbbecSDK.so.1.9

@@ -0,0 +1 @@
+libOrbbecSDK.so.1.9.1

orbbec_camera/SDK/lib/arm64/libudev.so

@@ -1 +0,0 @@
-libudev.so.1.6.3

orbbec_camera/SDK/lib/arm64/libudev.so.1

@@ -1 +0,0 @@
-libudev.so.1.6.3

orbbec_camera/SDK/lib/x64/libOrbbecSDK.so

@@ -1 +1 @@
-libOrbbecSDK.so.1.8
+libOrbbecSDK.so.1.9

orbbec_camera/SDK/lib/x64/libOrbbecSDK.so.1.8

@@ -1 +0,0 @@
-libOrbbecSDK.so.1.8.3

orbbec_camera/SDK/lib/x64/libOrbbecSDK.so.1.9

@@ -0,0 +1 @@
+libOrbbecSDK.so.1.9.1