/* * Copyright (C) 2010-2011, Mathieu Labbe and IntRoLab - Universite de Sherbrooke * * This file is part of RTAB-Map. * * RTAB-Map is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * RTAB-Map is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with RTAB-Map. If not, see . */ #include "rtabmap/core/CameraRGBD.h" #include "rtabmap/core/DBDriver.h" #include #include #include #include #include #include #include #include #include #include #ifdef WITH_FREENECT #include #ifdef FREENECT_DASH_INCLUDES #include #else #include #endif #endif #ifdef WITH_OPENNI2 #include #endif namespace rtabmap { CameraRGBD::CameraRGBD(float imageRate, const Transform & localTransform, float fx, float fy, float cx, float cy) : _imageRate(imageRate), _localTransform(localTransform), _frameRateTimer(new UTimer()), _fx(fx), _fy(fy), _cx(cx), _cy(cy) { } CameraRGBD::~CameraRGBD() { if(_frameRateTimer) { delete _frameRateTimer; } } void CameraRGBD::takeImage(cv::Mat & rgb, cv::Mat & depth, float & fx, float & fy, float & cx, float & cy) { float imageRate = _imageRate==0.0f?33.0f:_imageRate; // limit to 33Hz if infinity if(imageRate>0) { int sleepTime = (1000.0f/imageRate - 1000.0f*_frameRateTimer->getElapsedTime()); if(sleepTime > 2) { uSleep(sleepTime-2); } // Add precision at the cost of a small overhead while(_frameRateTimer->getElapsedTime() < 1.0/double(imageRate)-0.000001) { // } double slept = _frameRateTimer->getElapsedTime(); _frameRateTimer->start(); UDEBUG("slept=%fs vs target=%fs", slept, 1.0/double(imageRate)); } UTimer timer; this->captureImage(rgb, depth, fx, fy, cx, cy); if(_fx) { fx = _fx; // override if set } if(_fy) { fy = _fy; // override if set } if(_cx) { cx = _cx; // override if set } if(_cy) { cy = _cy; // override if set } UDEBUG("Time capturing image = %fs", timer.ticks()); } ///////////////////////// // CameraOpenNIPCL ///////////////////////// CameraOpenni::CameraOpenni(const std::string & deviceId, float imageRate, const Transform & localTransform, float fx, float fy, float cx, float cy) : CameraRGBD(imageRate, localTransform, fx, fy, cx, cy), interface_(0), deviceId_(deviceId), depthConstant_(0.0f) { } CameraOpenni::~CameraOpenni() { UDEBUG(""); if(connection_.connected()) { connection_.disconnect(); } if(interface_) { interface_->stop(); uSleep(1000); // make sure it is stopped delete interface_; interface_ = 0; } } void CameraOpenni::image_cb ( const boost::shared_ptr& rgb, const boost::shared_ptr& depth, float constant) { UScopeMutex s(dataMutex_); bool notify = rgb_.empty(); cv::Mat rgbFrame(rgb->getHeight(), rgb->getWidth(), CV_8UC3); rgb->fillRGB(rgb->getWidth(), rgb->getHeight(), rgbFrame.data); cv::cvtColor(rgbFrame, rgb_, CV_RGB2BGR); depth_ = cv::Mat(rgb->getHeight(), rgb->getWidth(), CV_16UC1); depth->fillDepthImageRaw(rgb->getWidth(), rgb->getHeight(), (unsigned short*)depth_.data); depthConstant_ = constant; if(notify) { dataReady_.release(); } } bool CameraOpenni::init() { if(interface_) { interface_->stop(); uSleep(100); // make sure it is stopped delete interface_; interface_ = 0; } try { interface_ = new pcl::OpenNIGrabber(deviceId_); boost::function&, const boost::shared_ptr&, float)> f = boost::bind (&CameraOpenni::image_cb, this, _1, _2, _3); connection_ = interface_->registerCallback (f); interface_->start (); } catch(const pcl::IOException& ex) { UERROR("OpenNI exception: %s", ex.what()); if(interface_) { delete interface_; interface_ = 0; } return false; } return true; } void CameraOpenni::captureImage(cv::Mat & rgb, cv::Mat & depth, float & fx, float & fy, float & cx, float & cy) { if(interface_ && interface_->isRunning()) { dataReady_.acquire(); UScopeMutex s(dataMutex_); if(depthConstant_) { depth = depth_; rgb = rgb_; fx = 1.0f/depthConstant_; fy = 1.0f/depthConstant_; cx = float(depth_.cols/2) - 0.5f; cy = float(depth_.rows/2) - 0.5f; } depth_ = cv::Mat(); rgb_ = cv::Mat(); depthConstant_ = 0.0f; } } ///////////////////////// // CameraOpenNICV ///////////////////////// bool CameraOpenNICV::available() { return cv::getBuildInformation().find("OpenNI: YES") != std::string::npos; } CameraOpenNICV::CameraOpenNICV(bool asus, float imageRate, const rtabmap::Transform & localTransform, float fx, float fy, float cx, float cy) : CameraRGBD(imageRate, localTransform, fx, fy, cx, cy), _asus(asus), _depthFocal(0.0f) { } CameraOpenNICV::~CameraOpenNICV() { _capture.release(); } bool CameraOpenNICV::init() { if(_capture.isOpened()) { _capture.release(); } ULOGGER_DEBUG("CameraRGBD::init()"); _capture.open( _asus?CV_CAP_OPENNI_ASUS:CV_CAP_OPENNI ); if(_capture.isOpened()) { _capture.set( CV_CAP_OPENNI_IMAGE_GENERATOR_OUTPUT_MODE, CV_CAP_OPENNI_VGA_30HZ ); _depthFocal = _capture.get( CV_CAP_OPENNI_DEPTH_GENERATOR_FOCAL_LENGTH ); // Print some avalible device settings. UINFO("Depth generator output mode:"); UINFO("FRAME_WIDTH %f", _capture.get( CV_CAP_PROP_FRAME_WIDTH )); UINFO("FRAME_HEIGHT %f", _capture.get( CV_CAP_PROP_FRAME_HEIGHT )); UINFO("FRAME_MAX_DEPTH %f mm", _capture.get( CV_CAP_PROP_OPENNI_FRAME_MAX_DEPTH )); UINFO("BASELINE %f mm", _capture.get( CV_CAP_PROP_OPENNI_BASELINE )); UINFO("FPS %f", _capture.get( CV_CAP_PROP_FPS )); UINFO("Focal %f", _capture.get( CV_CAP_OPENNI_DEPTH_GENERATOR_FOCAL_LENGTH )); UINFO("REGISTRATION %f", _capture.get( CV_CAP_PROP_OPENNI_REGISTRATION )); if(_capture.get( CV_CAP_PROP_OPENNI_REGISTRATION ) == 0.0) { UERROR("Depth registration is not activated on this device!"); } if( _capture.get( CV_CAP_OPENNI_IMAGE_GENERATOR_PRESENT ) ) { UINFO("Image generator output mode:"); UINFO("FRAME_WIDTH %f", _capture.get( CV_CAP_OPENNI_IMAGE_GENERATOR+CV_CAP_PROP_FRAME_WIDTH )); UINFO("FRAME_HEIGHT %f", _capture.get( CV_CAP_OPENNI_IMAGE_GENERATOR+CV_CAP_PROP_FRAME_HEIGHT )); UINFO("FPS %f", _capture.get( CV_CAP_OPENNI_IMAGE_GENERATOR+CV_CAP_PROP_FPS )); } else { UERROR("CameraRGBD: Device doesn't contain image generator."); _capture.release(); return false; } } else { ULOGGER_ERROR("CameraRGBD: Failed to create a capture object!"); _capture.release(); return false; } return true; } void CameraOpenNICV::captureImage(cv::Mat & rgb, cv::Mat & depth, float & fx, float & fy, float & cx, float & cy) { if(_capture.isOpened()) { _capture.grab(); _capture.retrieve( depth, CV_CAP_OPENNI_DEPTH_MAP ); _capture.retrieve( rgb, CV_CAP_OPENNI_BGR_IMAGE ); depth = depth.clone(); rgb = rgb.clone(); UASSERT(_depthFocal > 0.0f); fx = _depthFocal; fy = _depthFocal; cx = float(depth.cols/2) - 0.5f; cy = float(depth.rows/2) - 0.5f; } else { ULOGGER_WARN("The camera must be initialized before requesting an image."); } } ///////////////////////// // CameraOpenNI2 ///////////////////////// bool CameraOpenNI2::available() { #ifdef WITH_OPENNI2 return true; #else return false; #endif } CameraOpenNI2::CameraOpenNI2(float imageRate, const rtabmap::Transform & localTransform, float fx, float fy, float cx, float cy) : CameraRGBD(imageRate, localTransform, fx, fy, cx, cy), #ifdef WITH_OPENNI2 _device(new openni::Device()), _color(new openni::VideoStream()), _depth(new openni::VideoStream()), #else _device(0), _color(0), _depth(0), #endif _depthFx(0.0f), _depthFy(0.0f) { } CameraOpenNI2::~CameraOpenNI2() { #ifdef WITH_OPENNI2 _color->stop(); _color->destroy(); _depth->stop(); _depth->destroy(); _device->close(); openni::OpenNI::shutdown(); delete _device; delete _color; delete _depth; #endif } bool CameraOpenNI2::init() { #ifdef WITH_OPENNI2 openni::OpenNI::initialize(); if(_device->open(openni::ANY_DEVICE) != openni::STATUS_OK) { UERROR("CameraOpenNI2: Cannot open device."); _device->close(); openni::OpenNI::shutdown(); return false; } if(!_device->isImageRegistrationModeSupported(openni::IMAGE_REGISTRATION_DEPTH_TO_COLOR)) { UERROR("CameraOpenNI2: Device doesn't support depth/color registration."); _device->close(); openni::OpenNI::shutdown(); return false; } if(_device->getSensorInfo(openni::SENSOR_DEPTH) == NULL || _device->getSensorInfo(openni::SENSOR_COLOR) == NULL) { UERROR("CameraOpenNI2: Cannot get sensor info for depth and color."); _device->close(); openni::OpenNI::shutdown(); return false; } if(_depth->create(*_device, openni::SENSOR_DEPTH) != openni::STATUS_OK) { UERROR("CameraOpenNI2: Cannot create depth stream."); _device->close(); openni::OpenNI::shutdown(); return false; } if(_color->create(*_device, openni::SENSOR_COLOR) != openni::STATUS_OK) { UERROR("CameraOpenNI2: Cannot create color stream."); _depth->destroy(); _device->close(); openni::OpenNI::shutdown(); return false; } if(_device->setImageRegistrationMode(openni::IMAGE_REGISTRATION_DEPTH_TO_COLOR ) != openni::STATUS_OK) { UERROR("CameraOpenNI2: Failed to set depth/color registration."); } _depth->setMirroringEnabled(false); _color->setMirroringEnabled(false); const openni::Array& depthVideoModes = _depth->getSensorInfo().getSupportedVideoModes(); for(int i=0; i& colorVideoModes = _color->getSensorInfo().getSupportedVideoModes(); for(int i=0; isetVideoMode(mMode); openni::VideoMode mModeColor; mModeColor.setFps(30); mModeColor.setResolution(640,480); mModeColor.setPixelFormat(openni::PIXEL_FORMAT_RGB888); _color->setVideoMode(mModeColor); UINFO("CameraOpenNI2: Using depth video mode: fps=%d, pixel=%d, w=%d, h=%d, H-FOV=%f rad, V-FOV=%f rad", _depth->getVideoMode().getFps(), _depth->getVideoMode().getPixelFormat(), _depth->getVideoMode().getResolutionX(), _depth->getVideoMode().getResolutionY(), _depth->getHorizontalFieldOfView(), _depth->getVerticalFieldOfView()); bool registered = true; if(registered) { _depthFx = float(_color->getVideoMode().getResolutionX()/2) / std::tan(_color->getHorizontalFieldOfView()/2.0f); _depthFy = float(_color->getVideoMode().getResolutionY()/2) / std::tan(_color->getVerticalFieldOfView()/2.0f); } else { _depthFx = float(_depth->getVideoMode().getResolutionX()/2) / std::tan(_depth->getHorizontalFieldOfView()/2.0f); _depthFy = float(_depth->getVideoMode().getResolutionY()/2) / std::tan(_depth->getVerticalFieldOfView()/2.0f); } UINFO("depth fx=%f fy=%f", _depthFx, _depthFy); UINFO("CameraOpenNI2: Using color video mode: fps=%d, pixel=%d, w=%d, h=%d, H-FOV=%f rad, V-FOV=%f rad", _color->getVideoMode().getFps(), _color->getVideoMode().getPixelFormat(), _color->getVideoMode().getResolutionX(), _color->getVideoMode().getResolutionY(), _color->getHorizontalFieldOfView(), _color->getVerticalFieldOfView()); if(_depth->start() != openni::STATUS_OK || _color->start() != openni::STATUS_OK) { UERROR("CameraOpenNI2: Cannot start depth and/or color streams."); _depth->stop(); _color->stop(); _depth->destroy(); _color->destroy(); _device->close(); openni::OpenNI::shutdown(); return false; } uSleep(1000); // just to make sure the sensor is correctly initialized return true; #else UERROR("CameraOpenNI2: RTAB-Map is not built with OpenNI2 support!"); return false; #endif } void CameraOpenNI2::captureImage(cv::Mat & rgb, cv::Mat & depth, float & fx, float & fy, float & cx, float & cy) { #ifdef WITH_OPENNI2 if(_device->isValid() && _depth->isValid() && _color->isValid() && _device->getSensorInfo(openni::SENSOR_DEPTH) != NULL && _device->getSensorInfo(openni::SENSOR_COLOR) != NULL) { openni::VideoFrameRef depthFrame, colorFrame; _depth->readFrame(&depthFrame); _color->readFrame(&colorFrame); if(depthFrame.isValid() && colorFrame.isValid()) { int h=depthFrame.getHeight(); int w=depthFrame.getWidth(); depth = cv::Mat(h, w, CV_16U, (void*)depthFrame.getData()).clone(); h=colorFrame.getHeight(); w=colorFrame.getWidth(); cv::Mat tmp(h, w, CV_8UC3, (void *)colorFrame.getData()); cv::cvtColor(tmp, rgb, CV_RGB2BGR); } UASSERT(_depthFx != 0.0f && _depthFy != 0.0f); fx = _depthFx; fy = _depthFy; cx = float(depth.cols/2) - 0.5f; cy = float(depth.rows/2) - 0.5f; } else { ULOGGER_WARN("The camera must be initialized before requesting an image."); } #else UERROR("CameraOpenNI2: RTAB-Map is not built with OpenNI2 support!"); #endif } #ifdef WITH_FREENECT // // FreenectDevice // class FreenectDevice : public UThread { public: FreenectDevice(freenect_context * ctx, int index) : index_(index), ctx_(ctx), device_(0), depthFocal_(0.0f) { UASSERT(ctx_ != 0); } ~FreenectDevice() { this->join(true); if(device_ && freenect_close_device(device_) < 0){} //FN_WARNING("Device did not shutdown in a clean fashion"); } bool init() { if(freenect_open_device(ctx_, &device_, index_) < 0) { UERROR("FreenectDevice: Cannot open Kinect"); return false; } freenect_set_user(device_, this); freenect_set_video_mode(device_, freenect_find_video_mode(FREENECT_RESOLUTION_MEDIUM, FREENECT_VIDEO_RGB)); freenect_set_depth_mode(device_, freenect_find_depth_mode(FREENECT_RESOLUTION_MEDIUM, FREENECT_DEPTH_REGISTERED)); depthBuffer_ = cv::Mat(cv::Size(640,480),CV_16UC1); rgbBuffer_ = cv::Mat(cv::Size(640,480), CV_8UC3); freenect_set_depth_buffer(device_, depthBuffer_.data); freenect_set_video_buffer(device_, rgbBuffer_.data); freenect_set_depth_callback(device_, freenect_depth_callback); freenect_set_video_callback(device_, freenect_video_callback); bool registered = true; float rgb_focal_length_sxga = 1050.0f; float width_sxga = 1280.0f; float width = freenect_get_current_depth_mode(device_).width; float scale = width / width_sxga; if(registered) { depthFocal_ = rgb_focal_length_sxga * scale; } else { freenect_registration reg = freenect_copy_registration(device_); float depth_focal_length_sxga = reg.zero_plane_info.reference_distance / reg.zero_plane_info.reference_pixel_size; freenect_destroy_registration(®); depthFocal_ = depth_focal_length_sxga * scale; } UINFO("FreenectDevice: Depth focal = %f", depthFocal_); return true; } float getDepthFocal() const {return depthFocal_;} void getData(cv::Mat & rgb, cv::Mat & depth) { if(this->isRunning()) { dataReady_.acquire(); UScopeMutex s(dataMutex_); rgb = rgbLastFrame_; depth = depthLastFrame_; rgbLastFrame_ = cv::Mat(); depthLastFrame_= cv::Mat(); } } private: // Do not call directly even in child void VideoCallback(void* rgb) { UASSERT(rgbBuffer_.data == rgb); UScopeMutex s(dataMutex_); bool notify = rgbLastFrame_.empty(); cv::cvtColor(rgbBuffer_, rgbLastFrame_, CV_RGB2BGR); if(!depthLastFrame_.empty() && notify) { dataReady_.release(); } } // Do not call directly even in child void DepthCallback(void* depth) { UASSERT(depthBuffer_.data == depth); UScopeMutex s(dataMutex_); bool notify = depthLastFrame_.empty(); depthLastFrame_ = depthBuffer_.clone(); if(!rgbLastFrame_.empty() && notify) { dataReady_.release(); } } void startVideo() { if(device_ && freenect_start_video(device_) < 0) UERROR("Cannot start RGB callback"); } void stopVideo() { if(device_ && freenect_stop_video(device_) < 0) UERROR("Cannot stop RGB callback"); } void startDepth() { if(device_ && freenect_start_depth(device_) < 0) UERROR("Cannot start depth callback"); } void stopDepth() { if(device_ && freenect_stop_depth(device_) < 0) UERROR("Cannot stop depth callback"); } virtual void mainLoopBegin() { this->startDepth(); this->startVideo(); } virtual void mainLoop() { timeval t; t.tv_sec = 0; t.tv_usec = 10000; if(freenect_process_events_timeout(ctx_, &t) < 0) { UERROR("FreenectDevice: Cannot process freenect events"); this->kill(); } } virtual void mainLoopEnd() { this->stopDepth(); this->stopVideo(); dataReady_.release(); } static void freenect_depth_callback(freenect_device *dev, void *depth, uint32_t timestamp) { FreenectDevice* device = static_cast(freenect_get_user(dev)); device->DepthCallback(depth); } static void freenect_video_callback(freenect_device *dev, void *video, uint32_t timestamp) { FreenectDevice* device = static_cast(freenect_get_user(dev)); device->VideoCallback(video); } //noncopyable FreenectDevice( const FreenectDevice& ); const FreenectDevice& operator=( const FreenectDevice& ); private: int index_; freenect_context * ctx_; freenect_device * device_; cv::Mat depthBuffer_; cv::Mat rgbBuffer_; UMutex dataMutex_; cv::Mat depthLastFrame_; cv::Mat rgbLastFrame_; float depthFocal_; USemaphore dataReady_; }; #endif // // CameraFreenect // bool CameraFreenect::available() { #ifdef WITH_FREENECT return true; #else return false; #endif } CameraFreenect::CameraFreenect(int deviceId, float imageRate, const Transform & localTransform, float fx, float fy, float cx, float cy) : CameraRGBD(imageRate, localTransform, fx, fy, cx, cy), deviceId_(deviceId), ctx_(0), freenectDevice_(0) { #ifdef WITH_FREENECT if(freenect_init(&ctx_, NULL) < 0) UERROR("Cannot initialize freenect library"); // We claim both the motor and camera devices, since this class exposes both. // It does not support audio, so we do not claim it. freenect_select_subdevices(ctx_, static_cast(FREENECT_DEVICE_MOTOR | FREENECT_DEVICE_CAMERA)); #endif } CameraFreenect::~CameraFreenect() { #ifdef WITH_FREENECT if(freenectDevice_) { freenectDevice_->join(true); delete freenectDevice_; freenectDevice_ = 0; } if(ctx_) { if(freenect_shutdown(ctx_) < 0){} //FN_WARNING("Freenect did not shutdown in a clean fashion"); } #endif } bool CameraFreenect::init() { #ifdef WITH_FREENECT if(freenectDevice_) { freenectDevice_->join(true); delete freenectDevice_; freenectDevice_ = 0; } if(ctx_ && freenect_num_devices(ctx_) > 0) { freenectDevice_ = new FreenectDevice(ctx_, deviceId_); if(freenectDevice_->init()) { freenectDevice_->start(); uSleep(3000); return true; } else { UERROR("CameraFreenect: Init failed!"); } delete freenectDevice_; freenectDevice_ = 0; } else { UERROR("CameraFreenect: No devices connected!"); } #else UERROR("CameraFreenect: RTAB-Map is not built with Freenect support!"); #endif return false; } void CameraFreenect::captureImage(cv::Mat & rgb, cv::Mat & depth, float & fx, float & fy, float & cx, float & cy) { #ifdef WITH_FREENECT if(ctx_ && freenectDevice_) { if(freenectDevice_->isRunning()) { freenectDevice_->getData(rgb, depth); UASSERT(freenectDevice_->getDepthFocal() != 0.0f); fx = freenectDevice_->getDepthFocal(); fy = freenectDevice_->getDepthFocal(); cx = float(depth.cols/2) - 0.5f; cy = float(depth.rows/2) - 0.5f; if(depth.empty()) { UWARN("CameraFreenect: Data not ready! Try to reduce the image rate to avoid this warning..."); } } else { UERROR("CameraFreenect: Re-initialization needed!"); delete freenectDevice_; freenectDevice_ = 0; } if(depth.empty() || rgb.empty()) { rgb = cv::Mat(); depth = cv::Mat(); fx = 0.0f; fy = 0.0f; cx = 0.0f; cy = 0.0f; } } #else UERROR("CameraFreenect: RTAB-Map is not built with Freenect support!"); #endif } } // namespace rtabmap