diff --git a/app/android/jni/RTABMapApp.cpp b/app/android/jni/RTABMapApp.cpp index dcdf8eba..5ee12681 100644 --- a/app/android/jni/RTABMapApp.cpp +++ b/app/android/jni/RTABMapApp.cpp @@ -1290,7 +1290,7 @@ int RTABMapApp::Render() arViewMatrix = glm::inverse(rtabmap::glmFromTransform(mapCorrection)*glm::inverse(arViewMatrix)); } } - if(!visualizingMesh_ && main_scene_.GetCameraType() == tango_gl::GestureCamera::kFirstPerson && !main_scene_.isMeshRendering()) + if(!visualizingMesh_ && main_scene_.GetCameraType() == tango_gl::GestureCamera::kFirstPerson) { rtabmap::CameraModel occlusionModel; cv::Mat occlusionImage = ((rtabmap::CameraMobile*)camera_)->getOcclusionImage(&occlusionModel); @@ -2069,7 +2069,7 @@ int RTABMapApp::Render() fpsTime.restart(); main_scene_.setFrustumVisible(camera_!=0); - lastDrawnCloudsCount_ = main_scene_.Render(uvsTransformed, arViewMatrix, arProjectionMatrix, occlusionMesh); + lastDrawnCloudsCount_ = main_scene_.Render(uvsTransformed, arViewMatrix, arProjectionMatrix, occlusionMesh, true); if(renderingTime_ < fpsTime.elapsed()) { renderingTime_ = fpsTime.elapsed(); diff --git a/app/android/jni/background_renderer.cc b/app/android/jni/background_renderer.cc index 8a2aa527..fb19a9f7 100644 --- a/app/android/jni/background_renderer.cc +++ b/app/android/jni/background_renderer.cc @@ -39,22 +39,63 @@ const std::string kFragmentShaderOES = "precision mediump float;\n" "varying vec2 v_TexCoord;\n" "uniform samplerExternalOES sTexture;\n" + "uniform bool uRedUnknown;\n" "void main() {\n" " vec4 sample = texture2D(sTexture, v_TexCoord);\n" - " float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n" - " gl_FragColor = vec4(grey, grey, grey, 0.5);\n" + " float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n" + " gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n" "}\n"; +const std::string kFragmentShaderBlendingOES = + "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + "varying vec2 v_TexCoord;\n" + "uniform samplerExternalOES sTexture;\n" + "uniform bool uRedUnknown;\n" + "void main() {\n" + " vec4 sample = texture2D(sTexture, v_TexCoord);\n" + " vec2 coord = uScreenScale * gl_FragCoord.xy;\n;" + " vec4 depthPacked = texture2D(uDepthTexture, coord);\n" + " float depth = dot(depthPacked, 1./vec4(1.,255.,65025.,16581375.));\n" + " if(depth > 0.0)\n" + " gl_FragColor = vec4(sample.r, sample.g, sample.b, 0.5);\n" + " else {\n" + " float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n" + " gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n" + " }\n" + "}\n"; + const std::string kFragmentShader = "precision mediump float;\n" "varying vec2 v_TexCoord;\n" "uniform sampler2D sTexture;\n" + "uniform bool uRedUnknown;\n" "void main() {\n" " vec4 sample = texture2D(sTexture, v_TexCoord);\n" " float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n" - " gl_FragColor = vec4(grey, grey, grey, 0.5);\n" + " gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n" "}\n"; +const std::string kFragmentShaderBlending = + "precision mediump float;\n" + "varying vec2 v_TexCoord;\n" + "uniform sampler2D sTexture;\n" + "uniform sampler2D uDepthTexture;\n" + "uniform vec2 uScreenScale;\n" + "uniform bool uRedUnknown;\n" + "void main() {\n" + " vec4 sample = texture2D(sTexture, v_TexCoord);\n" + " vec2 coord = uScreenScale * gl_FragCoord.xy;\n;" + " vec4 depthPacked = texture2D(uDepthTexture, coord);\n" + " float depth = dot(depthPacked, 1./vec4(1.,255.,65025.,16581375.));\n" + " if(depth > 0.0)\n" + " gl_FragColor = vec4(sample.r, sample.g, sample.b, 0.5);\n" + " else {\n" + " float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n" + " gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n" + " }\n" + "}\n"; + /* To debug depth texture const std::string kFragmentShader = "precision mediump float;\n" @@ -77,6 +118,8 @@ const std::string kFragmentShader = } // namespace +std::vector BackgroundRenderer::shaderPrograms_; + void BackgroundRenderer::InitializeGlContent(GLuint textureId, bool oes) { texture_id_ = textureId; @@ -84,22 +127,26 @@ void BackgroundRenderer::InitializeGlContent(GLuint textureId, bool oes) oes_ = oes; #endif - shader_program_ = tango_gl::util::CreateProgram( - kVertexShader.c_str(), - oes_?kFragmentShaderOES.c_str():kFragmentShader.c_str()); - if (!shader_program_) { - LOGE("Could not create program."); - } - glUseProgram(shader_program_); - attribute_vertices_ = glGetAttribLocation(shader_program_, "a_Position"); - attribute_uvs_ = glGetAttribLocation(shader_program_, "a_TexCoord"); - glUseProgram(0); + if(shaderPrograms_.empty()) + { + shaderPrograms_.resize(2,0); + shaderPrograms_[0] = tango_gl::util::CreateProgram( + kVertexShader.c_str(), + oes_?kFragmentShaderOES.c_str():kFragmentShader.c_str()); + UASSERT(shaderPrograms_[0]!=0); + shaderPrograms_[1] = tango_gl::util::CreateProgram( + kVertexShader.c_str(), + oes_?kFragmentShaderBlendingOES.c_str():kFragmentShaderBlending.c_str()); + UASSERT(shaderPrograms_[1]!=0); + } } -void BackgroundRenderer::Draw(const float * transformed_uvs) { +void BackgroundRenderer::Draw(const float * transformed_uvs, const GLuint & depthTexture, int screenWidth, int screenHeight, bool redUnknown) { static_assert(std::extent::value == kNumVertices * 2, "Incorrect kVertices length"); - glUseProgram(shader_program_); + GLuint program = shaderPrograms_[depthTexture>0?1:0]; + + glUseProgram(program); glDepthMask(GL_FALSE); glEnable (GL_BLEND); @@ -110,7 +157,27 @@ void BackgroundRenderer::Draw(const float * transformed_uvs) { else #endif glBindTexture(GL_TEXTURE_2D, texture_id_); + + if(depthTexture>0) + { + // Texture activate unit 1 + glActiveTexture(GL_TEXTURE1); + // Bind the texture to this unit. + glBindTexture(GL_TEXTURE_2D, depthTexture); + // Tell the texture uniform sampler to use this texture in the shader by binding to texture unit 1. + GLuint depth_texture_handle = glGetUniformLocation(program, "uDepthTexture"); + glUniform1i(depth_texture_handle, 1); + + GLuint screenScale_handle = glGetUniformLocation(program, "uScreenScale"); + glUniform2f(screenScale_handle, 1.0f/(float)screenWidth, 1.0f/(float)screenHeight); + } + + GLuint screenScale_handle = glGetUniformLocation(program, "uRedUnknown"); + glUniform1i(screenScale_handle, redUnknown); + GLuint attribute_vertices_ = glGetAttribLocation(program, "a_Position"); + GLuint attribute_uvs_ = glGetAttribLocation(program, "a_TexCoord"); + glVertexAttribPointer(attribute_vertices_, 2, GL_FLOAT, GL_FALSE, 0, BackgroundRenderer_kVertices); glVertexAttribPointer(attribute_uvs_, 2, GL_FLOAT, GL_FALSE, 0, transformed_uvs?transformed_uvs:BackgroundRenderer_kTexCoord); diff --git a/app/android/jni/background_renderer.h b/app/android/jni/background_renderer.h index 59c1e409..225efdc1 100644 --- a/app/android/jni/background_renderer.h +++ b/app/android/jni/background_renderer.h @@ -59,16 +59,12 @@ public: // Draws the background image. This methods must be called for every ArFrame // returned by ArSession_update() to catch display geometry change events. - void Draw(const float * transformed_uvs); + void Draw(const float * transformed_uvs, const GLuint & depthTexture, int screenWidth, int screenHeight, bool redUnknown); - private: - - GLuint shader_program_; +private: + static std::vector shaderPrograms_; GLuint texture_id_; bool oes_ = false; - - GLuint attribute_vertices_; - GLuint attribute_uvs_; }; #endif // C_ARCORE_AUGMENTED_IMAGE_BACKGROUND_RENDERER_H_ diff --git a/app/android/jni/scene.cpp b/app/android/jni/scene.cpp index c15810eb..3a2a5d40 100644 --- a/app/android/jni/scene.cpp +++ b/app/android/jni/scene.cpp @@ -381,7 +381,7 @@ bool intersectFrustumAABB( } //Should only be called in OpenGL thread! -int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat4 arProjectionMatrix, const rtabmap::Mesh & occlusionMesh) { +int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat4 arProjectionMatrix, const rtabmap::Mesh & occlusionMesh, bool mapping) { UASSERT(gesture_camera_ != 0); if(currentPose_ == 0) @@ -492,7 +492,7 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat glClearColor(0, 0, 0, 0); glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT); - if(renderBackgroundCamera && occlusionMesh.cloud.get() && occlusionMesh.cloud->size()) + if(renderBackgroundCamera && !meshRendering_ && occlusionMesh.cloud.get() && occlusionMesh.cloud->size()) { PointCloudDrawable drawable(occlusionMesh); drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f, 0, 0, 0, 0, 0, true); @@ -548,15 +548,15 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat glClearColor(r_, g_, b_, 1.0f); glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT); + + if(renderBackgroundCamera && (!onlineBlending || !meshRendering_)) + { + background_renderer_->Draw(uvsTransformed, 0, screenWidth_, screenHeight_, false); - if(renderBackgroundCamera) - { - background_renderer_->Draw(uvsTransformed); - - //To debug occlusion image: - //PointCloudDrawable drawable(occlusionMesh); - //drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f); - } + //To debug occlusion image: + //PointCloudDrawable drawable(occlusionMesh); + //drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f); + } if(!currentPose_->isNull()) { @@ -624,10 +624,15 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat if(onlineBlending) { + if(renderBackgroundCamera && meshRendering_) + { + background_renderer_->Draw(uvsTransformed, depthTexture_, screenWidth_, screenHeight_, mapping); + } + glDisable (GL_BLEND); glDepthMask(GL_TRUE); } - + //draw markers on foreground for(std::map::const_iterator iter=markers_.begin(); iter!=markers_.end(); ++iter) { diff --git a/app/android/jni/scene.h b/app/android/jni/scene.h index 3859f4db..c4e8e25e 100644 --- a/app/android/jni/scene.h +++ b/app/android/jni/scene.h @@ -74,7 +74,7 @@ class Scene { // frame's timestamp. // @param: point_cloud_vertices, point cloud's vertices of the current point // frame. - int Render(const float * uvsTransformed = 0, glm::mat4 arViewMatrix = glm::mat4(0), glm::mat4 arProjectionMatrix=glm::mat4(0), const rtabmap::Mesh & occlusionMesh=rtabmap::Mesh()); + int Render(const float * uvsTransformed = 0, glm::mat4 arViewMatrix = glm::mat4(0), glm::mat4 arProjectionMatrix=glm::mat4(0), const rtabmap::Mesh & occlusionMesh=rtabmap::Mesh(), bool mapping=false); // Set render camera's viewing angle, first person, third person or top down. // @@ -155,6 +155,7 @@ class Scene { float getPointSize() const {return pointSize_;} bool isLighting() const {return lighting_;} bool isBackfaceCulling() const {return backfaceCulling_;} + bool isWireframe() const {return wireFrame_;} BackgroundRenderer * background_renderer_; diff --git a/app/android/jni/tango-gl/gesture_camera.cpp b/app/android/jni/tango-gl/gesture_camera.cpp index a52c2647..ef3cf6b4 100644 --- a/app/android/jni/tango-gl/gesture_camera.cpp +++ b/app/android/jni/tango-gl/gesture_camera.cpp @@ -183,7 +183,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) { case kFirstPerson: SetOrthoMode(false); SetFieldOfView(kLowestFov); - SetNearFarClipPlanes(0.5, 50); + SetNearFarClipPlanes(0.3, 50); SetPosition(glm::vec3(0.0f, 0.0f, 0.0f)); SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f)); cam_cur_dist_ = 0.0f; @@ -198,7 +198,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) { case kThirdPersonFollow: SetOrthoMode(false); SetFieldOfView(kLowFov); - SetNearFarClipPlanes(0.5, 50); + SetNearFarClipPlanes(1, 50); SetPosition(glm::vec3(0.0f, 0.0f, 0.0f)); SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f)); cam_cur_dist_ = camera_index==kThirdPersonFollow?kThirdPersonFollowCameraDist:kThirdPersonCameraDist; @@ -213,7 +213,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) { SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f)); SetOrthoMode(false); SetFieldOfView(kLowFov); - SetNearFarClipPlanes(0.5, 50); + SetNearFarClipPlanes(1, 50); cam_cur_dist_ = kTopDownCameraDist; anchor_offset_ = glm::vec3(0.0f,0.0f,0.0f); cam_cur_angle_.x = -M_PI / 2.0f; @@ -228,7 +228,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) { SetOrthoScale(kTopDownCameraDist); SetOrthoCropFactor(-1.0f); SetFieldOfView(kLowFov); - SetNearFarClipPlanes(0.5, 50); + SetNearFarClipPlanes(1, 50); cam_cur_dist_ = kTopDownCameraDist; anchor_offset_ = glm::vec3(0.0f,0.0f,0.0f); cam_cur_angle_.x = -M_PI / 2.0f; diff --git a/app/ios/RTABMapApp/RTABMap.swift b/app/ios/RTABMapApp/RTABMap.swift index bab1f531..c2fa7775 100644 --- a/app/ios/RTABMapApp/RTABMap.swift +++ b/app/ios/RTABMapApp/RTABMap.swift @@ -248,6 +248,12 @@ class RTABMap { let quatv = GLKQuaternionMakeWithMatrix3(rotation) + let texX1 = (1-(2*frame.camera.intrinsics[0,0]/Float(frame.camera.imageResolution.width)) / p[0,0])/2 + let texY1 = (1-(2*frame.camera.intrinsics[1,1]/Float(frame.camera.imageResolution.height)) / p[1,1])/2 + + let texX2 = (1-(2*frame.camera.intrinsics[0,0]/Float(frame.camera.imageResolution.width)) / p[1,1])/2 + let texY2 = (1-(2*frame.camera.intrinsics[1,1]/Float(frame.camera.imageResolution.height)) / p[0,0])/2 + //11 10 01 00 // portrait //01 11 00 10 // right //10 00 11 01 // left @@ -255,13 +261,13 @@ class RTABMap { var texCoord: [Float] switch orientation { case .portrait: - texCoord = [1, 1, 1, 0, 0, 1, 0, 0] + texCoord = [1-texX2, 1-texY2, 1-texX2,texY2, texX2, 1-texY2, texX2, texY2] case .landscapeRight: - texCoord = [0, 1, 1, 1, 0, 0, 1, 0] + texCoord = [texX1, 1-texY1, 1-texX1, 1-texY1, texX1, texY1, 1-texX1, texY1] case .landscapeLeft: - texCoord = [1, 0, 0, 0, 1, 1, 0, 1] + texCoord = [1-texX1, texY1, texX1, texY1, 1-texX1, 1-texY1, texX1, 1-texY1] default: // down - texCoord = [0, 0, 0, 1, 1, 0, 1, 1] + texCoord = [texX2, texY2, texX2, 1-texY2, 1-texX2, texY2, 1-texX2, 1-texY2] } frame.rawFeaturePoints?.points.withUnsafeBufferPointer { bufferPoints in diff --git a/app/ios/RTABMapApp/ViewController.swift b/app/ios/RTABMapApp/ViewController.swift index 15253fa7..c5dd27c2 100644 --- a/app/ios/RTABMapApp/ViewController.swift +++ b/app/ios/RTABMapApp/ViewController.swift @@ -105,7 +105,7 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP @IBOutlet weak var exportOBJPLYButton: UIButton! @IBOutlet weak var orthoDistanceSlider: UISlider!{ didSet{ - orthoDistanceSlider.transform = CGAffineTransform(rotationAngle: CGFloat(-M_PI_2)) + orthoDistanceSlider.transform = CGAffineTransform(rotationAngle: CGFloat(-Double.pi/2)) } } @IBOutlet weak var orthoGridSlider: UISlider! @@ -275,11 +275,10 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP self.showToast(message: "Optimized mesh detected in the database, it is shown while the database is loading...", seconds: 3) } - let (usedMem, freeMem) = self.getMemoryUsage() + let usedMem = self.getMemoryUsage() self.statusLabel.text = "Status: " + (status == 1 && msg.isEmpty ? self.mState == State.STATE_CAMERA ? "Camera Preview" : "Idle" : msg) + "\n" + - "Used Memory: \(usedMem) MB\n" + - "Free Memory: \(freeMem) MB" + "Memory Usage: \(usedMem) MB" } } @@ -312,7 +311,7 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP pitch: Float, yaw: Float) { - let (usedMem, freeMem) = self.getMemoryUsage() + let usedMem = self.getMemoryUsage() if(loopClosureId > 0) { @@ -335,8 +334,7 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP self.statusLabel.text = self.statusLabel.text! + "Status: \(self.getStateString(state: self.mState))\n" + - "Used Memory (MB): \(usedMem)\n" + - "Free Memory (MB): \(freeMem)" + "Memory Usage : \(usedMem) MB" } if self.debugShown { self.statusLabel.text = @@ -403,29 +401,23 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP } } - func getMemoryUsage() -> (UInt64, UInt64) { - var pagesize: vm_size_t = 0 - - let host_port: mach_port_t = mach_host_self() - var host_size: mach_msg_type_number_t = mach_msg_type_number_t(MemoryLayout.stride / MemoryLayout.stride) - host_page_size(host_port, &pagesize) - - var vm_stat: vm_statistics = vm_statistics_data_t() - withUnsafeMutablePointer(to: &vm_stat) { (vmStatPointer) -> Void in - vmStatPointer.withMemoryRebound(to: integer_t.self, capacity: Int(host_size)) { - if (host_statistics(host_port, HOST_VM_INFO, $0, &host_size) != KERN_SUCCESS) { - NSLog("Error: Failed to fetch vm statistics") - } + func getMemoryUsage() -> UInt64 { + var taskInfo = mach_task_basic_info() + var count = mach_msg_type_number_t(MemoryLayout.size)/4 + let kerr: kern_return_t = withUnsafeMutablePointer(to: &taskInfo) { + $0.withMemoryRebound(to: integer_t.self, capacity: 1) { + task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), $0, &count) } } - /* Stats in bytes */ - let mem_used: UInt64 = UInt64(vm_stat.active_count + - vm_stat.inactive_count + - vm_stat.wire_count) * UInt64(pagesize) - let mem_free: UInt64 = UInt64(vm_stat.free_count) * UInt64(pagesize) - - return (mem_used/(1024*1024), mem_free/(1024*1024)) + if kerr == KERN_SUCCESS { + return taskInfo.resident_size / (1024*1024) + } + else { + print("Error with task_info(): " + + (String(cString: mach_error_string(kerr), encoding: String.Encoding.ascii) ?? "unknown error")) + return 0 + } } @objc func appMovedToBackground() {