iOS: improved feedback in first-person mapping mode (red background), fixed not matching camera overlay with 3D model, updated how memory usage is computed.

app/android/jni/RTABMapApp.cpp

@@ -1290,7 +1290,7 @@ int RTABMapApp::Render()
                             arViewMatrix = glm::inverse(rtabmap::glmFromTransform(mapCorrection)*glm::inverse(arViewMatrix));
                         }
                     }
-                    if(!visualizingMesh_ && main_scene_.GetCameraType() == tango_gl::GestureCamera::kFirstPerson && !main_scene_.isMeshRendering())
+                    if(!visualizingMesh_ && main_scene_.GetCameraType() == tango_gl::GestureCamera::kFirstPerson)
                     {
                         rtabmap::CameraModel occlusionModel;
                         cv::Mat occlusionImage = ((rtabmap::CameraMobile*)camera_)->getOcclusionImage(&occlusionModel);
@@ -2069,7 +2069,7 @@ int RTABMapApp::Render()
 
 			fpsTime.restart();
 			main_scene_.setFrustumVisible(camera_!=0);
-			lastDrawnCloudsCount_ = main_scene_.Render(uvsTransformed, arViewMatrix, arProjectionMatrix, occlusionMesh);
+			lastDrawnCloudsCount_ = main_scene_.Render(uvsTransformed, arViewMatrix, arProjectionMatrix, occlusionMesh, true);
 			if(renderingTime_ < fpsTime.elapsed())
 			{
 				renderingTime_ = fpsTime.elapsed();

app/android/jni/background_renderer.cc

@@ -39,22 +39,63 @@ const std::string kFragmentShaderOES =
 	"precision mediump float;\n"
 	"varying vec2 v_TexCoord;\n"
 	"uniform samplerExternalOES sTexture;\n"
+    "uniform bool uRedUnknown;\n"
 	"void main() {\n"
 	"    vec4 sample = texture2D(sTexture, v_TexCoord);\n"
-	"    float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
-	"    gl_FragColor = vec4(grey, grey, grey, 0.5);\n"
+    "    float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
+    "    gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
 	"}\n";
 
+const std::string kFragmentShaderBlendingOES =
+    "#extension GL_OES_EGL_image_external : require\n"
+    "precision mediump float;\n"
+    "varying vec2 v_TexCoord;\n"
+    "uniform samplerExternalOES sTexture;\n"
+    "uniform bool uRedUnknown;\n"
+    "void main() {\n"
+    "    vec4 sample = texture2D(sTexture, v_TexCoord);\n"
+    "    vec2 coord = uScreenScale * gl_FragCoord.xy;\n;"
+    "    vec4 depthPacked = texture2D(uDepthTexture, coord);\n"
+    "    float depth = dot(depthPacked, 1./vec4(1.,255.,65025.,16581375.));\n"
+    "    if(depth > 0.0)\n"
+    "        gl_FragColor = vec4(sample.r, sample.g, sample.b, 0.5);\n"
+    "    else {\n"
+    "        float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
+    "        gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
+    "    }\n"
+    "}\n";
+
 const std::string kFragmentShader =
 	"precision mediump float;\n"
 	"varying vec2 v_TexCoord;\n"
     "uniform sampler2D sTexture;\n"
+    "uniform bool uRedUnknown;\n"
 	"void main() {\n"
 	"    vec4 sample = texture2D(sTexture, v_TexCoord);\n"
 	"    float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
-	"    gl_FragColor = vec4(grey, grey, grey, 0.5);\n"
+	"    gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
 	"}\n";
 
+const std::string kFragmentShaderBlending =
+    "precision mediump float;\n"
+    "varying vec2 v_TexCoord;\n"
+    "uniform sampler2D sTexture;\n"
+    "uniform sampler2D uDepthTexture;\n"
+    "uniform vec2 uScreenScale;\n"
+    "uniform bool uRedUnknown;\n"
+    "void main() {\n"
+    "    vec4 sample = texture2D(sTexture, v_TexCoord);\n"
+    "    vec2 coord = uScreenScale * gl_FragCoord.xy;\n;"
+    "    vec4 depthPacked = texture2D(uDepthTexture, coord);\n"
+    "    float depth = dot(depthPacked, 1./vec4(1.,255.,65025.,16581375.));\n"
+    "    if(depth > 0.0)\n"
+    "        gl_FragColor = vec4(sample.r, sample.g, sample.b, 0.5);\n"
+    "    else {\n"
+    "        float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
+    "        gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
+    "    }\n"
+    "}\n";
+
 /* To debug depth texture
  const std::string kFragmentShader =
      "precision mediump float;\n"
@@ -77,6 +118,8 @@ const std::string kFragmentShader =
 
 }  // namespace
 
+std::vector<GLuint> BackgroundRenderer::shaderPrograms_;
+
 void BackgroundRenderer::InitializeGlContent(GLuint textureId, bool oes)
 {
   texture_id_ = textureId;
@@ -84,22 +127,26 @@ void BackgroundRenderer::InitializeGlContent(GLuint textureId, bool oes)
   oes_ = oes;
 #endif
 
-  shader_program_ = tango_gl::util::CreateProgram(
-		  kVertexShader.c_str(),
-		  oes_?kFragmentShaderOES.c_str():kFragmentShader.c_str());
-  if (!shader_program_) {
-    LOGE("Could not create program.");
-  }
-  glUseProgram(shader_program_);
-  attribute_vertices_ = glGetAttribLocation(shader_program_, "a_Position");
-  attribute_uvs_ = glGetAttribLocation(shader_program_, "a_TexCoord");
-  glUseProgram(0);
+    if(shaderPrograms_.empty())
+    {
+        shaderPrograms_.resize(2,0);
+        shaderPrograms_[0] = tango_gl::util::CreateProgram(
+              kVertexShader.c_str(),
+              oes_?kFragmentShaderOES.c_str():kFragmentShader.c_str());
+        UASSERT(shaderPrograms_[0]!=0);
+        shaderPrograms_[1] = tango_gl::util::CreateProgram(
+              kVertexShader.c_str(),
+              oes_?kFragmentShaderBlendingOES.c_str():kFragmentShaderBlending.c_str());
+        UASSERT(shaderPrograms_[1]!=0);
+    }
 }
 
-void BackgroundRenderer::Draw(const float * transformed_uvs) {
+void BackgroundRenderer::Draw(const float * transformed_uvs, const GLuint & depthTexture, int screenWidth, int screenHeight, bool redUnknown) {
   static_assert(std::extent<decltype(BackgroundRenderer_kVertices)>::value == kNumVertices * 2, "Incorrect kVertices length");
 
-  glUseProgram(shader_program_);
+  GLuint program = shaderPrograms_[depthTexture>0?1:0];
+    
+  glUseProgram(program);
   glDepthMask(GL_FALSE);
   glEnable (GL_BLEND);
 
@@ -110,7 +157,27 @@ void BackgroundRenderer::Draw(const float * transformed_uvs) {
   else
 #endif
     glBindTexture(GL_TEXTURE_2D, texture_id_);
+    
+  if(depthTexture>0)
+  {
+    // Texture activate unit 1
+    glActiveTexture(GL_TEXTURE1);
+    // Bind the texture to this unit.
+    glBindTexture(GL_TEXTURE_2D, depthTexture);
+    // Tell the texture uniform sampler to use this texture in the shader by binding to texture unit 1.
+    GLuint depth_texture_handle = glGetUniformLocation(program, "uDepthTexture");
+    glUniform1i(depth_texture_handle, 1);
+      
+    GLuint screenScale_handle = glGetUniformLocation(program, "uScreenScale");
+    glUniform2f(screenScale_handle, 1.0f/(float)screenWidth, 1.0f/(float)screenHeight);
+  }
+    
+  GLuint screenScale_handle = glGetUniformLocation(program, "uRedUnknown");
+  glUniform1i(screenScale_handle, redUnknown);
 
+  GLuint attribute_vertices_ = glGetAttribLocation(program, "a_Position");
+  GLuint attribute_uvs_ = glGetAttribLocation(program, "a_TexCoord");
+    
   glVertexAttribPointer(attribute_vertices_, 2, GL_FLOAT, GL_FALSE, 0, BackgroundRenderer_kVertices);
   glVertexAttribPointer(attribute_uvs_, 2, GL_FLOAT, GL_FALSE, 0, transformed_uvs?transformed_uvs:BackgroundRenderer_kTexCoord);
 

app/android/jni/background_renderer.h

@@ -59,16 +59,12 @@ public:
 
   // Draws the background image.  This methods must be called for every ArFrame
   // returned by ArSession_update() to catch display geometry change events.
-  void Draw(const float * transformed_uvs);
+  void Draw(const float * transformed_uvs, const GLuint & depthTexture, int screenWidth, int screenHeight, bool redUnknown);
 
- private:
-
-  GLuint shader_program_;
+private:
+  static std::vector<GLuint> shaderPrograms_;
   GLuint texture_id_;
   bool oes_ = false;
-
-  GLuint attribute_vertices_;
-  GLuint attribute_uvs_;
 };
 
 #endif  // C_ARCORE_AUGMENTED_IMAGE_BACKGROUND_RENDERER_H_

app/android/jni/scene.cpp

@@ -381,7 +381,7 @@ bool intersectFrustumAABB(
 }
 
 //Should only be called in OpenGL thread!
-int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat4 arProjectionMatrix, const rtabmap::Mesh & occlusionMesh) {
+int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat4 arProjectionMatrix, const rtabmap::Mesh & occlusionMesh, bool mapping) {
 	UASSERT(gesture_camera_ != 0);
 
 	if(currentPose_ == 0)
@@ -492,7 +492,7 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat
 		glClearColor(0, 0, 0, 0);
 		glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
 
-		if(renderBackgroundCamera && occlusionMesh.cloud.get() && occlusionMesh.cloud->size())
+		if(renderBackgroundCamera && !meshRendering_ && occlusionMesh.cloud.get() && occlusionMesh.cloud->size())
 		{
 			PointCloudDrawable drawable(occlusionMesh);
 			drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f, 0, 0, 0, 0, 0, true);
@@ -548,15 +548,15 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat
 
 	glClearColor(r_, g_, b_, 1.0f);
 	glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
+    
+    if(renderBackgroundCamera && (!onlineBlending || !meshRendering_))
+    {
+        background_renderer_->Draw(uvsTransformed, 0, screenWidth_, screenHeight_, false);
 
-	if(renderBackgroundCamera)
-	{
-		background_renderer_->Draw(uvsTransformed);
-
-		//To debug occlusion image:
-		//PointCloudDrawable drawable(occlusionMesh);
-		//drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f);
-	}
+        //To debug occlusion image:
+        //PointCloudDrawable drawable(occlusionMesh);
+        //drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f);
+    }
 
 	if(!currentPose_->isNull())
 	{
@@ -624,10 +624,15 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat
 
 	if(onlineBlending)
 	{
+        if(renderBackgroundCamera && meshRendering_)
+        {
+            background_renderer_->Draw(uvsTransformed, depthTexture_, screenWidth_, screenHeight_, mapping);
+        }
+        
 		glDisable (GL_BLEND);
 		glDepthMask(GL_TRUE);
 	}
-
+    
 	//draw markers on foreground
 	for(std::map<int, tango_gl::Axis*>::const_iterator iter=markers_.begin(); iter!=markers_.end(); ++iter)
 	{

app/android/jni/scene.h

@@ -74,7 +74,7 @@ class Scene {
   //         frame's timestamp.
   // @param: point_cloud_vertices, point cloud's vertices of the current point
   //         frame.
-  int Render(const float * uvsTransformed = 0, glm::mat4 arViewMatrix = glm::mat4(0), glm::mat4 arProjectionMatrix=glm::mat4(0), const rtabmap::Mesh & occlusionMesh=rtabmap::Mesh());
+  int Render(const float * uvsTransformed = 0, glm::mat4 arViewMatrix = glm::mat4(0), glm::mat4 arProjectionMatrix=glm::mat4(0), const rtabmap::Mesh & occlusionMesh=rtabmap::Mesh(), bool mapping=false);
 
   // Set render camera's viewing angle, first person, third person or top down.
   //
@@ -155,6 +155,7 @@ class Scene {
   float getPointSize() const {return pointSize_;}
   bool isLighting() const {return lighting_;}
   bool isBackfaceCulling() const {return backfaceCulling_;}
+  bool isWireframe() const {return wireFrame_;}
 
   BackgroundRenderer * background_renderer_;
 

app/android/jni/tango-gl/gesture_camera.cpp

@@ -183,7 +183,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
     case kFirstPerson:
       SetOrthoMode(false);
       SetFieldOfView(kLowestFov);
-      SetNearFarClipPlanes(0.5, 50);
+      SetNearFarClipPlanes(0.3, 50);
       SetPosition(glm::vec3(0.0f, 0.0f, 0.0f));
       SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f));
       cam_cur_dist_ = 0.0f;
@@ -198,7 +198,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
     case kThirdPersonFollow:
       SetOrthoMode(false);
       SetFieldOfView(kLowFov);
-      SetNearFarClipPlanes(0.5, 50);
+      SetNearFarClipPlanes(1, 50);
       SetPosition(glm::vec3(0.0f, 0.0f, 0.0f));
       SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f));
       cam_cur_dist_ = camera_index==kThirdPersonFollow?kThirdPersonFollowCameraDist:kThirdPersonCameraDist;
@@ -213,7 +213,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
       SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f));
       SetOrthoMode(false);
       SetFieldOfView(kLowFov);
-      SetNearFarClipPlanes(0.5, 50);
+      SetNearFarClipPlanes(1, 50);
       cam_cur_dist_ = kTopDownCameraDist;
       anchor_offset_ = glm::vec3(0.0f,0.0f,0.0f);
       cam_cur_angle_.x = -M_PI / 2.0f;
@@ -228,7 +228,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
 	  SetOrthoScale(kTopDownCameraDist);
 	  SetOrthoCropFactor(-1.0f);
       SetFieldOfView(kLowFov);
-      SetNearFarClipPlanes(0.5, 50);
+      SetNearFarClipPlanes(1, 50);
 	  cam_cur_dist_ = kTopDownCameraDist;
 	  anchor_offset_ = glm::vec3(0.0f,0.0f,0.0f);
 	  cam_cur_angle_.x = -M_PI / 2.0f;