i/rtabmap
1
0
Fork 0
mirror of https://github.com/introlab/rtabmap.git synced 2026-09-02 09:30:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

iOS: improved feedback in first-person mapping mode (red background), fixed not matching camera overlay with 3D model, updated how memory usage is computed.

Browse Source
This commit is contained in:
matlabbe
2021-06-11 21:41:59 -04:00
parent a6f3a81999
commit 3f633081cb
8 changed files with 138 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/android/jni/RTABMapApp.cpp
View File

@@ -1290,7 +1290,7 @@ int RTABMapApp::Render()
arViewMatrix = glm::inverse(rtabmap::glmFromTransform(mapCorrection)*glm::inverse(arViewMatrix));
}
}
if(!visualizingMesh_ && main_scene_.GetCameraType() == tango_gl::GestureCamera::kFirstPerson && !main_scene_.isMeshRendering())
if(!visualizingMesh_ && main_scene_.GetCameraType() == tango_gl::GestureCamera::kFirstPerson)
{
rtabmap::CameraModel occlusionModel;
cv::Mat occlusionImage = ((rtabmap::CameraMobile*)camera_)->getOcclusionImage(&occlusionModel);
@@ -2069,7 +2069,7 @@ int RTABMapApp::Render()
fpsTime.restart();
main_scene_.setFrustumVisible(camera_!=0);
lastDrawnCloudsCount_ = main_scene_.Render(uvsTransformed, arViewMatrix, arProjectionMatrix, occlusionMesh);
lastDrawnCloudsCount_ = main_scene_.Render(uvsTransformed, arViewMatrix, arProjectionMatrix, occlusionMesh, true);
if(renderingTime_ < fpsTime.elapsed())
{
renderingTime_ = fpsTime.elapsed();

97
app/android/jni/background_renderer.cc
View File

@@ -39,22 +39,63 @@ const std::string kFragmentShaderOES =
"precision mediump float;\n"
"varying vec2 v_TexCoord;\n"
"uniform samplerExternalOES sTexture;\n"
"uniform bool uRedUnknown;\n"
"void main() {\n"
" vec4 sample = texture2D(sTexture, v_TexCoord);\n"
" float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
" gl_FragColor = vec4(grey, grey, grey, 0.5);\n"
" float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
" gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
"}\n";
const std::string kFragmentShaderBlendingOES =
"#extension GL_OES_EGL_image_external : require\n"
"precision mediump float;\n"
"varying vec2 v_TexCoord;\n"
"uniform samplerExternalOES sTexture;\n"
"uniform bool uRedUnknown;\n"
"void main() {\n"
" vec4 sample = texture2D(sTexture, v_TexCoord);\n"
" vec2 coord = uScreenScale * gl_FragCoord.xy;\n;"
" vec4 depthPacked = texture2D(uDepthTexture, coord);\n"
" float depth = dot(depthPacked, 1./vec4(1.,255.,65025.,16581375.));\n"
" if(depth > 0.0)\n"
" gl_FragColor = vec4(sample.r, sample.g, sample.b, 0.5);\n"
" else {\n"
" float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
" gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
" }\n"
"}\n";
const std::string kFragmentShader =
"precision mediump float;\n"
"varying vec2 v_TexCoord;\n"
"uniform sampler2D sTexture;\n"
"uniform bool uRedUnknown;\n"
"void main() {\n"
" vec4 sample = texture2D(sTexture, v_TexCoord);\n"
" float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
" gl_FragColor = vec4(grey, grey, grey, 0.5);\n"
" gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
"}\n";
const std::string kFragmentShaderBlending =
"precision mediump float;\n"
"varying vec2 v_TexCoord;\n"
"uniform sampler2D sTexture;\n"
"uniform sampler2D uDepthTexture;\n"
"uniform vec2 uScreenScale;\n"
"uniform bool uRedUnknown;\n"
"void main() {\n"
" vec4 sample = texture2D(sTexture, v_TexCoord);\n"
" vec2 coord = uScreenScale * gl_FragCoord.xy;\n;"
" vec4 depthPacked = texture2D(uDepthTexture, coord);\n"
" float depth = dot(depthPacked, 1./vec4(1.,255.,65025.,16581375.));\n"
" if(depth > 0.0)\n"
" gl_FragColor = vec4(sample.r, sample.g, sample.b, 0.5);\n"
" else {\n"
" float grey = 0.21 * sample.r + 0.71 * sample.g + 0.07 * sample.b;\n"
" gl_FragColor = vec4(grey, uRedUnknown?0.0:grey, uRedUnknown?0.0:grey, 0.5);\n"
" }\n"
"}\n";
/* To debug depth texture
const std::string kFragmentShader =
"precision mediump float;\n"
@@ -77,6 +118,8 @@ const std::string kFragmentShader =
} // namespace
std::vector<GLuint> BackgroundRenderer::shaderPrograms_;
void BackgroundRenderer::InitializeGlContent(GLuint textureId, bool oes)
{
texture_id_ = textureId;
@@ -84,22 +127,26 @@ void BackgroundRenderer::InitializeGlContent(GLuint textureId, bool oes)
oes_ = oes;
#endif
shader_program_ = tango_gl::util::CreateProgram(
kVertexShader.c_str(),
oes_?kFragmentShaderOES.c_str():kFragmentShader.c_str());
if (!shader_program_) {
LOGE("Could not create program.");
}
glUseProgram(shader_program_);
attribute_vertices_ = glGetAttribLocation(shader_program_, "a_Position");
attribute_uvs_ = glGetAttribLocation(shader_program_, "a_TexCoord");
glUseProgram(0);
if(shaderPrograms_.empty())
{
shaderPrograms_.resize(2,0);
shaderPrograms_[0] = tango_gl::util::CreateProgram(
kVertexShader.c_str(),
oes_?kFragmentShaderOES.c_str():kFragmentShader.c_str());
UASSERT(shaderPrograms_[0]!=0);
shaderPrograms_[1] = tango_gl::util::CreateProgram(
kVertexShader.c_str(),
oes_?kFragmentShaderBlendingOES.c_str():kFragmentShaderBlending.c_str());
UASSERT(shaderPrograms_[1]!=0);
}
}
void BackgroundRenderer::Draw(const float * transformed_uvs) {
void BackgroundRenderer::Draw(const float * transformed_uvs, const GLuint & depthTexture, int screenWidth, int screenHeight, bool redUnknown) {
static_assert(std::extent<decltype(BackgroundRenderer_kVertices)>::value == kNumVertices * 2, "Incorrect kVertices length");
glUseProgram(shader_program_);
GLuint program = shaderPrograms_[depthTexture>0?1:0];
glUseProgram(program);
glDepthMask(GL_FALSE);
glEnable (GL_BLEND);
@@ -110,7 +157,27 @@ void BackgroundRenderer::Draw(const float * transformed_uvs) {
else
#endif
glBindTexture(GL_TEXTURE_2D, texture_id_);
if(depthTexture>0)
{
// Texture activate unit 1
glActiveTexture(GL_TEXTURE1);
// Bind the texture to this unit.
glBindTexture(GL_TEXTURE_2D, depthTexture);
// Tell the texture uniform sampler to use this texture in the shader by binding to texture unit 1.
GLuint depth_texture_handle = glGetUniformLocation(program, "uDepthTexture");
glUniform1i(depth_texture_handle, 1);
GLuint screenScale_handle = glGetUniformLocation(program, "uScreenScale");
glUniform2f(screenScale_handle, 1.0f/(float)screenWidth, 1.0f/(float)screenHeight);
}
GLuint screenScale_handle = glGetUniformLocation(program, "uRedUnknown");
glUniform1i(screenScale_handle, redUnknown);
GLuint attribute_vertices_ = glGetAttribLocation(program, "a_Position");
GLuint attribute_uvs_ = glGetAttribLocation(program, "a_TexCoord");
glVertexAttribPointer(attribute_vertices_, 2, GL_FLOAT, GL_FALSE, 0, BackgroundRenderer_kVertices);
glVertexAttribPointer(attribute_uvs_, 2, GL_FLOAT, GL_FALSE, 0, transformed_uvs?transformed_uvs:BackgroundRenderer_kTexCoord);

10
app/android/jni/background_renderer.h
View File

@@ -59,16 +59,12 @@ public:
// Draws the background image. This methods must be called for every ArFrame
// returned by ArSession_update() to catch display geometry change events.
void Draw(const float * transformed_uvs);
void Draw(const float * transformed_uvs, const GLuint & depthTexture, int screenWidth, int screenHeight, bool redUnknown);
private:
GLuint shader_program_;
private:
static std::vector<GLuint> shaderPrograms_;
GLuint texture_id_;
bool oes_ = false;
GLuint attribute_vertices_;
GLuint attribute_uvs_;
};
#endif // C_ARCORE_AUGMENTED_IMAGE_BACKGROUND_RENDERER_H_

27
app/android/jni/scene.cpp
View File

@@ -381,7 +381,7 @@ bool intersectFrustumAABB(
}
//Should only be called in OpenGL thread!
int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat4 arProjectionMatrix, const rtabmap::Mesh & occlusionMesh) {
int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat4 arProjectionMatrix, const rtabmap::Mesh & occlusionMesh, bool mapping) {
UASSERT(gesture_camera_ != 0);
if(currentPose_ == 0)
@@ -492,7 +492,7 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat
glClearColor(0, 0, 0, 0);
glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
if(renderBackgroundCamera && occlusionMesh.cloud.get() && occlusionMesh.cloud->size())
if(renderBackgroundCamera && !meshRendering_ && occlusionMesh.cloud.get() && occlusionMesh.cloud->size())
{
PointCloudDrawable drawable(occlusionMesh);
drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f, 0, 0, 0, 0, 0, true);
@@ -548,15 +548,15 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat
glClearColor(r_, g_, b_, 1.0f);
glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
if(renderBackgroundCamera && (!onlineBlending || !meshRendering_))
{
background_renderer_->Draw(uvsTransformed, 0, screenWidth_, screenHeight_, false);
if(renderBackgroundCamera)
{
background_renderer_->Draw(uvsTransformed);
//To debug occlusion image:
//PointCloudDrawable drawable(occlusionMesh);
//drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f);
}
//To debug occlusion image:
//PointCloudDrawable drawable(occlusionMesh);
//drawable.Render(projectionMatrix, viewMatrix, true, pointSize_, false, false, 999.0f);
}
if(!currentPose_->isNull())
{
@@ -624,10 +624,15 @@ int Scene::Render(const float * uvsTransformed, glm::mat4 arViewMatrix, glm::mat
if(onlineBlending)
{
if(renderBackgroundCamera && meshRendering_)
{
background_renderer_->Draw(uvsTransformed, depthTexture_, screenWidth_, screenHeight_, mapping);
}
glDisable (GL_BLEND);
glDepthMask(GL_TRUE);
}
//draw markers on foreground
for(std::map<int, tango_gl::Axis*>::const_iterator iter=markers_.begin(); iter!=markers_.end(); ++iter)
{

3
app/android/jni/scene.h
View File

@@ -74,7 +74,7 @@ class Scene {
// frame's timestamp.
// @param: point_cloud_vertices, point cloud's vertices of the current point
// frame.
int Render(const float * uvsTransformed = 0, glm::mat4 arViewMatrix = glm::mat4(0), glm::mat4 arProjectionMatrix=glm::mat4(0), const rtabmap::Mesh & occlusionMesh=rtabmap::Mesh());
int Render(const float * uvsTransformed = 0, glm::mat4 arViewMatrix = glm::mat4(0), glm::mat4 arProjectionMatrix=glm::mat4(0), const rtabmap::Mesh & occlusionMesh=rtabmap::Mesh(), bool mapping=false);
// Set render camera's viewing angle, first person, third person or top down.
//
@@ -155,6 +155,7 @@ class Scene {
float getPointSize() const {return pointSize_;}
bool isLighting() const {return lighting_;}
bool isBackfaceCulling() const {return backfaceCulling_;}
bool isWireframe() const {return wireFrame_;}
BackgroundRenderer * background_renderer_;

8
app/android/jni/tango-gl/gesture_camera.cpp
View File

@@ -183,7 +183,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
case kFirstPerson:
SetOrthoMode(false);
SetFieldOfView(kLowestFov);
SetNearFarClipPlanes(0.5, 50);
SetNearFarClipPlanes(0.3, 50);
SetPosition(glm::vec3(0.0f, 0.0f, 0.0f));
SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f));
cam_cur_dist_ = 0.0f;
@@ -198,7 +198,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
case kThirdPersonFollow:
SetOrthoMode(false);
SetFieldOfView(kLowFov);
SetNearFarClipPlanes(0.5, 50);
SetNearFarClipPlanes(1, 50);
SetPosition(glm::vec3(0.0f, 0.0f, 0.0f));
SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f));
cam_cur_dist_ = camera_index==kThirdPersonFollow?kThirdPersonFollowCameraDist:kThirdPersonCameraDist;
@@ -213,7 +213,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
SetRotation(glm::quat(1.0f, 0.0f, 0.0f, 0.0f));
SetOrthoMode(false);
SetFieldOfView(kLowFov);
SetNearFarClipPlanes(0.5, 50);
SetNearFarClipPlanes(1, 50);
cam_cur_dist_ = kTopDownCameraDist;
anchor_offset_ = glm::vec3(0.0f,0.0f,0.0f);
cam_cur_angle_.x = -M_PI / 2.0f;
@@ -228,7 +228,7 @@ void GestureCamera::SetCameraType(CameraType camera_index) {
SetOrthoScale(kTopDownCameraDist);
SetOrthoCropFactor(-1.0f);
SetFieldOfView(kLowFov);
SetNearFarClipPlanes(0.5, 50);
SetNearFarClipPlanes(1, 50);
cam_cur_dist_ = kTopDownCameraDist;
anchor_offset_ = glm::vec3(0.0f,0.0f,0.0f);
cam_cur_angle_.x = -M_PI / 2.0f;

14
app/ios/RTABMapApp/RTABMap.swift
View File

@@ -248,6 +248,12 @@ class RTABMap {
let quatv = GLKQuaternionMakeWithMatrix3(rotation)
let texX1 = (1-(2*frame.camera.intrinsics[0,0]/Float(frame.camera.imageResolution.width)) / p[0,0])/2
let texY1 = (1-(2*frame.camera.intrinsics[1,1]/Float(frame.camera.imageResolution.height)) / p[1,1])/2
let texX2 = (1-(2*frame.camera.intrinsics[0,0]/Float(frame.camera.imageResolution.width)) / p[1,1])/2
let texY2 = (1-(2*frame.camera.intrinsics[1,1]/Float(frame.camera.imageResolution.height)) / p[0,0])/2
//11 10 01 00 // portrait
//01 11 00 10 // right
//10 00 11 01 // left
@@ -255,13 +261,13 @@ class RTABMap {
var texCoord: [Float]
switch orientation {
case .portrait:
texCoord = [1, 1, 1, 0, 0, 1, 0, 0]
texCoord = [1-texX2, 1-texY2, 1-texX2,texY2, texX2, 1-texY2, texX2, texY2]
case .landscapeRight:
texCoord = [0, 1, 1, 1, 0, 0, 1, 0]
texCoord = [texX1, 1-texY1, 1-texX1, 1-texY1, texX1, texY1, 1-texX1, texY1]
case .landscapeLeft:
texCoord = [1, 0, 0, 0, 1, 1, 0, 1]
texCoord = [1-texX1, texY1, texX1, texY1, 1-texX1, 1-texY1, texX1, 1-texY1]
default: // down
texCoord = [0, 0, 0, 1, 1, 0, 1, 1]
texCoord = [texX2, texY2, texX2, 1-texY2, 1-texX2, texY2, 1-texX2, 1-texY2]
}
frame.rawFeaturePoints?.points.withUnsafeBufferPointer { bufferPoints in

46
app/ios/RTABMapApp/ViewController.swift
View File

@@ -105,7 +105,7 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP
@IBOutlet weak var exportOBJPLYButton: UIButton!
@IBOutlet weak var orthoDistanceSlider: UISlider!{
didSet{
orthoDistanceSlider.transform = CGAffineTransform(rotationAngle: CGFloat(-M_PI_2))
orthoDistanceSlider.transform = CGAffineTransform(rotationAngle: CGFloat(-Double.pi/2))
}
}
@IBOutlet weak var orthoGridSlider: UISlider!
@@ -275,11 +275,10 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP
self.showToast(message: "Optimized mesh detected in the database, it is shown while the database is loading...", seconds: 3)
}
let (usedMem, freeMem) = self.getMemoryUsage()
let usedMem = self.getMemoryUsage()
self.statusLabel.text =
"Status: " + (status == 1 && msg.isEmpty ? self.mState == State.STATE_CAMERA ? "Camera Preview" : "Idle" : msg) + "\n" +
"Used Memory: \(usedMem) MB\n" +
"Free Memory: \(freeMem) MB"
"Memory Usage: \(usedMem) MB"
}
}
@@ -312,7 +311,7 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP
pitch: Float,
yaw: Float)
{
let (usedMem, freeMem) = self.getMemoryUsage()
let usedMem = self.getMemoryUsage()
if(loopClosureId > 0)
{
@@ -335,8 +334,7 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP
self.statusLabel.text =
self.statusLabel.text! +
"Status: \(self.getStateString(state: self.mState))\n" +
"Used Memory (MB): \(usedMem)\n" +
"Free Memory (MB): \(freeMem)"
"Memory Usage : \(usedMem) MB"
}
if self.debugShown {
self.statusLabel.text =
@@ -403,29 +401,23 @@ class ViewController: GLKViewController, ARSessionDelegate, RTABMapObserver, UIP
}
}
func getMemoryUsage() -> (UInt64, UInt64) {
var pagesize: vm_size_t = 0
let host_port: mach_port_t = mach_host_self()
var host_size: mach_msg_type_number_t = mach_msg_type_number_t(MemoryLayout<vm_statistics_data_t>.stride / MemoryLayout<integer_t>.stride)
host_page_size(host_port, &pagesize)
var vm_stat: vm_statistics = vm_statistics_data_t()
withUnsafeMutablePointer(to: &vm_stat) { (vmStatPointer) -> Void in
vmStatPointer.withMemoryRebound(to: integer_t.self, capacity: Int(host_size)) {
if (host_statistics(host_port, HOST_VM_INFO, $0, &host_size) != KERN_SUCCESS) {
NSLog("Error: Failed to fetch vm statistics")
}
func getMemoryUsage() -> UInt64 {
var taskInfo = mach_task_basic_info()
var count = mach_msg_type_number_t(MemoryLayout<mach_task_basic_info>.size)/4
let kerr: kern_return_t = withUnsafeMutablePointer(to: &taskInfo) {
$0.withMemoryRebound(to: integer_t.self, capacity: 1) {
task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), $0, &count)
}
}
/* Stats in bytes */
let mem_used: UInt64 = UInt64(vm_stat.active_count +
vm_stat.inactive_count +
vm_stat.wire_count) * UInt64(pagesize)
let mem_free: UInt64 = UInt64(vm_stat.free_count) * UInt64(pagesize)
return (mem_used/(1024*1024), mem_free/(1024*1024))
if kerr == KERN_SUCCESS {
return taskInfo.resident_size / (1024*1024)
}
else {
print("Error with task_info(): " +
(String(cString: mach_error_string(kerr), encoding: String.Encoding.ascii) ?? "unknown error"))
return 0
}
}
@objc func appMovedToBackground() {