Tango: added online blending option, background color option, double tap, fixed meshes jiggering on touch, removed raw mesh option

This commit is contained in:
matlabbe
2017-04-17 21:42:01 -04:00
parent 6f148c47e2
commit 0ccba3281e
21 changed files with 563 additions and 220 deletions
+302 -93
View File
@@ -19,6 +19,7 @@
#include <rtabmap/utilite/ULogger.h>
#include <rtabmap/utilite/UStl.h>
#include <rtabmap/utilite/UTimer.h>
#include <rtabmap/core/util3d_filtering.h>
#include <pcl/common/transforms.h>
@@ -72,16 +73,42 @@ const std::string kPointCloudVertexShader =
" vColor = aColor;\n"
"}\n";
const std::string kPointCloudFragmentShader =
"precision mediump float;\n"
"precision highp float;\n"
"precision mediump int;\n"
"uniform float uGainR;\n"
"uniform float uGainG;\n"
"uniform float uGainB;\n"
"uniform sampler2D uDepthTexture;\n"
"uniform bool uBlending;\n"
"uniform vec2 uScreenScale;\n"
"uniform bool uPackDepthToColor;\n"
"varying vec3 vColor;\n"
"varying float vLightWeighting;\n"
"void main() {\n"
" vec4 textureColor = vec4(vColor.z, vColor.y, vColor.x, 1.0);\n"
" gl_FragColor = vec4(textureColor.r * uGainR * vLightWeighting, textureColor.g * uGainG * vLightWeighting, textureColor.b * uGainB * vLightWeighting, textureColor.a);\n"
" float alpha = 1.0;\n"
" if(uBlending) {\n"
" vec2 coord = uScreenScale * gl_FragCoord.xy;\n;"
" float depth = texture2D(uDepthTexture, coord).r;\n"
// " alpha = 0.5;\n"
" float zNear = 0.2;\n"
" float zFar = 1000.0;\n"
" float ndcDepth = depth * 2.0 - 1.0;\n" // Back to NDC
" float linearDepth = (2.0 * zNear * zFar) / (zFar + zNear - ndcDepth * (zFar - zNear));\n"
" float ndcFragz = gl_FragCoord.z * 2.0 - 1.0;\n" // Back to NDC
" float linearFragz = (2.0 * zNear * zFar) / (zFar + zNear - ndcFragz * (zFar - zNear));\n"
" if(linearFragz > linearDepth + 0.05)\n"
" alpha=0.0;\n"
" }\n"
" if(uPackDepthToColor) {\n"
" float toFixed = 255.0/256.0;\n"
" vec4 enc = vec4(1.0, 255.0, 65025.0, 160581375.0) * toFixed * gl_FragCoord.z;\n"
" enc = fract(enc);\n"
" gl_FragColor = enc;\n"
" }\n"
" else {"
" gl_FragColor = vec4(textureColor.r * uGainR * vLightWeighting, textureColor.g * uGainG * vLightWeighting, textureColor.b * uGainB * vLightWeighting, alpha);\n"
" }\n"
"}\n";
const std::string kTextureMeshVertexShader =
@@ -119,17 +146,36 @@ const std::string kTextureMeshVertexShader =
" }\n"
"}\n";
const std::string kTextureMeshFragmentShader =
"precision mediump float;\n"
"precision highp float;\n"
"precision mediump int;\n"
"uniform sampler2D uTexture;\n"
"uniform sampler2D uDepthTexture;\n"
"uniform float uGainR;\n"
"uniform float uGainG;\n"
"uniform float uGainB;\n"
"uniform bool uBlending;\n"
"uniform vec2 uScreenScale;\n"
"varying vec2 vTexCoord;\n"
"varying float vLightWeighting;\n"
""
"void main() {\n"
" vec4 textureColor = texture2D(uTexture, vTexCoord);\n"
" gl_FragColor = vec4(textureColor.r * uGainR * vLightWeighting, textureColor.g * uGainG * vLightWeighting, textureColor.b * uGainB * vLightWeighting, textureColor.a);\n"
" float alpha = 1.0;\n"
" if(uBlending) {\n"
" vec2 coord = uScreenScale * gl_FragCoord.xy;\n;"
" float depth = texture2D(uDepthTexture, coord).r;\n"
// " alpha = 0.5;\n"
//Linearize depth: http://stackoverflow.com/questions/6652253/getting-the-true-z-value-from-the-depth-buffer
" float zNear = 0.2;\n"
" float zFar = 1000.0;\n"
" float ndcDepth = depth * 2.0 - 1.0;\n" // Back to NDC
" float linearDepth = (2.0 * zNear * zFar) / (zFar + zNear - ndcDepth * (zFar - zNear));\n"
" float ndcFragz = gl_FragCoord.z * 2.0 - 1.0;\n" // Back to NDC
" float linearFragz = (2.0 * zNear * zFar) / (zFar + zNear - ndcFragz * (zFar - zNear));\n"
" if(linearFragz > linearDepth + 0.05)\n"
" alpha=0.0;\n"
" }\n"
" gl_FragColor = vec4(textureColor.r * uGainR * vLightWeighting, textureColor.g * uGainG * vLightWeighting, textureColor.b * uGainB * vLightWeighting, alpha);\n"
"}\n";
const std::string kGraphVertexShader =
@@ -169,17 +215,22 @@ Scene::Scene() :
cloud_shader_program_(0),
texture_mesh_shader_program_(0),
graph_shader_program_(0),
blending_(true),
mapRendering_(true),
meshRendering_(true),
meshRenderingTexture_(true),
pointSize_(5.0f),
frustumCulling_(true),
boundingBoxRendering_(false),
lighting_(false),
backfaceCulling_(true),
r_(0.0f),
g_(0.0f),
b_(0.0f)
b_(0.0f),
fboId_(0),
depthTexture_(0),
screenWidth_(0),
screenHeight_(0),
doubleTapOn_(false)
{
gesture_camera_ = new tango_gl::GestureCamera();
gesture_camera_->SetCameraType(
@@ -242,27 +293,35 @@ void Scene::DeleteResources() {
LOGI("Scene::DeleteResources()");
if(axis_)
{
delete axis_;
axis_ = 0;
delete frustum_;
delete trace_;
delete grid_;
delete currentPose_;
delete box_;
delete axis_;
axis_ = 0;
delete frustum_;
delete trace_;
delete grid_;
delete currentPose_;
delete box_;
}
if (cloud_shader_program_) {
glDeleteShader(cloud_shader_program_);
cloud_shader_program_ = 0;
}
}
if (texture_mesh_shader_program_) {
glDeleteShader(texture_mesh_shader_program_);
texture_mesh_shader_program_ = 0;
}
}
if (graph_shader_program_) {
glDeleteShader(graph_shader_program_);
graph_shader_program_ = 0;
}
}
if(fboId_>0)
{
glDeleteFramebuffers(1, &fboId_);
fboId_ = 0;
glDeleteTextures(1, &depthTexture_);
depthTexture_ = 0;
}
clear();
}
@@ -289,13 +348,49 @@ void Scene::clear()
//Should only be called in OpenGL thread!
void Scene::SetupViewPort(int w, int h) {
if (h == 0) {
LOGE("Setup graphic height not valid");
}
UASSERT(gesture_camera_ != 0);
gesture_camera_->SetAspectRatio(static_cast<float>(w) /
static_cast<float>(h));
glViewport(0, 0, w, h);
if (h == 0) {
LOGE("Setup graphic height not valid");
}
UASSERT(gesture_camera_ != 0);
gesture_camera_->SetAspectRatio(static_cast<float>(w) / static_cast<float>(h));
glViewport(0, 0, w, h);
if(screenWidth_ != w || fboId_ == 0)
{
if(fboId_>0)
{
glDeleteFramebuffers(1, &fboId_);
fboId_ = 0;
glDeleteTextures(1, &depthTexture_);
depthTexture_ = 0;
}
// Create depth texture
glGenTextures(1, &depthTexture_);
glBindTexture(GL_TEXTURE_2D, depthTexture_);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, w, h, 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
// regenerate fbo texture
// create a framebuffer object, you need to delete them when program exits.
glGenFramebuffers(1, &fboId_);
glBindFramebuffer(GL_FRAMEBUFFER, fboId_);
// Set the texture to be at the depth attachment point of the FBO
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depthTexture_, 0);
GLuint status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if ( status != GL_FRAMEBUFFER_COMPLETE)
{
LOGE("Frame buffer cannot be generated! Status: %in", status);
}
glBindFramebuffer(GL_FRAMEBUFFER,0);
}
screenWidth_ = w;
screenHeight_ = h;
}
std::vector<glm::vec4> computeFrustumPlanes(const glm::mat4 & mat, bool normalize = true)
@@ -409,26 +504,12 @@ bool intersectFrustumAABB(
int Scene::Render() {
UASSERT(gesture_camera_ != 0);
glEnable(GL_DEPTH_TEST);
if(backfaceCulling_)
{
glEnable(GL_CULL_FACE);
}
else
{
glDisable(GL_CULL_FACE);
}
glClearColor(r_, g_, b_, 1.0f);
glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
glm::vec3 position(currentPose_->x(), currentPose_->y(), currentPose_->z());
Eigen::Quaternionf quat = currentPose_->getQuaternionf();
glm::quat rotation(quat.w(), quat.x(), quat.y(), quat.z());
glm::mat4 rotateM;
if(!currentPose_->isNull())
{
glm::vec3 position(currentPose_->x(), currentPose_->y(), currentPose_->z());
Eigen::Quaternionf quat = currentPose_->getQuaternionf();
glm::quat rotation(quat.w(), quat.x(), quat.y(), quat.z());
glm::mat4 rotateM;
rotateM = glm::rotate<float>(float(color_camera_to_display_rotation_)*-1.57079632679489661923132169163975144, glm::vec3(0.0f, 0.0f, 1.0f));
if (gesture_camera_->GetCameraType() == tango_gl::GestureCamera::kFirstPerson)
@@ -441,47 +522,52 @@ int Scene::Render() {
{
// In third person or top down mode, we follow the camera movement.
gesture_camera_->SetAnchorPosition(position, rotation*glm::quat(rotateM));
frustum_->SetPosition(position);
frustum_->SetRotation(rotation);
// Set the frustum scale to 4:3, this doesn't necessarily match the physical
// camera's aspect ratio, this is just for visualization purposes.
frustum_->SetScale(kFrustumScale);
frustum_->Render(gesture_camera_->GetProjectionMatrix(),
gesture_camera_->GetViewMatrix());
axis_->SetPosition(position);
axis_->SetRotation(rotation);
axis_->Render(gesture_camera_->GetProjectionMatrix(),
gesture_camera_->GetViewMatrix());
}
trace_->UpdateVertexArray(position);
if(traceVisible_)
{
trace_->Render(gesture_camera_->GetProjectionMatrix(),
gesture_camera_->GetViewMatrix());
}
if(gridVisible_)
{
grid_->Render(gesture_camera_->GetProjectionMatrix(),
gesture_camera_->GetViewMatrix());
}
}
float fov = 45.0f;
rtabmap::Transform openglCamera = GetOpenGLCameraPose(&fov);//*rtabmap::Transform(0.0f, 0.0f, 3.0f, 0.0f, 0.0f, 0.0f);
glm::mat4 projectionMatrix = gesture_camera_->GetProjectionMatrix();
glm::mat4 viewMatrix = gesture_camera_->GetViewMatrix();
rtabmap::Transform openglCamera = GetOpenGLCameraPose();//*rtabmap::Transform(0.0f, 0.0f, 3.0f, 0.0f, 0.0f, 0.0f);
// transform in same coordinate as frustum filtering
openglCamera *= rtabmap::Transform(
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f,
-1.0f, 0.0f, 0.0f, 0.0f);
int cloudDrawn=0;
if(mapRendering_ && frustumCulling_)
std::vector<glm::vec4> planes = computeFrustumPlanes(projectionMatrix*viewMatrix, true);
// First rendering to get depth texture
glEnable(GL_DEPTH_TEST);
glDepthFunc(GL_LESS);
glDepthMask(GL_TRUE);
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
glDisable (GL_BLEND);
glBlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
if(backfaceCulling_)
{
std::vector<glm::vec4> planes = computeFrustumPlanes(gesture_camera_->GetProjectionMatrix()*gesture_camera_->GetViewMatrix(), true);
glEnable(GL_CULL_FACE);
}
else
{
glDisable(GL_CULL_FACE);
}
UTimer timer;
bool onlineBlending = blending_ && mapRendering_ && meshRendering_ && pointClouds_.size()>1;
std::set<int> usedForDepth;
if(onlineBlending && fboId_)
{
// set the rendering destination to FBO
glBindFramebuffer(GL_FRAMEBUFFER, fboId_);
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glClearColor(1, 1, 1, 1);
glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
// Draw scene
for(std::map<int, PointCloudDrawable*>::const_iterator iter=pointClouds_.begin(); iter!=pointClouds_.end(); ++iter)
{
if(iter->second->isVisible())
@@ -490,29 +576,136 @@ int Scene::Render() {
iter->second->aabbMinWorld(),
iter->second->aabbMaxWorld()))
{
if(boundingBoxRendering_)
{
box_->updateVertices(iter->second->aabbMinWorld(), iter->second->aabbMaxWorld());
box_->Render(gesture_camera_->GetProjectionMatrix(),
gesture_camera_->GetViewMatrix());
}
++cloudDrawn;
usedForDepth.insert(iter->first);
Eigen::Vector3f cloudToCamera(
iter->second->getPose().x() - openglCamera.x(),
iter->second->getPose().y() - openglCamera.y(),
iter->second->getPose().z() - openglCamera.z());
float distanceToCameraSqr = cloudToCamera[0]*cloudToCamera[0] + cloudToCamera[1]*cloudToCamera[1] + cloudToCamera[2]*cloudToCamera[2];
iter->second->Render(gesture_camera_->GetProjectionMatrix(), gesture_camera_->GetViewMatrix(), meshRendering_, pointSize_, meshRenderingTexture_, lighting_, distanceToCameraSqr);
float distanceToCameraSqr = 999.0f; // set it large to use low res polygons for fast processing
iter->second->Render(projectionMatrix, viewMatrix, meshRendering_, pointSize_, false, false, distanceToCameraSqr);
}
}
}
// back to normal window-system-provided framebuffer
glBindFramebuffer(GL_FRAMEBUFFER, 0); // unbind
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
}
if(doubleTapOn_ && gesture_camera_->GetCameraType() != tango_gl::GestureCamera::kFirstPerson)
{
glClearColor(0, 0, 0, 0);
glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
for(std::map<int, PointCloudDrawable*>::const_iterator iter=pointClouds_.begin(); iter!=pointClouds_.end(); ++iter)
{
if((onlineBlending && usedForDepth.find(iter->first) != usedForDepth.end()) ||
(!onlineBlending && iter->second->isVisible() &&
intersectFrustumAABB(planes,
iter->second->aabbMinWorld(),
iter->second->aabbMaxWorld())))
{
float distanceToCameraSqr = 999.0f; // set it large to use low res polygons for fast processing
iter->second->Render(projectionMatrix, viewMatrix, meshRendering_, pointSize_*10.0f, false, false, distanceToCameraSqr, 0, 0, 0, true);
}
}
GLubyte zValue[4];
glReadPixels(doubleTapPos_.x*screenWidth_, screenHeight_-doubleTapPos_.y*screenHeight_, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, zValue);
float fromFixed = 256.0f/255.0f;
float zValueF = float(zValue[0]/255.0f)*fromFixed + float(zValue[1]/255.0f)*fromFixed/255.0f + float(zValue[2]/255.0f)*fromFixed/65025.0f + float(zValue[3]/255.0f)*fromFixed/160581375.0f;
if(zValueF != 0.0f)
{
zValueF = zValueF*2.0-1.0;//NDC
glm::vec4 point = glm::inverse(projectionMatrix*viewMatrix)*glm::vec4(doubleTapPos_.x*2.0f-1.0f, (1.0f-doubleTapPos_.y)*2.0f-1.0f, zValueF, 1.0f);
point /= point.w;
gesture_camera_->SetAnchorOffset(glm::vec3(point.x, point.y, point.z) - position);
}
}
doubleTapOn_ = false;
glClearColor(r_, g_, b_, 1.0f);
glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
if(!currentPose_->isNull())
{
if (gesture_camera_->GetCameraType() != tango_gl::GestureCamera::kFirstPerson)
{
frustum_->SetPosition(position);
frustum_->SetRotation(rotation);
// Set the frustum scale to 4:3, this doesn't necessarily match the physical
// camera's aspect ratio, this is just for visualization purposes.
frustum_->SetScale(kFrustumScale);
frustum_->Render(projectionMatrix, viewMatrix);
axis_->SetPosition(position);
axis_->SetRotation(rotation);
axis_->Render(projectionMatrix, viewMatrix);
}
trace_->UpdateVertexArray(position);
if(traceVisible_)
{
trace_->Render(projectionMatrix, viewMatrix);
}
if(gridVisible_)
{
grid_->Render(projectionMatrix, viewMatrix);
}
}
if(graphVisible_ && graph_)
{
graph_->Render(projectionMatrix, viewMatrix);
}
int cloudDrawn=0;
if(mapRendering_)
{
if(onlineBlending)
{
glEnable (GL_BLEND);
glDepthMask(GL_FALSE);
}
for(std::map<int, PointCloudDrawable*>::const_iterator iter=pointClouds_.begin(); iter!=pointClouds_.end(); ++iter)
{
if((onlineBlending && usedForDepth.find(iter->first) != usedForDepth.end()) ||
(!onlineBlending && iter->second->isVisible() &&
intersectFrustumAABB(planes,
iter->second->aabbMinWorld(),
iter->second->aabbMaxWorld())))
{
if(boundingBoxRendering_)
{
box_->updateVertices(iter->second->aabbMinWorld(), iter->second->aabbMaxWorld());
box_->Render(projectionMatrix, viewMatrix);
}
Eigen::Vector3f cloudToCamera(
iter->second->getPose().x() - openglCamera.x(),
iter->second->getPose().y() - openglCamera.y(),
iter->second->getPose().z() - openglCamera.z());
float distanceToCameraSqr = cloudToCamera[0]*cloudToCamera[0] + cloudToCamera[1]*cloudToCamera[1] + cloudToCamera[2]*cloudToCamera[2];
iter->second->Render(projectionMatrix, viewMatrix, meshRendering_, pointSize_, meshRenderingTexture_, lighting_, distanceToCameraSqr, onlineBlending?depthTexture_:0, screenWidth_, screenHeight_);
++cloudDrawn;
}
}
if(onlineBlending)
{
glDisable (GL_BLEND);
glDepthMask(GL_TRUE);
}
}
else
{
for(std::map<int, PointCloudDrawable*>::const_iterator iter=pointClouds_.begin(); iter!=pointClouds_.end(); ++iter)
{
if(!mapRendering_ && iter->first > 0)
if(iter->first > 0)
{
break;
}
@@ -522,8 +715,7 @@ int Scene::Render() {
if(boundingBoxRendering_)
{
box_->updateVertices(iter->second->aabbMinWorld(), iter->second->aabbMaxWorld());
box_->Render(gesture_camera_->GetProjectionMatrix(),
gesture_camera_->GetViewMatrix());
box_->Render(projectionMatrix, viewMatrix);
}
++cloudDrawn;
@@ -532,17 +724,12 @@ int Scene::Render() {
iter->second->getPose().y() - openglCamera.y(),
iter->second->getPose().z() - openglCamera.z());
float distanceToCameraSqr = cloudToCamera[0]*cloudToCamera[0] + cloudToCamera[1]*cloudToCamera[1] + cloudToCamera[2]*cloudToCamera[2];
iter->second->Render(gesture_camera_->GetProjectionMatrix(), gesture_camera_->GetViewMatrix(), meshRendering_, pointSize_, meshRenderingTexture_, lighting_, distanceToCameraSqr);
iter->second->Render(projectionMatrix, viewMatrix, meshRendering_, pointSize_, meshRenderingTexture_, lighting_, distanceToCameraSqr, onlineBlending?depthTexture_:0, screenWidth_, screenHeight_);
}
}
}
if(graphVisible_ && graph_)
{
graph_->Render(gesture_camera_->GetProjectionMatrix(), gesture_camera_->GetViewMatrix());
}
return cloudDrawn;
return 1;
}
void Scene::SetCameraType(tango_gl::GestureCamera::CameraType camera_type) {
@@ -570,7 +757,21 @@ void Scene::OnTouchEvent(int touch_count,
tango_gl::GestureCamera::TouchEvent event, float x0,
float y0, float x1, float y1) {
UASSERT(gesture_camera_ != 0);
gesture_camera_->OnTouchEvent(touch_count, event, x0, y0, x1, y1);
if(touch_count == 3)
{
//doubletap
if(!doubleTapOn_)
{
doubleTapPos_.x = x0;
doubleTapPos_.y = y0;
doubleTapOn_ = true;
}
}
else
{
// rotate/translate/zoom
gesture_camera_->OnTouchEvent(touch_count, event, x0, y0, x1, y1);
}
}
void Scene::updateGraph(
@@ -722,3 +923,11 @@ void Scene::updateGains(int id, float gainR, float gainG, float gainB)
iter->second->setGains(gainR, gainG, gainB);
}
}
void Scene::setGridColor(float r, float g, float b)
{
if(grid_)
{
grid_->SetColor(r, g, b);
}
}