From 95545979481fe3dc924f1566c7be8e245fce699f Mon Sep 17 00:00:00 2001 From: matlabbe Date: Sat, 10 Jun 2017 21:45:51 -0400 Subject: [PATCH] Tango: added multi-texturing support (#189) --- app/android/jni/RTABMapApp.cpp | 161 +++++++++++++----- app/android/jni/RTABMapApp.h | 4 +- app/android/jni/jni_interface.cpp | 2 + app/android/res/layout/activity_settings.xml | 8 + app/android/res/values/strings.xml | 24 +++ .../com/introlab/rtabmap/RTABMapActivity.java | 73 +++++--- .../src/com/introlab/rtabmap/RTABMapLib.java | 1 + .../introlab/rtabmap/SettingsActivity.java | 2 + 8 files changed, 215 insertions(+), 60 deletions(-) diff --git a/app/android/jni/RTABMapApp.cpp b/app/android/jni/RTABMapApp.cpp index bd2224be..a65d7a65 100644 --- a/app/android/jni/RTABMapApp.cpp +++ b/app/android/jni/RTABMapApp.cpp @@ -1873,14 +1873,15 @@ double sqr(uchar v) return double(v)*double(v); } -cv::Mat RTABMapApp::mergeTextures( +std::vector RTABMapApp::mergeTextures( pcl::TextureMesh & mesh, int textureSize, + int textureCount, const std::vector > & vertexToPixels) const { UASSERT(textureSize> 0); - LOGD("textureSize = %d materials=%d", textureSize, mesh.tex_materials.size()); - cv::Mat globalTexture; + LOGD("textureSize = %d textureCount=%d materials=%d", textureSize, textureCount, mesh.tex_materials.size()); + std::vector globalTextures; if(mesh.tex_materials.size() >= 1) { std::vector textures(mesh.tex_materials.size(), -1); @@ -1914,14 +1915,19 @@ cv::Mat RTABMapApp::mergeTextures( { float scale = 0.0f; std::vector materialsKept; - rtabmap::util3d::concatenateTextureMaterials(mesh, imageSize, textureSize, 1, scale, &materialsKept); + rtabmap::util3d::concatenateTextureMaterials(mesh, imageSize, textureSize, textureCount, scale, &materialsKept); LOGD("scale=%f materials=%d", scale, (int)mesh.tex_materials.size()); - if(scale && mesh.tex_materials.size()==1) + if(scale && mesh.tex_materials.size()) { + int materials = (int)mesh.tex_materials.size(); int cols = float(textureSize)/(scale*imageSize.width); + int rows = float(textureSize)/(scale*imageSize.height); - globalTexture = cv::Mat(textureSize, textureSize, imageType, cv::Scalar::all(255)); - cv::Mat globalTextureMask = cv::Mat(textureSize, textureSize, CV_8UC1, cv::Scalar::all(0)); + globalTextures.resize(materials); + for(int i=0; i(oi); cv::Mat_ N(num_images, num_images); N.setTo(0); @@ -2019,13 +2028,15 @@ cv::Mat RTABMapApp::mergeTextures( N(i, j) += 1; N(j, i) += 1; + int indexMaterial = i / (cols*rows); + // uv in globalTexture int ui = iter->second.x*emptyImage.cols + imageOrigin[iter->first].x; int vi = (1.0-iter->second.y)*emptyImage.rows + imageOrigin[iter->first].y; int uj = jter->second.x*emptyImage.cols + imageOrigin[jter->first].x; int vj = (1.0-jter->second.y)*emptyImage.rows + imageOrigin[jter->first].y; - cv::Vec3b * pt1 = globalTexture.ptr(vi,ui); - cv::Vec3b * pt2 = globalTexture.ptr(vj,uj); + cv::Vec3b * pt1 = globalTextures[indexMaterial].ptr(vi,ui); + cv::Vec3b * pt2 = globalTextures[indexMaterial].ptr(vj,uj); I(i, j) += std::sqrt(static_cast(sqr(pt1->val[0]) + sqr(pt1->val[1]) + sqr(pt1->val[2]))); I(j, i) += std::sqrt(static_cast(sqr(pt2->val[0]) + sqr(pt2->val[1]) + sqr(pt2->val[2]))); @@ -2044,9 +2055,16 @@ cv::Mat RTABMapApp::mergeTextures( for(int i=0; i channels; cv::split(roi, channels); @@ -2127,6 +2146,7 @@ cv::Mat RTABMapApp::mergeTextures( progressionStatus_.increment(); // blending BGR + LOGD("blending"); int decimation = 0; // determinate decimation to apply @@ -2168,7 +2188,13 @@ cv::Mat RTABMapApp::mergeTextures( if(decimation>0) { - cv::Mat blendGains(globalTexture.rows/decimation, globalTexture.cols/decimation, CV_32FC3, cv::Scalar::all(1.0f)); + LOGD("blending decimation=%d", decimation); + std::vector blendGains(materials); + for(int i=0; i 1) @@ -2191,7 +2217,8 @@ cv::Mat RTABMapApp::mergeTextures( { weight = 0.0f; } - cv::Vec3b * pt = globalTexture.ptr(v,u); + int indexMaterial = newCamIndex[iter->first] / (cols*rows); + cv::Vec3b * pt = globalTextures[indexMaterial].ptr(v,u); gainsB[k] = static_cast(pt->val[0]) * weight; gainsG[k] = static_cast(pt->val[1]) * weight; gainsR[k] = static_cast(pt->val[2]) * weight; @@ -2215,11 +2242,12 @@ cv::Mat RTABMapApp::mergeTextures( { int u = iter->second.x*emptyImage.cols + imageOrigin[iter->first].x; int v = (1.0-iter->second.y)*emptyImage.rows + imageOrigin[iter->first].y; - cv::Vec3b * pt = globalTexture.ptr(v,u); + int indexMaterial = newCamIndex[iter->first] / (cols*rows); + cv::Vec3b * pt = globalTextures[indexMaterial].ptr(v,u); float gB = targetColor[0]/(pt->val[0]==0?1.0f:pt->val[0]); float gG = targetColor[1]/(pt->val[1]==0?1.0f:pt->val[1]); float gR = targetColor[2]/(pt->val[2]==0?1.0f:pt->val[2]); - cv::Vec3f * ptr = blendGains.ptr(v/decimation, u/decimation); + cv::Vec3f * ptr = blendGains[indexMaterial].ptr(v/decimation, u/decimation); ptr->val[0] = (gB>1.3f)?1.3f:(gB<0.7f)?0.7f:gB; ptr->val[1] = (gG>1.3f)?1.3f:(gG<0.7f)?0.7f:gG; ptr->val[2] = (gR>1.3f)?1.3f:(gR<0.7f)?0.7f:gR; @@ -2229,11 +2257,15 @@ cv::Mat RTABMapApp::mergeTextures( } } - cv::Mat dst; - cv::blur(blendGains, dst, cv::Size(3,3)); - cv::resize(dst, blendGains, globalTexture.size(), 0, 0, cv::INTER_LINEAR); + LOGD("blending multiply"); + for(int i=0; i > vertexToPixels; - cv::Mat globalTexture; + std::vector globalTextures; int totalPolygons = 0; { if(optimized) @@ -3046,7 +3079,7 @@ bool RTABMapApp::exportMesh( if(textureSize>0 && totalPolygons && textureMesh->tex_materials.size()) { LOGI("Merging %d textures...", (int)textureMesh->tex_materials.size()); - globalTexture = mergeTextures(*textureMesh, textureSize, vertexToPixels); + globalTextures = mergeTextures(*textureMesh, textureSize, textureCount, vertexToPixels); if(progressionStatus_.isCanceled()) { @@ -3058,18 +3091,28 @@ bool RTABMapApp::exportMesh( return false; } + LOGD("Saving texture(s) (%d)", (int)globalTextures.size()); std::string baseName = uSplit(UFile::getName(filePath), '.').front(); std::string textureDirectory = UDirectory::getDir(filePath); - std::string fullPath = textureDirectory+UDirectory::separator()+baseName+".jpg"; - textureMesh->tex_materials[0].tex_file = baseName+".jpg"; - LOGI("Saving texture to %s.", fullPath.c_str()); - if(!cv::imwrite(fullPath, globalTexture)) + UASSERT(textureMesh->tex_materials.size() == globalTextures.size()); + for(unsigned int i=0; itex_materials.size(); ++i) { - LOGI("Failed saving %s!", fullPath.c_str()); - } - else - { - LOGI("Saved %s (%d bytes).", fullPath.c_str(), globalTexture.total()*globalTexture.channels()); + std::string baseNameNum = baseName; + if(textureMesh->tex_materials.size()>1) + { + baseNameNum+=uNumber2Str(i); + } + std::string fullPath = textureDirectory+UDirectory::separator()+baseNameNum+".jpg"; + textureMesh->tex_materials[i].tex_file = baseNameNum+".jpg"; + LOGI("Saving texture to %s.", fullPath.c_str()); + if(!cv::imwrite(fullPath, globalTextures[i])) + { + LOGI("Failed saving %s!", fullPath.c_str()); + } + else + { + LOGI("Saved %s (%d bytes).", fullPath.c_str(), globalTextures[i].total()*globalTextures[i].channels()); + } } } if(progressionStatus_.isCanceled()) @@ -3108,14 +3151,54 @@ bool RTABMapApp::exportMesh( } else if(textureMesh->tex_materials.size()) { - UASSERT(textureMesh->tex_polygons.size() && (int)textureMesh->tex_polygons[0].size() == totalPolygons); LOGI("Saving obj (%d vertices, %d polygons) to %s.", (int)textureMesh->cloud.data.size()/textureMesh->cloud.point_step, totalPolygons, filePath.c_str()); success = pcl::io::saveOBJFile(filePath, *textureMesh) == 0; if(success) { LOGI("Saved obj to %s!", filePath.c_str()); exportedMesh_ = textureMesh; - exportedTexture_ = globalTexture; + if(globalTextures.size() == 1) + { + exportedTexture_ = globalTextures[0]; + } + else if(globalTextures.size() > 1) + { + // Visualization doesn't support more than one material, so concatenate to 1 + std::vector materialsKept; + float scale = 0.0f; + cv::Size imageSize = globalTextures[0].size(); + int imageType = CV_8UC3; + rtabmap::util3d::concatenateTextureMaterials(*exportedMesh_, imageSize, textureSize, 1, scale, &materialsKept); + if(scale && exportedMesh_->tex_materials.size() == 1) + { + int cols = float(textureSize)/(scale*imageSize.width); + int rows = float(textureSize)/(scale*imageSize.height); + + exportedTexture_ = cv::Mat(textureSize, textureSize, imageType, cv::Scalar::all(255)); + + // make a blank texture + cv::Size resizedImageSize(int(imageSize.width*scale), int(imageSize.height*scale)); + int oi=0; + for(int i=0; i<(int)globalTextures.size(); ++i) + { + if(materialsKept.at(i)) + { + int u = oi%cols * resizedImageSize.width; + int v = ((oi/cols) % rows ) * resizedImageSize.height; + UASSERT(u < textureSize-resizedImageSize.width); + UASSERT(v < textureSize-resizedImageSize.height); + + cv::Mat resizedImage; + cv::resize(globalTextures[i], resizedImage, resizedImageSize, 0.0f, 0.0f, cv::INTER_AREA); + + UASSERT(resizedImage.type() == exportedTexture_.type()); + resizedImage.copyTo(exportedTexture_(cv::Rect(u, v, resizedImage.cols, resizedImage.rows))); + + ++oi; + } + } + } + } } else { diff --git a/app/android/jni/RTABMapApp.h b/app/android/jni/RTABMapApp.h index 50227a73..865815ca 100644 --- a/app/android/jni/RTABMapApp.h +++ b/app/android/jni/RTABMapApp.h @@ -146,8 +146,9 @@ class RTABMapApp : public UEventsHandler { void resetMapping(); void save(const std::string & databasePath); - cv::Mat mergeTextures(pcl::TextureMesh & mesh, + std::vector mergeTextures(pcl::TextureMesh & mesh, int textureSize, + int textureCount, const std::vector > & vertexToPixels) const; void cancelProcessing(); bool exportMesh( @@ -156,6 +157,7 @@ class RTABMapApp : public UEventsHandler { bool regenerateCloud, bool meshing, int textureSize, + int textureCount, int normalK, bool optimized, float optimizedVoxelSize, diff --git a/app/android/jni/jni_interface.cpp b/app/android/jni/jni_interface.cpp index d98e2174..4f2ae04c 100644 --- a/app/android/jni/jni_interface.cpp +++ b/app/android/jni/jni_interface.cpp @@ -335,6 +335,7 @@ Java_com_introlab_rtabmap_RTABMapLib_exportMesh( bool regenerateCloud, bool meshing, int textureSize, + int textureCount, int normalK, bool optimized, float optimizedVoxelSize, @@ -355,6 +356,7 @@ Java_com_introlab_rtabmap_RTABMapLib_exportMesh( regenerateCloud, meshing, textureSize, + textureCount, normalK, optimized, optimizedVoxelSize, diff --git a/app/android/res/layout/activity_settings.xml b/app/android/res/layout/activity_settings.xml index d2f76155..a8554229 100644 --- a/app/android/res/layout/activity_settings.xml +++ b/app/android/res/layout/activity_settings.xml @@ -227,6 +227,14 @@ android:entryValues="@array/pref_texture_size_values" android:defaultValue="@string/pref_default_texture_size"/> + + 0.01 pref_key_texture_size 4096 + pref_key_texture_count + 1 pref_key_normal_k 18 pref_key_max_texture_distance @@ -565,6 +567,8 @@ If you don\'t need a very precise point cloud, you can set this to reduce the output point cloud size. This is also used for optimized mesh. Texture Size If the map is large, you may want to increase this to maximize the texture resolution. + Maximum Output Textures + Note that all textures are still merged into one for visualization, but exported in multiple files. Normal K K-nearest neighbors used for normal computation when a mesh is created. Max Texture Distance @@ -606,6 +610,26 @@ "2048" "1024" + + "8" + "7" + "6" + "5" + "4" + "3" + "2" + "1" + + + "8" + "7" + "6" + "5" + "4" + "3" + "2" + "1" + "30" "24" diff --git a/app/android/src/com/introlab/rtabmap/RTABMapActivity.java b/app/android/src/com/introlab/rtabmap/RTABMapActivity.java index 57830a5c..87a34bf4 100644 --- a/app/android/src/com/introlab/rtabmap/RTABMapActivity.java +++ b/app/android/src/com/introlab/rtabmap/RTABMapActivity.java @@ -109,7 +109,7 @@ public class RTABMapActivity extends Activity implements OnClickListener { public static final String EXTRA_VALUE_ADF = "ADF_LOAD_SAVE_PERMISSION"; public static final String RTABMAP_TMP_DB = "rtabmap.tmp.db"; - public static final String RTABMAP_TMP_DIR = "tmp/"; + public static final String RTABMAP_TMP_DIR = "tmp"; public static final String RTABMAP_TMP_FILENAME = "map"; public static final String RTABMAP_SDCARD_PATH = "/sdcard/"; public static final String RTABMAP_EXPORT_DIR = "Export/"; @@ -627,6 +627,7 @@ public class RTABMapActivity extends Activity implements OnClickListener { break; case R.id.close_visualization_button: updateState(State.STATE_IDLE); + RTABMapLib.postExportation(false); break; case R.id.button_saveOnDevice: saveOnDevice(); @@ -1094,7 +1095,7 @@ public class RTABMapActivity extends Activity implements OnClickListener { private RTABMapActivity getActivity() {return this;} - private String[] loadFileList(String directory) { + private String[] loadFileList(String directory, final boolean databasesOnly) { File path = new File(directory); String fileList[]; try { @@ -1109,7 +1110,14 @@ public class RTABMapActivity extends Activity implements OnClickListener { @Override public boolean accept(File dir, String filename) { File sel = new File(dir, filename); - return filename.compareTo(RTABMAP_TMP_DB) != 0 && filename.endsWith(".db"); + if(databasesOnly) + { + return filename.compareTo(RTABMAP_TMP_DB) != 0 && filename.endsWith(".db"); + } + else + { + return sel.isFile(); + } } }; @@ -1242,7 +1250,6 @@ public class RTABMapActivity extends Activity implements OnClickListener { mItemModes.setEnabled(true); mButtonPause.setVisibility(mHudVisible?View.VISIBLE:View.INVISIBLE); mItemDataRecorderMode.setEnabled(mButtonPause.isChecked()); - RTABMapLib.postExportation(false); break; } mButtonFirst.setVisibility(mHudVisible?View.VISIBLE:View.INVISIBLE); @@ -1713,7 +1720,7 @@ public class RTABMapActivity extends Activity implements OnClickListener { } else if(itemId == R.id.open) { - final String[] files = loadFileList(mWorkingDirectory); + final String[] files = loadFileList(mWorkingDirectory, true); if(files.length > 0) { String[] filesWithSize = new String[files.length]; @@ -1774,6 +1781,7 @@ public class RTABMapActivity extends Activity implements OnClickListener { final String cloudVoxelSizeStr = sharedPref.getString(getString(R.string.pref_key_cloud_voxel), getString(R.string.pref_default_cloud_voxel)); final float cloudVoxelSize = Float.parseFloat(cloudVoxelSizeStr); final int textureSize = isOBJ?Integer.parseInt(sharedPref.getString(getString(R.string.pref_key_texture_size), getString(R.string.pref_default_texture_size))):0; + final int textureCount = Integer.parseInt(sharedPref.getString(getString(R.string.pref_key_texture_count), getString(R.string.pref_default_texture_count))); final int normalK = Integer.parseInt(sharedPref.getString(getString(R.string.pref_key_normal_k), getString(R.string.pref_default_normal_k))); final float maxTextureDistance = Float.parseFloat(sharedPref.getString(getString(R.string.pref_key_max_texture_distance), getString(R.string.pref_default_max_texture_distance))); final int minTextureClusterSize = Integer.parseInt(sharedPref.getString(getString(R.string.pref_key_min_texture_cluster_size), getString(R.string.pref_default_min_texture_cluster_size))); @@ -1793,10 +1801,24 @@ public class RTABMapActivity extends Activity implements OnClickListener { mExportProgressDialog.show(); updateState(State.STATE_PROCESSING); - final String tmpPath = mWorkingDirectory + RTABMAP_TMP_DIR + RTABMAP_TMP_FILENAME + extension; + final String tmpPath = mWorkingDirectory + RTABMAP_TMP_DIR + "/" + RTABMAP_TMP_FILENAME + extension; File tmpDir = new File(mWorkingDirectory + RTABMAP_TMP_DIR); tmpDir.mkdirs(); + String[] fileNames = loadFileList(mWorkingDirectory + RTABMAP_TMP_DIR, false); + if(!DISABLE_LOG) Log.i(TAG, String.format("Deleting %d files in \"%s\"", fileNames.length, mWorkingDirectory + RTABMAP_TMP_DIR)); + for(int i=0; i 0) { - Util.zip(filesToZip, zipOutput); - mToast.makeText(getActivity(), String.format("Mesh \"%s\" successfully exported!", pathHuman), mToast.LENGTH_LONG).show(); + String[] filesToZip = new String[fileNames.length]; + for(int i=0; i