Texturing: using depth image to filter wrongly projected textures

This commit is contained in:
matlabbe
2017-06-06 11:42:42 -04:00
parent fa95ede3e9
commit 7a0a82bcb4
6 changed files with 376 additions and 131 deletions

View File

@@ -170,6 +170,8 @@ ExportCloudsDialog::ExportCloudsDialog(QWidget *parent) :
connect(_ui->comboBox_meshingTextureSize, SIGNAL(currentIndexChanged(int)), this, SIGNAL(configChanged()));
connect(_ui->spinBox_mesh_maxTextures, SIGNAL(valueChanged(int)), this, SIGNAL(configChanged()));
connect(_ui->doubleSpinBox_meshingTextureMaxDistance, SIGNAL(valueChanged(double)), this, SIGNAL(configChanged()));
connect(_ui->doubleSpinBox_meshingTextureMaxDepthError, SIGNAL(valueChanged(double)), this, SIGNAL(configChanged()));
connect(_ui->doubleSpinBox_meshingTextureMaxAngle, SIGNAL(valueChanged(double)), this, SIGNAL(configChanged()));
connect(_ui->spinBox_mesh_minTextureClusterSize, SIGNAL(valueChanged(int)), this, SIGNAL(configChanged()));
connect(_ui->lineEdit_meshingTextureRoiRatios, SIGNAL(textChanged(const QString &)), this, SIGNAL(configChanged()));
connect(_ui->checkBox_cameraFilter, SIGNAL(stateChanged(int)), this, SIGNAL(configChanged()));
@@ -310,6 +312,8 @@ void ExportCloudsDialog::saveSettings(QSettings & settings, const QString & grou
settings.setValue("mesh_textureSize", _ui->comboBox_meshingTextureSize->currentIndex());
settings.setValue("mesh_textureMaxCount", _ui->spinBox_mesh_maxTextures->value());
settings.setValue("mesh_textureMaxDistance", _ui->doubleSpinBox_meshingTextureMaxDistance->value());
settings.setValue("mesh_textureMaxDepthError", _ui->doubleSpinBox_meshingTextureMaxDepthError->value());
settings.setValue("mesh_textureMaxAngle", _ui->doubleSpinBox_meshingTextureMaxAngle->value());
settings.setValue("mesh_textureMinCluster", _ui->spinBox_mesh_minTextureClusterSize->value());
settings.setValue("mesh_textureRoiRatios", _ui->lineEdit_meshingTextureRoiRatios->text());
settings.setValue("mesh_textureCameraFiltering", _ui->checkBox_cameraFilter->isChecked());
@@ -421,6 +425,8 @@ void ExportCloudsDialog::loadSettings(QSettings & settings, const QString & grou
_ui->comboBox_meshingTextureSize->setCurrentIndex(settings.value("mesh_textureSize", _ui->comboBox_meshingTextureSize->currentIndex()).toInt());
_ui->spinBox_mesh_maxTextures->setValue(settings.value("mesh_textureMaxCount", _ui->spinBox_mesh_maxTextures->value()).toInt());
_ui->doubleSpinBox_meshingTextureMaxDistance->setValue(settings.value("mesh_textureMaxDistance", _ui->doubleSpinBox_meshingTextureMaxDistance->value()).toDouble());
_ui->doubleSpinBox_meshingTextureMaxDepthError->setValue(settings.value("mesh_textureMaxDepthError", _ui->doubleSpinBox_meshingTextureMaxDepthError->value()).toDouble());
_ui->doubleSpinBox_meshingTextureMaxAngle->setValue(settings.value("mesh_textureMaxAngle", _ui->doubleSpinBox_meshingTextureMaxAngle->value()).toDouble());
_ui->spinBox_mesh_minTextureClusterSize->setValue(settings.value("mesh_textureMinCluster", _ui->spinBox_mesh_minTextureClusterSize->value()).toDouble());
_ui->lineEdit_meshingTextureRoiRatios->setText(settings.value("mesh_textureRoiRatios", _ui->lineEdit_meshingTextureRoiRatios->text()).toString());
_ui->checkBox_cameraFilter->setChecked(settings.value("mesh_textureCameraFiltering", _ui->checkBox_cameraFilter->isChecked()).toBool());
@@ -532,6 +538,8 @@ void ExportCloudsDialog::restoreDefaults()
_ui->comboBox_meshingTextureSize->setCurrentIndex(5); // 4096
_ui->spinBox_mesh_maxTextures->setValue(1);
_ui->doubleSpinBox_meshingTextureMaxDistance->setValue(3.0);
_ui->doubleSpinBox_meshingTextureMaxDepthError->setValue(0.0);
_ui->doubleSpinBox_meshingTextureMaxAngle->setValue(0.0);
_ui->spinBox_mesh_minTextureClusterSize->setValue(50);
_ui->lineEdit_meshingTextureRoiRatios->setText("0.0 0.0 0.0 0.0");
_ui->checkBox_cameraFilter->setChecked(false);
@@ -1928,6 +1936,7 @@ bool ExportCloudsDialog::getExportedClouds(
}
std::map<int, Transform> cameraPoses;
std::map<int, std::vector<CameraModel> > cameraModels;
std::map<int, cv::Mat > cameraDepths;
for(std::map<int, Transform>::iterator jter=cameras.begin(); jter!=cameras.end(); ++jter)
{
if(validCameras.find(jter->first) != validCameras.end())
@@ -1947,8 +1956,10 @@ bool ExportCloudsDialog::getExportedClouds(
_dbDriver->getCalibration(jter->first, models, stereoModel);
}
bool stereo=false;
if(stereoModel.isValidForProjection())
{
stereo = true;
models.clear();
models.push_back(stereoModel.left());
}
@@ -1956,8 +1967,11 @@ bool ExportCloudsDialog::getExportedClouds(
{
models.clear();
}
if(!jter->second.isNull() && models.size())
{
cv::Mat depth;
bool getDepth = !stereo && _ui->doubleSpinBox_meshingTextureMaxDepthError->value() >= 0.0f;
if(models[0].imageWidth() == 0 || models[0].imageHeight() == 0)
{
// we are using an old database format (image size not saved in calibrations), we should
@@ -1965,13 +1979,13 @@ bool ExportCloudsDialog::getExportedClouds(
cv::Mat img;
if(cacheHasCompressedImage)
{
cachedSignatures.find(jter->first)->sensorData().uncompressDataConst(&img, 0);
cachedSignatures.find(jter->first)->sensorData().uncompressDataConst(&img, getDepth?&depth:0);
}
else if(_dbDriver)
{
SensorData data;
_dbDriver->getNodeData(jter->first, data, true, false, false, false);
data.uncompressDataConst(&img, 0);
data.uncompressDataConst(&img, getDepth?&depth:0);
}
cv::Size imageSize = img.size();
imageSize.width /= models.size();
@@ -1979,13 +1993,30 @@ bool ExportCloudsDialog::getExportedClouds(
{
models[i].setImageSize(imageSize);
}
}
else
{
// get just the depth
if(cacheHasCompressedImage)
{
cachedSignatures.find(jter->first)->sensorData().uncompressDataConst(0, getDepth?&depth:0);
}
else if(_dbDriver)
{
SensorData data;
_dbDriver->getNodeData(jter->first, data, true, false, false, false);
data.uncompressDataConst(0, getDepth?&depth:0);
}
}
if(models[0].imageWidth() != 0 && models[0].imageHeight() != 0)
{
cameraPoses.insert(std::make_pair(jter->first, jter->second));
cameraModels.insert(std::make_pair(jter->first, models));
if(!depth.empty())
{
cameraDepths.insert(std::make_pair(jter->first, depth));
}
}
}
}
@@ -2058,6 +2089,7 @@ bool ExportCloudsDialog::getExportedClouds(
if(cameraPoses.find(modelIter->first)==cameraPoses.end())
{
cameraModels.erase(modelIter++);
cameraDepths.erase(modelIter->first);
}
else
{
@@ -2114,7 +2146,10 @@ bool ExportCloudsDialog::getExportedClouds(
iter->second,
cameraPoses,
cameraModels,
cameraDepths,
_ui->doubleSpinBox_meshingTextureMaxDistance->value(),
_ui->doubleSpinBox_meshingTextureMaxDepthError->value(),
_ui->doubleSpinBox_meshingTextureMaxAngle->value()*M_PI/180.0,
_ui->spinBox_mesh_minTextureClusterSize->value(),
roiRatios,
&texturingState,
@@ -3097,7 +3132,7 @@ std::vector<cv::Mat> ExportCloudsDialog::mergeTextures(
}
}
}
else if(mesh.tex_polygons[i].size())
else if(mesh.tex_polygons[i].size() && mesh.tex_materials[i].tex_file.compare("occluded")!=0)
{
UWARN("Failed parsing texture file name: %s", mesh.tex_materials[i].tex_file.c_str());
}
@@ -3278,9 +3313,16 @@ std::vector<cv::Mat> ExportCloudsDialog::mergeTextures(
for(int i=0; i<num_images; ++i)
{
for(int j=i+1; j<num_images; ++j)
for(int j=i; j<num_images; ++j)
{
if(N(i, j))
if(i == j)
{
if(N(i,j) == 0)
{
N(i,j) = 1;
}
}
else if(N(i, j))
{
I(i, j) /= N(i, j);
I(j, i) /= N(j, i);

View File

@@ -23,9 +23,9 @@
<property name="geometry">
<rect>
<x>0</x>
<y>-1412</y>
<y>-2146</y>
<width>773</width>
<height>3953</height>
<height>4045</height>
</rect>
</property>
<layout class="QVBoxLayout" name="verticalLayout_13">
@@ -1486,7 +1486,111 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
<layout class="QVBoxLayout" name="verticalLayout_16">
<item>
<layout class="QGridLayout" name="gridLayout_15" columnstretch="0,1">
<item row="7" column="1">
<item row="12" column="0">
<widget class="QSpinBox" name="spinBox_textureBrightnessContrastRatioHigh">
<property name="suffix">
<string> %</string>
</property>
<property name="minimum">
<number>0</number>
</property>
<property name="maximum">
<number>49</number>
</property>
</widget>
</item>
<item row="12" column="1">
<widget class="QLabel" name="label_meshingTextureSize_6">
<property name="text">
<string>Brightness and contrast balance high ratio. Only used when textures are merged.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="10" column="0">
<widget class="QComboBox" name="comboBox_blendingDecimation">
<item>
<property name="text">
<string>Auto</string>
</property>
</item>
<item>
<property name="text">
<string>1</string>
</property>
</item>
<item>
<property name="text">
<string>2</string>
</property>
</item>
<item>
<property name="text">
<string>4</string>
</property>
</item>
<item>
<property name="text">
<string>8</string>
</property>
</item>
<item>
<property name="text">
<string>16</string>
</property>
</item>
<item>
<property name="text">
<string>32</string>
</property>
</item>
<item>
<property name="text">
<string>64</string>
</property>
</item>
</widget>
</item>
<item row="9" column="0">
<widget class="QCheckBox" name="checkBox_blending">
<property name="text">
<string/>
</property>
</widget>
</item>
<item row="10" column="1">
<widget class="QLabel" name="label_exposureFusion_3">
<property name="text">
<string>Blending decimation. </string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="3" column="1">
<widget class="QLabel" name="label_meshingTextureSize_7">
<property name="text">
<string>ROI ratios [left right top bottom] between 0 and 1. This can be used to ignore black borders of RGB images caused by calibration. </string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QLabel" name="label_meshingTextureSize_8">
<property name="text">
<string>Maximum output textures.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="9" column="1">
<widget class="QLabel" name="label_exposureFusion_2">
<property name="text">
<string>Blending. Only used with dense reconstruction flavor and if clouds are assembled.</string>
@@ -1528,7 +1632,7 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</property>
</widget>
</item>
<item row="9" column="1">
<item row="11" column="1">
<widget class="QLabel" name="label_meshingTextureSize_5">
<property name="text">
<string>Brightness and contrast balance low ratio. Only used when textures are merged.</string>
@@ -1562,29 +1666,6 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</item>
</widget>
</item>
<item row="10" column="0">
<widget class="QSpinBox" name="spinBox_textureBrightnessContrastRatioHigh">
<property name="suffix">
<string> %</string>
</property>
<property name="minimum">
<number>0</number>
</property>
<property name="maximum">
<number>49</number>
</property>
</widget>
</item>
<item row="10" column="1">
<widget class="QLabel" name="label_meshingTextureSize_6">
<property name="text">
<string>Brightness and contrast balance high ratio. Only used when textures are merged.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QComboBox" name="comboBox_meshingTextureSize">
<item>
@@ -1634,14 +1715,14 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</item>
</widget>
</item>
<item row="12" column="0">
<item row="14" column="0">
<widget class="QCheckBox" name="checkBox_exposureFusion">
<property name="text">
<string/>
</property>
</widget>
</item>
<item row="5" column="1">
<item row="7" column="1">
<widget class="QLabel" name="label_meshingTextureSize_4">
<property name="text">
<string>Minimum polygon cluster size for texturing. This removes textures applied on sparse polygons. Only used with dense reconstruction flavor.</string>
@@ -1654,50 +1735,6 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
<item row="3" column="0">
<widget class="QLineEdit" name="lineEdit_meshingTextureRoiRatios"/>
</item>
<item row="8" column="0">
<widget class="QComboBox" name="comboBox_blendingDecimation">
<item>
<property name="text">
<string>Auto</string>
</property>
</item>
<item>
<property name="text">
<string>1</string>
</property>
</item>
<item>
<property name="text">
<string>2</string>
</property>
</item>
<item>
<property name="text">
<string>4</string>
</property>
</item>
<item>
<property name="text">
<string>8</string>
</property>
</item>
<item>
<property name="text">
<string>16</string>
</property>
</item>
<item>
<property name="text">
<string>32</string>
</property>
</item>
<item>
<property name="text">
<string>64</string>
</property>
</item>
</widget>
</item>
<item row="1" column="1">
<widget class="QLabel" name="label_meshingTextureSize">
<property name="text">
@@ -1708,21 +1745,14 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</property>
</widget>
</item>
<item row="7" column="0">
<widget class="QCheckBox" name="checkBox_blending">
<property name="text">
<string/>
</property>
</widget>
</item>
<item row="6" column="0">
<item row="8" column="0">
<widget class="QCheckBox" name="checkBox_cameraFilter">
<property name="text">
<string/>
</property>
</widget>
</item>
<item row="9" column="0">
<item row="11" column="0">
<widget class="QSpinBox" name="spinBox_textureBrightnessContrastRatioLow">
<property name="suffix">
<string> %</string>
@@ -1735,7 +1765,7 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</property>
</widget>
</item>
<item row="5" column="0">
<item row="7" column="0">
<widget class="QSpinBox" name="spinBox_mesh_minTextureClusterSize">
<property name="minimum">
<number>0</number>
@@ -1746,16 +1776,6 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</widget>
</item>
<item row="8" column="1">
<widget class="QLabel" name="label_exposureFusion_3">
<property name="text">
<string>Blending decimation. </string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="6" column="1">
<widget class="QLabel" name="label_meshingTextureSize_3">
<property name="text">
<string>Camera filtering. By comparing poses in the same area, only one camera in a fixed radius and angle is used for texturing.</string>
@@ -1765,7 +1785,7 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</property>
</widget>
</item>
<item row="12" column="1">
<item row="14" column="1">
<widget class="QLabel" name="label_exposureFusion">
<property name="text">
<string>Exposure fusion. Only used when textures are merged and brightness/contrast balance ratios are used (merging original texture + 1 for each ratios).</string>
@@ -1775,16 +1795,6 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</property>
</widget>
</item>
<item row="3" column="1">
<widget class="QLabel" name="label_meshingTextureSize_7">
<property name="text">
<string>ROI ratios [left right top bottom] between 0 and 1. This can be used to ignore black borders of RGB images caused by calibration. </string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QSpinBox" name="spinBox_mesh_maxTextures">
<property name="minimum">
@@ -1795,16 +1805,70 @@ Guidelines: 4 times the voxel size, 0.025 for voxel=0.</string>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QLabel" name="label_meshingTextureSize_8">
<item row="5" column="1">
<widget class="QLabel" name="label_meshingTextureSize_9">
<property name="text">
<string>Maximum output textures.</string>
<string>Maximum depth error between reconstructed mesh projection in camera and corresponding value in depth image to apply texturing (negative means disabled, 0 means that longest edge length of the face is used as maximum error).</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="5" column="0">
<widget class="QDoubleSpinBox" name="doubleSpinBox_meshingTextureMaxDepthError">
<property name="suffix">
<string> m</string>
</property>
<property name="decimals">
<number>2</number>
</property>
<property name="minimum">
<double>-1.000000000000000</double>
</property>
<property name="maximum">
<double>99.000000000000000</double>
</property>
<property name="singleStep">
<double>0.050000000000000</double>
</property>
<property name="value">
<double>0.000000000000000</double>
</property>
</widget>
</item>
<item row="6" column="1">
<widget class="QLabel" name="label_meshingTextureSize_10">
<property name="text">
<string>Maximum angle between camera and face to apply texture (0 means disabled).</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item row="6" column="0">
<widget class="QDoubleSpinBox" name="doubleSpinBox_meshingTextureMaxAngle">
<property name="suffix">
<string> deg</string>
</property>
<property name="decimals">
<number>0</number>
</property>
<property name="minimum">
<double>0.000000000000000</double>
</property>
<property name="maximum">
<double>180.000000000000000</double>
</property>
<property name="singleStep">
<double>1.000000000000000</double>
</property>
<property name="value">
<double>0.000000000000000</double>
</property>
</widget>
</item>
</layout>
</item>
<item>