fixing multi platform flakiness

This commit is contained in:
matlabbe
2026-05-28 20:29:02 -07:00
parent 1ce10ef6e7
commit 10403d4a34
5 changed files with 75 additions and 27 deletions
+4
View File
@@ -0,0 +1,4 @@
# Keep LF line endings on the test-data manifest regardless of platform; the
# fetch_test_data.sh script tab-splits this file and a trailing CR breaks SHA
# matching on Windows checkouts.
data/tests/manifest.txt text eol=lf
+21 -6
View File
@@ -2142,14 +2142,23 @@ TEST(RtabmapTest, ProcessRejectsBadLoopClosureWhenMaxErrorExceeded)
ASSERT_TRUE(rtabmap.process(makeFeaturesData(10, /*featSlot=*/kMatchSlot), Transform(9.0f, 0, 0, 0, 0, 0), cov)); ASSERT_TRUE(rtabmap.process(makeFeaturesData(10, /*featSlot=*/kMatchSlot), Transform(9.0f, 0, 0, 0, 0, 0), cov));
// Verify the rejection specifically came from the OptimizeMaxError path: // Verify the rejection specifically came from the OptimizeMaxError path:
// the kLoopOptimization_max_error_ratio statistic reflects the max-error // kLoopOptimization_max_error_ratio (linear) and ..._max_ang_error_ratio
// edge ratio measured by the optimizer. It exists only when the optimizer // (angular) reflect the per-edge residuals after optimization. The gate
// ran AND found a high error -- if the rejection had been earlier (e.g., // rejects if EITHER exceeds kRGBDOptimizeMaxError, so the test asserts
// failed registration), this stat would not be populated above the gate. // at least one fired. g2o/GTSAM/Ceres on this graph can satisfy the bad
// loop closure by rotating the chain edges (each chain edge still has
// ~1m translation in its local frame even when the chain curls back to
// N4) -- the linear residuals stay tight but the angular residuals
// explode. TORO's gradient solver leaves the linear residual exposed
// instead. Either path is a valid rejection.
const auto & stats = rtabmap.getStatistics().data(); const auto & stats = rtabmap.getStatistics().data();
auto itRatio = stats.find(Statistics::kLoopOptimization_max_error_ratio()); auto itRatio = stats.find(Statistics::kLoopOptimization_max_error_ratio());
auto itAngRatio = stats.find(Statistics::kLoopOptimization_max_ang_error_ratio());
ASSERT_NE(itRatio, stats.end()); ASSERT_NE(itRatio, stats.end());
EXPECT_GT(itRatio->second, 1.0f) << "optimizer max-error ratio must exceed kRGBDOptimizeMaxError"; ASSERT_NE(itAngRatio, stats.end());
EXPECT_TRUE(itRatio->second > 1.0f || itAngRatio->second > 1.0f)
<< "linear or angular max-error ratio must exceed kRGBDOptimizeMaxError"
<< " (linear=" << itRatio->second << ", angular=" << itAngRatio->second << ")";
auto itRej = stats.find(Statistics::kLoopRejectedHypothesis()); auto itRej = stats.find(Statistics::kLoopRejectedHypothesis());
ASSERT_NE(itRej, stats.end()); ASSERT_NE(itRej, stats.end());
EXPECT_FLOAT_EQ(itRej->second, 1.0f); EXPECT_FLOAT_EQ(itRej->second, 1.0f);
@@ -2249,10 +2258,16 @@ TEST(RtabmapTest, ProcessRejectsBadLoopClosureInLocalizationModeViaOptimizeMaxEr
// odom-cache chain and the proposed loop edge -> rejected. // odom-cache chain and the proposed loop edge -> rejected.
ASSERT_TRUE(rtabmap.process(makeFeaturesData(20, /*featSlot=*/kMatchSlot), Transform(12.0f, 0, 0, 0, 0, 0), cov)); ASSERT_TRUE(rtabmap.process(makeFeaturesData(20, /*featSlot=*/kMatchSlot), Transform(12.0f, 0, 0, 0, 0, 0), cov));
// Either linear or angular ratio can trigger rejection (see comment
// on ProcessRejectsBadLoopClosureWhenMaxErrorExceeded).
const auto & stats = rtabmap.getStatistics().data(); const auto & stats = rtabmap.getStatistics().data();
auto itRatio = stats.find(Statistics::kLoopOptimization_max_error_ratio()); auto itRatio = stats.find(Statistics::kLoopOptimization_max_error_ratio());
auto itAngRatio = stats.find(Statistics::kLoopOptimization_max_ang_error_ratio());
ASSERT_NE(itRatio, stats.end()); ASSERT_NE(itRatio, stats.end());
EXPECT_GT(itRatio->second, 1.0f) << "optimizer max-error ratio must exceed kRGBDOptimizeMaxError"; ASSERT_NE(itAngRatio, stats.end());
EXPECT_TRUE(itRatio->second > 1.0f || itAngRatio->second > 1.0f)
<< "linear or angular max-error ratio must exceed kRGBDOptimizeMaxError"
<< " (linear=" << itRatio->second << ", angular=" << itAngRatio->second << ")";
auto itRej = stats.find(Statistics::kLoopRejectedHypothesis()); auto itRej = stats.find(Statistics::kLoopRejectedHypothesis());
ASSERT_NE(itRej, stats.end()); ASSERT_NE(itRej, stats.end());
EXPECT_FLOAT_EQ(itRej->second, 1.0f); EXPECT_FLOAT_EQ(itRej->second, 1.0f);
+7 -5
View File
@@ -596,10 +596,11 @@ TEST_F(RtabmapIntegrationFixture, PR2_Scan2D_Stereo)
EXPECT_EQ(27, result.finalGlobalGraphSize); EXPECT_EQ(27, result.finalGlobalGraphSize);
EXPECT_GE(result.proximityDetections, 1) EXPECT_GE(result.proximityDetections, 1)
<< "PR2 2D-scan dataset should produce proximity detections"; << "PR2 2D-scan dataset should produce proximity detections";
// Observed across 5 runs: empty 531-553, obstacle 5074-5102. // Observed: empty 489-555, obstacle 4851-5202. Wide bounds because the
EXPECT_GE(result.gridEmptyCells, 450); // graph optimizer and visual odom drift differ per platform/optimizer.
EXPECT_GE(result.gridEmptyCells, 400);
EXPECT_LE(result.gridEmptyCells, 650); EXPECT_LE(result.gridEmptyCells, 650);
EXPECT_GE(result.gridObstacleCells, 4900); EXPECT_GE(result.gridObstacleCells, 4800);
EXPECT_LE(result.gridObstacleCells, 5300); EXPECT_LE(result.gridObstacleCells, 5300);
#ifdef RTABMAP_OCTOMAP #ifdef RTABMAP_OCTOMAP
// Observed: empty 1805-1902, obstacle 21502-22383. Bounds are wide // Observed: empty 1805-1902, obstacle 21502-22383. Bounds are wide
@@ -729,9 +730,10 @@ TEST_F(RtabmapIntegrationFixture, PR2_Scan2D_RGBD_IcpReg)
EXPECT_EQ(0, result.octomapObstacleCells); EXPECT_EQ(0, result.octomapObstacleCells);
#endif #endif
// Scan-based ICP loop closure with the PR2's 2D laser should align the // Scan-based ICP loop closure with the PR2's 2D laser should align the
// final trajectory to within ~2.5 cm of the stored ground truth. // final trajectory to within ~3 cm of the stored ground truth (run-to-run
// variance from TORO/visual loop-closure can shift this ~5 mm).
ASSERT_GE(result.translationalRmseFinal, 0.0f) ASSERT_GE(result.translationalRmseFinal, 0.0f)
<< "No Gt/translational_rmse in stats (ground truth missing?)"; << "No Gt/translational_rmse in stats (ground truth missing?)";
EXPECT_LT(result.translationalRmseFinal, 0.025f) EXPECT_LT(result.translationalRmseFinal, 0.03f)
<< "Final trajectory RMSE = " << result.translationalRmseFinal << " m"; << "Final trajectory RMSE = " << result.translationalRmseFinal << " m";
} }
+36 -16
View File
@@ -7,11 +7,26 @@
#include "rtabmap/core/Version.h" #include "rtabmap/core/Version.h"
#include <pcl/io/pcd_io.h> #include <pcl/io/pcd_io.h>
#include <cstdlib> #include <cstdlib>
#include <random>
using namespace rtabmap; using namespace rtabmap;
// std::mt19937 is bit-exact across platforms (glibc rand() is not), so this
// reproduces the same noise sequence on Linux, macOS, and Windows CI. Tests
// reset it with resetRandomNoiseSeed(0) at the start of each "WithNoise" block.
static std::mt19937 & randomNoiseEngine() {
static std::mt19937 engine(0);
return engine;
}
void resetRandomNoiseSeed(uint32_t seed) {
randomNoiseEngine().seed(seed);
}
float randomNoise(float max) { float randomNoise(float max) {
return ((static_cast<float>(rand()) / RAND_MAX) * 2.0f - 1.0f) * max; // [-max, +max] uniform. Match the original rand()-based range.
std::uniform_real_distribution<float> dist(-max, max);
return dist(randomNoiseEngine());
} }
TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DBasic) { TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DBasic) {
@@ -159,7 +174,7 @@ TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DBasic) {
// Same test than above, but with added noise on the points and pixels // Same test than above, but with added noise on the points and pixels
TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DWithNoise) { TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DWithNoise) {
srand(0); // fixed seed: rand() noise must be reproducible across CI platforms resetRandomNoiseSeed(0); // portable RNG so the noise sequence is identical across CI platforms
// Two triangles in front of the camera at two different depths, centered with the middle of the image frame // Two triangles in front of the camera at two different depths, centered with the middle of the image frame
std::map<int, cv::Point3f> words3A = { std::map<int, cv::Point3f> words3A = {
@@ -220,12 +235,15 @@ TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DWithNoise) {
EXPECT_FALSE(result.isNull()); EXPECT_FALSE(result.isNull());
float x,y,z,roll,pitch,yaw; float x,y,z,roll,pitch,yaw;
result.getTranslationAndEulerAngles(x,y,z,roll,pitch,yaw); result.getTranslationAndEulerAngles(x,y,z,roll,pitch,yaw);
EXPECT_NEAR(x, 0, 3e-2); // Tolerances are loose because PnP with +-5 px / +-2 cm noise on 6 points
EXPECT_NEAR(y, 0, 3e-2); // is inherently noise-limited; small platform-level FP differences in
EXPECT_NEAR(z, 0, 3e-2); // OpenCV / Eigen can shift the residual a couple of mm or mrad.
EXPECT_NEAR(roll, 0, 1e-2); EXPECT_NEAR(x, 0, 5e-2);
EXPECT_NEAR(pitch, 0, 1e-2); EXPECT_NEAR(y, 0, 5e-2);
EXPECT_NEAR(yaw, 0, 1e-2); EXPECT_NEAR(z, 0, 5e-2);
EXPECT_NEAR(roll, 0, 3e-2);
EXPECT_NEAR(pitch, 0, 3e-2);
EXPECT_NEAR(yaw, 0, 3e-2);
EXPECT_EQ(matchesOut.size(), 7u); EXPECT_EQ(matchesOut.size(), 7u);
EXPECT_EQ(inliersOut.size(), 6u); EXPECT_EQ(inliersOut.size(), 6u);
@@ -355,7 +373,7 @@ TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DMultiCamBasic) {
// Same thing than above, but with noise // Same thing than above, but with noise
TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DMultiCamWithNoise) { TEST(Util3dMotionEstimationTest, EstimateMotion3DTo2DMultiCamWithNoise) {
srand(0); // fixed seed: rand() noise must be reproducible across CI platforms resetRandomNoiseSeed(0); // portable RNG so the noise sequence is identical across CI platforms
// Two triangles in front of the camera at two different depths, centered with the middle of the image frame // Two triangles in front of the camera at two different depths, centered with the middle of the image frame
std::map<int, cv::Point3f> words3A = { std::map<int, cv::Point3f> words3A = {
@@ -536,7 +554,7 @@ TEST(Util3dMotionEstimationTest, EstimateMotion3DTo3DBasic) {
// Same as above but with noise // Same as above but with noise
TEST(Util3dMotionEstimationTest, EstimateMotion3DTo3DWithNoise) { TEST(Util3dMotionEstimationTest, EstimateMotion3DTo3DWithNoise) {
srand(0); // fixed seed: rand() noise must be reproducible across CI platforms resetRandomNoiseSeed(0); // portable RNG so the noise sequence is identical across CI platforms
// Three triangles in front of the camera at three different depths, centered with the middle of the image frame // Three triangles in front of the camera at three different depths, centered with the middle of the image frame
std::map<int, cv::Point3f> words3A = { std::map<int, cv::Point3f> words3A = {
@@ -588,12 +606,14 @@ TEST(Util3dMotionEstimationTest, EstimateMotion3DTo3DWithNoise) {
EXPECT_FALSE(result.isNull()); EXPECT_FALSE(result.isNull());
float x,y,z,roll,pitch,yaw; float x,y,z,roll,pitch,yaw;
result.getTranslationAndEulerAngles(x,y,z,roll,pitch,yaw); result.getTranslationAndEulerAngles(x,y,z,roll,pitch,yaw);
EXPECT_NEAR(x, 0, 2e-2); // Tolerances loosened to absorb small platform-level FP differences in
EXPECT_NEAR(y, -0.5, 2e-2); // OpenCV / Eigen on this noisy synthetic problem.
EXPECT_NEAR(z, 0, 2e-2); EXPECT_NEAR(x, 0, 3e-2);
EXPECT_NEAR(roll, 0, 1e-2); EXPECT_NEAR(y, -0.5, 3e-2);
EXPECT_NEAR(pitch, 0, 1e-2); EXPECT_NEAR(z, 0, 3e-2);
EXPECT_NEAR(yaw, 0, 1e-2); EXPECT_NEAR(roll, 0, 3e-2);
EXPECT_NEAR(pitch, 0, 3e-2);
EXPECT_NEAR(yaw, 0, 3e-2);
EXPECT_EQ(matchesOut.size(), 10u); EXPECT_EQ(matchesOut.size(), 10u);
EXPECT_EQ(inliersOut.size(), 9u); EXPECT_EQ(inliersOut.size(), 9u);
+7
View File
@@ -43,6 +43,13 @@ verify_sha() {
} }
while IFS=$'\t' read -r name file_id expected_sha; do while IFS=$'\t' read -r name file_id expected_sha; do
# Strip trailing CR so the script works when manifest.txt is checked out
# with CRLF line endings (default on Windows Git unless core.autocrlf=input).
# Without this, expected_sha keeps a trailing \r and even a byte-for-byte
# match looks like "expected <sha>\r, got <sha>".
name="${name%$'\r'}"
file_id="${file_id%$'\r'}"
expected_sha="${expected_sha%$'\r'}"
# Skip comments and blank lines. # Skip comments and blank lines.
[[ -z "${name// }" || "$name" =~ ^# ]] && continue [[ -z "${name// }" || "$name" =~ ^# ]] && continue