From 428dbbab726d00b153c867adca1c56f01fe743eb Mon Sep 17 00:00:00 2001 From: Arkylin Date: Tue, 21 Apr 2026 20:26:21 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- README.md | 28 +++++ d2c_batch.py | 39 +++++-- main.py | 295 +++++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 302 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index 8639206..ffe2997 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ test/ Log/ .vscode/ camera_params.json -**.pyc** \ No newline at end of file +**.pyc** +data/ \ No newline at end of file diff --git a/README.md b/README.md index 64b2425..ea6c57e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,22 @@ 多个深度点映射到同一彩色像素时，保留最近点（小 Z 值覆盖大 Z 值）。 +### 前向映射的间隙问题与 3×3 Splat 修复 + +前向映射（forward scatter）存在固有缺陷：每个深度像素只写入 color 平面的**一个像素**。由于深度相机与彩色相机的视角差异，经外参变换 + 透视投影后，相邻深度像素在 color 平面上的落点会出现跳跃，中间留下空洞，输出图像呈现**离散小方块**而非连续深度面。 + +**修复方案：3×3 邻域扩散（splat）** + +对每个投影后的深度像素，同时写入以其为中心的 3×3 邻域，而不是单个像素。遍历顺序仍保持**远→近**，因此当多个 splat 重叠时，Z 值较小（较近）的深度最后写入并覆盖，遮挡关系依然正确： + +``` +for dv in -1, 0, +1: + for du in -1, 0, +1: + aligned[v + dv, u + du] = z # 远→近顺序，近处覆盖远处 +``` + +如果局部仍有残留空洞（如深度图有效范围边缘），可将 3×3 扩大为 5×5（`range(-2, 3)`）。 + --- ## 环境依赖 @@ -124,6 +140,18 @@ python d2c_batch.py --convert 使用已有的 `camera_params.json`，直接输入深度图目录开始转换。适合相机参数已获取、需要反复处理不同数据集的场景。 +### 关闭 3×3 Splat 填充 + +```bash +python d2c_batch.py --no-splat +``` + +默认开启 splat（前向映射后用 3×3 邻域扩散填补间隙）。加 `--no-splat` 可关闭，每个深度像素仅写入 color 平面的单个像素，适合对原始投影结果做对比分析。可与其他参数组合使用： + +```bash +python d2c_batch.py --convert --no-splat +``` + --- ## 交互示例 diff --git a/d2c_batch.py b/d2c_batch.py index b14abe4..8e67d96 100644 --- a/d2c_batch.py +++ b/d2c_batch.py @@ -24,6 +24,12 @@ import numpy as np import cv2 from pathlib import Path +try: + from PIL import Image as _PIL_Image + _HAS_PIL = True +except ImportError: + _HAS_PIL = False + PARAMS_FILE = "camera_params.json" @@ -219,7 +225,7 @@ def _undistort_points(u, v, intr, dist): return x_u, y_u -def software_d2c(depth_img, params): +def software_d2c(depth_img, params, splat=True): """ Transform a uint16 depth image (depth camera space) to an aligned uint16 depth image in color camera space. @@ -294,7 +300,19 @@ def software_d2c(depth_img, params): z_vals = z_vals[order] aligned = np.zeros((color_h, color_w), dtype=np.uint16) - aligned[v_vals, u_vals] = z_vals + + if splat: + # Splat each depth pixel to a 3×3 neighbourhood to fill sub-pixel gaps + # that arise from forward projection. Far→near order is preserved so + # the nearest surface always wins when splats overlap. + for dv in range(-1, 2): + for du in range(-1, 2): + us = np.clip(u_vals + du, 0, color_w - 1) + vs = np.clip(v_vals + dv, 0, color_h - 1) + aligned[vs, us] = z_vals + else: + aligned[v_vals, u_vals] = z_vals + return aligned @@ -315,7 +333,7 @@ def depth_to_colormap(aligned, min_depth_mm=200, max_depth_mm=5000, depth_scale= return colormap -def batch_convert(params, depth_dir_str): +def batch_convert(params, depth_dir_str, splat=True): depth_dir = Path(depth_dir_str.strip().strip('"\'')) if not depth_dir.exists(): print(f" Directory not found: {depth_dir}") @@ -343,12 +361,20 @@ def batch_convert(params, depth_dir_str): print(f" Files : {len(png_files)}") print(f" Depth scale : {depth_scale:.6f} mm/unit") print(f" Output size : {color_w}x{color_h} (color resolution)") + print(f" Splat (3×3) : {'on' if splat else 'off'}") print() t0 = time.time() ok = 0 for i, fpath in enumerate(png_files, 1): - depth_img = cv2.imread(str(fpath), cv2.IMREAD_UNCHANGED) + depth_img = None + if _HAS_PIL: + try: + depth_img = np.array(_PIL_Image.open(str(fpath))) + except Exception: + depth_img = None + if depth_img is None: + depth_img = cv2.imread(str(fpath), cv2.IMREAD_UNCHANGED) if depth_img is None: print(f" [{i:>4}/{len(png_files)}] SKIP (unreadable): {fpath.name}") continue @@ -356,7 +382,7 @@ def batch_convert(params, depth_dir_str): depth_img = depth_img[:, :, 0] depth_img = depth_img.astype(np.uint16) - aligned = software_d2c(depth_img, params) + aligned = software_d2c(depth_img, params, splat=splat) # Save aligned uint16 depth cv2.imwrite(str(out_dir / fpath.name), aligned) @@ -405,6 +431,7 @@ def print_params_summary(params): def main(): only_params = "--params" in sys.argv only_convert = "--convert" in sys.argv + no_splat = "--no-splat" in sys.argv params = None @@ -448,7 +475,7 @@ def main(): print("No directory entered. Exiting.") return - batch_convert(params, depth_dir) + batch_convert(params, depth_dir, splat=not no_splat) if __name__ == "__main__": diff --git a/main.py b/main.py index c0b5758..163b4e4 100644 --- a/main.py +++ b/main.py @@ -11,6 +11,8 @@ import sys import json import time import threading +import tempfile +import shutil import contextlib import io import numpy as np @@ -57,6 +59,26 @@ def ndarray_to_pixmap(img_bgr: np.ndarray, max_w: int, max_h: int) -> QPixmap: return QPixmap.fromImage(qimg) +def imwrite_utf8(path: str, img: np.ndarray) -> bool: + """cv2.imwrite 的 Unicode 路径安全版本（Windows 中文路径兼容）。""" + ext = Path(path).suffix.lower() or ".png" + ok, buf = cv2.imencode(ext, img) + if ok: + Path(path).write_bytes(buf.tobytes()) + return ok + + +def save_ply_utf8(target_path: str, pc_frame, save_func) -> None: + """将点云先写到 ASCII 临时路径，再移动到 Unicode 目标路径。""" + tmpdir = tempfile.mkdtemp() + try: + tmp_ply = os.path.join(tmpdir, "pc.ply") + save_func(tmp_ply, pc_frame) + shutil.move(tmp_ply, target_path) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def depth_to_vis(depth_uint16: np.ndarray, scale: float) -> np.ndarray: """uint16 深度图 → JET 伪彩 BGR 图（无效像素为黑色）。""" depth_mm = depth_uint16.astype(np.float32) * scale @@ -178,25 +200,31 @@ class ProfileDialog(QDialog): # Camera Worker Thread # ───────────────────────────────────────────────────────────────── +_DISCONNECT_TIMEOUTS = 6 # 6 × 500 ms = 3 s 无帧视为断开 + class CameraWorker(QThread): preview_ready = pyqtSignal(object, object) # color_bgr, depth_vis capture_done = pyqtSignal(str) # save_dir params_ready = pyqtSignal(dict) log_msg = pyqtSignal(str) + disconnected = pyqtSignal() def __init__(self, pipeline, color_profile, depth_profile): super().__init__() - self._pipeline = pipeline - self._color_profile = color_profile - self._depth_profile = depth_profile - self._lock = threading.Lock() - self._running = True - self._d2c = False - self._do_capture = False - self._capture_dir = "" - self._camera_params = None - self._has_dual_ir = False - self._last_emit_ts = 0.0 + self._pipeline = pipeline + self._color_profile = color_profile + self._depth_profile = depth_profile + self._lock = threading.Lock() + self._running = True + self._d2c = False + self._do_capture = False + self._capture_dir = "" + self._camera_params = None + self._has_dual_ir = False + self._last_emit_ts = 0.0 + self._serial_number = "unknown" + self._consec_timeouts = 0 + self._disconnect_emitted = False # ── Public controls ── @@ -231,9 +259,13 @@ class CameraWorker(QThread): config.enable_stream(self._color_profile) config.enable_stream(self._depth_profile) - # 探测 IR 传感器类型 + # 获取设备 SN + 探测 IR 传感器类型 try: - device = self._pipeline.get_device() + device = self._pipeline.get_device() + try: + self._serial_number = device.get_device_info().get_serial_number() + except Exception: + pass sensor_list = device.get_sensor_list() for i in range(len(sensor_list)): st = sensor_list[i].get_type() @@ -260,7 +292,14 @@ class CameraWorker(QThread): try: frames = self._pipeline.wait_for_frames(500) if frames is None: + self._consec_timeouts += 1 + if (self._consec_timeouts >= _DISCONNECT_TIMEOUTS + and not self._disconnect_emitted): + self._disconnect_emitted = True + self._running = False + self.disconnected.emit() continue + self._consec_timeouts = 0 with self._lock: d2c = self._d2c @@ -269,13 +308,15 @@ class CameraWorker(QThread): if do_cap: self._do_capture = False - # 首帧提取相机参数 + # 首帧提取相机参数（含 SN） if self._camera_params is None: cf = frames.get_color_frame() df = frames.get_depth_frame() if cf and df: try: - self._camera_params = self._extract_params(cf, df) + params = self._extract_params(cf, df) + params['device'] = {'serial_number': self._serial_number} + self._camera_params = params self.params_ready.emit(self._camera_params) except Exception: pass @@ -322,6 +363,12 @@ class CameraWorker(QThread): except Exception as exc: if self._running: self.log_msg.emit(f"相机错误：{exc}") + # 将异常也计入超时计数，快速检测断线 + self._consec_timeouts += _DISCONNECT_TIMEOUTS + if not self._disconnect_emitted: + self._disconnect_emitted = True + self._running = False + self.disconnected.emit() time.sleep(0.1) try: @@ -340,11 +387,13 @@ class CameraWorker(QThread): cf = frames.get_color_frame() df = frames.get_depth_frame() + base = Path(save_dir) + # ① 彩色图 if cf: bgr = frame_to_bgr_image(cf) if bgr is not None: - cv2.imwrite(str(Path(save_dir) / "color.png"), bgr) + imwrite_utf8(str(base / "color.png"), bgr) # ② 红外图 if self._has_dual_ir: @@ -352,21 +401,21 @@ class CameraWorker(QThread): rir = frames.get_frame(OBFrameType.RIGHT_IR_FRAME) img = process_ir_frame(lir) if img is not None: - cv2.imwrite(str(Path(save_dir) / "ir_left.png"), img) + imwrite_utf8(str(base / "ir_left.png"), img) img = process_ir_frame(rir) if img is not None: - cv2.imwrite(str(Path(save_dir) / "ir_right.png"), img) + imwrite_utf8(str(base / "ir_right.png"), img) else: ir = frames.get_frame(OBFrameType.IR_FRAME) img = process_ir_frame(ir) if img is not None: - cv2.imwrite(str(Path(save_dir) / "ir.png"), img) + imwrite_utf8(str(base / "ir.png"), img) # ③ 原始深度图 if df: raw = np.frombuffer(df.get_data(), dtype=np.uint16).reshape( df.get_height(), df.get_width()) - cv2.imwrite(str(Path(save_dir) / "depth.png"), raw) + imwrite_utf8(str(base / "depth.png"), raw) # ④ D2C 深度图 + 伪彩 + 点云 aligned = align_filter.process(frames) @@ -376,15 +425,16 @@ class CameraWorker(QThread): if adf: ad = np.frombuffer(adf.get_data(), dtype=np.uint16).reshape( adf.get_height(), adf.get_width()) - cv2.imwrite(str(Path(save_dir) / "depth_d2c.png"), ad) + imwrite_utf8(str(base / "depth_d2c.png"), ad) vis = depth_to_vis(ad, adf.get_depth_scale()) - cv2.imwrite(str(Path(save_dir) / "depth_d2c_vis.png"), vis) + imwrite_utf8(str(base / "depth_d2c_vis.png"), vis) try: point_cloud_filter.set_create_point_format(OBFormat.POINT) pc = point_cloud_filter.process(aligned) if pc: - save_point_cloud_to_ply(str(Path(save_dir) / "point_cloud.ply"), pc) + save_ply_utf8(str(base / "point_cloud.ply"), pc, + save_point_cloud_to_ply) except Exception as exc: self.log_msg.emit(f"点云警告：{exc}") @@ -392,15 +442,11 @@ class CameraWorker(QThread): point_cloud_filter.set_create_point_format(OBFormat.RGB_POINT) pc = point_cloud_filter.process(aligned) if pc: - save_point_cloud_to_ply(str(Path(save_dir) / "point_cloud_color.ply"), pc) + save_ply_utf8(str(base / "point_cloud_color.ply"), pc, + save_point_cloud_to_ply) except Exception as exc: self.log_msg.emit(f"彩色点云警告：{exc}") - # ⑤ 相机参数 - if self._camera_params: - with open(str(Path(save_dir) / "camera_params.json"), "w") as fh: - json.dump(self._camera_params, fh, indent=2) - self.capture_done.emit(save_dir) # ── Static helpers ── @@ -455,10 +501,11 @@ class BatchD2CWorker(QThread): log = pyqtSignal(str) done = pyqtSignal() - def __init__(self, params: dict, depth_dir: str): + def __init__(self, params: dict, depth_dir: str, splat: bool = True): super().__init__() self._params = params self._depth_dir = depth_dir + self._splat = splat def run(self): class _LineCapture(io.StringIO): @@ -473,7 +520,7 @@ class BatchD2CWorker(QThread): cap = _LineCapture(self.log) with contextlib.redirect_stdout(cap): try: - batch_convert(self._params, self._depth_dir) + batch_convert(self._params, self._depth_dir, splat=self._splat) except Exception as exc: self.log.emit(f"Error: {exc}") self.done.emit() @@ -484,6 +531,8 @@ class BatchD2CWorker(QThread): # ───────────────────────────────────────────────────────────────── class MainWindow(QMainWindow): + back_to_profiles = pyqtSignal() + def __init__(self, pipeline, color_profile, depth_profile): super().__init__() self.setWindowTitle("Orbbec 数据采集 & D2C") @@ -491,12 +540,27 @@ class MainWindow(QMainWindow): self._session_dir: Path | None = None self._batch_worker: BatchD2CWorker | None = None + self._params_filename = "camera_params.json" # 由 SN+分辨率确定后更新 + self._restart_mode = False # 正在重启，不触发 app.quit + + # 保存选中的流参数，供断线重连时匹配 + self._saved_color_info = { + "width": color_profile.get_width(), + "height": color_profile.get_height(), + "fps": color_profile.get_fps(), + } + self._saved_depth_info = { + "width": depth_profile.get_width(), + "height": depth_profile.get_height(), + "fps": depth_profile.get_fps(), + } self._worker = CameraWorker(pipeline, color_profile, depth_profile) self._worker.preview_ready.connect(self._update_preview) self._worker.capture_done.connect(self._on_capture_done) self._worker.params_ready.connect(self._on_params_ready) self._worker.log_msg.connect(self._append_log) + self._worker.disconnected.connect(self._on_camera_disconnected) self._build_ui() self._worker.start() @@ -630,6 +694,17 @@ class MainWindow(QMainWindow): row2.addWidget(br2) layout.addLayout(row2) + # Splat 选项 + opt_row = QHBoxLayout() + self.splat_check = QCheckBox("Splat 3×3 填充（减少正向投影空洞，默认开启）") + self.splat_check.setChecked(True) + self.splat_check.setToolTip( + "对应 d2c_batch.py 的 splat 参数：将每个深度像素扩散到 3×3 邻域，" + "有效减少软件 D2C 正向投影产生的空洞。") + opt_row.addWidget(self.splat_check) + opt_row.addStretch() + layout.addLayout(opt_row) + # 运行按钮 run_row = QHBoxLayout() run_row.addStretch() @@ -680,14 +755,23 @@ class MainWindow(QMainWindow): self.session_status.setStyleSheet("color: #2a9a2a; font-size: 12px;") def _on_params_ready(self, params: dict): + # 根据 SN + 分辨率/帧率构建参数文件名 + sn = params.get("device", {}).get("serial_number", "unknown") + cw = params["color"]["width"]; ch = params["color"]["height"] + cfps = params["color"]["fps"] + dw = params["depth"]["width"]; dh = params["depth"]["height"] + dfps = params["depth"]["fps"] + self._params_filename = ( + f"SN{sn}_RGB{cw}x{ch}@{cfps}_D{dw}x{dh}@{dfps}.json" + ) if self._session_dir and self._session_dir.exists(): self._save_params_to_session(params) def _save_params_to_session(self, params: dict): try: - p = self._session_dir / "camera_params.json" - with open(p, "w") as fh: - json.dump(params, fh, indent=2) + p = self._session_dir / self._params_filename + p.write_text(json.dumps(params, indent=2, ensure_ascii=False), + encoding="utf-8") except Exception: pass @@ -734,19 +818,26 @@ class MainWindow(QMainWindow): edit.setText(f) def _resolve_params_file(self) -> str | None: - """返回有效的 camera_params.json 路径，找不到返回 None。""" + """返回有效的相机参数 JSON 路径，找不到返回 None。""" explicit = self.params_path_edit.text().strip() if explicit and Path(explicit).is_file(): return explicit - # 默认路径 - for candidate in [ - Path(PARAMS_FILE), - *(([self._session_dir / "camera_params.json"] - if self._session_dir else [])), - ]: - if candidate.is_file(): - return str(candidate) + # 优先在 session 目录查找 SN 命名的文件 + if self._session_dir and self._session_dir.exists(): + sn_files = sorted(self._session_dir.glob("SN*.json")) + if sn_files: + return str(sn_files[0]) + fallback = self._session_dir / "camera_params.json" + if fallback.is_file(): + return str(fallback) + + # 工作目录里的 SN 命名文件或旧版 camera_params.json + sn_files = sorted(Path(".").glob("SN*.json")) + if sn_files: + return str(sn_files[0]) + if Path(PARAMS_FILE).is_file(): + return PARAMS_FILE return None def _run_batch_d2c(self): @@ -771,7 +862,8 @@ class MainWindow(QMainWindow): self.d2c_log.append(f"[深度图目录] {depth_dir}\n") self.run_d2c_btn.setEnabled(False) - self._batch_worker = BatchD2CWorker(params, depth_dir) + splat = self.splat_check.isChecked() + self._batch_worker = BatchD2CWorker(params, depth_dir, splat=splat) self._batch_worker.log.connect(self.d2c_log.append) self._batch_worker.done.connect(self._on_batch_done) self._batch_worker.start() @@ -780,10 +872,94 @@ class MainWindow(QMainWindow): self.run_d2c_btn.setEnabled(True) self.d2c_log.append("\n[ 转换完成 ]") + # ── Camera disconnect / reconnect ── + + def _on_camera_disconnected(self): + self.capture_btn.setEnabled(False) + + dlg = QMessageBox(self) + dlg.setWindowTitle("相机断开连接") + dlg.setText("检测到相机已断开连接，请选择操作：") + dlg.setIcon(QMessageBox.Warning) + reconnect_btn = dlg.addButton("重新连接相机", QMessageBox.AcceptRole) + back_btn = dlg.addButton("返回分辨率选择", QMessageBox.RejectRole) + exit_btn = dlg.addButton("退出程序", QMessageBox.DestructiveRole) + dlg.exec_() + + clicked = dlg.clickedButton() + if clicked == reconnect_btn: + self._do_reconnect() + elif clicked == back_btn: + self._restart_mode = True + self.hide() + self.back_to_profiles.emit() # → _open_session → 新窗口 + self._worker.stop() + self.close() + else: + QApplication.quit() + + def _do_reconnect(self): + """用相同的分辨率配置重新连接相机。""" + prog = QMessageBox(QMessageBox.Information, "重新连接", + "正在重新连接相机，请稍候…", + QMessageBox.NoButton, self) + prog.setWindowModality(Qt.ApplicationModal) + prog.show() + QApplication.processEvents() + time.sleep(2) + + try: + from pyorbbecsdk import Pipeline, OBSensorType + + new_pipeline = Pipeline() + + def _best_profile(profile_list, target: dict): + for i in range(profile_list.get_count()): + p = profile_list.get_stream_profile_by_index(i) + vp = p.as_video_stream_profile() + if vp is None: + continue + if (vp.get_width() == target["width"] + and vp.get_height() == target["height"] + and vp.get_fps() == target["fps"]): + return vp + # 没找到精确匹配时用第一个 + p = profile_list.get_stream_profile_by_index(0) + return p.as_video_stream_profile() + + new_color = _best_profile( + new_pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR), + self._saved_color_info) + new_depth = _best_profile( + new_pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR), + self._saved_depth_info) + + prog.accept() + + old_worker = self._worker + self._worker = CameraWorker(new_pipeline, new_color, new_depth) + self._worker.preview_ready.connect(self._update_preview) + self._worker.capture_done.connect(self._on_capture_done) + self._worker.params_ready.connect(self._on_params_ready) + self._worker.log_msg.connect(self._append_log) + self._worker.disconnected.connect(self._on_camera_disconnected) + self._worker.start() + + self.capture_btn.setEnabled(True) + old_worker.deleteLater() + + except Exception as exc: + prog.accept() + QMessageBox.critical(self, "重新连接失败", + f"无法重新连接相机：{exc}\n\n请检查连接后重试。") + # 再次弹出选项 + self._on_camera_disconnected() + # ── Window close ── def closeEvent(self, event): self._worker.stop() + # _restart_mode 时由 _open_session 保持事件循环，不退出 app super().closeEvent(event) @@ -791,11 +967,28 @@ class MainWindow(QMainWindow): # Entry Point # ───────────────────────────────────────────────────────────────── +def _open_session(app: QApplication) -> None: + """显示分辨率选择对话框，成功后打开主窗口；取消则退出 app。""" + dlg = ProfileDialog() + if dlg.exec_() != QDialog.Accepted: + app.quit() + return + + if dlg.pipeline is None or dlg.selected_color is None or dlg.selected_depth is None: + QMessageBox.critical(None, "错误", "相机初始化失败，请检查设备连接后重启程序。") + app.quit() + return + + win = MainWindow(dlg.pipeline, dlg.selected_color, dlg.selected_depth) + # 断线→返回选择界面时，递归调用本函数 + win.back_to_profiles.connect(lambda: _open_session(app)) + win.show() + + def main(): app = QApplication(sys.argv) app.setStyle("Fusion") - # 深色调色板（可选，让界面更现代） from PyQt5.QtGui import QPalette, QColor palette = QPalette() palette.setColor(QPalette.Window, QColor(45, 45, 45)) @@ -813,17 +1006,7 @@ def main(): palette.setColor(QPalette.HighlightedText, QColor(0, 0, 0)) app.setPalette(palette) - # 分辨率选择 - dlg = ProfileDialog() - if dlg.exec_() != QDialog.Accepted: - sys.exit(0) - - if dlg.pipeline is None or dlg.selected_color is None or dlg.selected_depth is None: - QMessageBox.critical(None, "错误", "相机初始化失败，请检查设备连接后重启程序。") - sys.exit(1) - - win = MainWindow(dlg.pipeline, dlg.selected_color, dlg.selected_depth) - win.show() + _open_session(app) sys.exit(app.exec_())