diff --git a/my_model/image.png b/my_model/image.png index 2ce763e..f0d4d3b 100644 Binary files a/my_model/image.png and b/my_model/image.png differ diff --git a/src/vlm_detect/config/vlm_detect.yaml b/src/vlm_detect/config/vlm_detect.yaml index ebce0bf..e7f073d 100644 --- a/src/vlm_detect/config/vlm_detect.yaml +++ b/src/vlm_detect/config/vlm_detect.yaml @@ -1,18 +1,18 @@ -# vlm_detect 参数配置 - -vlm_node: - ros__parameters: - vlm_host: "http://192.168.10.189:8000" - vlm_model: "./OpenGVLab/InternVL3-1B/" - image_topic: "/image_mjpeg" - trigger_topic: "/sign4return" - trigger_sign: 9 - result_topic: "/vlm_result" - prompt_text: "描述图片中有一个病人的特征，字数控制在20字以内。" - max_tokens: 100 - tts_node: ros__parameters: - audio_sink: "alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo" - result_topic: "/vlm_result" + audio_sink: alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo + result_topic: /vlm_result tts_speed: 1.5 +vlm_detect: + ros__parameters: + crop_ratio: 0.45 + image_max_dim: 96 + image_topic: /image + max_tokens: 30 + prompt_text: 忽略白色边框。描述图中医院病房场景：一个人在医院病床上，盖着白色被子，画风为2D动漫插画。对人物外观特征高度抽象，称呼为「一个病人」。30字以内。 + result_topic: /vlm_result + temperature: 0.1 + trigger_sign: 9 + trigger_topic: /sign4return + vlm_host: http://192.168.10.189:8000 + vlm_model: /home/wisdom/models/gguf/Qwen2-VL-2B-Instruct-Q4_K_M.gguf diff --git a/src/vlm_detect/dataflow.md b/src/vlm_detect/dataflow.md new file mode 100644 index 0000000..9a79cd7 --- /dev/null +++ b/src/vlm_detect/dataflow.md @@ -0,0 +1,222 @@ +# VLM + QR → TTS 数据流关系 + +## 两条数据流总览 + +``` + ┌──────────────┐ + │ USB Camera │ + │ /dev/video0 │ + └──────┬───────┘ + │ + hobot_usb_cam (car_usb_cam) + │ + ┌────────────┼────────────┐ + │ │ │ + /image_mjpeg /image (其他话题) + │ │ + ▼ ▼ + ┌─────────────┐ ┌──────────────┐ + │ vlm_node │ │ qr_dete_node │ + │ (Python) │ │ (C++) │ + └──────┬──────┘ └──────┬───────┘ + │ │ + │ /qr_results + │ │ + │ ┌──────▼───────┐ + │ │ qr_tts_bridge│ + │ │ (Python) │ + │ └──────┬───────┘ + │ │ + ▼ ▼ + ┌────────────────────────────────┐ + │ /tts/speak │ + │ (origincar_msg/Speak) │ + │ │ + │ tts_server │ + │ Piper TTS + paplay │ + └────────────────────────────────┘ +``` + +--- + +## 数据流 1：VLM 图生文 → TTS + +### 触发条件 + +| 话题 | 类型 | 说明 | +|------|------|------| +| `/sign4return` | `std_msgs/Int32` | `data=9` 触发 VLM 推理 | + +### 数据流 + +``` +/image_mjpeg /sign4return (9) +(CompressedImage) (Int32) + │ │ + ▼ ▼ +┌─────────────────────────────────────────────────┐ +│ vlm_node │ +│ │ +│ 1. image_callback(): 缓存最新帧到 self.latest_image │ +│ 2. sign_callback(): 收到 data==9 时触发 │ +│ 3. process_image(): │ +│ a. (可选) 自动裁剪照片区域 (auto_crop) │ +│ b. 缩放到 max_dim (默认 96px) │ +│ c. JPEG 压缩 (quality=60) │ +│ d. Base64 编码 │ +│ e. POST → VLM HTTP API (/v1/chat/completions) │ +│ f. 返回文字描述 │ +│ 4. 发布结果到 /vlm_result │ +│ 5. 调用 /tts/speak 服务播报文字 │ +└────────────┬──────────────────┬──────────────────┘ + │ │ + ▼ ▼ + /vlm_result /tts/speak 服务 + (std_msgs/String) (origincar_msg/Speak) +``` + +### 关键参数 + +| 参数 | 值 | 说明 | +|------|-----|------| +| `image_max_dim` | 96 | 图片最大边长，越小越快 | +| `max_tokens` | 30 | VLM 生成最大 token 数 | +| `temperature` | 0.1 | 低温度 = 更确定 | +| `auto_crop` | false | CPU 优化，默认关闭自动裁剪 | +| `vlm_host` | `http://192.168.10.189:8000` | VLM 服务地址 (Windows WSL) | +| `vlm_model` | `Qwen2-VL-2B-Instruct-Q4_K_M` | 视觉语言模型 | + +--- + +## 数据流 2：QR 二维码识别 → TTS + +### 触发条件 + +| 话题 | 类型 | 说明 | +|------|------|------| +| `/sign4return` | `std_msgs/Int32` | `data=0` 开启二维码检测，`data=5` 关闭 | + +### 数据流 + +``` +/image (摄像头发布) +(CompressedImage) + │ + ▼ +┌──────────────────────────────┐ +│ hobot_codec (可选) │ +│ NV12 解码 → 共享内存 │ +│ /hbmem_img │ +└──────────────┬───────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────┐ +│ qr_detect / qr_dete_node (C++) │ +│ │ +│ 1. 订阅图像话题 │ +│ 2. ZBar / OpenCV 二维码检测 │ +│ 3. 发布识别结果到 /qr_results (std_msgs/String) │ +│ 4. 控制 /sign4return: │ +│ - 检测到时发 0 开启 │ +│ - 检测后发 5 关闭 │ +└────────────┬──────────────────────────────────────┘ + │ + ▼ /qr_results +┌──────────────────────────────────────────────────┐ +│ qr_tts_bridge (Python) │ +│ │ +│ 1. 订阅 /qr_results │ +│ 2. 收到二维码文字后调用 /tts/speak 服务 │ +└────────────┬──────────────────────────────────────┘ + │ + ▼ + /tts/speak 服务 + (origincar_msg/Speak) +``` + +--- + +## TTS 服务 (两条流共享) + +### 服务定义 + +``` +# origincar_msg/srv/Speak +string text # 要播报的文字 +--- +bool success # 是否成功 +string message # 错误信息 +``` + +### 服务端 tts_server + +``` +/tts/speak 请求 (text) + │ + ▼ +┌─────────────────────────────────────┐ +│ tts_server │ +│ │ +│ 1. Piper TTS 引擎 │ +│ 模型: zh_CN-huayan-medium.onnx │ +│ 生成 WAV 文件到 /tmp/tts_out.wav│ +│ 2. paplay 播放到 USB 音频设备 │ +│ 设备: USB Audio Device │ +│ 3. 失败回退: espeak-ng (cmn) │ +└─────────────────────────────────────┘ +``` + +| 参数 | 值 | 说明 | +|------|-----|------| +| `tts_speed` | 1.5 | 语速倍数 | +| `audio_sink` | USB Audio Device | PulseAudio 音频输出设备 | +| `voice` | zh_CN-huayan-medium | 中文女声 Piper 模型 | + +--- + +## /sign4return 统一控制协议 + +| 值 | 作用 | 使用者 | +|----|------|--------| +| 0 | 开启二维码检测 | `qr_detection` | +| 5 | 关闭二维码检测 | `qr_detection` | +| 9 | 触发 VLM 图像分析 | `vlm_node` | +| 10 | 切换 Nav2 普通档 | `nav2_profile_tuner` | +| 11 | 切换 Nav2 任务二档 | `nav2_profile_tuner` | + +--- + +## 完整比赛流程 (racing_control 编排) + +``` +1. 启动各节点 → 机器人开始寻线 +2. 摄像头检测到二维码 + → /qr_results = "1" + → qr_tts_bridge → /tts/speak("检测到一号二维码") + → /sign4return = 5 (关二维码) +3. 导航到闸门前 (entry_pose) +4. 通过闸门 → 切换赛道 +5. VLM 图像分析 + → /sign4return = 9 + → vlm_node 拍照 → VLM 推理 → /tts/speak(图像描述) +6. 导航到目标点 (vlm_pose) +7. 返回起点 (origin_pose) +``` + +--- + +## 一键启动命令 + +```bash +# VLM + TTS (两条流的核心服务) +source /opt/ros/humble/setup.bash && source /home/sunrise/yiliao_ws/install/setup.bash && ros2 launch vlm_detect vlm_detect.launch.py + +# QT + TTS 桥接 +source /opt/ros/humble/setup.bash && source /home/sunrise/yiliao_ws/install/setup.bash && ros2 launch vlm_detect vlm_detect.launch.py use_qr_tts:=true + +# 二维码检测 +source /opt/ros/humble/setup.bash && source /home/sunrise/yiliao_ws/install/setup.bash && ros2 launch qr_detection qr_detect.launch.py + +# 测试 VLM 推理 +source /opt/ros/humble/setup.bash && source /home/sunrise/yiliao_ws/install/setup.bash && ros2 run vlm_detect test_publisher --ros-args -p image_path:="/home/sunrise/yiliao_ws/my_model/image.png" +``` diff --git a/src/vlm_detect/launch/vlm_detect.launch.py b/src/vlm_detect/launch/vlm_detect.launch.py index 4dc0152..66ae7c4 100644 --- a/src/vlm_detect/launch/vlm_detect.launch.py +++ b/src/vlm_detect/launch/vlm_detect.launch.py @@ -70,10 +70,10 @@ def generate_launch_description(): 'vlm_host', default_value='http://192.168.10.189:8000', description='VLM 服务器地址') declare_vlm_model = DeclareLaunchArgument( - 'vlm_model', default_value='./OpenGVLab/InternVL3-1B/', + 'vlm_model', default_value='/home/wisdom/models/gguf/Qwen2-VL-2B-Instruct-Q4_K_M.gguf', description='VLM 模型名称') declare_image_topic = DeclareLaunchArgument( - 'image_topic', default_value='/image_mjpeg', + 'image_topic', default_value='/image', description='输入的压缩图像话题') declare_trigger_topic = DeclareLaunchArgument( 'trigger_topic', default_value='/sign4return', @@ -116,7 +116,6 @@ def generate_launch_description(): 'trigger_topic': trigger_topic, 'trigger_sign': trigger_sign, 'result_topic': result_topic, - 'prompt_text': prompt_text, 'max_tokens': max_tokens, }], ) diff --git a/src/vlm_detect/vlm_detect/test_publisher.py b/src/vlm_detect/vlm_detect/test_publisher.py index a29a8c1..70ba8aa 100644 --- a/src/vlm_detect/vlm_detect/test_publisher.py +++ b/src/vlm_detect/vlm_detect/test_publisher.py @@ -27,7 +27,7 @@ class TestPublisher(Node): interval = self.get_parameter("interval").value # 发布者 - self.image_pub = self.create_publisher(CompressedImage, "/image_mjpeg", 10) + self.image_pub = self.create_publisher(CompressedImage, "/image", 10) self.sign_pub = self.create_publisher(Int32, "/sign4return", 10) # 订阅结果 diff --git a/src/vlm_detect/vlm_detect/vlm_node.py b/src/vlm_detect/vlm_detect/vlm_node.py index b9c3eb5..bb80fcd 100644 --- a/src/vlm_detect/vlm_detect/vlm_node.py +++ b/src/vlm_detect/vlm_detect/vlm_node.py @@ -1,37 +1,33 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" -VLM 图生文节点 —— 收到触发信号后拍图发给 VLM 服务，结果调用 TTS 服务播报 -""" import rclpy from rclpy.node import Node from std_msgs.msg import Int32, String from sensor_msgs.msg import CompressedImage -from cv_bridge import CvBridge import cv2 import base64 import threading from openai import OpenAI -import os import time import numpy as np - from origincar_msg.srv import Speak class VLMProcessor(Node): def __init__(self): super().__init__('vlm_detect') - - # 声明 ROS2 参数 self.declare_parameter('vlm_host', 'http://192.168.10.189:8000') - self.declare_parameter('vlm_model', './OpenGVLab/InternVL3-1B/') + self.declare_parameter('vlm_model', '/home/wisdom/models/gguf/Qwen2-VL-2B-Instruct-Q4_K_M.gguf') self.declare_parameter('image_topic', '/image_mjpeg') self.declare_parameter('trigger_topic', '/sign4return') self.declare_parameter('trigger_sign', 9) self.declare_parameter('result_topic', '/vlm_result') - self.declare_parameter('prompt_text', '请描述这张图片的内容，用一句简短的话概括，不超过20个字。') - self.declare_parameter('max_tokens', 100) + self.declare_parameter('prompt_text', '') + self.declare_parameter('max_tokens', 30) + self.declare_parameter('image_max_dim', 96) + self.declare_parameter('temperature', 0.1) + self.declare_parameter('crop_ratio', 0.45) + self.declare_parameter('auto_crop', False) vlm_host = self.get_parameter('vlm_host').value vlm_model = self.get_parameter('vlm_model').value @@ -41,111 +37,108 @@ class VLMProcessor(Node): result_topic = self.get_parameter('result_topic').value self.prompt_text = self.get_parameter('prompt_text').value self.max_tokens = self.get_parameter('max_tokens').value + self.image_max_dim = self.get_parameter('image_max_dim').value + self.temperature = self.get_parameter('temperature').value + self.crop_ratio = self.get_parameter('crop_ratio').value + self.auto_crop = self.get_parameter('auto_crop').value - # 初始化 OpenAI 客户端 - self.client = OpenAI( - base_url=f"{vlm_host}/v1", - api_key="EMPTY", - ) + self.client = OpenAI(base_url=f"{vlm_host}/v1", api_key="EMPTY") self.vlm_model = vlm_model - - # ROS2 通信 - self.bridge = CvBridge() self.latest_image = None self.image_lock = threading.Lock() + self._busy = False - self.image_sub = self.create_subscription( - CompressedImage, image_topic, self.image_callback, 10 - ) - self.sign_sub = self.create_subscription( - Int32, trigger_topic, self.sign_callback, 10 - ) + self.image_sub = self.create_subscription(CompressedImage, image_topic, self.image_callback, 10) + self.sign_sub = self.create_subscription(Int32, trigger_topic, self.sign_callback, 10) self.result_pub = self.create_publisher(String, result_topic, 10) - # TTS 服务客户端 self.tts_client = self.create_client(Speak, '/tts/speak') while not self.tts_client.wait_for_service(timeout_sec=5.0): self.get_logger().info('Waiting for TTS service...') - self.get_logger().info( - f"VLM Processor 就绪 | host={vlm_host} | model={vlm_model} | " - f"image={image_topic} | trigger={trigger_topic}(sign={self.trigger_sign})" - ) + self.get_logger().info(f"VLM ready | host={vlm_host} | dim={self.image_max_dim} | crop={self.auto_crop}") def image_callback(self, msg): with self.image_lock: try: np_arr = np.frombuffer(msg.data, np.uint8) - cv_image = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) - self.latest_image = cv_image - self.get_logger().debug("图片已接收") + self.latest_image = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) except Exception as e: - self.get_logger().error(f"图片接收错误: {e}") + self.get_logger().error(f"Decode error: {e}") def sign_callback(self, msg): - if msg.data == self.trigger_sign: - self.get_logger().info(f"收到触发信号 ({msg.data}), 开始推理...") - with self.image_lock: - if self.latest_image is None: - self.get_logger().warning("无可用图片") - return - temp_path = "/tmp/vlm_temp_image.jpg" - cv2.imwrite(temp_path, self.latest_image) - self.get_logger().info(f"临时图片已保存: {temp_path}") - - try: - description = self.process_image(temp_path) - self.get_logger().info(f"图像描述: {description}") - - # 发布结果到话题 - result_msg = String() - result_msg.data = description - self.result_pub.publish(result_msg) - - # 调用 TTS 服务播报 - if self.tts_client.service_is_ready(): - req = Speak.Request() - req.text = description - future = self.tts_client.call_async(req) - future.add_done_callback(self._tts_done_callback) - else: - self.get_logger().warning('TTS service not available') - - os.remove(temp_path) - except Exception as e: - self.get_logger().error(f"图像推理失败: {e}") - - def _tts_done_callback(self, future): + if msg.data != self.trigger_sign: + return + if self._busy: + self.get_logger().warn("Busy, skip") + return + self.get_logger().info(f"Trigger {msg.data}") + with self.image_lock: + if self.latest_image is None: + self.get_logger().warning("No image") + return + img = self.latest_image.copy() + self._busy = True try: - resp = future.result() - if resp.success: - self.get_logger().debug(f'TTS OK: {resp.message}') - else: - self.get_logger().warning(f'TTS failed: {resp.message}') + t0 = time.time() + desc = self.process_image(img) + self.get_logger().info(f"Result({time.time()-t0:.1f}s): {desc}") + msg = String() + msg.data = desc + self.result_pub.publish(msg) + if self.tts_client.service_is_ready(): + req = Speak.Request() + req.text = desc + self.tts_client.call_async(req) except Exception as e: - self.get_logger().error(f'TTS call error: {e}') + self.get_logger().error(f"Inference: {e}") + finally: + self._busy = False - def process_image(self, image_path): - with open(image_path, "rb") as image_file: - base64_image = base64.b64encode(image_file.read()).decode('utf-8') - - start_time = time.time() - response = self.client.chat.completions.create( + def process_image(self, img): + if self.auto_crop: + img = self._detect_and_crop(img) + h, w = img.shape[:2] + max_dim = self.image_max_dim + if max(h, w) > max_dim: + scale = max_dim / max(h, w) + new_w, new_h = int(w * scale), int(h * scale) + img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_AREA) + else: + new_w, new_h = w, h + _, jpeg = cv2.imencode('.jpg', img, [cv2.IMWRITE_JPEG_QUALITY, 60]) + b64 = base64.b64encode(jpeg.tobytes()).decode('utf-8') + t0 = time.time() + resp = self.client.chat.completions.create( model=self.vlm_model, - messages=[{ - "role": "user", - "content": [ - {"type": "text", "text": self.prompt_text}, - {"type": "image_url", "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}" - }}, - ] - }], - max_tokens=self.max_tokens, - timeout=30, - ) - self.get_logger().info(f"VLM 推理耗时 {time.time() - start_time:.1f}s") - return response.choices[0].message.content + messages=[{"role": "user", "content": [ + {"type": "text", "text": self.prompt_text}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}} + ]}], + max_tokens=self.max_tokens, temperature=self.temperature, timeout=60) + self.get_logger().info(f"API {time.time()-t0:.1f}s {new_w}x{new_h} {len(jpeg)//1024}KB") + return resp.choices[0].message.content + + def _detect_and_crop(self, img): + h, w = img.shape[:2] + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + blurred = cv2.GaussianBlur(gray, (5, 5), 0) + edges = cv2.Canny(blurred, 50, 150) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) + edges = cv2.dilate(edges, kernel, iterations=1) + contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + best_rect, best_area = None, 0 + for cnt in contours: + peri = cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, 0.02 * peri, True) + area = cv2.contourArea(approx) + if len(approx) == 4 and area > w * h * 0.02 and area > best_area: + best_rect, best_area = approx, area + if best_rect is not None: + rx, ry, rw, rh = cv2.boundingRect(best_rect) + self.get_logger().info(f'Crop: {rw}x{rh}') + return img[ry:ry + rh, rx:rx + rw] + return img def main(args=None):