cyy/yiliao2026
1
0
Fork 0
forked from zbw/yiliao2026
Code Pull Requests Activity

vlm和tts

Browse Source
This commit is contained in:
Orange
2026-07-23 14:51:51 +08:00
parent 90761b40b2
commit a5cee5e142
35 changed files with 12694 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/vlm_detect/config/vlm_detect.yaml
View File

@@ -1,36 +1,18 @@
# vlm_detect 参数配置
# 使用: ros2 launch vlm_detect vlm_detect.launch.py
vlm_node:
ros__parameters:
# VLM 推理服务地址
vlm_host: "http://192.168.10.189:8000"
# 模型名称 (OpenAI 格式)
vlm_model: "./OpenGVLab/InternVL3-1B/"
# 订阅的压缩图像话题
image_topic: "/image_mjpeg"
# 订阅的触发信号话题
trigger_topic: "/sign4return"
# 触发信号值
trigger_sign: 9
# 发布结果的话题
result_topic: "/vlm_result"
# 发送给 VLM 的提示词
prompt_text: "描述图片中有一个病人的特征，字数控制在20字以内。"
# 最大输出 token 数
max_tokens: 100
tts_node:
ros__parameters:
# VLM 推理服务地址 (需与 vlm_node 一致)
vlm_host: "http://192.168.10.189:8000"
# 音频输出设备 (PulseAudio sink)
audio_sink: "alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo"
# 订阅 VLM 结果的话题 (需与 vlm_node 一致)
result_topic: "/vlm_result"
# TTS 语音 (edge-tts 语音名)
tts_voice: "zh-CN-XiaoxiaoNeural"
# 临时 MP3 存储路径
tmp_mp3_path: "/tmp/tts_out.mp3"
# 播放速度 (ffplay atempo, 范围 0.5~2.0)
tts_speed: 1.5

80
src/vlm_detect/launch/vlm_detect.launch.py
View File

@@ -1,13 +1,15 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
vlm_detect 联合启动文件
同时启动 vlm_node (图生文) 和 tts_node (语音播报)
vlm_detect 启动文件
同时启动 vlm_node (图生文) + tts_server (语音播报服务) + qr_tts_bridge (二维码播报)
用法:
ros2 launch vlm_detect vlm_detect.launch.py # 默认配置
ros2 launch vlm_detect vlm_detect.launch.py vlm_host:=http://... # 覆盖 VLM 服务地址
ros2 launch vlm_detect vlm_detect.launch.py use_tts:=false # 只启动 vlm_node
ros2 launch vlm_detect vlm_detect.launch.py # 默认全部
ros2 launch vlm_detect vlm_detect.launch.py vlm_host:=http://... # 指定 VLM 服务地址
ros2 launch vlm_detect vlm_detect.launch.py use_tts:=false # 关闭语音播报
ros2 launch vlm_detect vlm_detect.launch.py use_qr_tts:=false # 关闭二维码播报桥接
ros2 launch vlm_detect vlm_detect.launch.py use_vlm:=false # 只启动语音服务
"""
import os
@@ -23,7 +25,9 @@ from launch_ros.actions import Node
def generate_launch_description():
# ==================== Launch 参数 ====================
use_vlm = LaunchConfiguration('use_vlm')
use_tts = LaunchConfiguration('use_tts')
use_qr_tts = LaunchConfiguration('use_qr_tts')
config_file = LaunchConfiguration('config_file')
@@ -37,15 +41,22 @@ def generate_launch_description():
prompt_text = LaunchConfiguration('prompt_text')
max_tokens = LaunchConfiguration('max_tokens')
# tts_node 可覆盖参数
# tts_server 可覆盖参数
audio_sink = LaunchConfiguration('audio_sink')
tts_voice = LaunchConfiguration('tts_voice')
tts_speed = LaunchConfiguration('tts_speed')
# ==================== 声明参数 ====================
# ==================== 参数声明 ====================
declare_use_vlm = DeclareLaunchArgument(
'use_vlm', default_value='true',
description='启动 VLM 图生文节点')
declare_use_tts = DeclareLaunchArgument(
'use_tts', default_value='true',
description='是否同时启动 TTS 语音播报节点')
description='启动 TTS 语音播报服务')
declare_use_qr_tts = DeclareLaunchArgument(
'use_qr_tts', default_value='true',
description='启动二维码 → TTS 桥接节点')
declare_config_file = DeclareLaunchArgument(
'config_file',
@@ -57,47 +68,46 @@ def generate_launch_description():
# vlm_node 参数
declare_vlm_host = DeclareLaunchArgument(
'vlm_host', default_value='http://192.168.10.189:8000',
description='VLM 推理服务地址')
description='VLM 服务器地址')
declare_vlm_model = DeclareLaunchArgument(
'vlm_model', default_value='./OpenGVLab/InternVL3-1B/',
description='VLM 模型名称')
declare_image_topic = DeclareLaunchArgument(
'image_topic', default_value='/image_mjpeg',
description='订阅的压缩图像话题')
description='输入的压缩图像话题')
declare_trigger_topic = DeclareLaunchArgument(
'trigger_topic', default_value='/sign4return',
description='订阅的触发信号话题')
description='输入的触发信号话题')
declare_trigger_sign = DeclareLaunchArgument(
'trigger_sign', default_value='9',
description='触发信号值 (Int32)')
declare_result_topic = DeclareLaunchArgument(
'result_topic', default_value='/vlm_result',
description='发布 VLM 结果的话题')
description='输出 VLM 结果的话题')
declare_prompt_text = DeclareLaunchArgument(
'prompt_text', default_value='描述图片中有一个病人的特征，字数控制在20字以内。',
'prompt_text', default_value='请描述这张图片的内容，用一句简短的话概括，不超过20个字。',
description='发送给 VLM 的提示词')
declare_max_tokens = DeclareLaunchArgument(
'max_tokens', default_value='100',
description='最大输出 token 数')
description='最大生成 token 数')
# tts_node 参数
# tts_server 参数
declare_audio_sink = DeclareLaunchArgument(
'audio_sink',
default_value='alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo',
description='音频输出设备 (PulseAudio sink)')
declare_tts_voice = DeclareLaunchArgument(
'tts_voice', default_value='zh-CN-XiaoxiaoNeural',
description='TTS 语音名称 (edge-tts)')
declare_tts_speed = DeclareLaunchArgument(
'tts_speed', default_value='1.5',
description='播放速度倍率 (0.5~2.0)')
description='语速倍率 (0.5~2.0)')
# ==================== 节点 ====================
# VLM 图生文节点 (内部自带 TTS 服务客户端)
vlm_node = Node(
package='vlm_detect',
executable='vlm_node',
name='vlm_detect',
output='screen',
condition=IfCondition(use_vlm),
parameters=[config_file,
{
'vlm_host': vlm_host,
@@ -111,26 +121,35 @@ def generate_launch_description():
}],
)
tts_node = Node(
# TTS 语音播报服务端
tts_server = Node(
package='vlm_detect',
executable='tts_node',
name='tts_node',
executable='tts_server',
name='tts_server',
output='screen',
condition=IfCondition(use_tts),
parameters=[config_file,
{
'vlm_host': vlm_host,
'audio_sink': audio_sink,
'result_topic': result_topic,
'tts_voice': tts_voice,
'tts_speed': tts_speed,
}],
)
# ==================== 启动描述 ====================
# 二维码 → TTS 桥接 (订阅 qr_results，调用 /tts/speak)
qr_tts_bridge = Node(
package='vlm_detect',
executable='qr_tts_bridge',
name='qr_tts_bridge',
output='screen',
condition=IfCondition(use_qr_tts),
)
# ==================== 组装 ====================
return LaunchDescription([
# 参数声明
declare_use_vlm,
declare_use_tts,
declare_use_qr_tts,
declare_config_file,
declare_vlm_host,
declare_vlm_model,
@@ -141,12 +160,13 @@ def generate_launch_description():
declare_prompt_text,
declare_max_tokens,
declare_audio_sink,
declare_tts_voice,
declare_tts_speed,
# 节点
LogInfo(msg=['配置文件: ', config_file]),
LogInfo(msg=['VLM 服务: ', vlm_host]),
LogInfo(msg=['TTS 播报: ', use_tts]),
LogInfo(msg=['TTS 服务: ', use_tts]),
LogInfo(msg=['QR-TTS 桥接: ', use_qr_tts]),
vlm_node,
tts_node,
tts_server,
qr_tts_bridge,
])

2
src/vlm_detect/setup.py
View File

@@ -27,6 +27,8 @@ setup(
'vlm_node = vlm_detect.vlm_node:main',
'test_publisher = vlm_detect.test_publisher:main',
'tts_node = vlm_detect.tts_node:main',
'tts_server = vlm_detect.tts_server:main',
'qr_tts_bridge = vlm_detect.qr_tts_bridge:main',
],
},
)

BIN
src/vlm_detect/vlm_detect/__pycache__/test_publisher.cpython-310.pyc
View File

Binary file not shown.

BIN
src/vlm_detect/vlm_detect/__pycache__/tts_node.cpython-310.pyc
View File

Binary file not shown.

BIN
src/vlm_detect/vlm_detect/__pycache__/vlm_node.cpython-310.pyc
View File

Binary file not shown.

66
src/vlm_detect/vlm_detect/qr_tts_bridge.py Normal file
View File

@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""
QR 识别 → TTS 语音播报桥接节点
订阅 qr_results，调用 /tts/speak 服务
"""
import rclpy
from rclpy.node import Node
from std_msgs.msg import String
from origincar_msg.srv import Speak
class QrTtsBridge(Node):
def __init__(self):
super().__init__('qr_tts_bridge')
# TTS 服务客户端
self.tts_client = self.create_client(Speak, '/tts/speak')
while not self.tts_client.wait_for_service(timeout_sec=5.0):
self.get_logger().info('Waiting for TTS service...')
self.get_logger().info('TTS service connected')
# 订阅 QR 识别结果
self.sub = self.create_subscription(
String, 'qr_results', self.callback, 10)
self.get_logger().info('QR-TTS Bridge ready, listening on qr_results')
def callback(self, msg):
text = msg.data.strip()
if not text:
return
self.get_logger().info(f'QR result: {text}')
if not self.tts_client.service_is_ready():
self.get_logger().warning('TTS service not available')
return
req = Speak.Request()
req.text = text
future = self.tts_client.call_async(req)
future.add_done_callback(self._tts_done_callback)
def _tts_done_callback(self, future):
try:
resp = future.result()
if not resp.success:
self.get_logger().warning(f'TTS failed: {resp.message}')
except Exception as e:
self.get_logger().error(f'TTS call error: {e}')
def main(args=None):
rclpy.init(args=args)
node = QrTtsBridge()
try:
rclpy.spin(node)
except KeyboardInterrupt:
pass
finally:
node.destroy_node()
rclpy.shutdown()
if __name__ == '__main__':
main()

2
src/vlm_detect/vlm_detect/test_publisher.py
View File

@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
"""
测试发布者：发送图片和触发信号给 VLM 节点
用法: ros2 run vlm_detect test_publisher --ros-args -p image_path:="/path/to/image.jpg"
用法: ros2 run vlm_detect test_publisher --ros-args -p image_path:="/home/sunrise/yiliao_ws/my_model/image.png"
"""
import rclpy

70
src/vlm_detect/vlm_detect/tts_node.py
View File

@@ -1,60 +1,50 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import rclpy, subprocess, requests, os
import rclpy, subprocess, os, wave
from rclpy.node import Node
from std_msgs.msg import String
from piper import PiperVoice
from piper.config import SynthesisConfig
MODEL_PATH = '/home/sunrise/tts_model/zh_CN-huayan-medium.onnx'
class TTSNode(Node):
def __init__(self):
super().__init__("tts_node")
self.declare_parameter('vlm_host', 'http://192.168.10.189:8000')
self.declare_parameter('audio_sink',
'alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo')
super().__init__('tts_node')
self.declare_parameter('audio_sink', 'alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo')
self.declare_parameter('result_topic', '/vlm_result')
self.declare_parameter('tts_voice', 'zh-CN-XiaoxiaoNeural')
self.declare_parameter('tmp_mp3_path', '/tmp/tts_out.mp3')
self.declare_parameter('tts_speed', 1.5)
self.vlm_host = self.get_parameter('vlm_host').value
audio_sink = self.get_parameter('audio_sink').value
self.audio_sink = self.get_parameter('audio_sink').value
result_topic = self.get_parameter('result_topic').value
self.tts_voice = self.get_parameter('tts_voice').value
self.tmp_mp3 = self.get_parameter('tmp_mp3_path').value
self.tts_speed = self.get_parameter('tts_speed').value
self.audio_env = {**os.environ, "PULSE_SINK": audio_sink}
tts_speed = self.get_parameter('tts_speed').value
self.length_scale = 1.0 / tts_speed
self.espeak_speed = int(175 * tts_speed) # espeak default=175wpm, scale with tts_speed
self.sub = self.create_subscription(String, result_topic, self.callback, 10)
self.get_logger().info(
f"TTS 节点启动 | host={self.vlm_host} | sink={audio_sink} | "
f"voice={self.tts_voice} | speed={self.tts_speed}x"
)
self.get_logger().info(f'Piper TTS started | voice=zh_CN-huayan | speed={tts_speed}x | sink={self.audio_sink}')
self.get_logger().info(f'Loading model: {MODEL_PATH}')
self.voice = PiperVoice.load(MODEL_PATH)
self.get_logger().info('Model loaded OK')
def callback(self, msg):
text = msg.data
self.get_logger().info(f"语音播报: {text}")
text = msg.data.strip()
if not text:
return
self.get_logger().info(f'TTS: {text}')
try:
resp = requests.post(
f"{self.vlm_host}/v1/tts",
json={"text": text, "voice": self.tts_voice},
timeout=60
)
resp.raise_for_status()
with open(self.tmp_mp3, "wb") as f:
f.write(resp.content)
speed_str = f"atempo={self.tts_speed}"
syn_config = SynthesisConfig(length_scale=self.length_scale)
wav_path = '/tmp/tts_out.wav'
with wave.open(wav_path, 'wb') as wf:
self.voice.synthesize_wav(text, wf, syn_config=syn_config)
subprocess.Popen(
["ffplay", "-nodisp", "-autoexit", "-af", speed_str, self.tmp_mp3],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
env=self.audio_env
)
['paplay', f'--device={self.audio_sink}', wav_path],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except Exception as e:
self.get_logger().error(f"TTS 失败, 降级 espeak: {e}")
self.get_logger().error(f'Piper TTS failed, fallback espeak: {e}')
subprocess.Popen(
["espeak-ng", "-v", "zh", "-s", "150", text],
env=self.audio_env
)
['espeak-ng', '-v', 'cmn', '-s', str(self.espeak_speed), text],
env={**os.environ, 'PULSE_SINK': self.audio_sink},
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def main(args=None):
rclpy.init(args=args)
@@ -67,5 +57,5 @@ def main(args=None):
node.destroy_node()
rclpy.shutdown()
if __name__ == "__main__":
if __name__ == '__main__':
main()

90
src/vlm_detect/vlm_detect/tts_server.py Normal file
View File

@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""
TTS 语音播报服务端 (ROS2 Service Server)
服务类型: origincar_msg/srv/Speak
主 TTS: Piper 离线模型 降级: espeak-ng
"""
import rclpy
import subprocess
import os
import wave
from rclpy.node import Node
from origincar_msg.srv import Speak
from piper import PiperVoice
from piper.config import SynthesisConfig
MODEL_PATH = '/home/sunrise/tts_model/zh_CN-huayan-medium.onnx'
class TTSServer(Node):
def __init__(self):
super().__init__('tts_server')
self.declare_parameter('audio_sink',
'alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo')
self.declare_parameter('tts_speed', 1.5)
self.audio_sink = self.get_parameter('audio_sink').value
tts_speed = self.get_parameter('tts_speed').value
self.length_scale = 1.0 / tts_speed
self.espeak_speed = int(175 * tts_speed)
self.srv = self.create_service(Speak, '/tts/speak', self.handle_speak)
self.get_logger().info(f'TTS Server ready | voice=zh_CN-huayan | speed={tts_speed}x | sink={self.audio_sink}')
self.get_logger().info(f'Loading model: {MODEL_PATH}')
self.voice = PiperVoice.load(MODEL_PATH)
self.get_logger().info('Model loaded OK')
def handle_speak(self, request, response):
text = request.text.strip()
if not text:
response.success = False
response.message = 'empty text'
return response
self.get_logger().info(f'TTS: {text}')
try:
syn_config = SynthesisConfig(length_scale=self.length_scale)
wav_path = '/tmp/tts_out.wav'
with wave.open(wav_path, 'wb') as wf:
self.voice.synthesize_wav(text, wf, syn_config=syn_config)
subprocess.Popen(
['paplay', f'--device={self.audio_sink}', wav_path],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
response.success = True
response.message = 'ok'
except Exception as e:
self.get_logger().error(f'Piper TTS failed, fallback espeak: {e}')
try:
subprocess.Popen(
['espeak-ng', '-v', 'cmn', '-s', str(self.espeak_speed), text],
env={**os.environ, 'PULSE_SINK': self.audio_sink},
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
response.success = True
response.message = 'ok (espeak fallback)'
except Exception as e2:
self.get_logger().error(f'espeak also failed: {e2}')
response.success = False
response.message = str(e2)
return response
def main(args=None):
rclpy.init(args=args)
node = TTSServer()
try:
rclpy.spin(node)
except KeyboardInterrupt:
pass
finally:
node.destroy_node()
rclpy.shutdown()
if __name__ == '__main__':
main()

46
src/vlm_detect/vlm_detect/vlm_node.py
View File

@@ -1,5 +1,8 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
VLM 图生文节点 —— 收到触发信号后拍图发给 VLM 服务，结果调用 TTS 服务播报
"""
import rclpy
from rclpy.node import Node
from std_msgs.msg import Int32, String
@@ -13,6 +16,9 @@ import os
import time
import numpy as np
from origincar_msg.srv import Speak
class VLMProcessor(Node):
def __init__(self):
super().__init__('vlm_detect')
@@ -24,7 +30,7 @@ class VLMProcessor(Node):
self.declare_parameter('trigger_topic', '/sign4return')
self.declare_parameter('trigger_sign', 9)
self.declare_parameter('result_topic', '/vlm_result')
self.declare_parameter('prompt_text', '描述图片中有一个病人的特征，字数控制在20字以内。')
self.declare_parameter('prompt_text', '请描述这张图片的内容，用一句简短的话概括，不超过20个字。')
self.declare_parameter('max_tokens', 100)
vlm_host = self.get_parameter('vlm_host').value
@@ -43,7 +49,7 @@ class VLMProcessor(Node):
)
self.vlm_model = vlm_model
# ROS2 组件
# ROS2 通信
self.bridge = CvBridge()
self.latest_image = None
self.image_lock = threading.Lock()
@@ -56,8 +62,13 @@ class VLMProcessor(Node):
)
self.result_pub = self.create_publisher(String, result_topic, 10)
# TTS 服务客户端
self.tts_client = self.create_client(Speak, '/tts/speak')
while not self.tts_client.wait_for_service(timeout_sec=5.0):
self.get_logger().info('Waiting for TTS service...')
self.get_logger().info(
f"VLM Processor 启动 | host={vlm_host} | model={vlm_model} | "
f"VLM Processor 就绪 | host={vlm_host} | model={vlm_model} | "
f"image={image_topic} | trigger={trigger_topic}(sign={self.trigger_sign})"
)
@@ -73,7 +84,7 @@ class VLMProcessor(Node):
def sign_callback(self, msg):
if msg.data == self.trigger_sign:
self.get_logger().info(f"收到触发信号 ({msg.data}), 开始处理...")
self.get_logger().info(f"收到触发信号 ({msg.data}), 开始推理...")
with self.image_lock:
if self.latest_image is None:
self.get_logger().warning("无可用图片")
@@ -85,12 +96,34 @@ class VLMProcessor(Node):
try:
description = self.process_image(temp_path)
self.get_logger().info(f"图像描述: {description}")
# 发布结果到话题
result_msg = String()
result_msg.data = description
self.result_pub.publish(result_msg)
# 调用 TTS 服务播报
if self.tts_client.service_is_ready():
req = Speak.Request()
req.text = description
future = self.tts_client.call_async(req)
future.add_done_callback(self._tts_done_callback)
else:
self.get_logger().warning('TTS service not available')
os.remove(temp_path)
except Exception as e:
self.get_logger().error(f"图像处理出错: {e}")
self.get_logger().error(f"图像推理失败: {e}")
def _tts_done_callback(self, future):
try:
resp = future.result()
if resp.success:
self.get_logger().debug(f'TTS OK: {resp.message}')
else:
self.get_logger().warning(f'TTS failed: {resp.message}')
except Exception as e:
self.get_logger().error(f'TTS call error: {e}')
def process_image(self, image_path):
with open(image_path, "rb") as image_file:
@@ -109,10 +142,12 @@ class VLMProcessor(Node):
]
}],
max_tokens=self.max_tokens,
timeout=30,
)
self.get_logger().info(f"VLM 推理耗时 {time.time() - start_time:.1f}s")
return response.choices[0].message.content
def main(args=None):
rclpy.init(args=args)
node = VLMProcessor()
@@ -124,5 +159,6 @@ def main(args=None):
node.destroy_node()
rclpy.shutdown()
if __name__ == '__main__':
main()