""" 高性能实时语音识别与命令生成系统 整合所有模块，实现从语音检测到命令发送的完整流程： 1. 音频采集（高性能模式） 2. 音频预处理（降噪+AGC） 3. VAD语音活动检测 4. STT语音识别 5. 文本预处理（纠错+参数提取） 6. 命令生成 7. Socket发送 性能优化： - 多线程异步处理 - 非阻塞音频采集 - LRU缓存优化 - 低延迟设计 """ import math import numpy as np import os import random import threading import queue import time from typing import Callable, Dict, List, Optional, Tuple, TYPE_CHECKING from voice_drone.core.audio import AudioCapture, AudioPreprocessor from voice_drone.core.vad import VAD from voice_drone.core.stt import STT from voice_drone.core.text_preprocessor import TextPreprocessor, get_preprocessor from voice_drone.core.command import Command from voice_drone.core.scoket_client import SocketClient from voice_drone.core.configuration import ( SYSTEM_AUDIO_CONFIG, SYSTEM_RECOGNIZER_CONFIG, SYSTEM_SOCKET_SERVER_CONFIG, ) from voice_drone.core.tts_ack_cache import ( compute_ack_pcm_fingerprint, load_cached_phrases, persist_phrases, ) from voice_drone.core.wake_word import WakeWordDetector, get_wake_word_detector from voice_drone.logging_ import get_logger if TYPE_CHECKING: from voice_drone.core.tts import KokoroOnnxTTS logger = get_logger("recognizer") class VoiceCommandRecognizer: """ 高性能实时语音命令识别器 完整的语音转命令系统，包括： - 音频采集和预处理 - 语音活动检测 - 语音识别 - 文本预处理和参数提取 - 命令生成 - Socket发送 """ def __init__(self, auto_connect_socket: bool = True): """ 初始化语音命令识别器 Args: auto_connect_socket: 是否自动连接Socket服务器 """ logger.info("初始化语音命令识别系统...") # 初始化各模块 self.audio_capture = AudioCapture() self.audio_preprocessor = AudioPreprocessor() self.vad = VAD() self.stt = STT() self.text_preprocessor = get_preprocessor() # 使用全局单例 self.wake_word_detector = get_wake_word_detector() # 使用全局单例 # Socket客户端 self.socket_client = SocketClient(SYSTEM_SOCKET_SERVER_CONFIG) self.auto_connect_socket = auto_connect_socket if self.auto_connect_socket: if not self.socket_client.connect(): logger.warning("Socket连接失败，将在发送命令时重试") # 语音段缓冲区 self.speech_buffer: list = [] # 存储语音音频块 self.speech_buffer_lock = threading.Lock() # 预缓冲区：保存语音检测前一小段音频，避免丢失开头 # 例如：pre_speech_max_seconds = 0.8 表示保留最近约 0.8 秒音频 self.pre_speech_buffer: list = [] # 存储最近的静音/背景音块 # 从系统配置读取（确保类型正确：YAML 可能把数值当字符串） self.pre_speech_max_seconds: float = float( SYSTEM_RECOGNIZER_CONFIG.get("pre_speech_max_seconds", 0.8) ) self.pre_speech_max_chunks: Optional[int] = None # 根据采样率和chunk大小动态计算 # 命令发送成功后的 TTS 反馈（懒加载 Kokoro，避免拖慢启动） self.ack_tts_enabled = bool(SYSTEM_RECOGNIZER_CONFIG.get("ack_tts_enabled", True)) self.ack_tts_text = str(SYSTEM_RECOGNIZER_CONFIG.get("ack_tts_text", "好的收到")).strip() self.ack_tts_phrases: Dict[str, List[str]] = self._normalize_ack_tts_phrases( SYSTEM_RECOGNIZER_CONFIG.get("ack_tts_phrases") ) # True：仅 ack_tts_phrases 中出现的命令会播报，且每次随机一句；False：全局 ack_tts_text（所有成功命令同一应答） self._ack_mode_phrases: bool = bool(self.ack_tts_phrases) self.ack_tts_prewarm = bool(SYSTEM_RECOGNIZER_CONFIG.get("ack_tts_prewarm", True)) self.ack_tts_prewarm_blocking = bool( SYSTEM_RECOGNIZER_CONFIG.get("ack_tts_prewarm_blocking", True) ) self.ack_pause_mic_for_playback = bool( SYSTEM_RECOGNIZER_CONFIG.get("ack_pause_mic_for_playback", True) ) self.ack_tts_disk_cache = bool( SYSTEM_RECOGNIZER_CONFIG.get("ack_tts_disk_cache", True) ) self._tts_engine: Optional["KokoroOnnxTTS"] = None # 阻塞预加载时缓存波形：全局单句 _tts_ack_pcm，或按命令随机模式下的 _tts_phrase_pcm_cache（每句一条） self._tts_ack_pcm: Optional[Tuple[np.ndarray, int]] = None self._tts_phrase_pcm_cache: Dict[str, Tuple[np.ndarray, int]] = {} self._tts_lock = threading.Lock() # 命令线程只入队，主线程 process_audio_stream 中统一播放（避免 Windows 下后台线程 sd.play 无声） self._ack_playback_queue: queue.Queue = queue.Queue(maxsize=8) # STT识别线程和队列 self.stt_queue = queue.Queue(maxsize=5) # STT识别队列 self.stt_thread: Optional[threading.Thread] = None # 命令处理线程和队列 self.command_queue = queue.Queue(maxsize=10) # 命令处理队列 self.command_thread: Optional[threading.Thread] = None # 运行状态 self.running = False # 命令序列号（用于去重和顺序保证） self.sequence_id = 0 self.sequence_lock = threading.Lock() logger.info( f"应答TTS配置: enabled={self.ack_tts_enabled}, " f"mode={'按命令随机短语' if self._ack_mode_phrases else '全局固定文案'}, " f"prewarm_blocking={self.ack_tts_prewarm_blocking}, " f"pause_mic={self.ack_pause_mic_for_playback}, " f"disk_cache={self.ack_tts_disk_cache}" ) if self._ack_mode_phrases: logger.info(f" 仅播报命令: {list(self.ack_tts_phrases.keys())}") # VAD 后端：silero（默认）或 energy（按块 RMS，Silero 在部分板载麦上长期无段时使用） _ev_env = os.environ.get("ROCKET_ENERGY_VAD", "").lower() in ( "1", "true", "yes", ) _yaml_backend = str( SYSTEM_RECOGNIZER_CONFIG.get("vad_backend", "silero") ).lower() self._use_energy_vad: bool = _ev_env or _yaml_backend == "energy" self._energy_rms_high: float = float( SYSTEM_RECOGNIZER_CONFIG.get("energy_vad_rms_high", 280) ) self._energy_rms_low: float = float( SYSTEM_RECOGNIZER_CONFIG.get("energy_vad_rms_low", 150) ) self._energy_start_chunks: int = int( SYSTEM_RECOGNIZER_CONFIG.get("energy_vad_start_chunks", 4) ) self._energy_end_chunks: int = int( SYSTEM_RECOGNIZER_CONFIG.get("energy_vad_end_chunks", 15) ) # 高噪底/AGC 下 RMS 几乎不低于 energy_vad_rms_low 时，用「相对本段峰值」辅助判停 self._energy_end_peak_ratio: float = float( SYSTEM_RECOGNIZER_CONFIG.get("energy_vad_end_peak_ratio", 0.88) ) # 说话过程中对 utt 峰值每块乘衰减再与当前 rms 取 max，避免前几个字特响导致后半句一直被判「相对衰减」而误切段 self._energy_utt_peak_decay: float = float( SYSTEM_RECOGNIZER_CONFIG.get("energy_vad_utt_peak_decay", 0.988) ) self._energy_utt_peak_decay = max(0.95, min(0.9999, self._energy_utt_peak_decay)) self._ev_speaking: bool = False self._ev_high_run: int = 0 self._ev_low_run: int = 0 self._ev_rms_peak: float = 0.0 self._ev_last_diag_time: float = 0.0 self._ev_utt_peak: float = 0.0 # 可选：能量 VAD 刚进入「正在说话」时回调（用于机端 PROMPT_LISTEN 计时清零等） self._vad_speech_start_hook: Optional[Callable[[], None]] = None _trail_raw = SYSTEM_RECOGNIZER_CONFIG.get("trailing_silence_seconds") if _trail_raw is not None: _trail = float(_trail_raw) if _trail > 0: fs = int(SYSTEM_AUDIO_CONFIG.get("frame_size", 1024)) sr = int(SYSTEM_AUDIO_CONFIG.get("sample_rate", 16000)) if fs > 0 and sr > 0: n_end = max(1, int(math.ceil(_trail * sr / fs))) self._energy_end_chunks = n_end self.vad.silence_end_frames = n_end logger.info( "VAD 句尾切段：trailing_silence_seconds=%.2f → 连续静音块数=%d " "（每块≈%.0fms，Silero 与 energy 共用）", _trail, n_end, 1000.0 * fs / sr, ) if self._use_energy_vad: logger.info( "VAD 后端: energy（RMS）" f" high={self._energy_rms_high} low={self._energy_rms_low} " f"start_chunks={self._energy_start_chunks} end_chunks={self._energy_end_chunks}" f" end_peak_ratio={self._energy_end_peak_ratio}" f" utt_peak_decay={self._energy_utt_peak_decay}" ) logger.info("语音命令识别系统初始化完成") @staticmethod def _normalize_ack_tts_phrases(raw) -> Dict[str, List[str]]: """YAML: ack_tts_phrases: { takeoff: [\"...\", ...], ... }""" result: Dict[str, List[str]] = {} if not isinstance(raw, dict): return result for k, v in raw.items(): key = str(k).strip() if not key: continue if isinstance(v, list): phrases = [str(x).strip() for x in v if str(x).strip()] elif isinstance(v, str) and v.strip(): phrases = [v.strip()] else: phrases = [] if phrases: result[key] = phrases return result def _has_ack_tts_content(self) -> bool: if self._ack_mode_phrases: return any(bool(v) for v in self.ack_tts_phrases.values()) return bool(self.ack_tts_text) def _pick_ack_phrase(self, command_name: str) -> Optional[str]: if self._ack_mode_phrases: phrases = self.ack_tts_phrases.get(command_name) if not phrases: return None return random.choice(phrases) return self.ack_tts_text or None def _get_cached_pcm_for_phrase(self, phrase: str) -> Optional[Tuple[np.ndarray, int]]: """若启动阶段已预合成该句，则返回缓存，播报时不再跑 ONNX（低延迟）。""" if self._ack_mode_phrases: return self._tts_phrase_pcm_cache.get(phrase) if self._tts_ack_pcm is not None: return self._tts_ack_pcm return None def _ensure_tts_engine(self) -> "KokoroOnnxTTS": """懒加载 Kokoro（双检锁，避免多线程重复加载）。""" from voice_drone.core.tts import KokoroOnnxTTS if self._tts_engine is not None: return self._tts_engine with self._tts_lock: if self._tts_engine is None: logger.info("TTS: 正在加载 Kokoro 模型（首次约需十余秒）…") self._tts_engine = KokoroOnnxTTS() logger.info("TTS: Kokoro 模型加载完成") assert self._tts_engine is not None return self._tts_engine def _enqueue_ack_playback(self, command_name: str) -> None: """ 命令已成功发出后，将待播音频交给主线程队列。 不在此线程直接调用 sounddevice：Windows 上后台线程常出现播放完全无声。 """ if not self.ack_tts_enabled: return phrase = self._pick_ack_phrase(command_name) if not phrase: return try: cached = self._get_cached_pcm_for_phrase(phrase) if cached is not None: audio, sr = cached self._ack_playback_queue.put(("pcm", audio.copy(), sr), block=False) logger.info( f"命令已发送，已排队语音应答（主线程播放，预缓存）: {phrase!r}" ) print(f"[TTS] 已排队语音应答（主线程播放，预缓存）: {phrase!r}", flush=True) else: self._ack_playback_queue.put(("synth", phrase), block=False) logger.info( f"命令已发送，已排队语音应答（主线程合成+播放，无缓存，可能有数秒延迟）: {phrase!r}" ) print( f"[TTS] 已排队语音应答（主线程合成+播放，无缓存）: {phrase!r}", flush=True, ) except queue.Full: logger.warning("应答语音播放队列已满，跳过本次") def _before_audio_iteration(self) -> None: """主循环每轮开头（主线程）：子类可扩展以播放其它排队 TTS。""" self._drain_ack_playback_queue() def _drain_ack_playback_queue(self, recover_mic: bool = True) -> None: """在主线程中播放队列中的应答（与麦克风采集同进程、同主循环线程）。 Args: recover_mic: 播完后是否恢复麦克风；退出 shutdown 时应为 False，避免与 stop() 中关流冲突。 """ from voice_drone.core.tts import play_tts_audio, speak_text items: list = [] while True: try: items.append(self._ack_playback_queue.get_nowait()) except queue.Empty: break if not items: return mic_stopped = False if self.ack_pause_mic_for_playback: try: logger.info( "TTS: 已暂停麦克风采集以便扬声器播放（避免 Windows 下输入/输出同时开无声）" ) self.audio_capture.stop_stream() mic_stopped = True except Exception as e: logger.warning(f"暂停麦克风失败，将尝试直接播放: {e}") try: for item in items: try: kind = item[0] if kind == "pcm": _, audio, sr = item logger.info("TTS: 主线程播放应答（预缓存波形）") play_tts_audio(audio, sr) logger.info("TTS: 播放完成") elif kind == "synth": logger.info("TTS: 主线程合成并播放应答（无预缓存）") tts = self._ensure_tts_engine() text = item[1] if len(item) >= 2 else (self.ack_tts_text or "") speak_text(text, tts=tts) except Exception as e: logger.warning(f"应答语音播放失败: {e}", exc_info=True) finally: if mic_stopped and recover_mic: try: self.audio_capture.start_stream() try: self.audio_preprocessor.reset() except Exception as e: # noqa: BLE001 logger.debug("audio_preprocessor.reset: %s", e) # TTS 暂停期间若未凑齐「尾静音」帧，VAD 会一直保持 is_speaking=True； # 恢复后 detect_speech_start 会直接放弃，表现为「能恢复采集但再也不识别」。 self.vad.reset() with self.speech_buffer_lock: self.speech_buffer.clear() self.pre_speech_buffer.clear() logger.info("TTS: 麦克风采集已恢复（已重置 VAD 与语音缓冲）") except Exception as e: logger.error(f"麦克风采集恢复失败，请重启程序: {e}", exc_info=True) def _prewarm_tts_async(self) -> None: """后台预加载 TTS（仅当未使用阻塞预加载时）。""" if not self.ack_tts_enabled or not self._has_ack_tts_content() or not self.ack_tts_prewarm: return def _run() -> None: try: self._ensure_tts_engine() if self._ack_mode_phrases: logger.warning( "TTS: 当前为「按命令随机短语」且未使用阻塞预加载，" "各句首次播报可能仍有数秒延迟；若需低延迟请将 ack_tts_prewarm_blocking 设为 true。" ) except Exception as e: logger.warning(f"TTS 预加载失败（将在首次播报时重试）: {e}", exc_info=True) threading.Thread(target=_run, daemon=True, name="tts-prewarm").start() def _prewarm_tts_blocking(self) -> None: """启动时准备应答 PCM：优先读磁盘缓存（文案与 TTS 配置未变则跳过合成）；必要时加载 Kokoro 并合成。""" if not self.ack_tts_enabled or not self._has_ack_tts_content() or not self.ack_tts_prewarm: return use_disk = self.ack_tts_disk_cache logger.info("TTS: 正在准备语音反馈（磁盘缓存 / 合成）…") print("正在加载语音反馈…") try: if self._ack_mode_phrases: self._tts_phrase_pcm_cache.clear() seen: set = set() unique: List[str] = [] for lst in self.ack_tts_phrases.values(): for t in lst: p = str(t).strip() if p and p not in seen: seen.add(p) unique.append(p) if not unique: return fingerprint = compute_ack_pcm_fingerprint(unique, mode_phrases=True) missing = list(unique) if use_disk: loaded, missing = load_cached_phrases(unique, fingerprint) for ph, pcm in loaded.items(): self._tts_phrase_pcm_cache[ph] = pcm if not missing: self._tts_ack_pcm = None logger.info( "TTS: 已从磁盘加载全部应答波形（%d 句），跳过 Kokoro 加载与合成", len(unique), ) print("语音反馈已就绪（本地缓存），可以开始说话下指令。") return self._ensure_tts_engine() assert self._tts_engine is not None need = [p for p in unique if p not in self._tts_phrase_pcm_cache] for j, phrase in enumerate(need, start=1): logger.info( f"TTS: 合成应答句 {j}/{len(need)}: {phrase!r}" ) audio, sr = self._tts_engine.synthesize(phrase) self._tts_phrase_pcm_cache[phrase] = (audio, sr) self._tts_ack_pcm = None if use_disk: persist_phrases(fingerprint, dict(self._tts_phrase_pcm_cache)) logger.info( "TTS: 语音反馈已就绪（随机应答已缓存，播报低延迟）" ) print("语音反馈引擎已就绪，可以开始说话下指令。") else: text = (self.ack_tts_text or "").strip() if not text: return fingerprint = compute_ack_pcm_fingerprint( [], global_text=text, mode_phrases=False ) missing = [text] if use_disk: loaded, missing = load_cached_phrases([text], fingerprint) if text in loaded: self._tts_ack_pcm = loaded[text] if not missing: logger.info( "TTS: 已从磁盘加载全局应答波形，跳过 Kokoro 加载与合成" ) print("语音反馈已就绪（本地缓存），可以开始说话下指令。") return self._ensure_tts_engine() assert self._tts_engine is not None audio, sr = self._tts_engine.synthesize(text) self._tts_ack_pcm = (audio, sr) if use_disk: persist_phrases(fingerprint, {text: self._tts_ack_pcm}) logger.info( "TTS: 语音反馈引擎已就绪；已缓存应答语音，命令成功后将快速播报" ) print("语音反馈引擎已就绪，可以开始说话下指令。") except Exception as e: logger.warning( f"TTS: 启动阶段预加载失败，命令成功后可能延迟或无语音反馈: {e}", exc_info=True, ) @staticmethod def _init_sounddevice_output_probe() -> None: """在主线程探测默认输出设备；应答播报必须在主线程调用 sd.play。""" try: from voice_drone.core.tts import log_sounddevice_output_devices log_sounddevice_output_devices() import sounddevice as sd # type: ignore from voice_drone.core.tts import _sounddevice_default_output_index out_idx = _sounddevice_default_output_index() if out_idx is not None and int(out_idx) >= 0: info = sd.query_devices(int(out_idx)) logger.info( f"sounddevice 默认输出设备: {info.get('name', '?')} (index={out_idx})" ) sd.check_output_settings(samplerate=24000, channels=1, dtype="float32") # 预解析 tts.output_device，启动日志中可见实际用于播放的设备 from voice_drone.core.tts import get_playback_output_device_id get_playback_output_device_id() except Exception as e: logger.warning(f"sounddevice 输出设备探测失败，可能导致无法播音: {e}") def _get_next_sequence_id(self) -> int: """获取下一个命令序列号""" with self.sequence_lock: self.sequence_id += 1 return self.sequence_id @staticmethod def _int16_chunk_rms(chunk: np.ndarray) -> float: if chunk.size == 0: return 0.0 return float(np.sqrt(np.mean(chunk.astype(np.float64) ** 2))) def _submit_concatenated_speech_to_stt(self) -> None: """在持有 speech_buffer_lock 时调用：合并 speech_buffer 并送 STT，然后清空。""" if len(self.speech_buffer) == 0: return speech_audio = np.concatenate(self.speech_buffer) self.speech_buffer.clear() min_samples = int(self.audio_capture.sample_rate * 0.5) if len(speech_audio) >= min_samples: try: self.stt_queue.put(speech_audio.copy(), block=False) logger.debug( f"提交语音段到STT队列，长度: {len(speech_audio)} 采样点" ) if os.environ.get("ROCKET_PRINT_VAD", "").lower() in ( "1", "true", "yes", ): print( f"[VAD] 已送 STT，{len(speech_audio)} 采样点（≈{len(speech_audio) / float(self.audio_capture.sample_rate):.2f}s）", flush=True, ) except queue.Full: logger.warning("STT队列已满，跳过本次识别") elif os.environ.get("ROCKET_PRINT_VAD", "").lower() in ( "1", "true", "yes", ): print( f"[VAD] 语音段太短已丢弃（{len(speech_audio)} < {min_samples} 采样）", flush=True, ) def _energy_vad_on_chunk(self, processed_chunk: np.ndarray) -> None: rms = self._int16_chunk_rms(processed_chunk) _vad_diag = os.environ.get("ROCKET_PRINT_VAD", "").lower() in ( "1", "true", "yes", ) if _vad_diag: self._ev_rms_peak = max(self._ev_rms_peak, rms) now = time.monotonic() if now - self._ev_last_diag_time >= 3.0: print( f"[VAD] energy 诊断：近 3s 块 RMS 峰值≈{self._ev_rms_peak:.0f} " f"（high={self._energy_rms_high} low={self._energy_rms_low}）", flush=True, ) self._ev_rms_peak = 0.0 self._ev_last_diag_time = now if not self._ev_speaking: if rms >= self._energy_rms_high: self._ev_high_run += 1 else: self._ev_high_run = 0 if self._ev_high_run >= self._energy_start_chunks: self._ev_speaking = True self._ev_high_run = 0 self._ev_low_run = 0 self._ev_utt_peak = rms hook = self._vad_speech_start_hook if hook is not None: try: hook() except Exception as e: # noqa: BLE001 logger.debug("vad_speech_start_hook: %s", e, exc_info=True) with self.speech_buffer_lock: if self.pre_speech_buffer: self.speech_buffer = list(self.pre_speech_buffer) else: self.speech_buffer.clear() self.speech_buffer.append(processed_chunk) logger.debug( "energy VAD: 开始收集语音段（含预缓冲约 %.2f s）", self.pre_speech_max_seconds, ) return with self.speech_buffer_lock: self.speech_buffer.append(processed_chunk) self._ev_utt_peak = max(rms, self._ev_utt_peak * self._energy_utt_peak_decay) below_abs = rms <= self._energy_rms_low below_rel = ( self._energy_end_peak_ratio > 0 and self._ev_utt_peak >= self._energy_rms_high and rms <= self._ev_utt_peak * self._energy_end_peak_ratio ) if below_abs or below_rel: self._ev_low_run += 1 else: self._ev_low_run = 0 if self._ev_low_run >= self._energy_end_chunks: self._ev_speaking = False self._ev_low_run = 0 self._ev_utt_peak = 0.0 with self.speech_buffer_lock: self._submit_concatenated_speech_to_stt() self._reset_agc_after_utterance_end() logger.debug("energy VAD: 语音段结束，已提交") def _reset_agc_after_utterance_end(self) -> None: """VAD 句尾：清 AGC 滑窗，避免巨响后 RMS 卡死。""" try: self.audio_preprocessor.reset_agc_state() except AttributeError: pass def discard_pending_stt_segments(self) -> int: """丢弃尚未被 STT 线程取走的整句，避免唤醒/播 TTS 关麦后仍识别旧段。""" n = 0 while True: try: self.stt_queue.get_nowait() self.stt_queue.task_done() n += 1 except queue.Empty: break if n: logger.info( "已丢弃 %s 条待 STT 的语音段（流程切换，避免与播 TTS 重叠）", n, ) return n def _stt_worker_thread(self): """STT识别工作线程（异步处理，不阻塞主流程）""" logger.info("STT识别线程已启动") while self.running: try: audio_data = self.stt_queue.get(timeout=0.1) except queue.Empty: continue except Exception as e: logger.error(f"STT工作线程错误: {e}", exc_info=True) continue try: if audio_data is None: break try: text = self.stt.invoke_numpy(audio_data) if os.environ.get("ROCKET_PRINT_STT", "").lower() in ( "1", "true", "yes", ): print( f"[STT] {text!r}" if (text and text.strip()) else "[STT] <空或不识别>", flush=True, ) if text and text.strip(): logger.info(f"🎤 STT识别结果: {text}") try: self.command_queue.put(text, block=False) logger.debug(f"文本已提交到命令处理队列: {text}") except queue.Full: logger.warning("命令处理队列已满，跳过本次识别结果") except Exception as e: logger.error(f"STT识别失败: {e}", exc_info=True) finally: self.stt_queue.task_done() logger.info("STT识别线程已停止") def _command_worker_thread(self): """命令处理工作线程（文本预处理+命令生成+Socket发送）""" logger.info("命令处理线程已启动") while self.running: try: text = self.command_queue.get(timeout=0.1) except queue.Empty: continue except Exception as e: logger.error(f"命令处理线程错误: {e}", exc_info=True) continue try: if text is None: break try: # 1. 检测唤醒词 is_wake, matched_wake_word = self.wake_word_detector.detect(text) if not is_wake: logger.debug(f"未检测到唤醒词，忽略文本: {text}") continue logger.info(f"🔔 检测到唤醒词: {matched_wake_word}") # 2. 提取命令文本（移除唤醒词） command_text = self.wake_word_detector.extract_command_text(text) if not command_text or not command_text.strip(): logger.warning(f"唤醒词后无命令内容: {text}") continue logger.debug(f"提取的命令文本: {command_text}") # 3. 文本预处理（快速模式，不进行分词） normalized_text, params = self.text_preprocessor.preprocess_fast(command_text) logger.debug(f"文本预处理结果:") logger.debug(f" 规范化文本: {normalized_text}") logger.debug(f" 命令关键词: {params.command_keyword}") logger.debug(f" 距离: {params.distance} 米") logger.debug(f" 速度: {params.speed} 米/秒") logger.debug(f" 时间: {params.duration} 秒") # 4. 检查是否识别到命令关键词 if not params.command_keyword: logger.warning(f"未识别到有效命令关键词: {normalized_text}") continue # 5. 生成命令 sequence_id = self._get_next_sequence_id() command = Command.create( command=params.command_keyword, sequence_id=sequence_id, distance=params.distance, speed=params.speed, duration=params.duration ) logger.info(f"📝 生成命令: {command.command}") logger.debug(f"命令详情: {command.to_dict()}") # 6. 发送命令到Socket服务器 if self.socket_client.send_command_with_retry(command): logger.info(f"✅ 命令已发送: {command.command} (序列号: {sequence_id})") self._enqueue_ack_playback(command.command) else: logger.warning( "命令未送达（已达 max_retries）: %s (序列号: %s)", command.command, sequence_id, ) except Exception as e: logger.error(f"命令处理失败: {e}", exc_info=True) finally: self.command_queue.task_done() logger.info("命令处理线程已停止") def start(self): """启动语音命令识别系统""" if self.running: logger.warning("语音命令识别系统已在运行") return # 先完成阻塞式 TTS 预加载，再开麦与识别线程，避免用户在预加载期间下指令导致无波形缓存、播报延迟 print("[TTS] 探测扬声器并预加载应答语音（可能需十余秒，请勿说话）…", flush=True) self._init_sounddevice_output_probe() if self.ack_tts_enabled and self._has_ack_tts_content() and self.ack_tts_prewarm: if self.ack_tts_prewarm_blocking: self._prewarm_tts_blocking() else: print( "[TTS] 已跳过启动预加载（ack_tts_enabled/应答文案/ack_tts_prewarm）", flush=True, ) self.running = True # 启动STT识别线程 self.stt_thread = threading.Thread(target=self._stt_worker_thread, daemon=True) self.stt_thread.start() # 启动命令处理线程 self.command_thread = threading.Thread(target=self._command_worker_thread, daemon=True) self.command_thread.start() # 启动音频采集 self.audio_capture.start_stream() if self.ack_tts_enabled and self._has_ack_tts_content() and self.ack_tts_prewarm: if not self.ack_tts_prewarm_blocking: self._prewarm_tts_async() logger.info("语音命令识别系统已启动") print("\n" + "=" * 70) print("🎙️ 高性能实时语音命令识别系统已启动") print("=" * 70) print("💡 功能说明：") print(" - 系统会自动检测语音并识别") print(f" - 🔔 唤醒词: {self.wake_word_detector.primary}") print(" - 只有包含唤醒词的语音才会被处理") print(" - 识别结果会自动转换为无人机控制命令") print(" - 命令会自动发送到Socket服务器") print(" - 按 Ctrl+C 退出") print("=" * 70 + "\n") def stop(self): """停止语音命令识别系统""" if not self.running: return self.running = False # 先通知工作线程结束，再播放尚未 drain 的应答（避免 Ctrl+C 时主循环未跑下一轮导致无声） if self.stt_thread is not None: self.stt_queue.put(None) if self.command_thread is not None: self.command_queue.put(None) if self.stt_thread is not None: self.stt_thread.join(timeout=2.0) if self.command_thread is not None: self.command_thread.join(timeout=2.0) if self.ack_tts_enabled: try: self._drain_ack_playback_queue(recover_mic=False) except Exception as e: logger.warning(f"退出前播放应答失败: {e}", exc_info=True) self.audio_capture.stop_stream() # 断开Socket连接 if self.socket_client.connected: self.socket_client.disconnect() logger.info("语音命令识别系统已停止") print("\n语音命令识别系统已停止") def process_audio_stream(self): """ 处理音频流（主循环） 高性能实时处理流程： 1. 采集音频块（非阻塞） 2. 预处理（降噪+AGC） 3. VAD检测语音开始/结束 4. 收集语音段 5. 异步STT识别（不阻塞主流程） """ try: while self.running: # 0. 主线程播放命令应答（必须在采集循环线程中执行 sd.play，见 tts.play_tts_audio 说明） self._before_audio_iteration() # 1. 采集音频块（非阻塞，高性能模式） chunk = self.audio_capture.read_chunk_numpy(timeout=0.1) if chunk is None: continue # 2. 音频预处理（降噪+AGC） processed_chunk = self.audio_preprocessor.process(chunk) # 初始化预缓冲区的最大块数（只需计算一次） if self.pre_speech_max_chunks is None: # 每个chunk包含的采样点数 samples_per_chunk = processed_chunk.shape[0] if samples_per_chunk > 0: # 0.8 秒需要的chunk数量 = 预缓冲秒数 * 采样率 / 每块采样数 chunks = int( self.pre_speech_max_seconds * self.audio_capture.sample_rate / samples_per_chunk ) # 至少保留 1 块，避免被算成 0 self.pre_speech_max_chunks = max(chunks, 1) else: self.pre_speech_max_chunks = 1 # 将当前块加入预缓冲区（环形缓冲） # 注意：预缓冲区保存的是“最近的一段音频”，无论当下是否在说话 self.pre_speech_buffer.append(processed_chunk) if ( self.pre_speech_max_chunks is not None and len(self.pre_speech_buffer) > self.pre_speech_max_chunks ): # 超出最大长度时，丢弃最早的块 self.pre_speech_buffer.pop(0) # 3. VAD：Silero 或能量（RMS）分段 if self._use_energy_vad: self._energy_vad_on_chunk(processed_chunk) else: chunk_bytes = processed_chunk.tobytes() if self.vad.detect_speech_start(chunk_bytes): hook = self._vad_speech_start_hook if hook is not None: try: hook() except Exception as e: # noqa: BLE001 logger.debug( "vad_speech_start_hook: %s", e, exc_info=True ) with self.speech_buffer_lock: if self.pre_speech_buffer: self.speech_buffer = list(self.pre_speech_buffer) else: self.speech_buffer.clear() self.speech_buffer.append(processed_chunk) logger.debug( "检测到语音开始，使用预缓冲音频（约 %.2f 秒）作为前缀，开始收集语音段", self.pre_speech_max_seconds, ) elif self.vad.is_speaking: with self.speech_buffer_lock: self.speech_buffer.append(processed_chunk) if self.vad.detect_speech_end(chunk_bytes): with self.speech_buffer_lock: self._submit_concatenated_speech_to_stt() self._reset_agc_after_utterance_end() logger.debug("检测到语音结束，提交识别") hook = getattr(self, "_after_processed_audio_chunk", None) if hook is not None: try: hook(processed_chunk) except Exception as e: # noqa: BLE001 logger.debug( "after_processed_audio_chunk: %s", e, exc_info=True ) except KeyboardInterrupt: logger.info("用户中断") except Exception as e: logger.error(f"处理音频流时发生错误: {e}", exc_info=True) raise def run(self): """运行语音命令识别系统（完整流程）""" try: self.start() self.process_audio_stream() finally: self.stop() if __name__ == "__main__": # 测试代码 recognizer = VoiceCommandRecognizer() recognizer.run()