717 lines
26 KiB
Python
717 lines
26 KiB
Python
"""
|
||
文本预处理模块 - 高性能实时语音转命令文本处理
|
||
|
||
本模块主要用于对语音识别输出的文本进行清洗、纠错、简繁转换、分词和参数提取,
|
||
便于后续命令意图分析和参数解析。
|
||
|
||
主要功能：
|
||
1. 文本清理：去除杂音、特殊字符、多余空格
|
||
2. 纠错：同音字纠正、常见错误修正
|
||
3. 简繁转换：统一文本格式(繁体转简体)
|
||
4. 分词:使用jieba分词,便于关键词匹配
|
||
5. 数字提取：提取距离(米)、速度(米/秒)、时间(秒)
|
||
6. 关键词识别：识别命令关键词(起飞、降落、前进等)
|
||
|
||
性能优化：
|
||
- LRU缓存常用处理结果(分词、中文数字解析、完整预处理)
|
||
- 预编译正则表达式
|
||
- 优化字符串操作(使用正则表达式批量替换)
|
||
- 延迟加载可选依赖
|
||
- 缓存关键词排序结果
|
||
"""
|
||
|
||
import re
|
||
from typing import Dict, Optional, List, Tuple, Set
|
||
from functools import lru_cache
|
||
from dataclasses import dataclass
|
||
from voice_drone.logging_ import get_logger
|
||
from voice_drone.core.configuration import KEYWORDS_CONFIG, SYSTEM_TEXT_PREPROCESSOR_CONFIG
|
||
import warnings
|
||
warnings.filterwarnings("ignore")
|
||
|
||
logger = get_logger("text_preprocessor")
|
||
|
||
# 延迟加载可选依赖
|
||
try:
|
||
from opencc import OpenCC
|
||
OPENCC_AVAILABLE = True
|
||
except ImportError:
|
||
OPENCC_AVAILABLE = False
|
||
logger.warning("opencc 未安装,将跳过简繁转换功能")
|
||
|
||
try:
|
||
import jieba
|
||
JIEBA_AVAILABLE = True
|
||
# 初始化jieba,加载词典
|
||
jieba.initialize()
|
||
except ImportError:
|
||
JIEBA_AVAILABLE = False
|
||
logger.warning("jieba 未安装,将跳过分词功能")
|
||
|
||
try:
|
||
from pypinyin import lazy_pinyin, Style
|
||
PYPINYIN_AVAILABLE = True
|
||
except ImportError:
|
||
PYPINYIN_AVAILABLE = False
|
||
logger.warning("pypinyin 未安装,将跳过拼音相关功能")
|
||
|
||
|
||
@dataclass
|
||
class ExtractedParams:
|
||
"""提取的参数信息"""
|
||
distance: Optional[float] = None # 距离(米)
|
||
speed: Optional[float] = None # 速度(米/秒)
|
||
duration: Optional[float] = None # 时间(秒)
|
||
command_keyword: Optional[str] = None # 识别的命令关键词
|
||
|
||
|
||
@dataclass
|
||
class PreprocessedText:
|
||
"""预处理后的文本结果"""
|
||
cleaned_text: str # 清理后的文本
|
||
normalized_text: str # 规范化后的文本(简繁转换后)
|
||
words: List[str] # 分词结果
|
||
params: ExtractedParams # 提取的参数
|
||
original_text: str # 原始文本
|
||
|
||
|
||
class TextPreprocessor:
|
||
"""
|
||
高性能文本预处理器
|
||
|
||
针对实时语音转命令场景优化,支持：
|
||
- 文本清理和规范化
|
||
- 同音字纠错
|
||
- 简繁转换
|
||
- 分词
|
||
- 数字和单位提取
|
||
- 命令关键词识别
|
||
"""
|
||
|
||
def __init__(self,
|
||
enable_traditional_to_simplified: Optional[bool] = None,
|
||
enable_segmentation: Optional[bool] = None,
|
||
enable_correction: Optional[bool] = None,
|
||
enable_number_extraction: Optional[bool] = None,
|
||
enable_keyword_detection: Optional[bool] = None,
|
||
lru_cache_size: Optional[int] = None):
|
||
"""
|
||
初始化文本预处理器
|
||
|
||
Args:
|
||
enable_traditional_to_simplified: 是否启用繁简转换(None时从配置读取)
|
||
enable_segmentation: 是否启用分词(None时从配置读取)
|
||
enable_correction: 是否启用纠错(None时从配置读取)
|
||
enable_number_extraction: 是否启用数字提取(None时从配置读取)
|
||
enable_keyword_detection: 是否启用关键词检测(None时从配置读取)
|
||
lru_cache_size: LRU缓存大小(None时从配置读取)
|
||
"""
|
||
# 从配置读取参数(如果未提供)
|
||
config = SYSTEM_TEXT_PREPROCESSOR_CONFIG or {}
|
||
|
||
self.enable_traditional_to_simplified = (
|
||
enable_traditional_to_simplified
|
||
if enable_traditional_to_simplified is not None
|
||
else config.get("enable_traditional_to_simplified", True)
|
||
) and OPENCC_AVAILABLE
|
||
|
||
self.enable_segmentation = (
|
||
enable_segmentation
|
||
if enable_segmentation is not None
|
||
else config.get("enable_segmentation", True)
|
||
) and JIEBA_AVAILABLE
|
||
|
||
self.enable_correction = (
|
||
enable_correction
|
||
if enable_correction is not None
|
||
else config.get("enable_correction", True)
|
||
)
|
||
|
||
self.enable_number_extraction = (
|
||
enable_number_extraction
|
||
if enable_number_extraction is not None
|
||
else config.get("enable_number_extraction", True)
|
||
)
|
||
|
||
self.enable_keyword_detection = (
|
||
enable_keyword_detection
|
||
if enable_keyword_detection is not None
|
||
else config.get("enable_keyword_detection", True)
|
||
)
|
||
|
||
cache_size = (
|
||
lru_cache_size
|
||
if lru_cache_size is not None
|
||
else config.get("lru_cache_size", 512)
|
||
)
|
||
|
||
# 初始化OpenCC(如果可用)
|
||
if self.enable_traditional_to_simplified:
|
||
self.opencc = OpenCC('t2s') # 繁体转简体
|
||
else:
|
||
self.opencc = None
|
||
|
||
# 加载关键词映射(命令关键词 -> 命令类型)
|
||
self._load_keyword_mapping()
|
||
|
||
# 预编译正则表达式(性能优化)
|
||
self._compile_regex_patterns()
|
||
|
||
# 加载纠错字典
|
||
self._load_correction_dict()
|
||
|
||
# 设置LRU缓存大小
|
||
self._cache_size = cache_size
|
||
|
||
# 创建缓存装饰器(用于分词、中文数字解析、完整预处理)
|
||
self._segment_text_cached = lru_cache(maxsize=cache_size)(self._segment_text_impl)
|
||
self._parse_chinese_number_cached = lru_cache(maxsize=128)(self._parse_chinese_number_impl)
|
||
self._preprocess_cached = lru_cache(maxsize=cache_size)(self._preprocess_impl)
|
||
self._preprocess_fast_cached = lru_cache(maxsize=cache_size)(self._preprocess_fast_impl)
|
||
|
||
logger.info(f"文本预处理器初始化完成")
|
||
logger.info(f" 繁简转换: {'启用' if self.enable_traditional_to_simplified else '禁用'}")
|
||
logger.info(f" 分词: {'启用' if self.enable_segmentation else '禁用'}")
|
||
logger.info(f" 纠错: {'启用' if self.enable_correction else '禁用'}")
|
||
logger.info(f" 数字提取: {'启用' if self.enable_number_extraction else '禁用'}")
|
||
logger.info(f" 关键词检测: {'启用' if self.enable_keyword_detection else '禁用'}")
|
||
|
||
def _load_keyword_mapping(self):
|
||
"""加载关键词映射表(命令关键词 -> 命令类型)"""
|
||
self.keyword_to_command: Dict[str, str] = {}
|
||
|
||
if KEYWORDS_CONFIG:
|
||
for command_type, keywords in KEYWORDS_CONFIG.items():
|
||
if isinstance(keywords, list):
|
||
for keyword in keywords:
|
||
self.keyword_to_command[keyword] = command_type
|
||
elif isinstance(keywords, str):
|
||
self.keyword_to_command[keywords] = command_type
|
||
|
||
# 预计算排序结果(按长度降序,优先匹配长关键词)
|
||
self.sorted_keywords = sorted(
|
||
self.keyword_to_command.keys(),
|
||
key=len,
|
||
reverse=True
|
||
)
|
||
|
||
logger.debug(f"加载了 {len(self.keyword_to_command)} 个关键词映射")
|
||
|
||
def _compile_regex_patterns(self):
|
||
"""预编译正则表达式(性能优化)"""
|
||
# 清理文本：去除特殊字符、多余空格
|
||
self.pattern_clean_special = re.compile(r'[^\u4e00-\u9fa5a-zA-Z0-9\s米每秒秒分小时\.]')
|
||
self.pattern_clean_spaces = re.compile(r'\s+')
|
||
|
||
# 数字提取模式
|
||
# 距离：数字 + (米|m|M|公尺)(排除速度单位)
|
||
self.pattern_distance = re.compile(
|
||
r'(\d+\.?\d*)\s*(?:米|m|M|公尺|meter|meters)(?!\s*[/每]?\s*秒)',
|
||
re.IGNORECASE
|
||
)
|
||
|
||
# 速度：数字 + (米每秒|m/s|米/秒|米秒|mps|MPS)(优先匹配)
|
||
# 支持"三米每秒"、"5米/秒"、"2.5米每秒"等格式
|
||
self.pattern_speed = re.compile(
|
||
r'(?:速度\s*[:：]?\s*)?(\d+\.?\d*|[零一二三四五六七八九十]+)\s*(?:米\s*[/每]?\s*秒|m\s*/\s*s|mps|MPS)',
|
||
re.IGNORECASE
|
||
)
|
||
|
||
# 时间：数字 + (秒|s|S|分钟|分|min|小时|时|h|H)
|
||
# 支持"持续10秒"、"5分钟"等格式
|
||
self.pattern_duration = re.compile(
|
||
r'(?:持续\s*|持续\s*)?(\d+\.?\d*|[零一二三四五六七八九十]+)\s*(?:秒|s|S|分钟|分|min|小时|时|h|H)',
|
||
re.IGNORECASE
|
||
)
|
||
|
||
# 中文数字映射(用于识别"十米"、"五秒"等)
|
||
self.chinese_numbers = {
|
||
'零': 0, '一': 1, '二': 2, '三': 3, '四': 4, '五': 5,
|
||
'六': 6, '七': 7, '八': 8, '九': 9, '十': 10,
|
||
'壹': 1, '贰': 2, '叁': 3, '肆': 4, '伍': 5,
|
||
'陆': 6, '柒': 7, '捌': 8, '玖': 9, '拾': 10,
|
||
'百': 100, '千': 1000, '万': 10000
|
||
}
|
||
|
||
# 中文数字模式(如"十米"、"五秒"、"二十米"、"三米每秒")
|
||
# 支持"二十"、"三十"等复合数字
|
||
self.pattern_chinese_number = re.compile(
|
||
r'([零一二三四五六七八九十壹贰叁肆伍陆柒捌玖拾百千万]+)\s*(?:米|秒|分|小时|米\s*[/每]?\s*秒)'
|
||
)
|
||
|
||
def _load_correction_dict(self):
|
||
"""加载纠错字典(同音字、常见错误)并编译正则表达式"""
|
||
# 无人机控制相关的常见同音字/错误字映射(只保留实际需要纠错的)
|
||
correction_pairs = [
|
||
# 动作相关(同音字纠错)
|
||
('起非', '起飞'),
|
||
('降洛', '降落'),
|
||
('悬廷', '悬停'),
|
||
('停只', '停止'),
|
||
]
|
||
|
||
# 构建正则表达式模式(按长度降序,优先匹配长模式)
|
||
if correction_pairs:
|
||
# 按长度降序排序,优先匹配长模式
|
||
sorted_pairs = sorted(correction_pairs, key=lambda x: len(x[0]), reverse=True)
|
||
|
||
# 构建替换映射字典(用于快速查找)
|
||
self.correction_replacements = {wrong: correct for wrong, correct in sorted_pairs}
|
||
|
||
# 编译单一正则表达式
|
||
patterns = [re.escape(wrong) for wrong, _ in sorted_pairs]
|
||
self.correction_pattern = re.compile('|'.join(patterns))
|
||
else:
|
||
self.correction_pattern = None
|
||
self.correction_replacements = {}
|
||
|
||
logger.debug(f"加载了 {len(correction_pairs)} 个纠错规则")
|
||
|
||
def clean_text(self, text: str) -> str:
|
||
"""
|
||
清理文本：去除特殊字符、多余空格
|
||
|
||
Args:
|
||
text: 原始文本
|
||
|
||
Returns:
|
||
清理后的文本
|
||
"""
|
||
if not text:
|
||
return ""
|
||
|
||
# 去除特殊字符(保留中文、英文、数字、空格、常用标点)
|
||
text = self.pattern_clean_special.sub('', text)
|
||
|
||
# 统一空格(多个空格合并为一个)
|
||
text = self.pattern_clean_spaces.sub(' ', text)
|
||
|
||
# 去除首尾空格
|
||
text = text.strip()
|
||
|
||
return text
|
||
|
||
def correct_text(self, text: str) -> str:
|
||
"""
|
||
纠错：同音字、常见错误修正(使用正则表达式优化)
|
||
|
||
Args:
|
||
text: 待纠错文本
|
||
|
||
Returns:
|
||
纠错后的文本
|
||
"""
|
||
if not self.enable_correction or not text or not self.correction_pattern:
|
||
return text
|
||
|
||
# 使用正则表达式一次性替换所有模式(性能优化)
|
||
def replacer(match):
|
||
matched = match.group(0)
|
||
# 直接从字典中查找替换(O(1)查找)
|
||
return self.correction_replacements.get(matched, matched)
|
||
|
||
return self.correction_pattern.sub(replacer, text)
|
||
|
||
def traditional_to_simplified(self, text: str) -> str:
|
||
"""
|
||
繁体转简体
|
||
|
||
Args:
|
||
text: 待转换文本
|
||
|
||
Returns:
|
||
转换后的文本
|
||
"""
|
||
if not self.enable_traditional_to_simplified or not self.opencc or not text:
|
||
return text
|
||
|
||
try:
|
||
return self.opencc.convert(text)
|
||
except Exception as e:
|
||
logger.warning(f"繁简转换失败: {e}")
|
||
return text
|
||
|
||
def _segment_text_impl(self, text: str) -> List[str]:
|
||
"""
|
||
分词实现(内部方法,不带缓存)
|
||
|
||
Args:
|
||
text: 待分词文本
|
||
|
||
Returns:
|
||
分词结果列表
|
||
"""
|
||
if not self.enable_segmentation or not text:
|
||
return [text] if text else []
|
||
|
||
try:
|
||
words = list(jieba.cut(text, cut_all=False))
|
||
# 过滤空字符串
|
||
words = [w.strip() for w in words if w.strip()]
|
||
return words
|
||
except Exception as e:
|
||
logger.warning(f"分词失败: {e}")
|
||
return [text] if text else []
|
||
|
||
def segment_text(self, text: str) -> List[str]:
|
||
"""
|
||
分词(带缓存)
|
||
|
||
Args:
|
||
text: 待分词文本
|
||
|
||
Returns:
|
||
分词结果列表
|
||
"""
|
||
return self._segment_text_cached(text)
|
||
|
||
def extract_numbers(self, text: str) -> ExtractedParams:
|
||
"""
|
||
提取数字和单位(距离、速度、时间)
|
||
|
||
Args:
|
||
text: 待提取文本
|
||
|
||
Returns:
|
||
ExtractedParams对象,包含提取的参数
|
||
"""
|
||
params = ExtractedParams()
|
||
|
||
if not self.enable_number_extraction or not text:
|
||
return params
|
||
|
||
# 优先提取速度(避免被误识别为距离)
|
||
speed_match = self.pattern_speed.search(text)
|
||
if speed_match:
|
||
try:
|
||
speed_str = speed_match.group(1)
|
||
# 尝试解析中文数字
|
||
if speed_str.isdigit() or '.' in speed_str:
|
||
params.speed = float(speed_str)
|
||
else:
|
||
# 中文数字(使用缓存)
|
||
chinese_speed = self._parse_chinese_number(speed_str)
|
||
if chinese_speed is not None:
|
||
params.speed = float(chinese_speed)
|
||
except (ValueError, AttributeError):
|
||
pass
|
||
|
||
# 提取距离(米,排除速度单位)
|
||
distance_match = self.pattern_distance.search(text)
|
||
if distance_match:
|
||
try:
|
||
params.distance = float(distance_match.group(1))
|
||
except ValueError:
|
||
pass
|
||
|
||
# 提取时间(秒)
|
||
duration_matches = self.pattern_duration.finditer(text) # 查找所有匹配
|
||
for duration_match in duration_matches:
|
||
try:
|
||
duration_str = duration_match.group(1)
|
||
duration_unit = duration_match.group(2).lower() if len(duration_match.groups()) > 1 else '秒'
|
||
|
||
# 解析数字(支持中文数字)
|
||
if duration_str.isdigit() or '.' in duration_str:
|
||
duration_value = float(duration_str)
|
||
else:
|
||
# 中文数字
|
||
chinese_duration = self._parse_chinese_number(duration_str)
|
||
if chinese_duration is None:
|
||
continue
|
||
duration_value = float(chinese_duration)
|
||
|
||
# 转换为秒
|
||
if '分' in duration_unit or 'min' in duration_unit:
|
||
params.duration = duration_value * 60
|
||
break # 取第一个匹配
|
||
elif '小时' in duration_unit or 'h' in duration_unit:
|
||
params.duration = duration_value * 3600
|
||
break
|
||
else: # 秒
|
||
params.duration = duration_value
|
||
break
|
||
except (ValueError, IndexError, AttributeError):
|
||
continue
|
||
|
||
# 尝试提取中文数字(如"十米"、"五秒"、"二十米"、"三米每秒")
|
||
chinese_matches = self.pattern_chinese_number.finditer(text)
|
||
for chinese_match in chinese_matches:
|
||
try:
|
||
chinese_num_str = chinese_match.group(1)
|
||
full_match = chinese_match.group(0)
|
||
|
||
# 解析中文数字(使用缓存)
|
||
num_value = self._parse_chinese_number(chinese_num_str)
|
||
|
||
if num_value is not None:
|
||
# 判断单位类型
|
||
if '米每秒' in full_match or '米/秒' in full_match or '米每' in full_match:
|
||
# 速度单位
|
||
if params.speed is None:
|
||
params.speed = float(num_value)
|
||
elif '米' in full_match and '秒' not in full_match:
|
||
# 距离单位(不包含"秒")
|
||
if params.distance is None:
|
||
params.distance = float(num_value)
|
||
elif '秒' in full_match and '米' not in full_match:
|
||
# 时间单位(不包含"米")
|
||
if params.duration is None:
|
||
params.duration = float(num_value)
|
||
elif '分' in full_match and '米' not in full_match:
|
||
# 时间单位(分钟)
|
||
if params.duration is None:
|
||
params.duration = float(num_value) * 60
|
||
except (ValueError, IndexError, AttributeError):
|
||
continue
|
||
|
||
return params
|
||
|
||
def _parse_chinese_number_impl(self, chinese_num: str) -> Optional[int]:
|
||
"""
|
||
解析中文数字实现(内部方法,不带缓存)
|
||
|
||
Args:
|
||
chinese_num: 中文数字字符串
|
||
|
||
Returns:
|
||
对应的阿拉伯数字,解析失败返回None
|
||
"""
|
||
if not chinese_num:
|
||
return None
|
||
|
||
# 单个数字
|
||
if chinese_num in self.chinese_numbers:
|
||
return self.chinese_numbers[chinese_num]
|
||
|
||
# "十" -> 10
|
||
if chinese_num == '十' or chinese_num == '拾':
|
||
return 10
|
||
|
||
# "十一" -> 11, "十二" -> 12, ...
|
||
if chinese_num.startswith('十') or chinese_num.startswith('拾'):
|
||
rest = chinese_num[1:]
|
||
if rest in self.chinese_numbers:
|
||
return 10 + self.chinese_numbers[rest]
|
||
|
||
# "二十" -> 20, "三十" -> 30, ...
|
||
if chinese_num.endswith('十') or chinese_num.endswith('拾'):
|
||
prefix = chinese_num[:-1]
|
||
if prefix in self.chinese_numbers:
|
||
return self.chinese_numbers[prefix] * 10
|
||
|
||
# "二十五" -> 25, "三十五" -> 35, ...
|
||
if '十' in chinese_num or '拾' in chinese_num:
|
||
parts = chinese_num.replace('拾', '十').split('十')
|
||
if len(parts) == 2:
|
||
tens_part = parts[0] if parts[0] else '一' # "十五" -> parts[0]为空
|
||
ones_part = parts[1] if parts[1] else ''
|
||
|
||
tens = self.chinese_numbers.get(tens_part, 1) if tens_part else 1
|
||
ones = self.chinese_numbers.get(ones_part, 0) if ones_part else 0
|
||
|
||
return tens * 10 + ones
|
||
|
||
return None
|
||
|
||
def _parse_chinese_number(self, chinese_num: str) -> Optional[int]:
|
||
"""
|
||
解析中文数字(支持"十"、"二十"、"三十"、"五"等,带缓存)
|
||
|
||
Args:
|
||
chinese_num: 中文数字字符串
|
||
|
||
Returns:
|
||
对应的阿拉伯数字,解析失败返回None
|
||
"""
|
||
return self._parse_chinese_number_cached(chinese_num)
|
||
|
||
def detect_keyword(self, text: str, words: Optional[List[str]] = None) -> Optional[str]:
|
||
"""
|
||
检测命令关键词(使用缓存的排序结果)
|
||
|
||
Args:
|
||
text: 待检测文本
|
||
words: 分词结果(如果已分词,可传入以提高性能)
|
||
|
||
Returns:
|
||
检测到的命令类型(如"takeoff"、"forward"等),未检测到返回None
|
||
"""
|
||
if not self.enable_keyword_detection or not text:
|
||
return None
|
||
|
||
# 如果已分词,优先使用分词结果匹配
|
||
if words:
|
||
for word in words:
|
||
if word in self.keyword_to_command:
|
||
return self.keyword_to_command[word]
|
||
|
||
# 使用缓存的排序结果(按长度降序,优先匹配长关键词)
|
||
for keyword in self.sorted_keywords:
|
||
if keyword in text:
|
||
return self.keyword_to_command[keyword]
|
||
|
||
return None
|
||
|
||
def preprocess(self, text: str) -> PreprocessedText:
|
||
"""
|
||
完整的文本预处理流程(带缓存)
|
||
|
||
Args:
|
||
text: 原始文本
|
||
|
||
Returns:
|
||
PreprocessedText对象,包含所有预处理结果
|
||
"""
|
||
return self._preprocess_cached(text)
|
||
|
||
def _preprocess_impl(self, text: str) -> PreprocessedText:
|
||
"""
|
||
完整的文本预处理流程实现(内部方法,不带缓存)
|
||
|
||
Args:
|
||
text: 原始文本
|
||
|
||
Returns:
|
||
PreprocessedText对象,包含所有预处理结果
|
||
"""
|
||
if not text:
|
||
return PreprocessedText(
|
||
cleaned_text="",
|
||
normalized_text="",
|
||
words=[],
|
||
params=ExtractedParams(),
|
||
original_text=text
|
||
)
|
||
|
||
original_text = text
|
||
|
||
# 1. 清理文本
|
||
cleaned_text = self.clean_text(text)
|
||
|
||
# 2. 纠错
|
||
corrected_text = self.correct_text(cleaned_text)
|
||
|
||
# 3. 繁简转换
|
||
normalized_text = self.traditional_to_simplified(corrected_text)
|
||
|
||
# 4. 分词
|
||
words = self.segment_text(normalized_text)
|
||
|
||
# 5. 提取数字和单位
|
||
params = self.extract_numbers(normalized_text)
|
||
|
||
# 6. 检测关键词
|
||
command_keyword = self.detect_keyword(normalized_text, words)
|
||
params.command_keyword = command_keyword
|
||
|
||
return PreprocessedText(
|
||
cleaned_text=cleaned_text,
|
||
normalized_text=normalized_text,
|
||
words=words,
|
||
params=params,
|
||
original_text=original_text
|
||
)
|
||
|
||
def preprocess_fast(self, text: str) -> Tuple[str, ExtractedParams]:
|
||
"""
|
||
快速预处理(仅返回规范化文本和参数,不进行分词,带缓存)
|
||
|
||
适用于实时场景,性能优先
|
||
|
||
Args:
|
||
text: 原始文本
|
||
|
||
Returns:
|
||
(规范化文本, 提取的参数)
|
||
"""
|
||
return self._preprocess_fast_cached(text)
|
||
|
||
def _preprocess_fast_impl(self, text: str) -> Tuple[str, ExtractedParams]:
|
||
"""
|
||
快速预处理实现(内部方法,不带缓存)
|
||
|
||
Args:
|
||
text: 原始文本
|
||
|
||
Returns:
|
||
(规范化文本, 提取的参数)
|
||
"""
|
||
if not text:
|
||
return "", ExtractedParams()
|
||
|
||
# 1. 清理
|
||
cleaned = self.clean_text(text)
|
||
|
||
# 2. 纠错
|
||
corrected = self.correct_text(cleaned)
|
||
|
||
# 3. 繁简转换
|
||
normalized = self.traditional_to_simplified(corrected)
|
||
|
||
# 4. 提取参数(不进行分词,提高性能)
|
||
params = self.extract_numbers(normalized)
|
||
|
||
# 5. 检测关键词(在完整文本中搜索)
|
||
params.command_keyword = self.detect_keyword(normalized, words=None)
|
||
|
||
return normalized, params
|
||
|
||
|
||
# 全局单例(可选,用于提高性能)
|
||
_global_preprocessor: Optional[TextPreprocessor] = None
|
||
|
||
|
||
def get_preprocessor() -> TextPreprocessor:
|
||
"""获取全局预处理器实例(单例模式)"""
|
||
global _global_preprocessor
|
||
if _global_preprocessor is None:
|
||
_global_preprocessor = TextPreprocessor()
|
||
return _global_preprocessor
|
||
|
||
|
||
if __name__ == "__main__":
|
||
from command import Command
|
||
|
||
# 测试代码
|
||
preprocessor = TextPreprocessor()
|
||
|
||
test_cases = [
|
||
"现在起飞往前飞,飞10米,速度为5米每秒",
|
||
"向前飞二十米,速度三米每秒",
|
||
"立刻降落",
|
||
"悬停五秒",
|
||
"向右飛十米", # 繁体测试
|
||
"往左飛,速度2.5米/秒,持續10秒",
|
||
]
|
||
|
||
print("=" * 60)
|
||
print("文本预处理器测试")
|
||
print("=" * 60)
|
||
|
||
for i, test_text in enumerate(test_cases, 1):
|
||
print(f"\n测试 {i}: {test_text}")
|
||
print("-" * 60)
|
||
|
||
# 完整预处理
|
||
result = preprocessor.preprocess(test_text)
|
||
print(f"原始文本: {result.original_text}")
|
||
print(f"清理后: {result.cleaned_text}")
|
||
print(f"规范化: {result.normalized_text}")
|
||
print(f"分词: {result.words}")
|
||
print(f"提取参数:")
|
||
print(f" 距离: {result.params.distance} 米")
|
||
print(f" 速度: {result.params.speed} 米/秒")
|
||
print(f" 时间: {result.params.duration} 秒")
|
||
print(f" 命令关键词: {result.params.command_keyword}")
|
||
|
||
|
||
command = Command.create(result.params.command_keyword, 1, result.params.distance, result.params.speed, result.params.duration)
|
||
print(f"命令: {command.to_dict()}")
|
||
|
||
# 快速预处理
|
||
fast_text, fast_params = preprocessor.preprocess_fast(test_text)
|
||
print(f"\n快速预处理结果:")
|
||
print(f" 规范化文本: {fast_text}")
|
||
print(f" 命令关键词: {fast_params.command_keyword}")
|