mirror of
https://devops.liangqichi.top/qichi.liang/Orbitin.git
synced 2026-02-10 07:41:29 +08:00
重构: 完成代码审查和架构优化
主要改进: 1. 模块化架构重构 - 创建Confluence模块目录结构 - 统一飞书模块架构 - 重构数据库模块 2. 代码质量提升 - 创建统一配置管理 - 实现统一日志配置 - 完善类型提示和异常处理 3. 功能优化 - 移除parse-test功能 - 删除DEBUG_MODE配置 - 更新命令行选项 4. 文档完善 - 更新README.md项目结构 - 添加开发指南和故障排除 - 完善配置说明 5. 系统验证 - 所有核心功能测试通过 - 模块导入验证通过 - 架构完整性验证通过
This commit is contained in:
337
main.py
337
main.py
@@ -2,130 +2,210 @@
|
||||
"""
|
||||
码头作业日志管理工具
|
||||
从 Confluence 获取交接班日志并保存到数据库
|
||||
更新依赖，使用新的模块结构
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
from src.confluence import ConfluenceClient
|
||||
from src.extractor import HTMLTextExtractor
|
||||
from src.parser import HandoverLogParser
|
||||
from src.database import DailyLogsDatabase
|
||||
from src.report import DailyReportGenerator
|
||||
from src.config import config
|
||||
from src.logging_config import setup_logging, get_logger
|
||||
from src.confluence import ConfluenceClient, ConfluenceClientError, HTMLTextExtractor, HTMLTextExtractorError, HandoverLogParser, ShipLog, LogParserError
|
||||
from src.database.daily_logs import DailyLogsDatabase
|
||||
from src.report import DailyReportGenerator, ReportGeneratorError
|
||||
|
||||
# 加载环境变量
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
# 配置（从环境变量读取）
|
||||
CONF_BASE_URL = os.getenv('CONFLUENCE_BASE_URL')
|
||||
CONF_TOKEN = os.getenv('CONFLUENCE_TOKEN')
|
||||
CONF_CONTENT_ID = os.getenv('CONFLUENCE_CONTENT_ID')
|
||||
|
||||
# 飞书配置（可选）
|
||||
FEISHU_BASE_URL = os.getenv('FEISHU_BASE_URL')
|
||||
FEISHU_TOKEN = os.getenv('FEISHU_TOKEN')
|
||||
FEISHU_SPREADSHEET_TOKEN = os.getenv('FEISHU_SPREADSHEET_TOKEN')
|
||||
|
||||
DEBUG_DIR = 'debug'
|
||||
# 初始化日志
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def ensure_debug_dir():
|
||||
"""确保debug目录存在"""
|
||||
if not os.path.exists(DEBUG_DIR):
|
||||
os.makedirs(DEBUG_DIR)
|
||||
if not os.path.exists(config.DEBUG_DIR):
|
||||
os.makedirs(config.DEBUG_DIR)
|
||||
logger.info(f"创建调试目录: {config.DEBUG_DIR}")
|
||||
|
||||
|
||||
def get_timestamp():
|
||||
def get_timestamp() -> str:
|
||||
"""获取时间戳用于文件名"""
|
||||
return datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
|
||||
|
||||
def fetch_html():
|
||||
"""获取HTML内容"""
|
||||
if not CONF_BASE_URL or not CONF_TOKEN or not CONF_CONTENT_ID:
|
||||
print('错误：未配置 Confluence 信息，请检查 .env 文件')
|
||||
def fetch_html() -> str:
|
||||
"""
|
||||
获取HTML内容
|
||||
|
||||
返回:
|
||||
HTML字符串
|
||||
|
||||
异常:
|
||||
SystemExit: 配置错误或获取失败
|
||||
"""
|
||||
# 验证配置
|
||||
if not config.validate():
|
||||
logger.error("配置验证失败，请检查 .env 文件")
|
||||
sys.exit(1)
|
||||
|
||||
print('正在从 Confluence 获取 HTML 内容...')
|
||||
client = ConfluenceClient(CONF_BASE_URL, CONF_TOKEN)
|
||||
html = client.get_html(CONF_CONTENT_ID)
|
||||
if not html:
|
||||
print('错误：未获取到 HTML 内容')
|
||||
try:
|
||||
logger.info("正在从 Confluence 获取 HTML 内容...")
|
||||
client = ConfluenceClient()
|
||||
html = client.get_html(config.CONFLUENCE_CONTENT_ID)
|
||||
logger.info(f"获取成功，共 {len(html)} 字符")
|
||||
return html
|
||||
|
||||
except ConfluenceClientError as e:
|
||||
logger.error(f"获取HTML失败: {e}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.error(f"未知错误: {e}")
|
||||
sys.exit(1)
|
||||
print(f'获取成功，共 {len(html)} 字符')
|
||||
return html
|
||||
|
||||
|
||||
def extract_text(html):
|
||||
"""提取布局文本"""
|
||||
print('正在提取布局文本...')
|
||||
extractor = HTMLTextExtractor()
|
||||
layout_text = extractor.extract(html)
|
||||
print(f'提取完成，共 {len(layout_text)} 字符')
|
||||
return layout_text
|
||||
def extract_text(html: str) -> str:
|
||||
"""
|
||||
提取布局文本
|
||||
|
||||
参数:
|
||||
html: HTML字符串
|
||||
|
||||
返回:
|
||||
提取的文本
|
||||
"""
|
||||
try:
|
||||
logger.info("正在提取布局文本...")
|
||||
extractor = HTMLTextExtractor()
|
||||
layout_text = extractor.extract(html)
|
||||
logger.info(f"提取完成，共 {len(layout_text)} 字符")
|
||||
return layout_text
|
||||
|
||||
except HTMLTextExtractorError as e:
|
||||
logger.error(f"提取文本失败: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"未知错误: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def save_debug_file(content, suffix=''):
|
||||
"""保存调试文件到debug目录"""
|
||||
def save_debug_file(content: str, suffix: str = '') -> str:
|
||||
"""
|
||||
保存调试文件到debug目录
|
||||
|
||||
参数:
|
||||
content: 要保存的内容
|
||||
suffix: 文件名后缀
|
||||
|
||||
返回:
|
||||
保存的文件路径
|
||||
"""
|
||||
ensure_debug_dir()
|
||||
filename = f'layout_output{suffix}.txt' if suffix else 'layout_output.txt'
|
||||
filepath = os.path.join(DEBUG_DIR, filename)
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
print(f'已保存到 {filepath}')
|
||||
return filepath
|
||||
filepath = os.path.join(config.DEBUG_DIR, filename)
|
||||
|
||||
try:
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
logger.info(f"已保存到 {filepath}")
|
||||
return filepath
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存调试文件失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def parse_logs(text):
|
||||
"""解析日志数据"""
|
||||
print('正在解析日志数据...')
|
||||
parser = HandoverLogParser()
|
||||
logs = parser.parse(text)
|
||||
print(f'解析到 {len(logs)} 条记录')
|
||||
return logs
|
||||
def parse_logs(text: str) -> List[ShipLog]:
|
||||
"""
|
||||
解析日志数据
|
||||
|
||||
参数:
|
||||
text: 日志文本
|
||||
|
||||
返回:
|
||||
船次日志列表
|
||||
"""
|
||||
try:
|
||||
logger.info("正在解析日志数据...")
|
||||
parser = HandoverLogParser()
|
||||
logs = parser.parse(text)
|
||||
logger.info(f"解析到 {len(logs)} 条记录")
|
||||
return logs
|
||||
|
||||
except LogParserError as e:
|
||||
logger.error(f"解析日志失败: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"未知错误: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def save_to_db(logs):
|
||||
"""保存到数据库"""
|
||||
def save_to_db(logs: List[ShipLog]) -> int:
|
||||
"""
|
||||
保存到数据库
|
||||
|
||||
参数:
|
||||
logs: 船次日志列表
|
||||
|
||||
返回:
|
||||
保存的记录数
|
||||
"""
|
||||
if not logs:
|
||||
print('没有记录可保存')
|
||||
logger.warning("没有记录可保存")
|
||||
return 0
|
||||
|
||||
db = DailyLogsDatabase()
|
||||
count = db.insert_many([log.to_dict() for log in logs])
|
||||
print(f'已保存 {count} 条记录到数据库')
|
||||
try:
|
||||
db = DailyLogsDatabase()
|
||||
count = db.insert_many([log.to_dict() for log in logs])
|
||||
logger.info(f"已保存 {count} 条记录到数据库")
|
||||
|
||||
stats = db.get_stats()
|
||||
logger.info(f"数据库统计: 总记录={stats['total']}, 船次={len(stats['ships'])}, "
|
||||
f"日期范围={stats['date_range']['start']}~{stats['date_range']['end']}")
|
||||
|
||||
return count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存到数据库失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def add_unaccounted(year_month: str, teu: int, note: str = ''):
|
||||
"""
|
||||
添加未统计数据
|
||||
|
||||
stats = db.get_stats()
|
||||
print(f'\n数据库统计:')
|
||||
print(f' 总记录: {stats["total"]}')
|
||||
print(f' 船次: {len(stats["ships"])}')
|
||||
print(f' 日期范围: {stats["date_range"]["start"]} ~ {stats["date_range"]["end"]}')
|
||||
参数:
|
||||
year_month: 年月字符串，格式 "2025-12"
|
||||
teu: 未统计TEU数量
|
||||
note: 备注
|
||||
"""
|
||||
try:
|
||||
db = DailyLogsDatabase()
|
||||
result = db.insert_unaccounted(year_month, teu, note)
|
||||
if result:
|
||||
logger.info(f"已添加 {year_month} 月未统计数据: {teu}TEU")
|
||||
else:
|
||||
logger.error("添加失败")
|
||||
except Exception as e:
|
||||
logger.error(f"添加未统计数据失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def show_stats(date: str):
|
||||
"""
|
||||
显示指定日期的统计
|
||||
|
||||
db.close()
|
||||
return count
|
||||
参数:
|
||||
date: 日期字符串，格式 "YYYY-MM-DD"
|
||||
"""
|
||||
try:
|
||||
generator = DailyReportGenerator()
|
||||
generator.print_report(date)
|
||||
except ReportGeneratorError as e:
|
||||
logger.error(f"生成统计失败: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"未知错误: {e}")
|
||||
|
||||
|
||||
def add_unaccounted(year_month, teu, note=''):
|
||||
"""添加未统计数据"""
|
||||
db = DailyLogsDatabase()
|
||||
result = db.insert_unaccounted(year_month, teu, note)
|
||||
if result:
|
||||
print(f'已添加 {year_month} 月未统计数据: {teu}TEU')
|
||||
else:
|
||||
print('添加失败')
|
||||
db.close()
|
||||
|
||||
|
||||
def show_stats(date):
|
||||
"""显示指定日期的统计"""
|
||||
g = DailyReportGenerator()
|
||||
g.print_report(date)
|
||||
g.close()
|
||||
|
||||
|
||||
def run_fetch():
|
||||
def run_fetch() -> str:
|
||||
"""执行：获取HTML并提取文本"""
|
||||
html = fetch_html()
|
||||
text = extract_text(html)
|
||||
@@ -140,7 +220,7 @@ def run_fetch_and_save():
|
||||
save_to_db(logs)
|
||||
|
||||
|
||||
def run_fetch_save_debug():
|
||||
def run_fetch_save_debug() -> str:
|
||||
"""执行：获取、提取、保存到debug目录"""
|
||||
html = fetch_html()
|
||||
text = extract_text(html)
|
||||
@@ -149,33 +229,37 @@ def run_fetch_save_debug():
|
||||
return text
|
||||
|
||||
|
||||
def run_report(date=None):
|
||||
def run_report(date: Optional[str] = None):
|
||||
"""执行：生成日报"""
|
||||
if not date:
|
||||
date = datetime.now().strftime('%Y-%m-%d')
|
||||
show_stats(date)
|
||||
|
||||
|
||||
def run_parser_test():
|
||||
"""执行：解析测试"""
|
||||
ensure_debug_file_path = os.path.join(DEBUG_DIR, 'layout_output.txt')
|
||||
if os.path.exists('layout_output.txt'):
|
||||
filepath = 'layout_output.txt'
|
||||
elif os.path.exists(ensure_debug_file_path):
|
||||
filepath = ensure_debug_file_path
|
||||
else:
|
||||
print('未找到 layout_output.txt 文件')
|
||||
return
|
||||
|
||||
|
||||
def run_config_test():
|
||||
"""执行：配置测试"""
|
||||
logger.info("配置测试:")
|
||||
config.print_summary()
|
||||
|
||||
print(f'使用文件: {filepath}')
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
# 测试Confluence连接
|
||||
try:
|
||||
client = ConfluenceClient()
|
||||
if client.test_connection():
|
||||
logger.info("Confluence连接测试: 成功")
|
||||
else:
|
||||
logger.warning("Confluence连接测试: 失败")
|
||||
except Exception as e:
|
||||
logger.error(f"Confluence连接测试失败: {e}")
|
||||
|
||||
parser = HandoverLogParser()
|
||||
logs = parser.parse(text)
|
||||
print(f'解析到 {len(logs)} 条记录')
|
||||
for log in logs[:5]:
|
||||
print(f' {log.date} {log.shift} {log.ship_name}: {log.teu}TEU')
|
||||
# 测试数据库连接
|
||||
try:
|
||||
db = DailyLogsDatabase()
|
||||
stats = db.get_stats()
|
||||
logger.info(f"数据库连接测试: 成功，总记录: {stats['total']}")
|
||||
except Exception as e:
|
||||
logger.error(f"数据库连接测试失败: {e}")
|
||||
|
||||
|
||||
# 功能映射
|
||||
@@ -185,7 +269,7 @@ FUNCTIONS = {
|
||||
'fetch-debug': run_fetch_save_debug,
|
||||
'report': lambda: run_report(),
|
||||
'report-today': lambda: run_report(datetime.now().strftime('%Y-%m-%d')),
|
||||
'parse-test': run_parser_test,
|
||||
'config-test': run_config_test,
|
||||
'stats': lambda: show_stats(datetime.now().strftime('%Y-%m-%d')),
|
||||
}
|
||||
|
||||
@@ -201,21 +285,21 @@ def main():
|
||||
fetch-debug 获取、提取并保存带时间戳的debug文件
|
||||
report 生成日报（默认今天）
|
||||
report-today 生成今日日报
|
||||
parse-test 解析测试（使用已有的layout_output.txt）
|
||||
config-test 配置测试
|
||||
stats 显示今日统计
|
||||
|
||||
示例:
|
||||
python3 main.py fetch
|
||||
python3 main.py fetch-save
|
||||
python3 main.py report 2025-12-28
|
||||
python3 main.py parse-test
|
||||
python3 main.py config-test
|
||||
'''
|
||||
)
|
||||
parser.add_argument(
|
||||
'function',
|
||||
nargs='?',
|
||||
default='fetch-save',
|
||||
choices=list(FUNCTIONS.keys()),
|
||||
choices=['fetch', 'fetch-save', 'fetch-debug', 'report', 'report-today', 'config-test', 'stats'],
|
||||
help='要执行的功能 (默认: fetch-save)'
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -242,15 +326,36 @@ def main():
|
||||
# 添加未统计数据
|
||||
if args.unaccounted:
|
||||
year_month = args.month or datetime.now().strftime('%Y-%m')
|
||||
add_unaccounted(year_month, args.unaccounted)
|
||||
try:
|
||||
add_unaccounted(year_month, args.unaccounted)
|
||||
except Exception as e:
|
||||
logger.error(f"添加未统计数据失败: {e}")
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
# 执行功能
|
||||
if args.function == 'report' and args.date:
|
||||
run_report(args.date)
|
||||
else:
|
||||
FUNCTIONS[args.function]()
|
||||
try:
|
||||
if args.function == 'report' and args.date:
|
||||
run_report(args.date)
|
||||
else:
|
||||
FUNCTIONS[args.function]()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("用户中断操作")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
logger.error(f"执行功能失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 初始化日志系统
|
||||
setup_logging()
|
||||
|
||||
# 打印启动信息
|
||||
logger.info("=" * 50)
|
||||
logger.info("码头作业日志管理工具 - OrbitIn")
|
||||
logger.info(f"启动时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
# 运行主程序
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user