diff --git a/batch_syncnet.sh b/batch_syncnet.sh new file mode 100644 index 0000000..a627fb1 --- /dev/null +++ b/batch_syncnet.sh @@ -0,0 +1,213 @@ +#!/bin/bash +# SyncNet 批量处理脚本 +# 用法: ./batch_syncnet.sh [--skip-failed] [--skip-video-failed] + +# ==================== 配置 ==================== +PYTHON_SCRIPT="./batch_syncnet.py" # 您的Python批量脚本 +DATA_ROOT="./data/work" # 输出数据根目录 +LOG_DIR="./batch_logs" # 批量日志目录 + +# ==================== 参数解析 ==================== +SKIP_FAILED="" +SKIP_VIDEO_FAILED="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --skip-failed) SKIP_FAILED="--skip-failed" ;; + --skip-video-failed) SKIP_VIDEO_FAILED="--skip-video-failed" ;; + *) echo "未知参数: $1" && exit 1 ;; + esac + shift +done + +# ==================== 批量任务定义 ==================== +# 定义要处理的目录列表 +# 格式: "输入目录路径|输出子目录名" +# 输出目录会创建在 $DATA_ROOT/输出子目录名 下 +tasks=( + "/path/to/videos/dir1|output_dir1" + "/path/to/videos/dir2|output_dir2" + "/path/to/videos/dir3|output_dir3" + "/path/to/videos/dir4|output_dir4" + "/path/to/videos/dir5|output_dir5" + "/path/to/videos/dir6|output_dir6" + "/path/to/videos/dir7|output_dir7" + "/path/to/videos/dir8|output_dir8" + "/path/to/videos/dir9|output_dir9" + "/path/to/videos/dir10|output_dir10" + "/path/to/videos/dir11|output_dir11" + "/path/to/videos/dir12|output_dir12" +) + +# ==================== 初始化 ==================== +echo "========================================" +echo "SyncNet 批量处理脚本" +echo "开始时间: $(date)" +echo "Python脚本: $PYTHON_SCRIPT" +echo "数据根目录: $DATA_ROOT" +echo "总任务数: ${#tasks[@]}" +echo "参数: $SKIP_FAILED $SKIP_VIDEO_FAILED" +echo "========================================" + +# 创建日志目录 +mkdir -p "$LOG_DIR" +timestamp=$(date +%Y%m%d_%H%M%S) +batch_log_file="$LOG_DIR/batch_$timestamp.log" + +# 写入日志头 +{ +echo "===== SyncNet 批量处理执行日志 =====" +echo "执行时间: $(date)" +echo "Python脚本: $PYTHON_SCRIPT" +echo "数据根目录: $DATA_ROOT" +echo "总任务数: ${#tasks[@]}" +echo "参数: $SKIP_FAILED $SKIP_VIDEO_FAILED" +echo "===================================" +echo "" +} | tee -a "$batch_log_file" + +# ==================== 处理每个任务 ==================== +total_success=0 +total_failed=0 +failed_tasks=() + +for i in "${!tasks[@]}"; do + task="${tasks[$i]}" + + # 解析任务参数 + IFS='|' read -r input_dir output_subdir <<< "$task" + + # 构造输出目录 + output_dir="$DATA_ROOT/$output_subdir" + + echo "" + echo "处理任务 $((i+1))/${#tasks[@]}:" + echo " 输入目录: $input_dir" + echo " 输出目录: $output_dir" + echo "----------------------------------------" + + # 检查输入目录是否存在 + if [[ ! -d "$input_dir" ]]; then + echo " ❌ 输入目录不存在,跳过此任务" + echo " ❌ 输入目录不存在,跳过此任务" >> "$batch_log_file" + total_failed=$((total_failed + 1)) + failed_tasks+=("$input_dir (目录不存在)") + continue + fi + + # 创建输出目录 + mkdir -p "$output_dir" + + # 构建Python命令 + PYTHON_CMD="python3 \"$PYTHON_SCRIPT\" \ + --input_dir \"$input_dir\" \ + --data_dir \"$output_dir\" \ + $SKIP_FAILED \ + $SKIP_VIDEO_FAILED" + + # 添加可选参数(如果需要) + # PYTHON_CMD="$PYTHON_CMD --facedet_scale 0.25 --batch_size 20" + + # 记录开始时间 + task_start_time=$(date +%s) + + echo " 开始执行..." + echo " 命令: $PYTHON_CMD" + echo " 任务开始时间: $(date)" + echo "" + + # 执行命令并记录到日志文件 + { + echo "========================================" + echo "任务 $((i+1))/${#tasks[@]} 开始" + echo "输入目录: $input_dir" + echo "输出目录: $output_dir" + echo "开始时间: $(date)" + echo "========================================" + } >> "$batch_log_file" + + # 执行Python脚本 + eval $PYTHON_CMD 2>&1 | tee -a "$batch_log_file" + exit_code=${PIPESTATUS[0]} + + # 记录结束时间 + task_end_time=$(date +%s) + task_duration=$((task_end_time - task_start_time)) + + # 统计结果 + if [[ $exit_code -eq 0 ]]; then + echo " ✅ 任务完成 (耗时: ${task_duration}秒)" + { + echo "========================================" + echo "任务 $((i+1))/${#tasks[@]} 完成" + echo "状态: 成功" + echo "耗时: ${task_duration}秒" + echo "结束时间: $(date)" + echo "========================================" + } >> "$batch_log_file" + total_success=$((total_success + 1)) + else + echo " ❌ 任务失败 (耗时: ${task_duration}秒)" + { + echo "========================================" + echo "任务 $((i+1))/${#tasks[@]} 失败" + echo "状态: 失败" + echo "耗时: ${task_duration}秒" + echo "结束时间: $(date)" + echo "========================================" + } >> "$batch_log_file" + total_failed=$((total_failed + 1)) + failed_tasks+=("$input_dir (退出码: $exit_code)") + + # 如果开启了skip-video-failed,则终止整个批处理 + if [[ -n "$SKIP_VIDEO_FAILED" ]]; then + echo " ⚠️ 已开启--skip-video-failed,终止批量处理" + echo " ⚠️ 已开启--skip-video-failed,终止批量处理" >> "$batch_log_file" + break + fi + fi + + echo "----------------------------------------" +done + +# ==================== 生成汇总报告 ==================== +echo "" +echo "========================================" +echo "批量处理完成!" +echo "总任务数: ${#tasks[@]}" +echo "成功任务: $total_success" +echo "失败任务: $total_failed" +echo "批量日志: $batch_log_file" +echo "完成时间: $(date)" +echo "========================================" + +if [[ ${#failed_tasks[@]} -gt 0 ]]; then + echo "" + echo "失败任务详情:" + for failed_task in "${failed_tasks[@]}"; do + echo " - $failed_task" + done +fi + +# 写入汇总到日志文件 +{ +echo "" +echo "===== 批量处理汇总 =====" +echo "总任务数: ${#tasks[@]}" +echo "成功任务: $total_success" +echo "失败任务: $total_failed" +echo "完成时间: $(date)" +echo "批量日志: $batch_log_file" +if [[ ${#failed_tasks[@]} -gt 0 ]]; then + echo "" + echo "失败任务列表:" + for failed_task in "${failed_tasks[@]}"; do + echo " - $failed_task" + done +fi +echo "========================" +} >> "$batch_log_file" + +# 输出日志文件路径(方便后续查看) +echo "" +echo "详细日志已保存到: $batch_log_file" diff --git a/multi_run_automation.py b/multi_run_automation.py new file mode 100644 index 0000000..3d2a5d8 --- /dev/null +++ b/multi_run_automation.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import argparse +import subprocess +import os +import time +import sys +from pathlib import Path +import glob + +# ==================== 基础配置 ==================== +# 日志目录 +LOG_DIR = Path("./logs") +# 要执行的脚本列表 +SCRIPTS = [ + "run_pipeline.py", + "run_syncnet.py", + "run_visualise.py" +] +# 支持的视频格式(可自行扩展) +SUPPORTED_VIDEO_EXT = ['.mp4', '.avi', '.mov', '.mkv', '.flv', '.wmv'] + +# ==================== 工具函数 ==================== +def get_timestamp(): + """生成时间戳(YYYYMMDD_HHMMSS)""" + return time.strftime("%Y%m%d_%H%M%S", time.localtime()) + +def init_log_dir(): + """初始化日志目录""" + LOG_DIR.mkdir(exist_ok=True, parents=True) + +def get_video_files(input_dir, recursive=True): + """递归/非递归获取目录下所有支持的视频文件""" + video_files = [] + input_path = Path(input_dir).resolve() + + if not input_path.exists(): + print(f"❌ 输入目录不存在: {input_dir}") + return video_files + + # 遍历所有支持的视频格式 + for ext in SUPPORTED_VIDEO_EXT: + glob_pattern = f"**/*{ext}" if recursive else f"*{ext}" + files = glob.glob(str(input_path / glob_pattern), recursive=recursive) + video_files.extend(files) + + # 去重并排序 + video_files = sorted(list(set(video_files))) + return video_files + +def run_command(cmd, log_file): + """执行命令并记录日志""" + # 记录命令执行信息 + log_content = f"\n{'='*50}\n执行命令: {' '.join(cmd)}\n开始时间: {time.ctime()}\n{'='*50}\n" + log_file.write(log_content) + log_file.flush() + + # 执行命令并捕获输出 + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8" + ) + + # 实时输出日志 + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if line: + log_file.write(line) + log_file.flush() + # 同时输出到控制台 + sys.stdout.write(line) + sys.stdout.flush() + + # 记录执行结果 + return_code = process.returncode + result = "成功" if return_code == 0 else "失败" + end_log = f"\n{'='*50}\n命令执行{result},返回码: {return_code}\n结束时间: {time.ctime()}\n{'='*50}\n" + log_file.write(end_log) + log_file.flush() + + return return_code + +# ==================== 参数解析 ==================== +def parse_args(): + parser = argparse.ArgumentParser(description="SyncNet 批量全管线自动化脚本", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + # ---------------- 批量处理核心参数 ---------------- + parser.add_argument("--input_dir", type=str, required=True, + help="视频文件目录(批量处理必填,会递归查找所有视频)") + parser.add_argument("--no-recursive", action="store_true", + help="是否禁用递归查找(仅处理input_dir下一级目录)") + + # ---------------- 共用核心参数 ---------------- + parser.add_argument("--data_dir", type=str, default="data/work", + help="输出数据根目录(每个视频会在该目录下按文件名创建子目录)") + + # ---------------- run_pipeline.py 特有参数 ---------------- + parser.add_argument("--facedet_scale", type=float, default=0.25, + help="[pipeline] 人脸检测缩放因子") + parser.add_argument("--crop_scale", type=float, default=0.40, + help="[pipeline] 裁剪框缩放因子") + parser.add_argument("--min_track", type=int, default=100, + help="[pipeline] 最小人脸跟踪时长") + parser.add_argument("--num_failed_det", type=int, default=25, + help="[pipeline] 允许的最大检测失败次数") + parser.add_argument("--min_face_size", type=int, default=100, + help="[pipeline] 最小人脸尺寸(像素)") + + # ---------------- run_syncnet.py 特有参数 ---------------- + parser.add_argument("--initial_model", type=str, default="data/syncnet_v2.model", + help="[syncnet] 初始模型路径") + parser.add_argument("--batch_size", type=int, default=20, + help="[syncnet] 批处理大小") + parser.add_argument("--vshift", type=int, default=15, + help="[syncnet] 视频偏移量") + + # ---------------- run_visualise.py 特有参数 ---------------- + parser.add_argument("--frame_rate", type=int, default=25, + help="[visualise/pipeline] 帧率") + + # ---------------- 脚本控制参数 ---------------- + parser.add_argument("--skip-failed", action="store_true", + help="某个脚本执行失败时,是否跳过该视频的后续脚本") + parser.add_argument("--skip-video-failed", action="store_true", + help="某个视频处理失败时,是否跳过下一个视频") + + return parser.parse_args() + +# ==================== 主执行逻辑 ==================== +def main(): + # 1. 解析参数 + args = parse_args() + + # 2. 初始化日志目录 + init_log_dir() + + # 3. 生成批量日志文件名(带时间戳) + timestamp = get_timestamp() + batch_log_file_path = LOG_DIR / f"syncnet_batch_automation_{timestamp}.log" + + # 4. 获取所有视频文件 + video_files = get_video_files(args.input_dir, not args.no_recursive) + if not video_files: + print(f"❌ 在目录 {args.input_dir} 下未找到支持的视频文件(支持格式:{SUPPORTED_VIDEO_EXT})") + sys.exit(1) + print(f"✅ 共找到 {len(video_files)} 个视频文件,开始批量处理...") + + # 5. 打开批量日志文件 + with open(batch_log_file_path, "a", encoding="utf-8") as batch_log_file: + # 写入批量执行头部信息 + batch_log_file.write(f"===== SyncNet 批量自动化管线执行日志 =====\n") + batch_log_file.write(f"执行时间: {time.ctime()}\n") + batch_log_file.write(f"输入目录: {args.input_dir}\n") + batch_log_file.write(f"递归查找: {not args.no_recursive}\n") + batch_log_file.write(f"输出根目录: {args.data_dir}\n") + batch_log_file.write(f"视频文件数量: {len(video_files)}\n") + batch_log_file.write(f"日志文件: {batch_log_file_path}\n") + batch_log_file.write(f"==========================================\n\n") + batch_log_file.flush() + + # 6. 遍历处理每个视频 + total_success = 0 + total_failed = 0 + failed_videos = [] + + for idx, videofile in enumerate(video_files, 1): + # 生成reference(视频文件名,不含路径和后缀) + video_path = Path(videofile) + reference = video_path.stem # 核心:用文件名作为reference + # 替换特殊字符(避免目录创建失败) + reference = reference.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + + batch_log_file.write(f"\n\n{'='*60}\n开始处理第 {idx}/{len(video_files)} 个视频:\n文件路径: {videofile}\nReference: {reference}\n{'='*60}\n") + batch_log_file.flush() + print(f"\n\n📌 开始处理第 {idx}/{len(video_files)} 个视频: {videofile} (reference: {reference})") + + # 标记当前视频是否处理成功 + video_success = True + + # 构造每个脚本的执行命令 + # 6.1 run_pipeline.py 命令 + pipeline_cmd = [ + sys.executable, "run_pipeline.py", + "--videofile", videofile, + "--reference", reference, + "--data_dir", args.data_dir, + "--facedet_scale", str(args.facedet_scale), + "--crop_scale", str(args.crop_scale), + "--min_track", str(args.min_track), + "--frame_rate", str(args.frame_rate), + "--num_failed_det", str(args.num_failed_det), + "--min_face_size", str(args.min_face_size) + ] + + # 6.2 run_syncnet.py 命令 + syncnet_cmd = [ + sys.executable, "run_syncnet.py", + "--videofile", videofile, + "--reference", reference, + "--data_dir", args.data_dir, + "--initial_model", args.initial_model, + "--batch_size", str(args.batch_size), + "--vshift", str(args.vshift) + ] + + # 6.3 run_visualise.py 命令 + visualise_cmd = [ + sys.executable, "run_visualise.py", + "--videofile", videofile, + "--reference", reference, + "--data_dir", args.data_dir, + "--frame_rate", str(args.frame_rate) + ] + + # 按顺序执行脚本 + scripts_cmds = [ + ("run_pipeline.py", pipeline_cmd), + ("run_syncnet.py", syncnet_cmd), + ("run_visualise.py", visualise_cmd) + ] + + for script_name, cmd in scripts_cmds: + batch_log_file.write(f"\n\n========== 开始执行 {script_name} ==========\n") + batch_log_file.flush() + + # 执行命令 + return_code = run_command(cmd, batch_log_file) + + # 检查执行结果 + if return_code != 0: + video_success = False + batch_log_file.write(f"\n❌ {script_name} 执行失败 (视频: {videofile})\n") + batch_log_file.flush() + print(f"\n❌ {script_name} 执行失败 (视频: {videofile})") + + # 若开启skip-failed,跳过该视频后续脚本 + if args.skip_failed: + batch_log_file.write(f"\n⚠️ 已开启--skip-failed,跳过该视频后续脚本\n") + batch_log_file.flush() + print(f"\n⚠️ 已开启--skip-failed,跳过该视频后续脚本") + break + + # 统计结果 + if video_success: + total_success += 1 + batch_log_file.write(f"\n✅ 视频 {videofile} 处理完成\n") + print(f"\n✅ 视频 {videofile} 处理完成") + else: + total_failed += 1 + failed_videos.append(videofile) + batch_log_file.write(f"\n❌ 视频 {videofile} 处理失败\n") + print(f"\n❌ 视频 {videofile} 处理失败") + + # 若开启skip-video-failed,跳过下一个视频 + if args.skip_video_failed: + batch_log_file.write(f"\n⚠️ 已开启--skip-video-failed,终止批量处理\n") + batch_log_file.flush() + print(f"\n⚠️ 已开启--skip-video-failed,终止批量处理") + break + + # 7. 批量处理完成,写入汇总信息 + batch_log_file.write(f"\n\n===== 批量处理汇总 =====\n") + batch_log_file.write(f"总视频数: {len(video_files)}\n") + batch_log_file.write(f"成功数: {total_success}\n") + batch_log_file.write(f"失败数: {total_failed}\n") + if failed_videos: + batch_log_file.write(f"失败视频列表: {failed_videos}\n") + batch_log_file.write(f"完成时间: {time.ctime()}\n") + batch_log_file.write(f"批量日志文件: {batch_log_file_path}\n") + batch_log_file.write(f"========================\n") + batch_log_file.flush() + + # 控制台输出汇总 + print(f"\n\n===== 批量处理汇总 =====") + print(f"总视频数: {len(video_files)}") + print(f"成功数: {total_success}") + print(f"失败数: {total_failed}") + if failed_videos: + print(f"失败视频列表: {failed_videos}") + print(f"批量日志文件: {batch_log_file_path}") + print(f"========================") + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"\n❌ 批量脚本执行出错: {str(e)}") + # 错误信息写入日志 + timestamp = get_timestamp() + init_log_dir() + error_log_path = LOG_DIR / f"syncnet_batch_automation_error_{timestamp}.log" + with open(error_log_path, "a", encoding="utf-8") as f: + f.write(f"执行出错时间: {time.ctime()}\n") + f.write(f"错误信息: {str(e)}\n") + sys.exit(1) diff --git a/run_automation.py b/run_automation.py new file mode 100644 index 0000000..2fb7c9e --- /dev/null +++ b/run_automation.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import argparse +import subprocess +import os +import time +import sys +from pathlib import Path + +# ==================== 基础配置 ==================== +# 日志目录 +LOG_DIR = Path("./logs") +# 要执行的脚本列表 +SCRIPTS = [ + "run_pipeline.py", + "run_syncnet.py", + "run_visualise.py" +] + +# ==================== 工具函数 ==================== +def get_timestamp(): + """生成时间戳(YYYYMMDD_HHMMSS)""" + return time.strftime("%Y%m%d_%H%M%S", time.localtime()) + +def init_log_dir(): + """初始化日志目录""" + LOG_DIR.mkdir(exist_ok=True, parents=True) + +def run_command(cmd, log_file): + """执行命令并记录日志""" + # 记录命令执行信息 + log_content = f"\n{'='*50}\n执行命令: {' '.join(cmd)}\n开始时间: {time.ctime()}\n{'='*50}\n" + log_file.write(log_content) + log_file.flush() + + # 执行命令并捕获输出 + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8" + ) + + # 实时输出日志 + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if line: + log_file.write(line) + log_file.flush() + # 同时输出到控制台 + sys.stdout.write(line) + sys.stdout.flush() + + # 记录执行结果 + return_code = process.returncode + result = "成功" if return_code == 0 else "失败" + end_log = f"\n{'='*50}\n命令执行{result},返回码: {return_code}\n结束时间: {time.ctime()}\n{'='*50}\n" + log_file.write(end_log) + log_file.flush() + + return return_code + +# ==================== 参数解析 ==================== +def parse_args(): + parser = argparse.ArgumentParser(description="SyncNet 全管线自动化脚本", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + # ---------------- 共用核心参数 ---------------- + parser.add_argument("--videofile", type=str, required=True, + help="输入视频文件路径(必填)") + parser.add_argument("--reference", type=str, required=True, + help="视频标识名称(必填)") + parser.add_argument("--data_dir", type=str, default="data/work", + help="输出数据目录") + + # ---------------- run_pipeline.py 特有参数 ---------------- + parser.add_argument("--facedet_scale", type=float, default=0.25, + help="[pipeline] 人脸检测缩放因子") + parser.add_argument("--crop_scale", type=float, default=0.40, + help="[pipeline] 裁剪框缩放因子") + parser.add_argument("--min_track", type=int, default=100, + help="[pipeline] 最小人脸跟踪时长") + parser.add_argument("--num_failed_det", type=int, default=25, + help="[pipeline] 允许的最大检测失败次数") + parser.add_argument("--min_face_size", type=int, default=100, + help="[pipeline] 最小人脸尺寸(像素)") + + # ---------------- run_syncnet.py 特有参数 ---------------- + parser.add_argument("--initial_model", type=str, default="data/syncnet_v2.model", + help="[syncnet] 初始模型路径") + parser.add_argument("--batch_size", type=int, default=20, + help="[syncnet] 批处理大小") + parser.add_argument("--vshift", type=int, default=15, + help="[syncnet] 视频偏移量") + + # ---------------- run_visualise.py 特有参数 ---------------- + parser.add_argument("--frame_rate", type=int, default=25, + help="[visualise/pipeline] 帧率") + + # ---------------- 脚本控制参数 ---------------- + parser.add_argument("--skip-failed", action="store_true", + help="某个脚本执行失败时,是否跳过后续脚本") + parser.add_argument("--log-level", type=str, default="all", + choices=["all", "error"], + help="日志级别:all(全部输出) / error(仅错误)") + + return parser.parse_args() + +# ==================== 主执行逻辑 ==================== +def main(): + # 1. 解析参数 + args = parse_args() + + # 2. 初始化日志目录 + init_log_dir() + + # 3. 生成日志文件名(带时间戳) + timestamp = get_timestamp() + log_file_path = LOG_DIR / f"syncnet_automation_{timestamp}.log" + + # 4. 打开日志文件 + with open(log_file_path, "a", encoding="utf-8") as log_file: + # 写入脚本执行头部信息 + log_file.write(f"===== SyncNet 自动化管线执行日志 =====\n") + log_file.write(f"执行时间: {time.ctime()}\n") + log_file.write(f"视频文件: {args.videofile}\n") + log_file.write(f"参考名称: {args.reference}\n") + log_file.write(f"数据目录: {args.data_dir}\n") + log_file.write(f"日志文件: {log_file_path}\n") + log_file.write(f"=======================================\n\n") + log_file.flush() + + # 5. 构造每个脚本的执行命令 + # 5.1 run_pipeline.py 命令 + pipeline_cmd = [ + sys.executable, "run_pipeline.py", + "--videofile", args.videofile, + "--reference", args.reference, + "--data_dir", args.data_dir, + "--facedet_scale", str(args.facedet_scale), + "--crop_scale", str(args.crop_scale), + "--min_track", str(args.min_track), + "--frame_rate", str(args.frame_rate), + "--num_failed_det", str(args.num_failed_det), + "--min_face_size", str(args.min_face_size) + ] + + # 5.2 run_syncnet.py 命令 + syncnet_cmd = [ + sys.executable, "run_syncnet.py", + "--videofile", args.videofile, + "--reference", args.reference, + "--data_dir", args.data_dir, + "--initial_model", args.initial_model, + "--batch_size", str(args.batch_size), + "--vshift", str(args.vshift) + ] + + # 5.3 run_visualise.py 命令 + visualise_cmd = [ + sys.executable, "run_visualise.py", + "--videofile", args.videofile, + "--reference", args.reference, + "--data_dir", args.data_dir, + "--frame_rate", str(args.frame_rate) + ] + + # 6. 按顺序执行脚本 + scripts_cmds = [ + ("run_pipeline.py", pipeline_cmd), + ("run_syncnet.py", syncnet_cmd), + ("run_visualise.py", visualise_cmd) + ] + + for script_name, cmd in scripts_cmds: + log_file.write(f"\n\n========== 开始执行 {script_name} ==========\n") + log_file.flush() + + # 执行命令 + return_code = run_command(cmd, log_file) + + # 检查执行结果,若失败且开启skip-failed则退出 + if return_code != 0 and args.skip_failed: + log_file.write(f"\n{script_name} 执行失败,已开启--skip-failed,终止后续执行\n") + print(f"\n❌ {script_name} 执行失败,日志文件: {log_file_path}") + sys.exit(return_code) + + # 7. 执行完成 + log_file.write(f"\n\n===== 所有脚本执行完成 =====\n") + log_file.write(f"完成时间: {time.ctime()}\n") + log_file.write(f"日志文件: {log_file_path}\n") + print(f"\n✅ 全管线执行完成!日志文件: {log_file_path}") + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"\n❌ 脚本执行出错: {str(e)}") + # 错误信息写入日志 + timestamp = get_timestamp() + init_log_dir() + error_log_path = LOG_DIR / f"syncnet_automation_error_{timestamp}.log" + with open(error_log_path, "a", encoding="utf-8") as f: + f.write(f"执行出错时间: {time.ctime()}\n") + f.write(f"错误信息: {str(e)}\n") + sys.exit(1) diff --git a/run_syncnet_update.py b/run_syncnet_update.py new file mode 100644 index 0000000..3514a93 --- /dev/null +++ b/run_syncnet_update.py @@ -0,0 +1,75 @@ +#!/usr/bin/python +#-*- coding: utf-8 -*- + +import time, pdb, argparse, subprocess, pickle, os, gzip, glob +import numpy as np # 新增:导入numpy + +from SyncNetInstance import * + +# ==================== PARSE ARGUMENT ==================== + +parser = argparse.ArgumentParser(description = "SyncNet"); +parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); +parser.add_argument('--batch_size', type=int, default='20', help=''); +parser.add_argument('--vshift', type=int, default='15', help=''); +parser.add_argument('--data_dir', type=str, default='data/work', help=''); +parser.add_argument('--videofile', type=str, default='', help=''); +parser.add_argument('--reference', type=str, default='', help=''); +opt = parser.parse_args(); + +setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi')) +setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp')) +setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork')) +setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop')) + + +# ==================== LOAD MODEL AND FILE LIST ==================== + +s = SyncNetInstance(); + +s.loadParameters(opt.initial_model); +print("Model %s loaded."%opt.initial_model); + +flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi')) +flist.sort() + +# ==================== GET OFFSETS ==================== + +dists = [] +offsets_list = [] # 新增:存储每个裁剪视频的偏移值 +confidences_list = [] # 新增:存储每个裁剪视频的置信度 + +for idx, fname in enumerate(flist): + print(f"\nProcessing crop video {idx}: {fname}") + offset, conf, dist = s.evaluate(opt,videofile=fname) + dists.append(dist) + offsets_list.append(offset) # 保存偏移值 + confidences_list.append(conf) # 保存置信度 + +# ==================== SAVE ACTIVESD.PCKL ==================== + +with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: + pickle.dump(dists, fil) +print(f"\nSaved raw distance matrix to: {os.path.join(opt.work_dir,opt.reference,'activesd.pckl')}") + +# ==================== 新增:解析并生成 offsets.txt ==================== +def generate_offsets_txt(opt, offsets, confidences): + """从activesd.pckl的原始数据/直接结果生成offsets.txt""" + txt_path = os.path.join(opt.work_dir, opt.reference, 'offsets.txt') + frame_rate = 25 # 固定帧率(与run_pipeline.py一致) + + with open(txt_path, 'w', encoding='utf-8') as f: + # 写入表头 + f.write("track_id\toffset_frames\toffset_seconds\tconfidence\n") + # 写入每个裁剪视频的结果 + for track_id, (offset, conf) in enumerate(zip(offsets, confidences)): + offset_sec = offset / frame_rate # 转换为秒 + f.write(f"{track_id}\t{offset}\t{offset_sec:.4f}\t{conf:.4f}\n") + + print(f"Saved offset results to: {txt_path}") + print("\n=== Final Offset Summary ===") + for track_id, (offset, conf) in enumerate(zip(offsets, confidences)): + print(f"Track {track_id}: Offset = {offset} frames ({offset/25:.4f} sec), Confidence = {conf:.4f}") + +# 调用生成函数 +generate_offsets_txt(opt, offsets_list, confidences_list) diff --git a/run_syncnet_update_0116.py b/run_syncnet_update_0116.py new file mode 100644 index 0000000..b6f42ce --- /dev/null +++ b/run_syncnet_update_0116.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +#-*- coding: utf-8 -*- + +import time, pdb, argparse, subprocess, pickle, os, gzip, glob +import numpy as np # 新增:导入numpy + +from SyncNetInstance import * + +# ==================== PARSE ARGUMENT ==================== + +parser = argparse.ArgumentParser(description = "SyncNet"); +parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); +parser.add_argument('--batch_size', type=int, default='20', help=''); +parser.add_argument('--vshift', type=int, default='15', help=''); +parser.add_argument('--data_dir', type=str, default='data/work', help=''); +parser.add_argument('--videofile', type=str, default='', help=''); +parser.add_argument('--reference', type=str, default='', help=''); +opt = parser.parse_args(); + +setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi')) +setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp')) +setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork')) +setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop')) + + +# ==================== LOAD MODEL AND FILE LIST ==================== + +s = SyncNetInstance(); + +s.loadParameters(opt.initial_model); +print("Model %s loaded."%opt.initial_model); + +flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi')) +flist.sort() + +# ==================== GET OFFSETS ==================== + +dists = [] +offsets_list = [] # 存储每个裁剪视频的偏移值 +confidences_list = [] # 存储每个裁剪视频的置信度 +avg_min_dist_list = [] # 新增:存储每个裁剪视频的「最优偏移平均同步差」 + +for idx, fname in enumerate(flist): + print(f"\nProcessing crop video {idx}: {fname}") + offset, conf, dist = s.evaluate(opt,videofile=fname) + dists.append(dist) + offsets_list.append(offset) + confidences_list.append(conf) + + # 新增:计算当前track的「最优偏移平均同步差(最小距离)」 + # dist.shape = [帧数, 2*vshift+1] → 按帧取最小距离 → 求均值 + min_vals_per_frame = np.min(dist, axis=1) # 每帧的最小距离(最优偏移对应的距离) + avg_min_dist = np.mean(min_vals_per_frame) # 所有帧的最小距离均值(平均同步差) + avg_min_dist_list.append(avg_min_dist) + print(f"Track {idx} - 最优偏移平均同步差(最小距离): {avg_min_dist:.4f}") + +# ==================== SAVE ACTIVESD.PCKL ==================== + +with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: + pickle.dump(dists, fil) +print(f"\nSaved raw distance matrix to: {os.path.join(opt.work_dir,opt.reference,'activesd.pckl')}") + +# ==================== 生成 offsets.txt(含平均同步差) ==================== +def generate_offsets_txt(opt, offsets, confidences, avg_min_dists): + """生成包含偏移、置信度、平均同步差的offsets.txt""" + txt_path = os.path.join(opt.work_dir, opt.reference, 'offsets.txt') + frame_rate = 25 # 固定帧率(与run_pipeline.py一致) + + with open(txt_path, 'w', encoding='utf-8') as f: + # 新增表头:avg_min_dist(最优偏移平均同步差) + f.write("track_id\toffset_frames\toffset_seconds\tconfidence\tavg_min_dist\n") + # 写入每个track的结果 + for track_id, (offset, conf, avg_min) in enumerate(zip(offsets, confidences, avg_min_dists)): + offset_sec = offset / frame_rate # 偏移转换为秒 + f.write(f"{track_id}\t{offset}\t{offset_sec:.4f}\t{conf:.4f}\t{avg_min:.4f}\n") + + print(f"Saved offset results to: {txt_path}") + print("\n=== Final Offset Summary ===") + for track_id, (offset, conf, avg_min) in enumerate(zip(offsets, confidences, avg_min_dists)): + print(f"Track {track_id}: " + f"Offset = {offset} frames ({offset/25:.4f} sec), " + f"Confidence = {conf:.4f}, " + f"Avg Min Dist (平均同步差) = {avg_min:.4f}") + +# 调用生成函数(传入新增的avg_min_dist_list) +generate_offsets_txt(opt, offsets_list, confidences_list, avg_min_dist_list) diff --git a/utils/syncnet_summary_mean_by_linecount.py b/utils/syncnet_summary_mean_by_linecount.py new file mode 100644 index 0000000..9cbfeea --- /dev/null +++ b/utils/syncnet_summary_mean_by_linecount.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import argparse +import numpy as np +from pathlib import Path +from collections import Counter + +def parse_offsets_txt(txt_path): + """ + 读取单个offsets.txt文件,返回: + - 有效数据行数量(不含表头) + - 按confidence降序排序后的列数据 (sorted_offset_frames, sorted_offset_seconds, sorted_confidence) + """ + offset_frames = [] + offset_seconds = [] + confidence = [] + + try: + with open(txt_path, 'r', encoding='utf-8') as f: + # 跳过表头 + header = f.readline().strip() + if not header.startswith("track_id"): + print(f"⚠️ {txt_path} 表头格式异常,跳过") + return (0, None) + + # 读取数据行 + for line in f: + line = line.strip() + if not line: + continue + # 按制表符/空格分割(兼容不同分隔符) + parts = line.split() + if len(parts) < 4: + print(f"⚠️ {txt_path} 行数据异常:{line},跳过") + continue + # 提取数值(忽略track_id) + try: + of = int(parts[1]) + os_val = float(parts[2]) + conf = float(parts[3]) + offset_frames.append(of) + offset_seconds.append(os_val) + confidence.append(conf) + except ValueError: + print(f"⚠️ {txt_path} 数值转换失败:{line},跳过") + continue + + # 统计有效数据行数量 + data_line_count = len(confidence) + if data_line_count == 0: + print(f"⚠️ {txt_path} 无有效数据行,跳过") + return (0, None) + + # 按confidence降序排序(所有列同步排序) + sorted_indices = np.argsort(confidence)[::-1] + sorted_of = [offset_frames[i] for i in sorted_indices] + sorted_os = [offset_seconds[i] for i in sorted_indices] + sorted_conf = [confidence[i] for i in sorted_indices] + + return (data_line_count, (sorted_of, sorted_os, sorted_conf)) + + except Exception as e: + print(f"❌ 读取 {txt_path} 失败:{str(e)}") + return (0, None) + +def main(): + # 解析命令行参数 + parser = argparse.ArgumentParser(description="汇总SyncNet批量处理结果的均值(按数据行数量分组计算)") + parser.add_argument("--output_dir", type=str, required=True, + help="SyncNet的输出根目录(如/path/to/output/)") + args = parser.parse_args() + + # 校验输出目录 + output_root = Path(args.output_dir).resolve() + pywork_dir = output_root / "pywork" + if not pywork_dir.exists(): + print(f"❌ 未找到pywork目录:{pywork_dir}") + return + + # 第一步:遍历所有视频子目录,收集数据行数量和排序后的数据 + all_data = {} # key: 视频名称, value: (data_line_count, sorted_data) + video_names = [] # 记录所有找到的视频名称 + + for video_subdir in pywork_dir.iterdir(): + if not video_subdir.is_dir(): + continue + offsets_txt = video_subdir / "offsets.txt" + if not offsets_txt.exists(): + print(f"⚠️ {video_subdir.name} 目录下无offsets.txt,跳过") + continue + + # 读取该视频的offsets.txt + data_line_count, sorted_data = parse_offsets_txt(str(offsets_txt)) + if data_line_count == 0 or sorted_data is None: + continue + + all_data[video_subdir.name] = (data_line_count, sorted_data) + video_names.append(video_subdir.name) + + if not all_data: + print(f"❌ 未找到任何有效offsets.txt文件") + return + + # 第二步:按数据行数量分组 + grouped_data = {} # key: 数据行数量, value: {"videos": [视频名列表], "data": [排序后数据列表]} + for video_name, (line_count, sorted_data) in all_data.items(): + if line_count not in grouped_data: + grouped_data[line_count] = {"videos": [], "data": []} + grouped_data[line_count]["videos"].append(video_name) + grouped_data[line_count]["data"].append(sorted_data) + + # 打印分组统计 + print(f"\n===== 数据行数量分组统计 =====") + for line_count in sorted(grouped_data.keys()): + video_list = grouped_data[line_count]["videos"] + print(f"数据行数量 {line_count}:共 {len(video_list)} 个文件 → {', '.join(video_list)}") + + # 第三步:逐组计算均值 + output_txt = output_root / "syncnet_summary_mean_by_linecount.txt" + with open(output_txt, 'w', encoding='utf-8') as f: + # 写入总统计信息 + f.write("===== SyncNet批量结果均值汇总(按数据行数量分组)=====\n") + f.write(f"汇总时间:{os.popen('date').read().strip()}\n") + f.write(f"输出根目录:{output_root}\n\n") + + # 逐组写入结果 + for line_count in sorted(grouped_data.keys()): + group_info = grouped_data[line_count] + video_list = group_info["videos"] + data_list = group_info["data"] + group_size = len(video_list) + + # 写入分组表头 + f.write(f"{'='*60}\n") + f.write(f"分组:数据行数量 = {line_count} 行\n") + f.write(f"参与计算的视频数:{group_size}\n") + f.write(f"参与计算的视频名称:{', '.join(video_list)}\n") + f.write(f"{'='*60}\n") + + # 写入该组均值表头 + f.write("排序索引\t均值_offset_frames\t均值_offset_seconds\t均值_confidence\t参与计算的视频数\n") + + # 计算该组均值(数据行数量一致,直接按索引遍历) + for idx in range(line_count): + # 收集该索引下所有文件的数值 + of_vals = [d[0][idx] for d in data_list] + os_vals = [d[1][idx] for d in data_list] + conf_vals = [d[2][idx] for d in data_list] + + # 计算均值 + mean_of = np.mean(of_vals) + mean_os = np.mean(os_vals) + mean_conf = np.mean(conf_vals) + + # 写入该行均值 + f.write(f"{idx+1}\t{mean_of:.4f}\t{mean_os:.4f}\t{mean_conf:.4f}\t{group_size}\n") + + f.write("\n") # 组间空行分隔 + + # 控制台输出完成信息 + print(f"\n✅ 汇总完成!结果已保存到:{output_txt}") + print(f"\n===== 最终汇总结果预览 =====") + with open(output_txt, 'r', encoding='utf-8') as f: + preview = f.read().splitlines()[:20] # 预览前20行 + print("\n".join(preview)) + if len(f.read().splitlines()) > 20: + print("...(更多内容请查看完整文件)") + +if __name__ == "__main__": + main()