From 8ec7ca2f2f0b87cfc13a637a5808ca5c2aa7d0e1 Mon Sep 17 00:00:00 2001 From: Devraj KB Date: Sun, 25 May 2025 01:45:03 +0000 Subject: [PATCH 1/2] variable offeset detection draft WIP --- multi_view_syncnet.py | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 multi_view_syncnet.py diff --git a/multi_view_syncnet.py b/multi_view_syncnet.py new file mode 100644 index 0000000..4843ff1 --- /dev/null +++ b/multi_view_syncnet.py @@ -0,0 +1,111 @@ +import os +import subprocess +import numpy as np +import matplotlib.pyplot as plt +from scipy.signal import medfilt +import soundfile as sf +from pathlib import Path +from shutil import rmtree +import argparse +from SyncNetInstance import SyncNetInstance +from types import SimpleNamespace + +# ================================ +# ARGUMENT PARSER +# ================================ +parser = argparse.ArgumentParser(description="Run SyncNet on a video segment and compute AV offset.") +parser.add_argument("--video_path", type=str, required=True, help="Path to input video file (.avi)") +parser.add_argument("--tmp_dir", type=str, required=True, help="Directory to store temporary segment files") +parser.add_argument("--model_path", type=str, required=True, help="Path to SyncNet model file (.model)") +parser.add_argument("--segment_len", type=int, default=5, help="Segment length in seconds") +parser.add_argument("--stride", type=int, default=5, help="Stride in seconds") +args = parser.parse_args() + +# ================================ +# MAIN LOGIC +# ================================ + +def run_cmd(command): + subprocess.call(command, shell=True) + +def extract_audio(video_path, audio_path): + run_cmd(f"ffmpeg -y -i {video_path} -ac 1 -ar 16000 -vn {audio_path}") + +def get_duration(filepath): + with sf.SoundFile(filepath) as f: + return len(f) / f.samplerate + +def save_segment(video_path, start_sec, duration, out_path): + run_cmd(f"ffmpeg -y -i {video_path} -ss {start_sec} -t {duration} -c copy {out_path}") + +def main(): + os.makedirs(args.tmp_dir, exist_ok=True) + full_audio_path = os.path.join(args.tmp_dir, "full.wav") + extract_audio(args.video_path, full_audio_path) + audio_duration = get_duration(full_audio_path) + video_duration = audio_duration # Assuming AV alignment + + print(f"[INFO] Full duration: {video_duration:.2f}s") + segment_offsets = [] + + s = SyncNetInstance() + s.loadParameters(args.model_path) + s.eval() + + segment_id = 0 + for start_sec in np.arange(0, video_duration, args.stride): + print(f"[DEBUG] start_sec = {start_sec}") + if start_sec + args.segment_len > video_duration: + print(f"[WARNING] Skipping segment at {start_sec:.2f}s due to insufficient length.") + continue + + segment_video = os.path.join(args.tmp_dir, f"segment_{segment_id}.avi") + segment_audio = os.path.join(args.tmp_dir, f"segment_{segment_id}.wav") + + save_segment(args.video_path, start_sec, args.segment_len, segment_video) + save_segment(full_audio_path, start_sec, args.segment_len, segment_audio) + + print(f"[INFO] Processing segment {segment_id} ({start_sec:.2f}-{start_sec + args.segment_len:.2f}s)...") + + opt = SimpleNamespace( + tmp_dir=args.tmp_dir, + reference=f"segment_{segment_id}", + batch_size=20, + vshift=15, + data_dir=args.tmp_dir, + saveframes=False + ) + + offset, conf, _ = s.evaluate(opt=opt, videofile=segment_video) + print(f" -> Offset: {offset:.3f}, Conf: {conf:.3f}") + segment_offsets.append((start_sec, offset, conf)) + + segment_id += 1 + + # Plotting + if segment_offsets: + times, offsets, confs = zip(*segment_offsets) + + frame_rate = 25 # assumed frame rate + offsets_sec = [frame / frame_rate for frame in offsets] # convert to seconds + + base_name = os.path.splitext(os.path.basename(args.video_path))[0] + output_path = os.path.join(args.tmp_dir, f"{base_name}_offset_plot.png") + + plt.figure(figsize=(10, 4)) + plt.plot(times, offsets_sec, marker='o') + plt.xlabel("Start Time (s)") + plt.ylabel("Offset (seconds)") + plt.title("Audio Offset Over Time") + plt.suptitle(f"Segment Len: {args.segment_len}s, Stride: {args.stride}s", fontsize=10, y=0.94) + plt.grid() + plt.tight_layout() + plt.savefig(output_path) + plt.show() + print(f"[INFO] Offset plot saved to: {output_path}") + else: + print("[WARNING] No valid segments were processed.") + +if __name__ == "__main__": + main() + From 9dadffd379d30c833a1e7dd8bd515e22d4059827 Mon Sep 17 00:00:00 2001 From: devrajkb <75446944+devrajkb@users.noreply.github.com> Date: Mon, 26 May 2025 06:32:22 +0530 Subject: [PATCH 2/2] scripts to introduced offset between video & audio --- generate_offset_single_video_clean_ascii.py | 171 ++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 generate_offset_single_video_clean_ascii.py diff --git a/generate_offset_single_video_clean_ascii.py b/generate_offset_single_video_clean_ascii.py new file mode 100644 index 0000000..4c1a92e --- /dev/null +++ b/generate_offset_single_video_clean_ascii.py @@ -0,0 +1,171 @@ + +import os +import json +import random +import numpy as np +from moviepy.video.io.VideoFileClip import VideoFileClip +from moviepy.audio.io.AudioFileClip import AudioFileClip +from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter +import soundfile as sf +import argparse +from pathlib import Path +import matplotlib.pyplot as plt +import pandas as pd + +def shift_audio(audio, sr, shift_sec): + shift_samples = int(sr * shift_sec) + if shift_samples > 0: + shifted = np.concatenate((np.zeros(shift_samples), audio[:-shift_samples])) + elif shift_samples < 0: + shifted = np.concatenate((audio[-shift_samples:], np.zeros(-shift_samples))) + else: + shifted = audio + return shifted + +def shift_audio_segmented(audio, sr, duration, interval_sec, shift_range): + segments = [] + offset_log = [] + total_segments = int(np.ceil(duration / interval_sec)) + + for i in range(total_segments): + start = int(i * interval_sec * sr) + end = int(min((i + 1) * interval_sec * sr, len(audio))) + segment = audio[start:end] + shift = round(random.uniform(*shift_range), 3) + shifted = shift_audio(segment, sr, shift) + offset_log.append({ + "start_sec": round(i * interval_sec, 2), + "end_sec": round(min((i + 1) * interval_sec, duration), 2), + "offset_sec": shift + }) + segments.append(shifted) + + shifted_audio = np.concatenate(segments) + return shifted_audio, offset_log + +def plot_offset(offset_log, output_path): + times = [entry["start_sec"] for entry in offset_log] + values = [entry["offset_sec"] for entry in offset_log] + + plt.figure(figsize=(10, 4)) + plt.plot(times, values, marker='o') + plt.title("Audio Offset Over Time") + plt.xlabel("Time (s)") + plt.ylabel("Offset (s)") + plt.grid(True) + plt.tight_layout() + plt.savefig(output_path) + plt.close() + +def save_offset_csv(offset_log, csv_path): + df = pd.DataFrame(offset_log) + df.to_csv(csv_path, index=False) + +def get_color_for_offset(offset): + abs_offset = abs(offset) + if abs_offset < 0.1: + return (0, 255, 0) + elif abs_offset < 0.2: + return (255, 165, 0) + else: + return (255, 0, 0) + +def save_frame_overlay_debug(video, offset_log, out_debug_path, frame_offset_csv_path): + fps = video.fps + interval_frames = int(offset_log[0]["end_sec"] * fps) + frames = list(video.iter_frames(fps=fps)) + + debug_writer = FFMPEG_VideoWriter(out_debug_path, size=video.size, fps=fps, codec="libx264") + + frame_records = [] + for i, frame in enumerate(frames): + segment_idx = min(i // interval_frames, len(offset_log) - 1) + offset_sec = offset_log[segment_idx]["offset_sec"] + offset_frames = int(round(offset_sec * fps)) + color = get_color_for_offset(offset_sec) + + # Draw text overlay + from PIL import Image, ImageDraw + img = Image.fromarray(frame) + draw = ImageDraw.Draw(img) + text = f"Offset: {offset_sec:+.2f}s ({offset_frames:+d} frames)" + draw.rectangle([10, 10, 310, 50], fill=(0, 0, 0, 128)) + draw.text((20, 20), text, fill=color) + debug_writer.write_frame(np.array(img)) + + frame_records.append({ + "frame_index": i, + "time_sec": round(i / fps, 3), + "offset_sec": offset_sec, + "offset_frames": offset_frames + }) + + debug_writer.close() + pd.DataFrame(frame_records).to_csv(frame_offset_csv_path, index=False) + +def process_single_video(video_file, output_dir, interval_sec, shift_range): + video_path = Path(video_file) + video_name = video_path.stem + input_ext = video_path.suffix.lower() + + output_video_path = os.path.join(output_dir, f"{video_name}_shifted{input_ext}") + offset_log_path = os.path.join(output_dir, f"{video_name}.offset_log.json") + offset_csv_path = os.path.join(output_dir, f"{video_name}.offset_log.csv") + offset_plot_path = os.path.join(output_dir, f"{video_name}.offset_plot.png") + debug_overlay_path = os.path.join(output_dir, f"{video_name}_debug_overlay.mp4") + frame_offset_csv_path = os.path.join(output_dir, f"{video_name}_frame_offset.csv") + + video = VideoFileClip(str(video_path)) + duration = video.duration + sr = 16000 + + temp_audio_path = os.path.join(output_dir, f"{video_name}_temp.wav") + video.audio.write_audiofile(temp_audio_path, fps=sr) + audio_array, _ = sf.read(temp_audio_path) + if audio_array.ndim > 1: + audio_array = audio_array.mean(axis=1) + + shifted_audio, offset_log = shift_audio_segmented( + audio_array, sr, duration, interval_sec, shift_range + ) + + shifted_audio_path = os.path.join(output_dir, f"{video_name}_shifted.wav") + sf.write(shifted_audio_path, shifted_audio, sr) + + with open(offset_log_path, 'w') as f: + json.dump(offset_log, f, indent=4) + + save_offset_csv(offset_log, offset_csv_path) + plot_offset(offset_log, offset_plot_path) + save_frame_overlay_debug(video, offset_log, debug_overlay_path, frame_offset_csv_path) + + new_audio = AudioFileClip(shifted_audio_path) + new_video = video.with_audio(new_audio) + + if input_ext == '.avi': + new_video.write_videofile(output_video_path, codec='mpeg4', audio_codec='libmp3lame') + else: + new_video.write_videofile(output_video_path, codec='libx264', audio_codec='aac') + + os.remove(temp_audio_path) + os.remove(shifted_audio_path) + + print("[OK] Saved shifted video:", output_video_path) + print("[INFO] Offset JSON:", offset_log_path) + print("[PLOT] Offset Plot:", offset_plot_path) + print("[CSV] Offset Table:", offset_csv_path) + print("[CSV] Frame-Level Offset Table:", frame_offset_csv_path) + print("[DEBUG] Debug Overlay Video:", debug_overlay_path) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--video_file", required=True, help="Single video file path (.mp4 or .avi)") + parser.add_argument("--output_dir", required=True, help="Where to save shifted video and logs") + parser.add_argument("--interval_sec", type=float, default=2.0, help="Interval in seconds for changing offset") + parser.add_argument("--min_shift", type=float, default=-0.3, help="Min audio shift in seconds") + parser.add_argument("--max_shift", type=float, default=0.3, help="Max audio shift in seconds") + args = parser.parse_args() + + shift_range = (args.min_shift, args.max_shift) + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + process_single_video(args.video_file, args.output_dir, args.interval_sec, shift_range)