From 16f20bd2cf97aa3fe4e595fbdade6c8f6d76b5a3 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sat, 2 Aug 2025 18:21:42 +0600 Subject: [PATCH 01/11] fix: cuda -> cpu for cpu support --- .gitignore | 1 - SyncNetInstance.py | 8 ++++---- detectors/s3fd/__init__.py | 2 +- detectors/s3fd/nets.py | 2 +- run_pipeline.py | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 350ada0..9504c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,4 @@ Thumbs.db ######################### data/ protos/ -utils/ *.pth diff --git a/SyncNetInstance.py b/SyncNetInstance.py index 497d44f..438ef6c 100644 --- a/SyncNetInstance.py +++ b/SyncNetInstance.py @@ -37,7 +37,7 @@ class SyncNetInstance(torch.nn.Module): def __init__(self, dropout = 0, num_layers_in_fc_layers = 1024): super(SyncNetInstance, self).__init__(); - self.__S__ = S(num_layers_in_fc_layers = num_layers_in_fc_layers).cuda(); + self.__S__ = S(num_layers_in_fc_layers = num_layers_in_fc_layers); def evaluate(self, opt, videofile): @@ -109,12 +109,12 @@ def evaluate(self, opt, videofile): im_batch = [ imtv[:,:,vframe:vframe+5,:,:] for vframe in range(i,min(lastframe,i+opt.batch_size)) ] im_in = torch.cat(im_batch,0) - im_out = self.__S__.forward_lip(im_in.cuda()); + im_out = self.__S__.forward_lip(im_in); im_feat.append(im_out.data.cpu()) cc_batch = [ cct[:,:,:,vframe*4:vframe*4+20] for vframe in range(i,min(lastframe,i+opt.batch_size)) ] cc_in = torch.cat(cc_batch,0) - cc_out = self.__S__.forward_aud(cc_in.cuda()) + cc_out = self.__S__.forward_aud(cc_in) cc_feat.append(cc_out.data.cpu()) im_feat = torch.cat(im_feat,0) @@ -184,7 +184,7 @@ def extract_feature(self, opt, videofile): im_batch = [ imtv[:,:,vframe:vframe+5,:,:] for vframe in range(i,min(lastframe,i+opt.batch_size)) ] im_in = torch.cat(im_batch,0) - im_out = self.__S__.forward_lipfeat(im_in.cuda()); + im_out = self.__S__.forward_lipfeat(im_in); im_feat.append(im_out.data.cpu()) im_feat = torch.cat(im_feat,0) diff --git a/detectors/s3fd/__init__.py b/detectors/s3fd/__init__.py index d7f35e0..c25e7ea 100644 --- a/detectors/s3fd/__init__.py +++ b/detectors/s3fd/__init__.py @@ -12,7 +12,7 @@ class S3FD(): - def __init__(self, device='cuda'): + def __init__(self, device='cpu'): tstamp = time.time() self.device = device diff --git a/detectors/s3fd/nets.py b/detectors/s3fd/nets.py index 85b5c82..75f0add 100644 --- a/detectors/s3fd/nets.py +++ b/detectors/s3fd/nets.py @@ -27,7 +27,7 @@ def forward(self, x): class S3FDNet(nn.Module): - def __init__(self, device='cuda'): + def __init__(self, device='cpu'): super(S3FDNet, self).__init__() self.device = device diff --git a/run_pipeline.py b/run_pipeline.py index f5fc22e..7b70553 100755 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -184,7 +184,7 @@ def crop_video(opt,track,cropfile): def inference_video(opt): - DET = S3FD(device='cuda') + DET = S3FD(device='cpu') flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg')) flist.sort() From 5d2d37fcc71e434b21e5cb2cb9cd2875caedf268 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sat, 2 Aug 2025 20:22:50 +0600 Subject: [PATCH 02/11] chore: new cli arg + updated readme --- README.md | 38 +++++++++++++++++++++++++++++++++++++ detectors/s3fd/box_utils.py | 2 +- run_syncnet.py | 12 ++++++++++++ run_visualise.py | 2 +- 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7da5354..93ad607 100755 --- a/README.md +++ b/README.md @@ -36,6 +36,44 @@ python run_syncnet.py --videofile /path/to/video.mp4 --reference name_of_video - python run_visualise.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output ``` +## Parameters + +### run_pipeline.py parameters: +- `--min_face_size`: Minimum face size in pixels (default: 100). Reduce this value for videos with smaller faces. +- `--facedet_scale`: Scale factor for face detection (default: 0.25) +- `--crop_scale`: Scale bounding box (default: 0.40) +- `--min_track`: Minimum facetrack duration (default: 100 frames) + +Example with smaller faces: +``` +python run_pipeline.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output --min_face_size 50 +``` + +## Troubleshooting + +### Issue: Empty pycrop directory / No bounding boxes in output video + +**Symptoms:** +- `$DATA_DIR/pycrop/$REFERENCE/` directory is empty +- No bounding boxes appear in the output video +- Face detection appears to work (faces are detected in console output) + +**Cause:** The detected faces are smaller than the minimum face size threshold. + +**Solution:** +1. Check your detected face sizes by examining the face detection output +2. Reduce the `--min_face_size` parameter in `run_pipeline.py` +3. For videos with small faces (< 100 pixels), try `--min_face_size 50` or lower + +**Example fix:** +```bash +# Instead of default parameters +python run_pipeline.py --videofile data/chunk_003.mp4 --reference chunk_003 --data_dir data/test/ + +# Use lower minimum face size +python run_pipeline.py --videofile data/chunk_003.mp4 --reference chunk_003 --data_dir data/test/ --min_face_size 50 +``` + Outputs: ``` $DATA_DIR/pycrop/$REFERENCE/*.avi - cropped face tracks diff --git a/detectors/s3fd/box_utils.py b/detectors/s3fd/box_utils.py index 0779bcd..1bf4be2 100644 --- a/detectors/s3fd/box_utils.py +++ b/detectors/s3fd/box_utils.py @@ -35,7 +35,7 @@ def nms_(dets, thresh): inds = np.where(ovr <= thresh)[0] order = order[inds + 1] - return np.array(keep).astype(np.int) + return np.array(keep).astype(int) def decode(loc, priors, variances): diff --git a/run_syncnet.py b/run_syncnet.py index 45099fd..a1c1053 100755 --- a/run_syncnet.py +++ b/run_syncnet.py @@ -35,11 +35,23 @@ # ==================== GET OFFSETS ==================== dists = [] +offsets = [] +confs = [] for idx, fname in enumerate(flist): offset, conf, dist = s.evaluate(opt,videofile=fname) + offsets.append(offset) + confs.append(conf) dists.append(dist) # ==================== PRINT RESULTS TO FILE ==================== with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: pickle.dump(dists, fil) + +# ==================== SAVE OFFSETS TO TXT FILE ==================== + +with open(os.path.join(opt.work_dir,opt.reference,'offsets.txt'), 'w') as fil: + for idx, (offset, conf) in enumerate(zip(offsets, confs)): + fil.write('TRACK %d: OFFSET %d, CONF %.3f\n'%(idx, offset, conf)) + +print("Offset results saved to %s" % os.path.join(opt.work_dir,opt.reference,'offsets.txt')) diff --git a/run_visualise.py b/run_visualise.py index 85d8925..59d3e0b 100644 --- a/run_visualise.py +++ b/run_visualise.py @@ -69,7 +69,7 @@ for face in faces[fidx]: - clr = max(min(face['conf']*25,255),0) + clr = int(max(min(face['conf']*25,255),0)) cv2.rectangle(image,(int(face['x']-face['s']),int(face['y']-face['s'])),(int(face['x']+face['s']),int(face['y']+face['s'])),(0,clr,255-clr),3) cv2.putText(image,'Track %d, Conf %.3f'%(face['track'],face['conf']), (int(face['x']-face['s']),int(face['y']-face['s'])),cv2.FONT_HERSHEY_SIMPLEX,0.5,(255,255,255),2) From 6558a45abbeed9cb436b1c51b690d45de6cf70fd Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sat, 2 Aug 2025 20:34:24 +0600 Subject: [PATCH 03/11] chore: created helpers --- utils/QUALITY_FILTERING_GUIDE.md | 203 +++++++++++++++ utils/README_PIPELINE.md | 161 ++++++++++++ utils/VIDEO_PROCESSING_README.md | 253 +++++++++++++++++++ utils/analyze_results.py | 243 ++++++++++++++++++ utils/batch_process.py | 285 +++++++++++++++++++++ utils/batch_processing_script.sh | 17 ++ utils/compare_filtering.py | 157 ++++++++++++ utils/process_long_video.py | 412 +++++++++++++++++++++++++++++++ utils/process_video_chunks.py | 271 ++++++++++++++++++++ utils/quality_presets.py | 209 ++++++++++++++++ utils/video_utils.py | 276 +++++++++++++++++++++ 11 files changed, 2487 insertions(+) create mode 100644 utils/QUALITY_FILTERING_GUIDE.md create mode 100644 utils/README_PIPELINE.md create mode 100644 utils/VIDEO_PROCESSING_README.md create mode 100644 utils/analyze_results.py create mode 100755 utils/batch_process.py create mode 100755 utils/batch_processing_script.sh create mode 100644 utils/compare_filtering.py create mode 100755 utils/process_long_video.py create mode 100644 utils/process_video_chunks.py create mode 100644 utils/quality_presets.py create mode 100755 utils/video_utils.py diff --git a/utils/QUALITY_FILTERING_GUIDE.md b/utils/QUALITY_FILTERING_GUIDE.md new file mode 100644 index 0000000..c2ae788 --- /dev/null +++ b/utils/QUALITY_FILTERING_GUIDE.md @@ -0,0 +1,203 @@ +# SyncNet Quality Filtering for Bengali Speech Dataset + +## ๐ŸŽฏ Overview + +Yes, **you can absolutely reject scenes/segments with low sync scores!** I've enhanced the SyncNet pipeline with comprehensive quality filtering capabilities that automatically filter out poor-quality chunks based on synchronization confidence and offset thresholds. + +## โœ… What We Accomplished + +### 1. **Enhanced SyncNet Pipeline with Quality Filtering** +- **Automatic rejection** of chunks with low confidence scores +- **Offset-based filtering** to remove poorly synchronized segments +- **Configurable thresholds** for different quality requirements +- **Filtered dataset creation** with only high-quality chunks + +### 2. **Quality Filter Criteria** +- **Confidence Score**: Measures how confident SyncNet is about the sync analysis +- **Offset Magnitude**: Measures how many frames audio/video are out of sync +- **Face Detection**: Automatically rejects chunks with no detectable faces + +### 3. **Results from Your Video** +Using strict quality filters (confidence โ‰ฅ 4.0, |offset| โ‰ค 5 frames): +- **Original**: 14 chunks total, 12 with successful analysis +- **Filtered**: 7 high-quality chunks accepted (50% acceptance rate) +- **Quality improvement**: + - Average confidence: 4.944 โ†’ 6.377 (+1.433) + - Average |offset|: 3.8 โ†’ 0.6 frames (-3.2 frames) + +## ๐Ÿ”ง Quality Filter Settings + +### Available Presets: + +| Preset | Min Confidence | Max |Offset| | Use Case | +|--------|---------------|--------------|----------| +| `strict` | โ‰ฅ6.0 | โ‰ค2 frames | Publication-ready, highest quality | +| `high` | โ‰ฅ4.0 | โ‰ค5 frames | Training data, good quality | +| `medium` | โ‰ฅ2.5 | โ‰ค8 frames | Balanced approach | +| `relaxed` | โ‰ฅ1.5 | โ‰ค12 frames | Keep most usable chunks | +| `none` | โ‰ฅ0.0 | โ‰ค50 frames | No filtering | + +## ๐Ÿš€ How to Use Quality Filtering + +### 1. **Basic Usage with Filtering** +```bash +python process_long_video.py \ + --input_video "your_video.mp4" \ + --output_dir "output_folder" \ + --min_confidence 4.0 \ + --max_abs_offset 5 +``` + +### 2. **Using Quality Presets** +```bash +# List available presets +python quality_presets.py --list_presets + +# Use a specific preset +python quality_presets.py \ + --input_video "your_video.mp4" \ + --output_dir "analysis" \ + --preset high + +# Compare multiple presets +python quality_presets.py \ + --input_video "your_video.mp4" \ + --output_dir "comparison" \ + --compare_presets strict high medium +``` + +### 3. **Disable Filtering (Keep All Chunks)** +```bash +python process_long_video.py \ + --input_video "your_video.mp4" \ + --output_dir "output_folder" \ + --no_filter +``` + +## ๐Ÿ“Š Understanding Quality Metrics + +### **Confidence Scores** +- **>6.0**: Excellent synchronization confidence +- **4.0-6.0**: Good synchronization confidence +- **2.0-4.0**: Moderate synchronization confidence +- **<2.0**: Poor synchronization confidence + +### **Offset Values** (in frames @ 25fps) +- **0-2 frames**: Perfect/Excellent sync (0-80ms) +- **3-5 frames**: Good sync (120-200ms) +- **6-10 frames**: Acceptable sync (240-400ms) +- **>10 frames**: Poor sync (>400ms) + +### **Your Video Results** (with high preset) +โœ… **Accepted Chunks (7):** +- `chunk_000`: confidence=7.183, offset=-1 frame โญ **Best** +- `chunk_002`: confidence=6.349, offset=0 frames +- `chunk_003`: confidence=6.993, offset=0 frames +- `chunk_004`: confidence=6.289, offset=0 frames +- `chunk_011`: confidence=6.161, offset=-1 frame +- `chunk_012`: confidence=7.151, offset=-1 frame +- `chunk_013`: confidence=4.511, offset=-1 frame + +โŒ **Rejected Chunks (7):** +- Low confidence: 5 chunks +- High offset: 3 chunks +- No faces detected: 2 chunks + +## ๐Ÿ“ Output Structure with Filtering + +``` +output_folder/ +โ”œโ”€โ”€ chunks/ # All video chunks +โ”œโ”€โ”€ audio/ # All audio files +โ”œโ”€โ”€ filtered_chunks/ # โœ… Only high-quality video chunks +โ”œโ”€โ”€ filtered_audio/ # โœ… Only high-quality audio files +โ”œโ”€โ”€ syncnet_data/ # SyncNet analysis results +โ””โ”€โ”€ processing_summary.json # Complete results with filtering info +``` + +## ๐ŸŽฏ Recommended Workflows + +### **For Training Data Creation:** +1. Use `high` preset (confidence โ‰ฅ 4.0, |offset| โ‰ค 5) +2. Extract audio from `filtered_audio/` folder +3. Use corresponding chunks from `filtered_chunks/` folder + +### **For Research/Publication:** +1. Use `strict` preset (confidence โ‰ฅ 6.0, |offset| โ‰ค 2) +2. Manual review of accepted chunks +3. Report filtering criteria in methodology + +### **For Quick Dataset Expansion:** +1. Use `medium` preset for balanced quality/quantity +2. Batch process multiple videos +3. Combine filtered results + +### **For Quality Assessment:** +```bash +# Analyze filtering results +python analyze_results.py output_folder/processing_summary.json --detailed + +# Compare before/after filtering +python compare_filtering.py original_summary.json filtered_summary.json +``` + +## ๐Ÿ’ก Quality Filtering Benefits + +### **Automatic Quality Control** +- Eliminates manual review of hundreds of chunks +- Consistent quality criteria across entire dataset +- Saves time in dataset preparation + +### **Improved Training Data** +- Higher average confidence scores +- Better audio-visual synchronization +- More reliable for speech recognition training + +### **Configurable Standards** +- Adjust criteria based on your requirements +- Different presets for different use cases +- Easy to experiment with different thresholds + +## ๐Ÿ”„ Batch Processing with Quality Filtering + +### **Process Multiple Videos:** +```bash +#!/bin/bash +for video in /path/to/videos/*.mp4; do + video_name=$(basename "$video" .mp4) + + python process_long_video.py \ + --input_video "$video" \ + --output_dir "${video_name}_analysis" \ + --min_confidence 4.0 \ + --max_abs_offset 5 + + # Analyze results + python analyze_results.py "${video_name}_analysis/processing_summary.json" +done +``` + +### **Combine Filtered Results:** +```bash +# Copy all filtered chunks to a master dataset +mkdir -p master_dataset/{videos,audio} + +for analysis_dir in *_analysis; do + if [ -d "$analysis_dir/filtered_chunks" ]; then + cp "$analysis_dir/filtered_chunks"/* master_dataset/videos/ + cp "$analysis_dir/filtered_audio"/* master_dataset/audio/ + fi +done +``` + +## ๐Ÿ“ˆ Quality Filtering Success + +**Your Bengali video processing with quality filtering:** +- **Input**: 340 seconds of video +- **After chunking**: 14 x 30-second chunks +- **After SyncNet**: 12 successful analyses +- **After quality filtering**: 7 high-quality chunks (210 seconds) +- **Quality improvement**: +1.433 confidence, -3.2 frames offset +- **Ready for training**: 7 synchronized audio-video pairs + +The quality filtering successfully identified and retained only the best-synchronized segments while automatically rejecting poor-quality chunks, giving you a clean, high-quality dataset for Bengali speech recognition training! ๐ŸŽ‰ diff --git a/utils/README_PIPELINE.md b/utils/README_PIPELINE.md new file mode 100644 index 0000000..651f80e --- /dev/null +++ b/utils/README_PIPELINE.md @@ -0,0 +1,161 @@ +# SyncNet Video Processing Pipeline + +## Overview +This pipeline processes long videos by cutting them into chunks, extracting audio, and analyzing audio-visual synchronization using SyncNet. It's specifically designed for Bengali speech audio-visual dataset processing. + +## What We Accomplished + +### โœ… Complete Pipeline Implementation +1. **Video Chunking**: Splits long videos into manageable 30-second chunks with 5-second overlap +2. **Audio Extraction**: Extracts 16kHz mono audio from each chunk +3. **SyncNet Analysis**: Runs complete face detection, tracking, and lip-sync analysis +4. **Visualization**: Generates output videos with face markings and tracking info +5. **Results Analysis**: Provides detailed sync quality metrics and recommendations + +### โœ… Successfully Processed Your Video +- **Input**: `/Users/darklord/Research/Audio Visual/Code/bengali-speech-audio-visual-dataset/downloads/aRHpoSebPPI.mp4` +- **Duration**: 340.61 seconds (5.7 minutes) +- **Chunks Created**: 14 chunks (30s each with 5s overlap) +- **Successful Analysis**: 12/14 chunks (85.7% success rate) +- **No Faces Detected**: 2 chunks (chunks 5 and 6) + +### ๐Ÿ“Š Quality Results +- **Average Confidence**: 4.944 (moderate quality) +- **Best Chunk**: chunk_000 (confidence: 7.183, offset: -1 frame) +- **Worst Chunk**: chunk_009 (confidence: 2.231, offset: -2 frames) +- **High Quality Chunks**: 6 (>5.0 confidence) +- **Medium Quality Chunks**: 6 (2.0-5.0 confidence) + +## Generated Files Structure + +``` +aRHpoSebPPI_analysis/ +โ”œโ”€โ”€ chunks/ # Video chunks (14 files) +โ”‚ โ”œโ”€โ”€ chunk_000.mp4 # 0-30 seconds +โ”‚ โ”œโ”€โ”€ chunk_001.mp4 # 25-55 seconds +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ audio/ # Extracted audio files +โ”‚ โ”œโ”€โ”€ chunk_000.wav # 16kHz mono audio +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ syncnet_data/ # SyncNet processing data +โ”‚ โ”œโ”€โ”€ pywork/ # Analysis results +โ”‚ โ”‚ โ”œโ”€โ”€ chunk_000/ +โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ offsets.txt # Sync analysis results +โ”‚ โ”‚ โ””โ”€โ”€ ... +โ”‚ โ”œโ”€โ”€ pyavi/ # Visualization videos +โ”‚ โ”‚ โ”œโ”€โ”€ chunk_000/ +โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ video_out.avi # Video with face markings +โ”‚ โ”‚ โ””โ”€โ”€ ... +โ”‚ โ”œโ”€โ”€ pycrop/ # Cropped face videos +โ”‚ โ””โ”€โ”€ pyframes/ # Individual frames +โ””โ”€โ”€ processing_summary.json # Complete results summary +``` + +## Key Scripts Created + +### 1. `process_long_video.py` - Main Processing Pipeline +```bash +python process_long_video.py \ + --input_video "/path/to/video.mp4" \ + --output_dir "analysis_output" \ + --chunk_duration 30 \ + --overlap 5 +``` + +### 2. `analyze_results.py` - Results Analysis +```bash +python analyze_results.py analysis_output/processing_summary.json --detailed --list-files +``` + +### 3. `process_video_chunks.py` - Alternative Processing Script +```bash +python process_video_chunks.py \ + --video "/path/to/video.mp4" \ + --chunk_duration 30 \ + --max_chunks 5 +``` + +## Understanding SyncNet Results + +### Offset Values +- **Negative offset**: Audio leads video (audio comes first) +- **Positive offset**: Video leads audio (video comes first) +- **Zero offset**: Perfect synchronization +- **ยฑ1-2 frames**: Excellent sync +- **ยฑ3-5 frames**: Good sync +- **>ยฑ10 frames**: Poor sync + +### Confidence Scores +- **>7.0**: Excellent confidence +- **5.0-7.0**: Good confidence +- **2.0-5.0**: Moderate confidence +- **<2.0**: Low confidence + +### Your Video Analysis +Most chunks show moderate to good synchronization: +- **Best segments**: chunks 0, 2, 3, 11, 12 (confidence >6.0) +- **Moderate segments**: chunks 1, 7, 8, 9, 10 (confidence 2-5) +- **Problem areas**: chunks 5, 6 (no faces detected) + +## Usage for Bengali Speech Dataset + +### For Training Data +1. **Use high-confidence chunks** (>5.0) for training data +2. **Extract audio from best chunks** for speech recognition training +3. **Use visualization videos** to verify face detection quality + +### For Dataset Expansion +```bash +# Process multiple videos +for video in /path/to/videos/*.mp4; do + python process_long_video.py \ + --input_video "$video" \ + --output_dir "$(basename "$video" .mp4)_analysis" +done + +# Analyze all results +for summary in *_analysis/processing_summary.json; do + python analyze_results.py "$summary" --detailed +done +``` + +### Batch Processing Script +Create a batch script to process multiple videos: + +```bash +#!/bin/bash +VIDEO_DIR="/Users/darklord/Research/Audio Visual/Code/bengali-speech-audio-visual-dataset/downloads" +OUTPUT_DIR="batch_analysis" + +for video in "$VIDEO_DIR"/*.mp4; do + video_name=$(basename "$video" .mp4) + echo "Processing: $video_name" + + python process_long_video.py \ + --input_video "$video" \ + --output_dir "$OUTPUT_DIR/${video_name}_analysis" \ + --chunk_duration 30 \ + --overlap 5 + + echo "Analyzing results for: $video_name" + python analyze_results.py "$OUTPUT_DIR/${video_name}_analysis/processing_summary.json" --detailed +done +``` + +## Next Steps + +1. **Review visualization videos** to ensure face detection quality +2. **Extract high-quality audio chunks** for speech training +3. **Process additional videos** in your dataset +4. **Use sync analysis** to filter quality training data +5. **Combine results** from multiple videos for comprehensive dataset + +## File Locations for Your Processed Video + +- **Chunks**: `aRHpoSebPPI_analysis/chunks/` +- **Audio**: `aRHpoSebPPI_analysis/audio/` +- **Sync Results**: `aRHpoSebPPI_analysis/syncnet_data/pywork/*/offsets.txt` +- **Videos with Face Markings**: `aRHpoSebPPI_analysis/syncnet_data/pyavi/*/video_out.avi` +- **Summary**: `aRHpoSebPPI_analysis/processing_summary.json` + +The pipeline is now ready to process your entire Bengali speech video dataset efficiently! diff --git a/utils/VIDEO_PROCESSING_README.md b/utils/VIDEO_PROCESSING_README.md new file mode 100644 index 0000000..0202067 --- /dev/null +++ b/utils/VIDEO_PROCESSING_README.md @@ -0,0 +1,253 @@ +# SyncNet Video Processing Pipeline + +This directory contains scripts for processing long videos with SyncNet analysis, including video chunking, audio extraction, and batch processing capabilities. + +## Scripts Overview + +### 1. `process_long_video.py` - Complete Pipeline +Automatically processes long video files by: +- Cutting video into chunks +- Extracting audio from each chunk +- Running SyncNet analysis on each chunk +- Generating comprehensive results + +**Usage:** +```bash +python process_long_video.py \ + --input_video /path/to/long_video.mp4 \ + --output_dir /path/to/output \ + --chunk_duration 30 \ + --overlap 5 +``` + +**Parameters:** +- `--input_video`: Path to input video file +- `--output_dir`: Output directory for all results +- `--chunk_duration`: Duration of each chunk in seconds (default: 30) +- `--overlap`: Overlap between chunks in seconds (default: 5) +- `--min_track`: Minimum track length for SyncNet (default: 5) +- `--min_face_size`: Minimum face size for SyncNet (default: 10) + +**Output Structure:** +``` +output_dir/ +โ”œโ”€โ”€ chunks/ # Video chunks +โ”‚ โ”œโ”€โ”€ chunk_000.mp4 +โ”‚ โ”œโ”€โ”€ chunk_001.mp4 +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ audio/ # Extracted audio files +โ”‚ โ”œโ”€โ”€ chunk_000.wav +โ”‚ โ”œโ”€โ”€ chunk_001.wav +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ syncnet_data/ # SyncNet analysis data +โ”‚ โ”œโ”€โ”€ pyavi/ +โ”‚ โ”œโ”€โ”€ pycrop/ +โ”‚ โ”œโ”€โ”€ pywork/ +โ”‚ โ””โ”€โ”€ ... +โ””โ”€โ”€ processing_summary.json # Complete results summary +``` + +### 2. `video_utils.py` - Video Utilities +Flexible utilities for video processing tasks. + +**Commands:** + +#### Get Video Information +```bash +python video_utils.py info --input video.mp4 +``` + +#### Extract Audio +```bash +python video_utils.py extract-audio \ + --input video.mp4 \ + --output audio.wav \ + --sample_rate 16000 \ + --channels 1 +``` + +#### Cut Video by Time +```bash +python video_utils.py chunk-time \ + --input video.mp4 \ + --output_dir chunks/ \ + --duration 30 \ + --overlap 5 +``` + +#### Cut Video by Silence +```bash +python video_utils.py chunk-silence \ + --input video.mp4 \ + --output_dir chunks/ \ + --threshold -30 \ + --min_silence 1.0 \ + --min_chunk 5.0 +``` + +### 3. `batch_process.py` - Batch Processing +Process multiple video chunks with SyncNet in parallel. + +**Usage:** +```bash +python batch_process.py \ + --chunks_dir /path/to/chunks \ + --output_dir /path/to/results \ + --max_workers 2 +``` + +**Parameters:** +- `--chunks_dir`: Directory containing video chunks +- `--output_dir`: Output directory for SyncNet results +- `--max_workers`: Maximum parallel workers (default: 2) +- `--min_track`: Minimum track length for SyncNet (default: 5) +- `--min_face_size`: Minimum face size for SyncNet (default: 10) + +## Complete Workflow Examples + +### Example 1: Process a Single Long Video +```bash +# Process 2-hour Bengali news video +python process_long_video.py \ + --input_video ~/videos/bengali_news_2hours.mp4 \ + --output_dir ~/results/bengali_news \ + --chunk_duration 45 \ + --overlap 10 \ + --min_track 10 \ + --min_face_size 15 +``` + +### Example 2: Manual Chunking + Batch Processing +```bash +# Step 1: Cut video by silence detection +python video_utils.py chunk-silence \ + --input long_video.mp4 \ + --output_dir chunks/ \ + --threshold -25 \ + --min_silence 2.0 \ + --min_chunk 10.0 + +# Step 2: Extract audio from all chunks +for chunk in chunks/*.mp4; do + python video_utils.py extract-audio \ + --input "$chunk" \ + --output "audio/$(basename "$chunk" .mp4).wav" +done + +# Step 3: Run batch SyncNet analysis +python batch_process.py \ + --chunks_dir chunks/ \ + --output_dir syncnet_results/ \ + --max_workers 3 +``` + +### Example 3: Process Multiple Videos +```bash +# Process all videos in a directory +for video in ~/videos/*.mp4; do + echo "Processing: $video" + python process_long_video.py \ + --input_video "$video" \ + --output_dir "~/results/$(basename "$video" .mp4)" \ + --chunk_duration 30 \ + --overlap 5 +done +``` + +## Output Files Explained + +### SyncNet Results +Each processed chunk generates: +- **Cropped face videos**: `syncnet_data/pycrop/chunk_XXX/*.avi` +- **Offsets file**: `syncnet_data/pywork/chunk_XXX/offsets.txt` +- **Visualization video**: `syncnet_data/pyavi/chunk_XXX/video_out.avi` + +### Summary Files +- **processing_summary.json**: Complete processing results +- **batch_processing_summary.json**: Batch processing statistics + +### Sample Offsets File +``` +TRACK 0: OFFSET 2, CONF 8.147 +``` +- **OFFSET**: Audio-visual synchronization offset in frames +- **CONF**: Confidence score (higher = better sync) + +## Performance Tips + +1. **Chunk Duration**: + - 30-60 seconds works well for most content + - Shorter chunks for fast-changing scenes + - Longer chunks for stable talking head videos + +2. **Overlap**: + - 5-10 seconds overlap helps with continuity + - More overlap for better temporal consistency + +3. **Parallel Processing**: + - Use 2-4 workers for batch processing + - Monitor CPU/memory usage + - SyncNet is computationally intensive + +4. **Face Detection**: + - Increase `min_face_size` for distant faces + - Increase `min_track` for stable tracking + - Check lighting and video quality + +## Troubleshooting + +### Common Issues + +1. **No faces detected**: + - Check video quality and lighting + - Reduce `min_face_size` parameter + - Try different chunk boundaries + +2. **Low confidence scores**: + - Video may have poor audio-visual sync + - Check for audio delays or processing artifacts + - Verify speaker is visible in video + +3. **Memory issues**: + - Reduce `max_workers` in batch processing + - Process shorter chunks + - Close other applications + +4. **FFmpeg errors**: + - Ensure FFmpeg is installed and in PATH + - Check video file format compatibility + - Verify sufficient disk space + +### Debugging + +Enable verbose output: +```bash +# Add debug flags to see detailed processing +python process_long_video.py --input video.mp4 --output_dir results/ -v +``` + +Check individual chunk processing: +```bash +# Test single chunk first +python run_pipeline.py --videofile chunk_000.mp4 --reference test_chunk +python run_syncnet.py --videofile data/work/pycrop/test_chunk/000.avi --reference test_chunk +``` + +## Dependencies + +Required packages (install with `pip install -r requirements.txt`): +- torch>=1.4.0 +- torchvision +- opencv-contrib-python +- scipy +- numpy +- Pillow +- python_speech_features +- librosa +- pydub +- scenedetect==0.5.1 + +System requirements: +- FFmpeg (for video/audio processing) +- CUDA-capable GPU (recommended for faster processing) +- Sufficient disk space for chunks and intermediate files diff --git a/utils/analyze_results.py b/utils/analyze_results.py new file mode 100644 index 0000000..b21ce13 --- /dev/null +++ b/utils/analyze_results.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +Results Analysis Script for SyncNet Processing +Analyzes and summarizes SyncNet results from processed video chunks +""" + +import os +import json +import argparse +from pathlib import Path + +def load_summary(summary_file): + """Load processing summary JSON file""" + with open(summary_file, 'r') as f: + return json.load(f) + +def analyze_sync_quality(results): + """Analyze synchronization quality across chunks""" + successful_chunks = [r for r in results if r['status'] == 'success'] + + if not successful_chunks: + return {"error": "No successful chunks found"} + + # Extract confidence scores and offsets + confidences = [r['confidence'] for r in successful_chunks] + offsets = [r['offset'] for r in successful_chunks] + + # Calculate statistics + avg_confidence = sum(confidences) / len(confidences) + max_confidence = max(confidences) + min_confidence = min(confidences) + + avg_offset = sum(abs(o) for o in offsets) / len(offsets) + + # Categorize sync quality + high_quality = [r for r in successful_chunks if r['confidence'] > 5.0] + medium_quality = [r for r in successful_chunks if 2.0 <= r['confidence'] <= 5.0] + low_quality = [r for r in successful_chunks if r['confidence'] < 2.0] + + return { + "total_successful": len(successful_chunks), + "average_confidence": avg_confidence, + "max_confidence": max_confidence, + "min_confidence": min_confidence, + "average_abs_offset": avg_offset, + "high_quality_chunks": len(high_quality), + "medium_quality_chunks": len(medium_quality), + "low_quality_chunks": len(low_quality), + "best_chunk": max(successful_chunks, key=lambda x: x['confidence']), + "worst_chunk": min(successful_chunks, key=lambda x: x['confidence']) + } + +def print_detailed_analysis(summary_data): + """Print detailed analysis of results""" + print("๐ŸŽฌ VIDEO PROCESSING ANALYSIS") + print("="*60) + + # Basic stats + print(f"๐Ÿ“น Input Video: {summary_data['input_video']}") + print(f"โฑ๏ธ Total Chunks: {summary_data['total_chunks']}") + print(f"โœ… Successful Analysis: {summary_data['successful_analysis']}") + print(f"โŒ No Faces Detected: {summary_data['no_faces_chunks']}") + print(f"โฐ Chunk Duration: {summary_data['chunk_duration']}s") + print(f"๐Ÿ”„ Overlap: {summary_data['overlap']}s") + + # Quality filtering stats (if available) + if 'accepted_chunks' in summary_data: + print(f"\n๐ŸŽฏ QUALITY FILTERING") + print("-"*40) + print(f"โœ… Accepted Chunks: {summary_data['accepted_chunks']}") + print(f"โŒ Rejected Chunks: {summary_data['rejected_chunks']}") + print(f"๐Ÿ“ˆ Acceptance Rate: {summary_data.get('acceptance_rate', 0):.1f}%") + + if 'quality_filters' in summary_data: + filters = summary_data['quality_filters'] + print(f"๐Ÿ” Filter Settings:") + print(f" - Min Confidence: {filters.get('min_confidence', 'N/A')}") + print(f" - Max |Offset|: {filters.get('max_abs_offset', 'N/A')} frames") + print(f" - Filtering: {'Enabled' if filters.get('filter_enabled', False) else 'Disabled'}") + + # Show rejection reasons + if 'rejected_chunks' in summary_data and summary_data['rejected_chunks']: + print(f"\n๐Ÿ—‘๏ธ REJECTION REASONS:") + rejection_counts = {} + for rejected in summary_data['rejected_chunks']: + if isinstance(rejected, dict) and 'reason' in rejected: + for reason in rejected['reason']: + rejection_counts[reason] = rejection_counts.get(reason, 0) + 1 + + for reason, count in rejection_counts.items(): + print(f" - {reason}: {count} chunks") + + # Sync quality analysis + print(f"\n๐ŸŽฏ SYNCHRONIZATION ANALYSIS") + print("-"*40) + + analysis = analyze_sync_quality(summary_data['results']) + + if 'error' in analysis: + print(f"โŒ {analysis['error']}") + return + + print(f"๐Ÿ“Š Average Confidence: {analysis['average_confidence']:.3f}") + print(f"๐Ÿ“ˆ Max Confidence: {analysis['max_confidence']:.3f}") + print(f"๐Ÿ“‰ Min Confidence: {analysis['min_confidence']:.3f}") + print(f"โš–๏ธ Average Offset: {analysis['average_abs_offset']:.1f} frames") + + print(f"\n๐Ÿ† QUALITY BREAKDOWN") + print("-"*40) + print(f"๐ŸŸข High Quality (>5.0): {analysis['high_quality_chunks']} chunks") + print(f"๐ŸŸก Medium Quality (2.0-5.0): {analysis['medium_quality_chunks']} chunks") + print(f"๐Ÿ”ด Low Quality (<2.0): {analysis['low_quality_chunks']} chunks") + + # Best and worst chunks + best = analysis['best_chunk'] + worst = analysis['worst_chunk'] + + print(f"\nโญ BEST CHUNK: {best['reference']}") + print(f" Confidence: {best['confidence']:.3f}, Offset: {best['offset']} frames") + print(f" Status: {best.get('quality_status', 'unknown')}") + + print(f"\nโš ๏ธ WORST CHUNK: {worst['reference']}") + print(f" Confidence: {worst['confidence']:.3f}, Offset: {worst['offset']} frames") + print(f" Status: {worst.get('quality_status', 'unknown')}") + +def list_output_files(base_dir): + """List all generated output files""" + print(f"\n๐Ÿ“ OUTPUT FILES") + print("="*60) + + # Video chunks + chunks_dir = os.path.join(base_dir, "chunks") + if os.path.exists(chunks_dir): + chunks = sorted(os.listdir(chunks_dir)) + print(f"๐Ÿ“น Video Chunks ({len(chunks)} files):") + for chunk in chunks[:5]: # Show first 5 + print(f" - {chunk}") + if len(chunks) > 5: + print(f" ... and {len(chunks) - 5} more") + + # Audio files + audio_dir = os.path.join(base_dir, "audio") + if os.path.exists(audio_dir): + audio_files = sorted(os.listdir(audio_dir)) + print(f"\n๐ŸŽต Audio Files ({len(audio_files)} files):") + for audio in audio_files[:5]: # Show first 5 + print(f" - {audio}") + if len(audio_files) > 5: + print(f" ... and {len(audio_files) - 5} more") + + # SyncNet outputs + syncnet_dir = os.path.join(base_dir, "syncnet_data") + if os.path.exists(syncnet_dir): + # Find offsets files + import glob + offsets_files = glob.glob(os.path.join(syncnet_dir, "**", "offsets.txt"), recursive=True) + print(f"\n๐Ÿ“Š SyncNet Analysis ({len(offsets_files)} results):") + for offset_file in sorted(offsets_files)[:5]: + rel_path = os.path.relpath(offset_file, syncnet_dir) + print(f" - {rel_path}") + if len(offsets_files) > 5: + print(f" ... and {len(offsets_files) - 5} more") + + # Find visualization videos + viz_videos = glob.glob(os.path.join(syncnet_dir, "**", "video_out.avi"), recursive=True) + print(f"\n๐ŸŽฌ Visualization Videos ({len(viz_videos)} files):") + for viz_video in sorted(viz_videos)[:5]: + rel_path = os.path.relpath(viz_video, syncnet_dir) + print(f" - {rel_path}") + if len(viz_videos) > 5: + print(f" ... and {len(viz_videos) - 5} more") + +def recommend_next_steps(analysis_data): + """Provide recommendations based on analysis""" + print(f"\n๐Ÿ’ก RECOMMENDATIONS") + print("="*60) + + analysis = analyze_sync_quality(analysis_data['results']) + + if 'error' in analysis: + print("โŒ No successful analysis found") + return + + # Quality recommendations + if analysis['high_quality_chunks'] >= analysis['total_successful'] * 0.7: + print("โœ… Excellent sync quality! Most chunks show good lip-sync.") + elif analysis['medium_quality_chunks'] >= analysis['total_successful'] * 0.5: + print("โš ๏ธ Moderate sync quality. Consider:") + print(" - Check audio-video alignment in source") + print(" - Verify face detection quality") + else: + print("๐Ÿ”ด Poor sync quality detected. Consider:") + print(" - Check source video quality") + print(" - Ensure clear face visibility") + print(" - Verify audio quality") + + # Offset patterns + successful_results = [r for r in analysis_data['results'] if r['status'] == 'success'] + offsets = [r['offset'] for r in successful_results] + + if all(abs(o) <= 2 for o in offsets): + print("โœ… Consistent sync across chunks (ยฑ2 frames)") + elif max(abs(o) for o in offsets) > 10: + print("โš ๏ธ Large offset variations detected") + print(" - May indicate source sync issues") + + # Usage suggestions + print(f"\n๐Ÿ› ๏ธ NEXT STEPS:") + print("1. Review high-quality chunks for best results") + print("2. Use visualization videos to verify face detection") + print("3. Extract audio from best-sync chunks for training") + print("4. Apply similar processing to other videos") + +def main(): + parser = argparse.ArgumentParser(description='Analyze SyncNet processing results') + parser.add_argument('summary_file', help='Path to processing_summary.json file') + parser.add_argument('--detailed', action='store_true', help='Show detailed analysis') + parser.add_argument('--list-files', action='store_true', help='List all output files') + + args = parser.parse_args() + + if not os.path.exists(args.summary_file): + print(f"โŒ Summary file not found: {args.summary_file}") + return 1 + + # Load summary + summary_data = load_summary(args.summary_file) + + # Print analysis + print_detailed_analysis(summary_data) + + if args.list_files: + base_dir = os.path.dirname(args.summary_file) + list_output_files(base_dir) + + if args.detailed: + recommend_next_steps(summary_data) + + return 0 + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/utils/batch_process.py b/utils/batch_process.py new file mode 100755 index 0000000..be2fe37 --- /dev/null +++ b/utils/batch_process.py @@ -0,0 +1,285 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import json +import argparse +import subprocess +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed +import time + +def process_single_chunk(chunk_info): + """ + Process a single video chunk with SyncNet + + Args: + chunk_info: Dictionary with chunk information + + Returns: + Dictionary with processing results + """ + chunk_path = chunk_info['chunk_path'] + reference_name = chunk_info['reference_name'] + data_dir = chunk_info['data_dir'] + syncnet_dir = chunk_info['syncnet_dir'] + min_track = chunk_info.get('min_track', 5) + min_face_size = chunk_info.get('min_face_size', 10) + + print(f"๐Ÿ”„ Processing {reference_name}...") + + result = { + 'reference_name': reference_name, + 'chunk_path': chunk_path, + 'status': 'failed', + 'start_time': time.time() + } + + try: + # Step 1: Run preprocessing pipeline + pipeline_cmd = f'python run_pipeline.py --videofile "{chunk_path}" --reference {reference_name} --data_dir "{data_dir}" --min_track {min_track} --min_face_size {min_face_size}' + + print(f"Debug: Executing command: {pipeline_cmd}") + print(f"Debug: Working directory: {syncnet_dir}") + + pipeline_result = subprocess.run( + pipeline_cmd, + capture_output=True, + text=True, + shell=True, + cwd=syncnet_dir + ) + + if pipeline_result.returncode != 0: + result['error'] = f"Pipeline failed: {pipeline_result.stderr}" + return result + + # Check if face tracks were found + crop_dir = os.path.join(data_dir, 'pycrop', reference_name) + crop_files = [] + if os.path.exists(crop_dir): + crop_files = [f for f in os.listdir(crop_dir) if f.endswith('.avi')] + + if not crop_files: + result['status'] = 'no_faces' + result['message'] = 'No face tracks found' + return result + + result['face_tracks'] = len(crop_files) + + # Step 2: Run SyncNet analysis + syncnet_cmd = f'python run_syncnet.py --videofile "{os.path.join(crop_dir, crop_files[0])}" --reference {reference_name} --data_dir "{data_dir}"' + + syncnet_result = subprocess.run( + syncnet_cmd, + capture_output=True, + text=True, + shell=True, + cwd=syncnet_dir + ) + + if syncnet_result.returncode != 0: + result['error'] = f"SyncNet failed: {syncnet_result.stderr}" + return result + + # Step 3: Parse results + offsets_file = os.path.join(data_dir, 'pywork', reference_name, 'offsets.txt') + + if os.path.exists(offsets_file): + with open(offsets_file, 'r') as f: + content = f.read().strip() + if content: + # Parse multiple tracks: "TRACK 0: OFFSET -1, CONF 7.183" + lines = content.split('\n') + tracks = [] + + for line in lines: + line = line.strip() + if line and 'TRACK' in line: + try: + # Parse: "TRACK 0: OFFSET -1, CONF 7.183" + parts = line.split(', ') + if len(parts) >= 2: + offset_part = parts[0].split(': OFFSET ') + conf_part = parts[1].split('CONF ') + + if len(offset_part) >= 2 and len(conf_part) >= 2: + track_info = { + "offset": int(offset_part[1]), + "confidence": float(conf_part[1]) + } + tracks.append(track_info) + except (ValueError, IndexError) as e: + print(f"Warning: Could not parse line '{line}': {e}") + + # Use the track with highest confidence + if tracks: + best_track = max(tracks, key=lambda x: x['confidence']) + result['offset'] = best_track["offset"] + result['confidence'] = best_track["confidence"] + result['all_tracks'] = tracks + + result['status'] = 'success' + result['offsets_file'] = offsets_file + + except Exception as e: + result['error'] = str(e) + + finally: + result['end_time'] = time.time() + result['processing_time'] = result['end_time'] - result['start_time'] + + return result + +def batch_process_chunks(chunks_dir, output_dir, syncnet_dir, max_workers=2, min_track=5, min_face_size=10): + """ + Process multiple video chunks in parallel + + Args: + chunks_dir: Directory containing video chunks + output_dir: Output directory for SyncNet results + syncnet_dir: Path to SyncNet python directory + max_workers: Maximum number of parallel workers + min_track: Minimum track length for SyncNet + min_face_size: Minimum face size for SyncNet + """ + # Find all video chunks + chunk_files = [] + for ext in ['*.mp4', '*.avi', '*.mov', '*.mkv']: + chunk_files.extend(Path(chunks_dir).glob(ext)) + + chunk_files = sorted(chunk_files) + + if not chunk_files: + print(f"โŒ No video files found in {chunks_dir}") + return + + print(f"๐Ÿ“น Found {len(chunk_files)} video chunks") + + # Create SyncNet data directory + syncnet_data_dir = os.path.join(output_dir, 'syncnet_data') + os.makedirs(syncnet_data_dir, exist_ok=True) + + # Prepare chunk information + chunk_infos = [] + for chunk_file in chunk_files: + chunk_name = chunk_file.stem + chunk_infos.append({ + 'chunk_path': os.path.abspath(str(chunk_file)), + 'reference_name': chunk_name, + 'data_dir': os.path.abspath(syncnet_data_dir), + 'syncnet_dir': syncnet_dir, + 'min_track': min_track, + 'min_face_size': min_face_size + }) + + # Process chunks in parallel + results = [] + completed = 0 + + print(f"๐Ÿš€ Starting batch processing with {max_workers} workers...") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all jobs + future_to_chunk = { + executor.submit(process_single_chunk, chunk_info): chunk_info + for chunk_info in chunk_infos + } + + # Collect results as they complete + for future in as_completed(future_to_chunk): + chunk_info = future_to_chunk[future] + + try: + result = future.result() + results.append(result) + completed += 1 + + # Print progress + status_emoji = "โœ…" if result['status'] == 'success' else "โš ๏ธ" if result['status'] == 'no_faces' else "โŒ" + status_msg = f"Offset={result.get('offset', '?')}, Conf={result.get('confidence', '?'):.3f}" if result['status'] == 'success' else result.get('message', result.get('error', 'Unknown')) + + print(f"{status_emoji} [{completed}/{len(chunk_infos)}] {result['reference_name']}: {status_msg}") + + except Exception as e: + print(f"โŒ [{completed}/{len(chunk_infos)}] {chunk_info['reference_name']}: Exception - {e}") + results.append({ + 'reference_name': chunk_info['reference_name'], + 'chunk_path': chunk_info['chunk_path'], + 'status': 'exception', + 'error': str(e) + }) + completed += 1 + + # Save results summary + summary = { + 'total_chunks': len(chunk_files), + 'successful': len([r for r in results if r['status'] == 'success']), + 'no_faces': len([r for r in results if r['status'] == 'no_faces']), + 'failed': len([r for r in results if r['status'] in ['failed', 'exception']]), + 'results': results, + 'processing_timestamp': time.time() + } + + summary_file = os.path.join(output_dir, 'batch_processing_summary.json') + with open(summary_file, 'w') as f: + json.dump(summary, f, indent=2) + + # Print final summary + print(f"\n{'='*60}") + print("๐Ÿ“Š BATCH PROCESSING SUMMARY") + print(f"{'='*60}") + print(f"Total chunks: {summary['total_chunks']}") + print(f"โœ… Successful: {summary['successful']}") + print(f"โš ๏ธ No faces: {summary['no_faces']}") + print(f"โŒ Failed: {summary['failed']}") + print(f"๐Ÿ“„ Summary saved: {summary_file}") + + # Print detailed results for successful analyses + successful_results = [r for r in results if r['status'] == 'success'] + if successful_results: + print(f"\n๐Ÿ“ˆ SUCCESSFUL SYNC ANALYSIS:") + for result in successful_results: + offset = result.get('offset', '?') + conf = result.get('confidence', '?') + time_taken = result.get('processing_time', '?') + print(f" {result['reference_name']}: Offset={offset}, Conf={conf:.3f}, Time={time_taken:.1f}s") + + return results + +def main(): + parser = argparse.ArgumentParser(description="Batch process video chunks with SyncNet") + parser.add_argument('--chunks_dir', required=True, help='Directory containing video chunks') + parser.add_argument('--output_dir', required=True, help='Output directory for results') + parser.add_argument('--syncnet_dir', required=True, help='Path to SyncNet python directory') + parser.add_argument('--max_workers', type=int, default=2, help='Maximum parallel workers (default: 2)') + parser.add_argument('--min_track', type=int, default=5, help='Minimum track length for SyncNet (default: 5)') + parser.add_argument('--min_face_size', type=int, default=10, help='Minimum face size for SyncNet (default: 10)') + + args = parser.parse_args() + + # Validate inputs + if not os.path.exists(args.chunks_dir): + print(f"โŒ Chunks directory not found: {args.chunks_dir}") + sys.exit(1) + + if not os.path.exists(args.syncnet_dir): + print(f"โŒ SyncNet directory not found: {args.syncnet_dir}") + sys.exit(1) + + # Create output directory + os.makedirs(args.output_dir, exist_ok=True) + + # Run batch processing + batch_process_chunks( + chunks_dir=args.chunks_dir, + output_dir=args.output_dir, + syncnet_dir=args.syncnet_dir, + max_workers=args.max_workers, + min_track=args.min_track, + min_face_size=args.min_face_size + ) + +if __name__ == "__main__": + main() diff --git a/utils/batch_processing_script.sh b/utils/batch_processing_script.sh new file mode 100755 index 0000000..721afc0 --- /dev/null +++ b/utils/batch_processing_script.sh @@ -0,0 +1,17 @@ +#!/bin/bash +VIDEO_DIR="/Users/darklord/Research/Audio Visual/Code/bengali-speech-audio-visual-dataset/poc/outputs/aRHpoSebPPI/chunks/video" +OUTPUT_DIR="batch_analysis" + +for video in "$VIDEO_DIR"/*.mp4; do + video_name=$(basename "$video" .mp4) + echo "Processing: $video_name" + + python process_long_video.py \ + --input_video "$video" \ + --output_dir "$OUTPUT_DIR/${video_name}_analysis" \ + --chunk_duration 30 \ + --overlap 5 + + echo "Analyzing results for: $video_name" + python analyze_results.py "$OUTPUT_DIR/${video_name}_analysis/processing_summary.json" --detailed +done \ No newline at end of file diff --git a/utils/compare_filtering.py b/utils/compare_filtering.py new file mode 100644 index 0000000..0d6305d --- /dev/null +++ b/utils/compare_filtering.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +Quality Filtering Comparison Script +Compare results before and after quality filtering +""" + +import os +import json +import argparse + +def load_summary(summary_file): + """Load processing summary JSON file""" + with open(summary_file, 'r') as f: + return json.load(f) + +def compare_filtering_results(unfiltered_summary, filtered_summary): + """Compare unfiltered vs filtered results""" + + print("๐Ÿ” QUALITY FILTERING COMPARISON") + print("="*60) + + # Basic stats comparison + print("๐Ÿ“Š BASIC STATISTICS") + print("-"*40) + print(f"Total chunks: {unfiltered_summary['total_chunks']}") + print(f"Successful analysis: {unfiltered_summary['successful_analysis']}") + print(f"No faces detected: {unfiltered_summary['no_faces_chunks']}") + + # Filtering comparison + if 'accepted_chunks' in filtered_summary: + print(f"\n๐ŸŽฏ FILTERING RESULTS") + print("-"*40) + print(f"Before filtering: {unfiltered_summary['successful_analysis']} usable chunks") + print(f"After filtering: {len(filtered_summary['accepted_chunks'])} high-quality chunks") + print(f"Acceptance rate: {filtered_summary['acceptance_rate']:.1f}%") + print(f"Rejected: {len(filtered_summary['rejected_chunks'])} chunks") + + # Quality improvement + unfiltered_results = [r for r in unfiltered_summary['results'] if r['status'] == 'success'] + filtered_results = [r for r in filtered_summary['results'] if r.get('quality_status') == 'accepted'] + + if unfiltered_results and filtered_results: + unfiltered_avg_conf = sum(r['confidence'] for r in unfiltered_results) / len(unfiltered_results) + filtered_avg_conf = sum(r['confidence'] for r in filtered_results) / len(filtered_results) + + unfiltered_avg_offset = sum(abs(r['offset']) for r in unfiltered_results) / len(unfiltered_results) + filtered_avg_offset = sum(abs(r['offset']) for r in filtered_results) / len(filtered_results) + + print(f"\n๐Ÿ“ˆ QUALITY IMPROVEMENT") + print("-"*40) + print(f"Average confidence:") + print(f" Before: {unfiltered_avg_conf:.3f}") + print(f" After: {filtered_avg_conf:.3f} (+{filtered_avg_conf - unfiltered_avg_conf:.3f})") + + print(f"Average |offset|:") + print(f" Before: {unfiltered_avg_offset:.1f} frames") + print(f" After: {filtered_avg_offset:.1f} frames ({filtered_avg_offset - unfiltered_avg_offset:+.1f})") + + # Filter criteria + if 'quality_filters' in filtered_summary: + filters = filtered_summary['quality_filters'] + print(f"\n๐Ÿ”ง FILTER CRITERIA") + print("-"*40) + print(f"Minimum confidence: โ‰ฅ{filters['min_confidence']}") + print(f"Maximum |offset|: โ‰ค{filters['max_abs_offset']} frames") + print(f"Filtering enabled: {filters['filter_enabled']}") + + # Accepted chunks details + if 'accepted_chunks' in filtered_summary: + print(f"\nโœ… ACCEPTED CHUNKS ({len(filtered_summary['accepted_chunks'])})") + print("-"*40) + accepted_refs = set(filtered_summary['accepted_chunks']) + + for result in filtered_summary['results']: + if result['reference'] in accepted_refs and result['status'] == 'success': + print(f" {result['reference']}: confidence={result['confidence']:.3f}, offset={result['offset']}") + + # Rejection reasons breakdown + if 'rejected_chunks' in filtered_summary and filtered_summary['rejected_chunks']: + print(f"\nโŒ REJECTION BREAKDOWN ({len(filtered_summary['rejected_chunks'])} chunks)") + print("-"*40) + + reason_counts = {} + for rejected in filtered_summary['rejected_chunks']: + if isinstance(rejected, dict) and 'reason' in rejected: + for reason in rejected['reason']: + # Clean up reason for counting + if 'low_confidence' in reason: + key = 'Low confidence' + elif 'high_offset' in reason: + key = 'High offset' + elif 'no_faces' in reason: + key = 'No faces detected' + else: + key = reason + + reason_counts[key] = reason_counts.get(key, 0) + 1 + + for reason, count in reason_counts.items(): + print(f" {reason}: {count} chunks") + + # Recommendations + if 'accepted_chunks' in filtered_summary: + acceptance_rate = filtered_summary['acceptance_rate'] + + print(f"\n๐Ÿ’ก RECOMMENDATIONS") + print("-"*40) + + if acceptance_rate >= 70: + print("โœ… Excellent filtering results! Most chunks meet quality standards.") + elif acceptance_rate >= 50: + print("โš ๏ธ Moderate filtering results. Consider:") + print(" - Relaxing filter criteria slightly") + print(" - Checking source video quality") + elif acceptance_rate >= 30: + print("๐Ÿ”ด Low acceptance rate. Consider:") + print(" - Significantly relaxing filter criteria") + print(" - Using different videos with better sync") + else: + print("โŒ Very low acceptance rate. Consider:") + print(" - Disabling filtering temporarily") + print(" - Checking video source quality") + print(" - Using manual quality assessment") + + print(f"\n๐ŸŽฏ DATASET USAGE:") + print(f" - Use {len(filtered_summary['accepted_chunks'])} high-quality chunks for training") + print(f" - Total training data: ~{len(filtered_summary['accepted_chunks']) * 30} seconds") + print(f" - Audio files ready in: filtered_audio/") + print(f" - Video files ready in: filtered_chunks/") + +def main(): + parser = argparse.ArgumentParser(description='Compare filtering results') + parser.add_argument('unfiltered_summary', help='Path to unfiltered processing_summary.json') + parser.add_argument('filtered_summary', help='Path to filtered processing_summary.json') + + args = parser.parse_args() + + if not os.path.exists(args.unfiltered_summary): + print(f"โŒ Unfiltered summary not found: {args.unfiltered_summary}") + return 1 + + if not os.path.exists(args.filtered_summary): + print(f"โŒ Filtered summary not found: {args.filtered_summary}") + return 1 + + # Load summaries + unfiltered_data = load_summary(args.unfiltered_summary) + filtered_data = load_summary(args.filtered_summary) + + # Compare results + compare_filtering_results(unfiltered_data, filtered_data) + + return 0 + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/utils/process_long_video.py b/utils/process_long_video.py new file mode 100755 index 0000000..058fc4d --- /dev/null +++ b/utils/process_long_video.py @@ -0,0 +1,412 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import argparse +import subprocess +import json +import time +from pathlib import Path + +def run_command(command, description): + """Run a shell command and handle errors""" + print(f"Running: {description}") + print(f"Command: {command}") + + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Error in {description}:") + print(f"stdout: {result.stdout}") + print(f"stderr: {result.stderr}") + return False + + print(f"โœ… {description} completed successfully") + return True + +def get_video_duration(video_path): + """Get video duration in seconds using ffprobe""" + command = f'ffprobe -v quiet -show_entries format=duration -of csv=p=0 "{video_path}"' + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.returncode == 0: + return float(result.stdout.strip()) + else: + print(f"Error getting video duration: {result.stderr}") + return None + +def cut_video_chunks(input_video, output_dir, chunk_duration=30, overlap=5): + """ + Cut video into chunks with optional overlap + + Args: + input_video: Path to input video file + output_dir: Directory to save chunks + chunk_duration: Duration of each chunk in seconds + overlap: Overlap between chunks in seconds + + Returns: + List of chunk file paths + """ + print(f"\n๐ŸŽฌ Cutting video into {chunk_duration}s chunks...") + + # Create output directory + os.makedirs(output_dir, exist_ok=True) + + # Get video duration + total_duration = get_video_duration(input_video) + if total_duration is None: + return [] + + print(f"Total video duration: {total_duration:.2f} seconds") + + chunk_files = [] + chunk_num = 0 + start_time = 0 + + while start_time < total_duration: + # Calculate end time + end_time = min(start_time + chunk_duration, total_duration) + actual_duration = end_time - start_time + + # Skip very short chunks + if actual_duration < 5: + break + + # Generate chunk filename + chunk_name = f"chunk_{chunk_num:03d}.mp4" + chunk_path = os.path.join(output_dir, chunk_name) + + # Extract chunk using ffmpeg + command = f'ffmpeg -y -i "{input_video}" -ss {start_time} -t {actual_duration} -c copy "{chunk_path}"' + + if run_command(command, f"Extracting chunk {chunk_num} ({start_time:.1f}s - {end_time:.1f}s)"): + chunk_files.append(chunk_path) + print(f" ๐Ÿ“น Created: {chunk_name}") + + chunk_num += 1 + start_time += chunk_duration - overlap # Move start with overlap + + print(f"โœ… Created {len(chunk_files)} video chunks") + return chunk_files + +def extract_audio_from_chunk(video_path, audio_output_dir): + """Extract audio from a video chunk""" + video_name = Path(video_path).stem + audio_path = os.path.join(audio_output_dir, f"{video_name}.wav") + + # Create audio output directory + os.makedirs(audio_output_dir, exist_ok=True) + + # Extract audio using ffmpeg + command = f'ffmpeg -y -i "{video_path}" -ac 1 -ar 16000 -vn "{audio_path}"' + + if run_command(command, f"Extracting audio from {video_name}"): + return audio_path + return None + +def run_syncnet_pipeline(video_path, reference_name, data_dir, min_track=5, min_face_size=10): + """ + Run complete SyncNet pipeline on a video chunk + + Returns: + Dictionary with results or None if failed + """ + print(f"\n๐Ÿ”„ Running SyncNet pipeline for {reference_name}...") + + # Step 1: Run preprocessing pipeline + pipeline_cmd = f'python run_pipeline.py --videofile "{video_path}" --reference {reference_name} --data_dir "{data_dir}" --min_track {min_track} --min_face_size {min_face_size}' + + if not run_command(pipeline_cmd, f"SyncNet preprocessing for {reference_name}"): + return None + + # Check if cropped face video was created + crop_dir = os.path.join(data_dir, 'pycrop', reference_name) + crop_files = [f for f in os.listdir(crop_dir) if f.endswith('.avi')] if os.path.exists(crop_dir) else [] + + if not crop_files: + print(f"โŒ No face tracks found for {reference_name}") + return {"status": "no_faces", "reference": reference_name} + + # Step 2: Run SyncNet analysis + syncnet_cmd = f'python run_syncnet.py --videofile "{os.path.join(crop_dir, crop_files[0])}" --reference {reference_name} --data_dir "{data_dir}"' + + if not run_command(syncnet_cmd, f"SyncNet analysis for {reference_name}"): + return None + + # Step 3: Generate visualization (optional) + viz_cmd = f'python run_visualise.py --videofile "{video_path}" --reference {reference_name} --data_dir "{data_dir}"' + run_command(viz_cmd, f"SyncNet visualization for {reference_name}") + + # Read results + offsets_file = os.path.join(data_dir, 'pywork', reference_name, 'offsets.txt') + + results = { + "status": "success", + "reference": reference_name, + "video_path": video_path, + "face_tracks": len(crop_files), + "offsets_file": offsets_file if os.path.exists(offsets_file) else None + } + + # Parse offset results + if os.path.exists(offsets_file): + with open(offsets_file, 'r') as f: + content = f.read().strip() + if content: + # Parse multiple tracks: "TRACK 0: OFFSET -1, CONF 7.183" + lines = content.split('\n') + tracks = [] + + for line in lines: + line = line.strip() + if line and 'TRACK' in line: + try: + # Parse: "TRACK 0: OFFSET -1, CONF 7.183" + parts = line.split(', ') + if len(parts) >= 2: + offset_part = parts[0].split(': OFFSET ') + conf_part = parts[1].split('CONF ') + + if len(offset_part) >= 2 and len(conf_part) >= 2: + track_info = { + "offset": int(offset_part[1]), + "confidence": float(conf_part[1]) + } + tracks.append(track_info) + except (ValueError, IndexError) as e: + print(f"Warning: Could not parse line '{line}': {e}") + + # Use the track with highest confidence + if tracks: + best_track = max(tracks, key=lambda x: x['confidence']) + results["offset"] = best_track["offset"] + results["confidence"] = best_track["confidence"] + results["all_tracks"] = tracks + + return results + +def process_long_video(input_video, output_base_dir, chunk_duration=30, overlap=5, min_track=5, min_face_size=10, + min_confidence=2.0, max_abs_offset=10, filter_low_quality=True): + """ + Main function to process a long video file + + Args: + input_video: Path to input video file + output_base_dir: Base directory for all outputs + chunk_duration: Duration of each chunk in seconds + overlap: Overlap between chunks in seconds + min_track: Minimum track length for SyncNet + min_face_size: Minimum face size for SyncNet + min_confidence: Minimum confidence score to accept chunk (default: 2.0) + max_abs_offset: Maximum absolute offset in frames to accept chunk (default: 10) + filter_low_quality: Whether to filter out low-quality chunks (default: True) + """ + print(f"๐ŸŽฅ Processing long video: {input_video}") + print(f"๐Ÿ“ Output directory: {output_base_dir}") + + # Create output directories + chunks_dir = os.path.join(output_base_dir, 'chunks') + audio_dir = os.path.join(output_base_dir, 'audio') + syncnet_data_dir = os.path.join(output_base_dir, 'syncnet_data') + + os.makedirs(output_base_dir, exist_ok=True) + + # Step 1: Cut video into chunks + chunk_files = cut_video_chunks(input_video, chunks_dir, chunk_duration, overlap) + + if not chunk_files: + print("โŒ No chunks created. Exiting.") + return + + # Step 2: Process each chunk + results = [] + accepted_chunks = [] + rejected_chunks = [] + + for i, chunk_path in enumerate(chunk_files): + chunk_name = Path(chunk_path).stem + print(f"\n{'='*60}") + print(f"Processing chunk {i+1}/{len(chunk_files)}: {chunk_name}") + print(f"{'='*60}") + + # Extract audio + audio_path = extract_audio_from_chunk(chunk_path, audio_dir) + + if audio_path: + print(f" ๐ŸŽต Audio extracted: {audio_path}") + + # Run SyncNet analysis + syncnet_result = run_syncnet_pipeline( + chunk_path, + chunk_name, + syncnet_data_dir, + min_track=min_track, + min_face_size=min_face_size + ) + + if syncnet_result: + syncnet_result["chunk_index"] = i + syncnet_result["audio_path"] = audio_path + results.append(syncnet_result) + + # Evaluate chunk quality and decide whether to accept/reject + if syncnet_result["status"] == "success": + offset = syncnet_result.get("offset", float('inf')) + confidence = syncnet_result.get("confidence", 0.0) + + # Quality check + abs_offset = abs(offset) + is_good_confidence = confidence >= min_confidence + is_good_offset = abs_offset <= max_abs_offset + + if filter_low_quality and (not is_good_confidence or not is_good_offset): + # Reject this chunk + syncnet_result["quality_status"] = "rejected" + syncnet_result["rejection_reason"] = [] + + if not is_good_confidence: + syncnet_result["rejection_reason"].append(f"low_confidence ({confidence:.3f} < {min_confidence})") + if not is_good_offset: + syncnet_result["rejection_reason"].append(f"high_offset ({abs_offset} > {max_abs_offset})") + + rejected_chunks.append(syncnet_result) + print(f" โŒ REJECTED: {', '.join(syncnet_result['rejection_reason'])}") + print(f" Offset={offset}, Confidence={confidence:.3f}") + else: + # Accept this chunk + syncnet_result["quality_status"] = "accepted" + accepted_chunks.append(syncnet_result) + print(f" โœ… ACCEPTED: Offset={offset}, Confidence={confidence:.3f}") + + elif syncnet_result["status"] == "no_faces": + syncnet_result["quality_status"] = "rejected" + syncnet_result["rejection_reason"] = ["no_faces_detected"] + rejected_chunks.append(syncnet_result) + print(f" โŒ REJECTED: No faces detected in this chunk") + + # Create filtered output directories for accepted chunks only + if filter_low_quality and accepted_chunks: + filtered_chunks_dir = os.path.join(output_base_dir, 'filtered_chunks') + filtered_audio_dir = os.path.join(output_base_dir, 'filtered_audio') + + os.makedirs(filtered_chunks_dir, exist_ok=True) + os.makedirs(filtered_audio_dir, exist_ok=True) + + print(f"\n๐Ÿ” Creating filtered dataset with {len(accepted_chunks)} high-quality chunks...") + + for chunk_result in accepted_chunks: + # Copy video chunk + src_video = chunk_result["video_path"] + dst_video = os.path.join(filtered_chunks_dir, os.path.basename(src_video)) + import shutil + shutil.copy2(src_video, dst_video) + + # Copy audio file + src_audio = chunk_result["audio_path"] + if src_audio and os.path.exists(src_audio): + dst_audio = os.path.join(filtered_audio_dir, os.path.basename(src_audio)) + shutil.copy2(src_audio, dst_audio) + + print(f" ๐Ÿ“ Filtered videos: {filtered_chunks_dir}") + print(f" ๐Ÿ“ Filtered audio: {filtered_audio_dir}") + + # Step 3: Save summary results + summary_file = os.path.join(output_base_dir, 'processing_summary.json') + + summary = { + "input_video": input_video, + "total_chunks": len(chunk_files), + "successful_analysis": len([r for r in results if r["status"] == "success"]), + "no_faces_chunks": len([r for r in results if r["status"] == "no_faces"]), + "accepted_chunks": len(accepted_chunks), + "rejected_chunks": len(rejected_chunks), + "acceptance_rate": len(accepted_chunks) / len(chunk_files) * 100 if chunk_files else 0, + "chunk_duration": chunk_duration, + "overlap": overlap, + "quality_filters": { + "min_confidence": min_confidence, + "max_abs_offset": max_abs_offset, + "filter_enabled": filter_low_quality + }, + "processing_time": time.time(), + "results": results, + "accepted_chunks": [r["reference"] for r in accepted_chunks], + "rejected_chunks": [{"reference": r["reference"], "reason": r.get("rejection_reason", [])} for r in rejected_chunks] + } + + with open(summary_file, 'w') as f: + json.dump(summary, f, indent=2) + + print(f"\n{'='*60}") + print("๐Ÿ“Š PROCESSING SUMMARY") + print(f"{'='*60}") + print(f"Total chunks processed: {summary['total_chunks']}") + print(f"Successful SyncNet analysis: {summary['successful_analysis']}") + print(f"Chunks with no faces: {summary['no_faces_chunks']}") + + if filter_low_quality: + print(f"โœ… Accepted chunks: {summary['accepted_chunks']} ({summary['acceptance_rate']:.1f}%)") + print(f"โŒ Rejected chunks: {summary['rejected_chunks']}") + print(f"๐ŸŽฏ Quality filters: confidenceโ‰ฅ{min_confidence}, |offset|โ‰ค{max_abs_offset}") + + print(f"Summary saved to: {summary_file}") + + # Print individual results + print(f"\n๐Ÿ“‹ DETAILED RESULTS:") + for result in results: + if result["status"] == "success": + offset = result.get("offset", "?") + conf = result.get("confidence", "?") + status = result.get("quality_status", "unknown") + status_emoji = "โœ…" if status == "accepted" else "โŒ" if status == "rejected" else "โš ๏ธ" + + print(f" {status_emoji} {result['reference']}: Offset={offset}, Conf={conf:.3f}") + + if status == "rejected" and "rejection_reason" in result: + print(f" Reason: {', '.join(result['rejection_reason'])}") + + # Print summary of rejected chunks + if rejected_chunks: + print(f"\n๐Ÿ—‘๏ธ REJECTED CHUNKS SUMMARY:") + rejection_reasons = {} + for chunk in rejected_chunks: + for reason in chunk.get("rejection_reason", []): + rejection_reasons[reason] = rejection_reasons.get(reason, 0) + 1 + + for reason, count in rejection_reasons.items(): + print(f" - {reason}: {count} chunks") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process long video with SyncNet analysis") + parser.add_argument('--input_video', type=str, required=True, help='Path to input video file') + parser.add_argument('--output_dir', type=str, required=True, help='Output directory for all results') + parser.add_argument('--chunk_duration', type=int, default=30, help='Duration of each chunk in seconds (default: 30)') + parser.add_argument('--overlap', type=int, default=5, help='Overlap between chunks in seconds (default: 5)') + parser.add_argument('--min_track', type=int, default=5, help='Minimum track length for SyncNet (default: 5)') + parser.add_argument('--min_face_size', type=int, default=10, help='Minimum face size for SyncNet (default: 10)') + parser.add_argument('--min_confidence', type=float, default=2.0, help='Minimum confidence score to accept chunk (default: 2.0)') + parser.add_argument('--max_abs_offset', type=int, default=10, help='Maximum absolute offset in frames to accept chunk (default: 10)') + parser.add_argument('--filter_low_quality', action='store_true', default=True, help='Filter out low-quality chunks (default: True)') + parser.add_argument('--no_filter', action='store_false', dest='filter_low_quality', help='Disable quality filtering') + + args = parser.parse_args() + + # Validate input + if not os.path.exists(args.input_video): + print(f"โŒ Input video file not found: {args.input_video}") + sys.exit(1) + + # Run processing + process_long_video( + input_video=args.input_video, + output_base_dir=args.output_dir, + chunk_duration=args.chunk_duration, + overlap=args.overlap, + min_track=args.min_track, + min_face_size=args.min_face_size, + min_confidence=args.min_confidence, + max_abs_offset=args.max_abs_offset, + filter_low_quality=args.filter_low_quality + ) diff --git a/utils/process_video_chunks.py b/utils/process_video_chunks.py new file mode 100644 index 0000000..462221b --- /dev/null +++ b/utils/process_video_chunks.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +""" +Complete Video Processing Pipeline with SyncNet +Cuts video into chunks, extracts audio, and runs SyncNet analysis +""" + +import os +import sys +import subprocess +import argparse +from pathlib import Path + +def run_command(command, description=""): + """Run a shell command and return success status""" + print(f"\n{'='*50}") + print(f"Running: {description}") + print(f"Command: {command}") + print(f"{'='*50}") + + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.stdout: + print("STDOUT:") + print(result.stdout) + + if result.stderr: + print("STDERR:") + print(result.stderr) + + if result.returncode == 0: + print(f"โœ… Success: {description}") + return True + else: + print(f"โŒ Failed: {description} (exit code: {result.returncode})") + return False + +def get_video_duration(video_path): + """Get video duration in seconds using ffprobe""" + command = f'ffprobe -v quiet -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "{video_path}"' + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.returncode == 0: + try: + return float(result.stdout.strip()) + except ValueError: + return None + return None + +def create_chunks(video_path, chunk_duration=30, output_dir="video_chunks"): + """Split video into chunks of specified duration""" + + # Create output directory + os.makedirs(output_dir, exist_ok=True) + + # Get video duration + total_duration = get_video_duration(video_path) + if total_duration is None: + print("โŒ Could not determine video duration") + return [] + + print(f"๐Ÿ“น Video duration: {total_duration:.2f} seconds") + + chunk_files = [] + start_time = 0 + chunk_idx = 0 + + while start_time < total_duration: + # Calculate chunk end time + end_time = min(start_time + chunk_duration, total_duration) + actual_duration = end_time - start_time + + # Generate chunk filename + video_name = Path(video_path).stem + chunk_filename = f"{video_name}_chunk_{chunk_idx:03d}.mp4" + chunk_path = os.path.join(output_dir, chunk_filename) + + # FFmpeg command to extract chunk + command = f'ffmpeg -y -i "{video_path}" -ss {start_time} -t {actual_duration} -c copy "{chunk_path}"' + + if run_command(command, f"Creating chunk {chunk_idx} ({start_time:.1f}s - {end_time:.1f}s)"): + chunk_files.append(chunk_path) + print(f"โœ… Created: {chunk_path}") + else: + print(f"โŒ Failed to create chunk {chunk_idx}") + + start_time = end_time + chunk_idx += 1 + + return chunk_files + +def extract_audio(video_path, audio_output_dir="audio_chunks"): + """Extract audio from video file""" + + # Create output directory + os.makedirs(audio_output_dir, exist_ok=True) + + # Generate audio filename + video_name = Path(video_path).stem + audio_filename = f"{video_name}.wav" + audio_path = os.path.join(audio_output_dir, audio_filename) + + # FFmpeg command to extract audio + command = f'ffmpeg -y -i "{video_path}" -acodec pcm_s16le -ar 16000 -ac 1 "{audio_path}"' + + if run_command(command, f"Extracting audio from {video_name}"): + print(f"โœ… Audio extracted: {audio_path}") + return audio_path + else: + print(f"โŒ Failed to extract audio from {video_name}") + return None + +def run_syncnet_on_chunk(chunk_path, reference_name=None): + """Run SyncNet pipeline on a video chunk""" + + if reference_name is None: + reference_name = Path(chunk_path).stem + + print(f"\n๐ŸŽฏ Running SyncNet on: {chunk_path}") + print(f"๐Ÿ“ Reference name: {reference_name}") + + # Step 1: Run the full pipeline + pipeline_command = f'python run_pipeline.py --videofile "{chunk_path}" --reference "{reference_name}"' + if not run_command(pipeline_command, "SyncNet Pipeline (face detection, tracking, cropping)"): + return False + + # Step 2: Run SyncNet analysis + syncnet_command = f'python run_syncnet.py --videofile "{chunk_path}" --reference "{reference_name}"' + if not run_command(syncnet_command, "SyncNet Analysis (audio-visual sync)"): + return False + + # Step 3: Generate visualization + visualize_command = f'python run_visualise.py --videofile "{chunk_path}" --reference "{reference_name}"' + if not run_command(visualize_command, "SyncNet Visualization"): + return False + + # Check for outputs + work_dir = f"data/work/pywork/{reference_name}" + avi_dir = f"data/work/pyavi/{reference_name}" + + offsets_file = os.path.join(work_dir, "offsets.txt") + output_video = os.path.join(avi_dir, "video_out.avi") + + results = { + 'reference': reference_name, + 'offsets_file': offsets_file if os.path.exists(offsets_file) else None, + 'output_video': output_video if os.path.exists(output_video) else None, + 'success': True + } + + if results['offsets_file']: + print(f"โœ… Offsets file: {offsets_file}") + # Read and display offsets + try: + with open(offsets_file, 'r') as f: + content = f.read().strip() + print(f"๐Ÿ“Š Sync Results: {content}") + except Exception as e: + print(f"โš ๏ธ Could not read offsets file: {e}") + + if results['output_video']: + print(f"โœ… Output video: {output_video}") + + return results + +def main(): + parser = argparse.ArgumentParser(description='Process video with SyncNet pipeline') + parser.add_argument('--video', type=str, required=True, help='Path to input video file') + parser.add_argument('--chunk_duration', type=int, default=30, help='Chunk duration in seconds (default: 30)') + parser.add_argument('--max_chunks', type=int, default=5, help='Maximum number of chunks to process (default: 5)') + parser.add_argument('--skip_chunking', action='store_true', help='Skip chunking and process entire video') + parser.add_argument('--output_dir', type=str, default='processed_output', help='Output directory for results') + + args = parser.parse_args() + + # Validate input video + if not os.path.exists(args.video): + print(f"โŒ Video file not found: {args.video}") + return 1 + + print(f"๐ŸŽฌ Processing video: {args.video}") + print(f"๐Ÿ“ Output directory: {args.output_dir}") + + # Create output directory + os.makedirs(args.output_dir, exist_ok=True) + + if args.skip_chunking: + # Process entire video + print("\n๐ŸŽฏ Processing entire video (no chunking)") + reference_name = Path(args.video).stem + results = run_syncnet_on_chunk(args.video, reference_name) + + if results and results['success']: + print(f"\nโœ… Video processing completed successfully!") + print(f"๐Ÿ“Š Results for {reference_name}:") + if results['offsets_file']: + print(f" - Sync analysis: {results['offsets_file']}") + if results['output_video']: + print(f" - Output video: {results['output_video']}") + else: + print(f"\nโŒ Video processing failed!") + return 1 + + else: + # Create chunks and process them + chunk_dir = os.path.join(args.output_dir, "chunks") + audio_dir = os.path.join(args.output_dir, "audio") + + print(f"\nโœ‚๏ธ Creating video chunks (duration: {args.chunk_duration}s)") + chunks = create_chunks(args.video, args.chunk_duration, chunk_dir) + + if not chunks: + print("โŒ No chunks were created") + return 1 + + print(f"\n๐Ÿ“Š Created {len(chunks)} chunks") + + # Limit number of chunks to process + chunks_to_process = chunks[:args.max_chunks] + if len(chunks) > args.max_chunks: + print(f"โš ๏ธ Processing only first {args.max_chunks} chunks (limit)") + + # Process each chunk + results = [] + for i, chunk_path in enumerate(chunks_to_process): + print(f"\n{'='*60}") + print(f"๐ŸŽฌ Processing chunk {i+1}/{len(chunks_to_process)}: {os.path.basename(chunk_path)}") + print(f"{'='*60}") + + # Extract audio from chunk + audio_path = extract_audio(chunk_path, audio_dir) + if not audio_path: + print(f"โš ๏ธ Skipping chunk {i+1} due to audio extraction failure") + continue + + # Run SyncNet on chunk + reference_name = Path(chunk_path).stem + chunk_results = run_syncnet_on_chunk(chunk_path, reference_name) + + if chunk_results and chunk_results['success']: + chunk_results['chunk_path'] = chunk_path + chunk_results['audio_path'] = audio_path + results.append(chunk_results) + print(f"โœ… Chunk {i+1} processed successfully!") + else: + print(f"โŒ Chunk {i+1} processing failed!") + + # Summary + print(f"\n{'='*60}") + print(f"๐Ÿ“‹ PROCESSING SUMMARY") + print(f"{'='*60}") + print(f"Total chunks created: {len(chunks)}") + print(f"Chunks processed: {len(chunks_to_process)}") + print(f"Successful results: {len(results)}") + + for i, result in enumerate(results): + print(f"\n๐Ÿ“Š Chunk {i+1}: {result['reference']}") + if result['offsets_file']: + try: + with open(result['offsets_file'], 'r') as f: + content = f.read().strip() + print(f" - Sync: {content}") + except: + print(f" - Sync: {result['offsets_file']}") + if result['output_video']: + print(f" - Video: {result['output_video']}") + + print(f"\n๐ŸŽ‰ Processing completed!") + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/quality_presets.py b/utils/quality_presets.py new file mode 100644 index 0000000..dae53f0 --- /dev/null +++ b/utils/quality_presets.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +Quality Filter Presets for SyncNet Processing +Provides different quality filtering presets for different use cases +""" + +import argparse +import subprocess +import sys + +# Define quality filter presets +FILTER_PRESETS = { + 'strict': { + 'min_confidence': 6.0, + 'max_abs_offset': 2, + 'description': 'Strict filtering - only highest quality chunks (publication ready)' + }, + 'high': { + 'min_confidence': 4.0, + 'max_abs_offset': 5, + 'description': 'High quality filtering - good for training data' + }, + 'medium': { + 'min_confidence': 2.5, + 'max_abs_offset': 8, + 'description': 'Medium quality filtering - balanced approach' + }, + 'relaxed': { + 'min_confidence': 1.5, + 'max_abs_offset': 12, + 'description': 'Relaxed filtering - keep most usable chunks' + }, + 'none': { + 'min_confidence': 0.0, + 'max_abs_offset': 50, + 'description': 'No filtering - keep all chunks with faces' + } +} + +def run_with_preset(input_video, output_base_dir, preset_name, **kwargs): + """Run processing with a specific quality preset""" + + if preset_name not in FILTER_PRESETS: + print(f"โŒ Unknown preset: {preset_name}") + print(f"Available presets: {', '.join(FILTER_PRESETS.keys())}") + return False + + preset = FILTER_PRESETS[preset_name] + + print(f"๐ŸŽฏ Using '{preset_name}' preset") + print(f"๐Ÿ“ {preset['description']}") + print(f"๐Ÿ”ง Filters: confidenceโ‰ฅ{preset['min_confidence']}, |offset|โ‰ค{preset['max_abs_offset']}") + + # Build command + cmd_parts = [ + 'python', 'process_long_video.py', + '--input_video', f'"{input_video}"', + '--output_dir', f'"{output_base_dir}_{preset_name}"', + '--min_confidence', str(preset['min_confidence']), + '--max_abs_offset', str(preset['max_abs_offset']) + ] + + # Add additional arguments + for key, value in kwargs.items(): + if key in ['chunk_duration', 'overlap', 'min_track', 'min_face_size']: + cmd_parts.extend([f'--{key}', str(value)]) + + # Disable filtering for 'none' preset + if preset_name == 'none': + cmd_parts.append('--no_filter') + + command = ' '.join(cmd_parts) + + print(f"\n๐Ÿš€ Running: {command}") + result = subprocess.run(command, shell=True) + + return result.returncode == 0 + +def compare_presets(input_video, base_output_dir, presets_to_test): + """Run multiple presets and compare results""" + + print(f"๐Ÿ” TESTING MULTIPLE QUALITY PRESETS") + print("="*60) + + results = {} + + for preset_name in presets_to_test: + print(f"\n{'='*40}") + print(f"Testing preset: {preset_name}") + print(f"{'='*40}") + + success = run_with_preset(input_video, base_output_dir, preset_name) + + if success: + # Try to load results + import json + import os + + summary_file = f"{base_output_dir}_{preset_name}/processing_summary.json" + + if os.path.exists(summary_file): + with open(summary_file, 'r') as f: + summary = json.load(f) + + results[preset_name] = { + 'total_chunks': summary['total_chunks'], + 'successful_analysis': summary['successful_analysis'], + 'accepted_chunks': len(summary.get('accepted_chunks', [])), + 'acceptance_rate': summary.get('acceptance_rate', 0), + 'preset_config': FILTER_PRESETS[preset_name] + } + + print(f"โœ… {preset_name}: {results[preset_name]['accepted_chunks']} chunks accepted ({results[preset_name]['acceptance_rate']:.1f}%)") + else: + print(f"โš ๏ธ {preset_name}: Results file not found") + else: + print(f"โŒ {preset_name}: Processing failed") + + # Print comparison summary + if results: + print(f"\n๐Ÿ“Š PRESET COMPARISON SUMMARY") + print("="*60) + print(f"{'Preset':<10} {'Confidence':<11} {'Max Offset':<10} {'Accepted':<8} {'Rate':<6}") + print("-" * 50) + + for preset_name, data in results.items(): + config = data['preset_config'] + print(f"{preset_name:<10} โ‰ฅ{config['min_confidence']:<10} โ‰ค{config['max_abs_offset']:<9} {data['accepted_chunks']:<8} {data['acceptance_rate']:<5.1f}%") + + # Recommendations + print(f"\n๐Ÿ’ก RECOMMENDATIONS") + print("-"*40) + + best_balance = None + best_score = 0 + + for preset_name, data in results.items(): + # Score based on acceptance rate and absolute number + score = data['acceptance_rate'] * 0.7 + data['accepted_chunks'] * 0.3 + if score > best_score: + best_score = score + best_balance = preset_name + + if best_balance: + print(f"๐Ÿ† Best balanced preset: '{best_balance}'") + print(f" - {results[best_balance]['accepted_chunks']} high-quality chunks") + print(f" - {results[best_balance]['acceptance_rate']:.1f}% acceptance rate") + + # Usage recommendations + highest_acceptance = max(results.values(), key=lambda x: x['acceptance_rate']) + most_chunks = max(results.values(), key=lambda x: x['accepted_chunks']) + + print(f"\n๐ŸŽฏ USAGE RECOMMENDATIONS:") + print(f" - For maximum data: Use preset with {most_chunks['accepted_chunks']} chunks") + print(f" - For best quality: Use preset with {highest_acceptance['acceptance_rate']:.1f}% acceptance") + print(f" - For training: Consider 'high' or 'medium' presets") + print(f" - For research: Consider 'strict' preset") + +def main(): + parser = argparse.ArgumentParser(description='Process video with quality filter presets') + parser.add_argument('--input_video', type=str, help='Path to input video file') + parser.add_argument('--output_dir', type=str, help='Base output directory') + parser.add_argument('--preset', type=str, choices=list(FILTER_PRESETS.keys()), + help='Quality filter preset to use') + parser.add_argument('--compare_presets', nargs='+', choices=list(FILTER_PRESETS.keys()), + help='Compare multiple presets') + parser.add_argument('--list_presets', action='store_true', help='List available presets') + parser.add_argument('--chunk_duration', type=int, default=30, help='Chunk duration in seconds') + parser.add_argument('--overlap', type=int, default=5, help='Overlap between chunks in seconds') + + args = parser.parse_args() + + if args.list_presets: + print("๐ŸŽฏ AVAILABLE QUALITY PRESETS") + print("="*50) + for name, config in FILTER_PRESETS.items(): + print(f"\n'{name}':") + print(f" {config['description']}") + print(f" Min confidence: {config['min_confidence']}") + print(f" Max |offset|: {config['max_abs_offset']} frames") + return 0 + + if not args.input_video: + print("โŒ --input_video is required (unless using --list_presets)") + return 1 + + if not args.output_dir: + print("โŒ --output_dir is required (unless using --list_presets)") + return 1 + + if args.compare_presets: + compare_presets(args.input_video, args.output_dir, args.compare_presets) + elif args.preset: + run_with_preset( + args.input_video, + args.output_dir, + args.preset, + chunk_duration=args.chunk_duration, + overlap=args.overlap + ) + else: + print("โŒ Either --preset or --compare_presets must be specified") + print("Use --list_presets to see available options") + return 1 + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/video_utils.py b/utils/video_utils.py new file mode 100755 index 0000000..a8796d4 --- /dev/null +++ b/utils/video_utils.py @@ -0,0 +1,276 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import argparse +import subprocess +from pathlib import Path + +def get_video_info(video_path): + """Get video information using ffprobe""" + command = f'ffprobe -v quiet -print_format json -show_format -show_streams "{video_path}"' + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.returncode == 0: + import json + info = json.loads(result.stdout) + + # Find video stream + video_stream = None + audio_stream = None + + for stream in info['streams']: + if stream['codec_type'] == 'video' and video_stream is None: + video_stream = stream + elif stream['codec_type'] == 'audio' and audio_stream is None: + audio_stream = stream + + duration = float(info['format']['duration']) if 'duration' in info['format'] else None + + return { + 'duration': duration, + 'video_stream': video_stream, + 'audio_stream': audio_stream, + 'format': info['format'] + } + + return None + +def cut_video_by_time(input_video, output_dir, start_time, duration, chunk_name=None): + """ + Cut a specific segment from video + + Args: + input_video: Path to input video + output_dir: Output directory + start_time: Start time in seconds + duration: Duration in seconds + chunk_name: Optional custom name for chunk + """ + os.makedirs(output_dir, exist_ok=True) + + if chunk_name is None: + chunk_name = f"chunk_{start_time:06.1f}s_{duration:06.1f}s.mp4" + + output_path = os.path.join(output_dir, chunk_name) + + # Use ffmpeg to extract segment + command = f'ffmpeg -y -i "{input_video}" -ss {start_time} -t {duration} -c copy "{output_path}"' + + print(f"Extracting: {start_time}s - {start_time + duration}s -> {chunk_name}") + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.returncode == 0: + print(f"โœ… Created: {output_path}") + return output_path + else: + print(f"โŒ Error: {result.stderr}") + return None + +def cut_video_by_silence(input_video, output_dir, silence_threshold=-30, min_silence_duration=1.0, min_chunk_duration=5.0): + """ + Cut video based on silence detection + + Args: + input_video: Path to input video + output_dir: Output directory + silence_threshold: Silence threshold in dB (default: -30dB) + min_silence_duration: Minimum silence duration to split (default: 1.0s) + min_chunk_duration: Minimum chunk duration (default: 5.0s) + """ + print(f"๐Ÿ” Detecting silence in: {input_video}") + + # Step 1: Detect silence using ffmpeg + silence_detect_cmd = f'ffmpeg -i "{input_video}" -af "silencedetect=noise={silence_threshold}dB:d={min_silence_duration}" -f null - 2>&1' + + result = subprocess.run(silence_detect_cmd, shell=True, capture_output=True, text=True) + + # Parse silence detection output + silence_starts = [] + silence_ends = [] + + for line in result.stderr.split('\n'): + if 'silence_start:' in line: + time_str = line.split('silence_start: ')[1].split(' ')[0] + silence_starts.append(float(time_str)) + elif 'silence_end:' in line: + time_str = line.split('silence_end: ')[1].split(' ')[0] + silence_ends.append(float(time_str)) + + print(f"Found {len(silence_starts)} silence regions") + + # Get video duration + video_info = get_video_info(input_video) + total_duration = video_info['duration'] if video_info else None + + if total_duration is None: + print("โŒ Could not get video duration") + return [] + + # Calculate chunk boundaries + chunk_boundaries = [0.0] # Start with beginning + + # Add silence boundaries + for start, end in zip(silence_starts, silence_ends): + # Use middle of silence as split point + split_point = (start + end) / 2 + chunk_boundaries.append(split_point) + + chunk_boundaries.append(total_duration) # End with video end + chunk_boundaries = sorted(set(chunk_boundaries)) # Remove duplicates and sort + + # Create chunks + chunks = [] + os.makedirs(output_dir, exist_ok=True) + + for i in range(len(chunk_boundaries) - 1): + start_time = chunk_boundaries[i] + end_time = chunk_boundaries[i + 1] + duration = end_time - start_time + + # Skip very short chunks + if duration < min_chunk_duration: + print(f"โญ๏ธ Skipping short chunk: {duration:.1f}s") + continue + + chunk_name = f"silence_chunk_{i:03d}_{start_time:06.1f}s.mp4" + chunk_path = cut_video_by_time(input_video, output_dir, start_time, duration, chunk_name) + + if chunk_path: + chunks.append(chunk_path) + + return chunks + +def extract_audio_from_video(video_path, audio_output_path=None, sample_rate=16000, channels=1): + """ + Extract audio from video file + + Args: + video_path: Path to input video + audio_output_path: Output audio path (auto-generated if None) + sample_rate: Audio sample rate (default: 16000 for speech) + channels: Number of audio channels (default: 1 for mono) + """ + if audio_output_path is None: + video_name = Path(video_path).stem + audio_output_path = f"{video_name}.wav" + + # Create output directory + os.makedirs(os.path.dirname(audio_output_path), exist_ok=True) + + # Extract audio using ffmpeg + command = f'ffmpeg -y -i "{video_path}" -ac {channels} -ar {sample_rate} -vn "{audio_output_path}"' + + print(f"๐ŸŽต Extracting audio: {video_path} -> {audio_output_path}") + result = subprocess.run(command, shell=True, capture_output=True, text=True) + + if result.returncode == 0: + print(f"โœ… Audio extracted: {audio_output_path}") + return audio_output_path + else: + print(f"โŒ Audio extraction failed: {result.stderr}") + return None + +def main(): + parser = argparse.ArgumentParser(description="Video chunking and audio extraction utilities") + subparsers = parser.add_subparsers(dest='command', help='Available commands') + + # Command: chunk by time + chunk_parser = subparsers.add_parser('chunk-time', help='Cut video into fixed-duration chunks') + chunk_parser.add_argument('--input', required=True, help='Input video file') + chunk_parser.add_argument('--output_dir', required=True, help='Output directory for chunks') + chunk_parser.add_argument('--duration', type=float, default=30.0, help='Chunk duration in seconds (default: 30)') + chunk_parser.add_argument('--overlap', type=float, default=0.0, help='Overlap between chunks in seconds (default: 0)') + + # Command: chunk by silence + silence_parser = subparsers.add_parser('chunk-silence', help='Cut video based on silence detection') + silence_parser.add_argument('--input', required=True, help='Input video file') + silence_parser.add_argument('--output_dir', required=True, help='Output directory for chunks') + silence_parser.add_argument('--threshold', type=float, default=-30, help='Silence threshold in dB (default: -30)') + silence_parser.add_argument('--min_silence', type=float, default=1.0, help='Minimum silence duration (default: 1.0s)') + silence_parser.add_argument('--min_chunk', type=float, default=5.0, help='Minimum chunk duration (default: 5.0s)') + + # Command: extract audio + audio_parser = subparsers.add_parser('extract-audio', help='Extract audio from video') + audio_parser.add_argument('--input', required=True, help='Input video file') + audio_parser.add_argument('--output', help='Output audio file (auto-generated if not specified)') + audio_parser.add_argument('--sample_rate', type=int, default=16000, help='Sample rate (default: 16000)') + audio_parser.add_argument('--channels', type=int, default=1, help='Number of channels (default: 1)') + + # Command: info + info_parser = subparsers.add_parser('info', help='Get video information') + info_parser.add_argument('--input', required=True, help='Input video file') + + args = parser.parse_args() + + if args.command == 'chunk-time': + # Cut video into fixed-duration chunks + video_info = get_video_info(args.input) + if not video_info: + print("โŒ Could not get video information") + return + + total_duration = video_info['duration'] + chunk_duration = args.duration + overlap = args.overlap + + print(f"๐Ÿ“น Video duration: {total_duration:.1f}s") + print(f"๐Ÿ”ช Chunk duration: {chunk_duration}s with {overlap}s overlap") + + chunks = [] + start_time = 0.0 + chunk_num = 0 + + while start_time < total_duration: + remaining = total_duration - start_time + actual_duration = min(chunk_duration, remaining) + + if actual_duration < 5.0: # Skip very short chunks + break + + chunk_name = f"chunk_{chunk_num:03d}.mp4" + chunk_path = cut_video_by_time(args.input, args.output_dir, start_time, actual_duration, chunk_name) + + if chunk_path: + chunks.append(chunk_path) + + chunk_num += 1 + start_time += chunk_duration - overlap + + print(f"โœ… Created {len(chunks)} chunks") + + elif args.command == 'chunk-silence': + # Cut video based on silence + chunks = cut_video_by_silence( + args.input, + args.output_dir, + args.threshold, + args.min_silence, + args.min_chunk + ) + print(f"โœ… Created {len(chunks)} chunks based on silence") + + elif args.command == 'extract-audio': + # Extract audio + extract_audio_from_video(args.input, args.output, args.sample_rate, args.channels) + + elif args.command == 'info': + # Show video info + info = get_video_info(args.input) + if info: + print(f"๐Ÿ“น Video Information for: {args.input}") + print(f"Duration: {info['duration']:.2f} seconds") + if info['video_stream']: + vs = info['video_stream'] + print(f"Video: {vs.get('codec_name', 'unknown')} {vs.get('width', '?')}x{vs.get('height', '?')} @ {vs.get('r_frame_rate', '?')} fps") + if info['audio_stream']: + aus = info['audio_stream'] + print(f"Audio: {aus.get('codec_name', 'unknown')} {aus.get('sample_rate', '?')} Hz {aus.get('channels', '?')} channels") + else: + print("โŒ Could not get video information") + + else: + parser.print_help() + +if __name__ == "__main__": + main() From 26c3ad24218a016a256d345ee3b9686fb34930a2 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sun, 3 Aug 2025 11:11:33 +0600 Subject: [PATCH 04/11] chore: preset filtering --- README.md | 42 ++++ VIDEO_FILTERING_GUIDE.md | 160 ++++++++++++++ examples_video_filtering.sh | 52 +++++ filter_videos_by_sync_score.py | 381 +++++++++++++++++++++++++++++++++ 4 files changed, 635 insertions(+) create mode 100644 VIDEO_FILTERING_GUIDE.md create mode 100755 examples_video_filtering.sh create mode 100644 filter_videos_by_sync_score.py diff --git a/README.md b/README.md index 93ad607..2845079 100755 --- a/README.md +++ b/README.md @@ -74,6 +74,48 @@ python run_pipeline.py --videofile data/chunk_003.mp4 --reference chunk_003 --da python run_pipeline.py --videofile data/chunk_003.mp4 --reference chunk_003 --data_dir data/test/ --min_face_size 50 ``` +## Batch Processing and Quality Filtering + +### Filter Videos by SyncNet Quality Scores + +Process multiple videos and automatically filter them based on audio-visual synchronization quality: + +```bash +# Basic filtering with default thresholds +python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/filtered_results + +# Using quality presets +python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --preset high + +# Custom quality thresholds +python filter_videos_by_sync_score.py \ + --input_dir /path/to/videos \ + --output_dir /path/to/results \ + --min_confidence 6.0 \ + --max_abs_offset 3 \ + --min_face_size 40 \ + --max_workers 4 +``` + +**Quality Presets:** +- `--preset strict`: confidenceโ‰ฅ8.0, |offset|โ‰ค2 (publication ready) +- `--preset high`: confidenceโ‰ฅ6.0, |offset|โ‰ค3 (training data quality) +- `--preset medium`: confidenceโ‰ฅ4.0, |offset|โ‰ค5 (balanced filtering) +- `--preset relaxed`: confidenceโ‰ฅ2.0, |offset|โ‰ค8 (keep most usable) + +**Output Structure:** +``` +output_dir/ +โ”œโ”€โ”€ good_quality/ # Videos that pass quality thresholds +โ”œโ”€โ”€ poor_quality/ # Videos filtered out for low quality +โ””โ”€โ”€ sync_filter_results.json # Detailed analysis results +``` + +**Parameters:** +- `--min_confidence`: Minimum SyncNet confidence score to keep video +- `--max_abs_offset`: Maximum absolute frame offset to keep video +- `--keep_all`: Analyze quality but don't copy files to separate folders + Outputs: ``` $DATA_DIR/pycrop/$REFERENCE/*.avi - cropped face tracks diff --git a/VIDEO_FILTERING_GUIDE.md b/VIDEO_FILTERING_GUIDE.md new file mode 100644 index 0000000..8f404cd --- /dev/null +++ b/VIDEO_FILTERING_GUIDE.md @@ -0,0 +1,160 @@ +# Video Filtering by SyncNet Quality - Complete Guide + +## Overview +The `filter_videos_by_sync_score.py` script processes multiple videos to assess their audio-visual synchronization quality using SyncNet, then filters them based on configurable quality thresholds. + +## Quick Start + +### Basic Usage +```bash +# Filter videos with default settings (confidenceโ‰ฅ5.0, |offset|โ‰ค3) +python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results +``` + +### Using Quality Presets +```bash +# High quality filtering (training data grade) +python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --preset high + +# Strict filtering (publication ready) +python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --preset strict +``` + +## Quality Presets + +| Preset | Min Confidence | Max Offset | Use Case | +|----------|-----------------|------------|----------| +| `strict` | 8.0 | 2 frames | Publication-ready, highest quality | +| `high` | 6.0 | 3 frames | Training data, research quality | +| `medium` | 4.0 | 5 frames | General purpose, balanced filtering | +| `relaxed`| 2.0 | 8 frames | Keep most usable content | + +## Custom Parameters + +```bash +python filter_videos_by_sync_score.py \ + --input_dir /path/to/videos \ + --output_dir /path/to/results \ + --min_confidence 6.5 \ + --max_abs_offset 2 \ + --min_face_size 40 \ + --min_track 30 \ + --max_workers 4 +``` + +### Parameter Descriptions + +- `--min_confidence`: Minimum SyncNet confidence score (higher = better sync quality) +- `--max_abs_offset`: Maximum absolute frame offset allowed (lower = better sync) +- `--min_face_size`: Minimum face size in pixels for detection (adjust for video resolution) +- `--min_track`: Minimum number of frames a face must be tracked +- `--max_workers`: Number of parallel processing threads + +## Output Structure + +``` +output_dir/ +โ”œโ”€โ”€ good_quality/ # Videos passing quality thresholds +โ”‚ โ”œโ”€โ”€ video1.mp4 +โ”‚ โ”œโ”€โ”€ video2.mp4 +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ poor_quality/ # Videos filtered out +โ”‚ โ”œโ”€โ”€ low_sync_video1.mp4 +โ”‚ โ”œโ”€โ”€ low_sync_video2.mp4 +โ”‚ โ””โ”€โ”€ ... +โ””โ”€โ”€ sync_filter_results.json # Detailed analysis results +``` + +## Understanding Results + +### sync_filter_results.json Contains: +- **Filter settings**: Thresholds used +- **Statistics**: Count of good/poor/failed videos +- **Individual results**: Per-video metrics including: + - Confidence score + - Frame offset + - Processing time + - Quality assessment + +### Example Results Analysis: +```json +{ + "filter_settings": { + "min_confidence": 6.0, + "max_abs_offset": 3 + }, + "total_videos": 100, + "good_quality": 75, + "poor_quality": 20, + "no_faces_detected": 3, + "processing_failed": 2 +} +``` + +## Troubleshooting + +### Common Issues: + +1. **"No faces detected"** + - Reduce `--min_face_size` (try 30-40 for low resolution videos) + - Check that videos actually contain visible faces + +2. **High processing time** + - Reduce `--max_workers` if system becomes unresponsive + - Consider processing smaller batches + +3. **Most videos filtered out** + - Use `--preset relaxed` or lower thresholds + - Check sample videos manually to understand quality distribution + +### Analysis-Only Mode: +```bash +# Analyze quality without copying files to separate folders +python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --keep_all +``` + +## Best Practices + +### 1. Start with Analysis +Run with `--keep_all` first to understand your data's quality distribution. + +### 2. Choose Appropriate Presets +- **Research/Training**: Use `--preset high` +- **Production/Publication**: Use `--preset strict` +- **Content Curation**: Use `--preset medium` + +### 3. Adjust for Video Characteristics +- **Low resolution videos**: Reduce `--min_face_size` +- **Short clips**: Reduce `--min_track` +- **Noisy environments**: Consider `--preset relaxed` + +### 4. Monitor Processing +- Start with small batches to verify settings +- Monitor CPU/memory usage with multiple workers +- Check sample results before processing large datasets + +## Example Workflows + +### Workflow 1: Research Dataset Curation +```bash +# Step 1: Analyze all videos +python filter_videos_by_sync_score.py --input_dir raw_videos --output_dir analysis --keep_all --preset medium + +# Step 2: Review results and adjust thresholds +# Check analysis/sync_filter_results.json + +# Step 3: Apply final filtering +python filter_videos_by_sync_score.py --input_dir raw_videos --output_dir final_dataset --preset high +``` + +### Workflow 2: Quick Quality Check +```bash +# Fast assessment with relaxed settings for small faces +python filter_videos_by_sync_score.py \ + --input_dir videos \ + --output_dir filtered \ + --min_confidence 3.0 \ + --max_abs_offset 8 \ + --min_face_size 30 \ + --max_workers 8 +``` diff --git a/examples_video_filtering.sh b/examples_video_filtering.sh new file mode 100755 index 0000000..e17aded --- /dev/null +++ b/examples_video_filtering.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Example usage script for video filtering by SyncNet scores + +echo "๐ŸŽฌ SyncNet Video Quality Filter - Usage Examples" +echo "================================================" + +# Basic usage - filter videos with default thresholds +echo "๐Ÿ“ Basic filtering (confidenceโ‰ฅ5.0, |offset|โ‰ค3):" +echo "python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/filtered_results" +echo "" + +# Using quality presets +echo "๐Ÿ“ Using quality presets:" +echo "# Strict filtering (high quality only)" +echo "python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --preset strict" +echo "" +echo "# Medium quality filtering" +echo "python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --preset medium" +echo "" + +# Custom thresholds +echo "๐Ÿ“ Custom quality thresholds:" +echo "python filter_videos_by_sync_score.py \\" +echo " --input_dir /path/to/videos \\" +echo " --output_dir /path/to/results \\" +echo " --min_confidence 4.5 \\" +echo " --max_abs_offset 5 \\" +echo " --min_face_size 40 \\" +echo " --max_workers 4" +echo "" + +# Analysis only (don't copy files) +echo "๐Ÿ“ Analysis only (don't copy files to separate folders):" +echo "python filter_videos_by_sync_score.py --input_dir /path/to/videos --output_dir /path/to/results --keep_all" +echo "" + +echo "๐Ÿ“Š Quality Presets Available:" +echo " --preset strict : confidenceโ‰ฅ8.0, |offset|โ‰ค2 (publication ready)" +echo " --preset high : confidenceโ‰ฅ6.0, |offset|โ‰ค3 (training data)" +echo " --preset medium : confidenceโ‰ฅ4.0, |offset|โ‰ค5 (balanced)" +echo " --preset relaxed : confidenceโ‰ฅ2.0, |offset|โ‰ค8 (keep most usable)" +echo "" + +echo "๐Ÿ“ Output Structure:" +echo " output_dir/" +echo " โ”œโ”€โ”€ good_quality/ # Videos that pass quality thresholds" +echo " โ”œโ”€โ”€ poor_quality/ # Videos that fail quality thresholds" +echo " โ””โ”€โ”€ sync_filter_results.json # Detailed analysis results" +echo "" + +echo "๐Ÿ”ง For videos with small faces, reduce --min_face_size (default: 50)" +echo "โšก Increase --max_workers for faster processing (but higher CPU usage)" diff --git a/filter_videos_by_sync_score.py b/filter_videos_by_sync_score.py new file mode 100644 index 0000000..9047e4e --- /dev/null +++ b/filter_videos_by_sync_score.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +""" +Filter Videos by SyncNet Score +Batch process videos and filter based on audio-visual synchronization quality +""" + +import os +import sys +import json +import argparse +import subprocess +import shutil +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed +import time + +class SyncNetFilter: + def __init__(self, min_confidence=5.0, max_abs_offset=3, min_face_size=50, min_track=50): + """ + Initialize SyncNet filter with quality thresholds + + Args: + min_confidence: Minimum confidence score to keep video + max_abs_offset: Maximum absolute offset (frames) to keep video + min_face_size: Minimum face size for detection + min_track: Minimum track length for processing + """ + self.min_confidence = min_confidence + self.max_abs_offset = max_abs_offset + self.min_face_size = min_face_size + self.min_track = min_track + + def process_single_video(self, video_path, output_dir, temp_dir): + """ + Process a single video with SyncNet and return quality metrics + + Args: + video_path: Path to input video file + output_dir: Directory for final results + temp_dir: Temporary directory for processing + + Returns: + Dict with processing results and quality metrics + """ + video_name = Path(video_path).stem + print(f"๐Ÿ”„ Processing: {video_name}") + + result = { + 'video_name': video_name, + 'video_path': str(video_path), + 'status': 'failed', + 'start_time': time.time() + } + + try: + # Create temporary processing directory + temp_video_dir = os.path.join(temp_dir, video_name) + os.makedirs(temp_video_dir, exist_ok=True) + + # Step 1: Run pipeline + python_exec = sys.executable + pipeline_cmd = [ + python_exec, 'run_pipeline.py', + '--videofile', str(video_path), + '--reference', video_name, + '--data_dir', temp_video_dir, + '--min_face_size', str(self.min_face_size), + '--min_track', str(self.min_track) + ] + + pipeline_result = subprocess.run( + pipeline_cmd, + capture_output=True, + text=True, + cwd=os.getcwd() + ) + + if pipeline_result.returncode != 0: + result['error'] = f"Pipeline failed: {pipeline_result.stderr}" + return result + + # Check if tracks were created + tracks_file = os.path.join(temp_video_dir, 'pywork', video_name, 'tracks.pckl') + if not os.path.exists(tracks_file): + result['status'] = 'no_faces' + result['message'] = 'No face tracks detected' + return result + + # Step 2: Run SyncNet + syncnet_cmd = [ + python_exec, 'run_syncnet.py', + '--videofile', str(video_path), + '--reference', video_name, + '--data_dir', temp_video_dir + ] + + syncnet_result = subprocess.run( + syncnet_cmd, + capture_output=True, + text=True, + cwd=os.getcwd() + ) + + if syncnet_result.returncode != 0: + result['error'] = f"SyncNet failed: {syncnet_result.stderr}" + return result + + # Step 3: Parse results + offsets_file = os.path.join(temp_video_dir, 'pywork', video_name, 'offsets.txt') + if os.path.exists(offsets_file): + with open(offsets_file, 'r') as f: + content = f.read().strip() + + # Parse the offsets file + lines = content.split('\n') + tracks = [] + + for line in lines: + if 'TRACK' in line and 'OFFSET' in line and 'CONF' in line: + try: + # Parse: "TRACK 0: OFFSET 0, CONF 7.996" + parts = line.split(': OFFSET ') + if len(parts) >= 2: + offset_conf_part = parts[1].split(', CONF ') + if len(offset_conf_part) >= 2: + offset = int(offset_conf_part[0]) + confidence = float(offset_conf_part[1]) + tracks.append({ + 'offset': offset, + 'confidence': confidence + }) + except (ValueError, IndexError) as e: + print(f"Warning: Could not parse line '{line}': {e}") + continue + + # Use the track with highest confidence + if tracks: + best_track = max(tracks, key=lambda x: x['confidence']) + result['offset'] = best_track['offset'] + result['confidence'] = best_track['confidence'] + result['all_tracks'] = tracks + + # Check if we got the metrics + if 'confidence' not in result: + result['error'] = 'Could not parse SyncNet results' + return result + + # Determine quality + abs_offset = abs(result['offset']) + passes_confidence = result['confidence'] >= self.min_confidence + passes_offset = abs_offset <= self.max_abs_offset + + result['passes_quality'] = passes_confidence and passes_offset + result['quality_reasons'] = [] + + if not passes_confidence: + result['quality_reasons'].append(f"Low confidence: {result['confidence']:.3f} < {self.min_confidence}") + if not passes_offset: + result['quality_reasons'].append(f"High offset: {abs_offset} > {self.max_abs_offset}") + + result['status'] = 'success' + + except Exception as e: + result['error'] = str(e) + finally: + result['end_time'] = time.time() + result['processing_time'] = result['end_time'] - result['start_time'] + + # Clean up temporary files + try: + if os.path.exists(temp_video_dir): + shutil.rmtree(temp_video_dir) + except: + pass + + return result + + def filter_videos(self, input_dir, output_dir, max_workers=2, keep_all=False): + """ + Batch process and filter videos from input directory + + Args: + input_dir: Directory containing input videos + output_dir: Directory for filtered results + max_workers: Number of parallel processing workers + keep_all: If True, keep all videos but mark quality in results + """ + # Find all video files + video_extensions = ['.mp4', '.avi', '.mov', '.mkv', '.mp3', '.wav'] + video_files = [] + + for ext in video_extensions: + video_files.extend(Path(input_dir).glob(f'*{ext}')) + video_files.extend(Path(input_dir).glob(f'*{ext.upper()}')) + + video_files = sorted(set(video_files)) + + if not video_files: + print(f"โŒ No video files found in {input_dir}") + return + + print(f"๐Ÿ“น Found {len(video_files)} video files") + print(f"๐ŸŽฏ Quality thresholds: confidenceโ‰ฅ{self.min_confidence}, |offset|โ‰ค{self.max_abs_offset}") + + # Create output directories + os.makedirs(output_dir, exist_ok=True) + + if not keep_all: + good_dir = os.path.join(output_dir, 'good_quality') + poor_dir = os.path.join(output_dir, 'poor_quality') + os.makedirs(good_dir, exist_ok=True) + os.makedirs(poor_dir, exist_ok=True) + + temp_dir = os.path.join(output_dir, 'temp') + os.makedirs(temp_dir, exist_ok=True) + + # Process videos + results = [] + completed = 0 + + print(f"๐Ÿš€ Starting batch processing with {max_workers} workers...") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all jobs + future_to_video = { + executor.submit(self.process_single_video, video_path, output_dir, temp_dir): video_path + for video_path in video_files + } + + # Collect results + for future in as_completed(future_to_video): + video_path = future_to_video[future] + + try: + result = future.result() + results.append(result) + completed += 1 + + # Print progress + if result['status'] == 'success': + quality = "โœ… GOOD" if result['passes_quality'] else "โŒ POOR" + conf = result.get('confidence', 0) + offset = result.get('offset', 0) + print(f"{quality} [{completed}/{len(video_files)}] {result['video_name']}: " + f"Conf={conf:.3f}, Offset={offset}") + + # Copy file to appropriate directory + if not keep_all: + src_path = result['video_path'] + dst_dir = good_dir if result['passes_quality'] else poor_dir + dst_path = os.path.join(dst_dir, os.path.basename(src_path)) + shutil.copy2(src_path, dst_path) + + else: + status_emoji = "โš ๏ธ" if result['status'] == 'no_faces' else "โŒ" + message = result.get('message', result.get('error', 'Unknown error')) + print(f"{status_emoji} [{completed}/{len(video_files)}] {result['video_name']}: {message}") + + except Exception as e: + print(f"โŒ [{completed+1}/{len(video_files)}] {os.path.basename(video_path)}: Exception - {e}") + completed += 1 + + # Clean up temp directory + try: + shutil.rmtree(temp_dir) + except: + pass + + # Generate summary + successful = [r for r in results if r['status'] == 'success'] + good_quality = [r for r in successful if r['passes_quality']] + poor_quality = [r for r in successful if not r['passes_quality']] + no_faces = [r for r in results if r['status'] == 'no_faces'] + failed = [r for r in results if r['status'] == 'failed'] + + summary = { + 'filter_settings': { + 'min_confidence': self.min_confidence, + 'max_abs_offset': self.max_abs_offset, + 'min_face_size': self.min_face_size, + 'min_track': self.min_track + }, + 'total_videos': len(video_files), + 'successful_processing': len(successful), + 'good_quality': len(good_quality), + 'poor_quality': len(poor_quality), + 'no_faces_detected': len(no_faces), + 'processing_failed': len(failed), + 'results': results, + 'timestamp': time.time() + } + + # Save detailed results + results_file = os.path.join(output_dir, 'sync_filter_results.json') + with open(results_file, 'w') as f: + json.dump(summary, f, indent=2) + + # Print summary + print(f"\\n{'='*80}") + print("๐Ÿ“Š SYNC QUALITY FILTERING SUMMARY") + print(f"{'='*80}") + print(f"Total videos processed: {summary['total_videos']}") + print(f"โœ… Successfully processed: {summary['successful_processing']}") + print(f"๐ŸŽฏ Good quality (kept): {summary['good_quality']}") + print(f"โŒ Poor quality (filtered): {summary['poor_quality']}") + print(f"โš ๏ธ No faces detected: {summary['no_faces_detected']}") + print(f"๐Ÿ’ฅ Processing failed: {summary['processing_failed']}") + print(f"๐Ÿ“„ Detailed results: {results_file}") + + if good_quality: + print(f"\\n๐Ÿ† TOP QUALITY VIDEOS:") + sorted_good = sorted(good_quality, key=lambda x: x['confidence'], reverse=True) + for result in sorted_good[:10]: # Show top 10 + print(f" {result['video_name']}: Conf={result['confidence']:.3f}, Offset={result['offset']}") + + if poor_quality: + print(f"\\nโš ๏ธ FILTERED OUT (reasons):") + for result in poor_quality[:10]: # Show first 10 + reasons = ', '.join(result['quality_reasons']) + print(f" {result['video_name']}: {reasons}") + + return summary + + +def main(): + parser = argparse.ArgumentParser(description="Filter videos by SyncNet quality scores") + parser.add_argument('--input_dir', required=True, help='Directory containing input videos') + parser.add_argument('--output_dir', required=True, help='Output directory for filtered results') + parser.add_argument('--min_confidence', type=float, default=5.0, + help='Minimum confidence score to keep video (default: 5.0)') + parser.add_argument('--max_abs_offset', type=int, default=3, + help='Maximum absolute offset (frames) to keep video (default: 3)') + parser.add_argument('--min_face_size', type=int, default=50, + help='Minimum face size for detection (default: 50)') + parser.add_argument('--min_track', type=int, default=50, + help='Minimum track length for processing (default: 50)') + parser.add_argument('--max_workers', type=int, default=2, + help='Number of parallel workers (default: 2)') + parser.add_argument('--keep_all', action='store_true', + help='Keep all videos, just analyze quality (don\'t copy to folders)') + parser.add_argument('--preset', choices=['strict', 'high', 'medium', 'relaxed'], + help='Use quality preset instead of manual thresholds') + + args = parser.parse_args() + + # Apply preset if specified + if args.preset: + presets = { + 'strict': {'min_confidence': 8.0, 'max_abs_offset': 2}, + 'high': {'min_confidence': 6.0, 'max_abs_offset': 3}, + 'medium': {'min_confidence': 4.0, 'max_abs_offset': 5}, + 'relaxed': {'min_confidence': 2.0, 'max_abs_offset': 8} + } + preset_config = presets[args.preset] + args.min_confidence = preset_config['min_confidence'] + args.max_abs_offset = preset_config['max_abs_offset'] + print(f"๐ŸŽฏ Using '{args.preset}' preset: confidenceโ‰ฅ{args.min_confidence}, |offset|โ‰ค{args.max_abs_offset}") + + # Validate inputs + if not os.path.exists(args.input_dir): + print(f"โŒ Input directory not found: {args.input_dir}") + sys.exit(1) + + # Create filter and run + filter_tool = SyncNetFilter( + min_confidence=args.min_confidence, + max_abs_offset=args.max_abs_offset, + min_face_size=args.min_face_size, + min_track=args.min_track + ) + + filter_tool.filter_videos( + input_dir=args.input_dir, + output_dir=args.output_dir, + max_workers=args.max_workers, + keep_all=args.keep_all + ) + + +if __name__ == "__main__": + main() From 4892d0464f964ddcf1eae7d688a69c4b09a45b25 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sun, 3 Aug 2025 11:35:16 +0600 Subject: [PATCH 05/11] chore: added docs for video utils --- {utils => Docs}/QUALITY_FILTERING_GUIDE.md | 0 {utils => Docs}/README_PIPELINE.md | 0 .../VIDEO_FILTERING_GUIDE.md | 0 {utils => Docs}/VIDEO_PROCESSING_README.md | 0 Docs/VIDEO_UTILS_GUIDE.md | 430 ++++++++++++++++++ README.md | 20 + 6 files changed, 450 insertions(+) rename {utils => Docs}/QUALITY_FILTERING_GUIDE.md (100%) rename {utils => Docs}/README_PIPELINE.md (100%) rename VIDEO_FILTERING_GUIDE.md => Docs/VIDEO_FILTERING_GUIDE.md (100%) rename {utils => Docs}/VIDEO_PROCESSING_README.md (100%) create mode 100644 Docs/VIDEO_UTILS_GUIDE.md diff --git a/utils/QUALITY_FILTERING_GUIDE.md b/Docs/QUALITY_FILTERING_GUIDE.md similarity index 100% rename from utils/QUALITY_FILTERING_GUIDE.md rename to Docs/QUALITY_FILTERING_GUIDE.md diff --git a/utils/README_PIPELINE.md b/Docs/README_PIPELINE.md similarity index 100% rename from utils/README_PIPELINE.md rename to Docs/README_PIPELINE.md diff --git a/VIDEO_FILTERING_GUIDE.md b/Docs/VIDEO_FILTERING_GUIDE.md similarity index 100% rename from VIDEO_FILTERING_GUIDE.md rename to Docs/VIDEO_FILTERING_GUIDE.md diff --git a/utils/VIDEO_PROCESSING_README.md b/Docs/VIDEO_PROCESSING_README.md similarity index 100% rename from utils/VIDEO_PROCESSING_README.md rename to Docs/VIDEO_PROCESSING_README.md diff --git a/Docs/VIDEO_UTILS_GUIDE.md b/Docs/VIDEO_UTILS_GUIDE.md new file mode 100644 index 0000000..d6fa12c --- /dev/null +++ b/Docs/VIDEO_UTILS_GUIDE.md @@ -0,0 +1,430 @@ +# Video Utils - Single Video Processing Guide + +The `video_utils.py` module provides essential utilities for video processing operations including video information extraction, chunking, and audio extraction. This guide focuses on processing single videos with various utilities. + +## Overview + +The video utilities module offers the following main functions: +- **Video Information**: Extract metadata from video files +- **Time-based Chunking**: Split videos into fixed-duration segments +- **Silence-based Chunking**: Split videos at silence boundaries +- **Audio Extraction**: Extract audio tracks from video files + +## Installation Requirements + +Ensure you have `ffmpeg` installed on your system: + +```bash +# macOS +brew install ffmpeg + +# Ubuntu/Debian +sudo apt update && sudo apt install ffmpeg + +# Windows +# Download from https://ffmpeg.org/download.html +``` + +## Command Line Usage + +The video utils script provides a command-line interface with multiple subcommands: + +```bash +python utils/video_utils.py [command] [options] +``` + +### Available Commands + +1. `info` - Get video information +2. `chunk-time` - Split video into fixed-duration chunks +3. `chunk-silence` - Split video based on silence detection +4. `extract-audio` - Extract audio from video + +--- + +## 1. Video Information Extraction + +Get detailed metadata about your video file including duration, codecs, resolution, and audio properties. + +### Command Syntax +```bash +python utils/video_utils.py info --input /path/to/video.mp4 +``` + +### Example +```bash +python utils/video_utils.py info --input data/chunk_003.mp4 +``` + +### Sample Output +``` +๐Ÿ“น Video Information for: data/chunk_003.mp4 +Duration: 5.48 seconds +Video: h264 640x360 @ 25/1 fps +Audio: aac 44100 Hz 2 channels +``` + +### Use Cases +- **Quality Assessment**: Check video properties before processing +- **Batch Analysis**: Verify video specifications across datasets +- **Preprocessing**: Determine optimal parameters for further processing + +--- + +## 2. Time-based Video Chunking + +Split a video into fixed-duration segments with optional overlap between chunks. + +### Command Syntax +```bash +python utils/video_utils.py chunk-time --input VIDEO_FILE --output_dir OUTPUT_DIR [options] +``` + +### Parameters +- `--input`: Input video file path +- `--output_dir`: Directory to save video chunks +- `--duration`: Chunk duration in seconds (default: 30) +- `--overlap`: Overlap between chunks in seconds (default: 0) + +### Examples + +#### Basic 30-second chunks +```bash +python utils/video_utils.py chunk-time \ + --input data/long_video.mp4 \ + --output_dir chunks/time_based/ +``` + +#### Custom duration with overlap +```bash +python utils/video_utils.py chunk-time \ + --input data/presentation.mp4 \ + --output_dir chunks/presentation/ \ + --duration 60 \ + --overlap 5 +``` + +#### Short chunks for analysis +```bash +python utils/video_utils.py chunk-time \ + --input data/interview.mp4 \ + --output_dir chunks/analysis/ \ + --duration 10 \ + --overlap 2 +``` + +### Output Structure +``` +output_dir/ +โ”œโ”€โ”€ chunk_000.mp4 +โ”œโ”€โ”€ chunk_001.mp4 +โ”œโ”€โ”€ chunk_002.mp4 +โ””โ”€โ”€ ... +``` + +### Use Cases +- **Training Data Preparation**: Create uniform-length segments for ML training +- **Analysis Windows**: Generate consistent segments for frame-by-frame analysis +- **Memory Management**: Process large videos in smaller, manageable chunks + +--- + +## 3. Silence-based Video Chunking + +Automatically split videos at natural silence boundaries, ideal for speech and conversation videos. + +### Command Syntax +```bash +python utils/video_utils.py chunk-silence --input VIDEO_FILE --output_dir OUTPUT_DIR [options] +``` + +### Parameters +- `--input`: Input video file path +- `--output_dir`: Directory to save video chunks +- `--threshold`: Silence threshold in dB (default: -30) +- `--min_silence`: Minimum silence duration to trigger split (default: 1.0s) +- `--min_chunk`: Minimum chunk duration to keep (default: 5.0s) + +### Examples + +#### Default silence detection +```bash +python utils/video_utils.py chunk-silence \ + --input data/podcast.mp4 \ + --output_dir chunks/podcast_segments/ +``` + +#### Sensitive silence detection +```bash +python utils/video_utils.py chunk-silence \ + --input data/quiet_conversation.mp4 \ + --output_dir chunks/conversation/ \ + --threshold -40 \ + --min_silence 0.5 +``` + +#### Less sensitive (for noisy environments) +```bash +python utils/video_utils.py chunk-silence \ + --input data/conference_call.mp4 \ + --output_dir chunks/conference/ \ + --threshold -20 \ + --min_silence 2.0 \ + --min_chunk 10 +``` + +### Output Structure +``` +output_dir/ +โ”œโ”€โ”€ silence_chunk_000_000000.0s.mp4 +โ”œโ”€โ”€ silence_chunk_001_024500.5s.mp4 +โ”œโ”€โ”€ silence_chunk_002_056200.2s.mp4 +โ””โ”€โ”€ ... +``` + +### Understanding Silence Parameters + +| Parameter | Description | Typical Values | +|-----------|-------------|----------------| +| `--threshold` | Audio level considered "silence" | -30dB (normal), -40dB (sensitive), -20dB (noisy) | +| `--min_silence` | How long silence must last to split | 1.0s (conversations), 0.5s (precise), 2.0s (robust) | +| `--min_chunk` | Minimum segment length to keep | 5.0s (analysis), 3.0s (short clips), 10.0s (substantial) | + +### Use Cases +- **Podcast Processing**: Split episodes into individual topics/segments +- **Interview Analysis**: Separate questions and answers +- **Meeting Transcription**: Create segments for individual speakers +- **Content Editing**: Identify natural break points for editing + +--- + +## 4. Audio Extraction + +Extract audio tracks from video files with customizable format and quality settings. + +### Command Syntax +```bash +python utils/video_utils.py extract-audio --input VIDEO_FILE [options] +``` + +### Parameters +- `--input`: Input video file path +- `--output`: Output audio file path (auto-generated if not specified) +- `--sample_rate`: Audio sample rate in Hz (default: 16000) +- `--channels`: Number of audio channels (default: 1 for mono) + +### Examples + +#### Basic audio extraction (16kHz mono for speech processing) +```bash +python utils/video_utils.py extract-audio --input data/interview.mp4 +``` + +#### High-quality stereo extraction +```bash +python utils/video_utils.py extract-audio \ + --input data/music_video.mp4 \ + --output audio/music_track.wav \ + --sample_rate 44100 \ + --channels 2 +``` + +#### Speech analysis format +```bash +python utils/video_utils.py extract-audio \ + --input data/presentation.mp4 \ + --output audio/speech.wav \ + --sample_rate 16000 \ + --channels 1 +``` + +### Sample Rate Guidelines + +| Use Case | Sample Rate | Channels | Reasoning | +|----------|-------------|----------|-----------| +| Speech Analysis | 16000 Hz | 1 (mono) | Sufficient for speech, smaller files | +| SyncNet Processing | 16000 Hz | 1 (mono) | Required by SyncNet model | +| Music Analysis | 44100 Hz | 2 (stereo) | CD quality, preserves stereo information | +| General Purpose | 22050 Hz | 1 (mono) | Good balance of quality and size | + +### Use Cases +- **SyncNet Preprocessing**: Extract audio for sync analysis +- **Transcription**: Prepare audio for speech-to-text +- **Audio Analysis**: Isolate audio track for processing +- **Backup/Archival**: Save audio separately from video + +--- + +## Programmatic Usage + +You can also use the video utils functions directly in Python scripts: + +### Import and Basic Usage +```python +from utils.video_utils import get_video_info, cut_video_by_time, extract_audio_from_video + +# Get video information +info = get_video_info("data/video.mp4") +print(f"Duration: {info['duration']} seconds") + +# Extract 30-second clip starting at 60 seconds +cut_video_by_time("data/video.mp4", "output/", start_time=60, duration=30, chunk_name="excerpt.mp4") + +# Extract audio for SyncNet processing +extract_audio_from_video("data/video.mp4", "audio/extracted.wav", sample_rate=16000, channels=1) +``` + +### Batch Processing Example +```python +import os +from pathlib import Path +from utils.video_utils import get_video_info, cut_video_by_time + +def process_video_collection(input_dir, output_dir, chunk_duration=30): + """Process all videos in directory""" + video_files = list(Path(input_dir).glob("*.mp4")) + + for video_file in video_files: + print(f"Processing: {video_file.name}") + + # Get video info + info = get_video_info(str(video_file)) + if not info: + continue + + # Create chunks + video_output_dir = os.path.join(output_dir, video_file.stem) + total_duration = info['duration'] + + num_chunks = int(total_duration // chunk_duration) + 1 + for i in range(num_chunks): + start_time = i * chunk_duration + if start_time >= total_duration: + break + + actual_duration = min(chunk_duration, total_duration - start_time) + chunk_name = f"chunk_{i:03d}.mp4" + + cut_video_by_time( + str(video_file), + video_output_dir, + start_time, + actual_duration, + chunk_name + ) + +# Usage +process_video_collection("raw_videos/", "processed_chunks/") +``` + +--- + +## Integration with SyncNet Pipeline + +The video utils integrate seamlessly with the SyncNet processing pipeline: + +### Complete Workflow Example +```bash +# Step 1: Get video information +python utils/video_utils.py info --input raw_video.mp4 + +# Step 2: Split video into manageable chunks (optional for long videos) +python utils/video_utils.py chunk-time \ + --input raw_video.mp4 \ + --output_dir video_chunks/ \ + --duration 60 \ + --overlap 5 + +# Step 3: Process each chunk with SyncNet (using adjusted face size) +for chunk in video_chunks/*.mp4; do + chunk_name=$(basename "$chunk" .mp4) + python run_pipeline.py \ + --videofile "$chunk" \ + --reference "$chunk_name" \ + --data_dir syncnet_output/ \ + --min_face_size 50 + + python run_syncnet.py \ + --videofile "$chunk" \ + --reference "$chunk_name" \ + --data_dir syncnet_output/ +done + +# Step 4: Filter results by quality +python filter_videos_by_sync_score.py \ + --input_dir video_chunks/ \ + --output_dir filtered_results/ \ + --preset high +``` + +--- + +## Best Practices + +### 1. **Choose Appropriate Chunk Sizes** +- **Short videos (< 2 min)**: Process as single file +- **Medium videos (2-10 min)**: 30-60 second chunks +- **Long videos (> 10 min)**: 60-120 second chunks with overlap + +### 2. **Silence Detection Tips** +- **Clean recordings**: Use -30dB threshold +- **Noisy environments**: Use -20dB or higher +- **Whispered speech**: Use -40dB or lower +- **Test different values** on sample data first + +### 3. **Audio Extraction Guidelines** +- **SyncNet processing**: Always use 16kHz mono +- **Transcription**: 16kHz mono sufficient +- **Music analysis**: Use original sample rate and channels +- **Storage optimization**: Use lowest acceptable quality + +### 4. **File Organization** +``` +project/ +โ”œโ”€โ”€ raw_videos/ # Original video files +โ”œโ”€โ”€ chunks/ # Video segments +โ”‚ โ”œโ”€โ”€ time_based/ # Fixed-duration chunks +โ”‚ โ””โ”€โ”€ silence_based/ # Natural break chunks +โ”œโ”€โ”€ audio/ # Extracted audio files +โ”œโ”€โ”€ syncnet_data/ # SyncNet processing results +โ””โ”€โ”€ filtered_results/ # Quality-filtered videos +``` + +--- + +## Troubleshooting + +### Common Issues + +1. **FFmpeg not found** + ``` + Error: ffmpeg command not found + Solution: Install ffmpeg or add to PATH + ``` + +2. **Permission errors** + ``` + Error: Permission denied writing to output directory + Solution: Check directory permissions or use different output path + ``` + +3. **No audio track found** + ``` + Error: Could not extract audio + Solution: Verify video has audio track using 'info' command + ``` + +4. **Very short chunks produced** + ``` + Issue: Silence detection creates tiny segments + Solution: Increase --min_chunk parameter + ``` + +### Performance Tips + +- **Large files**: Use time-based chunking first, then process chunks +- **Slow processing**: Reduce video resolution before chunking +- **Memory issues**: Process smaller chunks or reduce overlap +- **Storage concerns**: Use appropriate audio sample rates + +This comprehensive guide should help you effectively use the video utilities for single video processing in your SyncNet workflow! diff --git a/README.md b/README.md index 2845079..81243b6 100755 --- a/README.md +++ b/README.md @@ -49,6 +49,26 @@ Example with smaller faces: python run_pipeline.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output --min_face_size 50 ``` +## Video Processing Utilities + +For video preprocessing, chunking, and analysis, use the video utilities: + +```bash +# Get video information +python utils/video_utils.py info --input data/video.mp4 + +# Split video into 30-second chunks +python utils/video_utils.py chunk-time --input data/video.mp4 --output_dir chunks/ --duration 30 + +# Split video at silence boundaries (ideal for speech) +python utils/video_utils.py chunk-silence --input data/conversation.mp4 --output_dir chunks/ + +# Extract audio for processing +python utils/video_utils.py extract-audio --input data/video.mp4 --output audio/extracted.wav +``` + +**๐Ÿ“– See [VIDEO_UTILS_GUIDE.md](VIDEO_UTILS_GUIDE.md) for comprehensive usage examples and best practices.** + ## Troubleshooting ### Issue: Empty pycrop directory / No bounding boxes in output video From f5ceacc158a3570301d29f3a7527566732d6e56f Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Thu, 7 Aug 2025 11:40:58 +0600 Subject: [PATCH 06/11] chore: updated readme --- Docs/VIDEO_UTILS_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/VIDEO_UTILS_GUIDE.md b/Docs/VIDEO_UTILS_GUIDE.md index d6fa12c..e21624c 100644 --- a/Docs/VIDEO_UTILS_GUIDE.md +++ b/Docs/VIDEO_UTILS_GUIDE.md @@ -129,7 +129,7 @@ output_dir/ --- -## 3. Silence-based Video Chunking +## 3. Silence-based Video Chunking (Do not use this. This can not detect silence if there is a backgroud noise) Automatically split videos at natural silence boundaries, ideal for speech and conversation videos. From 3ee29654bb0fa762e2e7781d5d1a3dcab05b9109 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sun, 10 Aug 2025 20:57:11 +0600 Subject: [PATCH 07/11] chore: mod gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9504c8a..e5e649b 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ Thumbs.db data/ protos/ *.pth +results \ No newline at end of file From a6d97328977598f3a1ca0cbfda24bb4973c83f49 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sun, 10 Aug 2025 21:31:43 +0600 Subject: [PATCH 08/11] chore: kept face tracking & bounding box with the videos --- ENHANCEMENT_SUMMARY.md | 122 +++++++++++++++++++++ README.md | 46 ++++++++ filter_videos_by_sync_score.py | 139 +++++++++++++++++++++++- test_enhanced_filter.py | 190 +++++++++++++++++++++++++++++++++ 4 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 ENHANCEMENT_SUMMARY.md create mode 100644 test_enhanced_filter.py diff --git a/ENHANCEMENT_SUMMARY.md b/ENHANCEMENT_SUMMARY.md new file mode 100644 index 0000000..fdeeb07 --- /dev/null +++ b/ENHANCEMENT_SUMMARY.md @@ -0,0 +1,122 @@ +# Enhanced SyncNet Filter - Output Preservation Update + +## Summary of Changes + +This update enhances the `filter_videos_by_sync_score.py` script to preserve valuable SyncNet processing outputs instead of discarding them after analysis. This addresses the user's need to keep cropped videos and bounding boxes for active speaker annotation. + +## Key Enhancements + +### 1. Output Preservation System + +**New Method: `_preserve_syncnet_outputs(result, video_name, temp_video_dir, output_dir)`** +- Preserves cropped face videos from `pycrop/` directory +- Saves bounding box visualization videos +- Keeps analysis results (offsets.txt, tracks.pkl) +- Organizes outputs in structured directories + +**New Method: `_copy_syncnet_outputs_to_quality_dir(result, quality_dir, output_dir)`** +- Copies SyncNet outputs to quality-specific directories +- Enables easy access to processing results by quality level +- Maintains organized structure for further analysis + +### 2. Enhanced Processing Pipeline + +**Added Visualization Step:** +- Automatically runs `run_visualise.py` after SyncNet analysis +- Generates videos with bounding box overlays +- Creates visual annotations for active speaker detection + +**Improved Output Structure:** +``` +output_dir/ +โ”œโ”€โ”€ good_quality/ +โ”‚ โ”œโ”€โ”€ video1.mp4 # Original videos +โ”‚ โ””โ”€โ”€ syncnet_outputs/ # SyncNet results +โ”‚ โ””โ”€โ”€ video1/ +โ”‚ โ”œโ”€โ”€ cropped_faces/ # Individual face tracks +โ”‚ โ”œโ”€โ”€ video1_with_bboxes.avi # Bounding box visualization +โ”‚ โ””โ”€โ”€ analysis/ # Offsets, tracking data +โ”œโ”€โ”€ poor_quality/ +โ”‚ โ””โ”€โ”€ syncnet_outputs/ # Results for filtered videos +โ”œโ”€โ”€ syncnet_outputs/ # Master results directory +โ””โ”€โ”€ sync_filter_results.json # Analysis summary +``` + +### 3. New Files Created + +**`test_enhanced_filter.py`** +- Comprehensive test suite for the enhanced functionality +- Validates output structure and preservation +- Provides usage examples and verification + +**Updated Documentation:** +- Enhanced README.md with new output structure +- Added testing instructions +- Documented use cases for preserved outputs + +## Technical Implementation + +### Code Changes in `filter_videos_by_sync_score.py`: + +1. **Added visualization step** in `process_single_video()`: + ```python + # Generate visualization with bounding boxes + viz_cmd = [ + sys.executable, 'run_visualise.py', + '--videofile', video_path, + '--reference', video_name, + '--data_dir', temp_video_dir + ] + subprocess.run(viz_cmd, check=True) + ``` + +2. **Added output preservation** after successful processing: + ```python + # Preserve SyncNet outputs for further use + self._preserve_syncnet_outputs(result, video_name, temp_video_dir, output_dir) + ``` + +3. **Enhanced quality directory copying**: + ```python + # Also copy SyncNet outputs to the quality directory + self._copy_syncnet_outputs_to_quality_dir(result, dst_dir, output_dir) + ``` + +### Benefits for Users: + +1. **Active Speaker Annotation**: Direct access to cropped faces and bounding boxes +2. **Training Data Preparation**: Organized face tracks with quality metadata +3. **Analysis Workflow**: All processing artifacts preserved for further research +4. **Quality Comparison**: Easy comparison between good and poor quality results + +## Usage + +### Basic Usage (with preservation): +```bash +python filter_videos_by_sync_score.py \ + --input_dir /path/to/videos \ + --output_dir /path/to/results \ + --preset medium +``` + +### Testing: +```bash +python test_enhanced_filter.py +``` + +## Compatibility + +- **Backward Compatible**: All existing functionality preserved +- **No Breaking Changes**: Original API and behavior maintained +- **Optional Features**: Preservation doesn't interfere with filtering logic +- **Error Handling**: Graceful degradation if preservation fails + +## Future Enhancements + +This update provides a foundation for: +- Custom output format selection +- Metadata-driven organization +- Integration with annotation tools +- Batch processing pipelines + +The enhanced filter now serves as both a quality filtering tool and a comprehensive SyncNet output manager, making it more valuable for research and production workflows. diff --git a/README.md b/README.md index 81243b6..724793b 100755 --- a/README.md +++ b/README.md @@ -127,10 +127,56 @@ python filter_videos_by_sync_score.py \ ``` output_dir/ โ”œโ”€โ”€ good_quality/ # Videos that pass quality thresholds +โ”‚ โ”œโ”€โ”€ video1.mp4 # Original videos +โ”‚ โ”œโ”€โ”€ video2.mp4 +โ”‚ โ””โ”€โ”€ syncnet_outputs/ # SyncNet processing results (NEW!) +โ”‚ โ”œโ”€โ”€ video1/ +โ”‚ โ”‚ โ”œโ”€โ”€ cropped_faces/ # Cropped face track videos +โ”‚ โ”‚ โ”œโ”€โ”€ video1_with_bboxes.avi # Video with bounding boxes +โ”‚ โ”‚ โ””โ”€โ”€ analysis/ # Analysis results +โ”‚ โ”‚ โ”œโ”€โ”€ offsets.txt # Frame offset values +โ”‚ โ”‚ โ””โ”€โ”€ tracks.pkl # Face tracking data +โ”‚ โ””โ”€โ”€ video2/ +โ”‚ โ””โ”€โ”€ ... โ”œโ”€โ”€ poor_quality/ # Videos filtered out for low quality +โ”‚ โ”œโ”€โ”€ rejected_video.mp4 # Original videos +โ”‚ โ””โ”€โ”€ syncnet_outputs/ # SyncNet outputs for poor quality videos +โ”‚ โ””โ”€โ”€ rejected_video/ +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ syncnet_outputs/ # All SyncNet processing results +โ”‚ โ”œโ”€โ”€ video1/ # Individual video results +โ”‚ โ””โ”€โ”€ video2/ โ””โ”€โ”€ sync_filter_results.json # Detailed analysis results ``` +**๐ŸŽฏ Enhanced Output Preservation (NEW!):** + +The filtering tool now preserves valuable SyncNet processing outputs for further analysis: + +- **Cropped Face Videos**: Individual face tracks as separate video files (`cropped_faces/*.avi`) +- **Bounding Box Visualizations**: Original video with face detection overlays (`*_with_bboxes.avi`) +- **Analysis Data**: Frame offsets, confidence scores, and tracking information +- **Quality-Organized**: All outputs copied to `good_quality/` and `poor_quality/` folders + +This is especially useful for: +- **Active Speaker Detection**: Use cropped faces and bounding boxes for annotation +- **Training Data Preparation**: Access to pre-processed face tracks and metadata +- **Quality Analysis**: Compare outputs between good and poor quality videos + +**Testing the Enhanced Filter:** + +Test the enhanced filtering functionality: + +```bash +python test_enhanced_filter.py +``` + +This will verify that: +- All required dependencies are present +- Output preservation works correctly +- Quality directories are created with proper structure +- SyncNet outputs are preserved in organized folders + **Parameters:** - `--min_confidence`: Minimum SyncNet confidence score to keep video - `--max_abs_offset`: Maximum absolute frame offset to keep video diff --git a/filter_videos_by_sync_score.py b/filter_videos_by_sync_score.py index 9047e4e..5d0a939 100644 --- a/filter_videos_by_sync_score.py +++ b/filter_videos_by_sync_score.py @@ -158,6 +158,27 @@ def process_single_video(self, video_path, output_dir, temp_dir): if not passes_offset: result['quality_reasons'].append(f"High offset: {abs_offset} > {self.max_abs_offset}") + # Step 4: Generate visualization with bounding boxes + visualize_cmd = [ + python_exec, 'run_visualise.py', + '--videofile', str(video_path), + '--reference', video_name, + '--data_dir', temp_video_dir + ] + + visualize_result = subprocess.run( + visualize_cmd, + capture_output=True, + text=True, + cwd=os.getcwd() + ) + + if visualize_result.returncode == 0: + result['visualization_created'] = True + else: + result['visualization_created'] = False + result['visualization_error'] = visualize_result.stderr + result['status'] = 'success' except Exception as e: @@ -166,7 +187,11 @@ def process_single_video(self, video_path, output_dir, temp_dir): result['end_time'] = time.time() result['processing_time'] = result['end_time'] - result['start_time'] - # Clean up temporary files + # Preserve important outputs before cleanup + if result['status'] == 'success': + self._preserve_syncnet_outputs(temp_video_dir, video_name, output_dir, result) + + # Clean up temporary files (but keep preserved outputs) try: if os.path.exists(temp_video_dir): shutil.rmtree(temp_video_dir) @@ -175,6 +200,115 @@ def process_single_video(self, video_path, output_dir, temp_dir): return result + def _preserve_syncnet_outputs(self, temp_video_dir, video_name, output_dir, result): + """ + Preserve important SyncNet outputs before cleanup + + Args: + temp_video_dir: Temporary processing directory + video_name: Name of the video being processed + output_dir: Main output directory + result: Processing result dictionary + """ + try: + # Create output subdirectories + syncnet_outputs_dir = os.path.join(output_dir, 'syncnet_outputs', video_name) + os.makedirs(syncnet_outputs_dir, exist_ok=True) + + # Preserve cropped face videos + crop_dir = os.path.join(temp_video_dir, 'pycrop', video_name) + if os.path.exists(crop_dir): + crop_output_dir = os.path.join(syncnet_outputs_dir, 'cropped_faces') + os.makedirs(crop_output_dir, exist_ok=True) + + # Copy all cropped face videos + for crop_file in os.listdir(crop_dir): + if crop_file.endswith('.avi'): + src_path = os.path.join(crop_dir, crop_file) + dst_path = os.path.join(crop_output_dir, crop_file) + shutil.copy2(src_path, dst_path) + + result['cropped_faces_saved'] = crop_output_dir + + # Preserve visualization video with bounding boxes + viz_video = os.path.join(temp_video_dir, 'pyavi', video_name, 'video_out.avi') + if os.path.exists(viz_video): + viz_output_path = os.path.join(syncnet_outputs_dir, f'{video_name}_with_bboxes.avi') + shutil.copy2(viz_video, viz_output_path) + result['bbox_video_saved'] = viz_output_path + + # Preserve analysis results + analysis_dir = os.path.join(syncnet_outputs_dir, 'analysis') + os.makedirs(analysis_dir, exist_ok=True) + + # Copy offsets file + offsets_file = os.path.join(temp_video_dir, 'pywork', video_name, 'offsets.txt') + if os.path.exists(offsets_file): + dst_offsets = os.path.join(analysis_dir, 'offsets.txt') + shutil.copy2(offsets_file, dst_offsets) + + # Copy tracks file + tracks_file = os.path.join(temp_video_dir, 'pywork', video_name, 'tracks.pckl') + if os.path.exists(tracks_file): + dst_tracks = os.path.join(analysis_dir, 'tracks.pckl') + shutil.copy2(tracks_file, dst_tracks) + + # Copy face detection results + faces_file = os.path.join(temp_video_dir, 'pywork', video_name, 'faces.pckl') + if os.path.exists(faces_file): + dst_faces = os.path.join(analysis_dir, 'faces.pckl') + shutil.copy2(faces_file, dst_faces) + + # Copy scene detection results + scene_file = os.path.join(temp_video_dir, 'pywork', video_name, 'scene.pckl') + if os.path.exists(scene_file): + dst_scene = os.path.join(analysis_dir, 'scene.pckl') + shutil.copy2(scene_file, dst_scene) + + # Copy activesd file (sync analysis) + activesd_file = os.path.join(temp_video_dir, 'pywork', video_name, 'activesd.pckl') + if os.path.exists(activesd_file): + dst_activesd = os.path.join(analysis_dir, 'activesd.pckl') + shutil.copy2(activesd_file, dst_activesd) + + result['analysis_files_saved'] = analysis_dir + + except Exception as e: + # Don't fail the main process if preservation fails + result['preservation_error'] = str(e) + + def _copy_syncnet_outputs_to_quality_dir(self, result, quality_dir, output_dir): + """ + Copy SyncNet outputs to the quality directory for easy access + + Args: + result: Processing result dictionary + quality_dir: Quality directory (good_quality or poor_quality) + output_dir: Main output directory + """ + try: + video_name = result['video_name'] + syncnet_source_dir = os.path.join(output_dir, 'syncnet_outputs', video_name) + + if os.path.exists(syncnet_source_dir): + # Create syncnet subdirectory in quality folder + quality_syncnet_dir = os.path.join(quality_dir, 'syncnet_outputs', video_name) + os.makedirs(quality_syncnet_dir, exist_ok=True) + + # Copy the entire syncnet output directory + for item in os.listdir(syncnet_source_dir): + src_path = os.path.join(syncnet_source_dir, item) + dst_path = os.path.join(quality_syncnet_dir, item) + + if os.path.isdir(src_path): + shutil.copytree(src_path, dst_path, dirs_exist_ok=True) + else: + shutil.copy2(src_path, dst_path) + + except Exception as e: + # Don't fail the main process if copying fails + pass + def filter_videos(self, input_dir, output_dir, max_workers=2, keep_all=False): """ Batch process and filter videos from input directory @@ -251,6 +385,9 @@ def filter_videos(self, input_dir, output_dir, max_workers=2, keep_all=False): dst_path = os.path.join(dst_dir, os.path.basename(src_path)) shutil.copy2(src_path, dst_path) + # Also copy SyncNet outputs to the quality directory + self._copy_syncnet_outputs_to_quality_dir(result, dst_dir, output_dir) + else: status_emoji = "โš ๏ธ" if result['status'] == 'no_faces' else "โŒ" message = result.get('message', result.get('error', 'Unknown error')) diff --git a/test_enhanced_filter.py b/test_enhanced_filter.py new file mode 100644 index 0000000..fc66179 --- /dev/null +++ b/test_enhanced_filter.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +""" +Test script for the enhanced SyncNet filtering with output preservation +""" + +import os +import sys +import tempfile +import shutil +from pathlib import Path + +# Add the current directory to the path so we can import our modules +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from filter_videos_by_sync_score import SyncNetFilter + +def test_enhanced_filter(): + """Test the enhanced filtering functionality""" + + print("๐Ÿงช Testing Enhanced SyncNet Filter with Output Preservation") + print("=" * 70) + + # Check if we have test data + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + if not os.path.exists(test_data_dir): + print("โŒ No test data directory found. Please ensure 'data' directory exists.") + return False + + # Look for video files + video_files = [] + for ext in ['.mp4', '.avi', '.mov', '.mkv']: + video_files.extend(Path(test_data_dir).glob(f'*{ext}')) + + if not video_files: + print("โŒ No video files found in data directory.") + return False + + print(f"๐Ÿ“น Found {len(video_files)} video file(s) for testing") + for video_file in video_files: + print(f" - {video_file.name}") + + # Create temporary output directory + with tempfile.TemporaryDirectory(prefix='syncnet_test_') as temp_output: + print(f"\n๐Ÿ“ Using temporary output directory: {temp_output}") + + # Create filter with relaxed settings for testing + filter_tool = SyncNetFilter( + min_confidence=2.0, # Very relaxed for testing + max_abs_offset=10, # Very relaxed for testing + min_face_size=50, + min_track=50 + ) + + print("\n๐Ÿš€ Running enhanced filter with output preservation...") + + try: + # Run the filter + summary = filter_tool.filter_videos( + input_dir=test_data_dir, + output_dir=temp_output, + max_workers=1, # Single worker for testing + keep_all=False + ) + + print("\nโœ… Filtering completed successfully!") + + # Check the output structure + print("\n๐Ÿ“‹ Checking output structure:") + + expected_dirs = ['good_quality', 'poor_quality', 'syncnet_outputs'] + for expected_dir in expected_dirs: + dir_path = os.path.join(temp_output, expected_dir) + if os.path.exists(dir_path): + print(f" โœ… {expected_dir}/ directory exists") + + # List contents + contents = os.listdir(dir_path) + if contents: + print(f" ๐Ÿ“‚ Contains: {', '.join(contents[:5])}") + if len(contents) > 5: + print(f" ๐Ÿ“‚ ... and {len(contents) - 5} more items") + else: + print(f" ๐Ÿ“‚ (empty)") + else: + print(f" โŒ {expected_dir}/ directory missing") + + # Check for syncnet outputs in quality directories + for quality_dir in ['good_quality', 'poor_quality']: + quality_path = os.path.join(temp_output, quality_dir) + if os.path.exists(quality_path): + syncnet_path = os.path.join(quality_path, 'syncnet_outputs') + if os.path.exists(syncnet_path): + print(f" โœ… {quality_dir}/syncnet_outputs/ exists") + + # Check for specific outputs + for item in os.listdir(syncnet_path): + item_path = os.path.join(syncnet_path, item) + if os.path.isdir(item_path): + sub_contents = os.listdir(item_path) + print(f" ๐Ÿ“ {item}/ contains: {', '.join(sub_contents)}") + else: + print(f" โš ๏ธ {quality_dir}/syncnet_outputs/ not found") + + # Check results file + results_file = os.path.join(temp_output, 'sync_filter_results.json') + if os.path.exists(results_file): + print(f" โœ… sync_filter_results.json exists") + + # Read and display summary stats + import json + with open(results_file, 'r') as f: + results = json.load(f) + + print(f" ๐Ÿ“Š Processed: {results['total_videos']} videos") + print(f" ๐Ÿ“Š Good quality: {results['good_quality']}") + print(f" ๐Ÿ“Š Poor quality: {results['poor_quality']}") + print(f" ๐Ÿ“Š No faces: {results['no_faces_detected']}") + print(f" ๐Ÿ“Š Failed: {results['processing_failed']}") + else: + print(f" โŒ sync_filter_results.json missing") + + print(f"\n๐ŸŽ‰ Test completed successfully!") + print(f"๐Ÿ’ก You can examine the test results in: {temp_output}") + + # Ask if user wants to keep the test results + response = input("\n๐Ÿค” Keep test results for inspection? (y/N): ").strip().lower() + if response in ['y', 'yes']: + permanent_dir = os.path.join(os.path.dirname(__file__), 'test_results') + shutil.copytree(temp_output, permanent_dir, dirs_exist_ok=True) + print(f"๐Ÿ“ Test results saved to: {permanent_dir}") + + return True + + except Exception as e: + print(f"\nโŒ Test failed with error: {e}") + import traceback + traceback.print_exc() + return False + +def main(): + """Main test function""" + print("๐Ÿ”ง Enhanced SyncNet Filter Test Suite") + print("=" * 50) + + # Check dependencies + print("\n๐Ÿ” Checking dependencies...") + + required_files = [ + 'SyncNetInstance.py', + 'SyncNetModel.py', + 'run_pipeline.py', + 'run_syncnet.py', + 'run_visualise.py', + 'filter_videos_by_sync_score.py' + ] + + missing_files = [] + for file in required_files: + if not os.path.exists(file): + missing_files.append(file) + else: + print(f" โœ… {file}") + + if missing_files: + print(f"\nโŒ Missing required files: {', '.join(missing_files)}") + return False + + # Run the test + success = test_enhanced_filter() + + if success: + print(f"\n๐ŸŽ‰ All tests passed! The enhanced filter is ready to use.") + print(f"\n๐Ÿ“š Usage example:") + print(f" python filter_videos_by_sync_score.py \\") + print(f" --input_dir /path/to/videos \\") + print(f" --output_dir /path/to/output \\") + print(f" --preset medium") + print(f"\n๐ŸŽฏ This will now preserve:") + print(f" โ€ข Cropped face videos in syncnet_outputs/*/cropped_faces/") + print(f" โ€ข Bounding box visualizations as *_with_bboxes.avi") + print(f" โ€ข Analysis results (offsets.txt, tracks.pkl)") + print(f" โ€ข Copy everything to good_quality/ and poor_quality/ folders") + else: + print(f"\nโŒ Tests failed. Please check the errors above.") + + return success + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) From 588d00aca740b93ea4692626d5a53ecc62297eba Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Sun, 17 Aug 2025 23:55:53 +0600 Subject: [PATCH 09/11] chore: added todo --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 724793b..848fa7f 100755 --- a/README.md +++ b/README.md @@ -193,6 +193,9 @@ $DATA_DIR/pyavi/$REFERENCE/video_out.avi - output video (as shown below)

+##Todo +- add duplicate remover + ## Publications ``` From 9e0160464d2d251d6bf64d64d9d7bdc2f7f5fed3 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Fri, 22 Aug 2025 04:02:10 +0600 Subject: [PATCH 10/11] chore: added yt_id to the chunk names --- DIRECTORY_PREPARE_SUMMARY.md | 0 Docs/DIRECTORY_PREPARE_SUMMARY.md | 125 ++++++++ README.md | 19 ++ utils/DIRECTORY_PREPARE_README.md | 243 ++++++++++++++++ utils/directory_prepare.py | 457 ++++++++++++++++++++++++++++++ 5 files changed, 844 insertions(+) create mode 100644 DIRECTORY_PREPARE_SUMMARY.md create mode 100644 Docs/DIRECTORY_PREPARE_SUMMARY.md create mode 100644 utils/DIRECTORY_PREPARE_README.md create mode 100644 utils/directory_prepare.py diff --git a/DIRECTORY_PREPARE_SUMMARY.md b/DIRECTORY_PREPARE_SUMMARY.md new file mode 100644 index 0000000..e69de29 diff --git a/Docs/DIRECTORY_PREPARE_SUMMARY.md b/Docs/DIRECTORY_PREPARE_SUMMARY.md new file mode 100644 index 0000000..b935e72 --- /dev/null +++ b/Docs/DIRECTORY_PREPARE_SUMMARY.md @@ -0,0 +1,125 @@ +# Directory Preparation Script - Implementation Summary + +## โœ… **Complete Implementation** + +I have successfully created the `utils/directory_prepare.py` script that takes a SyncNet results directory as input and organizes it into 4 structured directories as requested. + +## ๐ŸŽฏ **Script Features** + +### **Input Structure Expected:** +``` +results/video1/good_quality/ +โ”œโ”€โ”€ chunk_000.mp4, chunk_001.mp4, ... # Original chunk videos +โ””โ”€โ”€ syncnet_outputs/ + โ”œโ”€โ”€ chunk_000/ + โ”‚ โ”œโ”€โ”€ chunk_000_with_bboxes.avi # Bounding box video + โ”‚ โ”œโ”€โ”€ cropped_faces/ + โ”‚ โ”‚ โ””โ”€โ”€ 00000.avi # Cropped face video + โ”‚ โ””โ”€โ”€ analysis/ # Analysis files + โ””โ”€โ”€ chunk_001/, chunk_002/, ... +``` + +### **Output Structure Created:** +``` +organized_output/ +โ”œโ”€โ”€ video_normal/ # 1. Original chunk videos (copied) +โ”‚ โ”œโ”€โ”€ chunk_000.mp4, chunk_001.mp4, ... +โ”œโ”€โ”€ video_bbox/ # 2. Bounding box videos (AVIโ†’MP4) +โ”‚ โ”œโ”€โ”€ chunk_000_with_bboxes.mp4, ... +โ”œโ”€โ”€ video_cropped/ # 3. Cropped faces (AVIโ†’MP4, renamed) +โ”‚ โ”œโ”€โ”€ chunk_000.mp4, chunk_001.mp4, ... +โ””โ”€โ”€ audio/ # 4. Extracted audio (16kHz mono WAV) + โ”œโ”€โ”€ chunk_000.wav, chunk_001.wav, ... +``` + +## ๐Ÿš€ **Usage** + +### **Basic Command:** +```bash +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized_output +``` + +### **Advanced Options:** +```bash +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized_output \ + --max_workers 8 # Use 8 parallel workers +``` + +## โšก **Performance Results** + +**Test Results with your data (47 chunks):** +- โœ… **Normal videos**: 47/47 (copied) +- ๐Ÿ“ฆ **Bbox videos**: 47/47 (AVIโ†’MP4 converted) +- ๐Ÿ‘ค **Cropped videos**: 47/47 (AVIโ†’MP4 converted, renamed) +- ๐Ÿ”Š **Audio files**: 47/47 (extracted to 16kHz mono WAV) +- โฑ๏ธ **Processing time**: ~19 seconds with 2 workers + +## ๐Ÿ”ง **Technical Implementation** + +### **Video Conversion (AVI โ†’ MP4):** +- **Video Codec**: H.264 (libx264) +- **Audio Codec**: AAC +- **Quality**: CRF 23 (high quality) +- **Optimization**: Fast start enabled for web compatibility + +### **Audio Extraction:** +- **Format**: WAV (PCM 16-bit) +- **Sample Rate**: 16kHz (optimized for speech) +- **Channels**: Mono +- **Source**: Original chunk videos + +### **Processing Features:** +- **Parallel Processing**: ThreadPoolExecutor for efficient batch conversion +- **Progress Tracking**: Real-time status updates and comprehensive summary +- **Error Handling**: Graceful error handling with detailed logging +- **Input Validation**: Structure validation and dependency checks + +## ๐Ÿ“š **Documentation Created** + +1. **`utils/directory_prepare.py`** - Main script with comprehensive features +2. **`utils/DIRECTORY_PREPARE_README.md`** - Detailed usage guide +3. **Updated main `README.md`** - Added directory preparation section + +## ๐ŸŽฏ **Use Cases Enabled** + +### **1. Active Speaker Detection** +- `video_bbox/`: Use pre-drawn bounding boxes for annotation +- `video_cropped/`: Analyze face-only regions +- `audio/`: Audio-visual synchronization analysis + +### **2. Training Data Preparation** +- Organized structure for ML dataset creation +- Consistent naming convention for batch processing +- MP4 format ensures compatibility with ML frameworks + +### **3. Annotation Workflows** +- `video_bbox/`: Start with detected faces for faster annotation +- `video_cropped/`: Focus annotation on face regions only +- `audio/`: Synchronized audio for multi-modal annotation + +### **4. Quality Analysis** +- Compare good vs poor quality results side by side +- Analyze correlation between video quality and audio clarity +- Study face detection accuracy across conditions + +## ๐Ÿ” **Quality Assurance** + +- โœ… **Syntax Check**: Script compiles without errors +- โœ… **Live Testing**: Successfully processed 47 real chunks +- โœ… **Output Validation**: All 4 directories created with correct content +- โœ… **Performance**: Efficient parallel processing +- โœ… **Error Handling**: Graceful failure handling +- โœ… **Documentation**: Comprehensive usage guide + +## ๐Ÿ“‹ **Requirements** + +- **Python 3.6+** with standard libraries +- **FFmpeg** (for video conversion and audio extraction) +- **Input**: SyncNet results directory with expected structure +- **Dependencies**: Threading, subprocess, pathlib (all standard library) + +The script is production-ready and successfully organizes your SyncNet results into the exactly 4 directories you requested: `video_normal`, `video_bbox`, `video_cropped`, and `audio`! ๐ŸŽ‰ diff --git a/README.md b/README.md index 848fa7f..d7d55c4 100755 --- a/README.md +++ b/README.md @@ -177,6 +177,25 @@ This will verify that: - Quality directories are created with proper structure - SyncNet outputs are preserved in organized folders +### Directory Preparation Utility + +Organize SyncNet results into structured directories for easy access: + +```bash +# Organize filtered results into 4 directories +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized_output +``` + +This creates: +- **`video_normal/`**: Original chunk videos +- **`video_bbox/`**: Bounding box visualizations (converted to MP4) +- **`video_cropped/`**: Cropped face videos (converted to MP4) +- **`audio/`**: Extracted audio files (16kHz mono WAV) + +See [`utils/DIRECTORY_PREPARE_README.md`](utils/DIRECTORY_PREPARE_README.md) for detailed usage guide. + **Parameters:** - `--min_confidence`: Minimum SyncNet confidence score to keep video - `--max_abs_offset`: Maximum absolute frame offset to keep video diff --git a/utils/DIRECTORY_PREPARE_README.md b/utils/DIRECTORY_PREPARE_README.md new file mode 100644 index 0000000..c3a4b46 --- /dev/null +++ b/utils/DIRECTORY_PREPARE_README.md @@ -0,0 +1,243 @@ +# Directory Preparation Utility + +This utility organizes SyncNet processing results into a structured format for easy access and analysis. + +## Overview + +The `directory_prepare.py` script takes a directory containing SyncNet results (such as `results/video1/good_quality`) and reorganizes the content into 4 well-structured directories: + +1. **`video_normal/`** - Original chunk videos (all chunks) +2. **`video_bbox/`** - Bounding box visualization videos (converted to MP4) +3. **`video_cropped/`** - Cropped face videos (converted to MP4, renamed to chunk IDs) +4. **`audio/`** - Audio extracted from normal chunk videos + +## Features + +- **Parallel Processing**: Uses ThreadPoolExecutor for efficient batch conversion +- **Format Conversion**: Automatically converts AVI files to MP4 for better compatibility +- **Audio Extraction**: Extracts mono 16kHz audio optimized for speech processing +- **Progress Tracking**: Real-time progress updates and comprehensive summary +- **Error Handling**: Graceful error handling with detailed logging +- **Validation**: Input structure validation and dependency checks + +## Requirements + +- **Python 3.6+** with required packages (see main `requirements.txt`) +- **FFmpeg** - For video conversion and audio extraction + - macOS: `brew install ffmpeg` + - Linux: `apt install ffmpeg` + - Windows: Download from [ffmpeg.org](https://ffmpeg.org/) + +## Usage + +### Basic Usage + +```bash +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized_output +``` + +### Advanced Usage + +```bash +# Use more workers for faster processing +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized_output \ + --max_workers 8 +``` + +### Command-Line Arguments + +- `--input_dir` (required): Input directory containing SyncNet results +- `--output_dir` (required): Output directory where organized folders will be created +- `--max_workers` (optional): Number of parallel workers (default: 4) + +## Input Structure + +The script expects the following input structure: + +``` +input_dir/ +โ”œโ”€โ”€ chunk_000.mp4 # Original chunk videos +โ”œโ”€โ”€ chunk_001.mp4 +โ”œโ”€โ”€ ... +โ””โ”€โ”€ syncnet_outputs/ + โ”œโ”€โ”€ chunk_000/ + โ”‚ โ”œโ”€โ”€ chunk_000_with_bboxes.avi # Bounding box visualization + โ”‚ โ”œโ”€โ”€ cropped_faces/ + โ”‚ โ”‚ โ””โ”€โ”€ 00000.avi # Cropped face video + โ”‚ โ””โ”€โ”€ analysis/ + โ””โ”€โ”€ chunk_001/ + โ””โ”€โ”€ ... +``` + +## Output Structure + +The script creates the following organized structure: + +``` +output_dir/ +โ”œโ”€โ”€ video_normal/ +โ”‚ โ”œโ”€โ”€ chunk_000.mp4 # Original videos (copied) +โ”‚ โ”œโ”€โ”€ chunk_001.mp4 +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ video_bbox/ +โ”‚ โ”œโ”€โ”€ chunk_000_with_bboxes.mp4 # Bounding box videos (AVIโ†’MP4) +โ”‚ โ”œโ”€โ”€ chunk_001_with_bboxes.mp4 +โ”‚ โ””โ”€โ”€ ... +โ”œโ”€โ”€ video_cropped/ +โ”‚ โ”œโ”€โ”€ chunk_000.mp4 # Cropped faces (AVIโ†’MP4, renamed) +โ”‚ โ”œโ”€โ”€ chunk_001.mp4 +โ”‚ โ””โ”€โ”€ ... +โ””โ”€โ”€ audio/ + โ”œโ”€โ”€ chunk_000.wav # Extracted audio (16kHz mono) + โ”œโ”€โ”€ chunk_001.wav + โ””โ”€โ”€ ... +``` + +## Processing Details + +### Video Conversion (AVI โ†’ MP4) +- **Video Codec**: H.264 (libx264) +- **Audio Codec**: AAC +- **Quality**: CRF 23 (high quality) +- **Optimization**: Fast start enabled for web compatibility + +### Audio Extraction +- **Format**: WAV (PCM 16-bit) +- **Sample Rate**: 16kHz (optimized for speech) +- **Channels**: Mono +- **Use Case**: Speech recognition, audio analysis + +### Parallel Processing +- **Default Workers**: 4 (adjust based on your system) +- **Processing Order**: Chunks processed in sorted order +- **Progress Tracking**: Real-time status updates +- **Error Resilience**: Individual chunk failures don't stop the process + +## Examples + +### Process Good Quality Results + +```bash +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized/video1_good +``` + +### Process Poor Quality Results for Analysis + +```bash +python utils/directory_prepare.py \ + --input_dir results/video1/poor_quality \ + --output_dir organized/video1_poor +``` + +### High-Performance Processing + +```bash +# Use 8 workers for faster processing (adjust based on CPU cores) +python utils/directory_prepare.py \ + --input_dir results/video1/good_quality \ + --output_dir organized/video1_good \ + --max_workers 8 +``` + +## Use Cases + +### 1. **Active Speaker Detection** +- Use `video_bbox/` for annotation with pre-drawn bounding boxes +- Use `video_cropped/` for face-only analysis +- Use `audio/` for audio-visual synchronization analysis + +### 2. **Training Data Preparation** +- Organized structure makes it easy to create training datasets +- Consistent naming convention for batch processing +- MP4 format ensures compatibility with most ML frameworks + +### 3. **Quality Analysis** +- Compare good vs poor quality results side by side +- Analyze correlation between video quality and audio clarity +- Study face detection accuracy across different conditions + +### 4. **Annotation Workflows** +- `video_bbox/`: Start with pre-detected faces for faster annotation +- `video_cropped/`: Focus annotation on face regions only +- `audio/`: Synchronized audio for multi-modal annotation + +## Performance + +### Typical Processing Times +- **47 chunks** (as tested): ~19 seconds with 2 workers +- **Scaling**: Processing time scales roughly with `total_chunks / workers` +- **Bottleneck**: Usually FFmpeg conversion, not I/O + +### Resource Usage +- **CPU**: Scales with worker count (FFmpeg is CPU-intensive) +- **Memory**: Low memory usage (~100MB typical) +- **Disk**: Temporary space needed during conversion + +## Troubleshooting + +### Common Issues + +1. **FFmpeg not found** + ``` + โŒ FFmpeg not found. Please install FFmpeg to use this script. + ``` + **Solution**: Install FFmpeg using your system package manager + +2. **Input directory not found** + ``` + โŒ Input directory not found: /path/to/input + ``` + **Solution**: Check the path and ensure it exists + +3. **No syncnet_outputs directory** + ``` + ValueError: syncnet_outputs directory not found in: /path/to/input + ``` + **Solution**: Ensure you're pointing to a directory that contains SyncNet results + +4. **Conversion failures** + ``` + โŒ FFmpeg conversion failed for input.avi + ``` + **Solution**: Check that input video files are not corrupted + +### Performance Optimization + +- **Worker Count**: Set `--max_workers` to your CPU core count for optimal performance +- **Disk Speed**: Use SSD storage for better I/O performance +- **Memory**: Ensure sufficient free disk space (roughly 2x input size) + +## Integration + +This utility integrates well with: + +- **SyncNet Pipeline**: Use after running `filter_videos_by_sync_score.py` +- **Annotation Tools**: Organized structure works with most video annotation software +- **ML Frameworks**: MP4 and WAV formats are widely supported +- **Batch Processing**: Easy to incorporate into larger processing pipelines + +## Output Validation + +The script provides comprehensive logging and summary statistics: + +``` +๐Ÿ“Š DIRECTORY PREPARATION SUMMARY +================================================================================ +Total chunks processed: 47 +โœ… Normal videos: 47/47 +๐Ÿ“ฆ Bbox videos: 47/47 +๐Ÿ‘ค Cropped videos: 47/47 +๐Ÿ”Š Audio files: 47/47 + +๐Ÿ“ Output directories: + video_normal: test_organized/video_normal (47 files) + video_bbox: test_organized/video_bbox (47 files) + video_cropped: test_organized/video_cropped (47 files) + audio: test_organized/audio (47 files) +``` diff --git a/utils/directory_prepare.py b/utils/directory_prepare.py new file mode 100644 index 0000000..78784d5 --- /dev/null +++ b/utils/directory_prepare.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +""" +Directory Preparation Script for SyncNet Results + +This script takes a directory containing SyncNet results (e.g., results/video1/good_quality) +and organizes the content into 4 structured directories with optional video ID prefixing: + +1. video_normal - Original chunk videos (all chunks) +2. video_bbox - Bounding box visualization videos (converted to MP4) +3. video_cropped - Cropped face videos (converted to MP4, renamed to chunk IDs) +4. audio - Audio extracted from normal chunk videos + +The script can automatically prefix all output files with a video ID (e.g., YouTube ID) +to create names like "h9OrOkhODmY_chunk_000.mp4" instead of "chunk_000.mp4". + +Usage: + python directory_prepare.py --input_dir results/video1/good_quality --output_dir h9OrOkhODmY + python directory_prepare.py --input_dir results/video1/good_quality --output_dir organized --video_id h9OrOkhODmY +""" + +import os +import sys +import argparse +import subprocess +import shutil +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + + +class DirectoryPreparer: + """ + Organizes SyncNet results into structured directories + """ + + def __init__(self, input_dir, output_dir, max_workers=4, video_id=None): + """ + Initialize the directory preparer + + Args: + input_dir: Path to input directory (e.g., results/video1/good_quality) + output_dir: Path to output directory where organized folders will be created + max_workers: Number of parallel workers for processing + video_id: Optional video ID to use as prefix for chunk names (e.g., YouTube ID) + """ + self.input_dir = Path(input_dir) + self.output_dir = Path(output_dir) + self.max_workers = max_workers + self.video_id = video_id or Path(output_dir).name # Use output_dir name as default ID + + # Define output subdirectories + self.video_normal_dir = self.output_dir / "video_normal" + self.video_bbox_dir = self.output_dir / "video_bbox" + self.video_cropped_dir = self.output_dir / "video_cropped" + self.audio_dir = self.output_dir / "audio" + + # Validate input + self._validate_input() + + logger.info(f"๐ŸŽฏ Using video ID prefix: {self.video_id}") + + def _get_output_name(self, chunk_name): + """ + Generate output name with video ID prefix + + Args: + chunk_name: Original chunk name (e.g., 'chunk_000') + + Returns: + str: Prefixed name (e.g., 'h9OrOkhODmY_chunk_000') + """ + return f"{self.video_id}_{chunk_name}" + + def _validate_input(self): + """Validate input directory structure""" + if not self.input_dir.exists(): + raise ValueError(f"Input directory does not exist: {self.input_dir}") + + # Check for expected structure + syncnet_outputs = self.input_dir / "syncnet_outputs" + if not syncnet_outputs.exists(): + raise ValueError(f"syncnet_outputs directory not found in: {self.input_dir}") + + logger.info(f"โœ… Input directory validated: {self.input_dir}") + + def _create_output_directories(self): + """Create all output directories""" + directories = [ + self.video_normal_dir, + self.video_bbox_dir, + self.video_cropped_dir, + self.audio_dir + ] + + for directory in directories: + directory.mkdir(parents=True, exist_ok=True) + logger.info(f"๐Ÿ“ Created directory: {directory}") + + def _get_chunk_list(self): + """Get list of all chunk directories""" + syncnet_outputs = self.input_dir / "syncnet_outputs" + chunks = [] + + for item in syncnet_outputs.iterdir(): + if item.is_dir() and item.name.startswith('chunk_'): + chunks.append(item.name) + + chunks.sort() # Sort to ensure consistent processing order + logger.info(f"๐Ÿ“‹ Found {len(chunks)} chunks to process") + return chunks + + def _convert_avi_to_mp4(self, input_path, output_path): + """ + Convert AVI video to MP4 using ffmpeg + + Args: + input_path: Path to input AVI file + output_path: Path to output MP4 file + """ + try: + cmd = [ + 'ffmpeg', + '-i', str(input_path), + '-c:v', 'libx264', # Video codec + '-c:a', 'aac', # Audio codec + '-preset', 'medium', # Encoding speed/quality balance + '-crf', '23', # Quality setting (18-28 is good range) + '-movflags', '+faststart', # Enable fast start for web + '-y', # Overwrite output file + str(output_path) + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + return True + + except subprocess.CalledProcessError as e: + logger.error(f"โŒ FFmpeg conversion failed for {input_path}: {e.stderr}") + return False + except Exception as e: + logger.error(f"โŒ Unexpected error converting {input_path}: {e}") + return False + + def _extract_audio(self, input_path, output_path): + """ + Extract audio from video file + + Args: + input_path: Path to input video file + output_path: Path to output audio file + """ + try: + cmd = [ + 'ffmpeg', + '-i', str(input_path), + '-vn', # No video + '-acodec', 'pcm_s16le', # Audio codec + '-ar', '16000', # Sample rate (16kHz for speech) + '-ac', '1', # Mono channel + '-y', # Overwrite output file + str(output_path) + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + return True + + except subprocess.CalledProcessError as e: + logger.error(f"โŒ Audio extraction failed for {input_path}: {e.stderr}") + return False + except Exception as e: + logger.error(f"โŒ Unexpected error extracting audio from {input_path}: {e}") + return False + + def _process_chunk_normal_video(self, chunk_name): + """Process normal video for a chunk""" + input_video = self.input_dir / f"{chunk_name}.mp4" + output_name = self._get_output_name(chunk_name) + output_video = self.video_normal_dir / f"{output_name}.mp4" + + if input_video.exists(): + try: + shutil.copy2(input_video, output_video) + logger.info(f"โœ… Copied normal video: {output_name}.mp4") + return True + except Exception as e: + logger.error(f"โŒ Failed to copy normal video {chunk_name}: {e}") + return False + else: + logger.warning(f"โš ๏ธ Normal video not found: {input_video}") + return False + + def _process_chunk_bbox_video(self, chunk_name): + """Process bounding box video for a chunk""" + input_bbox = self.input_dir / "syncnet_outputs" / chunk_name / f"{chunk_name}_with_bboxes.avi" + output_name = self._get_output_name(chunk_name) + output_bbox = self.video_bbox_dir / f"{output_name}_with_bboxes.mp4" + + if input_bbox.exists(): + success = self._convert_avi_to_mp4(input_bbox, output_bbox) + if success: + logger.info(f"โœ… Converted bbox video: {output_name}_with_bboxes.mp4") + return True + else: + logger.error(f"โŒ Failed to convert bbox video: {chunk_name}") + return False + else: + logger.warning(f"โš ๏ธ Bbox video not found: {input_bbox}") + return False + + def _process_chunk_cropped_video(self, chunk_name): + """Process cropped face video for a chunk""" + input_cropped = self.input_dir / "syncnet_outputs" / chunk_name / "cropped_faces" / "00000.avi" + output_name = self._get_output_name(chunk_name) + output_cropped = self.video_cropped_dir / f"{output_name}.mp4" + + if input_cropped.exists(): + success = self._convert_avi_to_mp4(input_cropped, output_cropped) + if success: + logger.info(f"โœ… Converted cropped video: {output_name}.mp4") + return True + else: + logger.error(f"โŒ Failed to convert cropped video: {chunk_name}") + return False + else: + logger.warning(f"โš ๏ธ Cropped video not found: {input_cropped}") + return False + + def _process_chunk_audio(self, chunk_name): + """Extract audio from normal video for a chunk""" + input_video = self.input_dir / f"{chunk_name}.mp4" + output_name = self._get_output_name(chunk_name) + output_audio = self.audio_dir / f"{output_name}.wav" + + if input_video.exists(): + success = self._extract_audio(input_video, output_audio) + if success: + logger.info(f"โœ… Extracted audio: {output_name}.wav") + return True + else: + logger.error(f"โŒ Failed to extract audio: {chunk_name}") + return False + else: + logger.warning(f"โš ๏ธ Normal video not found for audio extraction: {input_video}") + return False + + def _process_single_chunk(self, chunk_name): + """Process all outputs for a single chunk""" + results = { + 'chunk': chunk_name, + 'normal_video': False, + 'bbox_video': False, + 'cropped_video': False, + 'audio': False + } + + # Process each type + results['normal_video'] = self._process_chunk_normal_video(chunk_name) + results['bbox_video'] = self._process_chunk_bbox_video(chunk_name) + results['cropped_video'] = self._process_chunk_cropped_video(chunk_name) + results['audio'] = self._process_chunk_audio(chunk_name) + + return results + + def prepare_directories(self): + """ + Main method to prepare all directories + + Returns: + dict: Summary of processing results + """ + logger.info(f"๐Ÿš€ Starting directory preparation") + logger.info(f"๐Ÿ“ฅ Input: {self.input_dir}") + logger.info(f"๐Ÿ“ค Output: {self.output_dir}") + + # Create output directories + self._create_output_directories() + + # Get list of chunks to process + chunks = self._get_chunk_list() + + if not chunks: + logger.warning("โš ๏ธ No chunks found to process") + return {'total_chunks': 0, 'results': []} + + # Process chunks in parallel + results = [] + completed = 0 + + logger.info(f"๐Ÿ”„ Processing {len(chunks)} chunks with {self.max_workers} workers...") + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + # Submit all jobs + future_to_chunk = { + executor.submit(self._process_single_chunk, chunk): chunk + for chunk in chunks + } + + # Collect results + for future in as_completed(future_to_chunk): + chunk = future_to_chunk[future] + + try: + result = future.result() + results.append(result) + completed += 1 + + # Show progress + success_count = sum([ + result['normal_video'], + result['bbox_video'], + result['cropped_video'], + result['audio'] + ]) + + logger.info(f"๐Ÿ“Š [{completed}/{len(chunks)}] {chunk}: {success_count}/4 operations successful") + + except Exception as e: + logger.error(f"โŒ [{completed+1}/{len(chunks)}] {chunk}: Exception - {e}") + completed += 1 + + # Generate summary + summary = self._generate_summary(results) + + return summary + + def _generate_summary(self, results): + """Generate processing summary""" + total_chunks = len(results) + + counts = { + 'normal_video': sum(r['normal_video'] for r in results), + 'bbox_video': sum(r['bbox_video'] for r in results), + 'cropped_video': sum(r['cropped_video'] for r in results), + 'audio': sum(r['audio'] for r in results) + } + + summary = { + 'total_chunks': total_chunks, + 'successful_operations': counts, + 'results': results, + 'output_directories': { + 'video_normal': str(self.video_normal_dir), + 'video_bbox': str(self.video_bbox_dir), + 'video_cropped': str(self.video_cropped_dir), + 'audio': str(self.audio_dir) + } + } + + # Print summary + logger.info(f"\n{'='*80}") + logger.info("๐Ÿ“Š DIRECTORY PREPARATION SUMMARY") + logger.info(f"{'='*80}") + logger.info(f"Total chunks processed: {total_chunks}") + logger.info(f"โœ… Normal videos: {counts['normal_video']}/{total_chunks}") + logger.info(f"๐Ÿ“ฆ Bbox videos: {counts['bbox_video']}/{total_chunks}") + logger.info(f"๐Ÿ‘ค Cropped videos: {counts['cropped_video']}/{total_chunks}") + logger.info(f"๐Ÿ”Š Audio files: {counts['audio']}/{total_chunks}") + + logger.info(f"\n๐Ÿ“ Output directories:") + for name, path in summary['output_directories'].items(): + file_count = len(list(Path(path).glob('*'))) if Path(path).exists() else 0 + logger.info(f" {name}: {path} ({file_count} files)") + + return summary + + +def main(): + """Main function""" + parser = argparse.ArgumentParser( + description="Prepare SyncNet results into organized directory structure", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic usage (uses output directory name as video ID) + python directory_prepare.py --input_dir results/video1/good_quality --output_dir h9OrOkhODmY + + # With explicit video ID + python directory_prepare.py --input_dir results/video1/good_quality --output_dir organized_output --video_id h9OrOkhODmY + + # With custom worker count + python directory_prepare.py --input_dir results/video1/good_quality --output_dir h9OrOkhODmY --max_workers 8 + """ + ) + + parser.add_argument( + '--input_dir', + required=True, + help='Input directory containing SyncNet results (e.g., results/video1/good_quality)' + ) + + parser.add_argument( + '--output_dir', + required=True, + help='Output directory where organized folders will be created' + ) + + parser.add_argument( + '--max_workers', + type=int, + default=4, + help='Number of parallel workers for processing (default: 4)' + ) + + parser.add_argument( + '--video_id', + help='Video ID to use as prefix for chunk names (default: uses output directory name)' + ) + + args = parser.parse_args() + + # Validate inputs + if not os.path.exists(args.input_dir): + logger.error(f"โŒ Input directory not found: {args.input_dir}") + sys.exit(1) + + # Check ffmpeg availability + try: + subprocess.run(['ffmpeg', '-version'], capture_output=True, check=True) + except (subprocess.CalledProcessError, FileNotFoundError): + logger.error("โŒ FFmpeg not found. Please install FFmpeg to use this script.") + logger.error(" Install with: brew install ffmpeg (macOS) or apt install ffmpeg (Linux)") + sys.exit(1) + + # Create preparer and run + try: + preparer = DirectoryPreparer( + input_dir=args.input_dir, + output_dir=args.output_dir, + max_workers=args.max_workers, + video_id=args.video_id + ) + + summary = preparer.prepare_directories() + + logger.info(f"\n๐ŸŽ‰ Directory preparation completed successfully!") + logger.info(f"๐Ÿ“ Results available in: {args.output_dir}") + + except Exception as e: + logger.error(f"โŒ Directory preparation failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() From 3a3fd9c4c4613f3184aa32afba71632ab8dc6573 Mon Sep 17 00:00:00 2001 From: Moon1570 Date: Mon, 22 Sep 2025 23:25:16 +0600 Subject: [PATCH 11/11] chore: moved data & updated doc --- Docs/QUALITY_FILTERING_GUIDE.md | 2 +- filter_videos_by_sync_score.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/QUALITY_FILTERING_GUIDE.md b/Docs/QUALITY_FILTERING_GUIDE.md index c2ae788..eb75077 100644 --- a/Docs/QUALITY_FILTERING_GUIDE.md +++ b/Docs/QUALITY_FILTERING_GUIDE.md @@ -32,7 +32,7 @@ Using strict quality filters (confidence โ‰ฅ 4.0, |offset| โ‰ค 5 frames): | Preset | Min Confidence | Max |Offset| | Use Case | |--------|---------------|--------------|----------| | `strict` | โ‰ฅ6.0 | โ‰ค2 frames | Publication-ready, highest quality | -| `high` | โ‰ฅ4.0 | โ‰ค5 frames | Training data, good quality | +| `high` | โ‰ฅ5.0 | โ‰ค5 frames | Training data, good quality | | `medium` | โ‰ฅ2.5 | โ‰ค8 frames | Balanced approach | | `relaxed` | โ‰ฅ1.5 | โ‰ค12 frames | Keep most usable chunks | | `none` | โ‰ฅ0.0 | โ‰ค50 frames | No filtering | diff --git a/filter_videos_by_sync_score.py b/filter_videos_by_sync_score.py index 5d0a939..0ede775 100644 --- a/filter_videos_by_sync_score.py +++ b/filter_videos_by_sync_score.py @@ -484,7 +484,7 @@ def main(): if args.preset: presets = { 'strict': {'min_confidence': 8.0, 'max_abs_offset': 2}, - 'high': {'min_confidence': 6.0, 'max_abs_offset': 3}, + 'high': {'min_confidence': 5.0, 'max_abs_offset': 4}, 'medium': {'min_confidence': 4.0, 'max_abs_offset': 5}, 'relaxed': {'min_confidence': 2.0, 'max_abs_offset': 8} }