From dae7abec1d1d97827337d1e8b692ec5ce5e8f5e5 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Fri, 28 Feb 2020 12:05:50 +0000
Subject: [PATCH 1/5] change data structure and video loader

---
 SyncNetInstance.py |  47 ++++++++-----
 demo_syncnet.py    |   3 +-
 run_pipeline.py    | 170 ++++++++++++++++++++++++++-------------------
 run_syncnet.py     |  23 ++----
 run_visualise.py   |  69 ++++++++----------
 5 files changed, 164 insertions(+), 148 deletions(-)

diff --git a/SyncNetInstance.py b/SyncNetInstance.py
index 44ce49e..497d44f 100644
--- a/SyncNetInstance.py
+++ b/SyncNetInstance.py
@@ -4,13 +4,14 @@
 
 import torch
 import numpy
-import time, pdb, argparse, subprocess, os
+import time, pdb, argparse, subprocess, os, math, glob
 import cv2
 import python_speech_features
 
 from scipy import signal
 from scipy.io import wavfile
 from SyncNetModel import *
+from shutil import rmtree
 
 
 # ==================== Get OFFSET ====================
@@ -41,21 +42,33 @@ def __init__(self, dropout = 0, num_layers_in_fc_layers = 1024):
     def evaluate(self, opt, videofile):
 
         self.__S__.eval();
+
+        # ========== ==========
+        # Convert files
+        # ========== ==========
+
+        if os.path.exists(os.path.join(opt.tmp_dir,opt.reference)):
+          rmtree(os.path.join(opt.tmp_dir,opt.reference))
+
+        os.makedirs(os.path.join(opt.tmp_dir,opt.reference))
+
+        command = ("ffmpeg -y -i %s -threads 1 -f image2 %s" % (videofile,os.path.join(opt.tmp_dir,opt.reference,'%06d.jpg'))) 
+        output = subprocess.call(command, shell=True, stdout=None)
+
+        command = ("ffmpeg -y -i %s -async 1 -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (videofile,os.path.join(opt.tmp_dir,opt.reference,'audio.wav'))) 
+        output = subprocess.call(command, shell=True, stdout=None)
         
         # ========== ==========
         # Load video 
         # ========== ==========
-        cap = cv2.VideoCapture(videofile)
 
-        frame_num = 1;
         images = []
-        while frame_num:
-            frame_num += 1
-            ret, image = cap.read()
-            if ret == 0:
-                break
+        
+        flist = glob.glob(os.path.join(opt.tmp_dir,opt.reference,'*.jpg'))
+        flist.sort()
 
-            images.append(image)
+        for fname in flist:
+            images.append(cv2.imread(fname))
 
         im = numpy.stack(images,axis=3)
         im = numpy.expand_dims(im,axis=0)
@@ -67,12 +80,7 @@ def evaluate(self, opt, videofile):
         # Load audio
         # ========== ==========
 
-        audiotmp = os.path.join(opt.tmp_dir,'audio.wav')
-
-        command = ("ffmpeg -y -i %s -async 1 -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (videofile,audiotmp))
-        output = subprocess.call(command, shell=True, stdout=None)
-
-        sample_rate, audio = wavfile.read(audiotmp)
+        sample_rate, audio = wavfile.read(os.path.join(opt.tmp_dir,opt.reference,'audio.wav'))
         mfcc = zip(*python_speech_features.mfcc(audio,sample_rate))
         mfcc = numpy.stack([numpy.array(i) for i in mfcc])
 
@@ -83,15 +91,16 @@ def evaluate(self, opt, videofile):
         # Check audio and video input length
         # ========== ==========
 
-        if (float(len(audio))/16000) < (float(len(images))/25) :
-            print(" *** WARNING: The audio (%.4fs) is shorter than the video (%.4fs). Type 'cont' to continue. *** "%(float(len(audio))/16000,float(len(images))/25))
-            pdb.set_trace()
+        if (float(len(audio))/16000) != (float(len(images))/25) :
+            print("WARNING: Audio (%.4fs) and video (%.4fs) lengths are different."%(float(len(audio))/16000,float(len(images))/25))
+
+        min_length = min(len(images),math.floor(len(audio)/640))
         
         # ========== ==========
         # Generate video and audio feats
         # ========== ==========
 
-        lastframe = len(images)-5
+        lastframe = min_length-5
         im_feat = []
         cc_feat = []
 
diff --git a/demo_syncnet.py b/demo_syncnet.py
index 1d72b20..01c25a6 100755
--- a/demo_syncnet.py
+++ b/demo_syncnet.py
@@ -14,7 +14,8 @@
 parser.add_argument('--batch_size', type=int, default='20', help='');
 parser.add_argument('--vshift', type=int, default='15', help='');
 parser.add_argument('--videofile', type=str, default="data/example.avi", help='');
-parser.add_argument('--tmp_dir', type=str, default="data", help='');
+parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help='');
+parser.add_argument('--reference', type=str, default="demo", help='');
 
 opt = parser.parse_args();
 
diff --git a/run_pipeline.py b/run_pipeline.py
index 293b4c2..b4ae02b 100755
--- a/run_pipeline.py
+++ b/run_pipeline.py
@@ -1,9 +1,10 @@
 #!/usr/bin/python
 
-import sys, time, os, pdb, argparse, pickle, subprocess
+import sys, time, os, pdb, argparse, pickle, subprocess, glob
 import numpy as np
 import tensorflow as tf
 import cv2
+from shutil import rmtree
 
 import scenedetect
 from scenedetect.video_manager import VideoManager
@@ -22,17 +23,21 @@
 # ========== ========== ========== ==========
 
 parser = argparse.ArgumentParser(description = "FaceTracker");
-parser.add_argument('--data_dir', type=str, default='data/work', help='Output direcotry');
-parser.add_argument('--videofile', type=str, default='', help='Input video file');
-parser.add_argument('--reference', type=str, default='', help='Name of the video');
-parser.add_argument('--crop_scale', type=float, default=0.5, help='Scale bounding box');
-parser.add_argument('--min_track', type=int, default=100, help='Minimum facetrack duration');
+parser.add_argument('--data_dir',       type=str, default='data/work', help='Output direcotry');
+parser.add_argument('--videofile',      type=str, default='', help='Input video file');
+parser.add_argument('--reference',      type=str, default='', help='Name of the video');
+parser.add_argument('--crop_scale',     type=float, default=0.5, help='Scale bounding box');
+parser.add_argument('--min_track',      type=int, default=100, help='Minimum facetrack duration');
+parser.add_argument('--frame_rate',     type=int, default=25, help='Frame rate');
+parser.add_argument('--num_failed_det', type=int, default=25, help='Number of missed detections allowed');
+parser.add_argument('--min_face_size',  type=float, default=0.03, help='Minimum size of faces');
 opt = parser.parse_args();
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
 setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp'))
 setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork'))
 setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop'))
+setattr(opt,'frames_dir',os.path.join(opt.data_dir,'pyframes'))
 
 # ========== ========== ========== ==========
 # # IOU FUNCTION
@@ -61,34 +66,30 @@ def bb_intersection_over_union(boxA, boxB):
 def track_shot(opt,scenefaces):
 
   iouThres  = 0.5     # Minimum IOU between consecutive face detections
-  numFail   = 3       # Number of missed detections allowed
-  minSize   = 0.05    # Minimum size of faces
   tracks    = []
 
   while True:
     track     = []
-    for faces in scenefaces:
-      for face in faces:
+    for framefaces in scenefaces:
+      for face in framefaces:
         if track == []:
           track.append(face)
-          faces.remove(face)
-        elif face[0] - track[-1][0] <= numFail:
-          iou = bb_intersection_over_union(face[1], track[-1][1])
+          framefaces.remove(face)
+        elif face['frame'] - track[-1]['frame'] <= opt.num_failed_det:
+          iou = bb_intersection_over_union(face['bbox'], track[-1]['bbox'])
           if iou > iouThres:
             track.append(face)
-            faces.remove(face)
+            framefaces.remove(face)
             continue
         else:
           break
 
-    
-
     if track == []:
       break
     elif len(track) > opt.min_track:
       
-      framenum    = np.array([ f[0] for f in track ])
-      bboxes    = np.array([np.array(f[1]) for f in track])
+      framenum    = np.array([ f['frame'] for f in track ])
+      bboxes      = np.array([np.array(f['bbox']) for f in track])
 
       frame_i   = np.arange(framenum[0],framenum[-1]+1)
 
@@ -98,9 +99,8 @@ def track_shot(opt,scenefaces):
         bboxes_i.append(interpfn(frame_i))
       bboxes_i  = np.stack(bboxes_i, axis=1)
 
-      if np.mean(bboxes_i[:,3]-bboxes_i[:,1]) > minSize:
-        tracks.append([frame_i,bboxes_i])
-
+      if np.mean(bboxes_i[:,3]-bboxes_i[:,1]) > opt.min_face_size:
+        tracks.append({'frame':frame_i,'bbox':bboxes_i})
 
   return tracks
 
@@ -110,57 +110,56 @@ def track_shot(opt,scenefaces):
         
 def crop_video(opt,track,cropfile):
 
-  cap = cv2.VideoCapture(os.path.join(opt.avi_dir,opt.reference,'video.avi'))
-
-  total_frames = cap.get(7)
-  cap.set(1,track[0][0]) # CHANGE THIS !!!
+  flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg'))
+  flist.sort()
 
   fourcc = cv2.VideoWriter_fourcc(*'XVID')
-  vOut = cv2.VideoWriter(cropfile+'t.avi', fourcc, cap.get(5), (224,224))
+  vOut = cv2.VideoWriter(cropfile+'t.avi', fourcc, opt.frame_rate, (224,224))
+
+  first_image = cv2.imread(flist[track['frame'][0]])
 
-  fw = cap.get(3)
-  fh = cap.get(4)
+  fw = first_image.shape[1]
+  fh = first_image.shape[0]
 
-  dets = [[], [], []]
+  dets = {'x':[], 'y':[], 's':[]}
 
-  for det in track[1]:
+  for det in track['bbox']:
 
-    dets[0].append(((det[3]-det[1])*fw+(det[2]-det[0])*fh)/4) # H+W / 4
-    dets[1].append((det[1]+det[3])*fw/2) # crop center x 
-    dets[2].append((det[0]+det[2])*fh/2) # crop center y
+    dets['s'].append(((det[3]-det[1])*fw+(det[2]-det[0])*fh)/4) # H+W / 4
+    dets['x'].append((det[1]+det[3])*fw/2) # crop center x 
+    dets['y'].append((det[0]+det[2])*fh/2) # crop center y
 
   # Smooth detections
-  dets[0] = signal.medfilt(dets[0],kernel_size=5)   
-  dets[1] = signal.medfilt(dets[1],kernel_size=5)
-  dets[2] = signal.medfilt(dets[2],kernel_size=7)
+  dets['s'] = signal.medfilt(dets['s'],kernel_size=7)   
+  dets['x'] = signal.medfilt(dets['x'],kernel_size=5)
+  dets['y'] = signal.medfilt(dets['y'],kernel_size=5)
 
-  for det in zip(*dets):
+  for fidx, frame in enumerate(track['frame']):
 
     cs  = opt.crop_scale
 
-    bs  = det[0]            # Detection box size
+    bs  = dets['s'][fidx]   # Detection box size
     bsi = int(bs*(1+2*cs))  # Pad videos by this amount 
 
-    ret, frame = cap.read()
+    image = cv2.imread(flist[frame])
     
-    frame = np.pad(frame,((bsi,bsi),(bsi,bsi),(0,0)), 'constant', constant_values=(0,0))
-    my  = det[2]+bsi  # BBox center Y
-    mx  = det[1]+bsi  # BBox center X
+    frame = np.pad(image,((bsi,bsi),(bsi,bsi),(0,0)), 'constant', constant_values=(0,0))
+    my  = dets['y'][fidx]+bsi  # BBox center Y
+    mx  = dets['x'][fidx]+bsi  # BBox center X
 
     face = frame[int(my-bs):int(my+bs*(1+2*cs)),int(mx-bs*(1+cs)):int(mx+bs*(1+cs))]
     
     vOut.write(cv2.resize(face,(224,224)))
 
-  audiotmp  = os.path.join(opt.tmp_dir,opt.reference,'audio.wav')
-  audiostart  = track[0][0]/cap.get(5)
-  audioend  = (track[0][-1]+1)/cap.get(5)
+  audiotmp    = os.path.join(opt.tmp_dir,opt.reference,'audio.wav')
+  audiostart  = (track['frame'][0])/opt.frame_rate
+  audioend    = (track['frame'][-1]+1)/opt.frame_rate
 
-  cap.release()
   vOut.release()
 
   # ========== CROP AUDIO FILE ==========
 
-  command = ("ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 -ss %.3f -to %.3f %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),audiostart,audioend,audiotmp)) #-async 1 
+  command = ("ffmpeg -y -i %s -ss %.3f -to %.3f %s" % (os.path.join(opt.avi_dir,opt.reference,'audio.wav'),audiostart,audioend,audiotmp)) 
   output = subprocess.call(command, shell=True, stdout=None)
 
   if output != 0:
@@ -170,7 +169,7 @@ def crop_video(opt,track,cropfile):
 
   # ========== COMBINE AUDIO AND VIDEO FILES ==========
 
-  command = ("ffmpeg -y -i %st.avi -i %s -c:v copy -c:a copy %s.avi" % (cropfile,audiotmp,cropfile)) #-async 1 
+  command = ("ffmpeg -y -i %st.avi -i %s -c:v copy -c:a copy %s.avi" % (cropfile,audiotmp,cropfile))
   output = subprocess.call(command, shell=True, stdout=None)
 
   if output != 0:
@@ -180,8 +179,7 @@ def crop_video(opt,track,cropfile):
 
   os.remove(cropfile+'t.avi')
 
-  return [track,dets]
-
+  return {'track':track, 'proc_track':dets}
 
 # ========== ========== ========== ==========
 # # FACE DETECTION
@@ -208,7 +206,8 @@ def load_image_into_numpy_array(image):
     return np.array(image.getdata()).reshape(
         (im_height, im_width, 3)).astype(np.uint8)
 
-  cap = cv2.VideoCapture(os.path.join(opt.avi_dir,opt.reference,'video.avi'))
+  flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg'))
+  flist.sort()
 
   detection_graph = tf.Graph()
   with detection_graph.as_default():
@@ -224,12 +223,10 @@ def load_image_into_numpy_array(image):
     config = tf.ConfigProto()
     config.gpu_options.allow_growth = True
     with tf.Session(graph=detection_graph, config=config) as sess:
-      frame_num = 0;
-      while True:
+      
+      for fidx, fname in enumerate(flist):
         
-        ret, image = cap.read()
-        if ret == 0:
-            break
+        image = cv2.imread(fname)
 
         image_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 
@@ -254,12 +251,9 @@ def load_image_into_numpy_array(image):
         dets.append([]);
         for index in range(0,len(score)):
           if score[index] > MIN_CONF:
-            dets[-1].append([frame_num, boxes[0][index].tolist(),score[index]])
-
-        print('%s-%05d; %d dets; %.2f Hz' % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),frame_num,len(dets[-1]),(1/elapsed_time))) 
-        frame_num += 1
+            dets[-1].append({'frame':fidx, 'bbox':boxes[0][index].tolist(), 'conf':score[index]})
 
-      cap.release()
+        print('%s-%05d; %d dets; %.2f Hz' % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),fidx,len(dets[-1]),(1/elapsed_time))) 
 
       savepath = os.path.join(opt.work_dir,opt.reference,'faces.pckl')
 
@@ -306,27 +300,52 @@ def scene_detect(opt):
 # # EXECUTE DEMO
 # ========== ========== ========== ==========
 
-if not(os.path.exists(os.path.join(opt.work_dir,opt.reference))):
-  os.makedirs(os.path.join(opt.work_dir,opt.reference))
+# ========== DELETE EXISTING DIRECTORIES ==========
+
+if os.path.exists(os.path.join(opt.work_dir,opt.reference)):
+  rmtree(os.path.join(opt.work_dir,opt.reference))
+
+if os.path.exists(os.path.join(opt.crop_dir,opt.reference)):
+  rmtree(os.path.join(opt.crop_dir,opt.reference))
+
+if os.path.exists(os.path.join(opt.avi_dir,opt.reference)):
+  rmtree(os.path.join(opt.avi_dir,opt.reference))
+
+if os.path.exists(os.path.join(opt.frames_dir,opt.reference)):
+  rmtree(os.path.join(opt.frames_dir,opt.reference))
 
-if not(os.path.exists(os.path.join(opt.crop_dir,opt.reference))):
-  os.makedirs(os.path.join(opt.crop_dir,opt.reference))
+if os.path.exists(os.path.join(opt.tmp_dir,opt.reference)):
+  rmtree(os.path.join(opt.tmp_dir,opt.reference))
 
-if not(os.path.exists(os.path.join(opt.avi_dir,opt.reference))):
-  os.makedirs(os.path.join(opt.avi_dir,opt.reference))
+# ========== MAKE NEW DIRECTORIES ==========
 
-if not(os.path.exists(os.path.join(opt.tmp_dir,opt.reference))):
-  os.makedirs(os.path.join(opt.tmp_dir,opt.reference))
+os.makedirs(os.path.join(opt.work_dir,opt.reference))
+os.makedirs(os.path.join(opt.crop_dir,opt.reference))
+os.makedirs(os.path.join(opt.avi_dir,opt.reference))
+os.makedirs(os.path.join(opt.frames_dir,opt.reference))
+os.makedirs(os.path.join(opt.tmp_dir,opt.reference))
 
-command = ("ffmpeg -y -i %s -qscale:v 4 -r 25 %s" % (opt.videofile,os.path.join(opt.avi_dir,opt.reference,'video.avi'))) #-async 1  -deinterlace
+# ========== CONVERT VIDEO AND EXTRACT FRAMES ==========
+
+command = ("ffmpeg -y -i %s -async 1 -qscale:v 4 -r 25 %s" % (opt.videofile,os.path.join(opt.avi_dir,opt.reference,'video.avi'))) #-async 1  -deinterlace
+output = subprocess.call(command, shell=True, stdout=None)
+
+command = ("ffmpeg -y -i %s -threads 1 -f image2 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.frames_dir,opt.reference,'%06d.jpg'))) 
 output = subprocess.call(command, shell=True, stdout=None)
+
+command = ("ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.avi_dir,opt.reference,'audio.wav'))) 
+output = subprocess.call(command, shell=True, stdout=None)
+
+# ========== FACE DETECTION ==========
+
 faces = inference_video(opt)
 
-# with open(os.path.join(opt.work_dir,opt.reference,'faces.pckl'), 'rb') as fil:
-#   faces = pickle.load(fil, encoding='latin1')
+# ========== SCENE DETECTION ==========
 
 scene = scene_detect(opt)
 
+# ========== FACE TRACKING ==========
+
 alltracks = []
 vidtracks = []
 
@@ -335,11 +354,16 @@ def scene_detect(opt):
   if shot[1].frame_num - shot[0].frame_num >= opt.min_track :
     alltracks.extend(track_shot(opt,faces[shot[0].frame_num:shot[1].frame_num]))
 
-for ii, track in enumerate(alltracks):
+# ========== FACE TRACK CROP ==========
 
+for ii, track in enumerate(alltracks):
   vidtracks.append(crop_video(opt,track,os.path.join(opt.crop_dir,opt.reference,'%05d'%ii)))
 
+# ========== SAVE RESULTS ==========
+
 savepath = os.path.join(opt.work_dir,opt.reference,'tracks.pckl')
 
 with open(savepath, 'wb') as fil:
   pickle.dump(vidtracks, fil)
+
+rmtree(os.path.join(opt.tmp_dir,opt.reference))
diff --git a/run_syncnet.py b/run_syncnet.py
index 6532059..45099fd 100755
--- a/run_syncnet.py
+++ b/run_syncnet.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 #-*- coding: utf-8 -*-
 
-import time, pdb, argparse, subprocess, pickle, os, gzip
+import time, pdb, argparse, subprocess, pickle, os, gzip, glob
 
 from SyncNetInstance import *
 
@@ -22,33 +22,24 @@
 setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop'))
 
 
-# ==================== LOAD MODEL ====================
+# ==================== LOAD MODEL AND FILE LIST ====================
 
 s = SyncNetInstance();
 
 s.loadParameters(opt.initial_model);
 print("Model %s loaded."%opt.initial_model);
 
-# ==================== GET OFFSETS ====================
+flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi'))
+flist.sort()
 
-with open(os.path.join(opt.work_dir,opt.reference,'tracks.pckl'), 'rb') as fil:
-    tracks = pickle.load(fil, encoding='latin1')
+# ==================== GET OFFSETS ====================
 
 dists = []
-offsets = []
-confs = []
-for ii, track in enumerate(tracks):
-    offset, conf, dist = s.evaluate(opt,videofile=os.path.join(opt.crop_dir,opt.reference,'%05d.avi'%ii))
-    offsets.append(offset)
+for idx, fname in enumerate(flist):
+    offset, conf, dist = s.evaluate(opt,videofile=fname)
     dists.append(dist)
-    confs.append(conf)
       
 # ==================== PRINT RESULTS TO FILE ====================
 
-with open(os.path.join(opt.work_dir,opt.reference,'offsets.txt'), 'w') as fil:
-    fil.write('FILENAME\tOFFSET\tCONF\n')
-    for ii, track in enumerate(tracks):
-      fil.write('%05d.avi\t%d\t%.3f\n'%(ii, offsets[ii], confs[ii]))
-      
 with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil:
     pickle.dump(dists, fil)
diff --git a/run_visualise.py b/run_visualise.py
index 198c5fd..85d8925 100644
--- a/run_visualise.py
+++ b/run_visualise.py
@@ -3,7 +3,7 @@
 
 import torch
 import numpy
-import time, pdb, argparse, subprocess, pickle, os
+import time, pdb, argparse, subprocess, pickle, os, glob
 import cv2
 
 from scipy import signal
@@ -11,19 +11,17 @@
 # ==================== PARSE ARGUMENT ====================
 
 parser = argparse.ArgumentParser(description = "SyncNet");
-parser.add_argument('--initial_model', type=str, default="data/syncnet.model", help='');
-parser.add_argument('--batch_size', type=int, default='20', help='');
-parser.add_argument('--vshift', type=int, default='15', help='');
-parser.add_argument('--data_dir', type=str, default='data/work', help='');
-parser.add_argument('--videofile', type=str, default='', help='');
-parser.add_argument('--reference', type=str, default='', help='');
+parser.add_argument('--data_dir', 	type=str, default='data/work', help='');
+parser.add_argument('--videofile', 	type=str, default='', help='');
+parser.add_argument('--reference', 	type=str, default='', help='');
+parser.add_argument('--frame_rate', type=int, default=25, help='Frame rate');
 opt = parser.parse_args();
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
 setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp'))
 setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork'))
 setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop'))
-
+setattr(opt,'frames_dir',os.path.join(opt.data_dir,'pyframes'))
 
 # ==================== LOAD FILES ====================
 
@@ -33,65 +31,58 @@
 with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'rb') as fil:
     dists = pickle.load(fil, encoding='latin1')
 
+flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg'))
+flist.sort()
+
 # ==================== SMOOTH FACES ====================
 
-faces = [ [] for i in range(1000000) ]
+faces = [[] for i in range(len(flist))]
 
-for ii, track in enumerate(tracks):
+for tidx, track in enumerate(tracks):
 
-	mean_dists =  numpy.mean(numpy.stack(dists[ii],1),1)
-	minidx = numpy.argmin(mean_dists,0)
-	minval = mean_dists[minidx] 
+	mean_dists 	=  numpy.mean(numpy.stack(dists[tidx],1),1)
+	minidx 		= numpy.argmin(mean_dists,0)
+	minval 		= mean_dists[minidx] 
 	
-	fdist   	= numpy.stack([dist[minidx] for dist in dists[ii]])
+	fdist   	= numpy.stack([dist[minidx] for dist in dists[tidx]])
 	fdist   	= numpy.pad(fdist, (3,3), 'constant', constant_values=10)
 
 	fconf   = numpy.median(mean_dists) - fdist
 	fconfm  = signal.medfilt(fconf,kernel_size=9)
 
-	for ij, frame in enumerate(track[0][0].tolist()) :
-		faces[frame].append([ii, fconfm[ij], track[1][0][ij], track[1][1][ij], track[1][2][ij]])
+	for fidx, frame in enumerate(track['track']['frame'].tolist()) :
+		faces[frame].append({'track': tidx, 'conf':fconfm[fidx], 's':track['proc_track']['s'][fidx], 'x':track['proc_track']['x'][fidx], 'y':track['proc_track']['y'][fidx]})
 
 # ==================== ADD DETECTIONS TO VIDEO ====================
 
-cap = cv2.VideoCapture(os.path.join(opt.avi_dir,opt.reference,'video.avi'))
-fw = int(cap.get(3))
-fh = int(cap.get(4))
+first_image = cv2.imread(flist[0])
+
+fw = first_image.shape[1]
+fh = first_image.shape[0]
 
 fourcc = cv2.VideoWriter_fourcc(*'XVID')
-vOut = cv2.VideoWriter(os.path.join(opt.avi_dir,opt.reference,'video_only.avi'), fourcc, cap.get(5), (fw,fh))
+vOut = cv2.VideoWriter(os.path.join(opt.avi_dir,opt.reference,'video_only.avi'), fourcc, opt.frame_rate, (fw,fh))
 
-frame_num=0
+for fidx, fname in enumerate(flist):
 
-while True:
-	ret, image = cap.read()
-	if ret == 0:
-		break
+	image = cv2.imread(fname)
 
-	for face in faces[frame_num]:
+	for face in faces[fidx]:
 
-		clr = max(min(face[1]*30,255),0)
+		clr = max(min(face['conf']*25,255),0)
 
-		cv2.rectangle(image,(int(face[3]-face[2]),int(face[4]-face[2])),(int(face[3]+face[2]),int(face[4]+face[2])),(0,clr,255-clr),3)
-		cv2.putText(image,'Track %d, L2 Dist %.3f'%(face[0],face[1]), (int(face[3]-face[2]),int(face[4]-face[2])),cv2.FONT_HERSHEY_SIMPLEX,0.5,(255,255,255),2)
+		cv2.rectangle(image,(int(face['x']-face['s']),int(face['y']-face['s'])),(int(face['x']+face['s']),int(face['y']+face['s'])),(0,clr,255-clr),3)
+		cv2.putText(image,'Track %d, Conf %.3f'%(face['track'],face['conf']), (int(face['x']-face['s']),int(face['y']-face['s'])),cv2.FONT_HERSHEY_SIMPLEX,0.5,(255,255,255),2)
 
 	vOut.write(image)
 
-	print('Frame %d'%frame_num)
-
-	frame_num+=1
+	print('Frame %d'%fidx)
 
-cap.release()
 vOut.release()
 
-# ========== CROP AUDIO FILE ==========
-
-command = ("ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.avi_dir,opt.reference,'audio_only.avi'))) 
-output = subprocess.call(command, shell=True, stdout=None)
-
 # ========== COMBINE AUDIO AND VIDEO FILES ==========
 
-command = ("ffmpeg -y -i %s -i %s -c:v copy -c:a copy %s" % (os.path.join(opt.avi_dir,opt.reference,'video_only.avi'),os.path.join(opt.avi_dir,opt.reference,'audio_only.avi'),os.path.join(opt.avi_dir,opt.reference,'video_out.avi'))) #-async 1 
+command = ("ffmpeg -y -i %s -i %s -c:v copy -c:a copy %s" % (os.path.join(opt.avi_dir,opt.reference,'video_only.avi'),os.path.join(opt.avi_dir,opt.reference,'audio.wav'),os.path.join(opt.avi_dir,opt.reference,'video_out.avi'))) #-async 1 
 output = subprocess.call(command, shell=True, stdout=None)
 
 

From 6efbb1c305c23f47a62b09cf4215a8ac45e97d49 Mon Sep 17 00:00:00 2001
From: joonson <joon@robots.ox.ac.uk>
Date: Sun, 29 Mar 2020 09:31:48 +0000
Subject: [PATCH 2/5] new detector combined

---
 .gitignore                  |   4 +-
 LICENSE.md                  |  19 ++++
 README.md                   |  21 +---
 SyncNetModel.py             |   3 +
 detectors/README.md         |   3 +
 detectors/__init__.py       |   1 +
 detectors/s3fd/__init__.py  |  61 ++++++++++
 detectors/s3fd/box_utils.py | 217 ++++++++++++++++++++++++++++++++++++
 detectors/s3fd/nets.py      | 174 +++++++++++++++++++++++++++++
 download_model.sh           |  11 +-
 requirements.txt            |   7 ++
 run_pipeline.py             | 125 +++++++--------------
 utils/label_map_util.py     | 140 -----------------------
 13 files changed, 532 insertions(+), 254 deletions(-)
 create mode 100644 LICENSE.md
 create mode 100755 detectors/README.md
 create mode 100644 detectors/__init__.py
 create mode 100644 detectors/s3fd/__init__.py
 create mode 100644 detectors/s3fd/box_utils.py
 create mode 100644 detectors/s3fd/nets.py
 create mode 100644 requirements.txt
 delete mode 100755 utils/label_map_util.py

diff --git a/.gitignore b/.gitignore
index 43a267a..350ada0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,5 +41,5 @@ Thumbs.db
 #########################
 data/
 protos/
-utils/__init__.py
-
+utils/
+*.pth
diff --git a/LICENSE.md b/LICENSE.md
new file mode 100644
index 0000000..de4a545
--- /dev/null
+++ b/LICENSE.md
@@ -0,0 +1,19 @@
+Copyright (c) 2016-present Joon Son Chung.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/README.md b/README.md
index 96e879d..7da5354 100755
--- a/README.md
+++ b/README.md
@@ -4,28 +4,15 @@ This repository contains the demo for the audio-to-video synchronisation network
 1. Removing temporal lags between the audio and visual streams in a video;
 2. Determining who is speaking amongst multiple faces in a video. 
 
-The model can be used for research purposes under <a href="https://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution License</a>. Please cite the paper below if you make use of the software. 
+Please cite the paper below if you make use of the software. 
 
-## Prerequisites
-The following packages are required to run the SyncNet demo:
+## Dependencies
 ```
-python (2.7.12)
-pytorch (0.4.0)
-numpy (1.14.3)
-scipy (1.0.1)
-opencv-python (3.4.0) - via opencv-contrib-python
-python_speech_features (0.6)
-cuda (8.0)
-ffmpeg (3.4.2)
+pip install -r requirements.txt
 ```
 
-In addition to above, these are required to run the full pipeline:
-```
-tensorflow (1.2, 1.4)
-pyscenedetect (0.5) 
-```
+In addition, `ffmpeg` is required.
 
-The demo has been tested with the package versions shown above, but may also work on other versions.
 
 ## Demo
 
diff --git a/SyncNetModel.py b/SyncNetModel.py
index e239325..c21ce25 100755
--- a/SyncNetModel.py
+++ b/SyncNetModel.py
@@ -1,3 +1,6 @@
+#!/usr/bin/python
+#-*- coding: utf-8 -*-
+
 import torch
 import torch.nn as nn
 
diff --git a/detectors/README.md b/detectors/README.md
new file mode 100755
index 0000000..f5a8d4f
--- /dev/null
+++ b/detectors/README.md
@@ -0,0 +1,3 @@
+# Face detector
+
+This face detector is adapted from `https://github.com/cs-giung/face-detection-pytorch`.
diff --git a/detectors/__init__.py b/detectors/__init__.py
new file mode 100644
index 0000000..059d49b
--- /dev/null
+++ b/detectors/__init__.py
@@ -0,0 +1 @@
+from .s3fd import S3FD
\ No newline at end of file
diff --git a/detectors/s3fd/__init__.py b/detectors/s3fd/__init__.py
new file mode 100644
index 0000000..d7f35e0
--- /dev/null
+++ b/detectors/s3fd/__init__.py
@@ -0,0 +1,61 @@
+import time
+import numpy as np
+import cv2
+import torch
+from torchvision import transforms
+from .nets import S3FDNet
+from .box_utils import nms_
+
+PATH_WEIGHT = './detectors/s3fd/weights/sfd_face.pth'
+img_mean = np.array([104., 117., 123.])[:, np.newaxis, np.newaxis].astype('float32')
+
+
+class S3FD():
+
+    def __init__(self, device='cuda'):
+
+        tstamp = time.time()
+        self.device = device
+
+        print('[S3FD] loading with', self.device)
+        self.net = S3FDNet(device=self.device).to(self.device)
+        state_dict = torch.load(PATH_WEIGHT, map_location=self.device)
+        self.net.load_state_dict(state_dict)
+        self.net.eval()
+        print('[S3FD] finished loading (%.4f sec)' % (time.time() - tstamp))
+    
+    def detect_faces(self, image, conf_th=0.8, scales=[1]):
+
+        w, h = image.shape[1], image.shape[0]
+
+        bboxes = np.empty(shape=(0, 5))
+
+        with torch.no_grad():
+            for s in scales:
+                scaled_img = cv2.resize(image, dsize=(0, 0), fx=s, fy=s, interpolation=cv2.INTER_LINEAR)
+
+                scaled_img = np.swapaxes(scaled_img, 1, 2)
+                scaled_img = np.swapaxes(scaled_img, 1, 0)
+                scaled_img = scaled_img[[2, 1, 0], :, :]
+                scaled_img = scaled_img.astype('float32')
+                scaled_img -= img_mean
+                scaled_img = scaled_img[[2, 1, 0], :, :]
+                x = torch.from_numpy(scaled_img).unsqueeze(0).to(self.device)
+                y = self.net(x)
+
+                detections = y.data
+                scale = torch.Tensor([w, h, w, h])
+
+                for i in range(detections.size(1)):
+                    j = 0
+                    while detections[0, i, j, 0] > conf_th:
+                        score = detections[0, i, j, 0]
+                        pt = (detections[0, i, j, 1:] * scale).cpu().numpy()
+                        bbox = (pt[0], pt[1], pt[2], pt[3], score)
+                        bboxes = np.vstack((bboxes, bbox))
+                        j += 1
+
+            keep = nms_(bboxes, 0.1)
+            bboxes = bboxes[keep]
+
+        return bboxes
diff --git a/detectors/s3fd/box_utils.py b/detectors/s3fd/box_utils.py
new file mode 100644
index 0000000..0779bcd
--- /dev/null
+++ b/detectors/s3fd/box_utils.py
@@ -0,0 +1,217 @@
+import numpy as np
+from itertools import product as product
+import torch
+from torch.autograd import Function
+
+
+def nms_(dets, thresh):
+    """
+    Courtesy of Ross Girshick
+    [https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/nms/py_cpu_nms.py]
+    """
+    x1 = dets[:, 0]
+    y1 = dets[:, 1]
+    x2 = dets[:, 2]
+    y2 = dets[:, 3]
+    scores = dets[:, 4]
+
+    areas = (x2 - x1) * (y2 - y1)
+    order = scores.argsort()[::-1]
+
+    keep = []
+    while order.size > 0:
+        i = order[0]
+        keep.append(int(i))
+        xx1 = np.maximum(x1[i], x1[order[1:]])
+        yy1 = np.maximum(y1[i], y1[order[1:]])
+        xx2 = np.minimum(x2[i], x2[order[1:]])
+        yy2 = np.minimum(y2[i], y2[order[1:]])
+
+        w = np.maximum(0.0, xx2 - xx1)
+        h = np.maximum(0.0, yy2 - yy1)
+        inter = w * h
+        ovr = inter / (areas[i] + areas[order[1:]] - inter)
+
+        inds = np.where(ovr <= thresh)[0]
+        order = order[inds + 1]
+
+    return np.array(keep).astype(np.int)
+
+
+def decode(loc, priors, variances):
+    """Decode locations from predictions using priors to undo
+    the encoding we did for offset regression at train time.
+    Args:
+        loc (tensor): location predictions for loc layers,
+            Shape: [num_priors,4]
+        priors (tensor): Prior boxes in center-offset form.
+            Shape: [num_priors,4].
+        variances: (list[float]) Variances of priorboxes
+    Return:
+        decoded bounding box predictions
+    """
+
+    boxes = torch.cat((
+        priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
+        priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
+    boxes[:, :2] -= boxes[:, 2:] / 2
+    boxes[:, 2:] += boxes[:, :2]
+    return boxes
+
+
+def nms(boxes, scores, overlap=0.5, top_k=200):
+    """Apply non-maximum suppression at test time to avoid detecting too many
+    overlapping bounding boxes for a given object.
+    Args:
+        boxes: (tensor) The location preds for the img, Shape: [num_priors,4].
+        scores: (tensor) The class predscores for the img, Shape:[num_priors].
+        overlap: (float) The overlap thresh for suppressing unnecessary boxes.
+        top_k: (int) The Maximum number of box preds to consider.
+    Return:
+        The indices of the kept boxes with respect to num_priors.
+    """
+
+    keep = scores.new(scores.size(0)).zero_().long()
+    if boxes.numel() == 0:
+        return keep, 0
+    x1 = boxes[:, 0]
+    y1 = boxes[:, 1]
+    x2 = boxes[:, 2]
+    y2 = boxes[:, 3]
+    area = torch.mul(x2 - x1, y2 - y1)
+    v, idx = scores.sort(0)  # sort in ascending order
+    # I = I[v >= 0.01]
+    idx = idx[-top_k:]  # indices of the top-k largest vals
+    xx1 = boxes.new()
+    yy1 = boxes.new()
+    xx2 = boxes.new()
+    yy2 = boxes.new()
+    w = boxes.new()
+    h = boxes.new()
+
+    # keep = torch.Tensor()
+    count = 0
+    while idx.numel() > 0:
+        i = idx[-1]  # index of current largest val
+        # keep.append(i)
+        keep[count] = i
+        count += 1
+        if idx.size(0) == 1:
+            break
+        idx = idx[:-1]  # remove kept element from view
+        # load bboxes of next highest vals
+        torch.index_select(x1, 0, idx, out=xx1)
+        torch.index_select(y1, 0, idx, out=yy1)
+        torch.index_select(x2, 0, idx, out=xx2)
+        torch.index_select(y2, 0, idx, out=yy2)
+        # store element-wise max with next highest score
+        xx1 = torch.clamp(xx1, min=x1[i])
+        yy1 = torch.clamp(yy1, min=y1[i])
+        xx2 = torch.clamp(xx2, max=x2[i])
+        yy2 = torch.clamp(yy2, max=y2[i])
+        w.resize_as_(xx2)
+        h.resize_as_(yy2)
+        w = xx2 - xx1
+        h = yy2 - yy1
+        # check sizes of xx1 and xx2.. after each iteration
+        w = torch.clamp(w, min=0.0)
+        h = torch.clamp(h, min=0.0)
+        inter = w * h
+        # IoU = i / (area(a) + area(b) - i)
+        rem_areas = torch.index_select(area, 0, idx)  # load remaining areas)
+        union = (rem_areas - inter) + area[i]
+        IoU = inter / union  # store result in iou
+        # keep only elements with an IoU <= overlap
+        idx = idx[IoU.le(overlap)]
+    return keep, count
+
+
+class Detect(object):
+
+    def __init__(self, num_classes=2,
+                    top_k=750, nms_thresh=0.3, conf_thresh=0.05,
+                    variance=[0.1, 0.2], nms_top_k=5000):
+        
+        self.num_classes = num_classes
+        self.top_k = top_k
+        self.nms_thresh = nms_thresh
+        self.conf_thresh = conf_thresh
+        self.variance = variance
+        self.nms_top_k = nms_top_k
+
+    def forward(self, loc_data, conf_data, prior_data):
+
+        num = loc_data.size(0)
+        num_priors = prior_data.size(0)
+
+        conf_preds = conf_data.view(num, num_priors, self.num_classes).transpose(2, 1)
+        batch_priors = prior_data.view(-1, num_priors, 4).expand(num, num_priors, 4)
+        batch_priors = batch_priors.contiguous().view(-1, 4)
+
+        decoded_boxes = decode(loc_data.view(-1, 4), batch_priors, self.variance)
+        decoded_boxes = decoded_boxes.view(num, num_priors, 4)
+
+        output = torch.zeros(num, self.num_classes, self.top_k, 5)
+
+        for i in range(num):
+            boxes = decoded_boxes[i].clone()
+            conf_scores = conf_preds[i].clone()
+
+            for cl in range(1, self.num_classes):
+                c_mask = conf_scores[cl].gt(self.conf_thresh)
+                scores = conf_scores[cl][c_mask]
+                
+                if scores.dim() == 0:
+                    continue
+                l_mask = c_mask.unsqueeze(1).expand_as(boxes)
+                boxes_ = boxes[l_mask].view(-1, 4)
+                ids, count = nms(boxes_, scores, self.nms_thresh, self.nms_top_k)
+                count = count if count < self.top_k else self.top_k
+
+                output[i, cl, :count] = torch.cat((scores[ids[:count]].unsqueeze(1), boxes_[ids[:count]]), 1)
+
+        return output
+
+
+class PriorBox(object):
+
+    def __init__(self, input_size, feature_maps,
+                    variance=[0.1, 0.2],
+                    min_sizes=[16, 32, 64, 128, 256, 512],
+                    steps=[4, 8, 16, 32, 64, 128],
+                    clip=False):
+
+        super(PriorBox, self).__init__()
+
+        self.imh = input_size[0]
+        self.imw = input_size[1]
+        self.feature_maps = feature_maps
+
+        self.variance = variance
+        self.min_sizes = min_sizes
+        self.steps = steps
+        self.clip = clip
+
+    def forward(self):
+        mean = []
+        for k, fmap in enumerate(self.feature_maps):
+            feath = fmap[0]
+            featw = fmap[1]
+            for i, j in product(range(feath), range(featw)):
+                f_kw = self.imw / self.steps[k]
+                f_kh = self.imh / self.steps[k]
+
+                cx = (j + 0.5) / f_kw
+                cy = (i + 0.5) / f_kh
+
+                s_kw = self.min_sizes[k] / self.imw
+                s_kh = self.min_sizes[k] / self.imh
+
+                mean += [cx, cy, s_kw, s_kh]
+
+        output = torch.FloatTensor(mean).view(-1, 4)
+        
+        if self.clip:
+            output.clamp_(max=1, min=0)
+        
+        return output
diff --git a/detectors/s3fd/nets.py b/detectors/s3fd/nets.py
new file mode 100644
index 0000000..85b5c82
--- /dev/null
+++ b/detectors/s3fd/nets.py
@@ -0,0 +1,174 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.nn.init as init
+from .box_utils import Detect, PriorBox
+
+
+class L2Norm(nn.Module):
+
+    def __init__(self, n_channels, scale):
+        super(L2Norm, self).__init__()
+        self.n_channels = n_channels
+        self.gamma = scale or None
+        self.eps = 1e-10
+        self.weight = nn.Parameter(torch.Tensor(self.n_channels))
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        init.constant_(self.weight, self.gamma)
+
+    def forward(self, x):
+        norm = x.pow(2).sum(dim=1, keepdim=True).sqrt() + self.eps
+        x = torch.div(x, norm)
+        out = self.weight.unsqueeze(0).unsqueeze(2).unsqueeze(3).expand_as(x) * x
+        return out
+
+
+class S3FDNet(nn.Module):
+
+    def __init__(self, device='cuda'):
+        super(S3FDNet, self).__init__()
+        self.device = device
+
+        self.vgg = nn.ModuleList([
+            nn.Conv2d(3, 64, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(64, 64, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2),
+
+            nn.Conv2d(64, 128, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(128, 128, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2),
+            
+            nn.Conv2d(128, 256, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(256, 256, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(256, 256, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2, ceil_mode=True),
+            
+            nn.Conv2d(256, 512, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(512, 512, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(512, 512, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2),
+
+            nn.Conv2d(512, 512, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(512, 512, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(512, 512, 3, 1, padding=1),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(2, 2),
+
+            nn.Conv2d(512, 1024, 3, 1, padding=6, dilation=6),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(1024, 1024, 1, 1),
+            nn.ReLU(inplace=True),
+        ])
+
+        self.L2Norm3_3 = L2Norm(256, 10)
+        self.L2Norm4_3 = L2Norm(512, 8)
+        self.L2Norm5_3 = L2Norm(512, 5)
+
+        self.extras = nn.ModuleList([
+            nn.Conv2d(1024, 256, 1, 1),
+            nn.Conv2d(256, 512, 3, 2, padding=1),
+            nn.Conv2d(512, 128, 1, 1),
+            nn.Conv2d(128, 256, 3, 2, padding=1),
+        ])
+        
+        self.loc = nn.ModuleList([
+            nn.Conv2d(256, 4, 3, 1, padding=1),
+            nn.Conv2d(512, 4, 3, 1, padding=1),
+            nn.Conv2d(512, 4, 3, 1, padding=1),
+            nn.Conv2d(1024, 4, 3, 1, padding=1),
+            nn.Conv2d(512, 4, 3, 1, padding=1),
+            nn.Conv2d(256, 4, 3, 1, padding=1),
+        ])
+
+        self.conf = nn.ModuleList([
+            nn.Conv2d(256, 4, 3, 1, padding=1),
+            nn.Conv2d(512, 2, 3, 1, padding=1),
+            nn.Conv2d(512, 2, 3, 1, padding=1),
+            nn.Conv2d(1024, 2, 3, 1, padding=1),
+            nn.Conv2d(512, 2, 3, 1, padding=1),
+            nn.Conv2d(256, 2, 3, 1, padding=1),
+        ])
+
+        self.softmax = nn.Softmax(dim=-1)
+        self.detect = Detect()
+
+    def forward(self, x):
+        size = x.size()[2:]
+        sources = list()
+        loc = list()
+        conf = list()
+
+        for k in range(16):
+            x = self.vgg[k](x)
+        s = self.L2Norm3_3(x)
+        sources.append(s)
+
+        for k in range(16, 23):
+            x = self.vgg[k](x)
+        s = self.L2Norm4_3(x)
+        sources.append(s)
+
+        for k in range(23, 30):
+            x = self.vgg[k](x)
+        s = self.L2Norm5_3(x)
+        sources.append(s)
+
+        for k in range(30, len(self.vgg)):
+            x = self.vgg[k](x)
+        sources.append(x)
+        
+        # apply extra layers and cache source layer outputs
+        for k, v in enumerate(self.extras):
+            x = F.relu(v(x), inplace=True)
+            if k % 2 == 1:
+                sources.append(x)
+
+        # apply multibox head to source layers
+        loc_x = self.loc[0](sources[0])
+        conf_x = self.conf[0](sources[0])
+
+        max_conf, _ = torch.max(conf_x[:, 0:3, :, :], dim=1, keepdim=True)
+        conf_x = torch.cat((max_conf, conf_x[:, 3:, :, :]), dim=1)
+
+        loc.append(loc_x.permute(0, 2, 3, 1).contiguous())
+        conf.append(conf_x.permute(0, 2, 3, 1).contiguous())
+
+        for i in range(1, len(sources)):
+            x = sources[i]
+            conf.append(self.conf[i](x).permute(0, 2, 3, 1).contiguous())
+            loc.append(self.loc[i](x).permute(0, 2, 3, 1).contiguous())
+
+        features_maps = []
+        for i in range(len(loc)):
+            feat = []
+            feat += [loc[i].size(1), loc[i].size(2)]
+            features_maps += [feat]
+
+        loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
+        conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
+
+        with torch.no_grad():
+            self.priorbox = PriorBox(size, features_maps)
+            self.priors = self.priorbox.forward()
+
+        output = self.detect.forward(
+            loc.view(loc.size(0), -1, 4),
+            self.softmax(conf.view(conf.size(0), -1, 2)),
+            self.priors.type(type(x.data)).to(self.device)
+        )
+
+        return output
diff --git a/download_model.sh b/download_model.sh
index 7045f17..3e3a9dc 100755
--- a/download_model.sh
+++ b/download_model.sh
@@ -5,12 +5,5 @@ wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/syncnet_v2.model -O d
 wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/example.avi -O data/example.avi
 
 # For the pre-processing pipeline
-
-wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/face_detection_tf.zip -O facedet.zip
-
-mkdir protos
-unzip facedet.zip -d protos/
-rm -f facedet.zip
-
-cat /dev/null > protos/__init__.py
-cat /dev/null > utils/__init__.py
\ No newline at end of file
+mkdir detectors/s3fd/weights
+wget https://www.robots.ox.ac.uk/~vgg/software/lipsync/data/sfd_face.pth -O detectors/s3fd/weights/sfd_face.pth
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..8919740
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,7 @@
+torch>=1.4.0
+torchvision>=0.5.0
+numpy>=1.18.1
+scipy>=1.2.1
+scenedetect==0.5.1
+opencv-contrib-python
+python_speech_features
diff --git a/run_pipeline.py b/run_pipeline.py
index b4ae02b..f5fc22e 100755
--- a/run_pipeline.py
+++ b/run_pipeline.py
@@ -1,9 +1,7 @@
 #!/usr/bin/python
 
-import sys, time, os, pdb, argparse, pickle, subprocess, glob
+import sys, time, os, pdb, argparse, pickle, subprocess, glob, cv2
 import numpy as np
-import tensorflow as tf
-import cv2
 from shutil import rmtree
 
 import scenedetect
@@ -14,23 +12,25 @@
 from scenedetect.detectors import ContentDetector
 
 from scipy.interpolate import interp1d
-from utils import label_map_util
 from scipy.io import wavfile
 from scipy import signal
 
+from detectors import S3FD
+
 # ========== ========== ========== ==========
 # # PARSE ARGS
 # ========== ========== ========== ==========
 
 parser = argparse.ArgumentParser(description = "FaceTracker");
 parser.add_argument('--data_dir',       type=str, default='data/work', help='Output direcotry');
-parser.add_argument('--videofile',      type=str, default='', help='Input video file');
-parser.add_argument('--reference',      type=str, default='', help='Name of the video');
-parser.add_argument('--crop_scale',     type=float, default=0.5, help='Scale bounding box');
-parser.add_argument('--min_track',      type=int, default=100, help='Minimum facetrack duration');
-parser.add_argument('--frame_rate',     type=int, default=25, help='Frame rate');
-parser.add_argument('--num_failed_det', type=int, default=25, help='Number of missed detections allowed');
-parser.add_argument('--min_face_size',  type=float, default=0.03, help='Minimum size of faces');
+parser.add_argument('--videofile',      type=str, default='',   help='Input video file');
+parser.add_argument('--reference',      type=str, default='',   help='Video reference');
+parser.add_argument('--facedet_scale',  type=float, default=0.25, help='Scale factor for face detection');
+parser.add_argument('--crop_scale',     type=float, default=0.40, help='Scale bounding box');
+parser.add_argument('--min_track',      type=int, default=100,  help='Minimum facetrack duration');
+parser.add_argument('--frame_rate',     type=int, default=25,   help='Frame rate');
+parser.add_argument('--num_failed_det', type=int, default=25,   help='Number of missed detections allowed before tracking is stopped');
+parser.add_argument('--min_face_size',  type=int, default=100,  help='Minimum face size in pixels');
 opt = parser.parse_args();
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
@@ -99,7 +99,7 @@ def track_shot(opt,scenefaces):
         bboxes_i.append(interpfn(frame_i))
       bboxes_i  = np.stack(bboxes_i, axis=1)
 
-      if np.mean(bboxes_i[:,3]-bboxes_i[:,1]) > opt.min_face_size:
+      if max(np.mean(bboxes_i[:,2]-bboxes_i[:,0]), np.mean(bboxes_i[:,3]-bboxes_i[:,1])) > opt.min_face_size:
         tracks.append({'frame':frame_i,'bbox':bboxes_i})
 
   return tracks
@@ -116,23 +116,18 @@ def crop_video(opt,track,cropfile):
   fourcc = cv2.VideoWriter_fourcc(*'XVID')
   vOut = cv2.VideoWriter(cropfile+'t.avi', fourcc, opt.frame_rate, (224,224))
 
-  first_image = cv2.imread(flist[track['frame'][0]])
-
-  fw = first_image.shape[1]
-  fh = first_image.shape[0]
-
   dets = {'x':[], 'y':[], 's':[]}
 
   for det in track['bbox']:
 
-    dets['s'].append(((det[3]-det[1])*fw+(det[2]-det[0])*fh)/4) # H+W / 4
-    dets['x'].append((det[1]+det[3])*fw/2) # crop center x 
-    dets['y'].append((det[0]+det[2])*fh/2) # crop center y
+    dets['s'].append(max((det[3]-det[1]),(det[2]-det[0]))/2) 
+    dets['y'].append((det[1]+det[3])/2) # crop center x 
+    dets['x'].append((det[0]+det[2])/2) # crop center y
 
   # Smooth detections
-  dets['s'] = signal.medfilt(dets['s'],kernel_size=7)   
-  dets['x'] = signal.medfilt(dets['x'],kernel_size=5)
-  dets['y'] = signal.medfilt(dets['y'],kernel_size=5)
+  dets['s'] = signal.medfilt(dets['s'],kernel_size=13)   
+  dets['x'] = signal.medfilt(dets['x'],kernel_size=13)
+  dets['y'] = signal.medfilt(dets['y'],kernel_size=13)
 
   for fidx, frame in enumerate(track['frame']):
 
@@ -143,7 +138,7 @@ def crop_video(opt,track,cropfile):
 
     image = cv2.imread(flist[frame])
     
-    frame = np.pad(image,((bsi,bsi),(bsi,bsi),(0,0)), 'constant', constant_values=(0,0))
+    frame = np.pad(image,((bsi,bsi),(bsi,bsi),(0,0)), 'constant', constant_values=(110,110))
     my  = dets['y'][fidx]+bsi  # BBox center Y
     mx  = dets['x'][fidx]+bsi  # BBox center X
 
@@ -179,6 +174,8 @@ def crop_video(opt,track,cropfile):
 
   os.remove(cropfile+'t.avi')
 
+  print('Mean pos: x %.2f y %.2f s %.2f'%(np.mean(dets['x']),np.mean(dets['y']),np.mean(dets['s'])))
+
   return {'track':track, 'proc_track':dets}
 
 # ========== ========== ========== ==========
@@ -187,78 +184,34 @@ def crop_video(opt,track,cropfile):
 
 def inference_video(opt):
 
-
-  # Path to frozen detection graph. This is the actual model that is used for the object detection.
-  PATH_TO_CKPT = './protos/frozen_inference_graph_face.pb'
-
-  # List of the strings that is used to add correct label for each box.
-  PATH_TO_LABELS = './protos/face_label_map.pbtxt'
-
-  NUM_CLASSES = 2
-  MIN_CONF = 0.3
-
-  label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
-  categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
-  category_index = label_map_util.create_category_index(categories)
-
-  def load_image_into_numpy_array(image):
-    (im_width, im_height) = image.size
-    return np.array(image.getdata()).reshape(
-        (im_height, im_width, 3)).astype(np.uint8)
+  DET = S3FD(device='cuda')
 
   flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg'))
   flist.sort()
 
-  detection_graph = tf.Graph()
-  with detection_graph.as_default():
-      od_graph_def = tf.GraphDef()
-      with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
-          serialized_graph = fid.read()
-          od_graph_def.ParseFromString(serialized_graph)
-          tf.import_graph_def(od_graph_def, name='')
-
   dets = []
-
-  with detection_graph.as_default():
-    config = tf.ConfigProto()
-    config.gpu_options.allow_growth = True
-    with tf.Session(graph=detection_graph, config=config) as sess:
       
-      for fidx, fname in enumerate(flist):
-        
-        image = cv2.imread(fname)
-
-        image_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+  for fidx, fname in enumerate(flist):
 
-        image_np_expanded = np.expand_dims(image_np, axis=0)
-        image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
+    start_time = time.time()
+    
+    image = cv2.imread(fname)
 
-        boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
+    image_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    bboxes = DET.detect_faces(image_np, conf_th=0.9, scales=[opt.facedet_scale])
 
-        scores = detection_graph.get_tensor_by_name('detection_scores:0')
-        classes = detection_graph.get_tensor_by_name('detection_classes:0')
-        num_detections = detection_graph.get_tensor_by_name('num_detections:0')
-        
-        # Actual detection.
-        start_time = time.time()
-        (boxes, scores, classes, num_detections) = sess.run(
-            [boxes, scores, classes, num_detections],
-            feed_dict={image_tensor: image_np_expanded})
-        elapsed_time = time.time() - start_time
-        
-        score = scores[0]
+    dets.append([]);
+    for bbox in bboxes:
+      dets[-1].append({'frame':fidx, 'bbox':(bbox[:-1]).tolist(), 'conf':bbox[-1]})
 
-        dets.append([]);
-        for index in range(0,len(score)):
-          if score[index] > MIN_CONF:
-            dets[-1].append({'frame':fidx, 'bbox':boxes[0][index].tolist(), 'conf':score[index]})
+    elapsed_time = time.time() - start_time
 
-        print('%s-%05d; %d dets; %.2f Hz' % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),fidx,len(dets[-1]),(1/elapsed_time))) 
+    print('%s-%05d; %d dets; %.2f Hz' % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),fidx,len(dets[-1]),(1/elapsed_time))) 
 
-      savepath = os.path.join(opt.work_dir,opt.reference,'faces.pckl')
+  savepath = os.path.join(opt.work_dir,opt.reference,'faces.pckl')
 
-      with open(savepath, 'wb') as fil:
-        pickle.dump(dets, fil)
+  with open(savepath, 'wb') as fil:
+    pickle.dump(dets, fil)
 
   return dets
 
@@ -283,7 +236,7 @@ def scene_detect(opt):
 
   scene_list = scene_manager.get_scene_list(base_timecode)
 
-  savepath = os.path.join(opt.work_dir,'scene.pckl')
+  savepath = os.path.join(opt.work_dir,opt.reference,'scene.pckl')
 
   if scene_list == []:
     scene_list = [(video_manager.get_base_timecode(),video_manager.get_current_timecode())]
@@ -327,10 +280,10 @@ def scene_detect(opt):
 
 # ========== CONVERT VIDEO AND EXTRACT FRAMES ==========
 
-command = ("ffmpeg -y -i %s -async 1 -qscale:v 4 -r 25 %s" % (opt.videofile,os.path.join(opt.avi_dir,opt.reference,'video.avi'))) #-async 1  -deinterlace
+command = ("ffmpeg -y -i %s -qscale:v 2 -async 1 -r 25 %s" % (opt.videofile,os.path.join(opt.avi_dir,opt.reference,'video.avi')))
 output = subprocess.call(command, shell=True, stdout=None)
 
-command = ("ffmpeg -y -i %s -threads 1 -f image2 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.frames_dir,opt.reference,'%06d.jpg'))) 
+command = ("ffmpeg -y -i %s -qscale:v 2 -threads 1 -f image2 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.frames_dir,opt.reference,'%06d.jpg'))) 
 output = subprocess.call(command, shell=True, stdout=None)
 
 command = ("ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.avi_dir,opt.reference,'audio.wav'))) 
diff --git a/utils/label_map_util.py b/utils/label_map_util.py
deleted file mode 100755
index 3fd5316..0000000
--- a/utils/label_map_util.py
+++ /dev/null
@@ -1,140 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Label map utility functions."""
-
-import logging
-
-import tensorflow as tf
-from google.protobuf import text_format
-from protos import string_int_label_map_pb2    
-
-
-def _validate_label_map(label_map):
-  """Checks if a label map is valid.
-
-  Args:
-    label_map: StringIntLabelMap to validate.
-
-  Raises:
-    ValueError: if label map is invalid.
-  """
-  for item in label_map.item:
-    if item.id < 1:
-      raise ValueError('Label map ids should be >= 1.')
-
-
-def create_category_index(categories):
-  """Creates dictionary of COCO compatible categories keyed by category id.
-
-  Args:
-    categories: a list of dicts, each of which has the following keys:
-      'id': (required) an integer id uniquely identifying this category.
-      'name': (required) string representing category name
-        e.g., 'cat', 'dog', 'pizza'.
-
-  Returns:
-    category_index: a dict containing the same entries as categories, but keyed
-      by the 'id' field of each category.
-  """
-  category_index = {}
-  for cat in categories:
-    category_index[cat['id']] = cat
-  return category_index
-
-
-def convert_label_map_to_categories(label_map,
-                                    max_num_classes,
-                                    use_display_name=True):
-  """Loads label map proto and returns categories list compatible with eval.
-
-  This function loads a label map and returns a list of dicts, each of which
-  has the following keys:
-    'id': (required) an integer id uniquely identifying this category.
-    'name': (required) string representing category name
-      e.g., 'cat', 'dog', 'pizza'.
-  We only allow class into the list if its id-label_id_offset is
-  between 0 (inclusive) and max_num_classes (exclusive).
-  If there are several items mapping to the same id in the label map,
-  we will only keep the first one in the categories list.
-
-  Args:
-    label_map: a StringIntLabelMapProto or None.  If None, a default categories
-      list is created with max_num_classes categories.
-    max_num_classes: maximum number of (consecutive) label indices to include.
-    use_display_name: (boolean) choose whether to load 'display_name' field
-      as category name.  If False or if the display_name field does not exist,
-      uses 'name' field as category names instead.
-  Returns:
-    categories: a list of dictionaries representing all possible categories.
-  """
-  categories = []
-  list_of_ids_already_added = []
-  if not label_map:
-    label_id_offset = 1
-    for class_id in range(max_num_classes):
-      categories.append({
-          'id': class_id + label_id_offset,
-          'name': 'category_{}'.format(class_id + label_id_offset)
-      })
-    return categories
-  for item in label_map.item:
-    if not 0 < item.id <= max_num_classes:
-      logging.info('Ignore item %d since it falls outside of requested '
-                   'label range.', item.id)
-      continue
-    if use_display_name and item.HasField('display_name'):
-      name = item.display_name
-    else:
-      name = item.name
-    if item.id not in list_of_ids_already_added:
-      list_of_ids_already_added.append(item.id)
-      categories.append({'id': item.id, 'name': name})
-  return categories
-
-
-def load_labelmap(path):
-  """Loads label map proto.
-
-  Args:
-    path: path to StringIntLabelMap proto text file.
-  Returns:
-    a StringIntLabelMapProto
-  """
-  with tf.gfile.GFile(path, 'r') as fid:
-    label_map_string = fid.read()
-    label_map = string_int_label_map_pb2.StringIntLabelMap()
-    try:
-      text_format.Merge(label_map_string, label_map)
-    except text_format.ParseError:
-      label_map.ParseFromString(label_map_string)
-  _validate_label_map(label_map)
-  return label_map
-
-
-def get_label_map_dict(label_map_path):
-  """Reads a label map and returns a dictionary of label names to id.
-
-  Args:
-    label_map_path: path to label_map.
-
-  Returns:
-    A dictionary mapping label names to id.
-  """
-  label_map = load_labelmap(label_map_path)
-  label_map_dict = {}
-  for item in label_map.item:
-    label_map_dict[item.name] = item.id
-  return label_map_dict

From cdd20ba1c3280caf5191b36f68660d7a96d30008 Mon Sep 17 00:00:00 2001
From: joonson <joonson@kaist.ac.kr>
Date: Sat, 11 Apr 2026 15:18:26 +0900
Subject: [PATCH 3/5] Modernize codebase: auto-detect GPU/CPU, replace print
 with logging, update deps

---
 README.md                   |  18 ++--
 SyncNetInstance.py          |  89 ++++++++++---------
 SyncNetModel.py             |  52 +++++------
 demo_feature.py             |  29 ++++---
 demo_syncnet.py             |  29 ++++---
 detectors/s3fd/__init__.py  |  11 +--
 detectors/s3fd/box_utils.py |  35 +++-----
 detectors/s3fd/nets.py      |   4 +-
 download_model.sh           |   4 +-
 environment-cpu.yml         |  27 ++++++
 environment.yml             |  29 +++++++
 requirements.txt            |   7 --
 run_pipeline.py             | 169 ++++++++++++++++--------------------
 run_syncnet.py              |  31 ++++---
 run_visualise.py            |  37 ++++----
 15 files changed, 311 insertions(+), 260 deletions(-)
 create mode 100644 environment-cpu.yml
 create mode 100644 environment.yml
 delete mode 100644 requirements.txt

diff --git a/README.md b/README.md
index 7da5354..d3473b0 100755
--- a/README.md
+++ b/README.md
@@ -7,12 +7,18 @@ This repository contains the demo for the audio-to-video synchronisation network
 Please cite the paper below if you make use of the software. 
 
 ## Dependencies
+
 ```
-pip install -r requirements.txt
+conda env create -f environment.yml
 ```
 
-In addition, `ffmpeg` is required.
 
+## Getting Started
+
+Download the pretrained model:
+```
+sh download_model.sh
+```
 
 ## Demo
 
@@ -21,16 +27,17 @@ SyncNet demo:
 python demo_syncnet.py --videofile data/example.avi --tmp_dir /path/to/temp/directory
 ```
 
-Check that this script returns:
+Check that this script returns approximately the following values (minor differences are expected depending on your platform and package versions):
 ```
 AV offset:      3 
 Min dist:       5.353
 Confidence:     10.021
 ```
 
-Full pipeline:
+## Full Pipeline
+
+Run the three stages — face detection and tracking, sync offset estimation, and visualisation:
 ```
-sh download_model.sh
 python run_pipeline.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output
 python run_syncnet.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output
 python run_visualise.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output
@@ -39,7 +46,6 @@ python run_visualise.py --videofile /path/to/video.mp4 --reference name_of_video
 Outputs:
 ```
 $DATA_DIR/pycrop/$REFERENCE/*.avi - cropped face tracks
-$DATA_DIR/pywork/$REFERENCE/offsets.txt - audio-video offset values
 $DATA_DIR/pyavi/$REFERENCE/video_out.avi - output video (as shown below)
 ```
 <p align="center">
diff --git a/SyncNetInstance.py b/SyncNetInstance.py
index 497d44f..54c92ec 100644
--- a/SyncNetInstance.py
+++ b/SyncNetInstance.py
@@ -1,10 +1,10 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 # Video 25 FPS, Audio 16000HZ
 
 import torch
 import numpy
-import time, pdb, argparse, subprocess, os, math, glob
+import time, pdb, argparse, subprocess, os, math, glob, logging
 import cv2
 import python_speech_features
 
@@ -13,11 +13,13 @@
 from SyncNetModel import *
 from shutil import rmtree
 
+logger = logging.getLogger(__name__)
+
 
 # ==================== Get OFFSET ====================
 
 def calc_pdist(feat1, feat2, vshift=10):
-    
+
     win_size = vshift*2+1
 
     feat2p = torch.nn.functional.pad(feat2,(0,0,vshift,vshift))
@@ -34,14 +36,16 @@ def calc_pdist(feat1, feat2, vshift=10):
 
 class SyncNetInstance(torch.nn.Module):
 
-    def __init__(self, dropout = 0, num_layers_in_fc_layers = 1024):
-        super(SyncNetInstance, self).__init__();
+    def __init__(self, dropout = 0, num_layers_in_fc_layers = 1024, device=None):
+        super().__init__()
 
-        self.__S__ = S(num_layers_in_fc_layers = num_layers_in_fc_layers).cuda();
+        self.device = device or ('cuda' if torch.cuda.is_available() else 'cpu')
+        logger.info('Using device: %s', self.device)
+        self.__S__ = S(num_layers_in_fc_layers = num_layers_in_fc_layers).to(self.device)
 
     def evaluate(self, opt, videofile):
 
-        self.__S__.eval();
+        self.__S__.eval()
 
         # ========== ==========
         # Convert files
@@ -52,18 +56,21 @@ def evaluate(self, opt, videofile):
 
         os.makedirs(os.path.join(opt.tmp_dir,opt.reference))
 
-        command = ("ffmpeg -y -i %s -threads 1 -f image2 %s" % (videofile,os.path.join(opt.tmp_dir,opt.reference,'%06d.jpg'))) 
-        output = subprocess.call(command, shell=True, stdout=None)
+        command = ["ffmpeg", "-y", "-i", videofile, "-threads", "1", "-f", "image2",
+                   os.path.join(opt.tmp_dir, opt.reference, '%06d.jpg')]
+        subprocess.run(command, check=True)
+
+        command = ["ffmpeg", "-y", "-i", videofile, "-async", "1", "-ac", "1", "-vn",
+                   "-acodec", "pcm_s16le", "-ar", "16000",
+                   os.path.join(opt.tmp_dir, opt.reference, 'audio.wav')]
+        subprocess.run(command, check=True)
 
-        command = ("ffmpeg -y -i %s -async 1 -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (videofile,os.path.join(opt.tmp_dir,opt.reference,'audio.wav'))) 
-        output = subprocess.call(command, shell=True, stdout=None)
-        
         # ========== ==========
-        # Load video 
+        # Load video
         # ========== ==========
 
         images = []
-        
+
         flist = glob.glob(os.path.join(opt.tmp_dir,opt.reference,'*.jpg'))
         flist.sort()
 
@@ -74,7 +81,7 @@ def evaluate(self, opt, videofile):
         im = numpy.expand_dims(im,axis=0)
         im = numpy.transpose(im,(0,3,4,1,2))
 
-        imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float())
+        imtv = torch.from_numpy(im.astype(float)).float()
 
         # ========== ==========
         # Load audio
@@ -85,17 +92,17 @@ def evaluate(self, opt, videofile):
         mfcc = numpy.stack([numpy.array(i) for i in mfcc])
 
         cc = numpy.expand_dims(numpy.expand_dims(mfcc,axis=0),axis=0)
-        cct = torch.autograd.Variable(torch.from_numpy(cc.astype(float)).float())
+        cct = torch.from_numpy(cc.astype(float)).float()
 
         # ========== ==========
         # Check audio and video input length
         # ========== ==========
 
         if (float(len(audio))/16000) != (float(len(images))/25) :
-            print("WARNING: Audio (%.4fs) and video (%.4fs) lengths are different."%(float(len(audio))/16000,float(len(images))/25))
+            logger.warning("Audio (%.4fs) and video (%.4fs) lengths are different.",float(len(audio))/16000,float(len(images))/25)
 
         min_length = min(len(images),math.floor(len(audio)/640))
-        
+
         # ========== ==========
         # Generate video and audio feats
         # ========== ==========
@@ -106,15 +113,15 @@ def evaluate(self, opt, videofile):
 
         tS = time.time()
         for i in range(0,lastframe,opt.batch_size):
-            
+
             im_batch = [ imtv[:,:,vframe:vframe+5,:,:] for vframe in range(i,min(lastframe,i+opt.batch_size)) ]
             im_in = torch.cat(im_batch,0)
-            im_out  = self.__S__.forward_lip(im_in.cuda());
+            im_out  = self.__S__.forward_lip(im_in.to(self.device))
             im_feat.append(im_out.data.cpu())
 
             cc_batch = [ cct[:,:,:,vframe*4:vframe*4+20] for vframe in range(i,min(lastframe,i+opt.batch_size)) ]
             cc_in = torch.cat(cc_batch,0)
-            cc_out  = self.__S__.forward_aud(cc_in.cuda())
+            cc_out  = self.__S__.forward_aud(cc_in.to(self.device))
             cc_feat.append(cc_out.data.cpu())
 
         im_feat = torch.cat(im_feat,0)
@@ -123,8 +130,8 @@ def evaluate(self, opt, videofile):
         # ========== ==========
         # Compute offset
         # ========== ==========
-            
-        print('Compute time %.3f sec.' % (time.time()-tS))
+
+        logger.info('Compute time %.3f sec.', time.time()-tS)
 
         dists = calc_pdist(im_feat,cc_feat,vshift=opt.vshift)
         mdist = torch.mean(torch.stack(dists,1),1)
@@ -138,25 +145,27 @@ def evaluate(self, opt, videofile):
         # fdist   = numpy.pad(fdist, (3,3), 'constant', constant_values=15)
         fconf   = torch.median(mdist).numpy() - fdist
         fconfm  = signal.medfilt(fconf,kernel_size=9)
-        
+
         numpy.set_printoptions(formatter={'float': '{: 0.3f}'.format})
-        print('Framewise conf: ')
-        print(fconfm)
-        print('AV offset: \t%d \nMin dist: \t%.3f\nConfidence: \t%.3f' % (offset,minval,conf))
+        logger.info('Framewise conf: ')
+        logger.info(fconfm)
+        logger.info('AV offset: \t%d', offset)
+        logger.info('Min dist: \t%.3f', minval)
+        logger.info('Confidence: \t%.3f', conf)
 
         dists_npy = numpy.array([ dist.numpy() for dist in dists ])
         return offset.numpy(), conf.numpy(), dists_npy
 
     def extract_feature(self, opt, videofile):
 
-        self.__S__.eval();
-        
+        self.__S__.eval()
+
         # ========== ==========
-        # Load video 
+        # Load video
         # ========== ==========
         cap = cv2.VideoCapture(videofile)
 
-        frame_num = 1;
+        frame_num = 1
         images = []
         while frame_num:
             frame_num += 1
@@ -170,8 +179,8 @@ def extract_feature(self, opt, videofile):
         im = numpy.expand_dims(im,axis=0)
         im = numpy.transpose(im,(0,3,4,1,2))
 
-        imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float())
-        
+        imtv = torch.from_numpy(im.astype(float)).float()
+
         # ========== ==========
         # Generate video feats
         # ========== ==========
@@ -181,10 +190,10 @@ def extract_feature(self, opt, videofile):
 
         tS = time.time()
         for i in range(0,lastframe,opt.batch_size):
-            
+
             im_batch = [ imtv[:,:,vframe:vframe+5,:,:] for vframe in range(i,min(lastframe,i+opt.batch_size)) ]
             im_in = torch.cat(im_batch,0)
-            im_out  = self.__S__.forward_lipfeat(im_in.cuda());
+            im_out  = self.__S__.forward_lipfeat(im_in.to(self.device))
             im_feat.append(im_out.data.cpu())
 
         im_feat = torch.cat(im_feat,0)
@@ -192,17 +201,17 @@ def extract_feature(self, opt, videofile):
         # ========== ==========
         # Compute offset
         # ========== ==========
-            
-        print('Compute time %.3f sec.' % (time.time()-tS))
+
+        logger.info('Compute time %.3f sec.', time.time()-tS)
 
         return im_feat
 
 
     def loadParameters(self, path):
-        loaded_state = torch.load(path, map_location=lambda storage, loc: storage);
+        loaded_state = torch.load(path, map_location=lambda storage, loc: storage, weights_only=True)
 
-        self_state = self.__S__.state_dict();
+        self_state = self.__S__.state_dict()
 
         for name, param in loaded_state.items():
 
-            self_state[name].copy_(param);
+            self_state[name].copy_(param)
diff --git a/SyncNetModel.py b/SyncNetModel.py
index c21ce25..9953b10 100755
--- a/SyncNetModel.py
+++ b/SyncNetModel.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 
 import torch
@@ -6,20 +6,20 @@
 
 def save(model, filename):
     with open(filename, "wb") as f:
-        torch.save(model, f);
-        print("%s saved."%filename);
+        torch.save(model, f)
+        print(f"{filename} saved.")
 
 def load(filename):
-    net = torch.load(filename)
-    return net;
-    
+    net = torch.load(filename, weights_only=True)
+    return net
+
 class S(nn.Module):
     def __init__(self, num_layers_in_fc_layers = 1024):
-        super(S, self).__init__();
+        super().__init__()
 
-        self.__nFeatures__ = 24;
-        self.__nChs__ = 32;
-        self.__midChs__ = 32;
+        self.__nFeatures__ = 24
+        self.__nChs__ = 32
+        self.__midChs__ = 32
 
         self.netcnnaud = nn.Sequential(
             nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1)),
@@ -44,25 +44,25 @@ def __init__(self, num_layers_in_fc_layers = 1024):
             nn.BatchNorm2d(256),
             nn.ReLU(inplace=True),
             nn.MaxPool2d(kernel_size=(3,3), stride=(2,2)),
-            
+
             nn.Conv2d(256, 512, kernel_size=(5,4), padding=(0,0)),
             nn.BatchNorm2d(512),
             nn.ReLU(),
-        );
+        )
 
         self.netfcaud = nn.Sequential(
             nn.Linear(512, 512),
             nn.BatchNorm1d(512),
             nn.ReLU(),
             nn.Linear(512, num_layers_in_fc_layers),
-        );
+        )
 
         self.netfclip = nn.Sequential(
             nn.Linear(512, 512),
             nn.BatchNorm1d(512),
             nn.ReLU(),
             nn.Linear(512, num_layers_in_fc_layers),
-        );
+        )
 
         self.netcnnlip = nn.Sequential(
             nn.Conv3d(3, 96, kernel_size=(5,7,7), stride=(1,2,2), padding=0),
@@ -91,27 +91,27 @@ def __init__(self, num_layers_in_fc_layers = 1024):
             nn.Conv3d(256, 512, kernel_size=(1,6,6), padding=0),
             nn.BatchNorm3d(512),
             nn.ReLU(inplace=True),
-        );
+        )
 
     def forward_aud(self, x):
 
-        mid = self.netcnnaud(x); # N x ch x 24 x M
-        mid = mid.view((mid.size()[0], -1)); # N x (ch x 24)
-        out = self.netfcaud(mid);
+        mid = self.netcnnaud(x)  # N x ch x 24 x M
+        mid = mid.view((mid.size(0), -1))  # N x (ch x 24)
+        out = self.netfcaud(mid)
 
-        return out;
+        return out
 
     def forward_lip(self, x):
 
-        mid = self.netcnnlip(x); 
-        mid = mid.view((mid.size()[0], -1)); # N x (ch x 24)
-        out = self.netfclip(mid);
+        mid = self.netcnnlip(x)
+        mid = mid.view((mid.size(0), -1))  # N x (ch x 24)
+        out = self.netfclip(mid)
 
-        return out;
+        return out
 
     def forward_lipfeat(self, x):
 
-        mid = self.netcnnlip(x);
-        out = mid.view((mid.size()[0], -1)); # N x (ch x 24)
+        mid = self.netcnnlip(x)
+        out = mid.view((mid.size(0), -1))  # N x (ch x 24)
 
-        return out;
\ No newline at end of file
+        return out
diff --git a/demo_feature.py b/demo_feature.py
index e3bd290..416acfc 100755
--- a/demo_feature.py
+++ b/demo_feature.py
@@ -1,31 +1,34 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 
-import time, pdb, argparse, subprocess
+import time, pdb, argparse, subprocess, logging
 
 from SyncNetInstance import *
 
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
+
 # ==================== LOAD PARAMS ====================
 
 
-parser = argparse.ArgumentParser(description = "SyncNet");
+parser = argparse.ArgumentParser(description = "SyncNet")
 
-parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='');
-parser.add_argument('--batch_size', type=int, default='20', help='');
-parser.add_argument('--vshift', type=int, default='15', help='');
-parser.add_argument('--videofile', type=str, default="data/example.avi", help='');
-parser.add_argument('--tmp_dir', type=str, default="data", help='');
-parser.add_argument('--save_as', type=str, default="data/features.pt", help='');
+parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='')
+parser.add_argument('--batch_size', type=int, default='20', help='')
+parser.add_argument('--vshift', type=int, default='15', help='')
+parser.add_argument('--videofile', type=str, default="data/example.avi", help='')
+parser.add_argument('--tmp_dir', type=str, default="data", help='')
+parser.add_argument('--save_as', type=str, default="data/features.pt", help='')
 
-opt = parser.parse_args();
+opt = parser.parse_args()
 
 
 # ==================== RUN EVALUATION ====================
 
-s = SyncNetInstance();
+s = SyncNetInstance()
 
-s.loadParameters(opt.initial_model);
-print("Model %s loaded."%opt.initial_model);
+s.loadParameters(opt.initial_model)
+logger.info("Model %s loaded.", opt.initial_model)
 
 feats = s.extract_feature(opt, videofile=opt.videofile)
 
diff --git a/demo_syncnet.py b/demo_syncnet.py
index 01c25a6..8826b0a 100755
--- a/demo_syncnet.py
+++ b/demo_syncnet.py
@@ -1,30 +1,33 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 
-import time, pdb, argparse, subprocess
+import time, pdb, argparse, subprocess, logging
 
 from SyncNetInstance import *
 
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
+
 # ==================== LOAD PARAMS ====================
 
 
-parser = argparse.ArgumentParser(description = "SyncNet");
+parser = argparse.ArgumentParser(description = "SyncNet")
 
-parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='');
-parser.add_argument('--batch_size', type=int, default='20', help='');
-parser.add_argument('--vshift', type=int, default='15', help='');
-parser.add_argument('--videofile', type=str, default="data/example.avi", help='');
-parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help='');
-parser.add_argument('--reference', type=str, default="demo", help='');
+parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='')
+parser.add_argument('--batch_size', type=int, default='20', help='')
+parser.add_argument('--vshift', type=int, default='15', help='')
+parser.add_argument('--videofile', type=str, default="data/example.avi", help='')
+parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help='')
+parser.add_argument('--reference', type=str, default="demo", help='')
 
-opt = parser.parse_args();
+opt = parser.parse_args()
 
 
 # ==================== RUN EVALUATION ====================
 
-s = SyncNetInstance();
+s = SyncNetInstance()
 
-s.loadParameters(opt.initial_model);
-print("Model %s loaded."%opt.initial_model);
+s.loadParameters(opt.initial_model)
+logger.info("Model %s loaded.", opt.initial_model)
 
 s.evaluate(opt, videofile=opt.videofile)
diff --git a/detectors/s3fd/__init__.py b/detectors/s3fd/__init__.py
index d7f35e0..3c61da7 100644
--- a/detectors/s3fd/__init__.py
+++ b/detectors/s3fd/__init__.py
@@ -1,11 +1,12 @@
-import time
+import time, logging
 import numpy as np
 import cv2
 import torch
-from torchvision import transforms
 from .nets import S3FDNet
 from .box_utils import nms_
 
+logger = logging.getLogger(__name__)
+
 PATH_WEIGHT = './detectors/s3fd/weights/sfd_face.pth'
 img_mean = np.array([104., 117., 123.])[:, np.newaxis, np.newaxis].astype('float32')
 
@@ -17,12 +18,12 @@ def __init__(self, device='cuda'):
         tstamp = time.time()
         self.device = device
 
-        print('[S3FD] loading with', self.device)
+        logger.info('[S3FD] loading with %s', self.device)
         self.net = S3FDNet(device=self.device).to(self.device)
-        state_dict = torch.load(PATH_WEIGHT, map_location=self.device)
+        state_dict = torch.load(PATH_WEIGHT, map_location=self.device, weights_only=True)
         self.net.load_state_dict(state_dict)
         self.net.eval()
-        print('[S3FD] finished loading (%.4f sec)' % (time.time() - tstamp))
+        logger.info('[S3FD] finished loading (%.4f sec)', time.time() - tstamp)
     
     def detect_faces(self, image, conf_th=0.8, scales=[1]):
 
diff --git a/detectors/s3fd/box_utils.py b/detectors/s3fd/box_utils.py
index 0779bcd..00686a2 100644
--- a/detectors/s3fd/box_utils.py
+++ b/detectors/s3fd/box_utils.py
@@ -35,7 +35,7 @@ def nms_(dets, thresh):
         inds = np.where(ovr <= thresh)[0]
         order = order[inds + 1]
 
-    return np.array(keep).astype(np.int)
+    return np.array(keep).astype(np.intp)
 
 
 def decode(loc, priors, variances):
@@ -82,45 +82,32 @@ def nms(boxes, scores, overlap=0.5, top_k=200):
     v, idx = scores.sort(0)  # sort in ascending order
     # I = I[v >= 0.01]
     idx = idx[-top_k:]  # indices of the top-k largest vals
-    xx1 = boxes.new()
-    yy1 = boxes.new()
-    xx2 = boxes.new()
-    yy2 = boxes.new()
-    w = boxes.new()
-    h = boxes.new()
-
-    # keep = torch.Tensor()
+
     count = 0
     while idx.numel() > 0:
         i = idx[-1]  # index of current largest val
-        # keep.append(i)
         keep[count] = i
         count += 1
         if idx.size(0) == 1:
             break
         idx = idx[:-1]  # remove kept element from view
         # load bboxes of next highest vals
-        torch.index_select(x1, 0, idx, out=xx1)
-        torch.index_select(y1, 0, idx, out=yy1)
-        torch.index_select(x2, 0, idx, out=xx2)
-        torch.index_select(y2, 0, idx, out=yy2)
+        xx1 = torch.index_select(x1, 0, idx)
+        yy1 = torch.index_select(y1, 0, idx)
+        xx2 = torch.index_select(x2, 0, idx)
+        yy2 = torch.index_select(y2, 0, idx)
         # store element-wise max with next highest score
         xx1 = torch.clamp(xx1, min=x1[i])
         yy1 = torch.clamp(yy1, min=y1[i])
         xx2 = torch.clamp(xx2, max=x2[i])
         yy2 = torch.clamp(yy2, max=y2[i])
-        w.resize_as_(xx2)
-        h.resize_as_(yy2)
-        w = xx2 - xx1
-        h = yy2 - yy1
-        # check sizes of xx1 and xx2.. after each iteration
-        w = torch.clamp(w, min=0.0)
-        h = torch.clamp(h, min=0.0)
+        w = torch.clamp(xx2 - xx1, min=0.0)
+        h = torch.clamp(yy2 - yy1, min=0.0)
         inter = w * h
         # IoU = i / (area(a) + area(b) - i)
-        rem_areas = torch.index_select(area, 0, idx)  # load remaining areas)
+        rem_areas = torch.index_select(area, 0, idx)
         union = (rem_areas - inter) + area[i]
-        IoU = inter / union  # store result in iou
+        IoU = inter / union
         # keep only elements with an IoU <= overlap
         idx = idx[IoU.le(overlap)]
     return keep, count
@@ -181,7 +168,7 @@ def __init__(self, input_size, feature_maps,
                     steps=[4, 8, 16, 32, 64, 128],
                     clip=False):
 
-        super(PriorBox, self).__init__()
+        super().__init__()
 
         self.imh = input_size[0]
         self.imw = input_size[1]
diff --git a/detectors/s3fd/nets.py b/detectors/s3fd/nets.py
index 85b5c82..937a73f 100644
--- a/detectors/s3fd/nets.py
+++ b/detectors/s3fd/nets.py
@@ -8,7 +8,7 @@
 class L2Norm(nn.Module):
 
     def __init__(self, n_channels, scale):
-        super(L2Norm, self).__init__()
+        super().__init__()
         self.n_channels = n_channels
         self.gamma = scale or None
         self.eps = 1e-10
@@ -28,7 +28,7 @@ def forward(self, x):
 class S3FDNet(nn.Module):
 
     def __init__(self, device='cuda'):
-        super(S3FDNet, self).__init__()
+        super().__init__()
         self.device = device
 
         self.vgg = nn.ModuleList([
diff --git a/download_model.sh b/download_model.sh
index 3e3a9dc..34895d9 100755
--- a/download_model.sh
+++ b/download_model.sh
@@ -1,9 +1,9 @@
 # SyncNet model
 
-mkdir data
+mkdir -p data
 wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/syncnet_v2.model -O data/syncnet_v2.model
 wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/example.avi -O data/example.avi
 
 # For the pre-processing pipeline
-mkdir detectors/s3fd/weights
+mkdir -p detectors/s3fd/weights
 wget https://www.robots.ox.ac.uk/~vgg/software/lipsync/data/sfd_face.pth -O detectors/s3fd/weights/sfd_face.pth
\ No newline at end of file
diff --git a/environment-cpu.yml b/environment-cpu.yml
new file mode 100644
index 0000000..9dc3b99
--- /dev/null
+++ b/environment-cpu.yml
@@ -0,0 +1,27 @@
+name: syncnet
+channels:
+  - conda-forge
+  - pytorch
+  - defaults
+dependencies:
+  # Core Python and Math Libraries
+  - python=3.10
+  - numpy
+  - scipy
+  
+  # PyTorch Ecosystem
+  - pytorch::pytorch==2.5.1
+  - pytorch::torchvision==0.20.1
+  - pytorch::torchaudio==2.5.1
+  
+  # External Tools
+  - ffmpeg
+  
+  # Pip Installer
+  - pip
+  
+  # Pip-specific packages (Runs after Conda finishes)
+  - pip:
+    - scenedetect==0.6.7.1
+    - opencv-contrib-python==4.13.0.92
+    - python_speech_features==0.6
diff --git a/environment.yml b/environment.yml
new file mode 100644
index 0000000..edd13be
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,29 @@
+name: syncnet
+channels:
+  - conda-forge
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  # Core Python and Math Libraries
+  - python=3.10
+  - numpy
+  - scipy
+  
+  # PyTorch Ecosystem
+  - pytorch::pytorch==2.5.1
+  - pytorch::torchvision==0.20.1
+  - pytorch::torchaudio==2.5.1
+  - pytorch::pytorch-cuda=12.4
+  
+  # External Tools
+  - ffmpeg
+  
+  # Pip Installer
+  - pip
+  
+  # Pip-specific packages (Runs after Conda finishes)
+  - pip:
+    - scenedetect==0.6.7.1
+    - opencv-contrib-python==4.13.0.92
+    - python_speech_features==0.6
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 8919740..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-torch>=1.4.0
-torchvision>=0.5.0
-numpy>=1.18.1
-scipy>=1.2.1
-scenedetect==0.5.1
-opencv-contrib-python
-python_speech_features
diff --git a/run_pipeline.py b/run_pipeline.py
index f5fc22e..5632ed4 100755
--- a/run_pipeline.py
+++ b/run_pipeline.py
@@ -1,15 +1,14 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
-import sys, time, os, pdb, argparse, pickle, subprocess, glob, cv2
+import sys, time, os, pdb, argparse, pickle, subprocess, glob, cv2, logging
 import numpy as np
+import torch
 from shutil import rmtree
 
-import scenedetect
-from scenedetect.video_manager import VideoManager
-from scenedetect.scene_manager import SceneManager
-from scenedetect.frame_timecode import FrameTimecode
-from scenedetect.stats_manager import StatsManager
-from scenedetect.detectors import ContentDetector
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
+
+from scenedetect import open_video, SceneManager, ContentDetector
 
 from scipy.interpolate import interp1d
 from scipy.io import wavfile
@@ -21,17 +20,17 @@
 # # PARSE ARGS
 # ========== ========== ========== ==========
 
-parser = argparse.ArgumentParser(description = "FaceTracker");
-parser.add_argument('--data_dir',       type=str, default='data/work', help='Output direcotry');
-parser.add_argument('--videofile',      type=str, default='',   help='Input video file');
-parser.add_argument('--reference',      type=str, default='',   help='Video reference');
-parser.add_argument('--facedet_scale',  type=float, default=0.25, help='Scale factor for face detection');
-parser.add_argument('--crop_scale',     type=float, default=0.40, help='Scale bounding box');
-parser.add_argument('--min_track',      type=int, default=100,  help='Minimum facetrack duration');
-parser.add_argument('--frame_rate',     type=int, default=25,   help='Frame rate');
-parser.add_argument('--num_failed_det', type=int, default=25,   help='Number of missed detections allowed before tracking is stopped');
-parser.add_argument('--min_face_size',  type=int, default=100,  help='Minimum face size in pixels');
-opt = parser.parse_args();
+parser = argparse.ArgumentParser(description = "FaceTracker")
+parser.add_argument('--data_dir',       type=str, default='data/work', help='Output direcotry')
+parser.add_argument('--videofile',      type=str, default='',   help='Input video file')
+parser.add_argument('--reference',      type=str, default='',   help='Video reference')
+parser.add_argument('--facedet_scale',  type=float, default=0.25, help='Scale factor for face detection')
+parser.add_argument('--crop_scale',     type=float, default=0.40, help='Scale bounding box')
+parser.add_argument('--min_track',      type=int, default=100,  help='Minimum facetrack duration')
+parser.add_argument('--frame_rate',     type=int, default=25,   help='Frame rate')
+parser.add_argument('--num_failed_det', type=int, default=25,   help='Number of missed detections allowed before tracking is stopped')
+parser.add_argument('--min_face_size',  type=int, default=100,  help='Minimum face size in pixels')
+opt = parser.parse_args()
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
 setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp'))
@@ -44,19 +43,19 @@
 # ========== ========== ========== ==========
 
 def bb_intersection_over_union(boxA, boxB):
-  
+
   xA = max(boxA[0], boxB[0])
   yA = max(boxA[1], boxB[1])
   xB = min(boxA[2], boxB[2])
   yB = min(boxA[3], boxB[3])
- 
+
   interArea = max(0, xB - xA) * max(0, yB - yA)
- 
+
   boxAArea = (boxA[2] - boxA[0]) * (boxA[3] - boxA[1])
   boxBArea = (boxB[2] - boxB[0]) * (boxB[3] - boxB[1])
- 
+
   iou = interArea / float(boxAArea + boxBArea - interArea)
- 
+
   return iou
 
 # ========== ========== ========== ==========
@@ -87,7 +86,7 @@ def track_shot(opt,scenefaces):
     if track == []:
       break
     elif len(track) > opt.min_track:
-      
+
       framenum    = np.array([ f['frame'] for f in track ])
       bboxes      = np.array([np.array(f['bbox']) for f in track])
 
@@ -107,7 +106,7 @@ def track_shot(opt,scenefaces):
 # ========== ========== ========== ==========
 # # VIDEO CROP AND SAVE
 # ========== ========== ========== ==========
-        
+
 def crop_video(opt,track,cropfile):
 
   flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg'))
@@ -120,12 +119,12 @@ def crop_video(opt,track,cropfile):
 
   for det in track['bbox']:
 
-    dets['s'].append(max((det[3]-det[1]),(det[2]-det[0]))/2) 
-    dets['y'].append((det[1]+det[3])/2) # crop center x 
+    dets['s'].append(max((det[3]-det[1]),(det[2]-det[0]))/2)
+    dets['y'].append((det[1]+det[3])/2) # crop center x
     dets['x'].append((det[0]+det[2])/2) # crop center y
 
   # Smooth detections
-  dets['s'] = signal.medfilt(dets['s'],kernel_size=13)   
+  dets['s'] = signal.medfilt(dets['s'],kernel_size=13)
   dets['x'] = signal.medfilt(dets['x'],kernel_size=13)
   dets['y'] = signal.medfilt(dets['y'],kernel_size=13)
 
@@ -134,16 +133,16 @@ def crop_video(opt,track,cropfile):
     cs  = opt.crop_scale
 
     bs  = dets['s'][fidx]   # Detection box size
-    bsi = int(bs*(1+2*cs))  # Pad videos by this amount 
+    bsi = int(bs*(1+2*cs))  # Pad videos by this amount
 
     image = cv2.imread(flist[frame])
-    
+
     frame = np.pad(image,((bsi,bsi),(bsi,bsi),(0,0)), 'constant', constant_values=(110,110))
     my  = dets['y'][fidx]+bsi  # BBox center Y
     mx  = dets['x'][fidx]+bsi  # BBox center X
 
     face = frame[int(my-bs):int(my+bs*(1+2*cs)),int(mx-bs*(1+cs)):int(mx+bs*(1+cs))]
-    
+
     vOut.write(cv2.resize(face,(224,224)))
 
   audiotmp    = os.path.join(opt.tmp_dir,opt.reference,'audio.wav')
@@ -154,27 +153,25 @@ def crop_video(opt,track,cropfile):
 
   # ========== CROP AUDIO FILE ==========
 
-  command = ("ffmpeg -y -i %s -ss %.3f -to %.3f %s" % (os.path.join(opt.avi_dir,opt.reference,'audio.wav'),audiostart,audioend,audiotmp)) 
-  output = subprocess.call(command, shell=True, stdout=None)
-
-  if output != 0:
-    pdb.set_trace()
+  command = ["ffmpeg", "-y", "-i",
+             os.path.join(opt.avi_dir, opt.reference, 'audio.wav'),
+             "-ss", "%.3f" % audiostart, "-to", "%.3f" % audioend,
+             audiotmp]
+  subprocess.run(command, check=True)
 
   sample_rate, audio = wavfile.read(audiotmp)
 
   # ========== COMBINE AUDIO AND VIDEO FILES ==========
 
-  command = ("ffmpeg -y -i %st.avi -i %s -c:v copy -c:a copy %s.avi" % (cropfile,audiotmp,cropfile))
-  output = subprocess.call(command, shell=True, stdout=None)
-
-  if output != 0:
-    pdb.set_trace()
+  command = ["ffmpeg", "-y", "-i", cropfile+'t.avi', "-i", audiotmp,
+             "-c:v", "copy", "-c:a", "copy", cropfile+'.avi']
+  subprocess.run(command, check=True)
 
-  print('Written %s'%cropfile)
+  logger.info('Written %s', cropfile)
 
   os.remove(cropfile+'t.avi')
 
-  print('Mean pos: x %.2f y %.2f s %.2f'%(np.mean(dets['x']),np.mean(dets['y']),np.mean(dets['s'])))
+  logger.info('Mean pos: x %.2f y %.2f s %.2f', np.mean(dets['x']),np.mean(dets['y']),np.mean(dets['s']))
 
   return {'track':track, 'proc_track':dets}
 
@@ -184,29 +181,30 @@ def crop_video(opt,track,cropfile):
 
 def inference_video(opt):
 
-  DET = S3FD(device='cuda')
+  device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  DET = S3FD(device=device)
 
   flist = glob.glob(os.path.join(opt.frames_dir,opt.reference,'*.jpg'))
   flist.sort()
 
   dets = []
-      
+
   for fidx, fname in enumerate(flist):
 
     start_time = time.time()
-    
+
     image = cv2.imread(fname)
 
     image_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     bboxes = DET.detect_faces(image_np, conf_th=0.9, scales=[opt.facedet_scale])
 
-    dets.append([]);
+    dets.append([])
     for bbox in bboxes:
       dets[-1].append({'frame':fidx, 'bbox':(bbox[:-1]).tolist(), 'conf':bbox[-1]})
 
     elapsed_time = time.time() - start_time
 
-    print('%s-%05d; %d dets; %.2f Hz' % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),fidx,len(dets[-1]),(1/elapsed_time))) 
+    logger.info('%s-%05d; %d dets; %.2f Hz', os.path.join(opt.avi_dir,opt.reference,'video.avi'),fidx,len(dets[-1]),(1/elapsed_time))
 
   savepath = os.path.join(opt.work_dir,opt.reference,'faces.pckl')
 
@@ -221,33 +219,28 @@ def inference_video(opt):
 
 def scene_detect(opt):
 
-  video_manager = VideoManager([os.path.join(opt.avi_dir,opt.reference,'video.avi')])
-  stats_manager = StatsManager()
-  scene_manager = SceneManager(stats_manager)
-  # Add ContentDetector algorithm (constructor takes detector options like threshold).
-  scene_manager.add_detector(ContentDetector())
-  base_timecode = video_manager.get_base_timecode()
-
-  video_manager.set_downscale_factor()
-
-  video_manager.start()
+  video_path = os.path.join(opt.avi_dir,opt.reference,'video.avi')
+  video = open_video(video_path)
 
-  scene_manager.detect_scenes(frame_source=video_manager)
+  scene_manager = SceneManager()
+  scene_manager.add_detector(ContentDetector())
+  scene_manager.detect_scenes(video)
 
-  scene_list = scene_manager.get_scene_list(base_timecode)
+  scene_list = scene_manager.get_scene_list()
 
   savepath = os.path.join(opt.work_dir,opt.reference,'scene.pckl')
 
-  if scene_list == []:
-    scene_list = [(video_manager.get_base_timecode(),video_manager.get_current_timecode())]
+  if not scene_list:
+    video = open_video(video_path)
+    scene_list = [(video.base_timecode, video.base_timecode + video.duration)]
 
   with open(savepath, 'wb') as fil:
     pickle.dump(scene_list, fil)
 
-  print('%s - scenes detected %d'%(os.path.join(opt.avi_dir,opt.reference,'video.avi'),len(scene_list)))
+  logger.info('%s - scenes detected %d', video_path, len(scene_list))
 
   return scene_list
-    
+
 
 # ========== ========== ========== ==========
 # # EXECUTE DEMO
@@ -255,39 +248,31 @@ def scene_detect(opt):
 
 # ========== DELETE EXISTING DIRECTORIES ==========
 
-if os.path.exists(os.path.join(opt.work_dir,opt.reference)):
-  rmtree(os.path.join(opt.work_dir,opt.reference))
-
-if os.path.exists(os.path.join(opt.crop_dir,opt.reference)):
-  rmtree(os.path.join(opt.crop_dir,opt.reference))
-
-if os.path.exists(os.path.join(opt.avi_dir,opt.reference)):
-  rmtree(os.path.join(opt.avi_dir,opt.reference))
-
-if os.path.exists(os.path.join(opt.frames_dir,opt.reference)):
-  rmtree(os.path.join(opt.frames_dir,opt.reference))
-
-if os.path.exists(os.path.join(opt.tmp_dir,opt.reference)):
-  rmtree(os.path.join(opt.tmp_dir,opt.reference))
+for d in [opt.work_dir, opt.crop_dir, opt.avi_dir, opt.frames_dir, opt.tmp_dir]:
+  path = os.path.join(d, opt.reference)
+  if os.path.exists(path):
+    rmtree(path)
 
 # ========== MAKE NEW DIRECTORIES ==========
 
-os.makedirs(os.path.join(opt.work_dir,opt.reference))
-os.makedirs(os.path.join(opt.crop_dir,opt.reference))
-os.makedirs(os.path.join(opt.avi_dir,opt.reference))
-os.makedirs(os.path.join(opt.frames_dir,opt.reference))
-os.makedirs(os.path.join(opt.tmp_dir,opt.reference))
+for d in [opt.work_dir, opt.crop_dir, opt.avi_dir, opt.frames_dir, opt.tmp_dir]:
+  os.makedirs(os.path.join(d, opt.reference), exist_ok=True)
 
 # ========== CONVERT VIDEO AND EXTRACT FRAMES ==========
 
-command = ("ffmpeg -y -i %s -qscale:v 2 -async 1 -r 25 %s" % (opt.videofile,os.path.join(opt.avi_dir,opt.reference,'video.avi')))
-output = subprocess.call(command, shell=True, stdout=None)
+command = ["ffmpeg", "-y", "-i", opt.videofile, "-qscale:v", "2", "-async", "1", "-r", "25",
+           os.path.join(opt.avi_dir, opt.reference, 'video.avi')]
+subprocess.run(command, check=True)
 
-command = ("ffmpeg -y -i %s -qscale:v 2 -threads 1 -f image2 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.frames_dir,opt.reference,'%06d.jpg'))) 
-output = subprocess.call(command, shell=True, stdout=None)
+command = ["ffmpeg", "-y", "-i", os.path.join(opt.avi_dir, opt.reference, 'video.avi'),
+           "-qscale:v", "2", "-threads", "1", "-f", "image2",
+           os.path.join(opt.frames_dir, opt.reference, '%06d.jpg')]
+subprocess.run(command, check=True)
 
-command = ("ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (os.path.join(opt.avi_dir,opt.reference,'video.avi'),os.path.join(opt.avi_dir,opt.reference,'audio.wav'))) 
-output = subprocess.call(command, shell=True, stdout=None)
+command = ["ffmpeg", "-y", "-i", os.path.join(opt.avi_dir, opt.reference, 'video.avi'),
+           "-ac", "1", "-vn", "-acodec", "pcm_s16le", "-ar", "16000",
+           os.path.join(opt.avi_dir, opt.reference, 'audio.wav')]
+subprocess.run(command, check=True)
 
 # ========== FACE DETECTION ==========
 
@@ -304,8 +289,8 @@ def scene_detect(opt):
 
 for shot in scene:
 
-  if shot[1].frame_num - shot[0].frame_num >= opt.min_track :
-    alltracks.extend(track_shot(opt,faces[shot[0].frame_num:shot[1].frame_num]))
+  if shot[1].get_frames() - shot[0].get_frames() >= opt.min_track :
+    alltracks.extend(track_shot(opt,faces[shot[0].get_frames():shot[1].get_frames()]))
 
 # ========== FACE TRACK CROP ==========
 
diff --git a/run_syncnet.py b/run_syncnet.py
index 45099fd..dc9c2c0 100755
--- a/run_syncnet.py
+++ b/run_syncnet.py
@@ -1,20 +1,23 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 
-import time, pdb, argparse, subprocess, pickle, os, gzip, glob
+import time, pdb, argparse, subprocess, pickle, os, gzip, glob, logging
 
 from SyncNetInstance import *
 
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
+
 # ==================== PARSE ARGUMENT ====================
 
-parser = argparse.ArgumentParser(description = "SyncNet");
-parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='');
-parser.add_argument('--batch_size', type=int, default='20', help='');
-parser.add_argument('--vshift', type=int, default='15', help='');
-parser.add_argument('--data_dir', type=str, default='data/work', help='');
-parser.add_argument('--videofile', type=str, default='', help='');
-parser.add_argument('--reference', type=str, default='', help='');
-opt = parser.parse_args();
+parser = argparse.ArgumentParser(description = "SyncNet")
+parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='')
+parser.add_argument('--batch_size', type=int, default='20', help='')
+parser.add_argument('--vshift', type=int, default='15', help='')
+parser.add_argument('--data_dir', type=str, default='data/work', help='')
+parser.add_argument('--videofile', type=str, default='', help='')
+parser.add_argument('--reference', type=str, default='', help='')
+opt = parser.parse_args()
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
 setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp'))
@@ -24,10 +27,10 @@
 
 # ==================== LOAD MODEL AND FILE LIST ====================
 
-s = SyncNetInstance();
+s = SyncNetInstance()
 
-s.loadParameters(opt.initial_model);
-print("Model %s loaded."%opt.initial_model);
+s.loadParameters(opt.initial_model)
+logger.info("Model %s loaded.", opt.initial_model)
 
 flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi'))
 flist.sort()
@@ -38,7 +41,7 @@
 for idx, fname in enumerate(flist):
     offset, conf, dist = s.evaluate(opt,videofile=fname)
     dists.append(dist)
-      
+
 # ==================== PRINT RESULTS TO FILE ====================
 
 with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil:
diff --git a/run_visualise.py b/run_visualise.py
index 85d8925..6148963 100644
--- a/run_visualise.py
+++ b/run_visualise.py
@@ -1,21 +1,24 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #-*- coding: utf-8 -*-
 
 import torch
 import numpy
-import time, pdb, argparse, subprocess, pickle, os, glob
+import time, pdb, argparse, subprocess, pickle, os, glob, logging
 import cv2
 
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
+
 from scipy import signal
 
 # ==================== PARSE ARGUMENT ====================
 
-parser = argparse.ArgumentParser(description = "SyncNet");
-parser.add_argument('--data_dir', 	type=str, default='data/work', help='');
-parser.add_argument('--videofile', 	type=str, default='', help='');
-parser.add_argument('--reference', 	type=str, default='', help='');
-parser.add_argument('--frame_rate', type=int, default=25, help='Frame rate');
-opt = parser.parse_args();
+parser = argparse.ArgumentParser(description = "SyncNet")
+parser.add_argument('--data_dir', 	type=str, default='data/work', help='')
+parser.add_argument('--videofile', 	type=str, default='', help='')
+parser.add_argument('--reference', 	type=str, default='', help='')
+parser.add_argument('--frame_rate', type=int, default=25, help='Frame rate')
+opt = parser.parse_args()
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
 setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp'))
@@ -42,8 +45,8 @@
 
 	mean_dists 	=  numpy.mean(numpy.stack(dists[tidx],1),1)
 	minidx 		= numpy.argmin(mean_dists,0)
-	minval 		= mean_dists[minidx] 
-	
+	minval 		= mean_dists[minidx]
+
 	fdist   	= numpy.stack([dist[minidx] for dist in dists[tidx]])
 	fdist   	= numpy.pad(fdist, (3,3), 'constant', constant_values=10)
 
@@ -69,20 +72,22 @@
 
 	for face in faces[fidx]:
 
-		clr = max(min(face['conf']*25,255),0)
+		clr = int(max(min(face['conf']*25,255),0))
 
 		cv2.rectangle(image,(int(face['x']-face['s']),int(face['y']-face['s'])),(int(face['x']+face['s']),int(face['y']+face['s'])),(0,clr,255-clr),3)
 		cv2.putText(image,'Track %d, Conf %.3f'%(face['track'],face['conf']), (int(face['x']-face['s']),int(face['y']-face['s'])),cv2.FONT_HERSHEY_SIMPLEX,0.5,(255,255,255),2)
 
 	vOut.write(image)
 
-	print('Frame %d'%fidx)
+	logger.info('Frame %d', fidx)
 
 vOut.release()
 
 # ========== COMBINE AUDIO AND VIDEO FILES ==========
 
-command = ("ffmpeg -y -i %s -i %s -c:v copy -c:a copy %s" % (os.path.join(opt.avi_dir,opt.reference,'video_only.avi'),os.path.join(opt.avi_dir,opt.reference,'audio.wav'),os.path.join(opt.avi_dir,opt.reference,'video_out.avi'))) #-async 1 
-output = subprocess.call(command, shell=True, stdout=None)
-
-
+command = ["ffmpeg", "-y", "-i",
+           os.path.join(opt.avi_dir, opt.reference, 'video_only.avi'),
+           "-i", os.path.join(opt.avi_dir, opt.reference, 'audio.wav'),
+           "-c:v", "copy", "-c:a", "copy",
+           os.path.join(opt.avi_dir, opt.reference, 'video_out.avi')]
+subprocess.run(command, check=True)

From 1fbfc081bda41f27aab51c0fbaa0e5e337126cf3 Mon Sep 17 00:00:00 2001
From: joonson <joonson@kaist.ac.kr>
Date: Sat, 11 Apr 2026 22:55:06 +0900
Subject: [PATCH 4/5] Minor bug fixes

---
 SyncNetInstance.py | 6 +++---
 SyncNetModel.py    | 9 ---------
 run_pipeline.py    | 1 -
 3 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/SyncNetInstance.py b/SyncNetInstance.py
index 54c92ec..d23e1b4 100644
--- a/SyncNetInstance.py
+++ b/SyncNetInstance.py
@@ -149,9 +149,9 @@ def evaluate(self, opt, videofile):
         numpy.set_printoptions(formatter={'float': '{: 0.3f}'.format})
         logger.info('Framewise conf: ')
         logger.info(fconfm)
-        logger.info('AV offset: \t%d', offset)
-        logger.info('Min dist: \t%.3f', minval)
-        logger.info('Confidence: \t%.3f', conf)
+        logger.info('AV offset: \t%d', offset.item())
+        logger.info('Min dist: \t%.3f', minval.item())
+        logger.info('Confidence: \t%.3f', conf.item())
 
         dists_npy = numpy.array([ dist.numpy() for dist in dists ])
         return offset.numpy(), conf.numpy(), dists_npy
diff --git a/SyncNetModel.py b/SyncNetModel.py
index 9953b10..cf97caf 100755
--- a/SyncNetModel.py
+++ b/SyncNetModel.py
@@ -4,15 +4,6 @@
 import torch
 import torch.nn as nn
 
-def save(model, filename):
-    with open(filename, "wb") as f:
-        torch.save(model, f)
-        print(f"{filename} saved.")
-
-def load(filename):
-    net = torch.load(filename, weights_only=True)
-    return net
-
 class S(nn.Module):
     def __init__(self, num_layers_in_fc_layers = 1024):
         super().__init__()
diff --git a/run_pipeline.py b/run_pipeline.py
index 5632ed4..50fee52 100755
--- a/run_pipeline.py
+++ b/run_pipeline.py
@@ -231,7 +231,6 @@ def scene_detect(opt):
   savepath = os.path.join(opt.work_dir,opt.reference,'scene.pckl')
 
   if not scene_list:
-    video = open_video(video_path)
     scene_list = [(video.base_timecode, video.base_timecode + video.duration)]
 
   with open(savepath, 'wb') as fil:

From d3e3d44e7a64c336e92591266061d1a6860da106 Mon Sep 17 00:00:00 2001
From: joonson <joonson@kaist.ac.kr>
Date: Fri, 17 Apr 2026 15:40:13 +0900
Subject: [PATCH 5/5] logging changes

---
 README.md           |  8 ++++++++
 SyncNetInstance.py  |  6 ++++--
 environment-cpu.yml |  1 +
 environment.yml     |  1 +
 run_pipeline.py     | 43 +++++++++++++++++++++++++------------------
 run_visualise.py    | 10 +++++-----
 6 files changed, 44 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index d3473b0..b2e2772 100755
--- a/README.md
+++ b/README.md
@@ -8,10 +8,18 @@ Please cite the paper below if you make use of the software.
 
 ## Dependencies
 
+GPU (CUDA):
 ```
 conda env create -f environment.yml
 ```
 
+CPU only:
+```
+conda env create -f environment-cpu.yml
+```
+
+The code automatically detects and uses a CUDA GPU if available, and falls back to CPU otherwise.
+
 
 ## Getting Started
 
diff --git a/SyncNetInstance.py b/SyncNetInstance.py
index d23e1b4..3d6286a 100644
--- a/SyncNetInstance.py
+++ b/SyncNetInstance.py
@@ -56,11 +56,13 @@ def evaluate(self, opt, videofile):
 
         os.makedirs(os.path.join(opt.tmp_dir,opt.reference))
 
-        command = ["ffmpeg", "-y", "-i", videofile, "-threads", "1", "-f", "image2",
+        logger.info('Extracting video frames from %s', videofile)
+        command = ["ffmpeg", "-y", "-loglevel", "error", "-i", videofile, "-threads", "1", "-f", "image2",
                    os.path.join(opt.tmp_dir, opt.reference, '%06d.jpg')]
         subprocess.run(command, check=True)
 
-        command = ["ffmpeg", "-y", "-i", videofile, "-async", "1", "-ac", "1", "-vn",
+        logger.info('Extracting audio from %s', videofile)
+        command = ["ffmpeg", "-y", "-loglevel", "error", "-i", videofile, "-async", "1", "-ac", "1", "-vn",
                    "-acodec", "pcm_s16le", "-ar", "16000",
                    os.path.join(opt.tmp_dir, opt.reference, 'audio.wav')]
         subprocess.run(command, check=True)
diff --git a/environment-cpu.yml b/environment-cpu.yml
index 9dc3b99..1804714 100644
--- a/environment-cpu.yml
+++ b/environment-cpu.yml
@@ -25,3 +25,4 @@ dependencies:
     - scenedetect==0.6.7.1
     - opencv-contrib-python==4.13.0.92
     - python_speech_features==0.6
+    - tqdm
diff --git a/environment.yml b/environment.yml
index edd13be..03ce3c4 100644
--- a/environment.yml
+++ b/environment.yml
@@ -27,3 +27,4 @@ dependencies:
     - scenedetect==0.6.7.1
     - opencv-contrib-python==4.13.0.92
     - python_speech_features==0.6
+    - tqdm
diff --git a/run_pipeline.py b/run_pipeline.py
index 50fee52..8305fb3 100755
--- a/run_pipeline.py
+++ b/run_pipeline.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import sys, time, os, pdb, argparse, pickle, subprocess, glob, cv2, logging
+import sys, time, os, argparse, pickle, subprocess, glob, cv2, logging
 import numpy as np
 import torch
 from shutil import rmtree
@@ -9,6 +9,7 @@
 logger = logging.getLogger(__name__)
 
 from scenedetect import open_video, SceneManager, ContentDetector
+from tqdm import tqdm
 
 from scipy.interpolate import interp1d
 from scipy.io import wavfile
@@ -30,6 +31,7 @@
 parser.add_argument('--frame_rate',     type=int, default=25,   help='Frame rate')
 parser.add_argument('--num_failed_det', type=int, default=25,   help='Number of missed detections allowed before tracking is stopped')
 parser.add_argument('--min_face_size',  type=int, default=100,  help='Minimum face size in pixels')
+parser.add_argument('--overwrite',      action='store_true',    help='Overwrite existing output directories')
 opt = parser.parse_args()
 
 setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi'))
@@ -153,7 +155,8 @@ def crop_video(opt,track,cropfile):
 
   # ========== CROP AUDIO FILE ==========
 
-  command = ["ffmpeg", "-y", "-i",
+  logger.info('Cropping audio track for %s', cropfile)
+  command = ["ffmpeg", "-y", "-loglevel", "error", "-i",
              os.path.join(opt.avi_dir, opt.reference, 'audio.wav'),
              "-ss", "%.3f" % audiostart, "-to", "%.3f" % audioend,
              audiotmp]
@@ -163,7 +166,8 @@ def crop_video(opt,track,cropfile):
 
   # ========== COMBINE AUDIO AND VIDEO FILES ==========
 
-  command = ["ffmpeg", "-y", "-i", cropfile+'t.avi', "-i", audiotmp,
+  logger.info('Merging audio and video for %s', cropfile)
+  command = ["ffmpeg", "-y", "-loglevel", "error", "-i", cropfile+'t.avi', "-i", audiotmp,
              "-c:v", "copy", "-c:a", "copy", cropfile+'.avi']
   subprocess.run(command, check=True)
 
@@ -189,22 +193,19 @@ def inference_video(opt):
 
   dets = []
 
-  for fidx, fname in enumerate(flist):
+  with tqdm(enumerate(flist), total=len(flist), desc='Detecting faces') as pbar:
+    for fidx, fname in pbar:
 
-    start_time = time.time()
+      image = cv2.imread(fname)
 
-    image = cv2.imread(fname)
+      image_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+      bboxes = DET.detect_faces(image_np, conf_th=0.9, scales=[opt.facedet_scale])
 
-    image_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-    bboxes = DET.detect_faces(image_np, conf_th=0.9, scales=[opt.facedet_scale])
+      dets.append([])
+      for bbox in bboxes:
+        dets[-1].append({'frame':fidx, 'bbox':(bbox[:-1]).tolist(), 'conf':bbox[-1]})
 
-    dets.append([])
-    for bbox in bboxes:
-      dets[-1].append({'frame':fidx, 'bbox':(bbox[:-1]).tolist(), 'conf':bbox[-1]})
-
-    elapsed_time = time.time() - start_time
-
-    logger.info('%s-%05d; %d dets; %.2f Hz', os.path.join(opt.avi_dir,opt.reference,'video.avi'),fidx,len(dets[-1]),(1/elapsed_time))
+      pbar.set_postfix(dets=len(dets[-1]))
 
   savepath = os.path.join(opt.work_dir,opt.reference,'faces.pckl')
 
@@ -250,6 +251,9 @@ def scene_detect(opt):
 for d in [opt.work_dir, opt.crop_dir, opt.avi_dir, opt.frames_dir, opt.tmp_dir]:
   path = os.path.join(d, opt.reference)
   if os.path.exists(path):
+    if not opt.overwrite:
+      sys.exit(f"Output directory already exists: {path}. Use --overwrite to overwrite.")
+    logger.warning('Overwriting existing directory: %s', path)
     rmtree(path)
 
 # ========== MAKE NEW DIRECTORIES ==========
@@ -259,16 +263,19 @@ def scene_detect(opt):
 
 # ========== CONVERT VIDEO AND EXTRACT FRAMES ==========
 
-command = ["ffmpeg", "-y", "-i", opt.videofile, "-qscale:v", "2", "-async", "1", "-r", "25",
+logger.info('Converting video to 25fps: %s', opt.videofile)
+command = ["ffmpeg", "-y", "-loglevel", "error", "-i", opt.videofile, "-qscale:v", "2", "-async", "1", "-r", "25",
            os.path.join(opt.avi_dir, opt.reference, 'video.avi')]
 subprocess.run(command, check=True)
 
-command = ["ffmpeg", "-y", "-i", os.path.join(opt.avi_dir, opt.reference, 'video.avi'),
+logger.info('Extracting frames from video')
+command = ["ffmpeg", "-y", "-loglevel", "error", "-i", os.path.join(opt.avi_dir, opt.reference, 'video.avi'),
            "-qscale:v", "2", "-threads", "1", "-f", "image2",
            os.path.join(opt.frames_dir, opt.reference, '%06d.jpg')]
 subprocess.run(command, check=True)
 
-command = ["ffmpeg", "-y", "-i", os.path.join(opt.avi_dir, opt.reference, 'video.avi'),
+logger.info('Extracting audio from video')
+command = ["ffmpeg", "-y", "-loglevel", "error", "-i", os.path.join(opt.avi_dir, opt.reference, 'video.avi'),
            "-ac", "1", "-vn", "-acodec", "pcm_s16le", "-ar", "16000",
            os.path.join(opt.avi_dir, opt.reference, 'audio.wav')]
 subprocess.run(command, check=True)
diff --git a/run_visualise.py b/run_visualise.py
index 6148963..5f0f21e 100644
--- a/run_visualise.py
+++ b/run_visualise.py
@@ -3,8 +3,9 @@
 
 import torch
 import numpy
-import time, pdb, argparse, subprocess, pickle, os, glob, logging
+import time, argparse, subprocess, pickle, os, glob, logging
 import cv2
+from tqdm import tqdm
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)s %(levelname)s: %(message)s')
 logger = logging.getLogger(__name__)
@@ -66,7 +67,7 @@
 fourcc = cv2.VideoWriter_fourcc(*'XVID')
 vOut = cv2.VideoWriter(os.path.join(opt.avi_dir,opt.reference,'video_only.avi'), fourcc, opt.frame_rate, (fw,fh))
 
-for fidx, fname in enumerate(flist):
+for fidx, fname in tqdm(enumerate(flist), total=len(flist), desc='Rendering'):
 
 	image = cv2.imread(fname)
 
@@ -79,13 +80,12 @@
 
 	vOut.write(image)
 
-	logger.info('Frame %d', fidx)
-
 vOut.release()
 
 # ========== COMBINE AUDIO AND VIDEO FILES ==========
 
-command = ["ffmpeg", "-y", "-i",
+logger.info('Combining audio and video into output file')
+command = ["ffmpeg", "-y", "-loglevel", "error", "-i",
            os.path.join(opt.avi_dir, opt.reference, 'video_only.avi'),
            "-i", os.path.join(opt.avi_dir, opt.reference, 'audio.wav'),
            "-c:v", "copy", "-c:a", "copy",