From e2b0738b03e43d1d87afdd67758daecab9a5f188 Mon Sep 17 00:00:00 2001 From: ehsainit Date: Wed, 10 Jun 2020 16:51:40 +0200 Subject: [PATCH 01/10] intergrating single_object tracker into current DLC --- deeplabcut/gui/analyze_videos.py | 4 +- deeplabcut/gui/create_videos.py | 2 +- .../lib/single_object_tracker.py | 105 +++++++++ .../lib/trackingutils.py | 11 + .../predict_videos.py | 210 +++++++++--------- 5 files changed, 230 insertions(+), 102 deletions(-) create mode 100644 deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py diff --git a/deeplabcut/gui/analyze_videos.py b/deeplabcut/gui/analyze_videos.py index 75a1286ff8..8e75a07aa1 100644 --- a/deeplabcut/gui/analyze_videos.py +++ b/deeplabcut/gui/analyze_videos.py @@ -178,7 +178,7 @@ def __init__(self, parent, gui_size, cfg): self, label="Specify the Tracker Method (you can try each)" ) tracker_text_boxsizer = wx.StaticBoxSizer(tracker_text, wx.VERTICAL) - trackertypes = ["skeleton", "box"] + trackertypes = ["skeleton", "box", "single_object"] self.trackertypes = wx.ComboBox( self, choices=trackertypes, style=wx.CB_READONLY ) @@ -570,4 +570,4 @@ def chooseOption(self, event): self.sizer.Fit(self) def getbp(self, event): - self.bodyparts = list(self.trajectory_to_plot.GetCheckedStrings()) + self.bodyparts = list(self.trajectory_to_plot.GetCheckedStrings()) \ No newline at end of file diff --git a/deeplabcut/gui/create_videos.py b/deeplabcut/gui/create_videos.py index 7fde61fb0b..f7d7301909 100644 --- a/deeplabcut/gui/create_videos.py +++ b/deeplabcut/gui/create_videos.py @@ -207,7 +207,7 @@ def __init__(self, parent, gui_size, cfg): if self.cfg.get("multianimalproject", False): tracker_text = wx.StaticBox(self, label="Specify the Tracker Method!") tracker_text_boxsizer = wx.StaticBoxSizer(tracker_text, wx.VERTICAL) - trackertypes = ["skeleton", "box"] + trackertypes = ["skeleton", "box", "single_object"] self.trackertypes = wx.ComboBox( self, choices=trackertypes, style=wx.CB_READONLY ) diff --git a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py new file mode 100644 index 0000000000..2acdd493bb --- /dev/null +++ b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py @@ -0,0 +1,105 @@ +from deeplabcut.pose_estimation_tensorflow.lib.trackingutils import * + + +class TrackByDetectionTracker: + def __init__(self, max_frames_to_skip, jointsnum): + self.max_frames_to_skip = max_frames_to_skip + self.id_count = 0 + self.tracks = [] + self.iter = 0 + self.det = [] + self.jointnum = jointsnum + + def cost_metric(self, detections, N, M): + cost_matrix = np.zeros(shape=(N, M)) # Cost matrix + for i in range(len(self.tracks)): + for j in range(len(detections)): + z = convert_bbox_to_z(detections[j])[:2] + gating_dist = self.tracks[i].KF.mahalanobis_dist(z) + cost_matrix[i][j] = gating_dist + return cost_matrix + + def track(self, bbs): + self.iter += 1 + detection_bbs = bbs + + if len(self.tracks) == 0: + for i in range(len(detection_bbs)): + track = Track(detection_bbs[i], self.id_count) + self.id_count += 1 + self.tracks.append(track) + + N = len(self.tracks) + M = len(detection_bbs) + cost_matrix = self.cost_metric(detection_bbs, N, M) + rows, cols = linear_sum_assignment(cost_matrix) + matches = self._matching(rows, cols, bbs, N) + + # either predict or use detection bbs to track + for i in range(len(matches)): + if matches[i] is not None: + self.tracks[i].skipped_frames = 0 + self.tracks[i].predict(self.iter, detection_bbs[matches[i]]) + else: + self.tracks[i].predict(it=self.iter) + states = [] + for t, track in enumerate(self.tracks): + if matches[t] is not None: + states.append(np.concatenate((track.prediction, [track.track_id, matches[t]])).reshape(1, -1)[0]) + if len(states) > 0: + return np.stack(states) + else: + return np.empty((0, self.jointnum * 2 + 2)) + + def _matching(self, rows, cols, detection_bbs, N): + + # assign detections as matches according to the linear assignment solution + matches = [None] * N + for i in range(len(rows)): + matches[rows[i]] = cols[i] + + # validate matches : if the iou are unreasonable unmatch, if linear assignment didnt find + # a solution increase the number of skipped frames + for i in range(len(matches)): + if matches[i] is not None: + _iou = iou(self.tracks[i].prediction, detection_bbs[matches[i]]) + if _iou > 9: + print("unmatched found") + matches[i] = None + else: + self.tracks[i].skipped_frames += 1 + # handle tracks with skipped frame > max_frame_to_skip -> assign death + del_tracks = [] + for i in range(len(self.tracks)): + if self.tracks[i].skipped_frames > self.max_frames_to_skip: + del_tracks.append(i) + + if len(del_tracks) > 0: + for i in del_tracks: + del self.tracks[i] + + # for detections with no matches will be considered as births - new tracks + for i in range(len(detection_bbs)): + if i not in matches: + track = Track(detection_bbs[i], self.id_count) + self.id_count += 1 + self.tracks.append(track) + return matches + + +class Track: + def __init__(self, corr_bb, trackId): + self.track_id = trackId + self.KF = KalmanBoxTracker(corr_bb) + self.prediction = corr_bb + self.trace = dict() # trace path + self.skipped_frames = 0 + + def predict(self, it, detection=None): + if detection is not None: + self.KF.update(detection) + self.KF.predict() + self.prediction = self.KF.get_state()[0] + self.trace[it] = self.prediction # probably will need only the center + self.KF.predict() + return self.KF.get_state()[0] \ No newline at end of file diff --git a/deeplabcut/pose_estimation_tensorflow/lib/trackingutils.py b/deeplabcut/pose_estimation_tensorflow/lib/trackingutils.py index 3969cebb44..dc7b8dc018 100644 --- a/deeplabcut/pose_estimation_tensorflow/lib/trackingutils.py +++ b/deeplabcut/pose_estimation_tensorflow/lib/trackingutils.py @@ -166,6 +166,17 @@ def get_state(self): Returns the current bounding box estimate. """ return convert_x_to_bbox(self.kf.x) + + def mahalanobis_dist(self, z): + """ + Compute the mahalanobis distance + """ + x_hat = np.dot(self.kf.H[:2,:2], self.kf.x[:2]) + y = np.subtract(z, x_hat) + S = self.kf.R[:2,:2] + np.dot(self.kf.H[:2,:2], np.dot(self.kf.P[:2,:2], self.kf.H.T[:2,:2])) + Inv_S = np.linalg.inv(S) + d = np.dot(y.T, np.dot(Inv_S, y)) + return np.sqrt(d) class SkeletonTracker: diff --git a/deeplabcut/pose_estimation_tensorflow/predict_videos.py b/deeplabcut/pose_estimation_tensorflow/predict_videos.py index ac28296e14..1678a99218 100755 --- a/deeplabcut/pose_estimation_tensorflow/predict_videos.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_videos.py @@ -36,22 +36,22 @@ def analyze_videos( - config, - videos, - videotype="avi", - shuffle=1, - trainingsetindex=0, - gputouse=None, - save_as_csv=False, - destfolder=None, - batchsize=None, - cropping=None, - get_nframesfrommetadata=True, - TFGPUinference=True, - dynamic=(False, 0.5, 10), - modelprefix="", - c_engine=False, - robust_nframes=False, + config, + videos, + videotype="avi", + shuffle=1, + trainingsetindex=0, + gputouse=None, + save_as_csv=False, + destfolder=None, + batchsize=None, + cropping=None, + get_nframesfrommetadata=True, + TFGPUinference=True, + dynamic=(False, 0.5, 10), + modelprefix="", + c_engine=False, + robust_nframes=False, ): """ Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex') @@ -353,10 +353,10 @@ def checkcropping(cfg, cap): else: raise Exception("Please check the order of cropping parameter!") if ( - cfg["x1"] >= 0 - and cfg["x2"] < int(cap.get(3) + 1) - and cfg["y1"] >= 0 - and cfg["y2"] < int(cap.get(4) + 1) + cfg["x1"] >= 0 + and cfg["x2"] < int(cap.get(3) + 1) + and cfg["y1"] >= 0 + and cfg["y2"] < int(cap.get(4) + 1) ): pass # good cropping box else: @@ -389,7 +389,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frames[batch_ind] = img_as_ubyte(frame) @@ -397,7 +397,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): if batch_ind == batchsize - 1: pose = predict.getposeNP(frames, dlc_cfg, sess, inputs, outputs) PredictedData[ - batch_num * batchsize : (batch_num + 1) * batchsize, : + batch_num * batchsize: (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 batch_num += 1 @@ -411,7 +411,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frames, dlc_cfg, sess, inputs, outputs ) # process the whole batch (some frames might be from previous batch!) PredictedData[ - batch_num * batchsize : batch_num * batchsize + batch_ind, : + batch_num * batchsize: batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] break counter += 1 @@ -440,13 +440,13 @@ def GetPoseS(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frame = img_as_ubyte(frame) pose = predict.getpose(frame, dlc_cfg, sess, inputs, outputs) PredictedData[ - counter, : + counter, : ] = ( pose.flatten() ) # NOTE: thereby cfg['all_joints_names'] should be same order as bodyparts! @@ -480,7 +480,7 @@ def GetPoseS_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frame = img_as_ubyte(frame) @@ -492,7 +492,7 @@ def GetPoseS_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes): pose[:, [0, 1, 2]] = pose[:, [1, 0, 2]] # pose = predict.getpose(frame, dlc_cfg, sess, inputs, outputs) PredictedData[ - counter, : + counter, : ] = ( pose.flatten() ) # NOTE: thereby cfg['all_joints_names'] should be same order as bodyparts! @@ -531,7 +531,7 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frames[batch_ind] = img_as_ubyte(frame) @@ -540,13 +540,13 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): # pose = predict.getposeNP(frames,dlc_cfg, sess, inputs, outputs) pose = sess.run(pose_tensor, feed_dict={inputs: frames}) pose[:, [0, 1, 2]] = pose[ - :, [1, 0, 2] - ] # change order to have x,y,confidence + :, [1, 0, 2] + ] # change order to have x,y,confidence pose = np.reshape( pose, (batchsize, -1) ) # bring into batchsize times x,y,conf etc. PredictedData[ - batch_num * batchsize : (batch_num + 1) * batchsize, : + batch_num * batchsize: (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 @@ -562,7 +562,7 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): pose[:, [0, 1, 2]] = pose[:, [1, 0, 2]] pose = np.reshape(pose, (batchsize, -1)) PredictedData[ - batch_num * batchsize : batch_num * batchsize + batch_ind, : + batch_num * batchsize: batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] break @@ -581,7 +581,7 @@ def getboundingbox(x, y, nx, ny, margin): def GetPoseDynamic( - cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, detectiontreshold, margin + cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, detectiontreshold, margin ): """ Non batch wise pose estimation for video cap by dynamically cropping around previously detected parts.""" if cfg["cropping"]: @@ -606,7 +606,7 @@ def GetPoseDynamic( originalframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - originalframe[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] )[y1:y2, x1:x2] else: frame = img_as_ubyte(originalframe[y1:y2, x1:x2]) @@ -625,12 +625,12 @@ def GetPoseDynamic( detected = True # object detected else: if ( - detected and (x1 + y1 + y2 - ny + x2 - nx) != 0 + detected and (x1 + y1 + y2 - ny + x2 - nx) != 0 ): # was detected in last frame and dyn. cropping was performed >> but object lost in cropped variant >> re-run on full frame! # print("looking again, lost!") if cfg["cropping"]: frame = img_as_ubyte( - originalframe[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frame = img_as_ubyte(originalframe) @@ -653,20 +653,20 @@ def GetPoseDynamic( def AnalyzeVideo( - video, - DLCscorer, - DLCscorerlegacy, - trainFraction, - cfg, - dlc_cfg, - sess, - inputs, - outputs, - pdindex, - save_as_csv, - destfolder=None, - TFGPUinference=True, - dynamic=(False, 0.5, 10), + video, + DLCscorer, + DLCscorerlegacy, + trainFraction, + cfg, + dlc_cfg, + sess, + inputs, + outputs, + pdindex, + save_as_csv, + destfolder=None, + TFGPUinference=True, + dynamic=(False, 0.5, 10), ): """ Helper function for analyzing a video. """ print("Starting to analyze % ", video) @@ -796,7 +796,7 @@ def AnalyzeVideo( def GetPosesofFrames( - cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, nframes, batchsize, rgb + cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, nframes, batchsize, rgb ): """ Batchwise prediction of pose for frame list in directory""" # from skimage.io import imread @@ -833,10 +833,10 @@ def GetPosesofFrames( else: raise Exception("Please check the order of cropping parameter!") if ( - cfg["x1"] >= 0 - and cfg["x2"] < int(np.shape(im)[1]) - and cfg["y1"] >= 0 - and cfg["y2"] < int(np.shape(im)[0]) + cfg["x1"] >= 0 + and cfg["x2"] < int(np.shape(im)[1]) + and cfg["y1"] >= 0 + and cfg["y2"] < int(np.shape(im)[0]) ): pass # good cropping box else: @@ -859,7 +859,7 @@ def GetPosesofFrames( if cfg["cropping"]: frame = img_as_ubyte( - im[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"], :] + im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] ) else: frame = img_as_ubyte(im) @@ -881,7 +881,7 @@ def GetPosesofFrames( if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - im[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"], :] + im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] ) else: frames[batch_ind] = img_as_ubyte(im) @@ -889,7 +889,7 @@ def GetPosesofFrames( if batch_ind == batchsize - 1: pose = predict.getposeNP(frames, dlc_cfg, sess, inputs, outputs) PredictedData[ - batch_num * batchsize : (batch_num + 1) * batchsize, : + batch_num * batchsize: (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 batch_num += 1 @@ -897,13 +897,13 @@ def GetPosesofFrames( batch_ind += 1 if ( - batch_ind > 0 + batch_ind > 0 ): # take care of the last frames (the batch that might have been processed) pose = predict.getposeNP( frames, dlc_cfg, sess, inputs, outputs ) # process the whole batch (some frames might be from previous batch!) PredictedData[ - batch_num * batchsize : batch_num * batchsize + batch_ind, : + batch_num * batchsize: batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] pbar.close() @@ -911,15 +911,15 @@ def GetPosesofFrames( def analyze_time_lapse_frames( - config, - directory, - frametype=".png", - shuffle=1, - trainingsetindex=0, - gputouse=None, - save_as_csv=False, - rgb=True, - modelprefix="", + config, + directory, + frametype=".png", + shuffle=1, + trainingsetindex=0, + gputouse=None, + save_as_csv=False, + rgb=True, + modelprefix="", ): """ Analyzed all images (of type = frametype) in a folder and stores the output in one file. @@ -1146,21 +1146,21 @@ def analyze_time_lapse_frames( def convert_detections2tracklets( - config, - videos, - videotype="avi", - shuffle=1, - trainingsetindex=0, - overwrite=False, - destfolder=None, - BPTS=None, - iBPTS=None, - PAF=None, - printintermediate=False, - inferencecfg=None, - modelprefix="", - track_method="box", - edgewisecondition=True, + config, + videos, + videotype="avi", + shuffle=1, + trainingsetindex=0, + overwrite=False, + destfolder=None, + BPTS=None, + iBPTS=None, + PAF=None, + printintermediate=False, + inferencecfg=None, + modelprefix="", + track_method="box", + edgewisecondition=True, ): """ This should be called at the end of deeplabcut.analyze_videos for multianimal projects! @@ -1224,11 +1224,12 @@ def convert_detections2tracklets( """ from deeplabcut.pose_estimation_tensorflow.lib import inferenceutils, trackingutils + from deeplabcut.pose_estimation_tensorflow.lib.single_object_tracker import TrackByDetectionTracker from deeplabcut.utils import auxfun_multianimal from easydict import EasyDict as edict import pickle - if track_method not in ("box", "skeleton"): + if track_method not in ("box", "skeleton", "single_object"): raise ValueError( "Invalid tracking method. Only `box` and `skeleton` are currently supported." ) @@ -1273,7 +1274,7 @@ def convert_detections2tracklets( if edgewisecondition: path_inferencebounds_config = ( - Path(modelfolder) / "test" / "inferencebounds.yaml" + Path(modelfolder) / "test" / "inferencebounds.yaml" ) try: inferenceboundscfg = auxiliaryfunctions.read_plainconfig( @@ -1344,11 +1345,18 @@ def convert_detections2tracklets( vname = Path(video).stem dataname = os.path.join(videofolder, vname + DLCscorer + ".h5") data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(dataname) - method = "sk" if track_method == "skeleton" else "bx" + if track_method == 'skeleton': + method = 'sk' + elif track_method == 'single_object': + method = 'so' + elif track_method == 'box': + method = 'bx' + else: + raise ValueError trackname = dataname.split(".h5")[0] + f"_{method}.pickle" trackname = trackname.replace(videofolder, destfolder) if ( - os.path.isfile(trackname) and not overwrite + os.path.isfile(trackname) and not overwrite ): # TODO: check if metadata are identical (same parameters!) print("Tracklets already computed", trackname) print("Set overwrite = True to overwrite.") @@ -1417,15 +1425,15 @@ def convert_detections2tracklets( imnames = [fn for fn in data if fn != "metadata"] - if track_method == "box": + if track_method == 'box': mot_tracker = trackingutils.Sort(inferencecfg) + elif track_method == 'single_object': + mot_tracker = TrackByDetectionTracker(10, numjoints) else: - mot_tracker = trackingutils.SORT( - numjoints, - inferencecfg["max_age"], - inferencecfg["min_hits"], - inferencecfg.get("oks_threshold", 0.5), - ) + mot_tracker = trackingutils.SORT(numjoints, + inferencecfg['max_age'], + inferencecfg['min_hits'], + inferencecfg.get('oks_threshold', 0.5)) Tracks = {} for index, imname in tqdm(enumerate(imnames)): @@ -1448,6 +1456,11 @@ def convert_detections2tracklets( inferencecfg, animals, 0 ) # TODO: get cropping parameters and utilize! trackers = mot_tracker.update(bb) + print(trackers) + elif track_method == 'single_object': + bb = inferenceutils.individual2boundingbox(inferencecfg, animals, + 0) # TODO: get cropping parameters and utilize! + trackers = mot_tracker.track(bb) else: temp = [arr.reshape((-1, 3))[:, :2] for arr in animals] trackers = mot_tracker.track(temp) @@ -1459,8 +1472,8 @@ def convert_detections2tracklets( all_jointnames.index(bp) for bp in cfg["uniquebodyparts"] ] if not any( - np.isfinite(a.reshape((-1, 3))[inds_unique]).all() - for a in animals + np.isfinite(a.reshape((-1, 3))[inds_unique]).all() + for a in animals ): single = np.full((numjoints, 3), np.nan) single_dets = inferenceutils.convertdetectiondict2listoflist( @@ -1480,7 +1493,6 @@ def convert_detections2tracklets( tracklet_id += 1 Tracks[tracklet_id] = {} Tracks[tracklet_id][imname] = single.flatten() - Tracks["header"] = pdindex with open(trackname, "wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. @@ -1498,4 +1510,4 @@ def convert_detections2tracklets( parser = argparse.ArgumentParser() parser.add_argument("video") parser.add_argument("config") - cli_args = parser.parse_args() + cli_args = parser.parse_args() \ No newline at end of file From 97f91b5ef3ead6971798ea9b838c3d5939b65c27 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Thu, 11 Jun 2020 19:18:52 -0400 Subject: [PATCH 02/10] added header. --- .../lib/single_object_tracker.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py index 2acdd493bb..251d8eaca6 100644 --- a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py +++ b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py @@ -1,3 +1,14 @@ +""" +DeepLabCut 2.2 Toolbox (deeplabcut.org) +© A. & M. Mathis Labs +https://github.com/AlexEMG/DeepLabCut +Please see AUTHORS for contributors. +https://github.com/AlexEMG/DeepLabCut/blob/master/AUTHORS +Licensed under GNU Lesser General Public License v3.0 + +This module was contributed by Tabet Ehsainieh - https://github.com/ehsainit +""" + from deeplabcut.pose_estimation_tensorflow.lib.trackingutils import * @@ -102,4 +113,4 @@ def predict(self, it, detection=None): self.prediction = self.KF.get_state()[0] self.trace[it] = self.prediction # probably will need only the center self.KF.predict() - return self.KF.get_state()[0] \ No newline at end of file + return self.KF.get_state()[0] From 7ed03ae73afef39fe7911ea3b501964e23591c32 Mon Sep 17 00:00:00 2001 From: ehsainit Date: Fri, 12 Jun 2020 21:10:15 +0200 Subject: [PATCH 03/10] correcting code-format (was auto-formatted) --- .../predict_videos.py | 195 +++++++++--------- 1 file changed, 98 insertions(+), 97 deletions(-) diff --git a/deeplabcut/pose_estimation_tensorflow/predict_videos.py b/deeplabcut/pose_estimation_tensorflow/predict_videos.py index 1678a99218..93b88e5416 100755 --- a/deeplabcut/pose_estimation_tensorflow/predict_videos.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_videos.py @@ -36,22 +36,22 @@ def analyze_videos( - config, - videos, - videotype="avi", - shuffle=1, - trainingsetindex=0, - gputouse=None, - save_as_csv=False, - destfolder=None, - batchsize=None, - cropping=None, - get_nframesfrommetadata=True, - TFGPUinference=True, - dynamic=(False, 0.5, 10), - modelprefix="", - c_engine=False, - robust_nframes=False, + config, + videos, + videotype="avi", + shuffle=1, + trainingsetindex=0, + gputouse=None, + save_as_csv=False, + destfolder=None, + batchsize=None, + cropping=None, + get_nframesfrommetadata=True, + TFGPUinference=True, + dynamic=(False, 0.5, 10), + modelprefix="", + c_engine=False, + robust_nframes=False, ): """ Makes prediction based on a trained network. The index of the trained network is specified by parameters in the config file (in particular the variable 'snapshotindex') @@ -353,10 +353,10 @@ def checkcropping(cfg, cap): else: raise Exception("Please check the order of cropping parameter!") if ( - cfg["x1"] >= 0 - and cfg["x2"] < int(cap.get(3) + 1) - and cfg["y1"] >= 0 - and cfg["y2"] < int(cap.get(4) + 1) + cfg["x1"] >= 0 + and cfg["x2"] < int(cap.get(3) + 1) + and cfg["y1"] >= 0 + and cfg["y2"] < int(cap.get(4) + 1) ): pass # good cropping box else: @@ -389,7 +389,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frames[batch_ind] = img_as_ubyte(frame) @@ -397,7 +397,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): if batch_ind == batchsize - 1: pose = predict.getposeNP(frames, dlc_cfg, sess, inputs, outputs) PredictedData[ - batch_num * batchsize: (batch_num + 1) * batchsize, : + batch_num * batchsize: (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 batch_num += 1 @@ -411,7 +411,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frames, dlc_cfg, sess, inputs, outputs ) # process the whole batch (some frames might be from previous batch!) PredictedData[ - batch_num * batchsize: batch_num * batchsize + batch_ind, : + batch_num * batchsize: batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] break counter += 1 @@ -440,13 +440,13 @@ def GetPoseS(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] ) else: frame = img_as_ubyte(frame) pose = predict.getpose(frame, dlc_cfg, sess, inputs, outputs) PredictedData[ - counter, : + counter, : ] = ( pose.flatten() ) # NOTE: thereby cfg['all_joints_names'] should be same order as bodyparts! @@ -480,7 +480,7 @@ def GetPoseS_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] ) else: frame = img_as_ubyte(frame) @@ -540,13 +540,13 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): # pose = predict.getposeNP(frames,dlc_cfg, sess, inputs, outputs) pose = sess.run(pose_tensor, feed_dict={inputs: frames}) pose[:, [0, 1, 2]] = pose[ - :, [1, 0, 2] - ] # change order to have x,y,confidence + :, [1, 0, 2] + ] # change order to have x,y,confidence pose = np.reshape( pose, (batchsize, -1) ) # bring into batchsize times x,y,conf etc. PredictedData[ - batch_num * batchsize: (batch_num + 1) * batchsize, : + batch_num * batchsize: (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 @@ -562,7 +562,7 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): pose[:, [0, 1, 2]] = pose[:, [1, 0, 2]] pose = np.reshape(pose, (batchsize, -1)) PredictedData[ - batch_num * batchsize: batch_num * batchsize + batch_ind, : + batch_num * batchsize: batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] break @@ -581,7 +581,7 @@ def getboundingbox(x, y, nx, ny, margin): def GetPoseDynamic( - cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, detectiontreshold, margin + cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, detectiontreshold, margin ): """ Non batch wise pose estimation for video cap by dynamically cropping around previously detected parts.""" if cfg["cropping"]: @@ -606,7 +606,7 @@ def GetPoseDynamic( originalframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] )[y1:y2, x1:x2] else: frame = img_as_ubyte(originalframe[y1:y2, x1:x2]) @@ -625,12 +625,12 @@ def GetPoseDynamic( detected = True # object detected else: if ( - detected and (x1 + y1 + y2 - ny + x2 - nx) != 0 + detected and (x1 + y1 + y2 - ny + x2 - nx) != 0 ): # was detected in last frame and dyn. cropping was performed >> but object lost in cropped variant >> re-run on full frame! # print("looking again, lost!") if cfg["cropping"]: frame = img_as_ubyte( - originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] ) else: frame = img_as_ubyte(originalframe) @@ -653,20 +653,20 @@ def GetPoseDynamic( def AnalyzeVideo( - video, - DLCscorer, - DLCscorerlegacy, - trainFraction, - cfg, - dlc_cfg, - sess, - inputs, - outputs, - pdindex, - save_as_csv, - destfolder=None, - TFGPUinference=True, - dynamic=(False, 0.5, 10), + video, + DLCscorer, + DLCscorerlegacy, + trainFraction, + cfg, + dlc_cfg, + sess, + inputs, + outputs, + pdindex, + save_as_csv, + destfolder=None, + TFGPUinference=True, + dynamic=(False, 0.5, 10), ): """ Helper function for analyzing a video. """ print("Starting to analyze % ", video) @@ -796,7 +796,7 @@ def AnalyzeVideo( def GetPosesofFrames( - cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, nframes, batchsize, rgb + cfg, dlc_cfg, sess, inputs, outputs, directory, framelist, nframes, batchsize, rgb ): """ Batchwise prediction of pose for frame list in directory""" # from skimage.io import imread @@ -833,10 +833,10 @@ def GetPosesofFrames( else: raise Exception("Please check the order of cropping parameter!") if ( - cfg["x1"] >= 0 - and cfg["x2"] < int(np.shape(im)[1]) - and cfg["y1"] >= 0 - and cfg["y2"] < int(np.shape(im)[0]) + cfg["x1"] >= 0 + and cfg["x2"] < int(np.shape(im)[1]) + and cfg["y1"] >= 0 + and cfg["y2"] < int(np.shape(im)[0]) ): pass # good cropping box else: @@ -859,7 +859,7 @@ def GetPosesofFrames( if cfg["cropping"]: frame = img_as_ubyte( - im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] + im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] ) else: frame = img_as_ubyte(im) @@ -881,7 +881,7 @@ def GetPosesofFrames( if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] + im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] ) else: frames[batch_ind] = img_as_ubyte(im) @@ -889,7 +889,7 @@ def GetPosesofFrames( if batch_ind == batchsize - 1: pose = predict.getposeNP(frames, dlc_cfg, sess, inputs, outputs) PredictedData[ - batch_num * batchsize: (batch_num + 1) * batchsize, : + batch_num * batchsize: (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 batch_num += 1 @@ -897,13 +897,13 @@ def GetPosesofFrames( batch_ind += 1 if ( - batch_ind > 0 + batch_ind > 0 ): # take care of the last frames (the batch that might have been processed) pose = predict.getposeNP( frames, dlc_cfg, sess, inputs, outputs ) # process the whole batch (some frames might be from previous batch!) PredictedData[ - batch_num * batchsize: batch_num * batchsize + batch_ind, : + batch_num * batchsize: batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] pbar.close() @@ -911,15 +911,15 @@ def GetPosesofFrames( def analyze_time_lapse_frames( - config, - directory, - frametype=".png", - shuffle=1, - trainingsetindex=0, - gputouse=None, - save_as_csv=False, - rgb=True, - modelprefix="", + config, + directory, + frametype=".png", + shuffle=1, + trainingsetindex=0, + gputouse=None, + save_as_csv=False, + rgb=True, + modelprefix="", ): """ Analyzed all images (of type = frametype) in a folder and stores the output in one file. @@ -1146,21 +1146,21 @@ def analyze_time_lapse_frames( def convert_detections2tracklets( - config, - videos, - videotype="avi", - shuffle=1, - trainingsetindex=0, - overwrite=False, - destfolder=None, - BPTS=None, - iBPTS=None, - PAF=None, - printintermediate=False, - inferencecfg=None, - modelprefix="", - track_method="box", - edgewisecondition=True, + config, + videos, + videotype="avi", + shuffle=1, + trainingsetindex=0, + overwrite=False, + destfolder=None, + BPTS=None, + iBPTS=None, + PAF=None, + printintermediate=False, + inferencecfg=None, + modelprefix="", + track_method="box", + edgewisecondition=True, ): """ This should be called at the end of deeplabcut.analyze_videos for multianimal projects! @@ -1274,7 +1274,7 @@ def convert_detections2tracklets( if edgewisecondition: path_inferencebounds_config = ( - Path(modelfolder) / "test" / "inferencebounds.yaml" + Path(modelfolder) / "test" / "inferencebounds.yaml" ) try: inferenceboundscfg = auxiliaryfunctions.read_plainconfig( @@ -1345,18 +1345,18 @@ def convert_detections2tracklets( vname = Path(video).stem dataname = os.path.join(videofolder, vname + DLCscorer + ".h5") data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(dataname) - if track_method == 'skeleton': - method = 'sk' - elif track_method == 'single_object': - method = 'so' - elif track_method == 'box': - method = 'bx' + if track_method == "skeleton": + method = "sk" + elif track_method == "single_object": + method = "so" + elif track_method == "box": + method = "bx" else: raise ValueError trackname = dataname.split(".h5")[0] + f"_{method}.pickle" trackname = trackname.replace(videofolder, destfolder) if ( - os.path.isfile(trackname) and not overwrite + os.path.isfile(trackname) and not overwrite ): # TODO: check if metadata are identical (same parameters!) print("Tracklets already computed", trackname) print("Set overwrite = True to overwrite.") @@ -1425,9 +1425,9 @@ def convert_detections2tracklets( imnames = [fn for fn in data if fn != "metadata"] - if track_method == 'box': + if track_method == "box": mot_tracker = trackingutils.Sort(inferencecfg) - elif track_method == 'single_object': + elif track_method == "single_object": mot_tracker = TrackByDetectionTracker(10, numjoints) else: mot_tracker = trackingutils.SORT(numjoints, @@ -1456,8 +1456,7 @@ def convert_detections2tracklets( inferencecfg, animals, 0 ) # TODO: get cropping parameters and utilize! trackers = mot_tracker.update(bb) - print(trackers) - elif track_method == 'single_object': + elif track_method == "single_object": bb = inferenceutils.individual2boundingbox(inferencecfg, animals, 0) # TODO: get cropping parameters and utilize! trackers = mot_tracker.track(bb) @@ -1472,8 +1471,8 @@ def convert_detections2tracklets( all_jointnames.index(bp) for bp in cfg["uniquebodyparts"] ] if not any( - np.isfinite(a.reshape((-1, 3))[inds_unique]).all() - for a in animals + np.isfinite(a.reshape((-1, 3))[inds_unique]).all() + for a in animals ): single = np.full((numjoints, 3), np.nan) single_dets = inferenceutils.convertdetectiondict2listoflist( @@ -1493,6 +1492,7 @@ def convert_detections2tracklets( tracklet_id += 1 Tracks[tracklet_id] = {} Tracks[tracklet_id][imname] = single.flatten() + Tracks["header"] = pdindex with open(trackname, "wb") as f: # Pickle the 'labeled-data' dictionary using the highest protocol available. @@ -1510,4 +1510,5 @@ def convert_detections2tracklets( parser = argparse.ArgumentParser() parser.add_argument("video") parser.add_argument("config") - cli_args = parser.parse_args() \ No newline at end of file + cli_args = parser.parse_args() + From 058d4713698f49a13eff56e8ef96e0c73de1e475 Mon Sep 17 00:00:00 2001 From: ehsainit Date: Fri, 12 Jun 2020 21:20:27 +0200 Subject: [PATCH 04/10] further formatting --- .../predict_videos.py | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/deeplabcut/pose_estimation_tensorflow/predict_videos.py b/deeplabcut/pose_estimation_tensorflow/predict_videos.py index 93b88e5416..51ecbce986 100755 --- a/deeplabcut/pose_estimation_tensorflow/predict_videos.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_videos.py @@ -36,9 +36,9 @@ def analyze_videos( - config, + config, videos, - videotype="avi", + videotype="avi", shuffle=1, trainingsetindex=0, gputouse=None, @@ -389,7 +389,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] ) else: frames[batch_ind] = img_as_ubyte(frame) @@ -397,7 +397,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): if batch_ind == batchsize - 1: pose = predict.getposeNP(frames, dlc_cfg, sess, inputs, outputs) PredictedData[ - batch_num * batchsize: (batch_num + 1) * batchsize, : + batch_num * batchsize : (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 batch_num += 1 @@ -411,7 +411,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frames, dlc_cfg, sess, inputs, outputs ) # process the whole batch (some frames might be from previous batch!) PredictedData[ - batch_num * batchsize: batch_num * batchsize + batch_ind, : + batch_num * batchsize : batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] break counter += 1 @@ -492,7 +492,7 @@ def GetPoseS_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes): pose[:, [0, 1, 2]] = pose[:, [1, 0, 2]] # pose = predict.getpose(frame, dlc_cfg, sess, inputs, outputs) PredictedData[ - counter, : + counter, : ] = ( pose.flatten() ) # NOTE: thereby cfg['all_joints_names'] should be same order as bodyparts! @@ -531,7 +531,7 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - frame[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] ) else: frames[batch_ind] = img_as_ubyte(frame) @@ -546,7 +546,7 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): pose, (batchsize, -1) ) # bring into batchsize times x,y,conf etc. PredictedData[ - batch_num * batchsize: (batch_num + 1) * batchsize, : + batch_num * batchsize : (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 @@ -606,7 +606,7 @@ def GetPoseDynamic( originalframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frame = img_as_ubyte( - originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + originalframe[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] )[y1:y2, x1:x2] else: frame = img_as_ubyte(originalframe[y1:y2, x1:x2]) @@ -630,7 +630,7 @@ def GetPoseDynamic( # print("looking again, lost!") if cfg["cropping"]: frame = img_as_ubyte( - originalframe[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"]] + originalframe[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] ) else: frame = img_as_ubyte(originalframe) @@ -859,7 +859,7 @@ def GetPosesofFrames( if cfg["cropping"]: frame = img_as_ubyte( - im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] + im[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"], :] ) else: frame = img_as_ubyte(im) @@ -881,7 +881,7 @@ def GetPosesofFrames( if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - im[cfg["y1"]: cfg["y2"], cfg["x1"]: cfg["x2"], :] + im[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"], :] ) else: frames[batch_ind] = img_as_ubyte(im) @@ -889,7 +889,7 @@ def GetPosesofFrames( if batch_ind == batchsize - 1: pose = predict.getposeNP(frames, dlc_cfg, sess, inputs, outputs) PredictedData[ - batch_num * batchsize: (batch_num + 1) * batchsize, : + batch_num * batchsize : (batch_num + 1) * batchsize, : ] = pose batch_ind = 0 batch_num += 1 @@ -903,7 +903,7 @@ def GetPosesofFrames( frames, dlc_cfg, sess, inputs, outputs ) # process the whole batch (some frames might be from previous batch!) PredictedData[ - batch_num * batchsize: batch_num * batchsize + batch_ind, : + batch_num * batchsize : batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] pbar.close() @@ -1430,10 +1430,12 @@ def convert_detections2tracklets( elif track_method == "single_object": mot_tracker = TrackByDetectionTracker(10, numjoints) else: - mot_tracker = trackingutils.SORT(numjoints, - inferencecfg['max_age'], - inferencecfg['min_hits'], - inferencecfg.get('oks_threshold', 0.5)) + mot_tracker = trackingutils.SORT( + numjoints, + inferencecfg['max_age'], + inferencecfg['min_hits'], + inferencecfg.get('oks_threshold', 0.5) + ) Tracks = {} for index, imname in tqdm(enumerate(imnames)): @@ -1471,7 +1473,7 @@ def convert_detections2tracklets( all_jointnames.index(bp) for bp in cfg["uniquebodyparts"] ] if not any( - np.isfinite(a.reshape((-1, 3))[inds_unique]).all() + np.isfinite(a.reshape((-1, 3))[inds_unique]).all() for a in animals ): single = np.full((numjoints, 3), np.nan) @@ -1511,4 +1513,4 @@ def convert_detections2tracklets( parser.add_argument("video") parser.add_argument("config") cli_args = parser.parse_args() - + From c23a23c30de9145b5e9d81a6f58040a42e7ddae5 Mon Sep 17 00:00:00 2001 From: ehsainit Date: Fri, 12 Jun 2020 21:23:46 +0200 Subject: [PATCH 05/10] more format-correcting --- .../pose_estimation_tensorflow/predict_videos.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/deeplabcut/pose_estimation_tensorflow/predict_videos.py b/deeplabcut/pose_estimation_tensorflow/predict_videos.py index 51ecbce986..630cf8167e 100755 --- a/deeplabcut/pose_estimation_tensorflow/predict_videos.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_videos.py @@ -389,7 +389,7 @@ def GetPoseF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) if cfg["cropping"]: frames[batch_ind] = img_as_ubyte( - frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] + frame[cfg["y1"] : cfg["y2"], cfg["x1"] : cfg["x2"]] ) else: frames[batch_ind] = img_as_ubyte(frame) @@ -562,7 +562,7 @@ def GetPoseF_GTF(cfg, dlc_cfg, sess, inputs, outputs, cap, nframes, batchsize): pose[:, [0, 1, 2]] = pose[:, [1, 0, 2]] pose = np.reshape(pose, (batchsize, -1)) PredictedData[ - batch_num * batchsize: batch_num * batchsize + batch_ind, : + batch_num * batchsize : batch_num * batchsize + batch_ind, : ] = pose[:batch_ind, :] break @@ -1432,9 +1432,9 @@ def convert_detections2tracklets( else: mot_tracker = trackingutils.SORT( numjoints, - inferencecfg['max_age'], - inferencecfg['min_hits'], - inferencecfg.get('oks_threshold', 0.5) + inferencecfg["max_age"], + inferencecfg["min_hits"], + inferencecfg.get("oks_threshold", 0.5), ) Tracks = {} @@ -1513,4 +1513,3 @@ def convert_detections2tracklets( parser.add_argument("video") parser.add_argument("config") cli_args = parser.parse_args() - From d33bf3c126f8e32726c669a3f00c260a2bbd86ba Mon Sep 17 00:00:00 2001 From: ehsainit Date: Fri, 12 Jun 2020 21:31:47 +0200 Subject: [PATCH 06/10] adding new line --- deeplabcut/gui/analyze_videos.py | 2 +- deeplabcut/pose_estimation_tensorflow/predict_videos.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deeplabcut/gui/analyze_videos.py b/deeplabcut/gui/analyze_videos.py index 8e75a07aa1..0d492621e9 100644 --- a/deeplabcut/gui/analyze_videos.py +++ b/deeplabcut/gui/analyze_videos.py @@ -570,4 +570,4 @@ def chooseOption(self, event): self.sizer.Fit(self) def getbp(self, event): - self.bodyparts = list(self.trajectory_to_plot.GetCheckedStrings()) \ No newline at end of file + self.bodyparts = list(self.trajectory_to_plot.GetCheckedStrings()) diff --git a/deeplabcut/pose_estimation_tensorflow/predict_videos.py b/deeplabcut/pose_estimation_tensorflow/predict_videos.py index 630cf8167e..e70f54f5b6 100755 --- a/deeplabcut/pose_estimation_tensorflow/predict_videos.py +++ b/deeplabcut/pose_estimation_tensorflow/predict_videos.py @@ -1435,7 +1435,7 @@ def convert_detections2tracklets( inferencecfg["max_age"], inferencecfg["min_hits"], inferencecfg.get("oks_threshold", 0.5), - ) + ) Tracks = {} for index, imname in tqdm(enumerate(imnames)): From aa549f47a253d1ebc763484b6d98898cce03e00e Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Sat, 13 Jun 2020 13:02:43 -0400 Subject: [PATCH 07/10] header change --- .../pose_estimation_tensorflow/lib/single_object_tracker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py index 251d8eaca6..db5690bb06 100644 --- a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py +++ b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py @@ -1,9 +1,11 @@ """ DeepLabCut 2.2 Toolbox (deeplabcut.org) -© A. & M. Mathis Labs -https://github.com/AlexEMG/DeepLabCut +https://github.com/DeepLabCut/DeepLabCut + +© DeepLabCut authors Please see AUTHORS for contributors. https://github.com/AlexEMG/DeepLabCut/blob/master/AUTHORS + Licensed under GNU Lesser General Public License v3.0 This module was contributed by Tabet Ehsainieh - https://github.com/ehsainit From df101493dc560c84e88ee4e2d8b6ae3ec4073c8b Mon Sep 17 00:00:00 2001 From: ehsainit Date: Fri, 19 Jun 2020 01:29:05 +0200 Subject: [PATCH 08/10] bugfix: matches were not deleted upon deleting a corresponding track causing a out an range errors --- .../lib/single_object_tracker.py | 15 ++++++++------- deeplabcut/utils/auxiliaryfunctions.py | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py index 251d8eaca6..062befb614 100644 --- a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py +++ b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py @@ -45,17 +45,18 @@ def track(self, bbs): cost_matrix = self.cost_metric(detection_bbs, N, M) rows, cols = linear_sum_assignment(cost_matrix) matches = self._matching(rows, cols, bbs, N) - # either predict or use detection bbs to track for i in range(len(matches)): - if matches[i] is not None: - self.tracks[i].skipped_frames = 0 - self.tracks[i].predict(self.iter, detection_bbs[matches[i]]) - else: - self.tracks[i].predict(it=self.iter) + # would be better if delete from the matches upon deleting tracks + if i < len(self.tracks): + if matches[i] is not None: + self.tracks[i].skipped_frames = 0 + self.tracks[i].predict(self.iter, detection_bbs[matches[i]]) + else: + self.tracks[i].predict(it=self.iter) states = [] for t, track in enumerate(self.tracks): - if matches[t] is not None: + if t < len(matches) and matches[t] is not None: states.append(np.concatenate((track.prediction, [track.track_id, matches[t]])).reshape(1, -1)[0]) if len(states) > 0: return np.stack(states) diff --git a/deeplabcut/utils/auxiliaryfunctions.py b/deeplabcut/utils/auxiliaryfunctions.py index a41fe8240a..b840c9af9a 100644 --- a/deeplabcut/utils/auxiliaryfunctions.py +++ b/deeplabcut/utils/auxiliaryfunctions.py @@ -753,6 +753,8 @@ def load_detection_data(video, scorer, track_method): tracker = "sk" elif track_method == "box": tracker = "bx" + elif track_method == "single_object": + tracker = "so" else: raise ValueError(f"Unrecognized track_method={track_method}") From 53798d57d6cf12bf79d8f6fa59d277538687a909 Mon Sep 17 00:00:00 2001 From: ehsainit Date: Fri, 19 Jun 2020 01:31:59 +0200 Subject: [PATCH 09/10] bugfix: matches were not deleted upon deleting a corresponding track causing a out an range errors --- .../pose_estimation_tensorflow/lib/single_object_tracker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py index 062befb614..718f8b407e 100644 --- a/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py +++ b/deeplabcut/pose_estimation_tensorflow/lib/single_object_tracker.py @@ -57,6 +57,7 @@ def track(self, bbs): states = [] for t, track in enumerate(self.tracks): if t < len(matches) and matches[t] is not None: + # new assigned tracks in the n frames will be fist tracked in the n+1 frame - probably need fixing states.append(np.concatenate((track.prediction, [track.track_id, matches[t]])).reshape(1, -1)[0]) if len(states) > 0: return np.stack(states) From 6d6831046a3babbe8262e9c41609766a4971a85b Mon Sep 17 00:00:00 2001 From: ehsainit Date: Sun, 1 Nov 2020 12:02:23 +0100 Subject: [PATCH 10/10] bugfix: creating a training set for one bp per ind i.e so tracker needs no paf --- ...ple_individuals_trainingsetmanipulation.py | 7 +++- deeplabcut/utils/auxfun_multianimal.py | 41 ++++++++++--------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py b/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py index 9c620c1093..df17b6849f 100644 --- a/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py +++ b/deeplabcut/generate_training_dataset/multiple_individuals_trainingsetmanipulation.py @@ -124,8 +124,11 @@ def strip_cropped_image_name(path): # ATTENTION: order has to be multibodyparts, then uniquebodyparts (for indexing) print("Utilizing the following graph:", partaffinityfield_graph) num_limbs = len(partaffinityfield_graph) - partaffinityfield_predict = True - + # SO tracker one bp per ind => npo pafs + if len(partaffinityfield_graph) == 0: + partaffinityfield_predict = False + else: + partaffinityfield_predict = True # Loading the encoder (if necessary downloading from TF) dlcparent_path = auxiliaryfunctions.get_deeplabcut_path() defaultconfigfile = os.path.join(dlcparent_path, "pose_cfg.yaml") diff --git a/deeplabcut/utils/auxfun_multianimal.py b/deeplabcut/utils/auxfun_multianimal.py index 1d4a309869..7f9ee1d922 100644 --- a/deeplabcut/utils/auxfun_multianimal.py +++ b/deeplabcut/utils/auxfun_multianimal.py @@ -57,25 +57,28 @@ def getpafgraph(cfg, printnames=True): # CHECKS if each bpt is connected to at least one other bpt # TODO: check that there is a path leading from each (multi)bpt to each other (multi)bpt! connected = set() - partaffinityfield_graph = [] - for link in cfg["skeleton"]: - if link[0] in bodypartnames and link[1] in bodypartnames: - bp1 = int(lookupdict[link[0]]) - bp2 = int(lookupdict[link[1]]) - connected.add(bp1) - connected.add(bp2) - partaffinityfield_graph.append([bp1, bp2]) - else: - print("Attention, parts do not exist!", link) - - unconnected = set(range(len(multianimalbodyparts))).difference(connected) - if unconnected: - raise ValueError( - f'Unconnected {", ".join(multianimalbodyparts[i] for i in unconnected)}. ' - f"For multi-animal projects, all multianimalbodyparts should be connected. " - f"Ideally there should be at least one (multinode) path from each multianimalbodyparts to each other multianimalbodyparts. " - f"Please verify the skeleton in the config.yaml." - ) + if len(cfg["multianimalbodyparts"]) == 1: + partaffinityfield_graph = [] + else: + partaffinityfield_graph = [] + for link in cfg["skeleton"]: + if link[0] in bodypartnames and link[1] in bodypartnames: + bp1 = int(lookupdict[link[0]]) + bp2 = int(lookupdict[link[1]]) + connected.add(bp1) + connected.add(bp2) + partaffinityfield_graph.append([bp1, bp2]) + else: + print("Attention, parts do not exist!", link) + + unconnected = set(range(len(multianimalbodyparts))).difference(connected) + if unconnected: + raise ValueError( + f'Unconnected {", ".join(multianimalbodyparts[i] for i in unconnected)}. ' + f"For multi-animal projects, all multianimalbodyparts should be connected. " + f"Ideally there should be at least one (multinode) path from each multianimalbodyparts to each other multianimalbodyparts. " + f"Please verify the skeleton in the config.yaml." + ) if printnames: graph2names(cfg, partaffinityfield_graph)