From c5d773b4451eab79d9f0a651a978cbd8b338e9e9 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Wed, 4 Dec 2024 17:16:52 +0100 Subject: [PATCH 1/5] tmp --- deeplabcut/compat.py | 2 + deeplabcut/core/metrics/api.py | 6 ++ deeplabcut/core/metrics/distance_metrics.py | 2 + .../pose_estimation_pytorch/apis/evaluate.py | 65 +++++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/deeplabcut/compat.py b/deeplabcut/compat.py index 448d0211c4..d01c6e0cce 100644 --- a/deeplabcut/compat.py +++ b/deeplabcut/compat.py @@ -488,6 +488,8 @@ def evaluate_network( trainingsetindex=trainingsetindex, plotting=plotting, show_errors=show_errors, + comparison_bodyparts=comparisonbodyparts, + per_keypoint_evaluation=per_keypoint_evaluation, modelprefix=modelprefix, **torch_kwargs, ) diff --git a/deeplabcut/core/metrics/api.py b/deeplabcut/core/metrics/api.py index c15fe2cc7e..addd836c69 100644 --- a/deeplabcut/core/metrics/api.py +++ b/deeplabcut/core/metrics/api.py @@ -25,6 +25,7 @@ def compute_metrics( pcutoff: float = -1, oks_bbox_margin: int = 0, oks_sigma: float = 0.1, + per_keypoint_evaluation: bool = False, ) -> dict: """Computes pose estimation performance metrics @@ -70,6 +71,7 @@ def compute_metrics( oks_bbox_margin: The margin to add around keypoints to compute the area for OKS computation. oks_sigma: The OKS sigma to use to compute pose. + per_keypoint_evaluation: Compute per-keypoint RMSE values. Returns: A dictionary containing keys "rmse", "rmse_cutoff", "mAP" and "mAR" mapping @@ -79,6 +81,10 @@ def compute_metrics( "rmse_pcutoff_unique_bodyparts" are also returned, containing the metrics for the unique bodyparts head. + If `per_keypoint_evaluation=True`, "keypoint_rmse", "keypoint_rmse_cutoff" (and + optionally "unique_keypoint_rmse" and "unique_keypoint_rmse_cutoff") keys are + added, containing a list of floats representing the RMSE for each keypoint. + Examples: >>> # Define the p-cutoff, prediction, and target DataFrames >>> pcutoff = 0.5 diff --git a/deeplabcut/core/metrics/distance_metrics.py b/deeplabcut/core/metrics/distance_metrics.py index 85c52ff9e0..5b22961fbb 100644 --- a/deeplabcut/core/metrics/distance_metrics.py +++ b/deeplabcut/core/metrics/distance_metrics.py @@ -225,6 +225,8 @@ def compute_rmse( if np.any(~np.isnan(pixel_errors)): rmse = np.nanmean(pixel_errors).item() + # TODO: CHECK THAT THIS WORKS WITH np.nanmean(pixel_errors, axis=1).item() + keypoint_scores = np.stack([m.keypoint_scores() for m in matches]) pixel_errors_cutoff = pixel_errors[keypoint_scores >= pcutoff] if np.any(~np.isnan(pixel_errors_cutoff)): diff --git a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py index 40981865de..0bef108dd4 100755 --- a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py +++ b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py @@ -102,6 +102,8 @@ def evaluate( loader: Loader, mode: str, detector_runner: InferenceRunner | None = None, + comparison_bodyparts: str | list[str] | None = None, + per_keypoint_evaluation: bool = False, pcutoff: float = 1, ) -> tuple[dict[str, float], dict[str, dict[str, np.ndarray]]]: """ @@ -113,6 +115,11 @@ def evaluate( detector_runner: If task == 'TD', a detector can be given to compute bounding boxes for pose estimation. If no detector is given, ground truth bounding boxes are used + comparison_bodyparts: A subset of the bodyparts for which to compute the + evaluation metrics. + per_keypoint_evaluation: Compute the train and test RMSE for each keypoint, and + save the results to a {model_name}-keypoint-results.csv in the + evaluation-results-pytorch folder. pcutoff: The p-cutoff to use for evaluation Returns: @@ -145,6 +152,18 @@ def evaluate( } gt_unique_keypoints = loader.ground_truth_keypoints(mode, unique_bodypart=True) + if comparison_bodyparts is not None: + poses = _get_keypoint_subset(poses, parameters.bodyparts, comparison_bodyparts) + gt_keypoints = _get_keypoint_subset( + gt_keypoints, parameters.bodyparts, comparison_bodyparts + ) + unique_poses = _get_keypoint_subset( + unique_poses, parameters.unique_bpts, comparison_bodyparts + ) + gt_unique_keypoints = _get_keypoint_subset( + gt_unique_keypoints, parameters.unique_bpts, comparison_bodyparts + ) + results = metrics.compute_metrics( gt_keypoints, poses, @@ -152,6 +171,7 @@ def evaluate( pcutoff=pcutoff, unique_bodypart_poses=unique_poses, unique_bodypart_gt=gt_unique_keypoints, + per_keypoint_evaluation=per_keypoint_evaluation, ) if loader.model_cfg["metadata"]["with_identity"]: @@ -377,6 +397,8 @@ def evaluate_snapshot( transform: A.Compose | None = None, plotting: bool | str = False, show_errors: bool = True, + comparison_bodyparts: str | list[str] | None = None, + per_keypoint_evaluation: bool = False, detector_snapshot: Snapshot | None = None, ) -> pd.DataFrame: """Evaluates a snapshot. @@ -394,6 +416,11 @@ def evaluate_snapshot( be either ``True``, ``False``, ``"bodypart"``, or ``"individual"``. Setting to ``True`` defaults as ``"bodypart"`` for multi-animal projects. show_errors: whether to compare predictions and ground truth + comparison_bodyparts: A subset of the bodyparts for which to compute the + evaluation metrics. + per_keypoint_evaluation: Compute the train and test RMSE for each keypoint, and + save the results to a {model_name}-keypoint-results.csv in the + evaluation-results-pytorch folder. detector_snapshot: Only for TD models. If defined, evaluation metrics are computed using the detections made by this snapshot """ @@ -519,6 +546,8 @@ def evaluate_network( plotting: bool | str = False, show_errors: bool = True, transform: A.Compose = None, + comparison_bodyparts: str | list[str] | None = None, + per_keypoint_evaluation: bool = False, modelprefix: str = "", detector_snapshot_index: int | None = None, ) -> None: @@ -547,6 +576,11 @@ def evaluate_network( show_errors: display train and test errors. transform: transformation pipeline for evaluation ** Should normalise the data the same way it was normalised during training ** + comparison_bodyparts: A subset of the bodyparts for which to compute the + evaluation metrics. + per_keypoint_evaluation: Compute the train and test RMSE for each keypoint, and + save the results to a {model_name}-keypoint-results.csv in the + evaluation-results-pytorch folder. modelprefix: directory containing the deeplabcut models to use when evaluating the network. By default, they are assumed to exist in the project folder. detector_snapshot_index: Only for TD models. If defined, uses the detector with @@ -653,6 +687,8 @@ def evaluate_network( transform=transform, plotting=plotting, show_errors=show_errors, + comparison_bodyparts=comparison_bodyparts, + per_keypoint_evaluation=per_keypoint_evaluation, detector_snapshot=detector_snapshot, ) @@ -705,6 +741,35 @@ def save_evaluation_results( df_scores.to_csv(combined_scores_path) +def _get_keypoint_subset( + data: dict[str, np.ndarray] | None, + bodyparts: list[str], + bodypart_subset: str | list[str], +) -> dict[str, np.ndarray] | None: + """ + + Args: + data: + bodyparts: + bodypart_subset: + + Returns: + + """ + if data is None: + return None + + if isinstance(bodypart_subset, str): + bodypart_subset = [bodypart_subset] + + to_keep = set(bodypart_subset) + bpt_indices = [i for i, b in enumerate(bodyparts) if b in to_keep] + if len(bpt_indices) == 0: + return None + + return {image: kpts[:, bpt_indices] for image, kpts in data.items()} + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--config", type=str) From 9a5b6d347bb37b6a94e92312b98800b870a690ca Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Mon, 9 Dec 2024 14:15:38 +0100 Subject: [PATCH 2/5] finished implementing per-keypoint RMSE --- deeplabcut/core/metrics/api.py | 32 ++-- deeplabcut/core/metrics/distance_metrics.py | 162 ++++++++++++++++-- .../pose_estimation_pytorch/apis/evaluate.py | 88 +++++++++- tests/core/metrics/test_metrics_api.py | 57 +++++- .../metrics/test_metrics_rmse_computation.py | 147 +++++++++++++++- 5 files changed, 445 insertions(+), 41 deletions(-) diff --git a/deeplabcut/core/metrics/api.py b/deeplabcut/core/metrics/api.py index addd836c69..1805b87180 100644 --- a/deeplabcut/core/metrics/api.py +++ b/deeplabcut/core/metrics/api.py @@ -25,7 +25,7 @@ def compute_metrics( pcutoff: float = -1, oks_bbox_margin: int = 0, oks_sigma: float = 0.1, - per_keypoint_evaluation: bool = False, + per_keypoint_rmse: bool = False, ) -> dict: """Computes pose estimation performance metrics @@ -71,7 +71,7 @@ def compute_metrics( oks_bbox_margin: The margin to add around keypoints to compute the area for OKS computation. oks_sigma: The OKS sigma to use to compute pose. - per_keypoint_evaluation: Compute per-keypoint RMSE values. + per_keypoint_rmse: Compute per-keypoint RMSE values. Returns: A dictionary containing keys "rmse", "rmse_cutoff", "mAP" and "mAR" mapping @@ -100,27 +100,33 @@ def compute_metrics( } # Sample output scores """ data = prepare_evaluation_data(ground_truth, predictions) - rmse, rmse_pcutoff = distance_metrics.compute_rmse(data, single_animal, pcutoff) oks_scores = distance_metrics.compute_oks( data=data, oks_sigma=oks_sigma, oks_bbox_margin=oks_bbox_margin, ) - results = dict(rmse=rmse, rmse_pcutoff=rmse_pcutoff, **oks_scores) + + data_unique = None + if unique_bodypart_gt is not None: + assert unique_bodypart_poses is not None + data_unique = prepare_evaluation_data(unique_bodypart_gt, unique_bodypart_poses) + + rmse_scores = distance_metrics.compute_rmse( + data, + single_animal, + pcutoff, + data_unique=data_unique, + per_keypoint_results=per_keypoint_rmse, + ) + results = dict(**rmse_scores, **oks_scores) if not single_animal: - det_rmse, det_rmse_p = distance_metrics.compute_detection_rmse(data, pcutoff) + det_rmse, det_rmse_p = distance_metrics.compute_detection_rmse( + data, pcutoff, data_unique=data_unique, + ) results["rmse_detections"] = det_rmse results["rmse_detections_pcutoff"] = det_rmse_p - if unique_bodypart_gt is not None: - # TODO: We should integrate unique bodyparts to main RMSE computation - assert unique_bodypart_poses is not None - unique_bpt = prepare_evaluation_data(unique_bodypart_gt, unique_bodypart_poses) - unique_bpt_metrics = distance_metrics.compute_rmse(unique_bpt, True, pcutoff) - results["rmse_unique_bpts"] = unique_bpt_metrics[0] - results["rmse_unique_bpts_pcutoff"] = unique_bpt_metrics[1] - return results diff --git a/deeplabcut/core/metrics/distance_metrics.py b/deeplabcut/core/metrics/distance_metrics.py index 5b22961fbb..f570b6b0bd 100644 --- a/deeplabcut/core/metrics/distance_metrics.py +++ b/deeplabcut/core/metrics/distance_metrics.py @@ -152,13 +152,12 @@ def compute_oks( } -def compute_rmse( +def match_predictions_for_rmse( data: list[tuple[np.ndarray, np.ndarray]], single_animal: bool, - pcutoff: float, oks_bbox_margin: float = 0.0, -) -> tuple[float, float]: - """Computes the RMSE for pose predictions. +) -> list[matching.PotentialMatch]: + """Matches GT keypoints to predictions to compute RMSE. Single animal RMSE is computed by simply calculating the distance between each ground truth keypoint and the corresponding prediction. @@ -176,16 +175,16 @@ def compute_rmse( num_bpts, 3). For the GT, the 3 coordinates are (x, y, visibility) while for the pose they are (x, y, confidence score). single_animal: Whether this is a single animal dataset. - pcutoff: The p-cutoff to use to compute RMSE. oks_bbox_margin: When single_animal is False, predictions are matched to GT using OKS. This is the margin used to apply when computing the bbox from the pose to compute OKS. Returns: - The RMSE and RMSE after removing all detections with a score below the pcutoff. + A list containing the predictions matched to ground truth. Raises: - AssertionError + ValueError: If `single_animal=True` but more than one ground truth/predicted + keypoint is found for an entry """ matches = [] for gt, pred in data: @@ -217,27 +216,106 @@ def compute_rmse( matches.extend(image_matches) - rmse, rmse_cutoff = float("nan"), float("nan") - if len(matches) == 0: - return rmse, rmse_cutoff + return matches + - pixel_errors = np.stack([m.pixel_errors() for m in matches]) - if np.any(~np.isnan(pixel_errors)): - rmse = np.nanmean(pixel_errors).item() +def compute_rmse( + data: list[tuple[np.ndarray, np.ndarray]], + single_animal: bool, + pcutoff: float, + data_unique: list[tuple[np.ndarray, np.ndarray]] | None = None, + per_keypoint_results: bool = False, + oks_bbox_margin: float = 0.0, +) -> dict[str, float]: + """Computes the RMSE for pose predictions. + + Single animal RMSE is computed by simply calculating the distance between each + ground truth keypoint and the corresponding prediction. + + Multi-animal RMSE is computed differently: predictions are first matched to ground + truth individuals using greedy OKS matching. RMSE is then computed only between + predictions and the ground truth pose they are matched to, only when the OKS is + non-zero (greater than a small threshold). Predictions that cannot be matched to + any ground truth with non-zero OKS are not used to compute RMSE. + + Args: + data: The data for which to compute RMSE. This is a list containing (gt_poses, + predicted_poses), where gt_pose is an array of shape (num_gt_individuals, + num_bpts, 3) and predicted_poses is an array of shape (num_predictions, + num_bpts, 3). For the GT, the 3 coordinates are (x, y, visibility) while for + the pose they are (x, y, confidence score). + single_animal: Whether this is a single animal dataset. + pcutoff: The p-cutoff to use to compute RMSE. + data_unique: Unique bodypart ground truth and predictions to include in RMSE + computations, if there are any such bodyparts. + per_keypoint_results: Whether to compute the RMSE for each individual keypoint. + oks_bbox_margin: When single_animal is False, predictions are matched to GT + using OKS. This is the margin used to apply when computing the bbox from + the pose to compute OKS. - # TODO: CHECK THAT THIS WORKS WITH np.nanmean(pixel_errors, axis=1).item() + Returns: + A dictionary matching metric names to values. It will at least have "rmse" and + "rmse_cutoff" keys. If `per_keypoint_results=True` and there is at least one + non-NaN pixel error it will also contain "rmse_keypoint_X" and + "rmse_cutoff_keypoint_X" keys for each bodypart, where X is the index of the + bodypart. + Raises: + ValueError: If `single_animal=True` but more than one ground truth/predicted + keypoint is found for an entry + """ + matches = match_predictions_for_rmse(data, single_animal, oks_bbox_margin) + pixel_errors, keypoint_scores = None, None + if len(matches) > 0: + pixel_errors = np.stack([m.pixel_errors() for m in matches]) keypoint_scores = np.stack([m.keypoint_scores() for m in matches]) - pixel_errors_cutoff = pixel_errors[keypoint_scores >= pcutoff] - if np.any(~np.isnan(pixel_errors_cutoff)): - rmse_cutoff = np.nanmean(pixel_errors_cutoff).item() - return rmse, rmse_cutoff + error, support, cutoff_error, cutoff_support = 0, 0, 0, 0 + if pixel_errors is not None: + error, support, cutoff_error, cutoff_support = collect_pixel_errors( + pixel_errors, keypoint_scores, pcutoff, + ) + + unique_pixel_errors, unique_keypoint_scores = None, None + if data_unique is not None: + u_matches = match_predictions_for_rmse(data_unique, single_animal=True) + if len(u_matches) > 0: + unique_pixel_errors = np.stack([m.pixel_errors() for m in u_matches]) + unique_keypoint_scores = np.stack([m.keypoint_scores() for m in u_matches]) + + u_error, u_support, u_cutoff_error, u_cutoff_support = collect_pixel_errors( + unique_pixel_errors, unique_keypoint_scores, pcutoff, + ) + error += u_error + support += u_support + cutoff_error += u_cutoff_error + cutoff_support += u_cutoff_support + + results = dict(rmse=float("nan"), rmse_pcutoff=float("nan")) + if support > 0: + results["rmse"] = error / support + if cutoff_support > 0: + results["rmse_pcutoff"] = cutoff_error / cutoff_support + + if per_keypoint_results: + bodypart_errors = [("rmse_keypoint", pixel_errors)] + if unique_pixel_errors is not None: + bodypart_errors.append(("rmse_unique_keypoint", unique_pixel_errors)) + + for key_prefix, bpt_errors in bodypart_errors: + for idx, keypoint_error in enumerate(bpt_errors.T): + rmse = float("nan") + if np.any(~np.isnan(keypoint_error)): + rmse = np.nanmean(keypoint_error) + results[f"{key_prefix}_{idx}"] = rmse + + return results def compute_detection_rmse( data: list[tuple[np.ndarray, np.ndarray]], pcutoff: float, + data_unique: list[tuple[np.ndarray, np.ndarray]] | None = None, ) -> tuple[float, float]: """Computes the detection RMSE for pose predictions. @@ -254,6 +332,8 @@ def compute_detection_rmse( num_bpts, 3). For the GT, the 3 coordinates are (x, y, visibility) while for the pose they are (x, y, confidence score). pcutoff: The p-cutoff to use to compute RMSE. + data_unique: Unique bodypart ground truth and predictions to include in RMSE + computations, if there are any such bodyparts. Returns: The detection RMSE and detection RMSE after removing all detections with a @@ -281,6 +361,17 @@ def compute_detection_rmse( distances.append(np.linalg.norm(gt[:2] - pred[:2])) scores.append(bpt_pred[pred_index, 2]) + if data_unique is not None: + for image_gt, image_pred in data_unique: + assert len(image_gt) == len(image_pred) == 1, ( + f"Unique GT an predictions must have length 1! Found {image_gt.shape}, " + f"{image_pred.shape}." + ) + unique_gt, unique_pred = image_gt[0], image_pred[0] + for gt, pred in zip(unique_gt, unique_pred): + distances.append(np.linalg.norm(gt[:2] - pred[:2])) + scores.append(pred[2]) + rmse, rmse_cutoff = float("nan"), float("nan") if len(distances) == 0: return rmse, rmse_cutoff @@ -295,3 +386,38 @@ def compute_detection_rmse( rmse_cutoff = np.nanmean(pixel_errors_cutoff).item() return rmse, rmse_cutoff + + +def collect_pixel_errors( + pixel_errors: np.ndarray, + keypoint_scores: np.ndarray, + pcutoff: float, +) -> tuple[float, int, float, int]: + """Collects pixel errors for RMSE computation + + Args: + pixel_errors: The pixel errors to collect, of shape (num_matches, num_bodyparts) + keypoint_scores: The scores corresponding to the pixel errors, of shape + (num_matches, num_bodyparts). + pcutoff: The pcutoff to use when computing cutoff RMSE. + + Returns: error, support, cutoff_error, support_cutoff + error: The sum of all pixel errors. + support: The number of valid pixel errors. + cutoff_error: The sum of all pixel errors with score > pcutoff. + support_cutoff: The number of valid pixel errors with score > pcutoff. + """ + error = 0.0 + cutoff_error = 0.0 + support = np.sum(~np.isnan(pixel_errors)).item() + support_cutoff = 0 + if support > 0: + error += np.nansum(pixel_errors).item() + + cutoff_mask = keypoint_scores >= pcutoff + cutoff_pixel_errors = pixel_errors[cutoff_mask] + support_cutoff = np.sum(~np.isnan(cutoff_pixel_errors)).item() + if support_cutoff > 0: + cutoff_error = np.nansum(cutoff_pixel_errors) + + return error, support, cutoff_error, support_cutoff diff --git a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py index 0bef108dd4..c43d310d43 100755 --- a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py +++ b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py @@ -116,7 +116,7 @@ def evaluate( boxes for pose estimation. If no detector is given, ground truth bounding boxes are used comparison_bodyparts: A subset of the bodyparts for which to compute the - evaluation metrics. + evaluation metrics. Passing "all" or None evaluates on all bodyparts. per_keypoint_evaluation: Compute the train and test RMSE for each keypoint, and save the results to a {model_name}-keypoint-results.csv in the evaluation-results-pytorch folder. @@ -127,6 +127,9 @@ def evaluate( A dict mapping the paths of images for which predictions were computed to the different predictions made by each model head """ + if comparison_bodyparts == "all": + comparison_bodyparts = None + parameters = loader.get_dataset_parameters() predictions = predict( pose_task=pose_task, @@ -157,6 +160,13 @@ def evaluate( gt_keypoints = _get_keypoint_subset( gt_keypoints, parameters.bodyparts, comparison_bodyparts ) + if poses is None or gt_keypoints is None: + raise ValueError( + "comparison_bodyparts must include at least one bodypart defined in " + f"the project. Found {comparison_bodyparts} but project bodyparts are " + f"{parameters.bodyparts}" + ) + unique_poses = _get_keypoint_subset( unique_poses, parameters.unique_bpts, comparison_bodyparts ) @@ -171,7 +181,7 @@ def evaluate( pcutoff=pcutoff, unique_bodypart_poses=unique_poses, unique_bodypart_gt=gt_unique_keypoints, - per_keypoint_evaluation=per_keypoint_evaluation, + per_keypoint_rmse=per_keypoint_evaluation, ) if loader.model_cfg["metadata"]["with_identity"]: @@ -455,6 +465,7 @@ def evaluate_snapshot( ) predictions = {} + rmse_per_bodypart = {} scores = { "%Training dataset": loader.train_fraction, "Shuffle number": loader.shuffle, @@ -472,7 +483,16 @@ def evaluate_snapshot( mode=split, pcutoff=pcutoff, detector_runner=detector_runner, + comparison_bodyparts=comparison_bodyparts, + per_keypoint_evaluation=per_keypoint_evaluation, ) + if per_keypoint_evaluation: + rmse_per_bodypart[split] = _extract_rmse_per_keypoint( + results, + parameters.bodyparts, + parameters.unique_bpts, + ) + df_split_predictions = build_predictions_dataframe( scorer=scorer, predictions=predictions_for_split, @@ -503,6 +523,12 @@ def evaluate_snapshot( scores_filepath = scores_filepath.with_stem(scores_filepath.stem + "-results") save_evaluation_results(df_scores, scores_filepath, show_errors, pcutoff) + if per_keypoint_evaluation: + rmse_per_bpt_path = output_filename.with_name( + output_filename.stem + "-keypoint-results.csv" + ) + save_rmse_per_bodypart(rmse_per_bodypart, rmse_per_bpt_path, show_errors) + if plotting: folder_name = f"LabeledImages_{scorer}" folder_path = loader.evaluation_folder / folder_name @@ -741,6 +767,40 @@ def save_evaluation_results( df_scores.to_csv(combined_scores_path) +def save_rmse_per_bodypart( + rmse_per_bodypart: dict[str, dict[str, float]], + output_path: Path, + print_results: bool, +) -> None: + """ + Saves the evaluation results per bodypart to a CSV file. + + Args: + rmse_per_bodypart: The scores dataframe for a snapshot + output_path: The path of the file where + print_results: Whether to print results to the console + """ + index, data = [], [] + if print_results: + print(f"Per-bodypart evaluation results ({output_path.stem}):") + + for split, rmse_results in rmse_per_bodypart.items(): + key = split.capitalize() + " error (px)" + index.append(key) + data.append(rmse_results) + + if print_results: + print(f" {key}") + bpt_key_length = max([len(k) for k in rmse_results.keys()]) + 4 + for k, v in rmse_results.items(): + key = (k + ":").ljust(bpt_key_length) + print(f" {key}{v:3>.2f}px") + + # Save scores file + df_rmse_per_bodypart = pd.DataFrame(data, index=index) + df_rmse_per_bodypart.to_csv(output_path) + + def _get_keypoint_subset( data: dict[str, np.ndarray] | None, bodyparts: list[str], @@ -770,6 +830,30 @@ def _get_keypoint_subset( return {image: kpts[:, bpt_indices] for image, kpts in data.items()} +def _extract_rmse_per_keypoint( + results: dict[str, float], + bodyparts: list[str], + unique_bodyparts: list[str], +) -> dict[str, float]: + """ + + Args: + results: + bodyparts: + unique_bodyparts: + + Returns: + + """ + rmse_per_keypoint = {} + for bpt_idx, bpt in enumerate(bodyparts): + rmse_per_keypoint[bpt] = results.pop(f"rmse_keypoint_{bpt_idx}") + for bpt_idx, bpt in enumerate(unique_bodyparts): + rmse_per_keypoint[bpt] = results.pop(f"rmse_unique_keypoint_{bpt_idx}") + + return rmse_per_keypoint + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--config", type=str) diff --git a/tests/core/metrics/test_metrics_api.py b/tests/core/metrics/test_metrics_api.py index 48e5497dcb..5051794a13 100644 --- a/tests/core/metrics/test_metrics_api.py +++ b/tests/core/metrics/test_metrics_api.py @@ -16,6 +16,17 @@ import deeplabcut.core.metrics as metrics +def _get_gt_and_pred_with_constant_err( + num_idv: int, num_bpt: int, error: float +) -> tuple[np.ndarray, np.ndarray]: + gt = np.arange(num_idv * num_bpt * 3).astype(float).reshape((num_idv, num_bpt, 3)) + gt[..., 2] = 2 + predictions = gt.copy() + predictions[..., 2] = 0.9 + predictions[..., :2] += error + return gt, predictions + + def test_computing_metrics_with_no_predictions(): gt = np.arange(5 * 6 * 3).astype(float).reshape((5, 6, 3)) gt[..., 2] = 2 @@ -30,11 +41,7 @@ def test_computing_metrics_with_no_predictions(): @pytest.mark.parametrize("error", [0.5, 1, 2]) def test_computing_metrics_with_constant_error(error): # only works for small errors: otherwise another matching can be found - gt = np.arange(5 * 6 * 3).astype(float).reshape((5, 6, 3)) - gt[..., 2] = 2 - predictions = gt.copy() - predictions[..., 2] = 0.9 - predictions[..., :2] += error + gt, predictions = _get_gt_and_pred_with_constant_err(5, 6, error) results = metrics.compute_metrics( ground_truth={"image": gt}, predictions={"image": predictions}, @@ -45,6 +52,46 @@ def test_computing_metrics_with_constant_error(error): assert_almost_equal(results["rmse_pcutoff"], np.sqrt(2) * error) +@pytest.mark.parametrize("error", [0.5, 1, 2]) +def test_metrics_with_unique_with_constant_error(error): + # only works for small errors: otherwise another matching can be found + gt, predictions = _get_gt_and_pred_with_constant_err(5, 6, error) + gt_unique, pred_unique = _get_gt_and_pred_with_constant_err(1, 8, error) + results = metrics.compute_metrics( + ground_truth={"image": gt}, + predictions={"image": predictions}, + unique_bodypart_gt={"image": gt_unique}, + unique_bodypart_poses={"image": pred_unique}, + ) + assert_almost_equal(results["rmse"], np.sqrt(2) * error) + assert_almost_equal(results["rmse_pcutoff"], np.sqrt(2) * error) + + +@pytest.mark.parametrize("error", [0.5, 1, 2]) +def test_metrics_per_bpt_with_unique_with_constant_error(error): + # only works for small errors: otherwise another matching can be found + gt, predictions = _get_gt_and_pred_with_constant_err(5, 6, error) + gt_unique, pred_unique = _get_gt_and_pred_with_constant_err(1, 8, error) + results = metrics.compute_metrics( + ground_truth={"image": gt}, + predictions={"image": predictions}, + unique_bodypart_gt={"image": gt_unique}, + unique_bodypart_poses={"image": pred_unique}, + per_keypoint_rmse=True, + ) + assert_almost_equal(results["rmse"], np.sqrt(2) * error) + assert_almost_equal(results["rmse_pcutoff"], np.sqrt(2) * error) + + for bpt_idx in range(gt.shape[1]): + key = f"rmse_keypoint_{bpt_idx}" + assert key in results + assert_almost_equal(results[key], np.sqrt(2) * error) + for bpt_idx in range(gt_unique.shape[1]): + key = f"rmse_unique_keypoint_{bpt_idx}" + assert key in results + assert_almost_equal(results[key], np.sqrt(2) * error) + + @pytest.mark.parametrize("error", [0.5, 1, 2]) def test_computing_metrics_single_animal(error): # only works for small errors: otherwise another matching can be found diff --git a/tests/core/metrics/test_metrics_rmse_computation.py b/tests/core/metrics/test_metrics_rmse_computation.py index 61fed3d8e2..d9f2b7c350 100644 --- a/tests/core/metrics/test_metrics_rmse_computation.py +++ b/tests/core/metrics/test_metrics_rmse_computation.py @@ -57,7 +57,8 @@ ) def test_rmse_single_image(gt: list, pred: list, result: tuple[float, float]): data = [(np.asarray(gt), np.asarray(pred))] - rmse, rmse_cutoff = compute_rmse(data, False, pcutoff=0.6, oks_bbox_margin=10.0) + computed_results = compute_rmse(data, False, pcutoff=0.6, oks_bbox_margin=10.0) + rmse, rmse_cutoff = computed_results["rmse"], computed_results["rmse_pcutoff"] expected_rmse, expected_rmse_cutoff = result assert_almost_equal(rmse, expected_rmse) assert_almost_equal(rmse_cutoff, expected_rmse_cutoff) @@ -83,7 +84,8 @@ def test_rmse_pcutoff(gt: list, pred: list, result: tuple[float, float]): data = [(np.asarray(gt), np.asarray(pred))] expected_rmse, expected_rmse_cutoff = result - rmse, rmse_cutoff = compute_rmse(data, False, pcutoff=0.6, oks_bbox_margin=10.0) + computed_results = compute_rmse(data, False, pcutoff=0.6, oks_bbox_margin=10.0) + rmse, rmse_cutoff = computed_results["rmse"], computed_results["rmse_pcutoff"] assert_almost_equal(rmse, expected_rmse) assert_almost_equal(rmse_cutoff, expected_rmse_cutoff) @@ -117,7 +119,8 @@ def test_rmse_with_nans(gt: list, pred: list, result: tuple[float, float]): data = [(np.asarray(gt), np.asarray(pred))] expected_rmse, expected_rmse_cutoff = result - rmse, rmse_cutoff = compute_rmse(data, False, pcutoff=0.6, oks_bbox_margin=10.0) + results = compute_rmse(data, False, pcutoff=0.6, oks_bbox_margin=10.0) + rmse, rmse_cutoff = results["rmse"], results["rmse_pcutoff"] assert_almost_equal(rmse, expected_rmse) assert_almost_equal(rmse_cutoff, expected_rmse_cutoff) @@ -199,3 +202,141 @@ def test_detection_rmse(gt: list, pred: list, result: tuple[float, float]): rmse, rmse_cutoff = compute_detection_rmse(data, pcutoff=0.6) assert_almost_equal(rmse, expected_rmse) assert_almost_equal(rmse_cutoff, expected_rmse_cutoff) + + +@pytest.mark.parametrize( + "gt, pred, unique_gt, unique_pred, result", + [ + ( + [ # ground truth pose + [[10.0, 10.0, 2], [10.0, 10.0, 2], [10.0, 10.0, 2]], + [[20.0, 20.0, 2], [20.0, 20.0, 2], [20.0, 20.0, 2]], + ], + [ # predicted pose + [[10.0, 10.0, 0.9], [10.0, 10.0, 0.9], [10.0, 10.0, 0.9]], + [[20.0, 24.0, 0.2], [20.0, 24.0, 0.2], [20.0, 20.0, 0.2]], + ], + [ # Unique GT + [[10.0, 10.0, 2], [10.0, 10.0, 2]], + ], + [ # Unique Pred + [[10.0, 10.0, 0.9], [10.0, 10.0, 0.9]], + ], + # 4 pixel error on 2 keypoints, 0 error on 5 keypoints + (1.0, 0.0), + ), + ( + [np.zeros((0, 3, 2))], # no GT pose + [ # predicted pose + [[10.0, 10.0, 0.9], [10.0, 10.0, 0.9], [10.0, 10.0, 0.9]], + ], + [ # Unique GT + [[10.0, 10.0, 2], [10.0, 10.0, 2]], + ], + [ # Unique Pred + [[15.0, 10.0, 0.5], [11.0, 10.0, 0.9]], + ], + # 5 pixel error on 1 keypoint, 1 pixel error on the other + (3.0, 1.0), + ), + ], +) +def test_rmse_with_unique( + gt: list, + pred: list, + unique_gt: list, + unique_pred: list, + result: tuple[float, float] +) -> None: + data = [(np.asarray(gt), np.asarray(pred))] + data_unique = [(np.asarray(unique_gt), np.asarray(unique_pred))] + expected_rmse, expected_rmse_cutoff = result + + results = compute_rmse( + data, False, pcutoff=0.6, data_unique=data_unique, oks_bbox_margin=10.0, + ) + rmse, rmse_cutoff = results["rmse"], results["rmse_pcutoff"] + assert_almost_equal(rmse, expected_rmse) + assert_almost_equal(rmse_cutoff, expected_rmse_cutoff) + + +@pytest.mark.parametrize( + "gt, pred, unique_gt, unique_pred, result", + [ + ( + [ # ground truth pose + [[10.0, 10.0, 2], [10.0, 10.0, 2], [10.0, 10.0, 2]], + [[20.0, 20.0, 2], [20.0, 20.0, 2], [20.0, 20.0, 2]], + ], + [ # predicted pose + [[10.0, 10.0, 0.9], [10.0, 10.0, 0.9], [10.0, 10.0, 0.9]], + [[20.0, 24.0, 0.2], [20.0, 24.0, 0.2], [20.0, 20.0, 0.2]], + ], + [ # Unique GT + [[10.0, 10.0, 2], [10.0, 10.0, 2]], + ], + [ # Unique Pred + [[10.0, 10.0, 0.9], [10.0, 10.0, 0.9]], + ], + # 4 pixel error on 2 keypoints, 0 error on 5 keypoints + [ + (1.0, 0.0), + [2.0, 2.0, 0.0], + [0.0, 0.0] + ], + ), + ( + [ # ground truth pose + [[10.0, 10.0, 2], [10.0, 10.0, 2], [10.0, 10.0, 2]], + [[20.0, 20.0, 2], [20.0, 20.0, 2], [20.0, 20.0, 2]], + ], + [ # predicted pose + [[10.0, 12.0, 0.9], [10.0, 10.0, 0.9], [10.0, 10.0, 0.9]], + [[20.0, 24.0, 0.7], [20.0, 24.0, 0.6], [20.0, 20.0, 0.8]], + ], + [ # Unique GT + [[10.0, 10.0, 2], [10.0, 10.0, 2]], + ], + [ # Unique Pred + [[12.0, 10.0, 0.9], [11.0, 10.0, 0.9]], + ], + [ # errors: 3 with 0px, 1 with 1px, 2 with 2px, 2 with 4px => 13/8 + (1.625, 1.625), + [3.0, 2.0, 0.0], + [2.0, 1.0] + ], + ), + ], +) +def test_rmse_per_bodypart_with_unique( + gt: list, + pred: list, + unique_gt: list, + unique_pred: list, + result: tuple[tuple[float, float], list[float], list[float]] +) -> None: + data = [(np.asarray(gt), np.asarray(pred))] + data_unique = [(np.asarray(unique_gt), np.asarray(unique_pred))] + expected_rmse, expected_rmse_cutoff = result[0] + bodypart_rmse = result[1] + unique_rmse = result[2] + + results = compute_rmse( + data, + single_animal=False, + pcutoff=0.6, + data_unique=data_unique, + per_keypoint_results=True, + oks_bbox_margin=10.0, + ) + assert_almost_equal(results["rmse"], expected_rmse) + assert_almost_equal(results["rmse_pcutoff"], expected_rmse_cutoff) + for bpt_index, bpt_rmse in enumerate(bodypart_rmse): + key = f"rmse_keypoint_{bpt_index}" + assert key in results + assert_almost_equal(results[key], bpt_rmse) + + for bpt_index, bpt_rmse in enumerate(unique_rmse): + key = f"rmse_unique_keypoint_{bpt_index}" + assert key in results + assert_almost_equal(results[key], bpt_rmse) From 5816e120b96ccbebbfc82d7e83450ab00e9fafc6 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Mon, 9 Dec 2024 16:22:39 +0100 Subject: [PATCH 3/5] improved comparison bodyparts to function with superanimal models --- deeplabcut/pose_estimation_pytorch/README.md | 10 +- .../pose_estimation_pytorch/apis/evaluate.py | 134 ++++++++++++------ examples/utils.py | 1 + 3 files changed, 93 insertions(+), 52 deletions(-) diff --git a/deeplabcut/pose_estimation_pytorch/README.md b/deeplabcut/pose_estimation_pytorch/README.md index ef4b562e40..0c3eec28a1 100644 --- a/deeplabcut/pose_estimation_pytorch/README.md +++ b/deeplabcut/pose_estimation_pytorch/README.md @@ -269,7 +269,6 @@ from deeplabcut.pose_estimation_pytorch.data import ( build_transforms, DLCLoader, ) -from deeplabcut.pose_estimation_pytorch.task import Task loader = DLCLoader( config="/path/to/my/project/config.yaml", @@ -279,12 +278,12 @@ loader = DLCLoader( train_dataset = loader.create_dataset( transform=build_transforms(loader.model_cfg["data"]["train"]), mode="train", - task=Task.BOTTOM_UP, + task=loader.pose_task, ) valid_dataset = loader.create_dataset( transform=build_transforms(loader.model_cfg["data"]["train"]), mode="test", - task=Task.BOTTOM_UP, + task=loader.pose_task, ) ``` @@ -358,7 +357,6 @@ model_cfg = make_pytorch_pose_config( top_down=True, ) write_config(config_path=model_cfg_path, config=model_cfg) -task = Task(model_cfg["method"]) # Create the loader for the COCO dataset loader = COCOLoader( @@ -370,12 +368,12 @@ loader = COCOLoader( train_dataset = loader.create_dataset( transform=build_transforms(loader.model_cfg["data"]["train"]), mode="train", - task=task, + task=loader.pose_task, ) valid_dataset = loader.create_dataset( transform=build_transforms(loader.model_cfg["data"]["train"]), mode="test", - task=task, + task=loader.pose_task, ) ``` diff --git a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py index c43d310d43..c488c6efba 100755 --- a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py +++ b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py @@ -48,7 +48,6 @@ def predict( - pose_task: Task, pose_runner: InferenceRunner, loader: Loader, mode: str, @@ -57,7 +56,6 @@ def predict( """Predicts poses on data contained in a loader Args: - pose_task: Whether the model is a top-down or bottom-up model pose_runner: The runner to use for pose estimation loader: The loader containing the data to predict poses on mode: {"train", "test"} The mode to predict on @@ -72,7 +70,7 @@ def predict( image_paths = loader.image_filenames(mode) context = None - if pose_task == Task.TOP_DOWN: + if loader.pose_task == Task.TOP_DOWN: # Get bounding boxes for context if detector_runner is not None: bbox_predictions = detector_runner.inference(images=tqdm(image_paths)) @@ -97,24 +95,26 @@ def predict( def evaluate( - pose_task: Task, pose_runner: InferenceRunner, loader: Loader, mode: str, detector_runner: InferenceRunner | None = None, + parameters: PoseDatasetParameters | None = None, comparison_bodyparts: str | list[str] | None = None, per_keypoint_evaluation: bool = False, pcutoff: float = 1, ) -> tuple[dict[str, float], dict[str, dict[str, np.ndarray]]]: """ Args: - pose_task: Whether to run top-down or bottom-up pose_runner: The runner for pose estimation loader: The loader containing the data to evaluate mode: Either 'train' or 'test' detector_runner: If task == 'TD', a detector can be given to compute bounding boxes for pose estimation. If no detector is given, ground truth bounding - boxes are used + boxes are used. + parameters: PoseDatasetParameters to use. If None, the parameters will be + obtained from the given Loader. This can be used to change the names of + bodyparts, e.g. when a model is trained with memory replay. comparison_bodyparts: A subset of the bodyparts for which to compute the evaluation metrics. Passing "all" or None evaluates on all bodyparts. per_keypoint_evaluation: Compute the train and test RMSE for each keypoint, and @@ -130,25 +130,27 @@ def evaluate( if comparison_bodyparts == "all": comparison_bodyparts = None - parameters = loader.get_dataset_parameters() predictions = predict( - pose_task=pose_task, pose_runner=pose_runner, loader=loader, mode=mode, detector_runner=detector_runner, ) + + # For models trained with memory-replay from SuperAnimal, keep project bodyparts if weight_init_cfg := loader.model_cfg["train_settings"].get("weight_init"): weight_init = WeightInitialization.from_dict(weight_init_cfg) if weight_init.memory_replay: for _, pred in predictions.items(): pred["bodyparts"] = pred["bodyparts"][:, weight_init.conversion_array] + gt_keypoints = loader.ground_truth_keypoints(mode) poses = {filename: pred["bodyparts"] for filename, pred in predictions.items()} - gt_keypoints = loader.ground_truth_keypoints(mode) - unique_poses = None - gt_unique_keypoints = None + if parameters is None: + parameters = loader.get_dataset_parameters() + + gt_unique_keypoints, unique_poses = None, None if parameters.num_unique_bpts > 1: unique_poses = { filename: pred["unique_bodyparts"] for filename, pred in predictions.items() @@ -434,7 +436,6 @@ def evaluate_snapshot( detector_snapshot: Only for TD models. If defined, evaluation metrics are computed using the detections made by this snapshot """ - pose_task = Task(loader.model_cfg.get("method", "bu")) parameters = loader.get_dataset_parameters() pcutoff = cfg.get("pcutoff", 0.6) @@ -451,16 +452,26 @@ def evaluate_snapshot( with_identity=loader.model_cfg["metadata"]["with_identity"], transform=transform, detector_path=detector_path, - detector_transform=None, ) - # The project bodyparts might be different to the bodyparts the model was trained to - # output, if the model is fine-tuned from SuperAnimal with memory replay. - # For evaluation, we want to only use the project bodyparts - project_bodyparts = auxiliaryfunctions.get_bodyparts(cfg) - parameters = PoseDatasetParameters( - bodyparts=project_bodyparts, - unique_bpts=parameters.unique_bpts, + # For memory-replay SuperAnimal models, convert bodyparts to project bodyparts + if weight_init_cfg := loader.model_cfg["train_settings"].get("weight_init", None): + weight_init = WeightInitialization.from_dict(weight_init_cfg) + if weight_init.memory_replay: + bodyparts = weight_init.bodyparts + if bodyparts is None: + bodyparts = auxiliaryfunctions.get_bodyparts(cfg) + + parameters = PoseDatasetParameters( + bodyparts=bodyparts, + unique_bpts=parameters.unique_bpts, + individuals=parameters.individuals, + ) + + # get the names of bodyparts on which the model is evaluated + eval_parameters = PoseDatasetParameters( + bodyparts=_get_subset_bodyparts(parameters.bodyparts, comparison_bodyparts), + unique_bpts=_get_subset_bodyparts(parameters.unique_bpts, comparison_bodyparts), individuals=parameters.individuals, ) @@ -477,7 +488,6 @@ def evaluate_snapshot( } for split in ["train", "test"]: results, predictions_for_split = evaluate( - pose_task=pose_task, pose_runner=pose_runner, loader=loader, mode=split, @@ -485,18 +495,17 @@ def evaluate_snapshot( detector_runner=detector_runner, comparison_bodyparts=comparison_bodyparts, per_keypoint_evaluation=per_keypoint_evaluation, + parameters=parameters, ) if per_keypoint_evaluation: - rmse_per_bodypart[split] = _extract_rmse_per_keypoint( - results, - parameters.bodyparts, - parameters.unique_bpts, + rmse_per_bodypart[split] = _extract_rmse_per_bodypart( + results, eval_parameters.bodyparts, eval_parameters.unique_bpts, ) df_split_predictions = build_predictions_dataframe( scorer=scorer, predictions=predictions_for_split, - parameters=parameters, + parameters=eval_parameters, image_name_to_index=image_to_dlc_df_index, ) predictions[split] = df_split_predictions @@ -543,7 +552,6 @@ def evaluate_snapshot( df_combined = predictions[mode].merge( df_ground_truth, left_index=True, right_index=True ) - unique_bodyparts = loader.get_dataset_parameters().unique_bpts plot_evaluation_results( df_combined=df_combined, @@ -552,7 +560,7 @@ def evaluate_snapshot( model_name=scorer, output_folder=str(folder_path), in_train_set=mode == "train", - plot_unique_bodyparts=len(unique_bodyparts) > 0, + plot_unique_bodyparts=eval_parameters.num_unique_bpts > 0, mode=plot_mode, colormap=cfg["colormap"], dot_size=cfg["dotsize"], @@ -665,15 +673,14 @@ def evaluate_network( loader.model_cfg["device"] = device loader.model_cfg["device"] = utils.resolve_device(loader.model_cfg) - task = Task(loader.model_cfg["method"]) snapshots = get_model_snapshots( snapshotindex, model_folder=loader.model_folder, - task=task, + task=loader.pose_task, ) detector_snapshots = [None] - if task == Task.TOP_DOWN: + if loader.pose_task == Task.TOP_DOWN: if detector_snapshot_index is not None: det_snapshots = get_model_snapshots( "all", loader.model_folder, Task.DETECT @@ -806,20 +813,26 @@ def _get_keypoint_subset( bodyparts: list[str], bodypart_subset: str | list[str], ) -> dict[str, np.ndarray] | None: - """ + """Obtains the pose for a subset of bodyparts Args: - data: - bodyparts: - bodypart_subset: + data: The data for which to obtain the pose belonging to the subset. A dict + mapping image name to an array of shape (num_idv, len(bodyparts), ...). + bodyparts: The bodyparts corresponding to the columns of the arrays in the data + dict (in the same order). + bodypart_subset: The subset of bodyparts to keep. Returns: - + A dict containing the image keys, mapping to arrays of shape + (num_idv, len(subset), ...) containing the data subset. """ if data is None: return None if isinstance(bodypart_subset, str): + if bodypart_subset == "all": + return data + bodypart_subset = [bodypart_subset] to_keep = set(bodypart_subset) @@ -830,28 +843,57 @@ def _get_keypoint_subset( return {image: kpts[:, bpt_indices] for image, kpts in data.items()} -def _extract_rmse_per_keypoint( +def _get_subset_bodyparts( + bodyparts: list[str], subset: str | list[str] | None, +) -> list[str]: + """Gets a subset of bodyparts that were used. + + Args: + bodyparts: The bodyparts output by the model. + subset: The subset of bodyparts to keep. + + Returns: + The bodyparts that were used to evaluate the model. + """ + if isinstance(subset, str): + if subset == "all": + return bodyparts + subset = [subset] + + to_keep = set(subset) + return [b for b in bodyparts if b in to_keep] + + +def _extract_rmse_per_bodypart( results: dict[str, float], bodyparts: list[str], unique_bodyparts: list[str], ) -> dict[str, float]: - """ + """Extracts the RMSE per bodypart metrics from the results dict + + This method modifies the given dict in-place, removing all keys for RMSE per + bodypart or unique bodypart. Args: - results: - bodyparts: - unique_bodyparts: + results: The results returned by the evaluation method. + bodyparts: The bodyparts defined for the project. + unique_bodyparts: The unique bodyparts defined for the project. Returns: - + The per-bodypart RMSE. """ - rmse_per_keypoint = {} + rmse_per_bodypart = {} for bpt_idx, bpt in enumerate(bodyparts): - rmse_per_keypoint[bpt] = results.pop(f"rmse_keypoint_{bpt_idx}") + rmse = results.pop(f"rmse_keypoint_{bpt_idx}", None) + if rmse is not None: + rmse_per_bodypart[bpt] = rmse + for bpt_idx, bpt in enumerate(unique_bodyparts): - rmse_per_keypoint[bpt] = results.pop(f"rmse_unique_keypoint_{bpt_idx}") + rmse = results.pop(f"rmse_unique_keypoint_{bpt_idx}", None) + if rmse is not None: + rmse_per_bodypart[bpt] = rmse - return rmse_per_keypoint + return rmse_per_bodypart if __name__ == "__main__": diff --git a/examples/utils.py b/examples/utils.py index 6c61795169..e355b53020 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -374,6 +374,7 @@ def run( trainingsetindex=trainset_index, device=device, plotting=True, + per_keypoint_evaluation=True, ) times.append(time.time()) log_step(f"Evaluation time: {times[-1] - times[-2]} seconds") From 745d376f5b351f81aa943047db934f69eeea790b Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Mon, 9 Dec 2024 16:40:08 +0100 Subject: [PATCH 4/5] updated docs --- docs/pytorch/user_guide.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pytorch/user_guide.md b/docs/pytorch/user_guide.md index d33b8e5cf4..c5420d4e65 100644 --- a/docs/pytorch/user_guide.md +++ b/docs/pytorch/user_guide.md @@ -76,11 +76,11 @@ parameters are not valid for the DLC 3.0 PyTorch API. | API Method | Implemented | Parameters not yet implemented | Parameters invalid for pytorch | |--------------------------------|:-----------:|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------| -| `train_network` | 🟢 | `keepdeconvweights` | `maxiters`, `saveiters`, `allow_growth`, `autotune` | +| `train_network` | 🟠 | `keepdeconvweights` | `maxiters`, `saveiters`, `allow_growth`, `autotune` | | `return_train_network_path` | 🟢 | | | -| `evaluate_network` | 🟢 | `comparisonbodyparts`, `rescale`, `per_keypoint_evaluation` | | +| `evaluate_network` | 🟠 | `rescale` | | | `return_evaluate_network_data` | 🔴 | | `TFGPUinference`, `allow_growth` | -| `analyze_videos` | 🟢 | `in_random_order`, `dynamic`, `n_tracks`, `calibrate` | | +| `analyze_videos` | 🟠 | `in_random_order`, `dynamic`, `n_tracks`, `calibrate` | | | `create_tracking_dataset` | 🔴 | | | | `analyze_time_lapse_frames` | 🟠 | the name has changed to `analyze_images` to better reflect what it actually does (no video needed) | | | `convert_detections2tracklets` | 🟢 | `greedy`, `calibrate`, `window_size` | | From 5766038fe612afd6155b579ded65d2da899e9ffd Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Tue, 10 Dec 2024 15:51:21 +0100 Subject: [PATCH 5/5] updated docstrings --- deeplabcut/pose_estimation_pytorch/apis/evaluate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py index c488c6efba..14460ace21 100755 --- a/deeplabcut/pose_estimation_pytorch/apis/evaluate.py +++ b/deeplabcut/pose_estimation_pytorch/apis/evaluate.py @@ -59,9 +59,9 @@ def predict( pose_runner: The runner to use for pose estimation loader: The loader containing the data to predict poses on mode: {"train", "test"} The mode to predict on - detector_runner: If the task is "TD", a detector runner can be given to detect - individuals in the images. If no detector is given, ground truth bounding - boxes will be used to crop individuals before pose estimation + detector_runner: If the loader's `pose_task` is "TD", a detector runner can be + given to detect individuals in the images. If no detector is given, ground + truth bounding boxes will be used to crop individuals before pose estimation Returns: The paths of images for which predictions were computed mapping to the @@ -109,9 +109,9 @@ def evaluate( pose_runner: The runner for pose estimation loader: The loader containing the data to evaluate mode: Either 'train' or 'test' - detector_runner: If task == 'TD', a detector can be given to compute bounding - boxes for pose estimation. If no detector is given, ground truth bounding - boxes are used. + detector_runner: If the loader's `pose_task` is "TD", a detector can be given to + compute bounding boxes for pose estimation. If no detector is given, ground + truth bounding boxes are used. parameters: PoseDatasetParameters to use. If None, the parameters will be obtained from the given Loader. This can be used to change the names of bodyparts, e.g. when a model is trained with memory replay.