Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
64a2f4e
fix _generate_output_data crash on empty predictions
deruyter92 Sep 8, 2026
510090e
Allow create_df_from_prediction() to handle an empty predictions list.
deruyter92 Sep 8, 2026
4b45fb9
Treat zero predictions as a distinct warning case in `video_inference()`
deruyter92 Sep 8, 2026
c78f439
Don't write dataframe for empty predictions
deruyter92 Sep 9, 2026
16a2977
update test_videos: pop `"PAFinds"` before comparison
deruyter92 Sep 9, 2026
f534de6
update test_videos: assert no DF is written for empty predictions
deruyter92 Sep 9, 2026
624b1b1
Return empty-detection inference results
C-Achard Sep 9, 2026
c57dd35
Use CTD predictions for video output
C-Achard Sep 9, 2026
4e4dab7
Handle empty prediction outputs gracefully
C-Achard Sep 9, 2026
42098ac
Reuse shared frame key width helper
C-Achard Sep 9, 2026
292d068
Clarify video inference warnings
C-Achard Sep 9, 2026
d5ba290
Go back to ValueError for empty predictions
C-Achard Sep 9, 2026
bff4fea
Remove NaN fallback for empty video predictions
C-Achard Sep 9, 2026
06486de
Fix CTD export guard for empty predictions
C-Achard Sep 9, 2026
e456aeb
Clarify empty CTD prediction handling
C-Achard Sep 9, 2026
a162439
Centralize test fakes
C-Achard Sep 9, 2026
11ff4fd
Add test for async with no images
C-Achard Sep 9, 2026
a210b69
Clarify async inference test comment
C-Achard Sep 9, 2026
c7a8c34
Use shared frame key width helper
C-Achard Sep 9, 2026
8248a10
Merge branch 'dev' into jaap/fix-empty-video-predictions
C-Achard Sep 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 52 additions & 21 deletions deeplabcut/pose_estimation_pytorch/apis/videos.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
TopDownDynamicCropper,
)
from deeplabcut.pose_estimation_pytorch.runners.inference import InferenceConfig
from deeplabcut.pose_estimation_pytorch.runners.shelving import frame_key_width
from deeplabcut.pose_estimation_pytorch.task import Task
from deeplabcut.refine_training_dataset.stitch import stitch_tracklets
from deeplabcut.utils import VideoReader, auxiliaryfunctions
Expand Down Expand Up @@ -228,15 +229,29 @@ def video_inference(
if shelf_writer is not None:
shelf_writer.close()

if shelf_writer is None and len(predictions) != n_frames:
if shelf_writer is None:
tip_url = "https://deeplabcut.github.io/DeepLabCut/docs/recipes/io.html"
header = "#tips-on-video-re-encoding-and-preprocessing"
logging.warning(
f"The video metadata indicates that there {n_frames} in the video, but "
f"only {len(predictions)} were able to be processed. This can happen if "
"the video is corrupted. You can try to fix the issue by re-encoding your "
f"video (tips on how to do that: {tip_url}{header})"
reencoding_tip = (
"This can happen if the video is corrupted. You can try to fix the issue "
f"by re-encoding your video (tips on how to do that: {tip_url}{header})"
)
if len(predictions) == 0:
causes = (
"no animals were detected in any frame, or the video could not be read"
if detector_runner is not None
else "the video could not be read"
)
logging.warning(
f"No predictions were produced for {video.video_path}: {causes}. "
f"Check model performance if that is unexpected. {reencoding_tip}"
)
elif len(predictions) != n_frames:
logging.warning(
f"The video metadata indicates that there are {n_frames} frames in "
f"the video, but only {len(predictions)} were able to be processed. "
f"{reencoding_tip}"
)

return predictions

Expand Down Expand Up @@ -655,16 +670,28 @@ def analyze_videos(
# add poses to the predictions
ctd_predictions.append(dict(bodyparts=pose))

create_df_from_prediction(
predictions=predictions,
multi_animal=multi_animal,
model_cfg=loader.model_cfg,
dlc_scorer=dlc_scorer,
output_path=output_path,
output_prefix=output_prefix + "_ctd",
save_as_csv=save_as_csv,
)
h5_files_created = True # .h5 file was created for CTD tracking
# ``ctd_predictions`` holds one entry per frame in the full
# pickle, so it is empty only when that file reports 0 frames
# (a shelf that wrote nothing, or ``save_as_df=False``). Warn
# and skip rather than raise like the export above does:
# inference success cannot be determined here
if ctd_predictions:
create_df_from_prediction(
predictions=ctd_predictions,
multi_animal=multi_animal,
model_cfg=loader.model_cfg,
dlc_scorer=dlc_scorer,
output_path=output_path,
output_prefix=output_prefix + "_ctd",
save_as_csv=save_as_csv,
)
h5_files_created = True # .h5 file was created for CTD tracking
else:
logging.warning(
f"Skipping CTD dataframe export for {video}: {output_pkl} "
"contains no frames, so no results .h5 file will be "
"written."
)

elif auto_track:
convert_detections2tracklets(
Expand Down Expand Up @@ -720,10 +747,8 @@ def create_df_from_prediction(
output_prefix: str | Path,
save_as_csv: bool = False,
) -> pd.DataFrame:
pred_bodyparts = np.stack([p["bodyparts"][..., :3] for p in predictions])
pred_unique_bodyparts = None
if len(predictions) > 0 and "unique_bodyparts" in predictions[0]:
pred_unique_bodyparts = np.stack([p["unique_bodyparts"] for p in predictions])
if not predictions:
raise ValueError("Cannot create a results DataFrame from an empty predictions list.")

output_h5 = Path(output_path) / f"{output_prefix}.h5"
output_pkl = Path(output_path) / f"{output_prefix}_full.pickle"
Expand All @@ -733,6 +758,11 @@ def create_df_from_prediction(
individuals = model_cfg["metadata"]["individuals"]
n_individuals = len(individuals)

pred_bodyparts = np.stack([p["bodyparts"][..., :3] for p in predictions])
pred_unique_bodyparts = None
if "unique_bodyparts" in predictions[0]:
pred_unique_bodyparts = np.stack([p["unique_bodyparts"] for p in predictions])

print(f"Saving results in {output_h5} and {output_pkl}")
coords = ["x", "y", "likelihood"]
cols = [[dlc_scorer], bodyparts, coords]
Expand All @@ -743,6 +773,7 @@ def create_df_from_prediction(
cols_names.insert(1, "individuals")

results_df_index = pd.MultiIndex.from_product(cols, names=cols_names)

pred_bodyparts = pred_bodyparts[:, :n_individuals]
df = pd.DataFrame(
pred_bodyparts.reshape((len(pred_bodyparts), -1)),
Expand Down Expand Up @@ -887,7 +918,7 @@ def _generate_output_data(
pose_config: dict,
predictions: list[dict[str, np.ndarray]],
) -> dict:
str_width = int(np.ceil(np.log10(len(predictions))))
str_width = frame_key_width(len(predictions))
output = {
"metadata": {
"nms radius": pose_config.get("nmsradius"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ def _process_batch(
if batch:
_process_batch(batch)

if not predictions_2d:
raise RuntimeError(f"No pose predictions were made for video {video_path}. Were no individuals detected?")

output_prefix = f"{Path(video_path).stem}_{dlc_scorer}"
output_h5 = dest_folder / f"{output_prefix}.h5"

Expand Down
3 changes: 3 additions & 0 deletions deeplabcut/pose_estimation_pytorch/runners/inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,9 @@ def _async_inference(
# propagate any exception from the producer immediately
if self._exception is not None:
raise self._exception
# emit images with an available prediction,
# even if no detections were made (which does not produce a queue item)
results.extend(self._extract_results(shelf_writer))

except BaseException as e: # catches KeyboardInterrupt, SystemExit, etc.
# tell producer to quit
Expand Down
21 changes: 18 additions & 3 deletions deeplabcut/pose_estimation_pytorch/runners/shelving.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,23 @@ def __getitem__(self, item: str) -> dict:
return self._db[item]


def frame_key_width(num_frames: int | None, default: int = 5) -> int:
"""Returns the zero-fill width for frame keys in a full-data dict.

Args:
num_frames: The number of frames in the video, if known.
default: The width to use when the number of frames is unknown.

Returns:
The number of leading zeros to pad frame indices with.
"""
if num_frames is None:
return default
if num_frames <= 1:
return 1
return int(np.ceil(np.log10(num_frames)))


class ShelfWriter(ShelfManager):
"""Writes data to a shelf on-the-fly during video analysis.

Expand All @@ -93,9 +110,7 @@ def __init__(self, pose_cfg: dict, filepath: str | Path, num_frames: int | None
self._num_frames = num_frames
self._frame_index = 0

self._str_width = 5
if num_frames is not None:
self._str_width = int(np.ceil(np.log10(num_frames)))
self._str_width = frame_key_width(num_frames)

def add_prediction(
self,
Expand Down
159 changes: 159 additions & 0 deletions tests/pose_estimation_pytorch/apis/test_videos.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
from unittest.mock import Mock

import pytest

import deeplabcut.pose_estimation_pytorch.apis.videos as videos

POSE_CFG = {
"all_joints": [[0], [1], [2]],
"all_joints_names": ["snout", "leftear", "rightear"],
"nmsradius": 5,
"minconfidence": 0.1,
"sigma": 1,
}


def test_generate_output_data_handles_empty_predictions():
output = videos._generate_output_data(POSE_CFG, [])

assert output["metadata"].pop("PAFinds").tolist() == []
assert output == {
"metadata": {
"nms radius": 5,
"minimal confidence": 0.1,
"sigma": 1,
"PAFgraph": None,
"all_joints": [[0], [1], [2]],
"all_joints_names": ["snout", "leftear", "rightear"],
"nframes": 0,
"key_str_width": 1,
}
}
Comment thread
Copilot marked this conversation as resolved.


def test_create_df_from_prediction_rejects_empty_predictions(tmp_path):
with pytest.raises(ValueError, match="empty predictions list"):
videos.create_df_from_prediction(
predictions=[],
dlc_scorer="DLC_test",
multi_animal=False,
model_cfg={
"metadata": {
"bodyparts": ["snout", "leftear", "rightear"],
"unique_bodyparts": [],
"individuals": ["animal_0"],
}
},
output_path=tmp_path,
output_prefix="video",
save_as_csv=False,
)

assert not (tmp_path / "video.h5").exists()


@pytest.fixture
def patch_video_iterator(monkeypatch):
"""Patches ``VideoIterator`` with a stub reporting ``n_frames`` empty frames."""

def _patch(n_frames: int):
class FakeVideoIterator:
def __init__(self, video_path, cropping=None):
self.video_path = video_path
self.fps = 25
self.dimensions = (640, 480)

def get_n_frames(self, robust=False):
return n_frames

def set_context(self, context):
pass

def __iter__(self):
return iter(())

monkeypatch.setattr(videos, "VideoIterator", FakeVideoIterator)
return FakeVideoIterator

return _patch


def _pose_runner(predictions: list) -> Mock:
runner = Mock()
runner.batch_size = 2
runner.inference.return_value = predictions
return runner


def test_video_inference_zero_predictions_warns_about_unreadable_video(patch_video_iterator, caplog):
"""Without a detector, every readable frame yields a prediction, so an empty
result means the video could not be read - not that no animals were found."""
patch_video_iterator(3)

with caplog.at_level("WARNING"):
predictions = videos.video_inference("video.mp4", _pose_runner([]))

assert predictions == []
assert "No predictions were produced for video.mp4" in caplog.text
assert "the video could not be read" in caplog.text
assert "no animals were detected" not in caplog.text
# the re-encoding tip is the actionable advice in this case
assert "re-encoding your video" in caplog.text


def test_video_inference_zero_predictions_with_detector_warns_about_detections(patch_video_iterator, caplog):
patch_video_iterator(3)

with caplog.at_level("WARNING"):
predictions = videos.video_inference("video.mp4", _pose_runner([]), detector_runner=_pose_runner([]))

assert predictions == []
assert "no animals were detected in any frame" in caplog.text
assert "the video could not be read" in caplog.text
assert "re-encoding your video" in caplog.text


def test_video_inference_zero_predictions_warns_when_frame_count_is_zero(patch_video_iterator, caplog):
"""A video whose metadata reports 0 frames must still warn."""
patch_video_iterator(0)

with caplog.at_level("WARNING"):
predictions = videos.video_inference("video.mp4", _pose_runner([]))

assert predictions == []
assert "No predictions were produced for video.mp4" in caplog.text


def test_video_inference_warns_when_some_frames_are_missing(patch_video_iterator, caplog):
patch_video_iterator(3)

with caplog.at_level("WARNING"):
predictions = videos.video_inference("video.mp4", _pose_runner([{}, {}]))

assert len(predictions) == 2
assert "there are 3 frames in the video, but only 2" in caplog.text
assert "re-encoding your video" in caplog.text
assert "No predictions were produced" not in caplog.text


def test_video_inference_does_not_warn_when_all_frames_are_predicted(patch_video_iterator, caplog):
patch_video_iterator(3)

with caplog.at_level("WARNING"):
predictions = videos.video_inference("video.mp4", _pose_runner([{}, {}, {}]))

assert len(predictions) == 3
assert "No predictions were produced" not in caplog.text
assert "were able to be processed" not in caplog.text


def test_video_inference_does_not_warn_when_writing_to_a_shelf(patch_video_iterator, caplog):
"""The returned list is empty by design when a shelf writer is given."""
patch_video_iterator(3)

with caplog.at_level("WARNING"):
predictions = videos.video_inference("video.mp4", _pose_runner([]), shelf_writer=Mock())

assert predictions == []
assert "No predictions were produced" not in caplog.text
assert "were able to be processed" not in caplog.text
Loading
Loading