Skip to content
7 changes: 4 additions & 3 deletions deeplabcut/pose_estimation_3d/plotting3D.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,15 +215,16 @@ def create_labeled_video_3d(
bodyparts2plot = list(np.unique([val for sublist in bodyparts2connect for val in sublist]))

# Format data
# copy=True: under pandas 3 CoW, to_numpy() may be read-only.
mask2d = df_cam1.columns.get_level_values("bodyparts").isin(bodyparts2plot)
xy1 = df_cam1.iloc[: len(df_3d)].loc[:, mask2d].to_numpy().reshape((len(df_3d), -1, 3))
xy1 = df_cam1.iloc[: len(df_3d)].loc[:, mask2d].to_numpy(copy=True).reshape((len(df_3d), -1, 3))
visible1 = xy1[..., 2] >= pcutoff
xy1[~visible1] = np.nan
xy2 = df_cam2.iloc[: len(df_3d)].loc[:, mask2d].to_numpy().reshape((len(df_3d), -1, 3))
xy2 = df_cam2.iloc[: len(df_3d)].loc[:, mask2d].to_numpy(copy=True).reshape((len(df_3d), -1, 3))
visible2 = xy2[..., 2] >= pcutoff
xy2[~visible2] = np.nan
mask = df_3d.columns.get_level_values("bodyparts").isin(bodyparts2plot)
xyz = df_3d.loc[:, mask].to_numpy().reshape((len(df_3d), -1, 3))
xyz = df_3d.loc[:, mask].to_numpy(copy=True).reshape((len(df_3d), -1, 3))
xyz[~(visible1 & visible2)] = np.nan

bpts = df_3d.columns.get_level_values("bodyparts")[mask][::3]
Expand Down
5 changes: 3 additions & 2 deletions deeplabcut/pose_estimation_3d/triangulation.py
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,9 @@ def triangulate(
num_frames = dataFrame_camera1_undistort.shape[0]
### Assign nan to [X,Y] of low likelihood predictions ###
# Convert the data to a np array to easily mask out the low likelihood predictions
data_cam1_tmp = dataFrame_camera1_undistort.to_numpy().reshape((num_frames, -1, 3))
data_cam2_tmp = dataFrame_camera2_undistort.to_numpy().reshape((num_frames, -1, 3))
# copy=True: under pandas 3 CoW, to_numpy() may be read-only.
data_cam1_tmp = dataFrame_camera1_undistort.to_numpy(copy=True).reshape((num_frames, -1, 3))
data_cam2_tmp = dataFrame_camera2_undistort.to_numpy(copy=True).reshape((num_frames, -1, 3))
# Assign [X,Y] = nan to low likelihood predictions
data_cam1_tmp[data_cam1_tmp[..., 2] < pcutoff, :2] = np.nan
data_cam2_tmp[data_cam2_tmp[..., 2] < pcutoff, :2] = np.nan
Expand Down
5 changes: 3 additions & 2 deletions deeplabcut/pose_estimation_pytorch/data/ctd.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,9 @@ def load_conditions_h5(
"""

def _parse_row(df_row) -> np.ndarray:
# Row to numpy and reshape
pose = df_row.to_numpy().reshape((num_conditions, num_bodyparts, 3))
# Row to numpy and reshape. copy=True: under pandas 3 CoW,
# to_numpy() may return a read-only view.
pose = df_row.to_numpy(copy=True).reshape((num_conditions, num_bodyparts, 3))

# Remove missing data
missing_keypoints = np.any(np.isnan(pose) | (pose < 0), axis=2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ def evaluate_multianimal_full(
)

temp["sample"] = 0
peaks_gt = temp.loc[:, ["sample", "y", "x", "bodyparts"]].to_numpy()
peaks_gt = temp.loc[:, ["sample", "y", "x", "bodyparts"]].to_numpy(copy=True)
peaks_gt[:, 1:3] = (peaks_gt[:, 1:3] - stride // 2) / stride

pred = predictma.predict_batched_peaks_and_costs(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ def _load_pseudo_data_from_h5(self, cfg, threshold=0.5, mask_kpts_below_thresh=F
item = DataItem()
data = df.loc[imagename]
# 3 for likelihood
kpts = data.to_numpy().reshape(-1, 3)
# copy=True: `kpts` is masked in place below (mask_kpts_below_thresh);
# under pandas 3 CoW a single-row float Series returns a read-only view.
kpts = data.to_numpy(copy=True).reshape(-1, 3)
item.num_joints = kpts.shape[0]
joint_ids = np.arange(item.num_joints)[..., np.newaxis]
frame_name = "frame_" + str(int(imagename.split("frame")[1])) + ".png"
Expand Down
6 changes: 4 additions & 2 deletions deeplabcut/post_processing/filtering.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,8 +248,10 @@ def filterpredictions(
elif filtertype == "spline":
data = df.copy()
mask_data = data.columns.get_level_values("coords").isin(("x", "y"))
xy = data.loc[:, mask_data].values
prob = data.loc[:, ~mask_data].values
# copy=True: under pandas 3 CoW, homogeneous float selections
# via .values/.to_numpy() are read-only views.
xy = data.loc[:, mask_data].to_numpy(copy=True)
prob = data.loc[:, ~mask_data].to_numpy(copy=True)
missing = np.isnan(xy)
xy_filled = columnwise_spline_interp(xy, windowlength)
filled = ~np.isnan(xy_filled)
Expand Down
11 changes: 8 additions & 3 deletions deeplabcut/refine_training_dataset/tracklets.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,9 @@ def load_tracklets_from_hdf(self, filename):
self.filename = filename
df = pd.read_hdf(filename)

# Fill existing gaps
data = df.to_numpy()
# Fill existing gaps. copy=True: under pandas 3 CoW, to_numpy() may
# return a read-only view that cannot be mutated in place.
data = df.to_numpy(copy=True)
mask = ~df.columns.get_level_values(level="coords").str.contains("likelihood")
xy = data[:, mask]
prob = data[:, ~mask]
Expand All @@ -245,7 +246,11 @@ def load_tracklets_from_hdf(self, filename):
self.bodyparts = idx.get_level_values("bodyparts")
self.nframes = len(df)
self.times = np.arange(self.nframes)
self.data = df.values.reshape((self.nframes, -1, 3)).swapaxes(0, 1)
# copy=True: `self.xy`/`self.prob` are views into `self.data` and are
# mutated in place (e.g. by `swap_tracklets`), so the backing array must
# stay writable. Under pandas 3 CoW, `.values` on a homogeneous float
# DataFrame returns a read-only view.
self.data = df.to_numpy(copy=True).reshape((self.nframes, -1, 3)).swapaxes(0, 1)
self.xy = self.data[:, :, :2]
self.prob = self.data[:, :, 2]
individuals = idx.get_level_values("individuals")
Expand Down
4 changes: 3 additions & 1 deletion deeplabcut/utils/make_labeled_video.py
Original file line number Diff line number Diff line change
Expand Up @@ -1017,7 +1017,9 @@ def create_video_with_keypoints_only(
ny = int(np.nanmax(df.xs("y", axis=1, level="coords")))

n_frames = df.shape[0]
xyp = df.values.reshape((n_frames, -1, 3))
# copy=True: `coords` is a view into `xyp` and is masked in place below;
# under pandas 3 CoW `.values` would return a read-only view.
xyp = df.to_numpy(copy=True).reshape((n_frames, -1, 3))

if color_by == "bodypart":
map_ = bodyparts.map(dict(zip(bodypart_names, range(n_bodyparts), strict=False)))
Expand Down
2 changes: 1 addition & 1 deletion deeplabcut/utils/pandas_future_mode.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def configure_pandas_future_if_enabled() -> None:
raise RuntimeError(f"pandas future mode requires pandas 2.3.x, got {pd.__version__}")

pd.options.future.infer_string = True
pd.options.mode.copy_on_write = "warn"
pd.options.mode.copy_on_write = True

print(
f"pandas future mode enabled: pandas={pd.__version__}, "
Expand Down
108 changes: 108 additions & 0 deletions tests/utils/test_pandas_cow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#
# DeepLabCut Toolbox (deeplabcut.org)
# (c) A. & M.W. Mathis Labs
# https://github.com/DeepLabCut/DeepLabCut
#
# Please see AUTHORS for contributors.
# https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS
#
# Licensed under GNU Lesser General Public License v3.0
#

"""Verify that ``configure_pandas_future_if_enabled`` (copy_on_write=True)
surfaces mutation patterns that should be guarded by ``copy=True``.
"""

import numpy as np
import pandas as pd
import pytest

from deeplabcut.utils.pandas_future_mode import (
configure_pandas_future_if_enabled,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _configure_cow_true(monkeypatch) -> None:
"""Configure pandas CoW via the DLC future-mode machinery."""
monkeypatch.setattr(pd.options.future, "infer_string", pd.options.future.infer_string)
monkeypatch.setattr(pd.options.mode, "copy_on_write", pd.options.mode.copy_on_write)
monkeypatch.setenv("DLC_PANDAS_FUTURE", "1")
configure_pandas_future_if_enabled()


def _make_dlc_df(n_frames: int = 5) -> pd.DataFrame:
"""Build a small DLC-style MultiIndex DataFrame (scorer x bodypart x coord)."""
bodyparts = ["snout", "leftear", "rightear"]
coords = ["x", "y", "likelihood"]
arrays = {}
for bp in bodyparts:
for c in coords:
arrays[("scorer1", bp, c)] = np.random.randn(n_frames)
df = pd.DataFrame(arrays)
df.columns = pd.MultiIndex.from_tuples(
df.columns,
names=["scorer", "bodyparts", "coords"],
)
return df


# ---------------------------------------------------------------------------
# Sanity: the machinery actually engages.
# ---------------------------------------------------------------------------


def test_future_mode_sets_cow_true(monkeypatch):
"""``configure_pandas_future_if_enabled()`` sets copy_on_write=True."""
_configure_cow_true(monkeypatch)
assert pd.options.mode.copy_on_write is True, f"Expected copy_on_write=True, got {pd.options.mode.copy_on_write!r}"
assert pd.options.future.infer_string is True


# ---------------------------------------------------------------------------
# The PR's patterns: extracting a numpy array, then mutating it in place.
# Without copy=True each would blow up at runtime under pandas 3.
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
"extract",
[
pytest.param(
lambda df: df.to_numpy().reshape((len(df), -1, 3)),
id="to_numpy_reshape",
),
pytest.param(
lambda df: df.values.reshape((len(df), -1, 3)),
id="values_reshape",
),
pytest.param(
lambda df: df.loc[:, df.columns.get_level_values("coords").isin(("x", "y"))].to_numpy(),
id="column_subset",
),
pytest.param(
lambda df: df.loc[df.index[0]].to_numpy().reshape(-1, 3),
id="single_row",
),
],
)
def test_future_mode_makes_numpy_arrays_read_only(extract, monkeypatch):
"""``configure_pandas_future_if_enabled()`` with copy_on_write=True
returns read-only numpy arrays that cannot be mutated in place.

These are the patterns that this PR guards with ``copy=True``.
Without the guard, each would raise ``ValueError`` (read-only array)
under pandas 3.
"""
_configure_cow_true(monkeypatch)
df = _make_dlc_df()

arr = extract(df)

assert not arr.flags.writeable, "array unexpectedly writeable — CoW may not be active"

with pytest.raises(ValueError, match="read-only"):
arr.flat[0] = 999.0
Loading