diff --git a/deeplabcut/pose_estimation_3d/plotting3D.py b/deeplabcut/pose_estimation_3d/plotting3D.py index d39967a866..593e712035 100644 --- a/deeplabcut/pose_estimation_3d/plotting3D.py +++ b/deeplabcut/pose_estimation_3d/plotting3D.py @@ -215,15 +215,16 @@ def create_labeled_video_3d( bodyparts2plot = list(np.unique([val for sublist in bodyparts2connect for val in sublist])) # Format data + # copy=True: under pandas 3 CoW, to_numpy() may be read-only. mask2d = df_cam1.columns.get_level_values("bodyparts").isin(bodyparts2plot) - xy1 = df_cam1.iloc[: len(df_3d)].loc[:, mask2d].to_numpy().reshape((len(df_3d), -1, 3)) + xy1 = df_cam1.iloc[: len(df_3d)].loc[:, mask2d].to_numpy(copy=True).reshape((len(df_3d), -1, 3)) visible1 = xy1[..., 2] >= pcutoff xy1[~visible1] = np.nan - xy2 = df_cam2.iloc[: len(df_3d)].loc[:, mask2d].to_numpy().reshape((len(df_3d), -1, 3)) + xy2 = df_cam2.iloc[: len(df_3d)].loc[:, mask2d].to_numpy(copy=True).reshape((len(df_3d), -1, 3)) visible2 = xy2[..., 2] >= pcutoff xy2[~visible2] = np.nan mask = df_3d.columns.get_level_values("bodyparts").isin(bodyparts2plot) - xyz = df_3d.loc[:, mask].to_numpy().reshape((len(df_3d), -1, 3)) + xyz = df_3d.loc[:, mask].to_numpy(copy=True).reshape((len(df_3d), -1, 3)) xyz[~(visible1 & visible2)] = np.nan bpts = df_3d.columns.get_level_values("bodyparts")[mask][::3] diff --git a/deeplabcut/pose_estimation_3d/triangulation.py b/deeplabcut/pose_estimation_3d/triangulation.py index c213bfd312..3b945166c6 100644 --- a/deeplabcut/pose_estimation_3d/triangulation.py +++ b/deeplabcut/pose_estimation_3d/triangulation.py @@ -326,8 +326,9 @@ def triangulate( num_frames = dataFrame_camera1_undistort.shape[0] ### Assign nan to [X,Y] of low likelihood predictions ### # Convert the data to a np array to easily mask out the low likelihood predictions - data_cam1_tmp = dataFrame_camera1_undistort.to_numpy().reshape((num_frames, -1, 3)) - data_cam2_tmp = dataFrame_camera2_undistort.to_numpy().reshape((num_frames, -1, 3)) + # copy=True: under pandas 3 CoW, to_numpy() may be read-only. + data_cam1_tmp = dataFrame_camera1_undistort.to_numpy(copy=True).reshape((num_frames, -1, 3)) + data_cam2_tmp = dataFrame_camera2_undistort.to_numpy(copy=True).reshape((num_frames, -1, 3)) # Assign [X,Y] = nan to low likelihood predictions data_cam1_tmp[data_cam1_tmp[..., 2] < pcutoff, :2] = np.nan data_cam2_tmp[data_cam2_tmp[..., 2] < pcutoff, :2] = np.nan diff --git a/deeplabcut/pose_estimation_pytorch/data/ctd.py b/deeplabcut/pose_estimation_pytorch/data/ctd.py index 9cce0122ca..23f086befb 100644 --- a/deeplabcut/pose_estimation_pytorch/data/ctd.py +++ b/deeplabcut/pose_estimation_pytorch/data/ctd.py @@ -208,8 +208,9 @@ def load_conditions_h5( """ def _parse_row(df_row) -> np.ndarray: - # Row to numpy and reshape - pose = df_row.to_numpy().reshape((num_conditions, num_bodyparts, 3)) + # Row to numpy and reshape. copy=True: under pandas 3 CoW, + # to_numpy() may return a read-only view. + pose = df_row.to_numpy(copy=True).reshape((num_conditions, num_bodyparts, 3)) # Remove missing data missing_keypoints = np.any(np.isnan(pose) | (pose < 0), axis=2) diff --git a/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py b/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py index 5cd893f7e2..0dc290cc02 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py +++ b/deeplabcut/pose_estimation_tensorflow/core/evaluate_multianimal.py @@ -328,7 +328,7 @@ def evaluate_multianimal_full( ) temp["sample"] = 0 - peaks_gt = temp.loc[:, ["sample", "y", "x", "bodyparts"]].to_numpy() + peaks_gt = temp.loc[:, ["sample", "y", "x", "bodyparts"]].to_numpy(copy=True) peaks_gt[:, 1:3] = (peaks_gt[:, 1:3] - stride // 2) / stride pred = predictma.predict_batched_peaks_and_costs( diff --git a/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py b/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py index e9caaa495c..312188b14b 100644 --- a/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py +++ b/deeplabcut/pose_estimation_tensorflow/datasets/pose_multianimal_imgaug.py @@ -131,7 +131,9 @@ def _load_pseudo_data_from_h5(self, cfg, threshold=0.5, mask_kpts_below_thresh=F item = DataItem() data = df.loc[imagename] # 3 for likelihood - kpts = data.to_numpy().reshape(-1, 3) + # copy=True: `kpts` is masked in place below (mask_kpts_below_thresh); + # under pandas 3 CoW a single-row float Series returns a read-only view. + kpts = data.to_numpy(copy=True).reshape(-1, 3) item.num_joints = kpts.shape[0] joint_ids = np.arange(item.num_joints)[..., np.newaxis] frame_name = "frame_" + str(int(imagename.split("frame")[1])) + ".png" diff --git a/deeplabcut/post_processing/filtering.py b/deeplabcut/post_processing/filtering.py index 2f013aa833..16d77cf51d 100644 --- a/deeplabcut/post_processing/filtering.py +++ b/deeplabcut/post_processing/filtering.py @@ -248,8 +248,10 @@ def filterpredictions( elif filtertype == "spline": data = df.copy() mask_data = data.columns.get_level_values("coords").isin(("x", "y")) - xy = data.loc[:, mask_data].values - prob = data.loc[:, ~mask_data].values + # copy=True: under pandas 3 CoW, homogeneous float selections + # via .values/.to_numpy() are read-only views. + xy = data.loc[:, mask_data].to_numpy(copy=True) + prob = data.loc[:, ~mask_data].to_numpy(copy=True) missing = np.isnan(xy) xy_filled = columnwise_spline_interp(xy, windowlength) filled = ~np.isnan(xy_filled) diff --git a/deeplabcut/refine_training_dataset/tracklets.py b/deeplabcut/refine_training_dataset/tracklets.py index 4e07e02d0f..9297268559 100644 --- a/deeplabcut/refine_training_dataset/tracklets.py +++ b/deeplabcut/refine_training_dataset/tracklets.py @@ -221,8 +221,9 @@ def load_tracklets_from_hdf(self, filename): self.filename = filename df = pd.read_hdf(filename) - # Fill existing gaps - data = df.to_numpy() + # Fill existing gaps. copy=True: under pandas 3 CoW, to_numpy() may + # return a read-only view that cannot be mutated in place. + data = df.to_numpy(copy=True) mask = ~df.columns.get_level_values(level="coords").str.contains("likelihood") xy = data[:, mask] prob = data[:, ~mask] @@ -245,7 +246,11 @@ def load_tracklets_from_hdf(self, filename): self.bodyparts = idx.get_level_values("bodyparts") self.nframes = len(df) self.times = np.arange(self.nframes) - self.data = df.values.reshape((self.nframes, -1, 3)).swapaxes(0, 1) + # copy=True: `self.xy`/`self.prob` are views into `self.data` and are + # mutated in place (e.g. by `swap_tracklets`), so the backing array must + # stay writable. Under pandas 3 CoW, `.values` on a homogeneous float + # DataFrame returns a read-only view. + self.data = df.to_numpy(copy=True).reshape((self.nframes, -1, 3)).swapaxes(0, 1) self.xy = self.data[:, :, :2] self.prob = self.data[:, :, 2] individuals = idx.get_level_values("individuals") diff --git a/deeplabcut/utils/make_labeled_video.py b/deeplabcut/utils/make_labeled_video.py index a50a483f31..e6acfbeb98 100644 --- a/deeplabcut/utils/make_labeled_video.py +++ b/deeplabcut/utils/make_labeled_video.py @@ -1017,7 +1017,9 @@ def create_video_with_keypoints_only( ny = int(np.nanmax(df.xs("y", axis=1, level="coords"))) n_frames = df.shape[0] - xyp = df.values.reshape((n_frames, -1, 3)) + # copy=True: `coords` is a view into `xyp` and is masked in place below; + # under pandas 3 CoW `.values` would return a read-only view. + xyp = df.to_numpy(copy=True).reshape((n_frames, -1, 3)) if color_by == "bodypart": map_ = bodyparts.map(dict(zip(bodypart_names, range(n_bodyparts), strict=False))) diff --git a/deeplabcut/utils/pandas_future_mode.py b/deeplabcut/utils/pandas_future_mode.py index cfb9d952e7..fb8adf40aa 100644 --- a/deeplabcut/utils/pandas_future_mode.py +++ b/deeplabcut/utils/pandas_future_mode.py @@ -23,7 +23,7 @@ def configure_pandas_future_if_enabled() -> None: raise RuntimeError(f"pandas future mode requires pandas 2.3.x, got {pd.__version__}") pd.options.future.infer_string = True - pd.options.mode.copy_on_write = "warn" + pd.options.mode.copy_on_write = True print( f"pandas future mode enabled: pandas={pd.__version__}, " diff --git a/tests/utils/test_pandas_cow.py b/tests/utils/test_pandas_cow.py new file mode 100644 index 0000000000..84a5fee4d9 --- /dev/null +++ b/tests/utils/test_pandas_cow.py @@ -0,0 +1,108 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# (c) A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# + +"""Verify that ``configure_pandas_future_if_enabled`` (copy_on_write=True) +surfaces mutation patterns that should be guarded by ``copy=True``. +""" + +import numpy as np +import pandas as pd +import pytest + +from deeplabcut.utils.pandas_future_mode import ( + configure_pandas_future_if_enabled, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _configure_cow_true(monkeypatch) -> None: + """Configure pandas CoW via the DLC future-mode machinery.""" + monkeypatch.setattr(pd.options.future, "infer_string", pd.options.future.infer_string) + monkeypatch.setattr(pd.options.mode, "copy_on_write", pd.options.mode.copy_on_write) + monkeypatch.setenv("DLC_PANDAS_FUTURE", "1") + configure_pandas_future_if_enabled() + + +def _make_dlc_df(n_frames: int = 5) -> pd.DataFrame: + """Build a small DLC-style MultiIndex DataFrame (scorer x bodypart x coord).""" + bodyparts = ["snout", "leftear", "rightear"] + coords = ["x", "y", "likelihood"] + arrays = {} + for bp in bodyparts: + for c in coords: + arrays[("scorer1", bp, c)] = np.random.randn(n_frames) + df = pd.DataFrame(arrays) + df.columns = pd.MultiIndex.from_tuples( + df.columns, + names=["scorer", "bodyparts", "coords"], + ) + return df + + +# --------------------------------------------------------------------------- +# Sanity: the machinery actually engages. +# --------------------------------------------------------------------------- + + +def test_future_mode_sets_cow_true(monkeypatch): + """``configure_pandas_future_if_enabled()`` sets copy_on_write=True.""" + _configure_cow_true(monkeypatch) + assert pd.options.mode.copy_on_write is True, f"Expected copy_on_write=True, got {pd.options.mode.copy_on_write!r}" + assert pd.options.future.infer_string is True + + +# --------------------------------------------------------------------------- +# The PR's patterns: extracting a numpy array, then mutating it in place. +# Without copy=True each would blow up at runtime under pandas 3. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "extract", + [ + pytest.param( + lambda df: df.to_numpy().reshape((len(df), -1, 3)), + id="to_numpy_reshape", + ), + pytest.param( + lambda df: df.values.reshape((len(df), -1, 3)), + id="values_reshape", + ), + pytest.param( + lambda df: df.loc[:, df.columns.get_level_values("coords").isin(("x", "y"))].to_numpy(), + id="column_subset", + ), + pytest.param( + lambda df: df.loc[df.index[0]].to_numpy().reshape(-1, 3), + id="single_row", + ), + ], +) +def test_future_mode_makes_numpy_arrays_read_only(extract, monkeypatch): + """``configure_pandas_future_if_enabled()`` with copy_on_write=True + returns read-only numpy arrays that cannot be mutated in place. + + These are the patterns that this PR guards with ``copy=True``. + Without the guard, each would raise ``ValueError`` (read-only array) + under pandas 3. + """ + _configure_cow_true(monkeypatch) + df = _make_dlc_df() + + arr = extract(df) + + assert not arr.flags.writeable, "array unexpectedly writeable — CoW may not be active" + + with pytest.raises(ValueError, match="read-only"): + arr.flat[0] = 999.0