Skip to content

Updating early stopping for more intuitive use #2051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Adel-Moumen merged 4 commits into from
Aug 11, 2023
Merged

Updating early stopping for more intuitive use #2051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
daa08dd
Updating early stopping for more intuitive use
pplantinga Jun 24, 2023
4b8a2f1
Update docstring for EpochLoopWithStopper to show new usage
pplantinga Jun 24, 2023
3160924
Blacken epoch_loop
pplantinga Jun 24, 2023
171e703
Add checkpointer save/load hooks to EpochCounterWithStopper
pplantinga Aug 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions recipes/LibriSpeech/G2P/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -533,10 +533,7 @@ def on_stage_end(self, stage, stage_loss, epoch):
if self.hparams.enable_metrics:
self._write_reports(epoch, final=False)

if self.epoch_counter.should_stop(
current=epoch, current_metric=per,
):
self.epoch_counter.current = self.epoch_counter.limit
self.epoch_counter.update_metric(per)

if stage == sb.Stage.TEST:
test_stats = {"loss": stage_loss}
Expand Down
88 changes: 71 additions & 17 deletions speechbrain/utils/epoch_loop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from .checkpoints import mark_as_saver
from .checkpoints import mark_as_loader
import logging
import yaml

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -91,19 +92,27 @@ class EpochCounterWithStopper(EpochCounter):
>>> epoch_counter = EpochCounterWithStopper(limit, limit_to_stop, limit_warmup, direction)
>>> for epoch in epoch_counter:
... # Run training...
... # Track a validation metric,
... # Track a validation metric, (insert calculation here)
... current_valid_metric = 0
... # get the current valid metric (get current_valid_metric)
... if epoch_counter.should_stop(current=epoch,
... current_metric=current_valid_metric,):
... epoch_counter.current = epoch_counter.limit # skipping unpromising epochs
... # Update epoch counter so that we stop at the appropriate time
... epoch_counter.update_metric(current_valid_metric)
... print(epoch)
1
2
3
4
5
6
7
8
"""

def __init__(self, limit, limit_to_stop, limit_warmup, direction):
super().__init__(limit)
self.limit_to_stop = limit_to_stop
self.limit_warmup = limit_warmup
self.direction = direction
self.should_stop = False

self.best_limit = 0
self.min_delta = 1e-6
Expand All @@ -113,21 +122,66 @@ def __init__(self, limit, limit_to_stop, limit_warmup, direction):
if self.limit_warmup < 0:
raise ValueError("Stopper 'limit_warmup' must be >= 0")
if self.direction == "min":
self.th, self.sign = float("inf"), 1
self.best_score, self.sign = float("inf"), 1
elif self.direction == "max":
self.th, self.sign = -float("inf"), -1
self.best_score, self.sign = -float("inf"), -1
else:
raise ValueError("Stopper 'direction' must be 'min' or 'max'")

def should_stop(self, current, current_metric):
"""Returns True is training should stop (based on the performance
metrics)."""
should_stop = False
if current > self.limit_warmup:
def __next__(self):
"""Stop iteration if we've reached the condition."""
if self.should_stop:
raise StopIteration
else:
return super().__next__()

def update_metric(self, current_metric):
"""Update the state to reflect most recent value of the relevant metric.

NOTE: Should be called only once per validation loop.

Arguments
---------
current_metric : float
The metric used to make a stopping decision.
"""
if self.current > self.limit_warmup:
if self.sign * current_metric < self.sign * (
(1 - self.min_delta) * self.th
(1 - self.min_delta) * self.best_score
):
self.best_limit = current
self.th = current_metric
should_stop = (current - self.best_limit) >= self.limit_to_stop
return should_stop
self.best_limit = self.current
self.best_score = current_metric

epochs_without_improvement = self.current - self.best_limit
self.should_stop = epochs_without_improvement >= self.limit_to_stop
if self.should_stop:
logger.info(
f"{epochs_without_improvement} epochs without improvement.\n"
f"Patience of {self.limit_to_stop} is exhausted, stopping."
)

@mark_as_saver
def _save(self, path):
with open(path, "w") as fo:
yaml.dump(
{
"current_epoch": self.current,
"best_epoch": self.best_limit,
"best_score": self.best_score,
"should_stop": self.should_stop,
},
fo,
)

@mark_as_loader
def _recover(self, path, end_of_epoch=True, device=None):
del device # Not used.
with open(path) as fi:
saved_dict = yaml.safe_load(fi)
if end_of_epoch:
self.current = saved_dict["current_epoch"]
else:
self.current = saved_dict["current_epoch"] - 1
self.best_limit = saved_dict["best_epoch"]
self.best_score = saved_dict["best_score"]
self.should_stop = saved_dict["should_stop"]