Skip to content

Fix to support CV 18 #2647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asumagic merged 5 commits into from
Sep 25, 2024
Merged

Fix to support CV 18 #2647

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e8b77cc
Fix to support CV 18
Aug 22, 2024
548da22
Merge branch 'develop' into fix/support-common-voice-18
cvl01 Aug 29, 2024
2d07fdc
Fix: Split header line according to tsv in cv prep
asumagic Sep 25, 2024
c5856ab
Formatting
asumagic Sep 25, 2024
bd4e5c3
Document header_map parameter
asumagic Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions recipes/CommonVoice/common_voice_prepare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,9 @@ class CVRow:
words: str


def process_line(line, convert_to_wav, data_folder, language, accented_letters):
def process_line(
line, convert_to_wav, data_folder, language, accented_letters, header_map
):
"""Process a line of CommonVoice tsv file.

Arguments
Expand All @@ -211,22 +213,28 @@ def process_line(line, convert_to_wav, data_folder, language, accented_letters):
accented_letters : bool
Defines if accented letters will be kept as individual letters or
transformed to the closest non-accented letters.
header_map : Dict[str, int]
Map from column name to column indices

Returns
-------
CVRow
A dataclass containing the information about the line.
"""

columns = line.strip().split("\t")
spk_id = columns[header_map["client_id"]]
audio_path_filename = columns[header_map["path"]]
words = columns[header_map["sentence"]]

# Path is at indice 1 in Common Voice tsv files. And .mp3 files
# are located in datasets/lang/clips/
audio_path = data_folder + "/clips/" + line.split("\t")[1]
audio_path = data_folder + "/clips/" + audio_path_filename

if convert_to_wav:
audio_path = convert_mp3_to_wav(audio_path)

file_name = audio_path.split(".")[-2].split("/")[-1]
spk_id = line.split("\t")[0]
snt_id = file_name

# Reading the signal (to retrieve duration in seconds)
Expand All @@ -240,7 +248,6 @@ def process_line(line, convert_to_wav, data_folder, language, accented_letters):
duration = info.num_frames / info.sample_rate

# Getting transcript
words = line.split("\t")[2]

# Unicode Normalization
words = unicode_normalisation(words)
Expand Down Expand Up @@ -309,8 +316,15 @@ def create_csv(
raise FileNotFoundError(msg)

# We load and skip the header
loaded_csv = open(orig_tsv_file, "r").readlines()[1:]
nb_samples = len(loaded_csv)
csv_lines = open(orig_tsv_file, "r").readlines()
header_line = csv_lines[0]
csv_data_lines = csv_lines[1:]
nb_samples = len(csv_data_lines)

header_map = {
column_name: index
for index, column_name in enumerate(header_line.split("\t"))
}

msg = "Preparing CSV files for %s samples ..." % (str(nb_samples))
logger.info(msg)
Expand All @@ -328,6 +342,7 @@ def create_csv(
data_folder=data_folder,
language=language,
accented_letters=accented_letters,
header_map=header_map,
)

# Stream into a .tmp file, and rename it to the real path at the end.
Expand All @@ -340,7 +355,7 @@ def create_csv(

csv_writer.writerow(["ID", "duration", "wav", "spk_id", "wrd"])

for row in parallel_map(line_processor, loaded_csv):
for row in parallel_map(line_processor, csv_data_lines):
if row is None:
continue

Expand All @@ -360,7 +375,7 @@ def create_csv(
# Final prints
msg = "%s successfully created!" % (csv_file)
logger.info(msg)
msg = "Number of samples: %s " % (str(len(loaded_csv)))
msg = "Number of samples: %s " % (str(len(csv_data_lines)))
logger.info(msg)
msg = "Total duration: %s Hours" % (str(round(total_duration / 3600, 2)))
logger.info(msg)
Expand Down