kotoba-whisper, ReazonSpeech, Nue-ASRの文字起こし性能を比較してみた

name="python環境の名前"
conda create -n $name -y python=3.10 poetry ffmpeg
poetry install # pyproject.tomlを事前に用意してください

[tool.poetry.dependencies]
python = "3.10"
torch = { url = "https://download.pytorch.org/whl/cu118/torch-2.3.0%2Bcu118-cp310-cp310-linux_x86_64.whl#sha256=ceb81d79fc7b017f51b1613f83b878efb8974cc946631fb9cd577bbaa2873293" }
torchvision = { url = "https://download.pytorch.org/whl/cu118/torchvision-0.18.0%2Bcu118-cp310-cp310-linux_x86_64.whl#sha256=86b2970a75316a90e00ac66863b5414269ffd3e8c58683f1f40128da989285b6" }
torchaudio = { url = "https://download.pytorch.org/whl/cu118/torchaudio-2.3.0%2Bcu118-cp310-cp310-linux_x86_64.whl#sha256=f21f08ebf87510a09f4ee3698997eeec4b8f2520e95218defbcd88e5c28d5112" }
pandas = "^2.2"
librosa = "^0.10.2.post1"
jiwer = "^3.0.4"
mecab-python3 = "^1.0.9"
ruff = "^0.4.7"
num2words = "^0.5"
neologdn = "^0.5"

pip install --upgrade transformers accelerate

git clone https://github.com/reazon-research/ReazonSpeech
pip install ReazonSpeech/pkg/espnet-asr

git clone https://github.com/reazon-research/ReazonSpeech
pip install Cython
pip install ReazonSpeech/pkg/nemo-asr

pip install git+https://github.com/rinnakk/nue-asr.git

import time
from tqdm import tqdm
wav_dir = Path('/path/to/tedxjp')

def calc_duration(file_id):
    wav_path = wav_dir / f'{file_id}.wav'
    audio, sr = sf.read(wav_path)
    return len(audio) / sr

def main():
    #######################################################
    # モデルごとに記述を変える部分
    from reazonspeech.espnet.asr import load_model, transcribe, audio_from_path
		model = load_model()
    #######################################################
		
    df = pd.read_csv(data_dir / 'text', sep=' ', header=None, names=['id', 'text'])
    df['duration'] = df['id'].apply(calc_duration)
    id2text = df[['id', 'text']].set_index('id').to_dict()['text']
    paths2audio_files = [str(wav_dir / f'{file_id}.wav') for file_id in id2text.keys()]
    
    results = []
    tic = time.perf_counter()
    for path in tqdm(paths2audio_files):
	    #######################################################
	    # モデルごとに記述を変える部分
        audio = audio_from_path(path)
        ret = transcribe(model, audio)
        results.append({
	          'gt_text': id2text[Path(path).stem],
	          'pred_text': ret.text,
          },
	      )
	    #######################################################
    toc = time.perf_counter() - tic
    pd.DataFrame(results).to_csv('results_reazon-nemo.csv', index=False)
    print(f'Elapsed time: {toc:.2f} seconds')
    print(f'RTF: {toc / df["duration"].sum():.3f}')

model_id = "kotoba-tech/kotoba-whisper-v1.0"
torch_dtype = torch.bfloat16 if device == 'cuda' else torch.float32
model_kwargs = {"attn_implementation": "sdpa"} if device == 'cuda' else {}
model = pipeline(
    "automatic-speech-recognition",
    model=model_id,
    torch_dtype=torch_dtype,
    device=device,
    model_kwargs=model_kwargs
)
generate_kwargs = {"language": "japanese", "task": "transcribe"}

ret = model(path, generate_kwargs=generate_kwargs)
results.append({
    'gt_text': id2text[Path(path).stem],
    'pred_text': ret['text'],
  },
)

model = nue_asr.load_model("rinna/nue-asr")
tokenizer = nue_asr.load_tokenizer("rinna/nue-asr")

ret = nue_asr.transcribe(model, tokenizer, path)
results.append({
    'gt_text': id2text[Path(path).stem],
    'pred_text': ret.text,
   },
)

import pandas as pd
import jiwer
from argparse import ArgumentParser
import MeCab
from dataclasses import dataclass
from num2words import num2words
import re
import neologdn

symbols = [
    "、",
    "。",
    " ",
    "!"
	... # 省略
]

_SPECIALS = {ord(c): "" for c in symbols}

def normalize(utt_txt):
    """Normalize text.
    Use for Japanese text.
    Args:
        utt_txt: String of Japanese text.
    Returns:
        utt_txt: Normalized
    """
    # trim non-phonatory symbols in the text
    utt_txt = utt_txt.translate(_SPECIALS)
    
    # convert UTF-16 latin chars to ASCII
    utt_txt = neologdn.normalize(utt_txt)
    
    # replace all the numbers
    numbers = re.findall(r"\d+\.?\d*", utt_txt)
    transcribed_numbers = [num2words(item, lang="ja") for item in numbers]
    for nr in range(len(numbers)):
        old_nr = numbers[nr]
        new_nr = transcribed_numbers[nr]
        utt_txt = utt_txt.replace(old_nr, new_nr, 1)
    return utt_txt
    
@dataclass
class TotalErrors:
    deletions: int = 0
    insertions: int = 0
    substitutions: int = 0
    distance: int = 0
    length: int = 0

    @property
    def error_rate(self):
        return self.distance / self.length
        
def main():
    parser = ArgumentParser()
    parser.add_argument("input", type=str, help="Path to the result csv file")
    args = parser.parse_args()
    df = pd.read_csv(args.input)

    mecab = MeCab.Tagger("-Owakati")
    total_word_errors = TotalErrors()
    total_char_errors = TotalErrors()

    for row in df.itertuples():
        gt_text = normalize(row.gt_text)
        if pd.isna(row.pred_text):
            asr_text = ""
        else:
            asr_text = normalize(row.pred_text)

        gt_words = mecab.parse(gt_text)
        asr_words = mecab.parse(asr_text)
        word_errors = jiwer.process_words(gt_words, asr_words)
        total_word_errors.deletions += word_errors.deletions
        total_word_errors.insertions += word_errors.insertions
        total_word_errors.substitutions += word_errors.substitutions
        distance = (
            word_errors.deletions + word_errors.insertions + word_errors.substitutions
        )
        total_word_errors.distance += distance
        total_word_errors.length += len(gt_words.split())

        char_errors = jiwer.process_characters(gt_text, asr_text)
        total_char_errors.deletions += char_errors.deletions
        total_char_errors.insertions += char_errors.insertions
        total_char_errors.substitutions += char_errors.substitutions
        distance = (
            char_errors.deletions + char_errors.insertions + char_errors.substitutions
        )
        total_char_errors.distance += distance
        total_char_errors.length += len(gt_text)

    print("----------- Word Error Rate -----------")
    print(f"WER: {total_word_errors.error_rate:.3f}")
    print("deletions:", total_word_errors.deletions)
    print("insertions:", total_word_errors.insertions)
    print("substitutions:", total_word_errors.substitutions)

    print("---------- Character Error Rate ----------")
    print(f"CER: {total_char_errors.error_rate:.3f}")
    print("deletions:", total_char_errors.deletions)
    print("insertions:", total_char_errors.insertions)
    print("substitutions:", total_char_errors.substitutions)