OpenAI Whisper 모델 파인튜닝 완벽 가이드: 한국어 음성인식 정확도 높이기

pip install torch torchvision torchaudio
pip install transformers datasets accelerate evaluate jiwer
pip install tensorboard

from datasets import Dataset, Audio
import pandas as pd

# 데이터 구조 예시
data = {
    'audio': ['audio/sample1.wav', 'audio/sample2.wav'],
    'sentence': ['안녕하세요 반갑습니다', '오늘 날씨가 좋네요']
}

df = pd.DataFrame(data)
dataset = Dataset.from_pandas(df)

# 오디오 컬럼 타입 지정
dataset = dataset.cast_column('audio', Audio(sampling_rate=16000))

from transformers import WhisperProcessor, WhisperForConditionalGeneration

model_name = "openai/whisper-small"
processor = WhisperProcessor.from_pretrained(model_name, language="Korean", task="transcribe")
model = WhisperForConditionalGeneration.from_pretrained(model_name)

# 한국어로 언어 토큰 강제 설정
model.config.forced_decoder_ids = processor.get_decoder_prompt_ids(language="ko", task="transcribe")
model.config.suppress_tokens = []

def prepare_dataset(batch):
    # 오디오를 16kHz로 리샘플링
    audio = batch["audio"]

    # 입력 피처 추출 (log-mel spectrogram)
    batch["input_features"] = processor.feature_extractor(
        audio["array"], 
        sampling_rate=audio["sampling_rate"]
    ).input_features[0]

    # 텍스트 토큰화
    batch["labels"] = processor.tokenizer(batch["sentence"]).input_ids

    return batch

# 데이터셋 전처리 적용
dataset = dataset.map(prepare_dataset, remove_columns=dataset.column_names)

from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer
import evaluate

# CER 메트릭 정의
cer_metric = evaluate.load("cer")

def compute_metrics(pred):
    pred_ids = pred.predictions
    label_ids = pred.label_ids

    # -100을 패딩으로 처리
    label_ids[label_ids == -100] = processor.tokenizer.pad_token_id

    pred_str = processor.tokenizer.batch_decode(pred_ids, skip_special_tokens=True)
    label_str = processor.tokenizer.batch_decode(label_ids, skip_special_tokens=True)

    cer = cer_metric.compute(predictions=pred_str, references=label_str)
    return {"cer": cer}

# 학습 인자 설정
training_args = Seq2SeqTrainingArguments(
    output_dir="./whisper-ko-finetuned",
    per_device_train_batch_size=8,
    gradient_accumulation_steps=2,  # 실질 배치 크기 = 16
    learning_rate=1e-5,
    warmup_steps=500,
    num_train_epochs=3,
    evaluation_strategy="steps",
    eval_steps=500,
    save_steps=1000,
    logging_steps=100,
    fp16=True,  # Mixed Precision 학습
    predict_with_generate=True,
    generation_max_length=225,
    report_to=["tensorboard"],
    load_best_model_at_end=True,
    metric_for_best_model="cer",
    greater_is_better=False,
)

# Trainer 초기화
trainer = Seq2SeqTrainer(
    model=model,
    args=training_args,
    train_dataset=dataset["train"],
    eval_dataset=dataset["test"],
    tokenizer=processor.feature_extractor,
    compute_metrics=compute_metrics,
)

# 학습 시작
trainer.train()

# 모델 저장
model.save_pretrained("./whisper-ko-final")
processor.save_pretrained("./whisper-ko-final")

# 추론 예시
import torch
from transformers import pipeline

pipe = pipeline(
    "automatic-speech-recognition",
    model="./whisper-ko-final",
    device=0 if torch.cuda.is_available() else -1
)

result = pipe("test_audio.wav")
print(result["text"])

from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=32,  # LoRA rank
    lora_alpha=64,
    target_modules=["q_proj", "v_proj"],  # Attention 레이어에만 적용
    lora_dropout=0.05,
    bias="none",
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# 출력: trainable params: 3.4M || all params: 247M || trainable%: 1.38%