diff --git a/compute_pppl.py b/compute_pppl.py index 11b6efd14e53837fe810b8453af01c9753806f25..58a4ba66105bac415e6e7a40405217e7c8955c1a 100644 --- a/compute_pppl.py +++ b/compute_pppl.py @@ -5,6 +5,7 @@ import json import logging import argparse from tqdm import tqdm +from pathlib import Path from transformers import AutoTokenizer, AutoModelForMaskedLM @@ -193,6 +194,7 @@ def main(args): model_name = args.model_name window_size = args.window_size batch_size = args.batch_size + Path(args.output_path).mkdir(parents=True, exist_ok=True) for (root,dirs,files) in os.walk(args.input_path, topdown=True):