PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added support to resume workload

Browse Source
This commit is contained in:
Christian Risi 2025-10-02 08:48:28 +02:00
parent 0975c19e69
commit 17d82f0a4e
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Scripts/Training/bpe_trainer_pool.py
View File

@ -72,11 +72,17 @@ def train(args: ProgramArgs):
VOCABULARY_PATH = Path(args.output_file) VOCABULARY_PATH = Path(args.output_file)
CACHE_PATH = Path(args.cache_file) CACHE_PATH = Path(args.cache_file)
start_bpe = BPE.NanoSocratesBPE()
if CACHE_PATH.is_file():
voc = BPE.load_nanos_vocabulary(CACHE_PATH)
start_bpe = BPE.NanoSocratesBPE(voc)
print(f"Training BPE") print(f"Training BPE")
BPE_ENCODER = TRAINER.trainBPE( BPE_ENCODER = TRAINER.trainBPE(
DATASET_PATH, DATASET_PATH,
CACHE_PATH CACHE_PATH,
start_bpe
) )
VOCABULARY = BPE_ENCODER.vocabulary VOCABULARY = BPE_ENCODER.vocabulary