Added support to resume workload

2025-10-02 08:48:28 +02:00 · 2025-10-02 08:48:28 +02:00 · 17d82f0a4e
commit 17d82f0a4e
parent 0975c19e69
1 changed files with 7 additions and 1 deletions
--- a/Scripts/Training/bpe_trainer_pool.py
+++ b/Scripts/Training/bpe_trainer_pool.py
@ -72,11 +72,17 @@ def train(args: ProgramArgs):
    VOCABULARY_PATH = Path(args.output_file)
    CACHE_PATH = Path(args.cache_file)

+    start_bpe = BPE.NanoSocratesBPE()
+    if CACHE_PATH.is_file():
+        voc = BPE.load_nanos_vocabulary(CACHE_PATH)
+        start_bpe = BPE.NanoSocratesBPE(voc)
+
    print(f"Training BPE")

    BPE_ENCODER = TRAINER.trainBPE(
        DATASET_PATH,
-        CACHE_PATH
+        CACHE_PATH,
+        start_bpe
    )

    VOCABULARY = BPE_ENCODER.vocabulary