This commit is contained in:
GassiGiuseppe 2025-10-10 22:27:01 +02:00
parent e76dbeb9a7
commit 3446870291

View File

@ -41,7 +41,7 @@ class Batcher:
RNG = random.Random(self._seed)
self._masker.reseed(self._seed)
for batch in pd.read_csv(self._dataset_path, chunksize= int(batch_size)): #now we support 3 task
for batch in pd.read_csv(self._dataset_path, chunksize= batch_size):
tokenized_batch = pd.DataFrame()
# encode