From ee12f53f12bf796021aef89c9c18718f95a8ee76 Mon Sep 17 00:00:00 2001
From: GassiGiuseppe <g.gassi@studenti.poliba.it>
Date: Tue, 7 Oct 2025 22:47:59 +0200
Subject: [PATCH] Added EOS token

---
 Scripts/DataCleaning/pipeline/cleaner.py       | 4 ++--
 Scripts/DataCleaning/pipeline/pipeline.py      | 7 ++++---
 Scripts/Libs/CleaningPipeline/special_token.py | 1 +
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/Scripts/DataCleaning/pipeline/cleaner.py b/Scripts/DataCleaning/pipeline/cleaner.py
index ff0dc14..812d757 100644
--- a/Scripts/DataCleaning/pipeline/cleaner.py
+++ b/Scripts/DataCleaning/pipeline/cleaner.py
@@ -57,8 +57,8 @@ class PipelineApplier():
         # MovieID and Abstract are unique for each other 1 <-> 1
         RDF = RDF.groupby(["MovieID", "Abstract"])["Triple"].apply("".join).reset_index()
         # add special token for: start of triple, end of triple and start of abstract
-        RDF["Triple"] = SpecialToken.START_TRIPLE_LIST.value + RDF["Triple"] 
-        RDF["Abstract"] = SpecialToken.ABSTRACT.value + RDF["Abstract"]
+        RDF["Triple"] = SpecialToken.START_TRIPLE_LIST.value + RDF["Triple"]+SpecialToken.END_OF_SENTENCE.value
+        RDF["Abstract"] = SpecialToken.ABSTRACT.value + RDF["Abstract"] + SpecialToken.END_OF_SENTENCE.value
         return RDF[["MovieID","Triple","Abstract"]]
 
 
diff --git a/Scripts/DataCleaning/pipeline/pipeline.py b/Scripts/DataCleaning/pipeline/pipeline.py
index d350497..eb8ba8c 100644
--- a/Scripts/DataCleaning/pipeline/pipeline.py
+++ b/Scripts/DataCleaning/pipeline/pipeline.py
@@ -53,7 +53,7 @@ class Pipeline():
         self.task_rdf_completation = RDF_completation_task_dataset("./Assets/Dataset/Tmp/rdf_completation.csv")
 
         self._movie_filter.frequency_filter(50,3000)
-        self._relationship_filter.frequency_filter(20, 2395627) # from 2718 to 3069 
+        self._relationship_filter.frequency_filter(25, 2395627) # from 2718 to 3069 
         self._relationship_filter.delete_relationship_uri_by_list(RELATIONSHIP_FILTER_LIST)
 
     def other_filter(self):
@@ -131,7 +131,8 @@ class Pipeline():
         # Django Unchained  : 138952
         # Spirited Away     : 144137
         # Knives Out        : 148025
-        movie_list = [106465,106466,106467,106468,106469,106470,106471,106472,106473]#[117248, 147074, 113621, 1123, 117586, 90177, 71587, 138952, 144137, 148025]
+        # [106465,106466,106467,106468,106469,106470,106471,106472,106473]
+        movie_list = [117248, 147074, 113621, 1123, 117586, 90177, 71587, 138952, 144137, 148025]
         self._movie_filter.MOVIE_FILTER = pd.DataFrame({"MovieID": movie_list})
 
     def generate_csv_debug_file(self, debug_path:str):
@@ -144,7 +145,7 @@ class Pipeline():
 
 
 pipe = Pipeline()
-# pipe.use_toy_dataset()
+#pipe.use_toy_dataset()
 pipe.other_filter()
 pipe.execute_all_task()
 # pipe.generate_csv_debug_file("Assets/Dataset/Tmp/debug.csv")
\ No newline at end of file
diff --git a/Scripts/Libs/CleaningPipeline/special_token.py b/Scripts/Libs/CleaningPipeline/special_token.py
index 644ad71..a3dc418 100644
--- a/Scripts/Libs/CleaningPipeline/special_token.py
+++ b/Scripts/Libs/CleaningPipeline/special_token.py
@@ -9,6 +9,7 @@ class SpecialToken(str, Enum):
     RELATIONSHIP = "<PRED>"
     OBJECT = "<OBJ>"
     ABSTRACT = "<ABS>"
+    END_OF_SENTENCE = "<EOS>"
     CORPUS_END = "<END>"
 
     ## Tasks' Token