PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Added EOS token

Browse Source
This commit is contained in:
GassiGiuseppe
2025-10-07 22:47:59 +02:00
parent a04f4c7cb7
commit ee12f53f12
3 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Scripts/DataCleaning/pipeline/cleaner.py
View File

@@ -57,8 +57,8 @@ class PipelineApplier():
# MovieID and Abstract are unique for each other 1 <-> 1
RDF = RDF.groupby(["MovieID", "Abstract"])["Triple"].apply("".join).reset_index()
# add special token for: start of triple, end of triple and start of abstract
RDF["Triple"] = SpecialToken.START_TRIPLE_LIST.value + RDF["Triple"]
RDF["Abstract"] = SpecialToken.ABSTRACT.value + RDF["Abstract"]
RDF["Triple"] = SpecialToken.START_TRIPLE_LIST.value + RDF["Triple"]+SpecialToken.END_OF_SENTENCE.value
RDF["Abstract"] = SpecialToken.ABSTRACT.value + RDF["Abstract"] + SpecialToken.END_OF_SENTENCE.value
return RDF[["MovieID","Triple","Abstract"]]