Merge branch 'dev.embedder' of https://repositories.communitynotfound.work/PoliBa-DeepLearning/NanoSocrates into dev.embedder
This commit is contained in:
GassiGiuseppe
commit
ee253c39f4
@ -6,22 +6,22 @@
|
||||
"id": "adbd9598",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\Chris\\miniconda3\\envs\\deep_learning\\Lib\\site-packages\\torch\\utils\\_device.py:103: UserWarning: Aten Op fallback from XPU to CPU happends. This may have performance implications. If need debug the fallback ops please set environment variable `PYTORCH_DEBUG_XPU_FALLBACK=1` (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\build\\xpu\\ATen\\RegisterXPU_0.cpp:54528.)\n",
|
||||
" return func(*args, **kwargs)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"EPOCH 1\n",
|
||||
"\tLoss: 9.161508560180664\n",
|
||||
"ACTUAL:\n",
|
||||
"\tdbp-dbr:Category:Afpurl:linguistics/gold/hypernydbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Award=dbp-dbr:Category:Af city beginSt girdbp-dbr:Category:Films_set_in_a_movie_thea_C Colin performance. performance. In Kong,dbp-dbr:Category:Films_shot_in_Tokyo folkloredbp-dbr:Jamiedbp-dbr:Ingenious_Film_PRogRog Agat=ahlberdbp-dbr:I'm_Sdbp-dbr:Category:Films_about_paralledbp-dbr:Matt_Ddbp-dbr:Category:Films_about begin official secon slu\u0013 slu slujangodbp-dbr:Roy_Buchadbp-dbr:Matt_D official charact slu begin .. officialdbp-dbr:Category:American_films_about_9720.0dbp-dbr:Megamind_dbp-dbr:Roy_Buchauzanne_Pleshettedbp-dbr:Category:American_films_about_ Endgamso2.9dbp-dbr:Category:Films_about_American_s138.0so back gro ..ialldbp-dbr:Sweet_Dreams_(dbp-dbr:Nobody_but_Me_(The_Isley_Bdbp-dbr:Category:2000_inddbp-dbr:Jamie novel,am_Bergmaneldbp-dbr:Category:Films_featuring 21st Nai folklore direct-to cityam_Bergman_Actor_Golden_Globe_winning_performance-noun-1_crime_itdbp-dbo:sound folkloredbp-dbr:Category:American_films_about_dbp-dbr:Category:Films_set_in_acinematographer)dbp-dbr:Ingenious_Film_P dialogue official neighborhood. pdbp-dbr:Category:Best_Drama_Picture_cinematographer) reunrounddbp-dbr:Category:Sdbp-dbr:David_S._G manufacturers Winstdbp-dbr:Matt_D selectedeinstince ban_R see 21stdbp-dbr:Category:Films_based_on_works_by_Bret_dbp-dbr:List_of_accolades_receiveddbp-dbr:Category:Films_about_telepresence citych causedbp-dbr:Category:Films_about_cannibalman, creative p Martelldbp-dbr:Category:American_films_about_ictur styli city replacitrover Review roledbp-dbr:Category:Yō bedbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Award sets8.3am_Bergman tridbp-dbr:Category:Films_about_paraplegics_or_quadriplegdbp-dbr:Category:2000s_ official twodbp-dbr:Category:Films_featuring Blason Theroux,dbp-dbr:Category:Necrophilia_in_filmdbp-dbr:Category:Films_using_motion_c official operated8.3dbp-dbr:James_Remar save Nai official time.S ..dbp-dbr:Matt_D pdbp-dbr:Matt_D official Bla<START>dbp-dbr:Category:Films_featuring city two creativew3:2002/07/owl#round begin Begins. folkloreman, inspirhttp://www.w3.org/2006/03/wn/wn20/instances/synset-movie threedbp-dbr:MRC_(dbp-dbr:Reginald_Hudlindbp-dbr:Category:Yōs.dbp-dbr:Category:Cultdbp-dbr:Megamind_am_Bergman Yorksongdbp-dbr:Chadbp-dbr:Category:Films_set_in_adbp-dbr:Category:Cultarsgenre matchdbp-dbr:Category:Cult Batman novelhttp://www.w3.org/2006/03/wn/wn20/instances/synset-phonograph_re .. back acclaidbp-dbr:Charles_ officialdbp-dbr:Category:Cult Wyoming, city ten-yeardbp-dbr:Category:Films_set_in_adbp-dbr:Category:Films_featuring Theroux, sciencecontroversdbp-dbr:Category:Films_shot_in_Louisiana Cindydbp-dbr:Category:Yōdbp-dbr:Jamie friendtodbp-dbr:Category:2000s_chdbp-dbr:Category:Edgar_Award featur two1.006E9 Redbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Awarddbp-dbr:Category:Yō play,dbp-dbr:I'm_S8.3lan_Silvest Redbp-dbr:Philip_Proctorman,900000dbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Awarddbp-dbr:Category:Yōdbp-dbr:Category:Films_shot_in_Bedfordshiredbp-dbr:Category:Yō IsbinStephendbp-dbr:Category:Films_sho bot toget informatdbp-dbr:Category:Yōdbp-dbr:Category:The_Weinstein_Company_filmsling dialoguedbp-dbr:Category:Yōtra noveldbp-dbr:Category:American_films_about_ mastermindbp-dbr:Category:Cultmerican_1.006E9dbp-dbr:Category:Japanese_ghost_films novel dialogue1.006E9 Japanestrover<END>\n",
|
||||
"EXPECTED:\n",
|
||||
"\tThe Dark Knight is a 2008 superhero film directed by Christopher Nolan, from a screenplay co-written with his brother Jonathan. Based on the DC Comics superhero Batman, it is the sequel to Batman Begins (2005), and the second installment in The Dark Knight trilogy. The plot follows the vigilante Batman, police lieutenant James Gordon, and district attorney Harvey Dent, who form an alliance to dismantle organized crime in Gotham City. Their efforts are derailed by the Joker, an anarchistic mastermind who seeks to test how far Batman will go to save the city from chaos. The ensemble cast includes Christian Bale, Michael Caine, Heath Ledger, Gary Oldman, Aaron Eckhart, Maggie Gyllenhaal, and Morgan Freeman.Warner Bros. Pictures prioritized a sequel following the successful reinvention of the Batman film series with Batman Begins. Christopher and Batman Begins co-writer David S. Goyer developed the story elements, making Dent the central protagonist caught up in the battle between Batman and the Joker. In writing the screenplay, the Nolans were influenced by 1980s Batman comics and crime drama films, and sought to continue Batman Begins' heightened sense of realism. From April to November 2007, filming took place with a $185 million budget in Chicago and Hong Kong, and on sets<END><END>\n",
|
||||
"EPOCH 2\n",
|
||||
"\tLoss: 9.131484031677246\n"
|
||||
"\tLoss: 9.131484031677246\n",
|
||||
"ACTUAL:\n",
|
||||
"\tdbp-dbr:Category:Afpurl:linguistics/gold/hyperny2.9dbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_AwardGraham_ writdbp-dbr:20th_Century_St city beginSt girdbp-dbr:Category:Films_set_in_a_movie_thea_Co performance. performance.dbp-dbr:Category:Afscience_fictiondbp-dbr:Category:Films_shot_in_Tokyo folklore begin official BlaRog this= positivedbp-dbr:I'm_Sdbp-dbr:Category:Films_about_paralledbp-dbr:Matt_D Christ begin official.og match\u0013 slu slujangodbp-dbr:Roy_Buchadbp-dbr:Matt_D official charact back begincinematographer) officialdbp-dbr:Category:American_films_about_9720.0dbp-dbr:Megamind_uzanne_Pleshettedbp-dbr:Category:American_films_about_dbp-dbr:Category:Films_set_in_a folklore2.9comedy_dbp-dbr:One_Way execut back characters'dbp-dbr:Category:Best_Animated_Feature_Broadcast_ialldbp-dbr:Sweet_Dreams_(dbp-dbr:Nobody_but_Me_(The_Isley_Bdbp-dbr:Category:2000_inddbp-dbr:Jamie novel,3.0eldbp-dbr:Category:Films_featuring 21st Nai_F direct-to city BAFTA,_Actor_Golden_Globe_winning_performance-noun-1 Naiitdbp-dbo:sound inspirdbp-dbr:Category:American_films_about_ pcinematographer)dbp-dbr:Ingenious_Film_P dialogue ban p frecinematographer) reun_Actor_Golden_Globe_winning_performancedbp-dbr:Roy_Bsing manufacturers Winstdbp-dbr:Matt_D selectedeinstincears_R see 21stdbp-dbr:Category:Afdbp-dbp:namedbp-dbr:Category:Films_about_telepresence citych causedbp-dbr:Category:Films_about_cannibalman, creative p Martelldbp-dbr:Category:American_films_about_ emphasized styli Re replacitrovereghttp://www.w3.org/2006/03/wn/wn20/instances/synset-phonograph_redbp-dbr:Category:Yō targetsdbp-dbr:Category:Films_whose_editor_won_the_Best_Film_Editing_Academy_Award put8.3am_Bergman tridbp-dbr:Category:Yōdbp-dbr:Category:2000s_ official twodbp-dbr:Category:Films_featuring high-reso8.3 Theroux,dbp-dbr:Category:Necrophilia_in_filmdbp-dbr:Category:Films_using_motion_c official operated8.3drama_films save Nai official time.S ..dbp-dbr:Matt_Dent_dbp-dbr:Matt_D official Bla<START>dbp-dbr:Category:Films_featuringo twodbp-dbr:Category:Films_featuringcontroversies_ advertis begin Begins. folkloreman, inspirhttp://www.w3.org/2006/03/wn/wn20/instances/synset-movie .. noveldbp-dbr:Reginald_Hudlindbp-dbr:Category:Yōs.dbp-dbr:Category:Cultdbp-dbr:James_Remaram_Bergman Yorksongdbp-dbr:Chadbp-dbr:Category:Films_set_in_a (Evanars match Begins.genre matchdbp-dbr:Category:Cult Batman novelhttp://www.w3.org/2006/03/wn/wn20/instances/synset-phonograph_refter back acclai folklore officialdbp-dbr:The_Dark_Knight__Sound__1 noveldbp-dbr:Category:Films_featuring_a ten-yeardbp-dbr:Category:Films_set_in_adbp-dbr:Category:Films_featuring Theroux,dbp-dbr:Category:2010s_American_filmssuitle Cindydbp-dbr:Category:Yōdbp-dbr:Jamiemerican_todbp-dbr:Category:2000s_chdbp-dbr:Category:Edgar_Award featur two1.006E9 Redbp-dbr:Roy_Buchadbp-dbr:Category:Yōdbp-dbr:Category:Yōdbp-dbr:I'm_S8.3lan_Silvest (particularlydbp-dbr:Philip_Proctorman,900000 Naidbp-dbr:Category:Yō Goggindbp-dbr:Category:Yō IsbinStephendbp-dbr:Category:Films_sho botEnglish-languag informatdbp-dbr:Category:Yōdbp-dbr:Category:The_Weinstein_Company_films Japanesdbp-dbr:Category:Af BAFTA,tra noveldbp-dbr:Category:American_films_about_ mastermindbp-dbr:Category:Cultmerican_1.006E9dbp-dbr:Category:Japanese_ghost_films novel dialogue1.006E9 Japanes Behind<END>\n",
|
||||
"EXPECTED:\n",
|
||||
"\tThe Dark Knight is a 2008 superhero film directed by Christopher Nolan, from a screenplay co-written with his brother Jonathan. Based on the DC Comics superhero Batman, it is the sequel to Batman Begins (2005), and the second installment in The Dark Knight trilogy. The plot follows the vigilante Batman, police lieutenant James Gordon, and district attorney Harvey Dent, who form an alliance to dismantle organized crime in Gotham City. Their efforts are derailed by the Joker, an anarchistic mastermind who seeks to test how far Batman will go to save the city from chaos. The ensemble cast includes Christian Bale, Michael Caine, Heath Ledger, Gary Oldman, Aaron Eckhart, Maggie Gyllenhaal, and Morgan Freeman.Warner Bros. Pictures prioritized a sequel following the successful reinvention of the Batman film series with Batman Begins. Christopher and Batman Begins co-writer David S. Goyer developed the story elements, making Dent the central protagonist caught up in the battle between Batman and the Joker. In writing the screenplay, the Nolans were influenced by 1980s Batman comics and crime drama films, and sought to continue Batman Begins' heightened sense of realism. From April to November 2007, filming took place with a $185 million budget in Chicago and Hong Kong, and on sets<END><END>\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -92,7 +92,7 @@
|
||||
" output_tokens, SENTENCE_LENGTH, PAD_TOKEN, END_TOKEN\n",
|
||||
" )\n",
|
||||
" decoder_default_tokens, _ = Transformer.normalize_sequence(\n",
|
||||
" decoder_default_tokens, SENTENCE_LENGTH, PAD_TOKEN, END_TOKEN\n",
|
||||
" decoder_default_tokens, SENTENCE_LENGTH, PAD_TOKEN, END_TOKEN, False\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" TOY_BATCH_INPUT_LIST.append(input_tokens)\n",
|
||||
@ -121,23 +121,26 @@
|
||||
"\n",
|
||||
" encoder_list = torch.tensor([TOY_BATCH_INPUT_LIST[0]])\n",
|
||||
" decoder_list = torch.tensor([TOY_BATCH_DECODER_DEFAULT[0]])\n",
|
||||
" padding_list = torch.tensor([TOY_BATCH_PADDING_LIST[0]], dtype=torch.bool)\n",
|
||||
" src_padding = torch.tensor([TOY_BATCH_PADDING_LIST[0]], dtype=torch.bool)\n",
|
||||
"\n",
|
||||
" # Transform target into logits\n",
|
||||
" target_logits = torch.tensor([TOY_BATCH_TARGET_LIST[0]])\n",
|
||||
"\n",
|
||||
" last_loss = 0\n",
|
||||
" last_prediction: torch.Tensor\n",
|
||||
"\n",
|
||||
" for i in range(0, SENTENCE_LENGTH):\n",
|
||||
"\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" tgt_padding = decoder_list.eq(PAD_TOKEN)\n",
|
||||
"\n",
|
||||
" logits: torch.Tensor = NANOSOCRATES((encoder_list, padding_list, decoder_list))\n",
|
||||
" logits: torch.Tensor = NANOSOCRATES((encoder_list, src_padding, decoder_list, tgt_padding))\n",
|
||||
" prob = torch.softmax(logits, 2)\n",
|
||||
"\n",
|
||||
" most_probable_tokens = torch.argmax(prob, 2)\n",
|
||||
" last_prediction = most_probable_tokens\n",
|
||||
"\n",
|
||||
" logits = logits[:,0:i,:]\n",
|
||||
" logits = logits[:,:i,:]\n",
|
||||
" logits = logits.permute(0, 2, 1)\n",
|
||||
"\n",
|
||||
" loss : torch.Tensor = cross_entropy(logits, target_logits[:, 0:i])\n",
|
||||
@ -152,11 +155,23 @@
|
||||
" decoder_list[:,i+1] = target_logits[:,i]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" current_epoch += 1\n",
|
||||
"\n",
|
||||
" if current_epoch % 1 == 0:\n",
|
||||
" print(f\"EPOCH {current_epoch}\\n\\tLoss: {last_loss}\")\n",
|
||||
"\n",
|
||||
" for encoded_sentence, expected_sentence in zip(\n",
|
||||
" Transformer.tensor2token(last_prediction[:,:], END_TOKEN), # type: ignore\n",
|
||||
" Transformer.tensor2token(target_logits[:,:], END_TOKEN)\n",
|
||||
" ):\n",
|
||||
" decoded_sentence = TOKENANO.decode(encoded_sentence)\n",
|
||||
" decoded_target = TOKENANO.decode(expected_sentence)\n",
|
||||
" print(f\"ACTUAL:\\n\\t{decoded_sentence}\\nEXPECTED:\\n\\t{decoded_target}\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
|
||||
@ -189,7 +189,7 @@ class NanoSocratesBPE(Encoder):
|
||||
token_stack.appendleft(right_token)
|
||||
token_stack.appendleft(left_token)
|
||||
|
||||
return UTF_8_STRING_ARR.decode("utf-8")
|
||||
return UTF_8_STRING_ARR.decode("utf-8", errors="ignore")
|
||||
|
||||
def __token_decode(self, token_id: int) -> tuple[int, int]:
|
||||
|
||||
|
||||
@ -41,18 +41,19 @@ class Decoder(nn.Module):
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor
|
||||
]
|
||||
): # -> list[torch.Tensor]: # k_x = v_x . While x_q = x
|
||||
# WARNING: args is needed to have sequential
|
||||
x, k_x, v_x, padding_mask = args
|
||||
x, k_x, v_x, src_padding_mask, tgt_padding_mask = args
|
||||
|
||||
# build of attention mask
|
||||
attention_mask = get_causal_attention_mask(x.size(1))
|
||||
|
||||
# 1) Masked Attention
|
||||
MASKED_ATTENTION = self.__masked_attention(
|
||||
x, x, x, key_padding_mask=padding_mask, attention_mask=attention_mask
|
||||
x, x, x, key_padding_mask=tgt_padding_mask, attention_mask=attention_mask
|
||||
)
|
||||
|
||||
# 2) Dropout
|
||||
@ -68,7 +69,7 @@ class Decoder(nn.Module):
|
||||
|
||||
# 5) Encoder–decoder (cross) attention
|
||||
CROSS_ATTENTION = self.__cross_attention(
|
||||
x, k_x, v_x, key_padding_mask=padding_mask
|
||||
x, k_x, v_x, key_padding_mask=src_padding_mask
|
||||
)
|
||||
|
||||
# 6) Dropout
|
||||
@ -96,7 +97,7 @@ class Decoder(nn.Module):
|
||||
# 12) Layer Normalization
|
||||
x = self.__layer_norm_3(x)
|
||||
|
||||
return (x, k_x, v_x, padding_mask)
|
||||
return (x, k_x, v_x, src_padding_mask, tgt_padding_mask)
|
||||
|
||||
|
||||
# use eval to disable dropout ecc
|
||||
|
||||
@ -37,17 +37,17 @@ class TrainingModel(torch.nn.Module):
|
||||
|
||||
self.__detokener = DeToken(latent_space, vocabulary_size)
|
||||
|
||||
def forward(self, args: tuple[torch.Tensor, torch.Tensor, torch.Tensor]):
|
||||
def forward(self, args: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]):
|
||||
|
||||
encoder_embedder_input, padding_tensor, decoder_embedder_input = args
|
||||
encoder_embedder_input, src_padding, decoder_embedder_input, tgt_padding = args
|
||||
|
||||
encoder_tensor = self.__encoder_embedder(encoder_embedder_input)
|
||||
decoder_tensor = self.__decoder_embedder(decoder_embedder_input)
|
||||
|
||||
encoder_output, _ = self.__encoder((encoder_tensor, padding_tensor))
|
||||
encoder_output, _ = self.__encoder((encoder_tensor, src_padding))
|
||||
|
||||
decoder_output, _, _, _ = self.__decoder(
|
||||
(decoder_tensor, encoder_output, encoder_output, None)
|
||||
decoder_output, _, _, _, _ = self.__decoder(
|
||||
(decoder_tensor, encoder_output, encoder_output, src_padding, tgt_padding)
|
||||
)
|
||||
|
||||
logits: torch.Tensor = self.__detokener(decoder_output)
|
||||
|
||||
@ -3,6 +3,7 @@ from .task_type import TaskType
|
||||
from .post_tokenization import truncate_sequence, pad_sequence, normalize_sequence, create_padding_mask
|
||||
from .inference_masking import inference_masking
|
||||
from .truncate_rdf_list import truncate_rdf_list
|
||||
from .decode_out import tensor2token
|
||||
|
||||
__all__ = [
|
||||
"TaskType",
|
||||
@ -13,5 +14,6 @@ __all__ = [
|
||||
"create_padding_mask",
|
||||
"normalize_sequence",
|
||||
"inference_masking",
|
||||
"truncate_rdf_list"
|
||||
"truncate_rdf_list",
|
||||
"tensor2token"
|
||||
]
|
||||
27
Project_Model/Libs/Transformer/Utils/decode_out.py
Normal file
27
Project_Model/Libs/Transformer/Utils/decode_out.py
Normal file
@ -0,0 +1,27 @@
|
||||
from typing import Generator
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def tensor2token(tensor: torch.Tensor, end_token: int) -> Generator[list[int]]:
|
||||
|
||||
if len(tensor.shape) < 1 or len(tensor.shape) > 2:
|
||||
raise ValueError("Shape is not correct")
|
||||
|
||||
if len(tensor.shape) == 1:
|
||||
token_list: list[int] = tensor.tolist()
|
||||
token_list.append(end_token)
|
||||
yield token_list
|
||||
return
|
||||
|
||||
batch_len: int
|
||||
batch_len, _ = tensor.shape
|
||||
|
||||
for i in range(batch_len):
|
||||
|
||||
smaller_tensor = tensor[i, :]
|
||||
token_list: list[int] = smaller_tensor.tolist()
|
||||
token_list.append(end_token)
|
||||
yield token_list
|
||||
|
||||
|
||||
@ -1,16 +1,19 @@
|
||||
def truncate_sequence(
|
||||
sequence: list[int], truncate_at: int, end_token: int
|
||||
sequence: list[int], truncate_at: int, end_token: int, add_ending: bool
|
||||
) -> list[int]:
|
||||
|
||||
if len(sequence) < truncate_at - 1:
|
||||
if add_ending:
|
||||
sequence.append(end_token)
|
||||
return sequence
|
||||
|
||||
if len(sequence) < truncate_at:
|
||||
if add_ending:
|
||||
sequence[-1] = end_token
|
||||
return sequence
|
||||
|
||||
TRUNCATED_SEQUENCE = sequence[:truncate_at]
|
||||
if add_ending:
|
||||
TRUNCATED_SEQUENCE[-1] = end_token
|
||||
|
||||
return TRUNCATED_SEQUENCE
|
||||
@ -48,8 +51,9 @@ def normalize_sequence(
|
||||
max_length: int,
|
||||
pad_token: int,
|
||||
end_token: int,
|
||||
add_ending: bool = True
|
||||
) -> tuple[list[int], list[bool]]:
|
||||
new_sequence = truncate_sequence(sequence, max_length, end_token)
|
||||
new_sequence = truncate_sequence(sequence, max_length, end_token, add_ending)
|
||||
new_sequence = pad_sequence(new_sequence, max_length, pad_token)
|
||||
PADDING_MASK = create_padding_mask(new_sequence, pad_token)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user