113 lines
2.4 KiB
Plaintext
113 lines
2.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "4ae47336",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"B, T, D = 4, 7, 32\n",
|
|
"x = torch.randn(B, T, D)\n",
|
|
"attn_mask = torch.triu(torch.ones(T, T, dtype=torch.bool), diagonal=1) # [T,T]\n",
|
|
"pad_mask = torch.zeros(B, T, dtype=torch.bool) # no pads\n",
|
|
"mha = torch.nn.MultiheadAttention(D, num_heads=4, batch_first=True)\n",
|
|
"y, _ = mha(x, x, x, attn_mask=attn_mask, key_padding_mask=pad_mask) # should work\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "e38e3fb5",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([[[0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
|
|
" [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]],\n",
|
|
"\n",
|
|
" [[0, 0, 1, 0, 0, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
|
|
" [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]]])"
|
|
]
|
|
},
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"torch.nn.functional.one_hot(torch.tensor([\n",
|
|
" [4, 1, 9],\n",
|
|
" [2,4,5]\n",
|
|
"]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "7119ad53",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"device(type='cpu')"
|
|
]
|
|
},
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"torch.get_default_device()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "8c95691a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"xpu\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from Project_Model.Libs.TorchShims import get_default_device\n",
|
|
"\n",
|
|
"print(get_default_device())"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "deep_learning",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.13.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|