From 29677f7b18de9ebf9a237719726816440c7e59d6 Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 1 Feb 2025 10:54:29 +0900 Subject: [PATCH 1/7] Add files via upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - evaluation_test : 공식 깃헙대로 version 맞춰서 테스트 (timm=0.3.2) - version_updated : 업데이트된 version으로 공식 깃헙 pth 파일 불러와서 테스트 - Imagenet 2012 val 사용 --- evalutation_test.ipynb | 596 +++++++++++++++++++++++++++++++++++++++++ version_updated.ipynb | 489 +++++++++++++++++++++++++++++++++ 2 files changed, 1085 insertions(+) create mode 100644 evalutation_test.ipynb create mode 100644 version_updated.ipynb diff --git a/evalutation_test.ipynb b/evalutation_test.ipynb new file mode 100644 index 0000000..b88b218 --- /dev/null +++ b/evalutation_test.ipynb @@ -0,0 +1,596 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "https://github.com/facebookresearch/deit/blob/main/README_deit.md" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#bash\n", + "\n", + "pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 torchaudio==0.8.1 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html\n", + "pip install torchtext==0.9.1\n", + "pip install timm==0.3.2\n", + "\n", + "- ImportError: cannot import name 'container_abcs' from 'torch._six' 오류 해결 위해 다운그레이드 필요!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.8.1+cu111\n", + "0.3.2\n" + ] + } + ], + "source": [ + "import torch\n", + "import timm\n", + "\n", + "print(torch.__version__) # PyTorch 버전 확인\n", + "print(timm.__version__) # timm 버전 확인\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import timm\n", + "assert timm.__version__ == \"0.3.2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluation func" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import torch.nn as nn\n", + "from thop import profile # FLOPs\n", + "\n", + "# 모델 FLOPs 계산 함수\n", + "def compute_flops(model, input_size=(1, 3, 224, 224)):\n", + " flops, params = profile(model, inputs=(torch.randn(input_size),))\n", + " return flops\n", + "\n", + "# 모델 평가 함수 정의\n", + "def evaluate_model(model, dataloader):\n", + " model.eval()\n", + " correct_top1 = 0\n", + " correct_top5 = 0\n", + " total = 0\n", + " loss_fn = nn.CrossEntropyLoss()\n", + " total_loss = 0.0\n", + "\n", + " start_time = time.time() # 평가 시작 시간 기록\n", + "\n", + " with torch.no_grad():\n", + " for images, labels in dataloader:\n", + " images, labels = images.to('cuda' if torch.cuda.is_available() else 'cpu'), labels.to('cuda' if torch.cuda.is_available() else 'cpu')\n", + " outputs = model(images)\n", + " _, predicted_top1 = torch.max(outputs.data, 1)\n", + " predicted_top5 = torch.topk(outputs.data, 5)[1] # Top-5 예측\n", + " \n", + " total += labels.size(0)\n", + " correct_top1 += (predicted_top1 == labels).sum().item()\n", + " \n", + " # Top-5 비교\n", + " correct_top5 += (predicted_top5 == labels.view(-1, 1)).sum().item()\n", + " \n", + " loss = loss_fn(outputs, labels)\n", + " total_loss += loss.item()\n", + "\n", + " end_time = time.time() # 평가 종료 시간 기록\n", + " elapsed_time = end_time - start_time # 총 소요 시간 계산\n", + " \n", + " accuracy_top1 = 100 * correct_top1 / total\n", + " accuracy_top5 = 100 * correct_top5 / total\n", + " avg_loss = total_loss / len(dataloader) # 평균 손실 계산\n", + " \n", + " # 결과 출력\n", + " print(f'Acc@1: {accuracy_top1:.3f} Acc@5: {accuracy_top5:.3f} loss: {avg_loss:.3f}')\n", + " \n", + " # 이미지 처리 속도 계산\n", + " im_per_sec = total / elapsed_time\n", + " print(f'Processing speed: {im_per_sec:.2f} images/sec')\n", + " \n", + " # FLOPs 계산\n", + " flops = compute_flops(model)\n", + " print(f'FLOPs: {flops:.2f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeiT-tiny" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + } + ], + "source": [ + "model_tiny = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_tiny.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Acc@1: 69.952 Acc@5: 90.014 loss: 1.310\n", + "Processing speed: 24.46 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "FLOPs: 1078633728.00\n" + ] + } + ], + "source": [ + "evaluate_model(model_tiny, test_loader) #34min" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which should give\n", + "\n", + "* Acc@1 72.202 Acc@5 91.124 loss 1.219\n", + "\n", + "제발..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeiT-small" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + } + ], + "source": [ + "model_small = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Acc@1: 78.846 Acc@5: 94.488 loss: 0.928\n", + "Processing speed: 11.94 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "FLOPs: 4248783360.00\n" + ] + } + ], + "source": [ + "model_small.eval()\n", + "evaluate_model(model_small, test_loader) #69min" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "giving\n", + "\n", + "* Acc@1 79.854 Acc@5 94.968 loss 0.881" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeiT-base" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n", + "Downloading: \"https://dl.fbaipublicfiles.com/deit/deit_base_patch16_224-b5f2ef4d.pth\" to C:\\Users\\seonahryu/.cache\\torch\\hub\\checkpoints\\deit_base_patch16_224-b5f2ef4d.pth\n", + "100%|██████████| 330M/330M [00:09<00:00, 36.1MB/s] \n" + ] + } + ], + "source": [ + "model_base = torch.hub.load('facebookresearch/deit:main', 'deit_base_patch16_224', pretrained=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Acc@1: 81.252 Acc@5: 95.322 loss: 0.854\n", + "Processing speed: 3.16 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "FLOPs: 16863630336.00\n" + ] + } + ], + "source": [ + "model_base.eval()\n", + "evaluate_model(model_base, test_loader) #263min" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This should give\n", + "\n", + "* Acc@1 81.846 Acc@5 95.594 loss 0.820" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/version_updated.ipynb b/version_updated.ipynb new file mode 100644 index 0000000..30975dd --- /dev/null +++ b/version_updated.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from timm import create_model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "model = create_model('deit_tiny_patch16_224', pretrained=True, num_classes=1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\seonahryu\\AppData\\Local\\Temp\\ipykernel_86428\\3522241460.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", + " model.load_state_dict(torch.load(\"C:/Users/seonahryu/Downloads/deit_tiny_patch16_224-a1311bcf.pth\"), strict=False)\n" + ] + }, + { + "data": { + "text/plain": [ + "_IncompatibleKeys(missing_keys=['cls_token', 'pos_embed', 'patch_embed.proj.weight', 'patch_embed.proj.bias', 'blocks.0.norm1.weight', 'blocks.0.norm1.bias', 'blocks.0.attn.qkv.weight', 'blocks.0.attn.qkv.bias', 'blocks.0.attn.proj.weight', 'blocks.0.attn.proj.bias', 'blocks.0.norm2.weight', 'blocks.0.norm2.bias', 'blocks.0.mlp.fc1.weight', 'blocks.0.mlp.fc1.bias', 'blocks.0.mlp.fc2.weight', 'blocks.0.mlp.fc2.bias', 'blocks.1.norm1.weight', 'blocks.1.norm1.bias', 'blocks.1.attn.qkv.weight', 'blocks.1.attn.qkv.bias', 'blocks.1.attn.proj.weight', 'blocks.1.attn.proj.bias', 'blocks.1.norm2.weight', 'blocks.1.norm2.bias', 'blocks.1.mlp.fc1.weight', 'blocks.1.mlp.fc1.bias', 'blocks.1.mlp.fc2.weight', 'blocks.1.mlp.fc2.bias', 'blocks.2.norm1.weight', 'blocks.2.norm1.bias', 'blocks.2.attn.qkv.weight', 'blocks.2.attn.qkv.bias', 'blocks.2.attn.proj.weight', 'blocks.2.attn.proj.bias', 'blocks.2.norm2.weight', 'blocks.2.norm2.bias', 'blocks.2.mlp.fc1.weight', 'blocks.2.mlp.fc1.bias', 'blocks.2.mlp.fc2.weight', 'blocks.2.mlp.fc2.bias', 'blocks.3.norm1.weight', 'blocks.3.norm1.bias', 'blocks.3.attn.qkv.weight', 'blocks.3.attn.qkv.bias', 'blocks.3.attn.proj.weight', 'blocks.3.attn.proj.bias', 'blocks.3.norm2.weight', 'blocks.3.norm2.bias', 'blocks.3.mlp.fc1.weight', 'blocks.3.mlp.fc1.bias', 'blocks.3.mlp.fc2.weight', 'blocks.3.mlp.fc2.bias', 'blocks.4.norm1.weight', 'blocks.4.norm1.bias', 'blocks.4.attn.qkv.weight', 'blocks.4.attn.qkv.bias', 'blocks.4.attn.proj.weight', 'blocks.4.attn.proj.bias', 'blocks.4.norm2.weight', 'blocks.4.norm2.bias', 'blocks.4.mlp.fc1.weight', 'blocks.4.mlp.fc1.bias', 'blocks.4.mlp.fc2.weight', 'blocks.4.mlp.fc2.bias', 'blocks.5.norm1.weight', 'blocks.5.norm1.bias', 'blocks.5.attn.qkv.weight', 'blocks.5.attn.qkv.bias', 'blocks.5.attn.proj.weight', 'blocks.5.attn.proj.bias', 'blocks.5.norm2.weight', 'blocks.5.norm2.bias', 'blocks.5.mlp.fc1.weight', 'blocks.5.mlp.fc1.bias', 'blocks.5.mlp.fc2.weight', 'blocks.5.mlp.fc2.bias', 'blocks.6.norm1.weight', 'blocks.6.norm1.bias', 'blocks.6.attn.qkv.weight', 'blocks.6.attn.qkv.bias', 'blocks.6.attn.proj.weight', 'blocks.6.attn.proj.bias', 'blocks.6.norm2.weight', 'blocks.6.norm2.bias', 'blocks.6.mlp.fc1.weight', 'blocks.6.mlp.fc1.bias', 'blocks.6.mlp.fc2.weight', 'blocks.6.mlp.fc2.bias', 'blocks.7.norm1.weight', 'blocks.7.norm1.bias', 'blocks.7.attn.qkv.weight', 'blocks.7.attn.qkv.bias', 'blocks.7.attn.proj.weight', 'blocks.7.attn.proj.bias', 'blocks.7.norm2.weight', 'blocks.7.norm2.bias', 'blocks.7.mlp.fc1.weight', 'blocks.7.mlp.fc1.bias', 'blocks.7.mlp.fc2.weight', 'blocks.7.mlp.fc2.bias', 'blocks.8.norm1.weight', 'blocks.8.norm1.bias', 'blocks.8.attn.qkv.weight', 'blocks.8.attn.qkv.bias', 'blocks.8.attn.proj.weight', 'blocks.8.attn.proj.bias', 'blocks.8.norm2.weight', 'blocks.8.norm2.bias', 'blocks.8.mlp.fc1.weight', 'blocks.8.mlp.fc1.bias', 'blocks.8.mlp.fc2.weight', 'blocks.8.mlp.fc2.bias', 'blocks.9.norm1.weight', 'blocks.9.norm1.bias', 'blocks.9.attn.qkv.weight', 'blocks.9.attn.qkv.bias', 'blocks.9.attn.proj.weight', 'blocks.9.attn.proj.bias', 'blocks.9.norm2.weight', 'blocks.9.norm2.bias', 'blocks.9.mlp.fc1.weight', 'blocks.9.mlp.fc1.bias', 'blocks.9.mlp.fc2.weight', 'blocks.9.mlp.fc2.bias', 'blocks.10.norm1.weight', 'blocks.10.norm1.bias', 'blocks.10.attn.qkv.weight', 'blocks.10.attn.qkv.bias', 'blocks.10.attn.proj.weight', 'blocks.10.attn.proj.bias', 'blocks.10.norm2.weight', 'blocks.10.norm2.bias', 'blocks.10.mlp.fc1.weight', 'blocks.10.mlp.fc1.bias', 'blocks.10.mlp.fc2.weight', 'blocks.10.mlp.fc2.bias', 'blocks.11.norm1.weight', 'blocks.11.norm1.bias', 'blocks.11.attn.qkv.weight', 'blocks.11.attn.qkv.bias', 'blocks.11.attn.proj.weight', 'blocks.11.attn.proj.bias', 'blocks.11.norm2.weight', 'blocks.11.norm2.bias', 'blocks.11.mlp.fc1.weight', 'blocks.11.mlp.fc1.bias', 'blocks.11.mlp.fc2.weight', 'blocks.11.mlp.fc2.bias', 'norm.weight', 'norm.bias', 'head.weight', 'head.bias'], unexpected_keys=['epoch', 'accuracy', 'model'])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.load_state_dict(torch.load(\"C:/Users/seonahryu/Desktop/brp1/deit_tiny_patch16_224-a1311bcf.pth\"), strict=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " (norm): Identity()\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (patch_drop): Identity()\n", + " (norm_pre): Identity()\n", + " (blocks): Sequential(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (fc_norm): Identity()\n", + " (head_drop): Dropout(p=0.0, inplace=False)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "import torch.nn as nn\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "from thop import profile # FLOPs\n", + "\n", + "# 모델 FLOPs 계산 함수\n", + "def compute_flops(model, input_size=(1, 3, 224, 224)):\n", + " flops, params = profile(model, inputs=(torch.randn(input_size),))\n", + " return flops\n", + "\n", + "# 모델 평가 함수 정의\n", + "def evaluate_model(model, dataloader):\n", + " model.eval()\n", + " correct = 0\n", + " total = 0\n", + " loss_fn = nn.CrossEntropyLoss()\n", + "\n", + " start_time = time.time() # 평가 시작 시간 기록\n", + "\n", + " with torch.no_grad():\n", + " for images, labels in dataloader:\n", + " images, labels = images.to('cuda' if torch.cuda.is_available() else 'cpu'), labels.to('cuda' if torch.cuda.is_available() else 'cpu')\n", + " outputs = model(images)\n", + " _, predicted = torch.max(outputs.data, 1)\n", + " total += labels.size(0)\n", + " correct += (predicted == labels).sum().item()\n", + " loss = loss_fn(outputs, labels)\n", + "\n", + " end_time = time.time() # 평가 종료 시간 기록\n", + " elapsed_time = end_time - start_time # 총 소요 시간 계산\n", + "\n", + " accuracy = 100 * correct / total\n", + " print(f'Accuracy: {accuracy:.3f}%')\n", + " print(f'Loss: {loss.item():.3f}')\n", + " \n", + " # 이미지 처리 속도 계산\n", + " im_per_sec = total / elapsed_time\n", + " print(f'Processing speed: {im_per_sec:.2f} images/sec')\n", + " \n", + " # FLOPs 계산\n", + " flops = compute_flops(model)\n", + " print(f'FLOPs: {flops:.2f}')" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 69.956%\n", + "Loss: 2.464\n", + "Processing speed: 26.88 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "[INFO] Register zero_ops() for .\n", + "FLOPs: 1078633728.00\n" + ] + } + ], + "source": [ + "evaluate_model(model, test_loader)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 5aef8d0d6002ebd4ef7e8dc71baa6abe18a03afa Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 15 Feb 2025 16:46:01 +0900 Subject: [PATCH 2/7] Add files via upload --- attention_rollout.ipynb | 1453 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1453 insertions(+) create mode 100644 attention_rollout.ipynb diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb new file mode 100644 index 0000000..43cc091 --- /dev/null +++ b/attention_rollout.ipynb @@ -0,0 +1,1453 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "from tqdm import tqdm\n", + "import timm\n", + "assert timm.__version__ == \"0.3.2\"\n", + "from torchvision import transforms\n", + "from PIL import Image\n", + "import requests\n", + "import random\n", + "import numpy as np\n", + "\n", + "# 장치 설정\n", + "device = 'cuda' if torch.cuda.is_available() else 'cpu'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def my_forward_wrapper(attn_obj):\n", + " def my_forward(x):\n", + " B, N, C = x.shape\n", + " qkv = attn_obj.qkv(x).reshape(B, N, 3, attn_obj.num_heads, C // attn_obj.num_heads).permute(2, 0, 3, 1, 4)\n", + " q, k, v = qkv.unbind(0)\n", + "\n", + " attn = (q @ k.transpose(-2, -1)) * attn_obj.scale\n", + " attn = attn.softmax(dim=-1)\n", + " attn = attn_obj.attn_drop(attn)\n", + "\n", + " # attn_map을 저장\n", + " attn_maps.append(attn.detach()) # 주의 가중치를 저장\n", + " attn_obj.cls_attn_map = attn[:, :, 0, 2:] # cls attention map 저장\n", + "\n", + " x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n", + " x = attn_obj.proj(x)\n", + " x = attn_obj.proj_drop(x)\n", + " return x\n", + " return my_forward" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " # 현재 주의 맵의 크기 확인\n", + " current_size = attn_map.size(-1)\n", + " I = torch.eye(max(current_size, I_size)).to(attn_map.device) # 동적 아이덴티티 행렬 생성\n", + "\n", + " if current_size < I_size:\n", + " padding_size = I_size - current_size\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " I_padded = F.pad(I, (0, padding_size, 0, 0)) # 아이덴티티 행렬 패딩\n", + " prod = padded_attn_map + I_padded\n", + " else:\n", + " prod = attn_map + I\n", + "\n", + " attn_rollout.append(prod / prod.sum(dim=-1, keepdim=True)) # 정규화\n", + "\n", + " return attn_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-small" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 384, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=384, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 292.63it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Block 1: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 2: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 3: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 4: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 5: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 6: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 7: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 8: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 9: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 10: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 11: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 12: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "CLS Weight Shape: torch.Size([1, 194])\n", + "Attention Rollout: torch.Size([1, 6, 196, 196])\n", + "Combined CLS Weight Shape: torch.Size([1, 6, 194])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "attn_maps = [] # 전역 리스트를 사용하여 attn_map을 저장\n", + "cls_weights = []\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출\n", + "for block in tqdm(model.blocks):\n", + " wrapped_forward = my_forward_wrapper(block.attn)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성\n", + " x = torch.randn(1, 196, 384) # B=1, N=196, C=384\n", + " output = wrapped_forward(x)\n", + "\n", + " # attn_map이 저장되었는지 확인합니다.\n", + " if attn_maps:\n", + " print(f\"Block {len(attn_maps)}: Attention map collected with shape {attn_maps[-1].shape}\")\n", + " else:\n", + " print(f\"Block {len(attn_maps)}: No attention map collected.\")\n", + "\n", + " # cls_attn_map 저장\n", + " cls_attn_map = block.attn.cls_attn_map.detach() # cls_attn_map 저장\n", + " cls_weights.append(cls_attn_map)\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps:\n", + " attn_map = attn_maps[-1].mean(dim=1).squeeze(0).detach()\n", + "else:\n", + " print(\"attn_maps 내용:\", attn_maps) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "cls_weight = cls_weights[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_cpu = attn_map.cpu()\n", + "cls_weight_cpu = cls_weight.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_combined = torch.prod(torch.stack(cls_weights), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_prod = torch.prod(torch.stack(attn_maps), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_cpu = [attn_map.cpu() for attn_map in attn_maps]\n", + "cls_weights_cpu = [cls_weight.cpu() for cls_weight in cls_weights]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 12003.73it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Attention Rollout Length: 12\n", + "Attention Rollout Shapes: [torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196])]\n", + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " attn_rollout = []\n", + " I = torch.eye(attn_maps[0].shape[-1]) # Identity matrix\n", + " prod = I\n", + " for i, attn_map in enumerate(attn_maps):\n", + " prod = prod @ (attn_map + I) # Product of attention maps with identity matrix\n", + " \n", + " prod = prod / prod.sum(dim=-1, keepdim=True) # Normalize\n", + " attn_rollout.append(prod)\n", + " return attn_rollout\n", + "\n", + "# Attention Rollout\n", + "attn_rollout = attention_rollout_function(attn_maps_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout = []\n", + "\n", + "for i in tqdm(range(len(attn_rollout))): # attn_rollout의 크기로 반복\n", + " cls_weight_tensor = attn_rollout[i][0, 0, :, :] # 첫 번째 헤드 사용\n", + "\n", + " # 텐서의 크기를 출력하여 디버깅\n", + " print(f\"Class weight tensor shape before view: {cls_weight_tensor.shape}\")\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인합니다.\n", + " if cls_weight_tensor.numel() == 14 * 14:\n", + " cls_weights_rollout.append(cls_weight_tensor.view(14, 14))\n", + " elif cls_weight_tensor.numel() == 196 * 196:\n", + " # 196x196 텐서를 클래스 가중치로 사용\n", + " cls_weights_rollout.append(cls_weight_tensor) # 변환하지 않고 그대로 사용\n", + " else:\n", + " print(f\"Skipping view for index {i}, tensor size is {cls_weight_tensor.numel()}\")\n", + "\n", + "# 결과 확인\n", + "if isinstance(attn_rollout, list):\n", + " print(\"Attention Rollout Length:\", len(attn_rollout))\n", + " print(\"Attention Rollout Shapes:\", [tensor.shape for tensor in attn_rollout])\n", + "else:\n", + " print(\"Attention Rollout Shape:\", attn_rollout.shape)\n", + "\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0092, 0.0108, 0.0016, ..., 0.0033, 0.0031, 0.0041],\n", + " [0.0025, 0.0249, 0.0017, ..., 0.0045, 0.0037, 0.0045],\n", + " [0.0022, 0.0028, 0.0083, ..., 0.0025, 0.0050, 0.0159],\n", + " ...,\n", + " [0.0022, 0.0089, 0.0015, ..., 0.0199, 0.0061, 0.0046],\n", + " [0.0014, 0.0043, 0.0024, ..., 0.0043, 0.0238, 0.0065],\n", + " [0.0042, 0.0048, 0.0018, ..., 0.0042, 0.0047, 0.0244]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0028, 0.0103, 0.0145, 0.0100, 0.0049, 0.0047, 0.0083, 0.0101, 0.0075,\n", + " 0.0024, 0.0125, 0.0012, 0.0042, 0.0037, 0.0073, 0.0071, 0.0119, 0.0110,\n", + " 0.0049, 0.0063, 0.0066, 0.0083, 0.0065, 0.0068, 0.0030, 0.0534, 0.0020,\n", + " 0.0335, 0.0037, 0.0035, 0.0532, 0.0121, 0.0040, 0.0036, 0.0115, 0.0019,\n", + " 0.0060, 0.0106, 0.0159, 0.0096, 0.0117, 0.0069, 0.0179, 0.0196, 0.0382,\n", + " 0.0029, 0.0084, 0.0023, 0.0139, 0.0022, 0.0152, 0.0044, 0.0024, 0.0139,\n", + " 0.0097, 0.0583, 0.0025, 0.0032, 0.0253, 0.0018, 0.0032, 0.0034, 0.0056,\n", + " 0.0191, 0.0047, 0.0171, 0.0095, 0.0093, 0.0037, 0.0014, 0.0151, 0.0051,\n", + " 0.0043, 0.0063, 0.0054, 0.0231, 0.0095, 0.0082, 0.0100, 0.0046, 0.0108,\n", + " 0.0045, 0.0025, 0.0169, 0.0107, 0.0023, 0.0054, 0.0060, 0.0040, 0.0103,\n", + " 0.0049, 0.0091, 0.0056, 0.0041, 0.1168, 0.0315, 0.0022, 0.0107, 0.0085,\n", + " 0.0040, 0.0037, 0.0197, 0.0153, 0.0179, 0.0044, 0.0178, 0.0102, 0.0677,\n", + " 0.0103, 0.0061, 0.0065, 0.0028, 0.0051, 0.0014, 0.0034, 0.0241, 0.0049,\n", + " 0.0142, 0.0090, 0.0042, 0.0320, 0.0069, 0.0044, 0.0044, 0.0182, 0.0101,\n", + " 0.0042, 0.0026, 0.1099, 0.0097, 0.0284, 0.0165, 0.1040, 0.0059, 0.0176,\n", + " 0.0036, 0.0107, 0.0176, 0.0098, 0.0038, 0.0078, 0.0069, 0.0027, 0.0309,\n", + " 0.0057, 0.0041, 0.0194, 0.0011, 0.0053, 0.0026, 0.0038, 0.0306, 0.0064,\n", + " 0.0205, 0.0032, 0.0127, 0.0057, 0.0024, 0.0022, 0.0252, 0.0051, 0.0481,\n", + " 0.0108, 0.0292, 0.0301, 0.0071, 0.0176, 0.0060, 0.0037, 0.0065, 0.0056,\n", + " 0.0025, 0.0069, 0.0224, 0.0032, 0.0108, 0.0104, 0.0012, 0.0050, 0.0022,\n", + " 0.0089, 0.0038, 0.0131, 0.0068, 0.0124, 0.0128, 0.0195, 0.0075, 0.0050,\n", + " 0.0138, 0.0089, 0.0084, 0.0060, 0.0109]])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[5.0145e-01, 1.0773e-04, 1.9219e-09, ..., 3.0224e-07, 4.8908e-08,\n", + " 9.4525e-09],\n", + " [1.7469e-07, 7.3549e-01, 1.9689e-09, ..., 1.5312e-05, 1.8353e-07,\n", + " 9.8260e-07],\n", + " [2.0269e-04, 8.8949e-03, 7.8070e-01, ..., 8.4695e-05, 3.8640e-05,\n", + " 1.5120e-02],\n", + " ...,\n", + " [3.1366e-08, 1.4511e-03, 1.2052e-10, ..., 5.2665e-01, 1.4668e-05,\n", + " 1.2199e-06],\n", + " [6.7066e-11, 3.5859e-06, 1.0056e-12, ..., 1.3806e-06, 5.0115e-01,\n", + " 6.2253e-07],\n", + " [2.0210e-10, 8.3169e-06, 4.0810e-09, ..., 2.4031e-07, 3.3919e-06,\n", + " 5.2196e-01]]),\n", + " tensor([[2.8310e-01, 7.6627e-04, 1.8418e-03, ..., 1.2459e-04, 8.3037e-03,\n", + " 8.7034e-04],\n", + " [3.8865e-04, 3.8519e-01, 1.1187e-03, ..., 5.4312e-04, 1.4127e-02,\n", + " 8.7212e-04],\n", + " [2.2728e-04, 4.7906e-03, 7.5910e-01, ..., 1.1250e-04, 3.2742e-03,\n", + " 9.5180e-03],\n", + " ...,\n", + " [2.5900e-04, 1.3777e-03, 2.0527e-03, ..., 4.1472e-01, 1.6249e-03,\n", + " 2.4871e-04],\n", + " [2.2064e-04, 5.9052e-04, 1.7989e-03, ..., 1.3416e-04, 4.2058e-01,\n", + " 4.4896e-04],\n", + " [1.6595e-04, 5.9181e-04, 1.7374e-03, ..., 1.4127e-04, 6.7444e-02,\n", + " 3.2628e-01]]),\n", + " tensor([[0.1570, 0.0010, 0.0056, ..., 0.0006, 0.0064, 0.0009],\n", + " [0.0135, 0.1933, 0.0032, ..., 0.0010, 0.0105, 0.0013],\n", + " [0.0069, 0.0031, 0.3877, ..., 0.0009, 0.0040, 0.0054],\n", + " ...,\n", + " [0.0108, 0.0015, 0.0052, ..., 0.2111, 0.0023, 0.0007],\n", + " [0.0091, 0.0012, 0.0048, ..., 0.0011, 0.2260, 0.0007],\n", + " [0.0097, 0.0011, 0.0052, ..., 0.0007, 0.0395, 0.1640]]),\n", + " tensor([[0.1081, 0.0055, 0.0075, ..., 0.0018, 0.0058, 0.0013],\n", + " [0.0112, 0.1038, 0.0062, ..., 0.0022, 0.0086, 0.0024],\n", + " [0.0063, 0.0048, 0.1985, ..., 0.0013, 0.0051, 0.0036],\n", + " ...,\n", + " [0.0089, 0.0055, 0.0061, ..., 0.1082, 0.0036, 0.0019],\n", + " [0.0075, 0.0059, 0.0061, ..., 0.0019, 0.1328, 0.0011],\n", + " [0.0081, 0.0053, 0.0061, ..., 0.0017, 0.0251, 0.0829]]),\n", + " tensor([[0.0596, 0.0034, 0.0052, ..., 0.0040, 0.0040, 0.0009],\n", + " [0.0106, 0.0527, 0.0047, ..., 0.0043, 0.0053, 0.0014],\n", + " [0.0083, 0.0031, 0.1010, ..., 0.0042, 0.0034, 0.0020],\n", + " ...,\n", + " [0.0104, 0.0036, 0.0045, ..., 0.0581, 0.0028, 0.0011],\n", + " [0.0099, 0.0036, 0.0045, ..., 0.0044, 0.0673, 0.0008],\n", + " [0.0101, 0.0034, 0.0049, ..., 0.0041, 0.0135, 0.0417]]),\n", + " tensor([[0.0315, 0.0021, 0.0068, ..., 0.0030, 0.0032, 0.0032],\n", + " [0.0069, 0.0267, 0.0069, ..., 0.0031, 0.0038, 0.0035],\n", + " [0.0055, 0.0019, 0.0546, ..., 0.0032, 0.0028, 0.0039],\n", + " ...,\n", + " [0.0069, 0.0022, 0.0068, ..., 0.0300, 0.0025, 0.0039],\n", + " [0.0066, 0.0022, 0.0065, ..., 0.0033, 0.0347, 0.0031],\n", + " [0.0067, 0.0021, 0.0068, ..., 0.0031, 0.0079, 0.0236]]),\n", + " tensor([[0.0174, 0.0023, 0.0059, ..., 0.0040, 0.0037, 0.0039],\n", + " [0.0049, 0.0145, 0.0060, ..., 0.0040, 0.0040, 0.0042],\n", + " [0.0042, 0.0021, 0.0310, ..., 0.0039, 0.0035, 0.0043],\n", + " ...,\n", + " [0.0050, 0.0022, 0.0059, ..., 0.0175, 0.0033, 0.0044],\n", + " [0.0048, 0.0023, 0.0058, ..., 0.0041, 0.0198, 0.0039],\n", + " [0.0049, 0.0022, 0.0060, ..., 0.0041, 0.0060, 0.0143]]),\n", + " tensor([[0.0093, 0.0036, 0.0040, ..., 0.0063, 0.0032, 0.0035],\n", + " [0.0031, 0.0097, 0.0041, ..., 0.0063, 0.0034, 0.0036],\n", + " [0.0027, 0.0035, 0.0166, ..., 0.0062, 0.0031, 0.0037],\n", + " ...,\n", + " [0.0031, 0.0036, 0.0040, ..., 0.0135, 0.0030, 0.0038],\n", + " [0.0030, 0.0036, 0.0040, ..., 0.0063, 0.0113, 0.0034],\n", + " [0.0031, 0.0036, 0.0041, ..., 0.0063, 0.0044, 0.0090]]),\n", + " tensor([[0.0069, 0.0032, 0.0029, ..., 0.0067, 0.0042, 0.0037],\n", + " [0.0038, 0.0062, 0.0029, ..., 0.0067, 0.0042, 0.0038],\n", + " [0.0035, 0.0031, 0.0092, ..., 0.0067, 0.0042, 0.0038],\n", + " ...,\n", + " [0.0038, 0.0032, 0.0029, ..., 0.0103, 0.0041, 0.0039],\n", + " [0.0037, 0.0031, 0.0029, ..., 0.0067, 0.0082, 0.0037],\n", + " [0.0037, 0.0032, 0.0029, ..., 0.0067, 0.0047, 0.0065]]),\n", + " tensor([[0.0074, 0.0034, 0.0031, ..., 0.0049, 0.0035, 0.0039],\n", + " [0.0058, 0.0049, 0.0031, ..., 0.0049, 0.0036, 0.0040],\n", + " [0.0058, 0.0034, 0.0063, ..., 0.0049, 0.0036, 0.0040],\n", + " ...,\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0067, 0.0035, 0.0040],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0048, 0.0056, 0.0039],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0049, 0.0038, 0.0053]]),\n", + " tensor([[0.0092, 0.0024, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0032, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0062, ..., 0.0049, 0.0041, 0.0086],\n", + " ...,\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0058, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0051, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0042, 0.0093]]),\n", + " tensor([[0.0056, 0.0037, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0052, 0.0040, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0040, ..., 0.0038, 0.0035, 0.0080],\n", + " ...,\n", + " [0.0052, 0.0037, 0.0032, ..., 0.0043, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0032, ..., 0.0038, 0.0040, 0.0080],\n", + " [0.0051, 0.0037, 0.0032, ..., 0.0038, 0.0036, 0.0084]])]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-tiny -> student model로 가정" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)\n", + "model_st" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model_st.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 266.64it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collected attention maps: 12\n", + "CLS Weight Shape: torch.Size([1])\n", + "Attention Rollout: torch.Size([1, 196, 192])\n", + "Combined CLS Weight Shape: torch.Size([1, 196])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# 전역 리스트를 사용하여 attn_map과 cls_weight 저장\n", + "attn_maps_st = []\n", + "cls_weights_st = []\n", + "\n", + "# Forward hook을 설정하는 함수\n", + "def get_attention_map(module, input, output):\n", + " attn_maps_st.append(output.detach()) # 주의 맵 저장\n", + " cls_attn_map = output[:, :, 0] # CLS token에 대한 attention map을 추출\n", + " cls_weights_st.append(cls_attn_map.detach()) # 클래스 주의 맵 저장\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출합니다.\n", + "for block in tqdm(model_st.blocks):\n", + " # Forward hook 등록\n", + " hook = block.attn.register_forward_hook(get_attention_map)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성합니다.\n", + " x = torch.randn(1, 3, 224, 224).to(device) # B=1, C=3, H=224, W=224\n", + " x_patch = model_st.patch_embed(x) # 패치 임베딩 변환\n", + "\n", + " # wrapped_forward 함수로 출력 계산\n", + " output = block.attn(x_patch)\n", + "\n", + " # hook 해제\n", + " hook.remove()\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps_st:\n", + " attn_map_st = attn_maps_st[-1].mean(dim=1).squeeze(0).detach()\n", + " print(f\"Collected attention maps: {len(attn_maps_st)}\")\n", + "else:\n", + " print(\"attn_maps_st 내용:\", attn_maps_st) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "# cls_weight 계산\n", + "cls_weight_st = cls_weights_st[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_st_cpu = attn_map_st.cpu()\n", + "cls_weight_st_cpu = cls_weight_st.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight_st.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_st_combined = torch.prod(torch.stack(cls_weights_st), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_st_prod = torch.prod(torch.stack(attn_maps_st), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_st_cpu = [attn_map.cpu() for attn_map in attn_maps_st]\n", + "cls_weights_st_cpu = [cls_weight.cpu() for cls_weight in cls_weights_st]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_st_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_st_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Attention Rollout\n", + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " I = torch.eye(I_size).to(attn_maps[0].device) # Identity matrix\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " if attn_map.size(-1) == I_size:\n", + " prod = attn_map + I\n", + " elif attn_map.size(-1) < I_size:\n", + " padding_size = I_size - attn_map.size(-1)\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " prod = padded_attn_map + I\n", + " else:\n", + " raise ValueError(\"Unexpected attention map size.\")\n", + "\n", + " attn_rollout.append(prod)\n", + "\n", + " attn_rollout = [prod / prod.sum(dim=-1, keepdim=True) for prod in attn_rollout] # Normalize\n", + " return attn_rollout" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + } + ], + "source": [ + "# Attention Rollout 실행\n", + "attn_rollout_st = attention_rollout_function(attn_maps_st_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout_st = []\n", + "\n", + "for i in range(len(attn_rollout_st)): # attn_rollout_st의 크기로 반복\n", + " cls_weight_tensor_st = attn_rollout_st[i][0] # 첫 번째 배치의 텐서 사용\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인\n", + " if cls_weight_tensor_st.numel() == 14 * 14:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st.view(14, 14))\n", + " elif cls_weight_tensor_st.numel() == 196 * 196:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st) # 그대로 사용\n", + "\n", + "# 결과 확인\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout_st])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0.1842, 0.6513, 0.9481, -0.0575, -0.4596, 0.9378, -0.0630, 0.0913,\n", + " -0.2760, 0.8236, -0.3048, -0.5067, -0.2237, 0.0104, 0.2620, 0.4669,\n", + " 0.3408, -0.6330, -0.4091, -0.4896, -0.9567, -1.5660, -0.2059, -0.2811,\n", + " -0.2813, -0.8560, -0.1496, -0.0455, -0.0833, 0.3670, 0.2738, -0.3592,\n", + " 0.3951, 0.6094, 0.5716, -0.0526, 0.3808, -0.0236, -0.7082, 0.1859,\n", + " 0.8467, -0.0996, 0.4303, 0.0613, 0.7428, 0.1889, -0.0631, -0.1368,\n", + " 0.1214, 0.5921, 0.1423, -0.4978, 0.5646, -0.2145, 0.1315, -0.3321,\n", + " 0.1872, -0.0260, -0.5459, 0.0105, 0.7164, -0.4216, -0.2821, 0.2232,\n", + " -0.1487, 0.1884, -0.1680, 0.3930, -0.3026, 0.8395, -0.5283, -0.1191,\n", + " 0.5188, -1.0522, 0.4154, -0.1342, -0.8908, 1.1572, 0.0318, 0.0403,\n", + " -0.4454, 0.0902, -0.5127, 0.6663, -0.3861, 0.5882, 0.0156, 0.5951,\n", + " -0.7754, 0.4059, 0.1418, 0.9044, -1.3180, -0.1740, 0.9519, -0.7662,\n", + " -0.1940, -0.4566, 0.9794, 0.2629, 0.3623, -0.1166, 0.5362, -0.2732,\n", + " -0.5911, -0.4877, 0.0517, 0.5354, 0.2801, -0.0113, 0.1709, -0.7622,\n", + " -0.1145, -0.9085, 0.3520, -0.6743, 0.2920, 0.2248, -0.2348, -0.4925,\n", + " 0.1487, -0.0883, -0.2174, -0.1746, -0.0644, -0.1431, 0.5035, -0.4010,\n", + " -0.1813, -0.0585, -0.1536, 0.7160, -0.5562, -0.1020, 0.2226, 0.2361,\n", + " 0.5698, -0.2106, 0.9990, -1.1657, -1.0541, 0.4736, 0.7887, 0.2140,\n", + " -0.4918, -0.7655, 1.2153, -0.0707, -0.4830, 0.2197, 0.0948, 0.8601,\n", + " 0.0194, 0.3300, -0.4195, 0.2778, -0.2720, -0.9317, -0.2371, 0.2978,\n", + " -0.3443, 1.3332, -0.0444, -0.4376, -0.2210, -0.0033, -0.0126, -0.0402,\n", + " -1.3693, -0.4690, 0.2604, 0.3911, -0.3407, 0.6645, -0.1561, -0.1206,\n", + " 0.4401, -0.1128, 0.3281, 0.3493, 0.4086, 0.2105, -0.1624, -0.0317,\n", + " 0.2122, 0.2028, 0.5176, -0.5969, -0.6693, 0.1342, -0.5071, -0.3850])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map_st" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.2606])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight_st" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[-0.1554, -0.0586, -0.0729, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.0529, -0.2460, -0.0994, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1179, -0.0144, -0.2249, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0701, -0.1249, -0.0585, ..., -0.1516, -0.0000, -0.0000],\n", + " [ 0.0249, -0.1786, -0.0892, ..., -0.0000, -0.1801, -0.0000],\n", + " [-0.1276, -0.1296, -0.0653, ..., -0.0000, -0.0000, -0.1503]]),\n", + " tensor([[-0.5720, 0.0253, -0.8815, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1371, -0.5842, -0.8806, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 0.0658, -0.2696, -1.0837, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0265, -0.1658, -0.9726, ..., -0.4945, -0.0000, -0.0000],\n", + " [-0.0341, -0.0129, -0.8782, ..., -0.0000, -0.4722, -0.0000],\n", + " [-0.0291, 0.0190, -1.0195, ..., -0.0000, -0.0000, -0.5399]]),\n", + " tensor([[ -5.2143, -0.2073, -9.1526, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.0086, -6.6903, -8.8212, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.6930, -0.4107, -17.7767, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [ 1.9814, -0.8107, -13.0110, ..., -8.9967, -0.0000, -0.0000],\n", + " [ 0.9145, -0.1184, -6.4268, ..., -0.0000, -4.5341, -0.0000],\n", + " [ 1.8620, -1.2088, -13.1969, ..., -0.0000, -0.0000, -8.9710]]),\n", + " tensor([[ 0.4234, -0.1738, 0.4660, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0612, 0.4136, 0.4037, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0573, -0.1247, 0.9858, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0938, -0.0743, 0.4447, ..., 0.5810, 0.0000, 0.0000],\n", + " [-0.1542, -0.1397, 0.5147, ..., 0.0000, 0.5836, 0.0000],\n", + " [-0.1097, -0.1474, 0.4535, ..., 0.0000, 0.0000, 0.5374]]),\n", + " tensor([[ 0.9406, -0.2136, 0.8167, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1652, 0.5770, 0.8471, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1347, -0.2466, 1.5981, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.1693, -0.2130, 0.8331, ..., 0.8538, 0.0000, 0.0000],\n", + " [ 0.1927, -0.2818, 0.8108, ..., 0.0000, 0.8053, 0.0000],\n", + " [ 0.1685, -0.2378, 0.8006, ..., 0.0000, 0.0000, 0.7944]]),\n", + " tensor([[ 0.7328, 0.0801, 0.5444, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0403, 0.7458, 0.5566, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0786, 0.0814, 1.3531, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0527, 0.0911, 0.6341, ..., 0.6709, 0.0000, 0.0000],\n", + " [-0.0836, 0.1107, 0.5707, ..., 0.0000, 0.7083, 0.0000],\n", + " [-0.0249, 0.0901, 0.5255, ..., 0.0000, 0.0000, 0.6873]]),\n", + " tensor([[0.8320, 0.2851, 0.8615, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1175, 0.9609, 0.8269, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1498, 0.2262, 1.6439, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [0.1284, 0.2232, 0.8748, ..., 0.7061, 0.0000, 0.0000],\n", + " [0.1243, 0.2083, 0.9053, ..., 0.0000, 0.7440, 0.0000],\n", + " [0.1109, 0.2427, 0.8597, ..., 0.0000, 0.0000, 0.6755]]),\n", + " tensor([[ 0.7119, 0.2967, 0.5289, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1194, 1.2095, 0.6030, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1917, 0.3107, 1.4640, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.1499, 0.3103, 0.6443, ..., 0.8962, 0.0000, 0.0000],\n", + " [-0.1631, 0.3251, 0.5534, ..., 0.0000, 0.8595, 0.0000],\n", + " [-0.1243, 0.2940, 0.6102, ..., 0.0000, 0.0000, 0.8932]]),\n", + " tensor([[ 3.0915, -0.7572, 1.6884, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8447, 1.3794, 1.6422, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8184, -0.8285, 3.8009, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.7615, -0.8503, 1.5874, ..., 2.2260, 0.0000, 0.0000],\n", + " [ 0.8272, -0.8181, 1.6270, ..., 0.0000, 2.1608, 0.0000],\n", + " [ 0.8811, -0.7469, 1.4999, ..., 0.0000, 0.0000, 2.1590]]),\n", + " tensor([[ 2.4223, -0.3267, 0.9755, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.2824, 1.6931, 0.8163, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.3807, -0.1291, 2.9314, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.3839, -0.2936, 0.9776, ..., 2.0073, 0.0000, 0.0000],\n", + " [ 0.4626, -0.2595, 0.9444, ..., 0.0000, 2.0228, 0.0000],\n", + " [ 0.4498, -0.2101, 0.9627, ..., 0.0000, 0.0000, 1.8273]]),\n", + " tensor([[15.2405, 2.7610, 6.7306, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 9.2874, 22.5292, 11.1778, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 6.0527, 2.1433, 18.1478, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 8.6357, 3.6522, 9.4287, ..., 15.7923, 0.0000, 0.0000],\n", + " [ 4.7526, 2.0569, 6.1020, ..., 0.0000, 8.8777, 0.0000],\n", + " [ 3.7997, 1.8325, 4.7829, ..., 0.0000, 0.0000, 7.1644]]),\n", + " tensor([[ 3.2594, 1.9875, 2.6364, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.8766, 17.1216, 9.5131, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.7089, 5.7129, 15.9202, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.4791, 2.2901, 3.2946, ..., 3.3042, 0.0000, 0.0000],\n", + " [ 0.7041, 3.5211, 4.8167, ..., 0.0000, 5.4057, 0.0000],\n", + " [ 1.7779, 8.7794, 10.9012, ..., 0.0000, 0.0000, 12.3839]])]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout_st" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Loss: 453.0188903808594\n", + "Layer 0 Loss: 0.0445, Percentage of Total Loss: 0.0098%\n", + "Layer 1 Loss: 0.0991, Percentage of Total Loss: 0.0219%\n", + "Layer 2 Loss: 394.6398, Percentage of Total Loss: 87.1133%\n", + "Layer 3 Loss: 0.0411, Percentage of Total Loss: 0.0091%\n", + "Layer 4 Loss: 0.0765, Percentage of Total Loss: 0.0169%\n", + "Layer 5 Loss: 0.0602, Percentage of Total Loss: 0.0133%\n", + "Layer 6 Loss: 0.0887, Percentage of Total Loss: 0.0196%\n", + "Layer 7 Loss: 0.0799, Percentage of Total Loss: 0.0176%\n", + "Layer 8 Loss: 0.5019, Percentage of Total Loss: 0.1108%\n", + "Layer 9 Loss: 0.5323, Percentage of Total Loss: 0.1175%\n", + "Layer 10 Loss: 38.7059, Percentage of Total Loss: 8.5440%\n", + "Layer 11 Loss: 18.1490, Percentage of Total Loss: 4.0062%\n", + "Average Layer Loss: 37.7516\n", + "Minimum Layer Loss: 0.0411\n", + "Maximum Layer Loss: 394.6398\n" + ] + } + ], + "source": [ + "# 손실 계산\n", + "loss = 0\n", + "layer_losses = [] # 레이어별 손실을 저장할 리스트\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = F.mse_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss = loss.item()\n", + "print(f\"Total Loss: {total_loss}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages = [(layer_loss / total_loss) * 100 for layer_loss in layer_losses]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses):\n", + " print(f\"Layer {i} Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages[i]:.4f}%\")\n", + "\n", + "# 추가 정보 출력\n", + "average_loss = sum(layer_losses) / len(layer_losses)\n", + "min_loss = min(layer_losses)\n", + "max_loss = max(layer_losses)\n", + "\n", + "print(f\"Average Layer Loss: {average_loss:.4f}\")\n", + "print(f\"Minimum Layer Loss: {min_loss:.4f}\")\n", + "print(f\"Maximum Layer Loss: {max_loss:.4f}\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 22267a60e56ed58b732df3f21b9dbf3db6cbd254 Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 15 Feb 2025 17:08:08 +0900 Subject: [PATCH 3/7] Delete attention_rollout.ipynb --- attention_rollout.ipynb | 1453 --------------------------------------- 1 file changed, 1453 deletions(-) delete mode 100644 attention_rollout.ipynb diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb deleted file mode 100644 index 43cc091..0000000 --- a/attention_rollout.ipynb +++ /dev/null @@ -1,1453 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import torch\n", - "import torch.nn.functional as F\n", - "from tqdm import tqdm\n", - "import timm\n", - "assert timm.__version__ == \"0.3.2\"\n", - "from torchvision import transforms\n", - "from PIL import Image\n", - "import requests\n", - "import random\n", - "import numpy as np\n", - "\n", - "# 장치 설정\n", - "device = 'cuda' if torch.cuda.is_available() else 'cpu'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from torch.utils.data import DataLoader\n", - "from torchvision.datasets import ImageFolder\n", - "import torchvision.transforms as transforms\n", - "\n", - "# 데이터 전처리\n", - "transform = transforms.Compose([\n", - " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", - "])\n", - "\n", - "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", - "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "def my_forward_wrapper(attn_obj):\n", - " def my_forward(x):\n", - " B, N, C = x.shape\n", - " qkv = attn_obj.qkv(x).reshape(B, N, 3, attn_obj.num_heads, C // attn_obj.num_heads).permute(2, 0, 3, 1, 4)\n", - " q, k, v = qkv.unbind(0)\n", - "\n", - " attn = (q @ k.transpose(-2, -1)) * attn_obj.scale\n", - " attn = attn.softmax(dim=-1)\n", - " attn = attn_obj.attn_drop(attn)\n", - "\n", - " # attn_map을 저장\n", - " attn_maps.append(attn.detach()) # 주의 가중치를 저장\n", - " attn_obj.cls_attn_map = attn[:, :, 0, 2:] # cls attention map 저장\n", - "\n", - " x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n", - " x = attn_obj.proj(x)\n", - " x = attn_obj.proj_drop(x)\n", - " return x\n", - " return my_forward" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def attention_rollout_function(attn_maps):\n", - " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", - " attn_rollout = []\n", - "\n", - " for attn_map in attn_maps:\n", - " # 현재 주의 맵의 크기 확인\n", - " current_size = attn_map.size(-1)\n", - " I = torch.eye(max(current_size, I_size)).to(attn_map.device) # 동적 아이덴티티 행렬 생성\n", - "\n", - " if current_size < I_size:\n", - " padding_size = I_size - current_size\n", - " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", - " I_padded = F.pad(I, (0, padding_size, 0, 0)) # 아이덴티티 행렬 패딩\n", - " prod = padded_attn_map + I_padded\n", - " else:\n", - " prod = attn_map + I\n", - "\n", - " attn_rollout.append(prod / prod.sum(dim=-1, keepdim=True)) # 정규화\n", - "\n", - " return attn_rollout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "DeiT-small" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" - ] - }, - { - "data": { - "text/plain": [ - "VisionTransformer(\n", - " (patch_embed): PatchEmbed(\n", - " (proj): Conv2d(3, 384, kernel_size=(16, 16), stride=(16, 16))\n", - " )\n", - " (pos_drop): Dropout(p=0.0, inplace=False)\n", - " (blocks): ModuleList(\n", - " (0): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (1): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (2): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (3): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (4): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (5): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (6): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (7): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (8): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (9): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (10): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (11): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (norm): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (head): Linear(in_features=384, out_features=1000, bias=True)\n", - ")" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)\n", - "model" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of blocks in the model: 12\n" - ] - } - ], - "source": [ - "print(f\"Number of blocks in the model: {len(model.blocks)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.eval()\n", - "\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 12/12 [00:00<00:00, 292.63it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Block 1: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 2: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 3: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 4: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 5: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 6: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 7: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 8: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 9: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 10: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 11: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 12: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "CLS Weight Shape: torch.Size([1, 194])\n", - "Attention Rollout: torch.Size([1, 6, 196, 196])\n", - "Combined CLS Weight Shape: torch.Size([1, 6, 194])\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "attn_maps = [] # 전역 리스트를 사용하여 attn_map을 저장\n", - "cls_weights = []\n", - "\n", - "# 각 블록을 순회하며 attention map과 class attention map을 추출\n", - "for block in tqdm(model.blocks):\n", - " wrapped_forward = my_forward_wrapper(block.attn)\n", - " \n", - " # 입력 텐서가 필요하므로 임의의 입력을 생성\n", - " x = torch.randn(1, 196, 384) # B=1, N=196, C=384\n", - " output = wrapped_forward(x)\n", - "\n", - " # attn_map이 저장되었는지 확인합니다.\n", - " if attn_maps:\n", - " print(f\"Block {len(attn_maps)}: Attention map collected with shape {attn_maps[-1].shape}\")\n", - " else:\n", - " print(f\"Block {len(attn_maps)}: No attention map collected.\")\n", - "\n", - " # cls_attn_map 저장\n", - " cls_attn_map = block.attn.cls_attn_map.detach() # cls_attn_map 저장\n", - " cls_weights.append(cls_attn_map)\n", - "\n", - "# 주의 맵이 저장되었는지 확인합니다.\n", - "if attn_maps:\n", - " attn_map = attn_maps[-1].mean(dim=1).squeeze(0).detach()\n", - "else:\n", - " print(\"attn_maps 내용:\", attn_maps) # 디버깅 출력 추가\n", - " raise ValueError(\"No attention maps were collected.\")\n", - "\n", - "cls_weight = cls_weights[-1].max(dim=1).values.detach()\n", - "\n", - "# Ensure tensors are on the CPU\n", - "attn_map_cpu = attn_map.cpu()\n", - "cls_weight_cpu = cls_weight.cpu()\n", - "\n", - "# 클래스 가중치의 크기를 확인합니다.\n", - "print(\"CLS Weight Shape:\", cls_weight.shape)\n", - "\n", - "# Combine class scores of all blocks\n", - "try:\n", - " cls_weight_combined = torch.prod(torch.stack(cls_weights), dim=0)\n", - "except RuntimeError as e:\n", - " print(f\"Error combining class weights: {e}\")\n", - "\n", - "# 주의 맵의 곱을 계산합니다\n", - "attn_maps_prod = torch.prod(torch.stack(attn_maps), dim=0)\n", - "\n", - "# CPU로 전환\n", - "attn_maps_cpu = [attn_map.cpu() for attn_map in attn_maps]\n", - "cls_weights_cpu = [cls_weight.cpu() for cls_weight in cls_weights]\n", - "\n", - "# 결과 확인\n", - "print(\"Attention Rollout:\", attn_maps_prod.shape)\n", - "print(\"Combined CLS Weight Shape:\", cls_weight_combined.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 12/12 [00:00<00:00, 12003.73it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Attention Rollout Length: 12\n", - "Attention Rollout Shapes: [torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196])]\n", - "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "def attention_rollout_function(attn_maps):\n", - " attn_rollout = []\n", - " I = torch.eye(attn_maps[0].shape[-1]) # Identity matrix\n", - " prod = I\n", - " for i, attn_map in enumerate(attn_maps):\n", - " prod = prod @ (attn_map + I) # Product of attention maps with identity matrix\n", - " \n", - " prod = prod / prod.sum(dim=-1, keepdim=True) # Normalize\n", - " attn_rollout.append(prod)\n", - " return attn_rollout\n", - "\n", - "# Attention Rollout\n", - "attn_rollout = attention_rollout_function(attn_maps_cpu)\n", - "\n", - "# For Class Weights\n", - "cls_weights_rollout = []\n", - "\n", - "for i in tqdm(range(len(attn_rollout))): # attn_rollout의 크기로 반복\n", - " cls_weight_tensor = attn_rollout[i][0, 0, :, :] # 첫 번째 헤드 사용\n", - "\n", - " # 텐서의 크기를 출력하여 디버깅\n", - " print(f\"Class weight tensor shape before view: {cls_weight_tensor.shape}\")\n", - "\n", - " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인합니다.\n", - " if cls_weight_tensor.numel() == 14 * 14:\n", - " cls_weights_rollout.append(cls_weight_tensor.view(14, 14))\n", - " elif cls_weight_tensor.numel() == 196 * 196:\n", - " # 196x196 텐서를 클래스 가중치로 사용\n", - " cls_weights_rollout.append(cls_weight_tensor) # 변환하지 않고 그대로 사용\n", - " else:\n", - " print(f\"Skipping view for index {i}, tensor size is {cls_weight_tensor.numel()}\")\n", - "\n", - "# 결과 확인\n", - "if isinstance(attn_rollout, list):\n", - " print(\"Attention Rollout Length:\", len(attn_rollout))\n", - " print(\"Attention Rollout Shapes:\", [tensor.shape for tensor in attn_rollout])\n", - "else:\n", - " print(\"Attention Rollout Shape:\", attn_rollout.shape)\n", - "\n", - "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout])" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0.0092, 0.0108, 0.0016, ..., 0.0033, 0.0031, 0.0041],\n", - " [0.0025, 0.0249, 0.0017, ..., 0.0045, 0.0037, 0.0045],\n", - " [0.0022, 0.0028, 0.0083, ..., 0.0025, 0.0050, 0.0159],\n", - " ...,\n", - " [0.0022, 0.0089, 0.0015, ..., 0.0199, 0.0061, 0.0046],\n", - " [0.0014, 0.0043, 0.0024, ..., 0.0043, 0.0238, 0.0065],\n", - " [0.0042, 0.0048, 0.0018, ..., 0.0042, 0.0047, 0.0244]])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "attn_map" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0.0028, 0.0103, 0.0145, 0.0100, 0.0049, 0.0047, 0.0083, 0.0101, 0.0075,\n", - " 0.0024, 0.0125, 0.0012, 0.0042, 0.0037, 0.0073, 0.0071, 0.0119, 0.0110,\n", - " 0.0049, 0.0063, 0.0066, 0.0083, 0.0065, 0.0068, 0.0030, 0.0534, 0.0020,\n", - " 0.0335, 0.0037, 0.0035, 0.0532, 0.0121, 0.0040, 0.0036, 0.0115, 0.0019,\n", - " 0.0060, 0.0106, 0.0159, 0.0096, 0.0117, 0.0069, 0.0179, 0.0196, 0.0382,\n", - " 0.0029, 0.0084, 0.0023, 0.0139, 0.0022, 0.0152, 0.0044, 0.0024, 0.0139,\n", - " 0.0097, 0.0583, 0.0025, 0.0032, 0.0253, 0.0018, 0.0032, 0.0034, 0.0056,\n", - " 0.0191, 0.0047, 0.0171, 0.0095, 0.0093, 0.0037, 0.0014, 0.0151, 0.0051,\n", - " 0.0043, 0.0063, 0.0054, 0.0231, 0.0095, 0.0082, 0.0100, 0.0046, 0.0108,\n", - " 0.0045, 0.0025, 0.0169, 0.0107, 0.0023, 0.0054, 0.0060, 0.0040, 0.0103,\n", - " 0.0049, 0.0091, 0.0056, 0.0041, 0.1168, 0.0315, 0.0022, 0.0107, 0.0085,\n", - " 0.0040, 0.0037, 0.0197, 0.0153, 0.0179, 0.0044, 0.0178, 0.0102, 0.0677,\n", - " 0.0103, 0.0061, 0.0065, 0.0028, 0.0051, 0.0014, 0.0034, 0.0241, 0.0049,\n", - " 0.0142, 0.0090, 0.0042, 0.0320, 0.0069, 0.0044, 0.0044, 0.0182, 0.0101,\n", - " 0.0042, 0.0026, 0.1099, 0.0097, 0.0284, 0.0165, 0.1040, 0.0059, 0.0176,\n", - " 0.0036, 0.0107, 0.0176, 0.0098, 0.0038, 0.0078, 0.0069, 0.0027, 0.0309,\n", - " 0.0057, 0.0041, 0.0194, 0.0011, 0.0053, 0.0026, 0.0038, 0.0306, 0.0064,\n", - " 0.0205, 0.0032, 0.0127, 0.0057, 0.0024, 0.0022, 0.0252, 0.0051, 0.0481,\n", - " 0.0108, 0.0292, 0.0301, 0.0071, 0.0176, 0.0060, 0.0037, 0.0065, 0.0056,\n", - " 0.0025, 0.0069, 0.0224, 0.0032, 0.0108, 0.0104, 0.0012, 0.0050, 0.0022,\n", - " 0.0089, 0.0038, 0.0131, 0.0068, 0.0124, 0.0128, 0.0195, 0.0075, 0.0050,\n", - " 0.0138, 0.0089, 0.0084, 0.0060, 0.0109]])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weight" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([[5.0145e-01, 1.0773e-04, 1.9219e-09, ..., 3.0224e-07, 4.8908e-08,\n", - " 9.4525e-09],\n", - " [1.7469e-07, 7.3549e-01, 1.9689e-09, ..., 1.5312e-05, 1.8353e-07,\n", - " 9.8260e-07],\n", - " [2.0269e-04, 8.8949e-03, 7.8070e-01, ..., 8.4695e-05, 3.8640e-05,\n", - " 1.5120e-02],\n", - " ...,\n", - " [3.1366e-08, 1.4511e-03, 1.2052e-10, ..., 5.2665e-01, 1.4668e-05,\n", - " 1.2199e-06],\n", - " [6.7066e-11, 3.5859e-06, 1.0056e-12, ..., 1.3806e-06, 5.0115e-01,\n", - " 6.2253e-07],\n", - " [2.0210e-10, 8.3169e-06, 4.0810e-09, ..., 2.4031e-07, 3.3919e-06,\n", - " 5.2196e-01]]),\n", - " tensor([[2.8310e-01, 7.6627e-04, 1.8418e-03, ..., 1.2459e-04, 8.3037e-03,\n", - " 8.7034e-04],\n", - " [3.8865e-04, 3.8519e-01, 1.1187e-03, ..., 5.4312e-04, 1.4127e-02,\n", - " 8.7212e-04],\n", - " [2.2728e-04, 4.7906e-03, 7.5910e-01, ..., 1.1250e-04, 3.2742e-03,\n", - " 9.5180e-03],\n", - " ...,\n", - " [2.5900e-04, 1.3777e-03, 2.0527e-03, ..., 4.1472e-01, 1.6249e-03,\n", - " 2.4871e-04],\n", - " [2.2064e-04, 5.9052e-04, 1.7989e-03, ..., 1.3416e-04, 4.2058e-01,\n", - " 4.4896e-04],\n", - " [1.6595e-04, 5.9181e-04, 1.7374e-03, ..., 1.4127e-04, 6.7444e-02,\n", - " 3.2628e-01]]),\n", - " tensor([[0.1570, 0.0010, 0.0056, ..., 0.0006, 0.0064, 0.0009],\n", - " [0.0135, 0.1933, 0.0032, ..., 0.0010, 0.0105, 0.0013],\n", - " [0.0069, 0.0031, 0.3877, ..., 0.0009, 0.0040, 0.0054],\n", - " ...,\n", - " [0.0108, 0.0015, 0.0052, ..., 0.2111, 0.0023, 0.0007],\n", - " [0.0091, 0.0012, 0.0048, ..., 0.0011, 0.2260, 0.0007],\n", - " [0.0097, 0.0011, 0.0052, ..., 0.0007, 0.0395, 0.1640]]),\n", - " tensor([[0.1081, 0.0055, 0.0075, ..., 0.0018, 0.0058, 0.0013],\n", - " [0.0112, 0.1038, 0.0062, ..., 0.0022, 0.0086, 0.0024],\n", - " [0.0063, 0.0048, 0.1985, ..., 0.0013, 0.0051, 0.0036],\n", - " ...,\n", - " [0.0089, 0.0055, 0.0061, ..., 0.1082, 0.0036, 0.0019],\n", - " [0.0075, 0.0059, 0.0061, ..., 0.0019, 0.1328, 0.0011],\n", - " [0.0081, 0.0053, 0.0061, ..., 0.0017, 0.0251, 0.0829]]),\n", - " tensor([[0.0596, 0.0034, 0.0052, ..., 0.0040, 0.0040, 0.0009],\n", - " [0.0106, 0.0527, 0.0047, ..., 0.0043, 0.0053, 0.0014],\n", - " [0.0083, 0.0031, 0.1010, ..., 0.0042, 0.0034, 0.0020],\n", - " ...,\n", - " [0.0104, 0.0036, 0.0045, ..., 0.0581, 0.0028, 0.0011],\n", - " [0.0099, 0.0036, 0.0045, ..., 0.0044, 0.0673, 0.0008],\n", - " [0.0101, 0.0034, 0.0049, ..., 0.0041, 0.0135, 0.0417]]),\n", - " tensor([[0.0315, 0.0021, 0.0068, ..., 0.0030, 0.0032, 0.0032],\n", - " [0.0069, 0.0267, 0.0069, ..., 0.0031, 0.0038, 0.0035],\n", - " [0.0055, 0.0019, 0.0546, ..., 0.0032, 0.0028, 0.0039],\n", - " ...,\n", - " [0.0069, 0.0022, 0.0068, ..., 0.0300, 0.0025, 0.0039],\n", - " [0.0066, 0.0022, 0.0065, ..., 0.0033, 0.0347, 0.0031],\n", - " [0.0067, 0.0021, 0.0068, ..., 0.0031, 0.0079, 0.0236]]),\n", - " tensor([[0.0174, 0.0023, 0.0059, ..., 0.0040, 0.0037, 0.0039],\n", - " [0.0049, 0.0145, 0.0060, ..., 0.0040, 0.0040, 0.0042],\n", - " [0.0042, 0.0021, 0.0310, ..., 0.0039, 0.0035, 0.0043],\n", - " ...,\n", - " [0.0050, 0.0022, 0.0059, ..., 0.0175, 0.0033, 0.0044],\n", - " [0.0048, 0.0023, 0.0058, ..., 0.0041, 0.0198, 0.0039],\n", - " [0.0049, 0.0022, 0.0060, ..., 0.0041, 0.0060, 0.0143]]),\n", - " tensor([[0.0093, 0.0036, 0.0040, ..., 0.0063, 0.0032, 0.0035],\n", - " [0.0031, 0.0097, 0.0041, ..., 0.0063, 0.0034, 0.0036],\n", - " [0.0027, 0.0035, 0.0166, ..., 0.0062, 0.0031, 0.0037],\n", - " ...,\n", - " [0.0031, 0.0036, 0.0040, ..., 0.0135, 0.0030, 0.0038],\n", - " [0.0030, 0.0036, 0.0040, ..., 0.0063, 0.0113, 0.0034],\n", - " [0.0031, 0.0036, 0.0041, ..., 0.0063, 0.0044, 0.0090]]),\n", - " tensor([[0.0069, 0.0032, 0.0029, ..., 0.0067, 0.0042, 0.0037],\n", - " [0.0038, 0.0062, 0.0029, ..., 0.0067, 0.0042, 0.0038],\n", - " [0.0035, 0.0031, 0.0092, ..., 0.0067, 0.0042, 0.0038],\n", - " ...,\n", - " [0.0038, 0.0032, 0.0029, ..., 0.0103, 0.0041, 0.0039],\n", - " [0.0037, 0.0031, 0.0029, ..., 0.0067, 0.0082, 0.0037],\n", - " [0.0037, 0.0032, 0.0029, ..., 0.0067, 0.0047, 0.0065]]),\n", - " tensor([[0.0074, 0.0034, 0.0031, ..., 0.0049, 0.0035, 0.0039],\n", - " [0.0058, 0.0049, 0.0031, ..., 0.0049, 0.0036, 0.0040],\n", - " [0.0058, 0.0034, 0.0063, ..., 0.0049, 0.0036, 0.0040],\n", - " ...,\n", - " [0.0058, 0.0034, 0.0031, ..., 0.0067, 0.0035, 0.0040],\n", - " [0.0058, 0.0034, 0.0031, ..., 0.0048, 0.0056, 0.0039],\n", - " [0.0058, 0.0034, 0.0031, ..., 0.0049, 0.0038, 0.0053]]),\n", - " tensor([[0.0092, 0.0024, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", - " [0.0084, 0.0032, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", - " [0.0084, 0.0024, 0.0062, ..., 0.0049, 0.0041, 0.0086],\n", - " ...,\n", - " [0.0084, 0.0024, 0.0046, ..., 0.0058, 0.0041, 0.0086],\n", - " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0051, 0.0086],\n", - " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0042, 0.0093]]),\n", - " tensor([[0.0056, 0.0037, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", - " [0.0052, 0.0040, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", - " [0.0051, 0.0036, 0.0040, ..., 0.0038, 0.0035, 0.0080],\n", - " ...,\n", - " [0.0052, 0.0037, 0.0032, ..., 0.0043, 0.0035, 0.0080],\n", - " [0.0051, 0.0036, 0.0032, ..., 0.0038, 0.0040, 0.0080],\n", - " [0.0051, 0.0037, 0.0032, ..., 0.0038, 0.0036, 0.0084]])]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weights_rollout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "DeiT-tiny -> student model로 가정" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" - ] - }, - { - "data": { - "text/plain": [ - "VisionTransformer(\n", - " (patch_embed): PatchEmbed(\n", - " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", - " )\n", - " (pos_drop): Dropout(p=0.0, inplace=False)\n", - " (blocks): ModuleList(\n", - " (0): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (1): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (2): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (3): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (4): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (5): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (6): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (7): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (8): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (9): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (10): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (11): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (head): Linear(in_features=192, out_features=1000, bias=True)\n", - ")" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model_st = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)\n", - "model_st" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of blocks in the model: 12\n" - ] - } - ], - "source": [ - "print(f\"Number of blocks in the model: {len(model_st.blocks)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model_st.eval()\n", - "\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 12/12 [00:00<00:00, 266.64it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collected attention maps: 12\n", - "CLS Weight Shape: torch.Size([1])\n", - "Attention Rollout: torch.Size([1, 196, 192])\n", - "Combined CLS Weight Shape: torch.Size([1, 196])\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# 전역 리스트를 사용하여 attn_map과 cls_weight 저장\n", - "attn_maps_st = []\n", - "cls_weights_st = []\n", - "\n", - "# Forward hook을 설정하는 함수\n", - "def get_attention_map(module, input, output):\n", - " attn_maps_st.append(output.detach()) # 주의 맵 저장\n", - " cls_attn_map = output[:, :, 0] # CLS token에 대한 attention map을 추출\n", - " cls_weights_st.append(cls_attn_map.detach()) # 클래스 주의 맵 저장\n", - "\n", - "# 각 블록을 순회하며 attention map과 class attention map을 추출합니다.\n", - "for block in tqdm(model_st.blocks):\n", - " # Forward hook 등록\n", - " hook = block.attn.register_forward_hook(get_attention_map)\n", - " \n", - " # 입력 텐서가 필요하므로 임의의 입력을 생성합니다.\n", - " x = torch.randn(1, 3, 224, 224).to(device) # B=1, C=3, H=224, W=224\n", - " x_patch = model_st.patch_embed(x) # 패치 임베딩 변환\n", - "\n", - " # wrapped_forward 함수로 출력 계산\n", - " output = block.attn(x_patch)\n", - "\n", - " # hook 해제\n", - " hook.remove()\n", - "\n", - "# 주의 맵이 저장되었는지 확인합니다.\n", - "if attn_maps_st:\n", - " attn_map_st = attn_maps_st[-1].mean(dim=1).squeeze(0).detach()\n", - " print(f\"Collected attention maps: {len(attn_maps_st)}\")\n", - "else:\n", - " print(\"attn_maps_st 내용:\", attn_maps_st) # 디버깅 출력 추가\n", - " raise ValueError(\"No attention maps were collected.\")\n", - "\n", - "# cls_weight 계산\n", - "cls_weight_st = cls_weights_st[-1].max(dim=1).values.detach()\n", - "\n", - "# Ensure tensors are on the CPU\n", - "attn_map_st_cpu = attn_map_st.cpu()\n", - "cls_weight_st_cpu = cls_weight_st.cpu()\n", - "\n", - "# 클래스 가중치의 크기를 확인합니다.\n", - "print(\"CLS Weight Shape:\", cls_weight_st.shape)\n", - "\n", - "# Combine class scores of all blocks\n", - "try:\n", - " cls_weight_st_combined = torch.prod(torch.stack(cls_weights_st), dim=0)\n", - "except RuntimeError as e:\n", - " print(f\"Error combining class weights: {e}\")\n", - "\n", - "# 주의 맵의 곱을 계산합니다\n", - "attn_maps_st_prod = torch.prod(torch.stack(attn_maps_st), dim=0)\n", - "\n", - "# CPU로 전환\n", - "attn_maps_st_cpu = [attn_map.cpu() for attn_map in attn_maps_st]\n", - "cls_weights_st_cpu = [cls_weight.cpu() for cls_weight in cls_weights_st]\n", - "\n", - "# 결과 확인\n", - "print(\"Attention Rollout:\", attn_maps_st_prod.shape)\n", - "print(\"Combined CLS Weight Shape:\", cls_weight_st_combined.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "# Attention Rollout\n", - "def attention_rollout_function(attn_maps):\n", - " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", - " I = torch.eye(I_size).to(attn_maps[0].device) # Identity matrix\n", - " attn_rollout = []\n", - "\n", - " for attn_map in attn_maps:\n", - " if attn_map.size(-1) == I_size:\n", - " prod = attn_map + I\n", - " elif attn_map.size(-1) < I_size:\n", - " padding_size = I_size - attn_map.size(-1)\n", - " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", - " prod = padded_attn_map + I\n", - " else:\n", - " raise ValueError(\"Unexpected attention map size.\")\n", - "\n", - " attn_rollout.append(prod)\n", - "\n", - " attn_rollout = [prod / prod.sum(dim=-1, keepdim=True) for prod in attn_rollout] # Normalize\n", - " return attn_rollout" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" - ] - } - ], - "source": [ - "# Attention Rollout 실행\n", - "attn_rollout_st = attention_rollout_function(attn_maps_st_cpu)\n", - "\n", - "# For Class Weights\n", - "cls_weights_rollout_st = []\n", - "\n", - "for i in range(len(attn_rollout_st)): # attn_rollout_st의 크기로 반복\n", - " cls_weight_tensor_st = attn_rollout_st[i][0] # 첫 번째 배치의 텐서 사용\n", - "\n", - " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인\n", - " if cls_weight_tensor_st.numel() == 14 * 14:\n", - " cls_weights_rollout_st.append(cls_weight_tensor_st.view(14, 14))\n", - " elif cls_weight_tensor_st.numel() == 196 * 196:\n", - " cls_weights_rollout_st.append(cls_weight_tensor_st) # 그대로 사용\n", - "\n", - "# 결과 확인\n", - "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout_st])" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([ 0.1842, 0.6513, 0.9481, -0.0575, -0.4596, 0.9378, -0.0630, 0.0913,\n", - " -0.2760, 0.8236, -0.3048, -0.5067, -0.2237, 0.0104, 0.2620, 0.4669,\n", - " 0.3408, -0.6330, -0.4091, -0.4896, -0.9567, -1.5660, -0.2059, -0.2811,\n", - " -0.2813, -0.8560, -0.1496, -0.0455, -0.0833, 0.3670, 0.2738, -0.3592,\n", - " 0.3951, 0.6094, 0.5716, -0.0526, 0.3808, -0.0236, -0.7082, 0.1859,\n", - " 0.8467, -0.0996, 0.4303, 0.0613, 0.7428, 0.1889, -0.0631, -0.1368,\n", - " 0.1214, 0.5921, 0.1423, -0.4978, 0.5646, -0.2145, 0.1315, -0.3321,\n", - " 0.1872, -0.0260, -0.5459, 0.0105, 0.7164, -0.4216, -0.2821, 0.2232,\n", - " -0.1487, 0.1884, -0.1680, 0.3930, -0.3026, 0.8395, -0.5283, -0.1191,\n", - " 0.5188, -1.0522, 0.4154, -0.1342, -0.8908, 1.1572, 0.0318, 0.0403,\n", - " -0.4454, 0.0902, -0.5127, 0.6663, -0.3861, 0.5882, 0.0156, 0.5951,\n", - " -0.7754, 0.4059, 0.1418, 0.9044, -1.3180, -0.1740, 0.9519, -0.7662,\n", - " -0.1940, -0.4566, 0.9794, 0.2629, 0.3623, -0.1166, 0.5362, -0.2732,\n", - " -0.5911, -0.4877, 0.0517, 0.5354, 0.2801, -0.0113, 0.1709, -0.7622,\n", - " -0.1145, -0.9085, 0.3520, -0.6743, 0.2920, 0.2248, -0.2348, -0.4925,\n", - " 0.1487, -0.0883, -0.2174, -0.1746, -0.0644, -0.1431, 0.5035, -0.4010,\n", - " -0.1813, -0.0585, -0.1536, 0.7160, -0.5562, -0.1020, 0.2226, 0.2361,\n", - " 0.5698, -0.2106, 0.9990, -1.1657, -1.0541, 0.4736, 0.7887, 0.2140,\n", - " -0.4918, -0.7655, 1.2153, -0.0707, -0.4830, 0.2197, 0.0948, 0.8601,\n", - " 0.0194, 0.3300, -0.4195, 0.2778, -0.2720, -0.9317, -0.2371, 0.2978,\n", - " -0.3443, 1.3332, -0.0444, -0.4376, -0.2210, -0.0033, -0.0126, -0.0402,\n", - " -1.3693, -0.4690, 0.2604, 0.3911, -0.3407, 0.6645, -0.1561, -0.1206,\n", - " 0.4401, -0.1128, 0.3281, 0.3493, 0.4086, 0.2105, -0.1624, -0.0317,\n", - " 0.2122, 0.2028, 0.5176, -0.5969, -0.6693, 0.1342, -0.5071, -0.3850])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "attn_map_st" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([0.2606])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weight_st" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([[-0.1554, -0.0586, -0.0729, ..., -0.0000, -0.0000, -0.0000],\n", - " [-0.0529, -0.2460, -0.0994, ..., -0.0000, -0.0000, -0.0000],\n", - " [-0.1179, -0.0144, -0.2249, ..., -0.0000, -0.0000, -0.0000],\n", - " ...,\n", - " [-0.0701, -0.1249, -0.0585, ..., -0.1516, -0.0000, -0.0000],\n", - " [ 0.0249, -0.1786, -0.0892, ..., -0.0000, -0.1801, -0.0000],\n", - " [-0.1276, -0.1296, -0.0653, ..., -0.0000, -0.0000, -0.1503]]),\n", - " tensor([[-0.5720, 0.0253, -0.8815, ..., -0.0000, -0.0000, -0.0000],\n", - " [-0.1371, -0.5842, -0.8806, ..., -0.0000, -0.0000, -0.0000],\n", - " [ 0.0658, -0.2696, -1.0837, ..., -0.0000, -0.0000, -0.0000],\n", - " ...,\n", - " [-0.0265, -0.1658, -0.9726, ..., -0.4945, -0.0000, -0.0000],\n", - " [-0.0341, -0.0129, -0.8782, ..., -0.0000, -0.4722, -0.0000],\n", - " [-0.0291, 0.0190, -1.0195, ..., -0.0000, -0.0000, -0.5399]]),\n", - " tensor([[ -5.2143, -0.2073, -9.1526, ..., -0.0000, -0.0000, -0.0000],\n", - " [ 1.0086, -6.6903, -8.8212, ..., -0.0000, -0.0000, -0.0000],\n", - " [ 1.6930, -0.4107, -17.7767, ..., -0.0000, -0.0000, -0.0000],\n", - " ...,\n", - " [ 1.9814, -0.8107, -13.0110, ..., -8.9967, -0.0000, -0.0000],\n", - " [ 0.9145, -0.1184, -6.4268, ..., -0.0000, -4.5341, -0.0000],\n", - " [ 1.8620, -1.2088, -13.1969, ..., -0.0000, -0.0000, -8.9710]]),\n", - " tensor([[ 0.4234, -0.1738, 0.4660, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0612, 0.4136, 0.4037, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0573, -0.1247, 0.9858, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [-0.0938, -0.0743, 0.4447, ..., 0.5810, 0.0000, 0.0000],\n", - " [-0.1542, -0.1397, 0.5147, ..., 0.0000, 0.5836, 0.0000],\n", - " [-0.1097, -0.1474, 0.4535, ..., 0.0000, 0.0000, 0.5374]]),\n", - " tensor([[ 0.9406, -0.2136, 0.8167, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.1652, 0.5770, 0.8471, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.1347, -0.2466, 1.5981, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.1693, -0.2130, 0.8331, ..., 0.8538, 0.0000, 0.0000],\n", - " [ 0.1927, -0.2818, 0.8108, ..., 0.0000, 0.8053, 0.0000],\n", - " [ 0.1685, -0.2378, 0.8006, ..., 0.0000, 0.0000, 0.7944]]),\n", - " tensor([[ 0.7328, 0.0801, 0.5444, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0403, 0.7458, 0.5566, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0786, 0.0814, 1.3531, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [-0.0527, 0.0911, 0.6341, ..., 0.6709, 0.0000, 0.0000],\n", - " [-0.0836, 0.1107, 0.5707, ..., 0.0000, 0.7083, 0.0000],\n", - " [-0.0249, 0.0901, 0.5255, ..., 0.0000, 0.0000, 0.6873]]),\n", - " tensor([[0.8320, 0.2851, 0.8615, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.1175, 0.9609, 0.8269, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.1498, 0.2262, 1.6439, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.1284, 0.2232, 0.8748, ..., 0.7061, 0.0000, 0.0000],\n", - " [0.1243, 0.2083, 0.9053, ..., 0.0000, 0.7440, 0.0000],\n", - " [0.1109, 0.2427, 0.8597, ..., 0.0000, 0.0000, 0.6755]]),\n", - " tensor([[ 0.7119, 0.2967, 0.5289, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.1194, 1.2095, 0.6030, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.1917, 0.3107, 1.4640, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [-0.1499, 0.3103, 0.6443, ..., 0.8962, 0.0000, 0.0000],\n", - " [-0.1631, 0.3251, 0.5534, ..., 0.0000, 0.8595, 0.0000],\n", - " [-0.1243, 0.2940, 0.6102, ..., 0.0000, 0.0000, 0.8932]]),\n", - " tensor([[ 3.0915, -0.7572, 1.6884, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.8447, 1.3794, 1.6422, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.8184, -0.8285, 3.8009, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.7615, -0.8503, 1.5874, ..., 2.2260, 0.0000, 0.0000],\n", - " [ 0.8272, -0.8181, 1.6270, ..., 0.0000, 2.1608, 0.0000],\n", - " [ 0.8811, -0.7469, 1.4999, ..., 0.0000, 0.0000, 2.1590]]),\n", - " tensor([[ 2.4223, -0.3267, 0.9755, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.2824, 1.6931, 0.8163, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.3807, -0.1291, 2.9314, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.3839, -0.2936, 0.9776, ..., 2.0073, 0.0000, 0.0000],\n", - " [ 0.4626, -0.2595, 0.9444, ..., 0.0000, 2.0228, 0.0000],\n", - " [ 0.4498, -0.2101, 0.9627, ..., 0.0000, 0.0000, 1.8273]]),\n", - " tensor([[15.2405, 2.7610, 6.7306, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 9.2874, 22.5292, 11.1778, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 6.0527, 2.1433, 18.1478, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 8.6357, 3.6522, 9.4287, ..., 15.7923, 0.0000, 0.0000],\n", - " [ 4.7526, 2.0569, 6.1020, ..., 0.0000, 8.8777, 0.0000],\n", - " [ 3.7997, 1.8325, 4.7829, ..., 0.0000, 0.0000, 7.1644]]),\n", - " tensor([[ 3.2594, 1.9875, 2.6364, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 1.8766, 17.1216, 9.5131, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 1.7089, 5.7129, 15.9202, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.4791, 2.2901, 3.2946, ..., 3.3042, 0.0000, 0.0000],\n", - " [ 0.7041, 3.5211, 4.8167, ..., 0.0000, 5.4057, 0.0000],\n", - " [ 1.7779, 8.7794, 10.9012, ..., 0.0000, 0.0000, 12.3839]])]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weights_rollout_st" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total Loss: 453.0188903808594\n", - "Layer 0 Loss: 0.0445, Percentage of Total Loss: 0.0098%\n", - "Layer 1 Loss: 0.0991, Percentage of Total Loss: 0.0219%\n", - "Layer 2 Loss: 394.6398, Percentage of Total Loss: 87.1133%\n", - "Layer 3 Loss: 0.0411, Percentage of Total Loss: 0.0091%\n", - "Layer 4 Loss: 0.0765, Percentage of Total Loss: 0.0169%\n", - "Layer 5 Loss: 0.0602, Percentage of Total Loss: 0.0133%\n", - "Layer 6 Loss: 0.0887, Percentage of Total Loss: 0.0196%\n", - "Layer 7 Loss: 0.0799, Percentage of Total Loss: 0.0176%\n", - "Layer 8 Loss: 0.5019, Percentage of Total Loss: 0.1108%\n", - "Layer 9 Loss: 0.5323, Percentage of Total Loss: 0.1175%\n", - "Layer 10 Loss: 38.7059, Percentage of Total Loss: 8.5440%\n", - "Layer 11 Loss: 18.1490, Percentage of Total Loss: 4.0062%\n", - "Average Layer Loss: 37.7516\n", - "Minimum Layer Loss: 0.0411\n", - "Maximum Layer Loss: 394.6398\n" - ] - } - ], - "source": [ - "# 손실 계산\n", - "loss = 0\n", - "layer_losses = [] # 레이어별 손실을 저장할 리스트\n", - "\n", - "for i in range(len(cls_weights_rollout_st)):\n", - " # 레이어별 가중치 차이 계산\n", - " layer_loss = F.mse_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", - " layer_losses.append(layer_loss.item()) # 손실을 리스트에 추가\n", - " loss += layer_loss\n", - "\n", - "# 총 손실 출력\n", - "total_loss = loss.item()\n", - "print(f\"Total Loss: {total_loss}\")\n", - "\n", - "# 레이어별 손실 비율 계산\n", - "layer_loss_percentages = [(layer_loss / total_loss) * 100 for layer_loss in layer_losses]\n", - "\n", - "# 각 레이어 손실 및 비율 출력\n", - "for i, layer_loss in enumerate(layer_losses):\n", - " print(f\"Layer {i} Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages[i]:.4f}%\")\n", - "\n", - "# 추가 정보 출력\n", - "average_loss = sum(layer_losses) / len(layer_losses)\n", - "min_loss = min(layer_losses)\n", - "max_loss = max(layer_losses)\n", - "\n", - "print(f\"Average Layer Loss: {average_loss:.4f}\")\n", - "print(f\"Minimum Layer Loss: {min_loss:.4f}\")\n", - "print(f\"Maximum Layer Loss: {max_loss:.4f}\")\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "brp1", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.21" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From a3c2824bf7e05da0dbc87f1cdd79ca493217b2fa Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 15 Feb 2025 17:08:30 +0900 Subject: [PATCH 4/7] Add files via upload --- attention_rollout.ipynb | 1612 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1612 insertions(+) create mode 100644 attention_rollout.ipynb diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb new file mode 100644 index 0000000..a9b4f4a --- /dev/null +++ b/attention_rollout.ipynb @@ -0,0 +1,1612 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "from tqdm import tqdm\n", + "import timm\n", + "assert timm.__version__ == \"0.3.2\"\n", + "from torchvision import transforms\n", + "from PIL import Image\n", + "import requests\n", + "import random\n", + "import numpy as np\n", + "\n", + "# 장치 설정\n", + "device = 'cuda' if torch.cuda.is_available() else 'cpu'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def my_forward_wrapper(attn_obj):\n", + " def my_forward(x):\n", + " B, N, C = x.shape\n", + " qkv = attn_obj.qkv(x).reshape(B, N, 3, attn_obj.num_heads, C // attn_obj.num_heads).permute(2, 0, 3, 1, 4)\n", + " q, k, v = qkv.unbind(0)\n", + "\n", + " attn = (q @ k.transpose(-2, -1)) * attn_obj.scale\n", + " attn = attn.softmax(dim=-1)\n", + " attn = attn_obj.attn_drop(attn)\n", + "\n", + " # attn_map을 저장\n", + " attn_maps.append(attn.detach()) # 주의 가중치를 저장\n", + " attn_obj.cls_attn_map = attn[:, :, 0, 2:] # cls attention map 저장\n", + "\n", + " x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n", + " x = attn_obj.proj(x)\n", + " x = attn_obj.proj_drop(x)\n", + " return x\n", + " return my_forward" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " # 현재 주의 맵의 크기 확인\n", + " current_size = attn_map.size(-1)\n", + " I = torch.eye(max(current_size, I_size)).to(attn_map.device) # 동적 아이덴티티 행렬 생성\n", + "\n", + " if current_size < I_size:\n", + " padding_size = I_size - current_size\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " I_padded = F.pad(I, (0, padding_size, 0, 0)) # 아이덴티티 행렬 패딩\n", + " prod = padded_attn_map + I_padded\n", + " else:\n", + " prod = attn_map + I\n", + "\n", + " attn_rollout.append(prod / prod.sum(dim=-1, keepdim=True)) # 정규화\n", + "\n", + " return attn_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-small" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 384, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=384, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 292.63it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Block 1: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 2: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 3: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 4: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 5: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 6: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 7: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 8: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 9: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 10: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 11: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 12: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "CLS Weight Shape: torch.Size([1, 194])\n", + "Attention Rollout: torch.Size([1, 6, 196, 196])\n", + "Combined CLS Weight Shape: torch.Size([1, 6, 194])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "attn_maps = [] # 전역 리스트를 사용하여 attn_map을 저장\n", + "cls_weights = []\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출\n", + "for block in tqdm(model.blocks):\n", + " wrapped_forward = my_forward_wrapper(block.attn)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성\n", + " x = torch.randn(1, 196, 384) # B=1, N=196, C=384\n", + " output = wrapped_forward(x)\n", + "\n", + " # attn_map이 저장되었는지 확인합니다.\n", + " if attn_maps:\n", + " print(f\"Block {len(attn_maps)}: Attention map collected with shape {attn_maps[-1].shape}\")\n", + " else:\n", + " print(f\"Block {len(attn_maps)}: No attention map collected.\")\n", + "\n", + " # cls_attn_map 저장\n", + " cls_attn_map = block.attn.cls_attn_map.detach() # cls_attn_map 저장\n", + " cls_weights.append(cls_attn_map)\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps:\n", + " attn_map = attn_maps[-1].mean(dim=1).squeeze(0).detach()\n", + "else:\n", + " print(\"attn_maps 내용:\", attn_maps) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "cls_weight = cls_weights[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_cpu = attn_map.cpu()\n", + "cls_weight_cpu = cls_weight.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_combined = torch.prod(torch.stack(cls_weights), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_prod = torch.prod(torch.stack(attn_maps), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_cpu = [attn_map.cpu() for attn_map in attn_maps]\n", + "cls_weights_cpu = [cls_weight.cpu() for cls_weight in cls_weights]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 12003.73it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Attention Rollout Length: 12\n", + "Attention Rollout Shapes: [torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196])]\n", + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " attn_rollout = []\n", + " I = torch.eye(attn_maps[0].shape[-1]) # Identity matrix\n", + " prod = I\n", + " for i, attn_map in enumerate(attn_maps):\n", + " prod = prod @ (attn_map + I) # Product of attention maps with identity matrix\n", + " \n", + " prod = prod / prod.sum(dim=-1, keepdim=True) # Normalize\n", + " attn_rollout.append(prod)\n", + " return attn_rollout\n", + "\n", + "# Attention Rollout\n", + "attn_rollout = attention_rollout_function(attn_maps_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout = []\n", + "\n", + "for i in tqdm(range(len(attn_rollout))): # attn_rollout의 크기로 반복\n", + " cls_weight_tensor = attn_rollout[i][0, 0, :, :] # 첫 번째 헤드 사용\n", + "\n", + " # 텐서의 크기를 출력하여 디버깅\n", + " print(f\"Class weight tensor shape before view: {cls_weight_tensor.shape}\")\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인합니다.\n", + " if cls_weight_tensor.numel() == 14 * 14:\n", + " cls_weights_rollout.append(cls_weight_tensor.view(14, 14))\n", + " elif cls_weight_tensor.numel() == 196 * 196:\n", + " # 196x196 텐서를 클래스 가중치로 사용\n", + " cls_weights_rollout.append(cls_weight_tensor) # 변환하지 않고 그대로 사용\n", + " else:\n", + " print(f\"Skipping view for index {i}, tensor size is {cls_weight_tensor.numel()}\")\n", + "\n", + "# 결과 확인\n", + "if isinstance(attn_rollout, list):\n", + " print(\"Attention Rollout Length:\", len(attn_rollout))\n", + " print(\"Attention Rollout Shapes:\", [tensor.shape for tensor in attn_rollout])\n", + "else:\n", + " print(\"Attention Rollout Shape:\", attn_rollout.shape)\n", + "\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0092, 0.0108, 0.0016, ..., 0.0033, 0.0031, 0.0041],\n", + " [0.0025, 0.0249, 0.0017, ..., 0.0045, 0.0037, 0.0045],\n", + " [0.0022, 0.0028, 0.0083, ..., 0.0025, 0.0050, 0.0159],\n", + " ...,\n", + " [0.0022, 0.0089, 0.0015, ..., 0.0199, 0.0061, 0.0046],\n", + " [0.0014, 0.0043, 0.0024, ..., 0.0043, 0.0238, 0.0065],\n", + " [0.0042, 0.0048, 0.0018, ..., 0.0042, 0.0047, 0.0244]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0028, 0.0103, 0.0145, 0.0100, 0.0049, 0.0047, 0.0083, 0.0101, 0.0075,\n", + " 0.0024, 0.0125, 0.0012, 0.0042, 0.0037, 0.0073, 0.0071, 0.0119, 0.0110,\n", + " 0.0049, 0.0063, 0.0066, 0.0083, 0.0065, 0.0068, 0.0030, 0.0534, 0.0020,\n", + " 0.0335, 0.0037, 0.0035, 0.0532, 0.0121, 0.0040, 0.0036, 0.0115, 0.0019,\n", + " 0.0060, 0.0106, 0.0159, 0.0096, 0.0117, 0.0069, 0.0179, 0.0196, 0.0382,\n", + " 0.0029, 0.0084, 0.0023, 0.0139, 0.0022, 0.0152, 0.0044, 0.0024, 0.0139,\n", + " 0.0097, 0.0583, 0.0025, 0.0032, 0.0253, 0.0018, 0.0032, 0.0034, 0.0056,\n", + " 0.0191, 0.0047, 0.0171, 0.0095, 0.0093, 0.0037, 0.0014, 0.0151, 0.0051,\n", + " 0.0043, 0.0063, 0.0054, 0.0231, 0.0095, 0.0082, 0.0100, 0.0046, 0.0108,\n", + " 0.0045, 0.0025, 0.0169, 0.0107, 0.0023, 0.0054, 0.0060, 0.0040, 0.0103,\n", + " 0.0049, 0.0091, 0.0056, 0.0041, 0.1168, 0.0315, 0.0022, 0.0107, 0.0085,\n", + " 0.0040, 0.0037, 0.0197, 0.0153, 0.0179, 0.0044, 0.0178, 0.0102, 0.0677,\n", + " 0.0103, 0.0061, 0.0065, 0.0028, 0.0051, 0.0014, 0.0034, 0.0241, 0.0049,\n", + " 0.0142, 0.0090, 0.0042, 0.0320, 0.0069, 0.0044, 0.0044, 0.0182, 0.0101,\n", + " 0.0042, 0.0026, 0.1099, 0.0097, 0.0284, 0.0165, 0.1040, 0.0059, 0.0176,\n", + " 0.0036, 0.0107, 0.0176, 0.0098, 0.0038, 0.0078, 0.0069, 0.0027, 0.0309,\n", + " 0.0057, 0.0041, 0.0194, 0.0011, 0.0053, 0.0026, 0.0038, 0.0306, 0.0064,\n", + " 0.0205, 0.0032, 0.0127, 0.0057, 0.0024, 0.0022, 0.0252, 0.0051, 0.0481,\n", + " 0.0108, 0.0292, 0.0301, 0.0071, 0.0176, 0.0060, 0.0037, 0.0065, 0.0056,\n", + " 0.0025, 0.0069, 0.0224, 0.0032, 0.0108, 0.0104, 0.0012, 0.0050, 0.0022,\n", + " 0.0089, 0.0038, 0.0131, 0.0068, 0.0124, 0.0128, 0.0195, 0.0075, 0.0050,\n", + " 0.0138, 0.0089, 0.0084, 0.0060, 0.0109]])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[5.0145e-01, 1.0773e-04, 1.9219e-09, ..., 3.0224e-07, 4.8908e-08,\n", + " 9.4525e-09],\n", + " [1.7469e-07, 7.3549e-01, 1.9689e-09, ..., 1.5312e-05, 1.8353e-07,\n", + " 9.8260e-07],\n", + " [2.0269e-04, 8.8949e-03, 7.8070e-01, ..., 8.4695e-05, 3.8640e-05,\n", + " 1.5120e-02],\n", + " ...,\n", + " [3.1366e-08, 1.4511e-03, 1.2052e-10, ..., 5.2665e-01, 1.4668e-05,\n", + " 1.2199e-06],\n", + " [6.7066e-11, 3.5859e-06, 1.0056e-12, ..., 1.3806e-06, 5.0115e-01,\n", + " 6.2253e-07],\n", + " [2.0210e-10, 8.3169e-06, 4.0810e-09, ..., 2.4031e-07, 3.3919e-06,\n", + " 5.2196e-01]]),\n", + " tensor([[2.8310e-01, 7.6627e-04, 1.8418e-03, ..., 1.2459e-04, 8.3037e-03,\n", + " 8.7034e-04],\n", + " [3.8865e-04, 3.8519e-01, 1.1187e-03, ..., 5.4312e-04, 1.4127e-02,\n", + " 8.7212e-04],\n", + " [2.2728e-04, 4.7906e-03, 7.5910e-01, ..., 1.1250e-04, 3.2742e-03,\n", + " 9.5180e-03],\n", + " ...,\n", + " [2.5900e-04, 1.3777e-03, 2.0527e-03, ..., 4.1472e-01, 1.6249e-03,\n", + " 2.4871e-04],\n", + " [2.2064e-04, 5.9052e-04, 1.7989e-03, ..., 1.3416e-04, 4.2058e-01,\n", + " 4.4896e-04],\n", + " [1.6595e-04, 5.9181e-04, 1.7374e-03, ..., 1.4127e-04, 6.7444e-02,\n", + " 3.2628e-01]]),\n", + " tensor([[0.1570, 0.0010, 0.0056, ..., 0.0006, 0.0064, 0.0009],\n", + " [0.0135, 0.1933, 0.0032, ..., 0.0010, 0.0105, 0.0013],\n", + " [0.0069, 0.0031, 0.3877, ..., 0.0009, 0.0040, 0.0054],\n", + " ...,\n", + " [0.0108, 0.0015, 0.0052, ..., 0.2111, 0.0023, 0.0007],\n", + " [0.0091, 0.0012, 0.0048, ..., 0.0011, 0.2260, 0.0007],\n", + " [0.0097, 0.0011, 0.0052, ..., 0.0007, 0.0395, 0.1640]]),\n", + " tensor([[0.1081, 0.0055, 0.0075, ..., 0.0018, 0.0058, 0.0013],\n", + " [0.0112, 0.1038, 0.0062, ..., 0.0022, 0.0086, 0.0024],\n", + " [0.0063, 0.0048, 0.1985, ..., 0.0013, 0.0051, 0.0036],\n", + " ...,\n", + " [0.0089, 0.0055, 0.0061, ..., 0.1082, 0.0036, 0.0019],\n", + " [0.0075, 0.0059, 0.0061, ..., 0.0019, 0.1328, 0.0011],\n", + " [0.0081, 0.0053, 0.0061, ..., 0.0017, 0.0251, 0.0829]]),\n", + " tensor([[0.0596, 0.0034, 0.0052, ..., 0.0040, 0.0040, 0.0009],\n", + " [0.0106, 0.0527, 0.0047, ..., 0.0043, 0.0053, 0.0014],\n", + " [0.0083, 0.0031, 0.1010, ..., 0.0042, 0.0034, 0.0020],\n", + " ...,\n", + " [0.0104, 0.0036, 0.0045, ..., 0.0581, 0.0028, 0.0011],\n", + " [0.0099, 0.0036, 0.0045, ..., 0.0044, 0.0673, 0.0008],\n", + " [0.0101, 0.0034, 0.0049, ..., 0.0041, 0.0135, 0.0417]]),\n", + " tensor([[0.0315, 0.0021, 0.0068, ..., 0.0030, 0.0032, 0.0032],\n", + " [0.0069, 0.0267, 0.0069, ..., 0.0031, 0.0038, 0.0035],\n", + " [0.0055, 0.0019, 0.0546, ..., 0.0032, 0.0028, 0.0039],\n", + " ...,\n", + " [0.0069, 0.0022, 0.0068, ..., 0.0300, 0.0025, 0.0039],\n", + " [0.0066, 0.0022, 0.0065, ..., 0.0033, 0.0347, 0.0031],\n", + " [0.0067, 0.0021, 0.0068, ..., 0.0031, 0.0079, 0.0236]]),\n", + " tensor([[0.0174, 0.0023, 0.0059, ..., 0.0040, 0.0037, 0.0039],\n", + " [0.0049, 0.0145, 0.0060, ..., 0.0040, 0.0040, 0.0042],\n", + " [0.0042, 0.0021, 0.0310, ..., 0.0039, 0.0035, 0.0043],\n", + " ...,\n", + " [0.0050, 0.0022, 0.0059, ..., 0.0175, 0.0033, 0.0044],\n", + " [0.0048, 0.0023, 0.0058, ..., 0.0041, 0.0198, 0.0039],\n", + " [0.0049, 0.0022, 0.0060, ..., 0.0041, 0.0060, 0.0143]]),\n", + " tensor([[0.0093, 0.0036, 0.0040, ..., 0.0063, 0.0032, 0.0035],\n", + " [0.0031, 0.0097, 0.0041, ..., 0.0063, 0.0034, 0.0036],\n", + " [0.0027, 0.0035, 0.0166, ..., 0.0062, 0.0031, 0.0037],\n", + " ...,\n", + " [0.0031, 0.0036, 0.0040, ..., 0.0135, 0.0030, 0.0038],\n", + " [0.0030, 0.0036, 0.0040, ..., 0.0063, 0.0113, 0.0034],\n", + " [0.0031, 0.0036, 0.0041, ..., 0.0063, 0.0044, 0.0090]]),\n", + " tensor([[0.0069, 0.0032, 0.0029, ..., 0.0067, 0.0042, 0.0037],\n", + " [0.0038, 0.0062, 0.0029, ..., 0.0067, 0.0042, 0.0038],\n", + " [0.0035, 0.0031, 0.0092, ..., 0.0067, 0.0042, 0.0038],\n", + " ...,\n", + " [0.0038, 0.0032, 0.0029, ..., 0.0103, 0.0041, 0.0039],\n", + " [0.0037, 0.0031, 0.0029, ..., 0.0067, 0.0082, 0.0037],\n", + " [0.0037, 0.0032, 0.0029, ..., 0.0067, 0.0047, 0.0065]]),\n", + " tensor([[0.0074, 0.0034, 0.0031, ..., 0.0049, 0.0035, 0.0039],\n", + " [0.0058, 0.0049, 0.0031, ..., 0.0049, 0.0036, 0.0040],\n", + " [0.0058, 0.0034, 0.0063, ..., 0.0049, 0.0036, 0.0040],\n", + " ...,\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0067, 0.0035, 0.0040],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0048, 0.0056, 0.0039],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0049, 0.0038, 0.0053]]),\n", + " tensor([[0.0092, 0.0024, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0032, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0062, ..., 0.0049, 0.0041, 0.0086],\n", + " ...,\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0058, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0051, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0042, 0.0093]]),\n", + " tensor([[0.0056, 0.0037, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0052, 0.0040, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0040, ..., 0.0038, 0.0035, 0.0080],\n", + " ...,\n", + " [0.0052, 0.0037, 0.0032, ..., 0.0043, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0032, ..., 0.0038, 0.0040, 0.0080],\n", + " [0.0051, 0.0037, 0.0032, ..., 0.0038, 0.0036, 0.0084]])]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-tiny -> student model로 가정" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)\n", + "model_st" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model_st.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 266.64it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collected attention maps: 12\n", + "CLS Weight Shape: torch.Size([1])\n", + "Attention Rollout: torch.Size([1, 196, 192])\n", + "Combined CLS Weight Shape: torch.Size([1, 196])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# 전역 리스트를 사용하여 attn_map과 cls_weight 저장\n", + "attn_maps_st = []\n", + "cls_weights_st = []\n", + "\n", + "# Forward hook을 설정하는 함수\n", + "def get_attention_map(module, input, output):\n", + " attn_maps_st.append(output.detach()) # 주의 맵 저장\n", + " cls_attn_map = output[:, :, 0] # CLS token에 대한 attention map을 추출\n", + " cls_weights_st.append(cls_attn_map.detach()) # 클래스 주의 맵 저장\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출합니다.\n", + "for block in tqdm(model_st.blocks):\n", + " # Forward hook 등록\n", + " hook = block.attn.register_forward_hook(get_attention_map)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성합니다.\n", + " x = torch.randn(1, 3, 224, 224).to(device) # B=1, C=3, H=224, W=224\n", + " x_patch = model_st.patch_embed(x) # 패치 임베딩 변환\n", + "\n", + " # wrapped_forward 함수로 출력 계산\n", + " output = block.attn(x_patch)\n", + "\n", + " # hook 해제\n", + " hook.remove()\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps_st:\n", + " attn_map_st = attn_maps_st[-1].mean(dim=1).squeeze(0).detach()\n", + " print(f\"Collected attention maps: {len(attn_maps_st)}\")\n", + "else:\n", + " print(\"attn_maps_st 내용:\", attn_maps_st) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "# cls_weight 계산\n", + "cls_weight_st = cls_weights_st[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_st_cpu = attn_map_st.cpu()\n", + "cls_weight_st_cpu = cls_weight_st.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight_st.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_st_combined = torch.prod(torch.stack(cls_weights_st), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_st_prod = torch.prod(torch.stack(attn_maps_st), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_st_cpu = [attn_map.cpu() for attn_map in attn_maps_st]\n", + "cls_weights_st_cpu = [cls_weight.cpu() for cls_weight in cls_weights_st]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_st_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_st_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Attention Rollout\n", + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " I = torch.eye(I_size).to(attn_maps[0].device) # Identity matrix\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " if attn_map.size(-1) == I_size:\n", + " prod = attn_map + I\n", + " elif attn_map.size(-1) < I_size:\n", + " padding_size = I_size - attn_map.size(-1)\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " prod = padded_attn_map + I\n", + " else:\n", + " raise ValueError(\"Unexpected attention map size.\")\n", + "\n", + " attn_rollout.append(prod)\n", + "\n", + " attn_rollout = [prod / prod.sum(dim=-1, keepdim=True) for prod in attn_rollout] # Normalize\n", + " return attn_rollout" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + } + ], + "source": [ + "# Attention Rollout 실행\n", + "attn_rollout_st = attention_rollout_function(attn_maps_st_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout_st = []\n", + "\n", + "for i in range(len(attn_rollout_st)): # attn_rollout_st의 크기로 반복\n", + " cls_weight_tensor_st = attn_rollout_st[i][0] # 첫 번째 배치의 텐서 사용\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인\n", + " if cls_weight_tensor_st.numel() == 14 * 14:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st.view(14, 14))\n", + " elif cls_weight_tensor_st.numel() == 196 * 196:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st) # 그대로 사용\n", + "\n", + "# 결과 확인\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout_st])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0.1842, 0.6513, 0.9481, -0.0575, -0.4596, 0.9378, -0.0630, 0.0913,\n", + " -0.2760, 0.8236, -0.3048, -0.5067, -0.2237, 0.0104, 0.2620, 0.4669,\n", + " 0.3408, -0.6330, -0.4091, -0.4896, -0.9567, -1.5660, -0.2059, -0.2811,\n", + " -0.2813, -0.8560, -0.1496, -0.0455, -0.0833, 0.3670, 0.2738, -0.3592,\n", + " 0.3951, 0.6094, 0.5716, -0.0526, 0.3808, -0.0236, -0.7082, 0.1859,\n", + " 0.8467, -0.0996, 0.4303, 0.0613, 0.7428, 0.1889, -0.0631, -0.1368,\n", + " 0.1214, 0.5921, 0.1423, -0.4978, 0.5646, -0.2145, 0.1315, -0.3321,\n", + " 0.1872, -0.0260, -0.5459, 0.0105, 0.7164, -0.4216, -0.2821, 0.2232,\n", + " -0.1487, 0.1884, -0.1680, 0.3930, -0.3026, 0.8395, -0.5283, -0.1191,\n", + " 0.5188, -1.0522, 0.4154, -0.1342, -0.8908, 1.1572, 0.0318, 0.0403,\n", + " -0.4454, 0.0902, -0.5127, 0.6663, -0.3861, 0.5882, 0.0156, 0.5951,\n", + " -0.7754, 0.4059, 0.1418, 0.9044, -1.3180, -0.1740, 0.9519, -0.7662,\n", + " -0.1940, -0.4566, 0.9794, 0.2629, 0.3623, -0.1166, 0.5362, -0.2732,\n", + " -0.5911, -0.4877, 0.0517, 0.5354, 0.2801, -0.0113, 0.1709, -0.7622,\n", + " -0.1145, -0.9085, 0.3520, -0.6743, 0.2920, 0.2248, -0.2348, -0.4925,\n", + " 0.1487, -0.0883, -0.2174, -0.1746, -0.0644, -0.1431, 0.5035, -0.4010,\n", + " -0.1813, -0.0585, -0.1536, 0.7160, -0.5562, -0.1020, 0.2226, 0.2361,\n", + " 0.5698, -0.2106, 0.9990, -1.1657, -1.0541, 0.4736, 0.7887, 0.2140,\n", + " -0.4918, -0.7655, 1.2153, -0.0707, -0.4830, 0.2197, 0.0948, 0.8601,\n", + " 0.0194, 0.3300, -0.4195, 0.2778, -0.2720, -0.9317, -0.2371, 0.2978,\n", + " -0.3443, 1.3332, -0.0444, -0.4376, -0.2210, -0.0033, -0.0126, -0.0402,\n", + " -1.3693, -0.4690, 0.2604, 0.3911, -0.3407, 0.6645, -0.1561, -0.1206,\n", + " 0.4401, -0.1128, 0.3281, 0.3493, 0.4086, 0.2105, -0.1624, -0.0317,\n", + " 0.2122, 0.2028, 0.5176, -0.5969, -0.6693, 0.1342, -0.5071, -0.3850])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map_st" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.2606])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight_st" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[-0.1554, -0.0586, -0.0729, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.0529, -0.2460, -0.0994, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1179, -0.0144, -0.2249, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0701, -0.1249, -0.0585, ..., -0.1516, -0.0000, -0.0000],\n", + " [ 0.0249, -0.1786, -0.0892, ..., -0.0000, -0.1801, -0.0000],\n", + " [-0.1276, -0.1296, -0.0653, ..., -0.0000, -0.0000, -0.1503]]),\n", + " tensor([[-0.5720, 0.0253, -0.8815, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1371, -0.5842, -0.8806, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 0.0658, -0.2696, -1.0837, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0265, -0.1658, -0.9726, ..., -0.4945, -0.0000, -0.0000],\n", + " [-0.0341, -0.0129, -0.8782, ..., -0.0000, -0.4722, -0.0000],\n", + " [-0.0291, 0.0190, -1.0195, ..., -0.0000, -0.0000, -0.5399]]),\n", + " tensor([[ -5.2143, -0.2073, -9.1526, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.0086, -6.6903, -8.8212, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.6930, -0.4107, -17.7767, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [ 1.9814, -0.8107, -13.0110, ..., -8.9967, -0.0000, -0.0000],\n", + " [ 0.9145, -0.1184, -6.4268, ..., -0.0000, -4.5341, -0.0000],\n", + " [ 1.8620, -1.2088, -13.1969, ..., -0.0000, -0.0000, -8.9710]]),\n", + " tensor([[ 0.4234, -0.1738, 0.4660, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0612, 0.4136, 0.4037, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0573, -0.1247, 0.9858, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0938, -0.0743, 0.4447, ..., 0.5810, 0.0000, 0.0000],\n", + " [-0.1542, -0.1397, 0.5147, ..., 0.0000, 0.5836, 0.0000],\n", + " [-0.1097, -0.1474, 0.4535, ..., 0.0000, 0.0000, 0.5374]]),\n", + " tensor([[ 0.9406, -0.2136, 0.8167, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1652, 0.5770, 0.8471, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1347, -0.2466, 1.5981, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.1693, -0.2130, 0.8331, ..., 0.8538, 0.0000, 0.0000],\n", + " [ 0.1927, -0.2818, 0.8108, ..., 0.0000, 0.8053, 0.0000],\n", + " [ 0.1685, -0.2378, 0.8006, ..., 0.0000, 0.0000, 0.7944]]),\n", + " tensor([[ 0.7328, 0.0801, 0.5444, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0403, 0.7458, 0.5566, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0786, 0.0814, 1.3531, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0527, 0.0911, 0.6341, ..., 0.6709, 0.0000, 0.0000],\n", + " [-0.0836, 0.1107, 0.5707, ..., 0.0000, 0.7083, 0.0000],\n", + " [-0.0249, 0.0901, 0.5255, ..., 0.0000, 0.0000, 0.6873]]),\n", + " tensor([[0.8320, 0.2851, 0.8615, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1175, 0.9609, 0.8269, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1498, 0.2262, 1.6439, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [0.1284, 0.2232, 0.8748, ..., 0.7061, 0.0000, 0.0000],\n", + " [0.1243, 0.2083, 0.9053, ..., 0.0000, 0.7440, 0.0000],\n", + " [0.1109, 0.2427, 0.8597, ..., 0.0000, 0.0000, 0.6755]]),\n", + " tensor([[ 0.7119, 0.2967, 0.5289, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1194, 1.2095, 0.6030, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1917, 0.3107, 1.4640, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.1499, 0.3103, 0.6443, ..., 0.8962, 0.0000, 0.0000],\n", + " [-0.1631, 0.3251, 0.5534, ..., 0.0000, 0.8595, 0.0000],\n", + " [-0.1243, 0.2940, 0.6102, ..., 0.0000, 0.0000, 0.8932]]),\n", + " tensor([[ 3.0915, -0.7572, 1.6884, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8447, 1.3794, 1.6422, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8184, -0.8285, 3.8009, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.7615, -0.8503, 1.5874, ..., 2.2260, 0.0000, 0.0000],\n", + " [ 0.8272, -0.8181, 1.6270, ..., 0.0000, 2.1608, 0.0000],\n", + " [ 0.8811, -0.7469, 1.4999, ..., 0.0000, 0.0000, 2.1590]]),\n", + " tensor([[ 2.4223, -0.3267, 0.9755, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.2824, 1.6931, 0.8163, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.3807, -0.1291, 2.9314, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.3839, -0.2936, 0.9776, ..., 2.0073, 0.0000, 0.0000],\n", + " [ 0.4626, -0.2595, 0.9444, ..., 0.0000, 2.0228, 0.0000],\n", + " [ 0.4498, -0.2101, 0.9627, ..., 0.0000, 0.0000, 1.8273]]),\n", + " tensor([[15.2405, 2.7610, 6.7306, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 9.2874, 22.5292, 11.1778, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 6.0527, 2.1433, 18.1478, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 8.6357, 3.6522, 9.4287, ..., 15.7923, 0.0000, 0.0000],\n", + " [ 4.7526, 2.0569, 6.1020, ..., 0.0000, 8.8777, 0.0000],\n", + " [ 3.7997, 1.8325, 4.7829, ..., 0.0000, 0.0000, 7.1644]]),\n", + " tensor([[ 3.2594, 1.9875, 2.6364, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.8766, 17.1216, 9.5131, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.7089, 5.7129, 15.9202, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.4791, 2.2901, 3.2946, ..., 3.3042, 0.0000, 0.0000],\n", + " [ 0.7041, 3.5211, 4.8167, ..., 0.0000, 5.4057, 0.0000],\n", + " [ 1.7779, 8.7794, 10.9012, ..., 0.0000, 0.0000, 12.3839]])]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout_st" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "두 모델 간의 차이 Loss 계산 (L1, L2, Cosine Similarity, KL Divergence로 구현)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total L1 Loss: 14.616626739501953\n", + "Layer 0 L1 Loss: 0.1479, Percentage of Total Loss: 1.0115%\n", + "Layer 1 L1 Loss: 0.2261, Percentage of Total Loss: 1.5469%\n", + "Layer 2 L1 Loss: 5.1462, Percentage of Total Loss: 35.2075%\n", + "Layer 3 L1 Loss: 0.1554, Percentage of Total Loss: 1.0629%\n", + "Layer 4 L1 Loss: 0.2121, Percentage of Total Loss: 1.4508%\n", + "Layer 5 L1 Loss: 0.1961, Percentage of Total Loss: 1.3413%\n", + "Layer 6 L1 Loss: 0.2350, Percentage of Total Loss: 1.6081%\n", + "Layer 7 L1 Loss: 0.2223, Percentage of Total Loss: 1.5206%\n", + "Layer 8 L1 Loss: 0.5502, Percentage of Total Loss: 3.7645%\n", + "Layer 9 L1 Loss: 0.5793, Percentage of Total Loss: 3.9635%\n", + "Layer 10 L1 Loss: 4.3009, Percentage of Total Loss: 29.4244%\n", + "Layer 11 L1 Loss: 2.6453, Percentage of Total Loss: 18.0980%\n" + ] + } + ], + "source": [ + "# L1 (Mean Absolute Error): 두 모델의 각 레이어 가중치 간의 절대 차이 계산\n", + "# L1 손실 계산\n", + "loss_l1 = 0\n", + "layer_losses_l1 = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = F.l1_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses_l1.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_l1 += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_l1 = loss_l1.item()\n", + "print(f\"Total L1 Loss: {total_loss_l1}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_l1 = [(layer_loss / total_loss_l1) * 100 for layer_loss in layer_losses_l1]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_l1):\n", + " print(f\"Layer {i} L1 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l1[i]:.4f}%\")\n", + "## 두 모델 간의 차이가 주로 Layer 2와 Layer 10에서 발생" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total L2 Loss: 453.0188903808594\n", + "Layer 0 L2 Loss: 0.0445, Percentage of Total Loss: 0.0098%\n", + "Layer 1 L2 Loss: 0.0991, Percentage of Total Loss: 0.0219%\n", + "Layer 2 L2 Loss: 394.6398, Percentage of Total Loss: 87.1133%\n", + "Layer 3 L2 Loss: 0.0411, Percentage of Total Loss: 0.0091%\n", + "Layer 4 L2 Loss: 0.0765, Percentage of Total Loss: 0.0169%\n", + "Layer 5 L2 Loss: 0.0602, Percentage of Total Loss: 0.0133%\n", + "Layer 6 L2 Loss: 0.0887, Percentage of Total Loss: 0.0196%\n", + "Layer 7 L2 Loss: 0.0799, Percentage of Total Loss: 0.0176%\n", + "Layer 8 L2 Loss: 0.5019, Percentage of Total Loss: 0.1108%\n", + "Layer 9 L2 Loss: 0.5323, Percentage of Total Loss: 0.1175%\n", + "Layer 10 L2 Loss: 38.7059, Percentage of Total Loss: 8.5440%\n", + "Layer 11 L2 Loss: 18.1490, Percentage of Total Loss: 4.0062%\n" + ] + } + ], + "source": [ + "# L2 (Euclidean Distance): 두 모델의 각 레이어 가중치 간의 유클리드 거리 계산\n", + "# L2 손실 계산\n", + "loss_l2 = 0\n", + "layer_losses_l2 = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산 (L2 손실)\n", + " layer_loss = F.mse_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses_l2.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_l2 += layer_loss\n", + "\n", + "# 총 L2 손실 출력\n", + "total_loss_l2 = loss_l2.item()\n", + "print(f\"Total L2 Loss: {total_loss_l2}\")\n", + "\n", + "# 레이어별 L2 손실 비율 계산\n", + "layer_loss_percentages_l2 = [(layer_loss / total_loss_l2) * 100 for layer_loss in layer_losses_l2]\n", + "\n", + "# 각 레이어 L2 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_l2):\n", + " print(f\"Layer {i} L2 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l2[i]:.4f}%\")\n", + "\n", + "## Layer 2: 2번째 레이어의 L2 손실이 394.6398로, 전체 손실의 87.11% -> 레이어2에서 두 모델 간의 가중치 차이가 매우 크다\n", + "## 나머지 레이어들은 상대적으로 손실 값이 매우 작고, 전체 손실에서 차지하는 비율도 낮음.\n", + "## layer 2가 두 모델의 성능 차이에 큰 기여하고 있을 것이라 판단할 수 있음. 두 모델의 구조나 파라미터 조정시 해당 레이어 더 보기!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Cosine Loss: 11.714558601379395\n", + "Layer 0 Cosine Loss: 1.0442, Percentage of Total Loss: 8.9137%\n", + "Layer 1 Cosine Loss: 1.0548, Percentage of Total Loss: 9.0045%\n", + "Layer 2 Cosine Loss: 1.0349, Percentage of Total Loss: 8.8344%\n", + "Layer 3 Cosine Loss: 0.8728, Percentage of Total Loss: 7.4502%\n", + "Layer 4 Cosine Loss: 0.8025, Percentage of Total Loss: 6.8504%\n", + "Layer 5 Cosine Loss: 0.9400, Percentage of Total Loss: 8.0241%\n", + "Layer 6 Cosine Loss: 1.0328, Percentage of Total Loss: 8.8163%\n", + "Layer 7 Cosine Loss: 0.9947, Percentage of Total Loss: 8.4910%\n", + "Layer 8 Cosine Loss: 0.9695, Percentage of Total Loss: 8.2759%\n", + "Layer 9 Cosine Loss: 0.9745, Percentage of Total Loss: 8.3191%\n", + "Layer 10 Cosine Loss: 0.9839, Percentage of Total Loss: 8.3986%\n", + "Layer 11 Cosine Loss: 1.0100, Percentage of Total Loss: 8.6217%\n" + ] + } + ], + "source": [ + "# Cosine Similarity: 두 모델의 가중치 벡터 간의 코사인 유사도 계산\n", + "\n", + "from torch.nn.functional import cosine_similarity\n", + "\n", + "# 코사인 유사도 손실 계산\n", + "loss_cosine = 0\n", + "layer_losses_cosine = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = 1 - cosine_similarity(cls_weights_rollout_st[i].view(1, -1), cls_weights_rollout[i].view(1, -1))\n", + " layer_losses_cosine.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_cosine += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_cosine = loss_cosine.item()\n", + "print(f\"Total Cosine Loss: {total_loss_cosine}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_cosine = [(layer_loss / total_loss_cosine) * 100 for layer_loss in layer_losses_cosine]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_cosine):\n", + " print(f\"Layer {i} Cosine Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_cosine[i]:.4f}%\")\n", + "\n", + "## 각 레이어의 손실은 0.8에서 1.1 사이로 분포 -> 두 모델의 가중치가 레이어별로 상당히 유사\n", + "## 레이어 0에서 레이어 11까지의 손실 값이 비슷한 수준이므로, 전체적으로 두 모델의 가중치가 유사" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total KL Divergence Loss: 31.367801666259766\n", + "Layer 0 KL Loss: 0.0042, Percentage of Total Loss: 0.0133%\n", + "Layer 1 KL Loss: 0.0067, Percentage of Total Loss: 0.0214%\n", + "Layer 2 KL Loss: 15.2732, Percentage of Total Loss: 48.6907%\n", + "Layer 3 KL Loss: 0.0010, Percentage of Total Loss: 0.0032%\n", + "Layer 4 KL Loss: 0.0025, Percentage of Total Loss: 0.0080%\n", + "Layer 5 KL Loss: 0.0023, Percentage of Total Loss: 0.0073%\n", + "Layer 6 KL Loss: 0.0020, Percentage of Total Loss: 0.0063%\n", + "Layer 7 KL Loss: 0.0030, Percentage of Total Loss: 0.0096%\n", + "Layer 8 KL Loss: 0.0287, Percentage of Total Loss: 0.0915%\n", + "Layer 9 KL Loss: 0.0246, Percentage of Total Loss: 0.0785%\n", + "Layer 10 KL Loss: 9.7367, Percentage of Total Loss: 31.0406%\n", + "Layer 11 KL Loss: 6.2829, Percentage of Total Loss: 20.0297%\n" + ] + } + ], + "source": [ + "# KL Divergence: 두 모델의 가중치를 확률 분포로 간주하고, Kullback-Leibler Divergence 계산하여 두 분포 간의 차이 측정\n", + "# KL 발산 계산\n", + "loss_kl = 0\n", + "layer_losses_kl = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 확률 분포로 정규화\n", + " p = F.softmax(cls_weights_rollout_st[i], dim=0) + 1e-10 # 작은 값 추가\n", + " q = F.softmax(cls_weights_rollout[i], dim=0) + 1e-10 # 작은 값 추가\n", + " \n", + " # KL 발산 계산\n", + " layer_loss = F.kl_div(p.log(), q, reduction='batchmean')\n", + " layer_losses_kl.append(layer_loss.item())\n", + " loss_kl += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_kl = loss_kl.item()\n", + "print(f\"Total KL Divergence Loss: {total_loss_kl}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_kl = [(layer_loss / total_loss_kl) * 100 for layer_loss in layer_losses_kl]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_kl):\n", + " print(f\"Layer {i} KL Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_kl[i]:.4f}%\")\n", + "## 두 모델 간의 차이가 주로 Layer 2, Layer 10, 그리고 Layer 11에서 발생" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From e2fe43c626e209f874615779f5970f4e74f8c115 Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 15 Feb 2025 17:14:59 +0900 Subject: [PATCH 5/7] Update attention_rollout.ipynb --- attention_rollout.ipynb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb index a9b4f4a..4fadcee 100644 --- a/attention_rollout.ipynb +++ b/attention_rollout.ipynb @@ -1,3 +1,25 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- attn_map\n", + "\n", + "attention map은 모델이 입력의 각 부분에 대해 얼마나 집중하는지를 나타내는 텐서. 각 입력의 특징들이 서로 어떻게 상호작용하는지를 보여줌.\n", + "입력 패치 간의 상관관계를 시각화하거나, 어떤 입력 부분이 모델의 예측에 더 큰 영향을 미치는지를 분석하는 데 사용.\n", + "\n", + "- cls_weight\n", + "\n", + "cls_weight는 CLS (Classification) 토큰에 대한 주의 맵의 특정 값으로, 주로 클래스별 가중치를 나타냄. 이 값은 모델이 특정 클래스에 대해 얼마나 집중하는지를 나타내는 정보.\n", + "각 블록에서 CLS 토큰에 대한 attention map 추출하여 클래스에 대한 가중치를 계산하는 데 사용. 이 값은 분류 작업에서 모델의 신뢰도를 평가.\n", + "\n", + "- cls_weights_rollout\n", + "\n", + "각 블록에서 추출된 CLS 토큰에 대한 주의 맵을 기반으로 만든 클래스 가중치의 리스트. 이 리스트는 모델의 여러 블록에서의 CLS 토큰에 대한 가중치를 포함하고 있음.\n", + "모델이 여러 블록에서 CLS 토큰에 대해 어떻게 집중하는지를 종합적으로 보여줌. 이를 통해 특정 클래스에 대한 모델의 전반적인 집중도를 평가할 수 있음." + ] + }, { "cells": [ { From b5ec1ef2a1e9a43a379fa58456cb8dbc4b407a2b Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 15 Feb 2025 17:15:18 +0900 Subject: [PATCH 6/7] Delete attention_rollout.ipynb --- attention_rollout.ipynb | 1634 --------------------------------------- 1 file changed, 1634 deletions(-) delete mode 100644 attention_rollout.ipynb diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb deleted file mode 100644 index 4fadcee..0000000 --- a/attention_rollout.ipynb +++ /dev/null @@ -1,1634 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- attn_map\n", - "\n", - "attention map은 모델이 입력의 각 부분에 대해 얼마나 집중하는지를 나타내는 텐서. 각 입력의 특징들이 서로 어떻게 상호작용하는지를 보여줌.\n", - "입력 패치 간의 상관관계를 시각화하거나, 어떤 입력 부분이 모델의 예측에 더 큰 영향을 미치는지를 분석하는 데 사용.\n", - "\n", - "- cls_weight\n", - "\n", - "cls_weight는 CLS (Classification) 토큰에 대한 주의 맵의 특정 값으로, 주로 클래스별 가중치를 나타냄. 이 값은 모델이 특정 클래스에 대해 얼마나 집중하는지를 나타내는 정보.\n", - "각 블록에서 CLS 토큰에 대한 attention map 추출하여 클래스에 대한 가중치를 계산하는 데 사용. 이 값은 분류 작업에서 모델의 신뢰도를 평가.\n", - "\n", - "- cls_weights_rollout\n", - "\n", - "각 블록에서 추출된 CLS 토큰에 대한 주의 맵을 기반으로 만든 클래스 가중치의 리스트. 이 리스트는 모델의 여러 블록에서의 CLS 토큰에 대한 가중치를 포함하고 있음.\n", - "모델이 여러 블록에서 CLS 토큰에 대해 어떻게 집중하는지를 종합적으로 보여줌. 이를 통해 특정 클래스에 대한 모델의 전반적인 집중도를 평가할 수 있음." - ] - }, -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import torch\n", - "import torch.nn.functional as F\n", - "from tqdm import tqdm\n", - "import timm\n", - "assert timm.__version__ == \"0.3.2\"\n", - "from torchvision import transforms\n", - "from PIL import Image\n", - "import requests\n", - "import random\n", - "import numpy as np\n", - "\n", - "# 장치 설정\n", - "device = 'cuda' if torch.cuda.is_available() else 'cpu'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from torch.utils.data import DataLoader\n", - "from torchvision.datasets import ImageFolder\n", - "import torchvision.transforms as transforms\n", - "\n", - "# 데이터 전처리\n", - "transform = transforms.Compose([\n", - " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", - "])\n", - "\n", - "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", - "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "def my_forward_wrapper(attn_obj):\n", - " def my_forward(x):\n", - " B, N, C = x.shape\n", - " qkv = attn_obj.qkv(x).reshape(B, N, 3, attn_obj.num_heads, C // attn_obj.num_heads).permute(2, 0, 3, 1, 4)\n", - " q, k, v = qkv.unbind(0)\n", - "\n", - " attn = (q @ k.transpose(-2, -1)) * attn_obj.scale\n", - " attn = attn.softmax(dim=-1)\n", - " attn = attn_obj.attn_drop(attn)\n", - "\n", - " # attn_map을 저장\n", - " attn_maps.append(attn.detach()) # 주의 가중치를 저장\n", - " attn_obj.cls_attn_map = attn[:, :, 0, 2:] # cls attention map 저장\n", - "\n", - " x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n", - " x = attn_obj.proj(x)\n", - " x = attn_obj.proj_drop(x)\n", - " return x\n", - " return my_forward" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def attention_rollout_function(attn_maps):\n", - " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", - " attn_rollout = []\n", - "\n", - " for attn_map in attn_maps:\n", - " # 현재 주의 맵의 크기 확인\n", - " current_size = attn_map.size(-1)\n", - " I = torch.eye(max(current_size, I_size)).to(attn_map.device) # 동적 아이덴티티 행렬 생성\n", - "\n", - " if current_size < I_size:\n", - " padding_size = I_size - current_size\n", - " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", - " I_padded = F.pad(I, (0, padding_size, 0, 0)) # 아이덴티티 행렬 패딩\n", - " prod = padded_attn_map + I_padded\n", - " else:\n", - " prod = attn_map + I\n", - "\n", - " attn_rollout.append(prod / prod.sum(dim=-1, keepdim=True)) # 정규화\n", - "\n", - " return attn_rollout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "DeiT-small" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" - ] - }, - { - "data": { - "text/plain": [ - "VisionTransformer(\n", - " (patch_embed): PatchEmbed(\n", - " (proj): Conv2d(3, 384, kernel_size=(16, 16), stride=(16, 16))\n", - " )\n", - " (pos_drop): Dropout(p=0.0, inplace=False)\n", - " (blocks): ModuleList(\n", - " (0): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (1): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (2): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (3): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (4): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (5): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (6): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (7): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (8): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (9): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (10): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (11): Block(\n", - " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=384, out_features=384, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (norm): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", - " (head): Linear(in_features=384, out_features=1000, bias=True)\n", - ")" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)\n", - "model" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of blocks in the model: 12\n" - ] - } - ], - "source": [ - "print(f\"Number of blocks in the model: {len(model.blocks)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.eval()\n", - "\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 12/12 [00:00<00:00, 292.63it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Block 1: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 2: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 3: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 4: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 5: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 6: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 7: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 8: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 9: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 10: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 11: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "Block 12: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", - "CLS Weight Shape: torch.Size([1, 194])\n", - "Attention Rollout: torch.Size([1, 6, 196, 196])\n", - "Combined CLS Weight Shape: torch.Size([1, 6, 194])\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "attn_maps = [] # 전역 리스트를 사용하여 attn_map을 저장\n", - "cls_weights = []\n", - "\n", - "# 각 블록을 순회하며 attention map과 class attention map을 추출\n", - "for block in tqdm(model.blocks):\n", - " wrapped_forward = my_forward_wrapper(block.attn)\n", - " \n", - " # 입력 텐서가 필요하므로 임의의 입력을 생성\n", - " x = torch.randn(1, 196, 384) # B=1, N=196, C=384\n", - " output = wrapped_forward(x)\n", - "\n", - " # attn_map이 저장되었는지 확인합니다.\n", - " if attn_maps:\n", - " print(f\"Block {len(attn_maps)}: Attention map collected with shape {attn_maps[-1].shape}\")\n", - " else:\n", - " print(f\"Block {len(attn_maps)}: No attention map collected.\")\n", - "\n", - " # cls_attn_map 저장\n", - " cls_attn_map = block.attn.cls_attn_map.detach() # cls_attn_map 저장\n", - " cls_weights.append(cls_attn_map)\n", - "\n", - "# 주의 맵이 저장되었는지 확인합니다.\n", - "if attn_maps:\n", - " attn_map = attn_maps[-1].mean(dim=1).squeeze(0).detach()\n", - "else:\n", - " print(\"attn_maps 내용:\", attn_maps) # 디버깅 출력 추가\n", - " raise ValueError(\"No attention maps were collected.\")\n", - "\n", - "cls_weight = cls_weights[-1].max(dim=1).values.detach()\n", - "\n", - "# Ensure tensors are on the CPU\n", - "attn_map_cpu = attn_map.cpu()\n", - "cls_weight_cpu = cls_weight.cpu()\n", - "\n", - "# 클래스 가중치의 크기를 확인합니다.\n", - "print(\"CLS Weight Shape:\", cls_weight.shape)\n", - "\n", - "# Combine class scores of all blocks\n", - "try:\n", - " cls_weight_combined = torch.prod(torch.stack(cls_weights), dim=0)\n", - "except RuntimeError as e:\n", - " print(f\"Error combining class weights: {e}\")\n", - "\n", - "# 주의 맵의 곱을 계산합니다\n", - "attn_maps_prod = torch.prod(torch.stack(attn_maps), dim=0)\n", - "\n", - "# CPU로 전환\n", - "attn_maps_cpu = [attn_map.cpu() for attn_map in attn_maps]\n", - "cls_weights_cpu = [cls_weight.cpu() for cls_weight in cls_weights]\n", - "\n", - "# 결과 확인\n", - "print(\"Attention Rollout:\", attn_maps_prod.shape)\n", - "print(\"Combined CLS Weight Shape:\", cls_weight_combined.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 12/12 [00:00<00:00, 12003.73it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Class weight tensor shape before view: torch.Size([196, 196])\n", - "Attention Rollout Length: 12\n", - "Attention Rollout Shapes: [torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196])]\n", - "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "def attention_rollout_function(attn_maps):\n", - " attn_rollout = []\n", - " I = torch.eye(attn_maps[0].shape[-1]) # Identity matrix\n", - " prod = I\n", - " for i, attn_map in enumerate(attn_maps):\n", - " prod = prod @ (attn_map + I) # Product of attention maps with identity matrix\n", - " \n", - " prod = prod / prod.sum(dim=-1, keepdim=True) # Normalize\n", - " attn_rollout.append(prod)\n", - " return attn_rollout\n", - "\n", - "# Attention Rollout\n", - "attn_rollout = attention_rollout_function(attn_maps_cpu)\n", - "\n", - "# For Class Weights\n", - "cls_weights_rollout = []\n", - "\n", - "for i in tqdm(range(len(attn_rollout))): # attn_rollout의 크기로 반복\n", - " cls_weight_tensor = attn_rollout[i][0, 0, :, :] # 첫 번째 헤드 사용\n", - "\n", - " # 텐서의 크기를 출력하여 디버깅\n", - " print(f\"Class weight tensor shape before view: {cls_weight_tensor.shape}\")\n", - "\n", - " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인합니다.\n", - " if cls_weight_tensor.numel() == 14 * 14:\n", - " cls_weights_rollout.append(cls_weight_tensor.view(14, 14))\n", - " elif cls_weight_tensor.numel() == 196 * 196:\n", - " # 196x196 텐서를 클래스 가중치로 사용\n", - " cls_weights_rollout.append(cls_weight_tensor) # 변환하지 않고 그대로 사용\n", - " else:\n", - " print(f\"Skipping view for index {i}, tensor size is {cls_weight_tensor.numel()}\")\n", - "\n", - "# 결과 확인\n", - "if isinstance(attn_rollout, list):\n", - " print(\"Attention Rollout Length:\", len(attn_rollout))\n", - " print(\"Attention Rollout Shapes:\", [tensor.shape for tensor in attn_rollout])\n", - "else:\n", - " print(\"Attention Rollout Shape:\", attn_rollout.shape)\n", - "\n", - "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout])" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0.0092, 0.0108, 0.0016, ..., 0.0033, 0.0031, 0.0041],\n", - " [0.0025, 0.0249, 0.0017, ..., 0.0045, 0.0037, 0.0045],\n", - " [0.0022, 0.0028, 0.0083, ..., 0.0025, 0.0050, 0.0159],\n", - " ...,\n", - " [0.0022, 0.0089, 0.0015, ..., 0.0199, 0.0061, 0.0046],\n", - " [0.0014, 0.0043, 0.0024, ..., 0.0043, 0.0238, 0.0065],\n", - " [0.0042, 0.0048, 0.0018, ..., 0.0042, 0.0047, 0.0244]])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "attn_map" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0.0028, 0.0103, 0.0145, 0.0100, 0.0049, 0.0047, 0.0083, 0.0101, 0.0075,\n", - " 0.0024, 0.0125, 0.0012, 0.0042, 0.0037, 0.0073, 0.0071, 0.0119, 0.0110,\n", - " 0.0049, 0.0063, 0.0066, 0.0083, 0.0065, 0.0068, 0.0030, 0.0534, 0.0020,\n", - " 0.0335, 0.0037, 0.0035, 0.0532, 0.0121, 0.0040, 0.0036, 0.0115, 0.0019,\n", - " 0.0060, 0.0106, 0.0159, 0.0096, 0.0117, 0.0069, 0.0179, 0.0196, 0.0382,\n", - " 0.0029, 0.0084, 0.0023, 0.0139, 0.0022, 0.0152, 0.0044, 0.0024, 0.0139,\n", - " 0.0097, 0.0583, 0.0025, 0.0032, 0.0253, 0.0018, 0.0032, 0.0034, 0.0056,\n", - " 0.0191, 0.0047, 0.0171, 0.0095, 0.0093, 0.0037, 0.0014, 0.0151, 0.0051,\n", - " 0.0043, 0.0063, 0.0054, 0.0231, 0.0095, 0.0082, 0.0100, 0.0046, 0.0108,\n", - " 0.0045, 0.0025, 0.0169, 0.0107, 0.0023, 0.0054, 0.0060, 0.0040, 0.0103,\n", - " 0.0049, 0.0091, 0.0056, 0.0041, 0.1168, 0.0315, 0.0022, 0.0107, 0.0085,\n", - " 0.0040, 0.0037, 0.0197, 0.0153, 0.0179, 0.0044, 0.0178, 0.0102, 0.0677,\n", - " 0.0103, 0.0061, 0.0065, 0.0028, 0.0051, 0.0014, 0.0034, 0.0241, 0.0049,\n", - " 0.0142, 0.0090, 0.0042, 0.0320, 0.0069, 0.0044, 0.0044, 0.0182, 0.0101,\n", - " 0.0042, 0.0026, 0.1099, 0.0097, 0.0284, 0.0165, 0.1040, 0.0059, 0.0176,\n", - " 0.0036, 0.0107, 0.0176, 0.0098, 0.0038, 0.0078, 0.0069, 0.0027, 0.0309,\n", - " 0.0057, 0.0041, 0.0194, 0.0011, 0.0053, 0.0026, 0.0038, 0.0306, 0.0064,\n", - " 0.0205, 0.0032, 0.0127, 0.0057, 0.0024, 0.0022, 0.0252, 0.0051, 0.0481,\n", - " 0.0108, 0.0292, 0.0301, 0.0071, 0.0176, 0.0060, 0.0037, 0.0065, 0.0056,\n", - " 0.0025, 0.0069, 0.0224, 0.0032, 0.0108, 0.0104, 0.0012, 0.0050, 0.0022,\n", - " 0.0089, 0.0038, 0.0131, 0.0068, 0.0124, 0.0128, 0.0195, 0.0075, 0.0050,\n", - " 0.0138, 0.0089, 0.0084, 0.0060, 0.0109]])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weight" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([[5.0145e-01, 1.0773e-04, 1.9219e-09, ..., 3.0224e-07, 4.8908e-08,\n", - " 9.4525e-09],\n", - " [1.7469e-07, 7.3549e-01, 1.9689e-09, ..., 1.5312e-05, 1.8353e-07,\n", - " 9.8260e-07],\n", - " [2.0269e-04, 8.8949e-03, 7.8070e-01, ..., 8.4695e-05, 3.8640e-05,\n", - " 1.5120e-02],\n", - " ...,\n", - " [3.1366e-08, 1.4511e-03, 1.2052e-10, ..., 5.2665e-01, 1.4668e-05,\n", - " 1.2199e-06],\n", - " [6.7066e-11, 3.5859e-06, 1.0056e-12, ..., 1.3806e-06, 5.0115e-01,\n", - " 6.2253e-07],\n", - " [2.0210e-10, 8.3169e-06, 4.0810e-09, ..., 2.4031e-07, 3.3919e-06,\n", - " 5.2196e-01]]),\n", - " tensor([[2.8310e-01, 7.6627e-04, 1.8418e-03, ..., 1.2459e-04, 8.3037e-03,\n", - " 8.7034e-04],\n", - " [3.8865e-04, 3.8519e-01, 1.1187e-03, ..., 5.4312e-04, 1.4127e-02,\n", - " 8.7212e-04],\n", - " [2.2728e-04, 4.7906e-03, 7.5910e-01, ..., 1.1250e-04, 3.2742e-03,\n", - " 9.5180e-03],\n", - " ...,\n", - " [2.5900e-04, 1.3777e-03, 2.0527e-03, ..., 4.1472e-01, 1.6249e-03,\n", - " 2.4871e-04],\n", - " [2.2064e-04, 5.9052e-04, 1.7989e-03, ..., 1.3416e-04, 4.2058e-01,\n", - " 4.4896e-04],\n", - " [1.6595e-04, 5.9181e-04, 1.7374e-03, ..., 1.4127e-04, 6.7444e-02,\n", - " 3.2628e-01]]),\n", - " tensor([[0.1570, 0.0010, 0.0056, ..., 0.0006, 0.0064, 0.0009],\n", - " [0.0135, 0.1933, 0.0032, ..., 0.0010, 0.0105, 0.0013],\n", - " [0.0069, 0.0031, 0.3877, ..., 0.0009, 0.0040, 0.0054],\n", - " ...,\n", - " [0.0108, 0.0015, 0.0052, ..., 0.2111, 0.0023, 0.0007],\n", - " [0.0091, 0.0012, 0.0048, ..., 0.0011, 0.2260, 0.0007],\n", - " [0.0097, 0.0011, 0.0052, ..., 0.0007, 0.0395, 0.1640]]),\n", - " tensor([[0.1081, 0.0055, 0.0075, ..., 0.0018, 0.0058, 0.0013],\n", - " [0.0112, 0.1038, 0.0062, ..., 0.0022, 0.0086, 0.0024],\n", - " [0.0063, 0.0048, 0.1985, ..., 0.0013, 0.0051, 0.0036],\n", - " ...,\n", - " [0.0089, 0.0055, 0.0061, ..., 0.1082, 0.0036, 0.0019],\n", - " [0.0075, 0.0059, 0.0061, ..., 0.0019, 0.1328, 0.0011],\n", - " [0.0081, 0.0053, 0.0061, ..., 0.0017, 0.0251, 0.0829]]),\n", - " tensor([[0.0596, 0.0034, 0.0052, ..., 0.0040, 0.0040, 0.0009],\n", - " [0.0106, 0.0527, 0.0047, ..., 0.0043, 0.0053, 0.0014],\n", - " [0.0083, 0.0031, 0.1010, ..., 0.0042, 0.0034, 0.0020],\n", - " ...,\n", - " [0.0104, 0.0036, 0.0045, ..., 0.0581, 0.0028, 0.0011],\n", - " [0.0099, 0.0036, 0.0045, ..., 0.0044, 0.0673, 0.0008],\n", - " [0.0101, 0.0034, 0.0049, ..., 0.0041, 0.0135, 0.0417]]),\n", - " tensor([[0.0315, 0.0021, 0.0068, ..., 0.0030, 0.0032, 0.0032],\n", - " [0.0069, 0.0267, 0.0069, ..., 0.0031, 0.0038, 0.0035],\n", - " [0.0055, 0.0019, 0.0546, ..., 0.0032, 0.0028, 0.0039],\n", - " ...,\n", - " [0.0069, 0.0022, 0.0068, ..., 0.0300, 0.0025, 0.0039],\n", - " [0.0066, 0.0022, 0.0065, ..., 0.0033, 0.0347, 0.0031],\n", - " [0.0067, 0.0021, 0.0068, ..., 0.0031, 0.0079, 0.0236]]),\n", - " tensor([[0.0174, 0.0023, 0.0059, ..., 0.0040, 0.0037, 0.0039],\n", - " [0.0049, 0.0145, 0.0060, ..., 0.0040, 0.0040, 0.0042],\n", - " [0.0042, 0.0021, 0.0310, ..., 0.0039, 0.0035, 0.0043],\n", - " ...,\n", - " [0.0050, 0.0022, 0.0059, ..., 0.0175, 0.0033, 0.0044],\n", - " [0.0048, 0.0023, 0.0058, ..., 0.0041, 0.0198, 0.0039],\n", - " [0.0049, 0.0022, 0.0060, ..., 0.0041, 0.0060, 0.0143]]),\n", - " tensor([[0.0093, 0.0036, 0.0040, ..., 0.0063, 0.0032, 0.0035],\n", - " [0.0031, 0.0097, 0.0041, ..., 0.0063, 0.0034, 0.0036],\n", - " [0.0027, 0.0035, 0.0166, ..., 0.0062, 0.0031, 0.0037],\n", - " ...,\n", - " [0.0031, 0.0036, 0.0040, ..., 0.0135, 0.0030, 0.0038],\n", - " [0.0030, 0.0036, 0.0040, ..., 0.0063, 0.0113, 0.0034],\n", - " [0.0031, 0.0036, 0.0041, ..., 0.0063, 0.0044, 0.0090]]),\n", - " tensor([[0.0069, 0.0032, 0.0029, ..., 0.0067, 0.0042, 0.0037],\n", - " [0.0038, 0.0062, 0.0029, ..., 0.0067, 0.0042, 0.0038],\n", - " [0.0035, 0.0031, 0.0092, ..., 0.0067, 0.0042, 0.0038],\n", - " ...,\n", - " [0.0038, 0.0032, 0.0029, ..., 0.0103, 0.0041, 0.0039],\n", - " [0.0037, 0.0031, 0.0029, ..., 0.0067, 0.0082, 0.0037],\n", - " [0.0037, 0.0032, 0.0029, ..., 0.0067, 0.0047, 0.0065]]),\n", - " tensor([[0.0074, 0.0034, 0.0031, ..., 0.0049, 0.0035, 0.0039],\n", - " [0.0058, 0.0049, 0.0031, ..., 0.0049, 0.0036, 0.0040],\n", - " [0.0058, 0.0034, 0.0063, ..., 0.0049, 0.0036, 0.0040],\n", - " ...,\n", - " [0.0058, 0.0034, 0.0031, ..., 0.0067, 0.0035, 0.0040],\n", - " [0.0058, 0.0034, 0.0031, ..., 0.0048, 0.0056, 0.0039],\n", - " [0.0058, 0.0034, 0.0031, ..., 0.0049, 0.0038, 0.0053]]),\n", - " tensor([[0.0092, 0.0024, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", - " [0.0084, 0.0032, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", - " [0.0084, 0.0024, 0.0062, ..., 0.0049, 0.0041, 0.0086],\n", - " ...,\n", - " [0.0084, 0.0024, 0.0046, ..., 0.0058, 0.0041, 0.0086],\n", - " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0051, 0.0086],\n", - " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0042, 0.0093]]),\n", - " tensor([[0.0056, 0.0037, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", - " [0.0052, 0.0040, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", - " [0.0051, 0.0036, 0.0040, ..., 0.0038, 0.0035, 0.0080],\n", - " ...,\n", - " [0.0052, 0.0037, 0.0032, ..., 0.0043, 0.0035, 0.0080],\n", - " [0.0051, 0.0036, 0.0032, ..., 0.0038, 0.0040, 0.0080],\n", - " [0.0051, 0.0037, 0.0032, ..., 0.0038, 0.0036, 0.0084]])]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weights_rollout" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "DeiT-tiny -> student model로 가정" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" - ] - }, - { - "data": { - "text/plain": [ - "VisionTransformer(\n", - " (patch_embed): PatchEmbed(\n", - " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", - " )\n", - " (pos_drop): Dropout(p=0.0, inplace=False)\n", - " (blocks): ModuleList(\n", - " (0): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (1): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (2): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (3): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (4): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (5): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (6): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (7): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (8): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (9): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (10): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " (11): Block(\n", - " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (attn): Attention(\n", - " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", - " (attn_drop): Dropout(p=0.0, inplace=False)\n", - " (proj): Linear(in_features=192, out_features=192, bias=True)\n", - " (proj_drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (drop_path): Identity()\n", - " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (mlp): Mlp(\n", - " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", - " (act): GELU()\n", - " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", - " (drop): Dropout(p=0.0, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", - " (head): Linear(in_features=192, out_features=1000, bias=True)\n", - ")" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model_st = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)\n", - "model_st" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of blocks in the model: 12\n" - ] - } - ], - "source": [ - "print(f\"Number of blocks in the model: {len(model_st.blocks)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model_st.eval()\n", - "\n", - "seed = 42\n", - "random.seed(seed)\n", - "np.random.seed(seed)\n", - "torch.manual_seed(seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 12/12 [00:00<00:00, 266.64it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collected attention maps: 12\n", - "CLS Weight Shape: torch.Size([1])\n", - "Attention Rollout: torch.Size([1, 196, 192])\n", - "Combined CLS Weight Shape: torch.Size([1, 196])\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# 전역 리스트를 사용하여 attn_map과 cls_weight 저장\n", - "attn_maps_st = []\n", - "cls_weights_st = []\n", - "\n", - "# Forward hook을 설정하는 함수\n", - "def get_attention_map(module, input, output):\n", - " attn_maps_st.append(output.detach()) # 주의 맵 저장\n", - " cls_attn_map = output[:, :, 0] # CLS token에 대한 attention map을 추출\n", - " cls_weights_st.append(cls_attn_map.detach()) # 클래스 주의 맵 저장\n", - "\n", - "# 각 블록을 순회하며 attention map과 class attention map을 추출합니다.\n", - "for block in tqdm(model_st.blocks):\n", - " # Forward hook 등록\n", - " hook = block.attn.register_forward_hook(get_attention_map)\n", - " \n", - " # 입력 텐서가 필요하므로 임의의 입력을 생성합니다.\n", - " x = torch.randn(1, 3, 224, 224).to(device) # B=1, C=3, H=224, W=224\n", - " x_patch = model_st.patch_embed(x) # 패치 임베딩 변환\n", - "\n", - " # wrapped_forward 함수로 출력 계산\n", - " output = block.attn(x_patch)\n", - "\n", - " # hook 해제\n", - " hook.remove()\n", - "\n", - "# 주의 맵이 저장되었는지 확인합니다.\n", - "if attn_maps_st:\n", - " attn_map_st = attn_maps_st[-1].mean(dim=1).squeeze(0).detach()\n", - " print(f\"Collected attention maps: {len(attn_maps_st)}\")\n", - "else:\n", - " print(\"attn_maps_st 내용:\", attn_maps_st) # 디버깅 출력 추가\n", - " raise ValueError(\"No attention maps were collected.\")\n", - "\n", - "# cls_weight 계산\n", - "cls_weight_st = cls_weights_st[-1].max(dim=1).values.detach()\n", - "\n", - "# Ensure tensors are on the CPU\n", - "attn_map_st_cpu = attn_map_st.cpu()\n", - "cls_weight_st_cpu = cls_weight_st.cpu()\n", - "\n", - "# 클래스 가중치의 크기를 확인합니다.\n", - "print(\"CLS Weight Shape:\", cls_weight_st.shape)\n", - "\n", - "# Combine class scores of all blocks\n", - "try:\n", - " cls_weight_st_combined = torch.prod(torch.stack(cls_weights_st), dim=0)\n", - "except RuntimeError as e:\n", - " print(f\"Error combining class weights: {e}\")\n", - "\n", - "# 주의 맵의 곱을 계산합니다\n", - "attn_maps_st_prod = torch.prod(torch.stack(attn_maps_st), dim=0)\n", - "\n", - "# CPU로 전환\n", - "attn_maps_st_cpu = [attn_map.cpu() for attn_map in attn_maps_st]\n", - "cls_weights_st_cpu = [cls_weight.cpu() for cls_weight in cls_weights_st]\n", - "\n", - "# 결과 확인\n", - "print(\"Attention Rollout:\", attn_maps_st_prod.shape)\n", - "print(\"Combined CLS Weight Shape:\", cls_weight_st_combined.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "# Attention Rollout\n", - "def attention_rollout_function(attn_maps):\n", - " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", - " I = torch.eye(I_size).to(attn_maps[0].device) # Identity matrix\n", - " attn_rollout = []\n", - "\n", - " for attn_map in attn_maps:\n", - " if attn_map.size(-1) == I_size:\n", - " prod = attn_map + I\n", - " elif attn_map.size(-1) < I_size:\n", - " padding_size = I_size - attn_map.size(-1)\n", - " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", - " prod = padded_attn_map + I\n", - " else:\n", - " raise ValueError(\"Unexpected attention map size.\")\n", - "\n", - " attn_rollout.append(prod)\n", - "\n", - " attn_rollout = [prod / prod.sum(dim=-1, keepdim=True) for prod in attn_rollout] # Normalize\n", - " return attn_rollout" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" - ] - } - ], - "source": [ - "# Attention Rollout 실행\n", - "attn_rollout_st = attention_rollout_function(attn_maps_st_cpu)\n", - "\n", - "# For Class Weights\n", - "cls_weights_rollout_st = []\n", - "\n", - "for i in range(len(attn_rollout_st)): # attn_rollout_st의 크기로 반복\n", - " cls_weight_tensor_st = attn_rollout_st[i][0] # 첫 번째 배치의 텐서 사용\n", - "\n", - " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인\n", - " if cls_weight_tensor_st.numel() == 14 * 14:\n", - " cls_weights_rollout_st.append(cls_weight_tensor_st.view(14, 14))\n", - " elif cls_weight_tensor_st.numel() == 196 * 196:\n", - " cls_weights_rollout_st.append(cls_weight_tensor_st) # 그대로 사용\n", - "\n", - "# 결과 확인\n", - "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout_st])" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([ 0.1842, 0.6513, 0.9481, -0.0575, -0.4596, 0.9378, -0.0630, 0.0913,\n", - " -0.2760, 0.8236, -0.3048, -0.5067, -0.2237, 0.0104, 0.2620, 0.4669,\n", - " 0.3408, -0.6330, -0.4091, -0.4896, -0.9567, -1.5660, -0.2059, -0.2811,\n", - " -0.2813, -0.8560, -0.1496, -0.0455, -0.0833, 0.3670, 0.2738, -0.3592,\n", - " 0.3951, 0.6094, 0.5716, -0.0526, 0.3808, -0.0236, -0.7082, 0.1859,\n", - " 0.8467, -0.0996, 0.4303, 0.0613, 0.7428, 0.1889, -0.0631, -0.1368,\n", - " 0.1214, 0.5921, 0.1423, -0.4978, 0.5646, -0.2145, 0.1315, -0.3321,\n", - " 0.1872, -0.0260, -0.5459, 0.0105, 0.7164, -0.4216, -0.2821, 0.2232,\n", - " -0.1487, 0.1884, -0.1680, 0.3930, -0.3026, 0.8395, -0.5283, -0.1191,\n", - " 0.5188, -1.0522, 0.4154, -0.1342, -0.8908, 1.1572, 0.0318, 0.0403,\n", - " -0.4454, 0.0902, -0.5127, 0.6663, -0.3861, 0.5882, 0.0156, 0.5951,\n", - " -0.7754, 0.4059, 0.1418, 0.9044, -1.3180, -0.1740, 0.9519, -0.7662,\n", - " -0.1940, -0.4566, 0.9794, 0.2629, 0.3623, -0.1166, 0.5362, -0.2732,\n", - " -0.5911, -0.4877, 0.0517, 0.5354, 0.2801, -0.0113, 0.1709, -0.7622,\n", - " -0.1145, -0.9085, 0.3520, -0.6743, 0.2920, 0.2248, -0.2348, -0.4925,\n", - " 0.1487, -0.0883, -0.2174, -0.1746, -0.0644, -0.1431, 0.5035, -0.4010,\n", - " -0.1813, -0.0585, -0.1536, 0.7160, -0.5562, -0.1020, 0.2226, 0.2361,\n", - " 0.5698, -0.2106, 0.9990, -1.1657, -1.0541, 0.4736, 0.7887, 0.2140,\n", - " -0.4918, -0.7655, 1.2153, -0.0707, -0.4830, 0.2197, 0.0948, 0.8601,\n", - " 0.0194, 0.3300, -0.4195, 0.2778, -0.2720, -0.9317, -0.2371, 0.2978,\n", - " -0.3443, 1.3332, -0.0444, -0.4376, -0.2210, -0.0033, -0.0126, -0.0402,\n", - " -1.3693, -0.4690, 0.2604, 0.3911, -0.3407, 0.6645, -0.1561, -0.1206,\n", - " 0.4401, -0.1128, 0.3281, 0.3493, 0.4086, 0.2105, -0.1624, -0.0317,\n", - " 0.2122, 0.2028, 0.5176, -0.5969, -0.6693, 0.1342, -0.5071, -0.3850])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "attn_map_st" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([0.2606])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weight_st" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([[-0.1554, -0.0586, -0.0729, ..., -0.0000, -0.0000, -0.0000],\n", - " [-0.0529, -0.2460, -0.0994, ..., -0.0000, -0.0000, -0.0000],\n", - " [-0.1179, -0.0144, -0.2249, ..., -0.0000, -0.0000, -0.0000],\n", - " ...,\n", - " [-0.0701, -0.1249, -0.0585, ..., -0.1516, -0.0000, -0.0000],\n", - " [ 0.0249, -0.1786, -0.0892, ..., -0.0000, -0.1801, -0.0000],\n", - " [-0.1276, -0.1296, -0.0653, ..., -0.0000, -0.0000, -0.1503]]),\n", - " tensor([[-0.5720, 0.0253, -0.8815, ..., -0.0000, -0.0000, -0.0000],\n", - " [-0.1371, -0.5842, -0.8806, ..., -0.0000, -0.0000, -0.0000],\n", - " [ 0.0658, -0.2696, -1.0837, ..., -0.0000, -0.0000, -0.0000],\n", - " ...,\n", - " [-0.0265, -0.1658, -0.9726, ..., -0.4945, -0.0000, -0.0000],\n", - " [-0.0341, -0.0129, -0.8782, ..., -0.0000, -0.4722, -0.0000],\n", - " [-0.0291, 0.0190, -1.0195, ..., -0.0000, -0.0000, -0.5399]]),\n", - " tensor([[ -5.2143, -0.2073, -9.1526, ..., -0.0000, -0.0000, -0.0000],\n", - " [ 1.0086, -6.6903, -8.8212, ..., -0.0000, -0.0000, -0.0000],\n", - " [ 1.6930, -0.4107, -17.7767, ..., -0.0000, -0.0000, -0.0000],\n", - " ...,\n", - " [ 1.9814, -0.8107, -13.0110, ..., -8.9967, -0.0000, -0.0000],\n", - " [ 0.9145, -0.1184, -6.4268, ..., -0.0000, -4.5341, -0.0000],\n", - " [ 1.8620, -1.2088, -13.1969, ..., -0.0000, -0.0000, -8.9710]]),\n", - " tensor([[ 0.4234, -0.1738, 0.4660, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0612, 0.4136, 0.4037, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0573, -0.1247, 0.9858, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [-0.0938, -0.0743, 0.4447, ..., 0.5810, 0.0000, 0.0000],\n", - " [-0.1542, -0.1397, 0.5147, ..., 0.0000, 0.5836, 0.0000],\n", - " [-0.1097, -0.1474, 0.4535, ..., 0.0000, 0.0000, 0.5374]]),\n", - " tensor([[ 0.9406, -0.2136, 0.8167, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.1652, 0.5770, 0.8471, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.1347, -0.2466, 1.5981, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.1693, -0.2130, 0.8331, ..., 0.8538, 0.0000, 0.0000],\n", - " [ 0.1927, -0.2818, 0.8108, ..., 0.0000, 0.8053, 0.0000],\n", - " [ 0.1685, -0.2378, 0.8006, ..., 0.0000, 0.0000, 0.7944]]),\n", - " tensor([[ 0.7328, 0.0801, 0.5444, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0403, 0.7458, 0.5566, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0786, 0.0814, 1.3531, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [-0.0527, 0.0911, 0.6341, ..., 0.6709, 0.0000, 0.0000],\n", - " [-0.0836, 0.1107, 0.5707, ..., 0.0000, 0.7083, 0.0000],\n", - " [-0.0249, 0.0901, 0.5255, ..., 0.0000, 0.0000, 0.6873]]),\n", - " tensor([[0.8320, 0.2851, 0.8615, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.1175, 0.9609, 0.8269, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.1498, 0.2262, 1.6439, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.1284, 0.2232, 0.8748, ..., 0.7061, 0.0000, 0.0000],\n", - " [0.1243, 0.2083, 0.9053, ..., 0.0000, 0.7440, 0.0000],\n", - " [0.1109, 0.2427, 0.8597, ..., 0.0000, 0.0000, 0.6755]]),\n", - " tensor([[ 0.7119, 0.2967, 0.5289, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.1194, 1.2095, 0.6030, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.1917, 0.3107, 1.4640, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [-0.1499, 0.3103, 0.6443, ..., 0.8962, 0.0000, 0.0000],\n", - " [-0.1631, 0.3251, 0.5534, ..., 0.0000, 0.8595, 0.0000],\n", - " [-0.1243, 0.2940, 0.6102, ..., 0.0000, 0.0000, 0.8932]]),\n", - " tensor([[ 3.0915, -0.7572, 1.6884, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.8447, 1.3794, 1.6422, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.8184, -0.8285, 3.8009, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.7615, -0.8503, 1.5874, ..., 2.2260, 0.0000, 0.0000],\n", - " [ 0.8272, -0.8181, 1.6270, ..., 0.0000, 2.1608, 0.0000],\n", - " [ 0.8811, -0.7469, 1.4999, ..., 0.0000, 0.0000, 2.1590]]),\n", - " tensor([[ 2.4223, -0.3267, 0.9755, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.2824, 1.6931, 0.8163, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 0.3807, -0.1291, 2.9314, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.3839, -0.2936, 0.9776, ..., 2.0073, 0.0000, 0.0000],\n", - " [ 0.4626, -0.2595, 0.9444, ..., 0.0000, 2.0228, 0.0000],\n", - " [ 0.4498, -0.2101, 0.9627, ..., 0.0000, 0.0000, 1.8273]]),\n", - " tensor([[15.2405, 2.7610, 6.7306, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 9.2874, 22.5292, 11.1778, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 6.0527, 2.1433, 18.1478, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 8.6357, 3.6522, 9.4287, ..., 15.7923, 0.0000, 0.0000],\n", - " [ 4.7526, 2.0569, 6.1020, ..., 0.0000, 8.8777, 0.0000],\n", - " [ 3.7997, 1.8325, 4.7829, ..., 0.0000, 0.0000, 7.1644]]),\n", - " tensor([[ 3.2594, 1.9875, 2.6364, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 1.8766, 17.1216, 9.5131, ..., 0.0000, 0.0000, 0.0000],\n", - " [ 1.7089, 5.7129, 15.9202, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [ 0.4791, 2.2901, 3.2946, ..., 3.3042, 0.0000, 0.0000],\n", - " [ 0.7041, 3.5211, 4.8167, ..., 0.0000, 5.4057, 0.0000],\n", - " [ 1.7779, 8.7794, 10.9012, ..., 0.0000, 0.0000, 12.3839]])]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cls_weights_rollout_st" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "두 모델 간의 차이 Loss 계산 (L1, L2, Cosine Similarity, KL Divergence로 구현)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total L1 Loss: 14.616626739501953\n", - "Layer 0 L1 Loss: 0.1479, Percentage of Total Loss: 1.0115%\n", - "Layer 1 L1 Loss: 0.2261, Percentage of Total Loss: 1.5469%\n", - "Layer 2 L1 Loss: 5.1462, Percentage of Total Loss: 35.2075%\n", - "Layer 3 L1 Loss: 0.1554, Percentage of Total Loss: 1.0629%\n", - "Layer 4 L1 Loss: 0.2121, Percentage of Total Loss: 1.4508%\n", - "Layer 5 L1 Loss: 0.1961, Percentage of Total Loss: 1.3413%\n", - "Layer 6 L1 Loss: 0.2350, Percentage of Total Loss: 1.6081%\n", - "Layer 7 L1 Loss: 0.2223, Percentage of Total Loss: 1.5206%\n", - "Layer 8 L1 Loss: 0.5502, Percentage of Total Loss: 3.7645%\n", - "Layer 9 L1 Loss: 0.5793, Percentage of Total Loss: 3.9635%\n", - "Layer 10 L1 Loss: 4.3009, Percentage of Total Loss: 29.4244%\n", - "Layer 11 L1 Loss: 2.6453, Percentage of Total Loss: 18.0980%\n" - ] - } - ], - "source": [ - "# L1 (Mean Absolute Error): 두 모델의 각 레이어 가중치 간의 절대 차이 계산\n", - "# L1 손실 계산\n", - "loss_l1 = 0\n", - "layer_losses_l1 = []\n", - "\n", - "for i in range(len(cls_weights_rollout_st)):\n", - " # 레이어별 가중치 차이 계산\n", - " layer_loss = F.l1_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", - " layer_losses_l1.append(layer_loss.item()) # 손실을 리스트에 추가\n", - " loss_l1 += layer_loss\n", - "\n", - "# 총 손실 출력\n", - "total_loss_l1 = loss_l1.item()\n", - "print(f\"Total L1 Loss: {total_loss_l1}\")\n", - "\n", - "# 레이어별 손실 비율 계산\n", - "layer_loss_percentages_l1 = [(layer_loss / total_loss_l1) * 100 for layer_loss in layer_losses_l1]\n", - "\n", - "# 각 레이어 손실 및 비율 출력\n", - "for i, layer_loss in enumerate(layer_losses_l1):\n", - " print(f\"Layer {i} L1 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l1[i]:.4f}%\")\n", - "## 두 모델 간의 차이가 주로 Layer 2와 Layer 10에서 발생" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total L2 Loss: 453.0188903808594\n", - "Layer 0 L2 Loss: 0.0445, Percentage of Total Loss: 0.0098%\n", - "Layer 1 L2 Loss: 0.0991, Percentage of Total Loss: 0.0219%\n", - "Layer 2 L2 Loss: 394.6398, Percentage of Total Loss: 87.1133%\n", - "Layer 3 L2 Loss: 0.0411, Percentage of Total Loss: 0.0091%\n", - "Layer 4 L2 Loss: 0.0765, Percentage of Total Loss: 0.0169%\n", - "Layer 5 L2 Loss: 0.0602, Percentage of Total Loss: 0.0133%\n", - "Layer 6 L2 Loss: 0.0887, Percentage of Total Loss: 0.0196%\n", - "Layer 7 L2 Loss: 0.0799, Percentage of Total Loss: 0.0176%\n", - "Layer 8 L2 Loss: 0.5019, Percentage of Total Loss: 0.1108%\n", - "Layer 9 L2 Loss: 0.5323, Percentage of Total Loss: 0.1175%\n", - "Layer 10 L2 Loss: 38.7059, Percentage of Total Loss: 8.5440%\n", - "Layer 11 L2 Loss: 18.1490, Percentage of Total Loss: 4.0062%\n" - ] - } - ], - "source": [ - "# L2 (Euclidean Distance): 두 모델의 각 레이어 가중치 간의 유클리드 거리 계산\n", - "# L2 손실 계산\n", - "loss_l2 = 0\n", - "layer_losses_l2 = []\n", - "\n", - "for i in range(len(cls_weights_rollout_st)):\n", - " # 레이어별 가중치 차이 계산 (L2 손실)\n", - " layer_loss = F.mse_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", - " layer_losses_l2.append(layer_loss.item()) # 손실을 리스트에 추가\n", - " loss_l2 += layer_loss\n", - "\n", - "# 총 L2 손실 출력\n", - "total_loss_l2 = loss_l2.item()\n", - "print(f\"Total L2 Loss: {total_loss_l2}\")\n", - "\n", - "# 레이어별 L2 손실 비율 계산\n", - "layer_loss_percentages_l2 = [(layer_loss / total_loss_l2) * 100 for layer_loss in layer_losses_l2]\n", - "\n", - "# 각 레이어 L2 손실 및 비율 출력\n", - "for i, layer_loss in enumerate(layer_losses_l2):\n", - " print(f\"Layer {i} L2 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l2[i]:.4f}%\")\n", - "\n", - "## Layer 2: 2번째 레이어의 L2 손실이 394.6398로, 전체 손실의 87.11% -> 레이어2에서 두 모델 간의 가중치 차이가 매우 크다\n", - "## 나머지 레이어들은 상대적으로 손실 값이 매우 작고, 전체 손실에서 차지하는 비율도 낮음.\n", - "## layer 2가 두 모델의 성능 차이에 큰 기여하고 있을 것이라 판단할 수 있음. 두 모델의 구조나 파라미터 조정시 해당 레이어 더 보기!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total Cosine Loss: 11.714558601379395\n", - "Layer 0 Cosine Loss: 1.0442, Percentage of Total Loss: 8.9137%\n", - "Layer 1 Cosine Loss: 1.0548, Percentage of Total Loss: 9.0045%\n", - "Layer 2 Cosine Loss: 1.0349, Percentage of Total Loss: 8.8344%\n", - "Layer 3 Cosine Loss: 0.8728, Percentage of Total Loss: 7.4502%\n", - "Layer 4 Cosine Loss: 0.8025, Percentage of Total Loss: 6.8504%\n", - "Layer 5 Cosine Loss: 0.9400, Percentage of Total Loss: 8.0241%\n", - "Layer 6 Cosine Loss: 1.0328, Percentage of Total Loss: 8.8163%\n", - "Layer 7 Cosine Loss: 0.9947, Percentage of Total Loss: 8.4910%\n", - "Layer 8 Cosine Loss: 0.9695, Percentage of Total Loss: 8.2759%\n", - "Layer 9 Cosine Loss: 0.9745, Percentage of Total Loss: 8.3191%\n", - "Layer 10 Cosine Loss: 0.9839, Percentage of Total Loss: 8.3986%\n", - "Layer 11 Cosine Loss: 1.0100, Percentage of Total Loss: 8.6217%\n" - ] - } - ], - "source": [ - "# Cosine Similarity: 두 모델의 가중치 벡터 간의 코사인 유사도 계산\n", - "\n", - "from torch.nn.functional import cosine_similarity\n", - "\n", - "# 코사인 유사도 손실 계산\n", - "loss_cosine = 0\n", - "layer_losses_cosine = []\n", - "\n", - "for i in range(len(cls_weights_rollout_st)):\n", - " # 레이어별 가중치 차이 계산\n", - " layer_loss = 1 - cosine_similarity(cls_weights_rollout_st[i].view(1, -1), cls_weights_rollout[i].view(1, -1))\n", - " layer_losses_cosine.append(layer_loss.item()) # 손실을 리스트에 추가\n", - " loss_cosine += layer_loss\n", - "\n", - "# 총 손실 출력\n", - "total_loss_cosine = loss_cosine.item()\n", - "print(f\"Total Cosine Loss: {total_loss_cosine}\")\n", - "\n", - "# 레이어별 손실 비율 계산\n", - "layer_loss_percentages_cosine = [(layer_loss / total_loss_cosine) * 100 for layer_loss in layer_losses_cosine]\n", - "\n", - "# 각 레이어 손실 및 비율 출력\n", - "for i, layer_loss in enumerate(layer_losses_cosine):\n", - " print(f\"Layer {i} Cosine Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_cosine[i]:.4f}%\")\n", - "\n", - "## 각 레이어의 손실은 0.8에서 1.1 사이로 분포 -> 두 모델의 가중치가 레이어별로 상당히 유사\n", - "## 레이어 0에서 레이어 11까지의 손실 값이 비슷한 수준이므로, 전체적으로 두 모델의 가중치가 유사" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total KL Divergence Loss: 31.367801666259766\n", - "Layer 0 KL Loss: 0.0042, Percentage of Total Loss: 0.0133%\n", - "Layer 1 KL Loss: 0.0067, Percentage of Total Loss: 0.0214%\n", - "Layer 2 KL Loss: 15.2732, Percentage of Total Loss: 48.6907%\n", - "Layer 3 KL Loss: 0.0010, Percentage of Total Loss: 0.0032%\n", - "Layer 4 KL Loss: 0.0025, Percentage of Total Loss: 0.0080%\n", - "Layer 5 KL Loss: 0.0023, Percentage of Total Loss: 0.0073%\n", - "Layer 6 KL Loss: 0.0020, Percentage of Total Loss: 0.0063%\n", - "Layer 7 KL Loss: 0.0030, Percentage of Total Loss: 0.0096%\n", - "Layer 8 KL Loss: 0.0287, Percentage of Total Loss: 0.0915%\n", - "Layer 9 KL Loss: 0.0246, Percentage of Total Loss: 0.0785%\n", - "Layer 10 KL Loss: 9.7367, Percentage of Total Loss: 31.0406%\n", - "Layer 11 KL Loss: 6.2829, Percentage of Total Loss: 20.0297%\n" - ] - } - ], - "source": [ - "# KL Divergence: 두 모델의 가중치를 확률 분포로 간주하고, Kullback-Leibler Divergence 계산하여 두 분포 간의 차이 측정\n", - "# KL 발산 계산\n", - "loss_kl = 0\n", - "layer_losses_kl = []\n", - "\n", - "for i in range(len(cls_weights_rollout_st)):\n", - " # 확률 분포로 정규화\n", - " p = F.softmax(cls_weights_rollout_st[i], dim=0) + 1e-10 # 작은 값 추가\n", - " q = F.softmax(cls_weights_rollout[i], dim=0) + 1e-10 # 작은 값 추가\n", - " \n", - " # KL 발산 계산\n", - " layer_loss = F.kl_div(p.log(), q, reduction='batchmean')\n", - " layer_losses_kl.append(layer_loss.item())\n", - " loss_kl += layer_loss\n", - "\n", - "# 총 손실 출력\n", - "total_loss_kl = loss_kl.item()\n", - "print(f\"Total KL Divergence Loss: {total_loss_kl}\")\n", - "\n", - "# 레이어별 손실 비율 계산\n", - "layer_loss_percentages_kl = [(layer_loss / total_loss_kl) * 100 for layer_loss in layer_losses_kl]\n", - "\n", - "# 각 레이어 손실 및 비율 출력\n", - "for i, layer_loss in enumerate(layer_losses_kl):\n", - " print(f\"Layer {i} KL Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_kl[i]:.4f}%\")\n", - "## 두 모델 간의 차이가 주로 Layer 2, Layer 10, 그리고 Layer 11에서 발생" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "brp1", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.21" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 024b72abc3012e9a10158398a9273f2240825778 Mon Sep 17 00:00:00 2001 From: seonahryu <123479764+seonahryu@users.noreply.github.com> Date: Sat, 15 Feb 2025 17:15:30 +0900 Subject: [PATCH 7/7] Add files via upload --- attention_rollout.ipynb | 1632 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1632 insertions(+) create mode 100644 attention_rollout.ipynb diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb new file mode 100644 index 0000000..1b55012 --- /dev/null +++ b/attention_rollout.ipynb @@ -0,0 +1,1632 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- attn_map\n", + "\n", + "attention map은 모델이 입력의 각 부분에 대해 얼마나 집중하는지를 나타내는 텐서. 각 입력의 특징들이 서로 어떻게 상호작용하는지를 보여줌.\n", + "입력 패치 간의 상관관계를 시각화하거나, 어떤 입력 부분이 모델의 예측에 더 큰 영향을 미치는지를 분석하는 데 사용.\n", + "\n", + "- cls_weight\n", + "\n", + "cls_weight는 CLS (Classification) 토큰에 대한 주의 맵의 특정 값으로, 주로 클래스별 가중치를 나타냄. 이 값은 모델이 특정 클래스에 대해 얼마나 집중하는지를 나타내는 정보.\n", + "각 블록에서 CLS 토큰에 대한 attention map 추출하여 클래스에 대한 가중치를 계산하는 데 사용. 이 값은 분류 작업에서 모델의 신뢰도를 평가.\n", + "\n", + "- cls_weights_rollout\n", + "\n", + "각 블록에서 추출된 CLS 토큰에 대한 주의 맵을 기반으로 만든 클래스 가중치의 리스트. 이 리스트는 모델의 여러 블록에서의 CLS 토큰에 대한 가중치를 포함하고 있음.\n", + "모델이 여러 블록에서 CLS 토큰에 대해 어떻게 집중하는지를 종합적으로 보여줌. 이를 통해 특정 클래스에 대한 모델의 전반적인 집중도를 평가할 수 있음." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "from tqdm import tqdm\n", + "import timm\n", + "assert timm.__version__ == \"0.3.2\"\n", + "from torchvision import transforms\n", + "from PIL import Image\n", + "import requests\n", + "import random\n", + "import numpy as np\n", + "\n", + "# 장치 설정\n", + "device = 'cuda' if torch.cuda.is_available() else 'cpu'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def my_forward_wrapper(attn_obj):\n", + " def my_forward(x):\n", + " B, N, C = x.shape\n", + " qkv = attn_obj.qkv(x).reshape(B, N, 3, attn_obj.num_heads, C // attn_obj.num_heads).permute(2, 0, 3, 1, 4)\n", + " q, k, v = qkv.unbind(0)\n", + "\n", + " attn = (q @ k.transpose(-2, -1)) * attn_obj.scale\n", + " attn = attn.softmax(dim=-1)\n", + " attn = attn_obj.attn_drop(attn)\n", + "\n", + " # attn_map을 저장\n", + " attn_maps.append(attn.detach()) # 주의 가중치를 저장\n", + " attn_obj.cls_attn_map = attn[:, :, 0, 2:] # cls attention map 저장\n", + "\n", + " x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n", + " x = attn_obj.proj(x)\n", + " x = attn_obj.proj_drop(x)\n", + " return x\n", + " return my_forward" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " # 현재 주의 맵의 크기 확인\n", + " current_size = attn_map.size(-1)\n", + " I = torch.eye(max(current_size, I_size)).to(attn_map.device) # 동적 아이덴티티 행렬 생성\n", + "\n", + " if current_size < I_size:\n", + " padding_size = I_size - current_size\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " I_padded = F.pad(I, (0, padding_size, 0, 0)) # 아이덴티티 행렬 패딩\n", + " prod = padded_attn_map + I_padded\n", + " else:\n", + " prod = attn_map + I\n", + "\n", + " attn_rollout.append(prod / prod.sum(dim=-1, keepdim=True)) # 정규화\n", + "\n", + " return attn_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-small" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 384, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=384, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 292.63it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Block 1: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 2: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 3: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 4: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 5: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 6: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 7: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 8: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 9: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 10: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 11: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 12: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "CLS Weight Shape: torch.Size([1, 194])\n", + "Attention Rollout: torch.Size([1, 6, 196, 196])\n", + "Combined CLS Weight Shape: torch.Size([1, 6, 194])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "attn_maps = [] # 전역 리스트를 사용하여 attn_map을 저장\n", + "cls_weights = []\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출\n", + "for block in tqdm(model.blocks):\n", + " wrapped_forward = my_forward_wrapper(block.attn)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성\n", + " x = torch.randn(1, 196, 384) # B=1, N=196, C=384\n", + " output = wrapped_forward(x)\n", + "\n", + " # attn_map이 저장되었는지 확인합니다.\n", + " if attn_maps:\n", + " print(f\"Block {len(attn_maps)}: Attention map collected with shape {attn_maps[-1].shape}\")\n", + " else:\n", + " print(f\"Block {len(attn_maps)}: No attention map collected.\")\n", + "\n", + " # cls_attn_map 저장\n", + " cls_attn_map = block.attn.cls_attn_map.detach() # cls_attn_map 저장\n", + " cls_weights.append(cls_attn_map)\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps:\n", + " attn_map = attn_maps[-1].mean(dim=1).squeeze(0).detach()\n", + "else:\n", + " print(\"attn_maps 내용:\", attn_maps) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "cls_weight = cls_weights[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_cpu = attn_map.cpu()\n", + "cls_weight_cpu = cls_weight.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_combined = torch.prod(torch.stack(cls_weights), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_prod = torch.prod(torch.stack(attn_maps), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_cpu = [attn_map.cpu() for attn_map in attn_maps]\n", + "cls_weights_cpu = [cls_weight.cpu() for cls_weight in cls_weights]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 12003.73it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Attention Rollout Length: 12\n", + "Attention Rollout Shapes: [torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196])]\n", + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " attn_rollout = []\n", + " I = torch.eye(attn_maps[0].shape[-1]) # Identity matrix\n", + " prod = I\n", + " for i, attn_map in enumerate(attn_maps):\n", + " prod = prod @ (attn_map + I) # Product of attention maps with identity matrix\n", + " \n", + " prod = prod / prod.sum(dim=-1, keepdim=True) # Normalize\n", + " attn_rollout.append(prod)\n", + " return attn_rollout\n", + "\n", + "# Attention Rollout\n", + "attn_rollout = attention_rollout_function(attn_maps_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout = []\n", + "\n", + "for i in tqdm(range(len(attn_rollout))): # attn_rollout의 크기로 반복\n", + " cls_weight_tensor = attn_rollout[i][0, 0, :, :] # 첫 번째 헤드 사용\n", + "\n", + " # 텐서의 크기를 출력하여 디버깅\n", + " print(f\"Class weight tensor shape before view: {cls_weight_tensor.shape}\")\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인합니다.\n", + " if cls_weight_tensor.numel() == 14 * 14:\n", + " cls_weights_rollout.append(cls_weight_tensor.view(14, 14))\n", + " elif cls_weight_tensor.numel() == 196 * 196:\n", + " # 196x196 텐서를 클래스 가중치로 사용\n", + " cls_weights_rollout.append(cls_weight_tensor) # 변환하지 않고 그대로 사용\n", + " else:\n", + " print(f\"Skipping view for index {i}, tensor size is {cls_weight_tensor.numel()}\")\n", + "\n", + "# 결과 확인\n", + "if isinstance(attn_rollout, list):\n", + " print(\"Attention Rollout Length:\", len(attn_rollout))\n", + " print(\"Attention Rollout Shapes:\", [tensor.shape for tensor in attn_rollout])\n", + "else:\n", + " print(\"Attention Rollout Shape:\", attn_rollout.shape)\n", + "\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0092, 0.0108, 0.0016, ..., 0.0033, 0.0031, 0.0041],\n", + " [0.0025, 0.0249, 0.0017, ..., 0.0045, 0.0037, 0.0045],\n", + " [0.0022, 0.0028, 0.0083, ..., 0.0025, 0.0050, 0.0159],\n", + " ...,\n", + " [0.0022, 0.0089, 0.0015, ..., 0.0199, 0.0061, 0.0046],\n", + " [0.0014, 0.0043, 0.0024, ..., 0.0043, 0.0238, 0.0065],\n", + " [0.0042, 0.0048, 0.0018, ..., 0.0042, 0.0047, 0.0244]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0028, 0.0103, 0.0145, 0.0100, 0.0049, 0.0047, 0.0083, 0.0101, 0.0075,\n", + " 0.0024, 0.0125, 0.0012, 0.0042, 0.0037, 0.0073, 0.0071, 0.0119, 0.0110,\n", + " 0.0049, 0.0063, 0.0066, 0.0083, 0.0065, 0.0068, 0.0030, 0.0534, 0.0020,\n", + " 0.0335, 0.0037, 0.0035, 0.0532, 0.0121, 0.0040, 0.0036, 0.0115, 0.0019,\n", + " 0.0060, 0.0106, 0.0159, 0.0096, 0.0117, 0.0069, 0.0179, 0.0196, 0.0382,\n", + " 0.0029, 0.0084, 0.0023, 0.0139, 0.0022, 0.0152, 0.0044, 0.0024, 0.0139,\n", + " 0.0097, 0.0583, 0.0025, 0.0032, 0.0253, 0.0018, 0.0032, 0.0034, 0.0056,\n", + " 0.0191, 0.0047, 0.0171, 0.0095, 0.0093, 0.0037, 0.0014, 0.0151, 0.0051,\n", + " 0.0043, 0.0063, 0.0054, 0.0231, 0.0095, 0.0082, 0.0100, 0.0046, 0.0108,\n", + " 0.0045, 0.0025, 0.0169, 0.0107, 0.0023, 0.0054, 0.0060, 0.0040, 0.0103,\n", + " 0.0049, 0.0091, 0.0056, 0.0041, 0.1168, 0.0315, 0.0022, 0.0107, 0.0085,\n", + " 0.0040, 0.0037, 0.0197, 0.0153, 0.0179, 0.0044, 0.0178, 0.0102, 0.0677,\n", + " 0.0103, 0.0061, 0.0065, 0.0028, 0.0051, 0.0014, 0.0034, 0.0241, 0.0049,\n", + " 0.0142, 0.0090, 0.0042, 0.0320, 0.0069, 0.0044, 0.0044, 0.0182, 0.0101,\n", + " 0.0042, 0.0026, 0.1099, 0.0097, 0.0284, 0.0165, 0.1040, 0.0059, 0.0176,\n", + " 0.0036, 0.0107, 0.0176, 0.0098, 0.0038, 0.0078, 0.0069, 0.0027, 0.0309,\n", + " 0.0057, 0.0041, 0.0194, 0.0011, 0.0053, 0.0026, 0.0038, 0.0306, 0.0064,\n", + " 0.0205, 0.0032, 0.0127, 0.0057, 0.0024, 0.0022, 0.0252, 0.0051, 0.0481,\n", + " 0.0108, 0.0292, 0.0301, 0.0071, 0.0176, 0.0060, 0.0037, 0.0065, 0.0056,\n", + " 0.0025, 0.0069, 0.0224, 0.0032, 0.0108, 0.0104, 0.0012, 0.0050, 0.0022,\n", + " 0.0089, 0.0038, 0.0131, 0.0068, 0.0124, 0.0128, 0.0195, 0.0075, 0.0050,\n", + " 0.0138, 0.0089, 0.0084, 0.0060, 0.0109]])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[5.0145e-01, 1.0773e-04, 1.9219e-09, ..., 3.0224e-07, 4.8908e-08,\n", + " 9.4525e-09],\n", + " [1.7469e-07, 7.3549e-01, 1.9689e-09, ..., 1.5312e-05, 1.8353e-07,\n", + " 9.8260e-07],\n", + " [2.0269e-04, 8.8949e-03, 7.8070e-01, ..., 8.4695e-05, 3.8640e-05,\n", + " 1.5120e-02],\n", + " ...,\n", + " [3.1366e-08, 1.4511e-03, 1.2052e-10, ..., 5.2665e-01, 1.4668e-05,\n", + " 1.2199e-06],\n", + " [6.7066e-11, 3.5859e-06, 1.0056e-12, ..., 1.3806e-06, 5.0115e-01,\n", + " 6.2253e-07],\n", + " [2.0210e-10, 8.3169e-06, 4.0810e-09, ..., 2.4031e-07, 3.3919e-06,\n", + " 5.2196e-01]]),\n", + " tensor([[2.8310e-01, 7.6627e-04, 1.8418e-03, ..., 1.2459e-04, 8.3037e-03,\n", + " 8.7034e-04],\n", + " [3.8865e-04, 3.8519e-01, 1.1187e-03, ..., 5.4312e-04, 1.4127e-02,\n", + " 8.7212e-04],\n", + " [2.2728e-04, 4.7906e-03, 7.5910e-01, ..., 1.1250e-04, 3.2742e-03,\n", + " 9.5180e-03],\n", + " ...,\n", + " [2.5900e-04, 1.3777e-03, 2.0527e-03, ..., 4.1472e-01, 1.6249e-03,\n", + " 2.4871e-04],\n", + " [2.2064e-04, 5.9052e-04, 1.7989e-03, ..., 1.3416e-04, 4.2058e-01,\n", + " 4.4896e-04],\n", + " [1.6595e-04, 5.9181e-04, 1.7374e-03, ..., 1.4127e-04, 6.7444e-02,\n", + " 3.2628e-01]]),\n", + " tensor([[0.1570, 0.0010, 0.0056, ..., 0.0006, 0.0064, 0.0009],\n", + " [0.0135, 0.1933, 0.0032, ..., 0.0010, 0.0105, 0.0013],\n", + " [0.0069, 0.0031, 0.3877, ..., 0.0009, 0.0040, 0.0054],\n", + " ...,\n", + " [0.0108, 0.0015, 0.0052, ..., 0.2111, 0.0023, 0.0007],\n", + " [0.0091, 0.0012, 0.0048, ..., 0.0011, 0.2260, 0.0007],\n", + " [0.0097, 0.0011, 0.0052, ..., 0.0007, 0.0395, 0.1640]]),\n", + " tensor([[0.1081, 0.0055, 0.0075, ..., 0.0018, 0.0058, 0.0013],\n", + " [0.0112, 0.1038, 0.0062, ..., 0.0022, 0.0086, 0.0024],\n", + " [0.0063, 0.0048, 0.1985, ..., 0.0013, 0.0051, 0.0036],\n", + " ...,\n", + " [0.0089, 0.0055, 0.0061, ..., 0.1082, 0.0036, 0.0019],\n", + " [0.0075, 0.0059, 0.0061, ..., 0.0019, 0.1328, 0.0011],\n", + " [0.0081, 0.0053, 0.0061, ..., 0.0017, 0.0251, 0.0829]]),\n", + " tensor([[0.0596, 0.0034, 0.0052, ..., 0.0040, 0.0040, 0.0009],\n", + " [0.0106, 0.0527, 0.0047, ..., 0.0043, 0.0053, 0.0014],\n", + " [0.0083, 0.0031, 0.1010, ..., 0.0042, 0.0034, 0.0020],\n", + " ...,\n", + " [0.0104, 0.0036, 0.0045, ..., 0.0581, 0.0028, 0.0011],\n", + " [0.0099, 0.0036, 0.0045, ..., 0.0044, 0.0673, 0.0008],\n", + " [0.0101, 0.0034, 0.0049, ..., 0.0041, 0.0135, 0.0417]]),\n", + " tensor([[0.0315, 0.0021, 0.0068, ..., 0.0030, 0.0032, 0.0032],\n", + " [0.0069, 0.0267, 0.0069, ..., 0.0031, 0.0038, 0.0035],\n", + " [0.0055, 0.0019, 0.0546, ..., 0.0032, 0.0028, 0.0039],\n", + " ...,\n", + " [0.0069, 0.0022, 0.0068, ..., 0.0300, 0.0025, 0.0039],\n", + " [0.0066, 0.0022, 0.0065, ..., 0.0033, 0.0347, 0.0031],\n", + " [0.0067, 0.0021, 0.0068, ..., 0.0031, 0.0079, 0.0236]]),\n", + " tensor([[0.0174, 0.0023, 0.0059, ..., 0.0040, 0.0037, 0.0039],\n", + " [0.0049, 0.0145, 0.0060, ..., 0.0040, 0.0040, 0.0042],\n", + " [0.0042, 0.0021, 0.0310, ..., 0.0039, 0.0035, 0.0043],\n", + " ...,\n", + " [0.0050, 0.0022, 0.0059, ..., 0.0175, 0.0033, 0.0044],\n", + " [0.0048, 0.0023, 0.0058, ..., 0.0041, 0.0198, 0.0039],\n", + " [0.0049, 0.0022, 0.0060, ..., 0.0041, 0.0060, 0.0143]]),\n", + " tensor([[0.0093, 0.0036, 0.0040, ..., 0.0063, 0.0032, 0.0035],\n", + " [0.0031, 0.0097, 0.0041, ..., 0.0063, 0.0034, 0.0036],\n", + " [0.0027, 0.0035, 0.0166, ..., 0.0062, 0.0031, 0.0037],\n", + " ...,\n", + " [0.0031, 0.0036, 0.0040, ..., 0.0135, 0.0030, 0.0038],\n", + " [0.0030, 0.0036, 0.0040, ..., 0.0063, 0.0113, 0.0034],\n", + " [0.0031, 0.0036, 0.0041, ..., 0.0063, 0.0044, 0.0090]]),\n", + " tensor([[0.0069, 0.0032, 0.0029, ..., 0.0067, 0.0042, 0.0037],\n", + " [0.0038, 0.0062, 0.0029, ..., 0.0067, 0.0042, 0.0038],\n", + " [0.0035, 0.0031, 0.0092, ..., 0.0067, 0.0042, 0.0038],\n", + " ...,\n", + " [0.0038, 0.0032, 0.0029, ..., 0.0103, 0.0041, 0.0039],\n", + " [0.0037, 0.0031, 0.0029, ..., 0.0067, 0.0082, 0.0037],\n", + " [0.0037, 0.0032, 0.0029, ..., 0.0067, 0.0047, 0.0065]]),\n", + " tensor([[0.0074, 0.0034, 0.0031, ..., 0.0049, 0.0035, 0.0039],\n", + " [0.0058, 0.0049, 0.0031, ..., 0.0049, 0.0036, 0.0040],\n", + " [0.0058, 0.0034, 0.0063, ..., 0.0049, 0.0036, 0.0040],\n", + " ...,\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0067, 0.0035, 0.0040],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0048, 0.0056, 0.0039],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0049, 0.0038, 0.0053]]),\n", + " tensor([[0.0092, 0.0024, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0032, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0062, ..., 0.0049, 0.0041, 0.0086],\n", + " ...,\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0058, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0051, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0042, 0.0093]]),\n", + " tensor([[0.0056, 0.0037, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0052, 0.0040, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0040, ..., 0.0038, 0.0035, 0.0080],\n", + " ...,\n", + " [0.0052, 0.0037, 0.0032, ..., 0.0043, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0032, ..., 0.0038, 0.0040, 0.0080],\n", + " [0.0051, 0.0037, 0.0032, ..., 0.0038, 0.0036, 0.0084]])]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-tiny -> student model로 가정" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)\n", + "model_st" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model_st.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 266.64it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collected attention maps: 12\n", + "CLS Weight Shape: torch.Size([1])\n", + "Attention Rollout: torch.Size([1, 196, 192])\n", + "Combined CLS Weight Shape: torch.Size([1, 196])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# 전역 리스트를 사용하여 attn_map과 cls_weight 저장\n", + "attn_maps_st = []\n", + "cls_weights_st = []\n", + "\n", + "# Forward hook을 설정하는 함수\n", + "def get_attention_map(module, input, output):\n", + " attn_maps_st.append(output.detach()) # 주의 맵 저장\n", + " cls_attn_map = output[:, :, 0] # CLS token에 대한 attention map을 추출\n", + " cls_weights_st.append(cls_attn_map.detach()) # 클래스 주의 맵 저장\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출합니다.\n", + "for block in tqdm(model_st.blocks):\n", + " # Forward hook 등록\n", + " hook = block.attn.register_forward_hook(get_attention_map)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성합니다.\n", + " x = torch.randn(1, 3, 224, 224).to(device) # B=1, C=3, H=224, W=224\n", + " x_patch = model_st.patch_embed(x) # 패치 임베딩 변환\n", + "\n", + " # wrapped_forward 함수로 출력 계산\n", + " output = block.attn(x_patch)\n", + "\n", + " # hook 해제\n", + " hook.remove()\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps_st:\n", + " attn_map_st = attn_maps_st[-1].mean(dim=1).squeeze(0).detach()\n", + " print(f\"Collected attention maps: {len(attn_maps_st)}\")\n", + "else:\n", + " print(\"attn_maps_st 내용:\", attn_maps_st) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "# cls_weight 계산\n", + "cls_weight_st = cls_weights_st[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_st_cpu = attn_map_st.cpu()\n", + "cls_weight_st_cpu = cls_weight_st.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight_st.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_st_combined = torch.prod(torch.stack(cls_weights_st), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_st_prod = torch.prod(torch.stack(attn_maps_st), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_st_cpu = [attn_map.cpu() for attn_map in attn_maps_st]\n", + "cls_weights_st_cpu = [cls_weight.cpu() for cls_weight in cls_weights_st]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_st_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_st_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Attention Rollout\n", + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " I = torch.eye(I_size).to(attn_maps[0].device) # Identity matrix\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " if attn_map.size(-1) == I_size:\n", + " prod = attn_map + I\n", + " elif attn_map.size(-1) < I_size:\n", + " padding_size = I_size - attn_map.size(-1)\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " prod = padded_attn_map + I\n", + " else:\n", + " raise ValueError(\"Unexpected attention map size.\")\n", + "\n", + " attn_rollout.append(prod)\n", + "\n", + " attn_rollout = [prod / prod.sum(dim=-1, keepdim=True) for prod in attn_rollout] # Normalize\n", + " return attn_rollout" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + } + ], + "source": [ + "# Attention Rollout 실행\n", + "attn_rollout_st = attention_rollout_function(attn_maps_st_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout_st = []\n", + "\n", + "for i in range(len(attn_rollout_st)): # attn_rollout_st의 크기로 반복\n", + " cls_weight_tensor_st = attn_rollout_st[i][0] # 첫 번째 배치의 텐서 사용\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인\n", + " if cls_weight_tensor_st.numel() == 14 * 14:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st.view(14, 14))\n", + " elif cls_weight_tensor_st.numel() == 196 * 196:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st) # 그대로 사용\n", + "\n", + "# 결과 확인\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout_st])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0.1842, 0.6513, 0.9481, -0.0575, -0.4596, 0.9378, -0.0630, 0.0913,\n", + " -0.2760, 0.8236, -0.3048, -0.5067, -0.2237, 0.0104, 0.2620, 0.4669,\n", + " 0.3408, -0.6330, -0.4091, -0.4896, -0.9567, -1.5660, -0.2059, -0.2811,\n", + " -0.2813, -0.8560, -0.1496, -0.0455, -0.0833, 0.3670, 0.2738, -0.3592,\n", + " 0.3951, 0.6094, 0.5716, -0.0526, 0.3808, -0.0236, -0.7082, 0.1859,\n", + " 0.8467, -0.0996, 0.4303, 0.0613, 0.7428, 0.1889, -0.0631, -0.1368,\n", + " 0.1214, 0.5921, 0.1423, -0.4978, 0.5646, -0.2145, 0.1315, -0.3321,\n", + " 0.1872, -0.0260, -0.5459, 0.0105, 0.7164, -0.4216, -0.2821, 0.2232,\n", + " -0.1487, 0.1884, -0.1680, 0.3930, -0.3026, 0.8395, -0.5283, -0.1191,\n", + " 0.5188, -1.0522, 0.4154, -0.1342, -0.8908, 1.1572, 0.0318, 0.0403,\n", + " -0.4454, 0.0902, -0.5127, 0.6663, -0.3861, 0.5882, 0.0156, 0.5951,\n", + " -0.7754, 0.4059, 0.1418, 0.9044, -1.3180, -0.1740, 0.9519, -0.7662,\n", + " -0.1940, -0.4566, 0.9794, 0.2629, 0.3623, -0.1166, 0.5362, -0.2732,\n", + " -0.5911, -0.4877, 0.0517, 0.5354, 0.2801, -0.0113, 0.1709, -0.7622,\n", + " -0.1145, -0.9085, 0.3520, -0.6743, 0.2920, 0.2248, -0.2348, -0.4925,\n", + " 0.1487, -0.0883, -0.2174, -0.1746, -0.0644, -0.1431, 0.5035, -0.4010,\n", + " -0.1813, -0.0585, -0.1536, 0.7160, -0.5562, -0.1020, 0.2226, 0.2361,\n", + " 0.5698, -0.2106, 0.9990, -1.1657, -1.0541, 0.4736, 0.7887, 0.2140,\n", + " -0.4918, -0.7655, 1.2153, -0.0707, -0.4830, 0.2197, 0.0948, 0.8601,\n", + " 0.0194, 0.3300, -0.4195, 0.2778, -0.2720, -0.9317, -0.2371, 0.2978,\n", + " -0.3443, 1.3332, -0.0444, -0.4376, -0.2210, -0.0033, -0.0126, -0.0402,\n", + " -1.3693, -0.4690, 0.2604, 0.3911, -0.3407, 0.6645, -0.1561, -0.1206,\n", + " 0.4401, -0.1128, 0.3281, 0.3493, 0.4086, 0.2105, -0.1624, -0.0317,\n", + " 0.2122, 0.2028, 0.5176, -0.5969, -0.6693, 0.1342, -0.5071, -0.3850])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map_st" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.2606])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight_st" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[-0.1554, -0.0586, -0.0729, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.0529, -0.2460, -0.0994, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1179, -0.0144, -0.2249, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0701, -0.1249, -0.0585, ..., -0.1516, -0.0000, -0.0000],\n", + " [ 0.0249, -0.1786, -0.0892, ..., -0.0000, -0.1801, -0.0000],\n", + " [-0.1276, -0.1296, -0.0653, ..., -0.0000, -0.0000, -0.1503]]),\n", + " tensor([[-0.5720, 0.0253, -0.8815, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1371, -0.5842, -0.8806, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 0.0658, -0.2696, -1.0837, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0265, -0.1658, -0.9726, ..., -0.4945, -0.0000, -0.0000],\n", + " [-0.0341, -0.0129, -0.8782, ..., -0.0000, -0.4722, -0.0000],\n", + " [-0.0291, 0.0190, -1.0195, ..., -0.0000, -0.0000, -0.5399]]),\n", + " tensor([[ -5.2143, -0.2073, -9.1526, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.0086, -6.6903, -8.8212, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.6930, -0.4107, -17.7767, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [ 1.9814, -0.8107, -13.0110, ..., -8.9967, -0.0000, -0.0000],\n", + " [ 0.9145, -0.1184, -6.4268, ..., -0.0000, -4.5341, -0.0000],\n", + " [ 1.8620, -1.2088, -13.1969, ..., -0.0000, -0.0000, -8.9710]]),\n", + " tensor([[ 0.4234, -0.1738, 0.4660, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0612, 0.4136, 0.4037, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0573, -0.1247, 0.9858, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0938, -0.0743, 0.4447, ..., 0.5810, 0.0000, 0.0000],\n", + " [-0.1542, -0.1397, 0.5147, ..., 0.0000, 0.5836, 0.0000],\n", + " [-0.1097, -0.1474, 0.4535, ..., 0.0000, 0.0000, 0.5374]]),\n", + " tensor([[ 0.9406, -0.2136, 0.8167, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1652, 0.5770, 0.8471, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1347, -0.2466, 1.5981, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.1693, -0.2130, 0.8331, ..., 0.8538, 0.0000, 0.0000],\n", + " [ 0.1927, -0.2818, 0.8108, ..., 0.0000, 0.8053, 0.0000],\n", + " [ 0.1685, -0.2378, 0.8006, ..., 0.0000, 0.0000, 0.7944]]),\n", + " tensor([[ 0.7328, 0.0801, 0.5444, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0403, 0.7458, 0.5566, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0786, 0.0814, 1.3531, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0527, 0.0911, 0.6341, ..., 0.6709, 0.0000, 0.0000],\n", + " [-0.0836, 0.1107, 0.5707, ..., 0.0000, 0.7083, 0.0000],\n", + " [-0.0249, 0.0901, 0.5255, ..., 0.0000, 0.0000, 0.6873]]),\n", + " tensor([[0.8320, 0.2851, 0.8615, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1175, 0.9609, 0.8269, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1498, 0.2262, 1.6439, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [0.1284, 0.2232, 0.8748, ..., 0.7061, 0.0000, 0.0000],\n", + " [0.1243, 0.2083, 0.9053, ..., 0.0000, 0.7440, 0.0000],\n", + " [0.1109, 0.2427, 0.8597, ..., 0.0000, 0.0000, 0.6755]]),\n", + " tensor([[ 0.7119, 0.2967, 0.5289, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1194, 1.2095, 0.6030, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1917, 0.3107, 1.4640, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.1499, 0.3103, 0.6443, ..., 0.8962, 0.0000, 0.0000],\n", + " [-0.1631, 0.3251, 0.5534, ..., 0.0000, 0.8595, 0.0000],\n", + " [-0.1243, 0.2940, 0.6102, ..., 0.0000, 0.0000, 0.8932]]),\n", + " tensor([[ 3.0915, -0.7572, 1.6884, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8447, 1.3794, 1.6422, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8184, -0.8285, 3.8009, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.7615, -0.8503, 1.5874, ..., 2.2260, 0.0000, 0.0000],\n", + " [ 0.8272, -0.8181, 1.6270, ..., 0.0000, 2.1608, 0.0000],\n", + " [ 0.8811, -0.7469, 1.4999, ..., 0.0000, 0.0000, 2.1590]]),\n", + " tensor([[ 2.4223, -0.3267, 0.9755, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.2824, 1.6931, 0.8163, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.3807, -0.1291, 2.9314, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.3839, -0.2936, 0.9776, ..., 2.0073, 0.0000, 0.0000],\n", + " [ 0.4626, -0.2595, 0.9444, ..., 0.0000, 2.0228, 0.0000],\n", + " [ 0.4498, -0.2101, 0.9627, ..., 0.0000, 0.0000, 1.8273]]),\n", + " tensor([[15.2405, 2.7610, 6.7306, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 9.2874, 22.5292, 11.1778, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 6.0527, 2.1433, 18.1478, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 8.6357, 3.6522, 9.4287, ..., 15.7923, 0.0000, 0.0000],\n", + " [ 4.7526, 2.0569, 6.1020, ..., 0.0000, 8.8777, 0.0000],\n", + " [ 3.7997, 1.8325, 4.7829, ..., 0.0000, 0.0000, 7.1644]]),\n", + " tensor([[ 3.2594, 1.9875, 2.6364, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.8766, 17.1216, 9.5131, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.7089, 5.7129, 15.9202, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.4791, 2.2901, 3.2946, ..., 3.3042, 0.0000, 0.0000],\n", + " [ 0.7041, 3.5211, 4.8167, ..., 0.0000, 5.4057, 0.0000],\n", + " [ 1.7779, 8.7794, 10.9012, ..., 0.0000, 0.0000, 12.3839]])]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout_st" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "두 모델 간의 차이 Loss 계산 (L1, L2, Cosine Similarity, KL Divergence로 구현)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total L1 Loss: 14.616626739501953\n", + "Layer 0 L1 Loss: 0.1479, Percentage of Total Loss: 1.0115%\n", + "Layer 1 L1 Loss: 0.2261, Percentage of Total Loss: 1.5469%\n", + "Layer 2 L1 Loss: 5.1462, Percentage of Total Loss: 35.2075%\n", + "Layer 3 L1 Loss: 0.1554, Percentage of Total Loss: 1.0629%\n", + "Layer 4 L1 Loss: 0.2121, Percentage of Total Loss: 1.4508%\n", + "Layer 5 L1 Loss: 0.1961, Percentage of Total Loss: 1.3413%\n", + "Layer 6 L1 Loss: 0.2350, Percentage of Total Loss: 1.6081%\n", + "Layer 7 L1 Loss: 0.2223, Percentage of Total Loss: 1.5206%\n", + "Layer 8 L1 Loss: 0.5502, Percentage of Total Loss: 3.7645%\n", + "Layer 9 L1 Loss: 0.5793, Percentage of Total Loss: 3.9635%\n", + "Layer 10 L1 Loss: 4.3009, Percentage of Total Loss: 29.4244%\n", + "Layer 11 L1 Loss: 2.6453, Percentage of Total Loss: 18.0980%\n" + ] + } + ], + "source": [ + "# L1 (Mean Absolute Error): 두 모델의 각 레이어 가중치 간의 절대 차이 계산\n", + "# L1 손실 계산\n", + "loss_l1 = 0\n", + "layer_losses_l1 = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = F.l1_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses_l1.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_l1 += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_l1 = loss_l1.item()\n", + "print(f\"Total L1 Loss: {total_loss_l1}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_l1 = [(layer_loss / total_loss_l1) * 100 for layer_loss in layer_losses_l1]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_l1):\n", + " print(f\"Layer {i} L1 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l1[i]:.4f}%\")\n", + "## 두 모델 간의 차이가 주로 Layer 2와 Layer 10에서 발생" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total L2 Loss: 453.0188903808594\n", + "Layer 0 L2 Loss: 0.0445, Percentage of Total Loss: 0.0098%\n", + "Layer 1 L2 Loss: 0.0991, Percentage of Total Loss: 0.0219%\n", + "Layer 2 L2 Loss: 394.6398, Percentage of Total Loss: 87.1133%\n", + "Layer 3 L2 Loss: 0.0411, Percentage of Total Loss: 0.0091%\n", + "Layer 4 L2 Loss: 0.0765, Percentage of Total Loss: 0.0169%\n", + "Layer 5 L2 Loss: 0.0602, Percentage of Total Loss: 0.0133%\n", + "Layer 6 L2 Loss: 0.0887, Percentage of Total Loss: 0.0196%\n", + "Layer 7 L2 Loss: 0.0799, Percentage of Total Loss: 0.0176%\n", + "Layer 8 L2 Loss: 0.5019, Percentage of Total Loss: 0.1108%\n", + "Layer 9 L2 Loss: 0.5323, Percentage of Total Loss: 0.1175%\n", + "Layer 10 L2 Loss: 38.7059, Percentage of Total Loss: 8.5440%\n", + "Layer 11 L2 Loss: 18.1490, Percentage of Total Loss: 4.0062%\n" + ] + } + ], + "source": [ + "# L2 (Euclidean Distance): 두 모델의 각 레이어 가중치 간의 유클리드 거리 계산\n", + "# L2 손실 계산\n", + "loss_l2 = 0\n", + "layer_losses_l2 = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산 (L2 손실)\n", + " layer_loss = F.mse_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses_l2.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_l2 += layer_loss\n", + "\n", + "# 총 L2 손실 출력\n", + "total_loss_l2 = loss_l2.item()\n", + "print(f\"Total L2 Loss: {total_loss_l2}\")\n", + "\n", + "# 레이어별 L2 손실 비율 계산\n", + "layer_loss_percentages_l2 = [(layer_loss / total_loss_l2) * 100 for layer_loss in layer_losses_l2]\n", + "\n", + "# 각 레이어 L2 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_l2):\n", + " print(f\"Layer {i} L2 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l2[i]:.4f}%\")\n", + "\n", + "## Layer 2: 2번째 레이어의 L2 손실이 394.6398로, 전체 손실의 87.11% -> 레이어2에서 두 모델 간의 가중치 차이가 매우 크다\n", + "## 나머지 레이어들은 상대적으로 손실 값이 매우 작고, 전체 손실에서 차지하는 비율도 낮음.\n", + "## layer 2가 두 모델의 성능 차이에 큰 기여하고 있을 것이라 판단할 수 있음. 두 모델의 구조나 파라미터 조정시 해당 레이어 더 보기!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Cosine Loss: 11.714558601379395\n", + "Layer 0 Cosine Loss: 1.0442, Percentage of Total Loss: 8.9137%\n", + "Layer 1 Cosine Loss: 1.0548, Percentage of Total Loss: 9.0045%\n", + "Layer 2 Cosine Loss: 1.0349, Percentage of Total Loss: 8.8344%\n", + "Layer 3 Cosine Loss: 0.8728, Percentage of Total Loss: 7.4502%\n", + "Layer 4 Cosine Loss: 0.8025, Percentage of Total Loss: 6.8504%\n", + "Layer 5 Cosine Loss: 0.9400, Percentage of Total Loss: 8.0241%\n", + "Layer 6 Cosine Loss: 1.0328, Percentage of Total Loss: 8.8163%\n", + "Layer 7 Cosine Loss: 0.9947, Percentage of Total Loss: 8.4910%\n", + "Layer 8 Cosine Loss: 0.9695, Percentage of Total Loss: 8.2759%\n", + "Layer 9 Cosine Loss: 0.9745, Percentage of Total Loss: 8.3191%\n", + "Layer 10 Cosine Loss: 0.9839, Percentage of Total Loss: 8.3986%\n", + "Layer 11 Cosine Loss: 1.0100, Percentage of Total Loss: 8.6217%\n" + ] + } + ], + "source": [ + "# Cosine Similarity: 두 모델의 가중치 벡터 간의 코사인 유사도 계산\n", + "\n", + "from torch.nn.functional import cosine_similarity\n", + "\n", + "# 코사인 유사도 손실 계산\n", + "loss_cosine = 0\n", + "layer_losses_cosine = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = 1 - cosine_similarity(cls_weights_rollout_st[i].view(1, -1), cls_weights_rollout[i].view(1, -1))\n", + " layer_losses_cosine.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_cosine += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_cosine = loss_cosine.item()\n", + "print(f\"Total Cosine Loss: {total_loss_cosine}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_cosine = [(layer_loss / total_loss_cosine) * 100 for layer_loss in layer_losses_cosine]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_cosine):\n", + " print(f\"Layer {i} Cosine Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_cosine[i]:.4f}%\")\n", + "\n", + "## 각 레이어의 손실은 0.8에서 1.1 사이로 분포 -> 두 모델의 가중치가 레이어별로 상당히 유사\n", + "## 레이어 0에서 레이어 11까지의 손실 값이 비슷한 수준이므로, 전체적으로 두 모델의 가중치가 유사" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total KL Divergence Loss: 31.367801666259766\n", + "Layer 0 KL Loss: 0.0042, Percentage of Total Loss: 0.0133%\n", + "Layer 1 KL Loss: 0.0067, Percentage of Total Loss: 0.0214%\n", + "Layer 2 KL Loss: 15.2732, Percentage of Total Loss: 48.6907%\n", + "Layer 3 KL Loss: 0.0010, Percentage of Total Loss: 0.0032%\n", + "Layer 4 KL Loss: 0.0025, Percentage of Total Loss: 0.0080%\n", + "Layer 5 KL Loss: 0.0023, Percentage of Total Loss: 0.0073%\n", + "Layer 6 KL Loss: 0.0020, Percentage of Total Loss: 0.0063%\n", + "Layer 7 KL Loss: 0.0030, Percentage of Total Loss: 0.0096%\n", + "Layer 8 KL Loss: 0.0287, Percentage of Total Loss: 0.0915%\n", + "Layer 9 KL Loss: 0.0246, Percentage of Total Loss: 0.0785%\n", + "Layer 10 KL Loss: 9.7367, Percentage of Total Loss: 31.0406%\n", + "Layer 11 KL Loss: 6.2829, Percentage of Total Loss: 20.0297%\n" + ] + } + ], + "source": [ + "# KL Divergence: 두 모델의 가중치를 확률 분포로 간주하고, Kullback-Leibler Divergence 계산하여 두 분포 간의 차이 측정\n", + "# KL 발산 계산\n", + "loss_kl = 0\n", + "layer_losses_kl = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 확률 분포로 정규화\n", + " p = F.softmax(cls_weights_rollout_st[i], dim=0) + 1e-10 # 작은 값 추가\n", + " q = F.softmax(cls_weights_rollout[i], dim=0) + 1e-10 # 작은 값 추가\n", + " \n", + " # KL 발산 계산\n", + " layer_loss = F.kl_div(p.log(), q, reduction='batchmean')\n", + " layer_losses_kl.append(layer_loss.item())\n", + " loss_kl += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_kl = loss_kl.item()\n", + "print(f\"Total KL Divergence Loss: {total_loss_kl}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_kl = [(layer_loss / total_loss_kl) * 100 for layer_loss in layer_losses_kl]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_kl):\n", + " print(f\"Layer {i} KL Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_kl[i]:.4f}%\")\n", + "## 두 모델 간의 차이가 주로 Layer 2, Layer 10, 그리고 Layer 11에서 발생" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}