diff --git a/attention_rollout.ipynb b/attention_rollout.ipynb new file mode 100644 index 0000000..1b55012 --- /dev/null +++ b/attention_rollout.ipynb @@ -0,0 +1,1632 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- attn_map\n", + "\n", + "attention map은 모델이 입력의 각 부분에 대해 얼마나 집중하는지를 나타내는 텐서. 각 입력의 특징들이 서로 어떻게 상호작용하는지를 보여줌.\n", + "입력 패치 간의 상관관계를 시각화하거나, 어떤 입력 부분이 모델의 예측에 더 큰 영향을 미치는지를 분석하는 데 사용.\n", + "\n", + "- cls_weight\n", + "\n", + "cls_weight는 CLS (Classification) 토큰에 대한 주의 맵의 특정 값으로, 주로 클래스별 가중치를 나타냄. 이 값은 모델이 특정 클래스에 대해 얼마나 집중하는지를 나타내는 정보.\n", + "각 블록에서 CLS 토큰에 대한 attention map 추출하여 클래스에 대한 가중치를 계산하는 데 사용. 이 값은 분류 작업에서 모델의 신뢰도를 평가.\n", + "\n", + "- cls_weights_rollout\n", + "\n", + "각 블록에서 추출된 CLS 토큰에 대한 주의 맵을 기반으로 만든 클래스 가중치의 리스트. 이 리스트는 모델의 여러 블록에서의 CLS 토큰에 대한 가중치를 포함하고 있음.\n", + "모델이 여러 블록에서 CLS 토큰에 대해 어떻게 집중하는지를 종합적으로 보여줌. 이를 통해 특정 클래스에 대한 모델의 전반적인 집중도를 평가할 수 있음." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "from tqdm import tqdm\n", + "import timm\n", + "assert timm.__version__ == \"0.3.2\"\n", + "from torchvision import transforms\n", + "from PIL import Image\n", + "import requests\n", + "import random\n", + "import numpy as np\n", + "\n", + "# 장치 설정\n", + "device = 'cuda' if torch.cuda.is_available() else 'cpu'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def my_forward_wrapper(attn_obj):\n", + " def my_forward(x):\n", + " B, N, C = x.shape\n", + " qkv = attn_obj.qkv(x).reshape(B, N, 3, attn_obj.num_heads, C // attn_obj.num_heads).permute(2, 0, 3, 1, 4)\n", + " q, k, v = qkv.unbind(0)\n", + "\n", + " attn = (q @ k.transpose(-2, -1)) * attn_obj.scale\n", + " attn = attn.softmax(dim=-1)\n", + " attn = attn_obj.attn_drop(attn)\n", + "\n", + " # attn_map을 저장\n", + " attn_maps.append(attn.detach()) # 주의 가중치를 저장\n", + " attn_obj.cls_attn_map = attn[:, :, 0, 2:] # cls attention map 저장\n", + "\n", + " x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n", + " x = attn_obj.proj(x)\n", + " x = attn_obj.proj_drop(x)\n", + " return x\n", + " return my_forward" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " # 현재 주의 맵의 크기 확인\n", + " current_size = attn_map.size(-1)\n", + " I = torch.eye(max(current_size, I_size)).to(attn_map.device) # 동적 아이덴티티 행렬 생성\n", + "\n", + " if current_size < I_size:\n", + " padding_size = I_size - current_size\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " I_padded = F.pad(I, (0, padding_size, 0, 0)) # 아이덴티티 행렬 패딩\n", + " prod = padded_attn_map + I_padded\n", + " else:\n", + " prod = attn_map + I\n", + "\n", + " attn_rollout.append(prod / prod.sum(dim=-1, keepdim=True)) # 정규화\n", + "\n", + " return attn_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-small" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 384, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=384, out_features=1152, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=384, out_features=384, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=384, out_features=1536, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=1536, out_features=384, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((384,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=384, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 292.63it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Block 1: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 2: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 3: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 4: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 5: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 6: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 7: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 8: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 9: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 10: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 11: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "Block 12: Attention map collected with shape torch.Size([1, 6, 196, 196])\n", + "CLS Weight Shape: torch.Size([1, 194])\n", + "Attention Rollout: torch.Size([1, 6, 196, 196])\n", + "Combined CLS Weight Shape: torch.Size([1, 6, 194])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "attn_maps = [] # 전역 리스트를 사용하여 attn_map을 저장\n", + "cls_weights = []\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출\n", + "for block in tqdm(model.blocks):\n", + " wrapped_forward = my_forward_wrapper(block.attn)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성\n", + " x = torch.randn(1, 196, 384) # B=1, N=196, C=384\n", + " output = wrapped_forward(x)\n", + "\n", + " # attn_map이 저장되었는지 확인합니다.\n", + " if attn_maps:\n", + " print(f\"Block {len(attn_maps)}: Attention map collected with shape {attn_maps[-1].shape}\")\n", + " else:\n", + " print(f\"Block {len(attn_maps)}: No attention map collected.\")\n", + "\n", + " # cls_attn_map 저장\n", + " cls_attn_map = block.attn.cls_attn_map.detach() # cls_attn_map 저장\n", + " cls_weights.append(cls_attn_map)\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps:\n", + " attn_map = attn_maps[-1].mean(dim=1).squeeze(0).detach()\n", + "else:\n", + " print(\"attn_maps 내용:\", attn_maps) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "cls_weight = cls_weights[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_cpu = attn_map.cpu()\n", + "cls_weight_cpu = cls_weight.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_combined = torch.prod(torch.stack(cls_weights), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_prod = torch.prod(torch.stack(attn_maps), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_cpu = [attn_map.cpu() for attn_map in attn_maps]\n", + "cls_weights_cpu = [cls_weight.cpu() for cls_weight in cls_weights]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 12003.73it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Class weight tensor shape before view: torch.Size([196, 196])\n", + "Attention Rollout Length: 12\n", + "Attention Rollout Shapes: [torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196]), torch.Size([1, 6, 196, 196])]\n", + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "def attention_rollout_function(attn_maps):\n", + " attn_rollout = []\n", + " I = torch.eye(attn_maps[0].shape[-1]) # Identity matrix\n", + " prod = I\n", + " for i, attn_map in enumerate(attn_maps):\n", + " prod = prod @ (attn_map + I) # Product of attention maps with identity matrix\n", + " \n", + " prod = prod / prod.sum(dim=-1, keepdim=True) # Normalize\n", + " attn_rollout.append(prod)\n", + " return attn_rollout\n", + "\n", + "# Attention Rollout\n", + "attn_rollout = attention_rollout_function(attn_maps_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout = []\n", + "\n", + "for i in tqdm(range(len(attn_rollout))): # attn_rollout의 크기로 반복\n", + " cls_weight_tensor = attn_rollout[i][0, 0, :, :] # 첫 번째 헤드 사용\n", + "\n", + " # 텐서의 크기를 출력하여 디버깅\n", + " print(f\"Class weight tensor shape before view: {cls_weight_tensor.shape}\")\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인합니다.\n", + " if cls_weight_tensor.numel() == 14 * 14:\n", + " cls_weights_rollout.append(cls_weight_tensor.view(14, 14))\n", + " elif cls_weight_tensor.numel() == 196 * 196:\n", + " # 196x196 텐서를 클래스 가중치로 사용\n", + " cls_weights_rollout.append(cls_weight_tensor) # 변환하지 않고 그대로 사용\n", + " else:\n", + " print(f\"Skipping view for index {i}, tensor size is {cls_weight_tensor.numel()}\")\n", + "\n", + "# 결과 확인\n", + "if isinstance(attn_rollout, list):\n", + " print(\"Attention Rollout Length:\", len(attn_rollout))\n", + " print(\"Attention Rollout Shapes:\", [tensor.shape for tensor in attn_rollout])\n", + "else:\n", + " print(\"Attention Rollout Shape:\", attn_rollout.shape)\n", + "\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0092, 0.0108, 0.0016, ..., 0.0033, 0.0031, 0.0041],\n", + " [0.0025, 0.0249, 0.0017, ..., 0.0045, 0.0037, 0.0045],\n", + " [0.0022, 0.0028, 0.0083, ..., 0.0025, 0.0050, 0.0159],\n", + " ...,\n", + " [0.0022, 0.0089, 0.0015, ..., 0.0199, 0.0061, 0.0046],\n", + " [0.0014, 0.0043, 0.0024, ..., 0.0043, 0.0238, 0.0065],\n", + " [0.0042, 0.0048, 0.0018, ..., 0.0042, 0.0047, 0.0244]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0028, 0.0103, 0.0145, 0.0100, 0.0049, 0.0047, 0.0083, 0.0101, 0.0075,\n", + " 0.0024, 0.0125, 0.0012, 0.0042, 0.0037, 0.0073, 0.0071, 0.0119, 0.0110,\n", + " 0.0049, 0.0063, 0.0066, 0.0083, 0.0065, 0.0068, 0.0030, 0.0534, 0.0020,\n", + " 0.0335, 0.0037, 0.0035, 0.0532, 0.0121, 0.0040, 0.0036, 0.0115, 0.0019,\n", + " 0.0060, 0.0106, 0.0159, 0.0096, 0.0117, 0.0069, 0.0179, 0.0196, 0.0382,\n", + " 0.0029, 0.0084, 0.0023, 0.0139, 0.0022, 0.0152, 0.0044, 0.0024, 0.0139,\n", + " 0.0097, 0.0583, 0.0025, 0.0032, 0.0253, 0.0018, 0.0032, 0.0034, 0.0056,\n", + " 0.0191, 0.0047, 0.0171, 0.0095, 0.0093, 0.0037, 0.0014, 0.0151, 0.0051,\n", + " 0.0043, 0.0063, 0.0054, 0.0231, 0.0095, 0.0082, 0.0100, 0.0046, 0.0108,\n", + " 0.0045, 0.0025, 0.0169, 0.0107, 0.0023, 0.0054, 0.0060, 0.0040, 0.0103,\n", + " 0.0049, 0.0091, 0.0056, 0.0041, 0.1168, 0.0315, 0.0022, 0.0107, 0.0085,\n", + " 0.0040, 0.0037, 0.0197, 0.0153, 0.0179, 0.0044, 0.0178, 0.0102, 0.0677,\n", + " 0.0103, 0.0061, 0.0065, 0.0028, 0.0051, 0.0014, 0.0034, 0.0241, 0.0049,\n", + " 0.0142, 0.0090, 0.0042, 0.0320, 0.0069, 0.0044, 0.0044, 0.0182, 0.0101,\n", + " 0.0042, 0.0026, 0.1099, 0.0097, 0.0284, 0.0165, 0.1040, 0.0059, 0.0176,\n", + " 0.0036, 0.0107, 0.0176, 0.0098, 0.0038, 0.0078, 0.0069, 0.0027, 0.0309,\n", + " 0.0057, 0.0041, 0.0194, 0.0011, 0.0053, 0.0026, 0.0038, 0.0306, 0.0064,\n", + " 0.0205, 0.0032, 0.0127, 0.0057, 0.0024, 0.0022, 0.0252, 0.0051, 0.0481,\n", + " 0.0108, 0.0292, 0.0301, 0.0071, 0.0176, 0.0060, 0.0037, 0.0065, 0.0056,\n", + " 0.0025, 0.0069, 0.0224, 0.0032, 0.0108, 0.0104, 0.0012, 0.0050, 0.0022,\n", + " 0.0089, 0.0038, 0.0131, 0.0068, 0.0124, 0.0128, 0.0195, 0.0075, 0.0050,\n", + " 0.0138, 0.0089, 0.0084, 0.0060, 0.0109]])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[5.0145e-01, 1.0773e-04, 1.9219e-09, ..., 3.0224e-07, 4.8908e-08,\n", + " 9.4525e-09],\n", + " [1.7469e-07, 7.3549e-01, 1.9689e-09, ..., 1.5312e-05, 1.8353e-07,\n", + " 9.8260e-07],\n", + " [2.0269e-04, 8.8949e-03, 7.8070e-01, ..., 8.4695e-05, 3.8640e-05,\n", + " 1.5120e-02],\n", + " ...,\n", + " [3.1366e-08, 1.4511e-03, 1.2052e-10, ..., 5.2665e-01, 1.4668e-05,\n", + " 1.2199e-06],\n", + " [6.7066e-11, 3.5859e-06, 1.0056e-12, ..., 1.3806e-06, 5.0115e-01,\n", + " 6.2253e-07],\n", + " [2.0210e-10, 8.3169e-06, 4.0810e-09, ..., 2.4031e-07, 3.3919e-06,\n", + " 5.2196e-01]]),\n", + " tensor([[2.8310e-01, 7.6627e-04, 1.8418e-03, ..., 1.2459e-04, 8.3037e-03,\n", + " 8.7034e-04],\n", + " [3.8865e-04, 3.8519e-01, 1.1187e-03, ..., 5.4312e-04, 1.4127e-02,\n", + " 8.7212e-04],\n", + " [2.2728e-04, 4.7906e-03, 7.5910e-01, ..., 1.1250e-04, 3.2742e-03,\n", + " 9.5180e-03],\n", + " ...,\n", + " [2.5900e-04, 1.3777e-03, 2.0527e-03, ..., 4.1472e-01, 1.6249e-03,\n", + " 2.4871e-04],\n", + " [2.2064e-04, 5.9052e-04, 1.7989e-03, ..., 1.3416e-04, 4.2058e-01,\n", + " 4.4896e-04],\n", + " [1.6595e-04, 5.9181e-04, 1.7374e-03, ..., 1.4127e-04, 6.7444e-02,\n", + " 3.2628e-01]]),\n", + " tensor([[0.1570, 0.0010, 0.0056, ..., 0.0006, 0.0064, 0.0009],\n", + " [0.0135, 0.1933, 0.0032, ..., 0.0010, 0.0105, 0.0013],\n", + " [0.0069, 0.0031, 0.3877, ..., 0.0009, 0.0040, 0.0054],\n", + " ...,\n", + " [0.0108, 0.0015, 0.0052, ..., 0.2111, 0.0023, 0.0007],\n", + " [0.0091, 0.0012, 0.0048, ..., 0.0011, 0.2260, 0.0007],\n", + " [0.0097, 0.0011, 0.0052, ..., 0.0007, 0.0395, 0.1640]]),\n", + " tensor([[0.1081, 0.0055, 0.0075, ..., 0.0018, 0.0058, 0.0013],\n", + " [0.0112, 0.1038, 0.0062, ..., 0.0022, 0.0086, 0.0024],\n", + " [0.0063, 0.0048, 0.1985, ..., 0.0013, 0.0051, 0.0036],\n", + " ...,\n", + " [0.0089, 0.0055, 0.0061, ..., 0.1082, 0.0036, 0.0019],\n", + " [0.0075, 0.0059, 0.0061, ..., 0.0019, 0.1328, 0.0011],\n", + " [0.0081, 0.0053, 0.0061, ..., 0.0017, 0.0251, 0.0829]]),\n", + " tensor([[0.0596, 0.0034, 0.0052, ..., 0.0040, 0.0040, 0.0009],\n", + " [0.0106, 0.0527, 0.0047, ..., 0.0043, 0.0053, 0.0014],\n", + " [0.0083, 0.0031, 0.1010, ..., 0.0042, 0.0034, 0.0020],\n", + " ...,\n", + " [0.0104, 0.0036, 0.0045, ..., 0.0581, 0.0028, 0.0011],\n", + " [0.0099, 0.0036, 0.0045, ..., 0.0044, 0.0673, 0.0008],\n", + " [0.0101, 0.0034, 0.0049, ..., 0.0041, 0.0135, 0.0417]]),\n", + " tensor([[0.0315, 0.0021, 0.0068, ..., 0.0030, 0.0032, 0.0032],\n", + " [0.0069, 0.0267, 0.0069, ..., 0.0031, 0.0038, 0.0035],\n", + " [0.0055, 0.0019, 0.0546, ..., 0.0032, 0.0028, 0.0039],\n", + " ...,\n", + " [0.0069, 0.0022, 0.0068, ..., 0.0300, 0.0025, 0.0039],\n", + " [0.0066, 0.0022, 0.0065, ..., 0.0033, 0.0347, 0.0031],\n", + " [0.0067, 0.0021, 0.0068, ..., 0.0031, 0.0079, 0.0236]]),\n", + " tensor([[0.0174, 0.0023, 0.0059, ..., 0.0040, 0.0037, 0.0039],\n", + " [0.0049, 0.0145, 0.0060, ..., 0.0040, 0.0040, 0.0042],\n", + " [0.0042, 0.0021, 0.0310, ..., 0.0039, 0.0035, 0.0043],\n", + " ...,\n", + " [0.0050, 0.0022, 0.0059, ..., 0.0175, 0.0033, 0.0044],\n", + " [0.0048, 0.0023, 0.0058, ..., 0.0041, 0.0198, 0.0039],\n", + " [0.0049, 0.0022, 0.0060, ..., 0.0041, 0.0060, 0.0143]]),\n", + " tensor([[0.0093, 0.0036, 0.0040, ..., 0.0063, 0.0032, 0.0035],\n", + " [0.0031, 0.0097, 0.0041, ..., 0.0063, 0.0034, 0.0036],\n", + " [0.0027, 0.0035, 0.0166, ..., 0.0062, 0.0031, 0.0037],\n", + " ...,\n", + " [0.0031, 0.0036, 0.0040, ..., 0.0135, 0.0030, 0.0038],\n", + " [0.0030, 0.0036, 0.0040, ..., 0.0063, 0.0113, 0.0034],\n", + " [0.0031, 0.0036, 0.0041, ..., 0.0063, 0.0044, 0.0090]]),\n", + " tensor([[0.0069, 0.0032, 0.0029, ..., 0.0067, 0.0042, 0.0037],\n", + " [0.0038, 0.0062, 0.0029, ..., 0.0067, 0.0042, 0.0038],\n", + " [0.0035, 0.0031, 0.0092, ..., 0.0067, 0.0042, 0.0038],\n", + " ...,\n", + " [0.0038, 0.0032, 0.0029, ..., 0.0103, 0.0041, 0.0039],\n", + " [0.0037, 0.0031, 0.0029, ..., 0.0067, 0.0082, 0.0037],\n", + " [0.0037, 0.0032, 0.0029, ..., 0.0067, 0.0047, 0.0065]]),\n", + " tensor([[0.0074, 0.0034, 0.0031, ..., 0.0049, 0.0035, 0.0039],\n", + " [0.0058, 0.0049, 0.0031, ..., 0.0049, 0.0036, 0.0040],\n", + " [0.0058, 0.0034, 0.0063, ..., 0.0049, 0.0036, 0.0040],\n", + " ...,\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0067, 0.0035, 0.0040],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0048, 0.0056, 0.0039],\n", + " [0.0058, 0.0034, 0.0031, ..., 0.0049, 0.0038, 0.0053]]),\n", + " tensor([[0.0092, 0.0024, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0032, 0.0046, ..., 0.0048, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0062, ..., 0.0049, 0.0041, 0.0086],\n", + " ...,\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0058, 0.0041, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0051, 0.0086],\n", + " [0.0084, 0.0024, 0.0046, ..., 0.0048, 0.0042, 0.0093]]),\n", + " tensor([[0.0056, 0.0037, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0052, 0.0040, 0.0032, ..., 0.0038, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0040, ..., 0.0038, 0.0035, 0.0080],\n", + " ...,\n", + " [0.0052, 0.0037, 0.0032, ..., 0.0043, 0.0035, 0.0080],\n", + " [0.0051, 0.0036, 0.0032, ..., 0.0038, 0.0040, 0.0080],\n", + " [0.0051, 0.0037, 0.0032, ..., 0.0038, 0.0036, 0.0084]])]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DeiT-tiny -> student model로 가정" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + }, + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)\n", + "model_st" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of blocks in the model: 12\n" + ] + } + ], + "source": [ + "print(f\"Number of blocks in the model: {len(model_st.blocks)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_st.eval()\n", + "\n", + "seed = 42\n", + "random.seed(seed)\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 12/12 [00:00<00:00, 266.64it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collected attention maps: 12\n", + "CLS Weight Shape: torch.Size([1])\n", + "Attention Rollout: torch.Size([1, 196, 192])\n", + "Combined CLS Weight Shape: torch.Size([1, 196])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# 전역 리스트를 사용하여 attn_map과 cls_weight 저장\n", + "attn_maps_st = []\n", + "cls_weights_st = []\n", + "\n", + "# Forward hook을 설정하는 함수\n", + "def get_attention_map(module, input, output):\n", + " attn_maps_st.append(output.detach()) # 주의 맵 저장\n", + " cls_attn_map = output[:, :, 0] # CLS token에 대한 attention map을 추출\n", + " cls_weights_st.append(cls_attn_map.detach()) # 클래스 주의 맵 저장\n", + "\n", + "# 각 블록을 순회하며 attention map과 class attention map을 추출합니다.\n", + "for block in tqdm(model_st.blocks):\n", + " # Forward hook 등록\n", + " hook = block.attn.register_forward_hook(get_attention_map)\n", + " \n", + " # 입력 텐서가 필요하므로 임의의 입력을 생성합니다.\n", + " x = torch.randn(1, 3, 224, 224).to(device) # B=1, C=3, H=224, W=224\n", + " x_patch = model_st.patch_embed(x) # 패치 임베딩 변환\n", + "\n", + " # wrapped_forward 함수로 출력 계산\n", + " output = block.attn(x_patch)\n", + "\n", + " # hook 해제\n", + " hook.remove()\n", + "\n", + "# 주의 맵이 저장되었는지 확인합니다.\n", + "if attn_maps_st:\n", + " attn_map_st = attn_maps_st[-1].mean(dim=1).squeeze(0).detach()\n", + " print(f\"Collected attention maps: {len(attn_maps_st)}\")\n", + "else:\n", + " print(\"attn_maps_st 내용:\", attn_maps_st) # 디버깅 출력 추가\n", + " raise ValueError(\"No attention maps were collected.\")\n", + "\n", + "# cls_weight 계산\n", + "cls_weight_st = cls_weights_st[-1].max(dim=1).values.detach()\n", + "\n", + "# Ensure tensors are on the CPU\n", + "attn_map_st_cpu = attn_map_st.cpu()\n", + "cls_weight_st_cpu = cls_weight_st.cpu()\n", + "\n", + "# 클래스 가중치의 크기를 확인합니다.\n", + "print(\"CLS Weight Shape:\", cls_weight_st.shape)\n", + "\n", + "# Combine class scores of all blocks\n", + "try:\n", + " cls_weight_st_combined = torch.prod(torch.stack(cls_weights_st), dim=0)\n", + "except RuntimeError as e:\n", + " print(f\"Error combining class weights: {e}\")\n", + "\n", + "# 주의 맵의 곱을 계산합니다\n", + "attn_maps_st_prod = torch.prod(torch.stack(attn_maps_st), dim=0)\n", + "\n", + "# CPU로 전환\n", + "attn_maps_st_cpu = [attn_map.cpu() for attn_map in attn_maps_st]\n", + "cls_weights_st_cpu = [cls_weight.cpu() for cls_weight in cls_weights_st]\n", + "\n", + "# 결과 확인\n", + "print(\"Attention Rollout:\", attn_maps_st_prod.shape)\n", + "print(\"Combined CLS Weight Shape:\", cls_weight_st_combined.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Attention Rollout\n", + "def attention_rollout_function(attn_maps):\n", + " I_size = 196 # 주의 맵의 크기를 196으로 설정\n", + " I = torch.eye(I_size).to(attn_maps[0].device) # Identity matrix\n", + " attn_rollout = []\n", + "\n", + " for attn_map in attn_maps:\n", + " if attn_map.size(-1) == I_size:\n", + " prod = attn_map + I\n", + " elif attn_map.size(-1) < I_size:\n", + " padding_size = I_size - attn_map.size(-1)\n", + " padded_attn_map = F.pad(attn_map, (0, padding_size, 0, 0)) # 오른쪽에 패딩 추가\n", + " prod = padded_attn_map + I\n", + " else:\n", + " raise ValueError(\"Unexpected attention map size.\")\n", + "\n", + " attn_rollout.append(prod)\n", + "\n", + " attn_rollout = [prod / prod.sum(dim=-1, keepdim=True) for prod in attn_rollout] # Normalize\n", + " return attn_rollout" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class Weights Rollout Shapes: [torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196]), torch.Size([196, 196])]\n" + ] + } + ], + "source": [ + "# Attention Rollout 실행\n", + "attn_rollout_st = attention_rollout_function(attn_maps_st_cpu)\n", + "\n", + "# For Class Weights\n", + "cls_weights_rollout_st = []\n", + "\n", + "for i in range(len(attn_rollout_st)): # attn_rollout_st의 크기로 반복\n", + " cls_weight_tensor_st = attn_rollout_st[i][0] # 첫 번째 배치의 텐서 사용\n", + "\n", + " # 크기를 확인하고, 14x14로 변환할 수 있는지 확인\n", + " if cls_weight_tensor_st.numel() == 14 * 14:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st.view(14, 14))\n", + " elif cls_weight_tensor_st.numel() == 196 * 196:\n", + " cls_weights_rollout_st.append(cls_weight_tensor_st) # 그대로 사용\n", + "\n", + "# 결과 확인\n", + "print(\"Class Weights Rollout Shapes:\", [cls_weight.shape for cls_weight in cls_weights_rollout_st])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0.1842, 0.6513, 0.9481, -0.0575, -0.4596, 0.9378, -0.0630, 0.0913,\n", + " -0.2760, 0.8236, -0.3048, -0.5067, -0.2237, 0.0104, 0.2620, 0.4669,\n", + " 0.3408, -0.6330, -0.4091, -0.4896, -0.9567, -1.5660, -0.2059, -0.2811,\n", + " -0.2813, -0.8560, -0.1496, -0.0455, -0.0833, 0.3670, 0.2738, -0.3592,\n", + " 0.3951, 0.6094, 0.5716, -0.0526, 0.3808, -0.0236, -0.7082, 0.1859,\n", + " 0.8467, -0.0996, 0.4303, 0.0613, 0.7428, 0.1889, -0.0631, -0.1368,\n", + " 0.1214, 0.5921, 0.1423, -0.4978, 0.5646, -0.2145, 0.1315, -0.3321,\n", + " 0.1872, -0.0260, -0.5459, 0.0105, 0.7164, -0.4216, -0.2821, 0.2232,\n", + " -0.1487, 0.1884, -0.1680, 0.3930, -0.3026, 0.8395, -0.5283, -0.1191,\n", + " 0.5188, -1.0522, 0.4154, -0.1342, -0.8908, 1.1572, 0.0318, 0.0403,\n", + " -0.4454, 0.0902, -0.5127, 0.6663, -0.3861, 0.5882, 0.0156, 0.5951,\n", + " -0.7754, 0.4059, 0.1418, 0.9044, -1.3180, -0.1740, 0.9519, -0.7662,\n", + " -0.1940, -0.4566, 0.9794, 0.2629, 0.3623, -0.1166, 0.5362, -0.2732,\n", + " -0.5911, -0.4877, 0.0517, 0.5354, 0.2801, -0.0113, 0.1709, -0.7622,\n", + " -0.1145, -0.9085, 0.3520, -0.6743, 0.2920, 0.2248, -0.2348, -0.4925,\n", + " 0.1487, -0.0883, -0.2174, -0.1746, -0.0644, -0.1431, 0.5035, -0.4010,\n", + " -0.1813, -0.0585, -0.1536, 0.7160, -0.5562, -0.1020, 0.2226, 0.2361,\n", + " 0.5698, -0.2106, 0.9990, -1.1657, -1.0541, 0.4736, 0.7887, 0.2140,\n", + " -0.4918, -0.7655, 1.2153, -0.0707, -0.4830, 0.2197, 0.0948, 0.8601,\n", + " 0.0194, 0.3300, -0.4195, 0.2778, -0.2720, -0.9317, -0.2371, 0.2978,\n", + " -0.3443, 1.3332, -0.0444, -0.4376, -0.2210, -0.0033, -0.0126, -0.0402,\n", + " -1.3693, -0.4690, 0.2604, 0.3911, -0.3407, 0.6645, -0.1561, -0.1206,\n", + " 0.4401, -0.1128, 0.3281, 0.3493, 0.4086, 0.2105, -0.1624, -0.0317,\n", + " 0.2122, 0.2028, 0.5176, -0.5969, -0.6693, 0.1342, -0.5071, -0.3850])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attn_map_st" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.2606])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weight_st" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[-0.1554, -0.0586, -0.0729, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.0529, -0.2460, -0.0994, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1179, -0.0144, -0.2249, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0701, -0.1249, -0.0585, ..., -0.1516, -0.0000, -0.0000],\n", + " [ 0.0249, -0.1786, -0.0892, ..., -0.0000, -0.1801, -0.0000],\n", + " [-0.1276, -0.1296, -0.0653, ..., -0.0000, -0.0000, -0.1503]]),\n", + " tensor([[-0.5720, 0.0253, -0.8815, ..., -0.0000, -0.0000, -0.0000],\n", + " [-0.1371, -0.5842, -0.8806, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 0.0658, -0.2696, -1.0837, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [-0.0265, -0.1658, -0.9726, ..., -0.4945, -0.0000, -0.0000],\n", + " [-0.0341, -0.0129, -0.8782, ..., -0.0000, -0.4722, -0.0000],\n", + " [-0.0291, 0.0190, -1.0195, ..., -0.0000, -0.0000, -0.5399]]),\n", + " tensor([[ -5.2143, -0.2073, -9.1526, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.0086, -6.6903, -8.8212, ..., -0.0000, -0.0000, -0.0000],\n", + " [ 1.6930, -0.4107, -17.7767, ..., -0.0000, -0.0000, -0.0000],\n", + " ...,\n", + " [ 1.9814, -0.8107, -13.0110, ..., -8.9967, -0.0000, -0.0000],\n", + " [ 0.9145, -0.1184, -6.4268, ..., -0.0000, -4.5341, -0.0000],\n", + " [ 1.8620, -1.2088, -13.1969, ..., -0.0000, -0.0000, -8.9710]]),\n", + " tensor([[ 0.4234, -0.1738, 0.4660, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0612, 0.4136, 0.4037, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0573, -0.1247, 0.9858, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0938, -0.0743, 0.4447, ..., 0.5810, 0.0000, 0.0000],\n", + " [-0.1542, -0.1397, 0.5147, ..., 0.0000, 0.5836, 0.0000],\n", + " [-0.1097, -0.1474, 0.4535, ..., 0.0000, 0.0000, 0.5374]]),\n", + " tensor([[ 0.9406, -0.2136, 0.8167, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1652, 0.5770, 0.8471, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.1347, -0.2466, 1.5981, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.1693, -0.2130, 0.8331, ..., 0.8538, 0.0000, 0.0000],\n", + " [ 0.1927, -0.2818, 0.8108, ..., 0.0000, 0.8053, 0.0000],\n", + " [ 0.1685, -0.2378, 0.8006, ..., 0.0000, 0.0000, 0.7944]]),\n", + " tensor([[ 0.7328, 0.0801, 0.5444, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0403, 0.7458, 0.5566, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.0786, 0.0814, 1.3531, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.0527, 0.0911, 0.6341, ..., 0.6709, 0.0000, 0.0000],\n", + " [-0.0836, 0.1107, 0.5707, ..., 0.0000, 0.7083, 0.0000],\n", + " [-0.0249, 0.0901, 0.5255, ..., 0.0000, 0.0000, 0.6873]]),\n", + " tensor([[0.8320, 0.2851, 0.8615, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1175, 0.9609, 0.8269, ..., 0.0000, 0.0000, 0.0000],\n", + " [0.1498, 0.2262, 1.6439, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [0.1284, 0.2232, 0.8748, ..., 0.7061, 0.0000, 0.0000],\n", + " [0.1243, 0.2083, 0.9053, ..., 0.0000, 0.7440, 0.0000],\n", + " [0.1109, 0.2427, 0.8597, ..., 0.0000, 0.0000, 0.6755]]),\n", + " tensor([[ 0.7119, 0.2967, 0.5289, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1194, 1.2095, 0.6030, ..., 0.0000, 0.0000, 0.0000],\n", + " [-0.1917, 0.3107, 1.4640, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [-0.1499, 0.3103, 0.6443, ..., 0.8962, 0.0000, 0.0000],\n", + " [-0.1631, 0.3251, 0.5534, ..., 0.0000, 0.8595, 0.0000],\n", + " [-0.1243, 0.2940, 0.6102, ..., 0.0000, 0.0000, 0.8932]]),\n", + " tensor([[ 3.0915, -0.7572, 1.6884, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8447, 1.3794, 1.6422, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.8184, -0.8285, 3.8009, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.7615, -0.8503, 1.5874, ..., 2.2260, 0.0000, 0.0000],\n", + " [ 0.8272, -0.8181, 1.6270, ..., 0.0000, 2.1608, 0.0000],\n", + " [ 0.8811, -0.7469, 1.4999, ..., 0.0000, 0.0000, 2.1590]]),\n", + " tensor([[ 2.4223, -0.3267, 0.9755, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.2824, 1.6931, 0.8163, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 0.3807, -0.1291, 2.9314, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.3839, -0.2936, 0.9776, ..., 2.0073, 0.0000, 0.0000],\n", + " [ 0.4626, -0.2595, 0.9444, ..., 0.0000, 2.0228, 0.0000],\n", + " [ 0.4498, -0.2101, 0.9627, ..., 0.0000, 0.0000, 1.8273]]),\n", + " tensor([[15.2405, 2.7610, 6.7306, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 9.2874, 22.5292, 11.1778, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 6.0527, 2.1433, 18.1478, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 8.6357, 3.6522, 9.4287, ..., 15.7923, 0.0000, 0.0000],\n", + " [ 4.7526, 2.0569, 6.1020, ..., 0.0000, 8.8777, 0.0000],\n", + " [ 3.7997, 1.8325, 4.7829, ..., 0.0000, 0.0000, 7.1644]]),\n", + " tensor([[ 3.2594, 1.9875, 2.6364, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.8766, 17.1216, 9.5131, ..., 0.0000, 0.0000, 0.0000],\n", + " [ 1.7089, 5.7129, 15.9202, ..., 0.0000, 0.0000, 0.0000],\n", + " ...,\n", + " [ 0.4791, 2.2901, 3.2946, ..., 3.3042, 0.0000, 0.0000],\n", + " [ 0.7041, 3.5211, 4.8167, ..., 0.0000, 5.4057, 0.0000],\n", + " [ 1.7779, 8.7794, 10.9012, ..., 0.0000, 0.0000, 12.3839]])]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cls_weights_rollout_st" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "두 모델 간의 차이 Loss 계산 (L1, L2, Cosine Similarity, KL Divergence로 구현)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total L1 Loss: 14.616626739501953\n", + "Layer 0 L1 Loss: 0.1479, Percentage of Total Loss: 1.0115%\n", + "Layer 1 L1 Loss: 0.2261, Percentage of Total Loss: 1.5469%\n", + "Layer 2 L1 Loss: 5.1462, Percentage of Total Loss: 35.2075%\n", + "Layer 3 L1 Loss: 0.1554, Percentage of Total Loss: 1.0629%\n", + "Layer 4 L1 Loss: 0.2121, Percentage of Total Loss: 1.4508%\n", + "Layer 5 L1 Loss: 0.1961, Percentage of Total Loss: 1.3413%\n", + "Layer 6 L1 Loss: 0.2350, Percentage of Total Loss: 1.6081%\n", + "Layer 7 L1 Loss: 0.2223, Percentage of Total Loss: 1.5206%\n", + "Layer 8 L1 Loss: 0.5502, Percentage of Total Loss: 3.7645%\n", + "Layer 9 L1 Loss: 0.5793, Percentage of Total Loss: 3.9635%\n", + "Layer 10 L1 Loss: 4.3009, Percentage of Total Loss: 29.4244%\n", + "Layer 11 L1 Loss: 2.6453, Percentage of Total Loss: 18.0980%\n" + ] + } + ], + "source": [ + "# L1 (Mean Absolute Error): 두 모델의 각 레이어 가중치 간의 절대 차이 계산\n", + "# L1 손실 계산\n", + "loss_l1 = 0\n", + "layer_losses_l1 = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = F.l1_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses_l1.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_l1 += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_l1 = loss_l1.item()\n", + "print(f\"Total L1 Loss: {total_loss_l1}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_l1 = [(layer_loss / total_loss_l1) * 100 for layer_loss in layer_losses_l1]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_l1):\n", + " print(f\"Layer {i} L1 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l1[i]:.4f}%\")\n", + "## 두 모델 간의 차이가 주로 Layer 2와 Layer 10에서 발생" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total L2 Loss: 453.0188903808594\n", + "Layer 0 L2 Loss: 0.0445, Percentage of Total Loss: 0.0098%\n", + "Layer 1 L2 Loss: 0.0991, Percentage of Total Loss: 0.0219%\n", + "Layer 2 L2 Loss: 394.6398, Percentage of Total Loss: 87.1133%\n", + "Layer 3 L2 Loss: 0.0411, Percentage of Total Loss: 0.0091%\n", + "Layer 4 L2 Loss: 0.0765, Percentage of Total Loss: 0.0169%\n", + "Layer 5 L2 Loss: 0.0602, Percentage of Total Loss: 0.0133%\n", + "Layer 6 L2 Loss: 0.0887, Percentage of Total Loss: 0.0196%\n", + "Layer 7 L2 Loss: 0.0799, Percentage of Total Loss: 0.0176%\n", + "Layer 8 L2 Loss: 0.5019, Percentage of Total Loss: 0.1108%\n", + "Layer 9 L2 Loss: 0.5323, Percentage of Total Loss: 0.1175%\n", + "Layer 10 L2 Loss: 38.7059, Percentage of Total Loss: 8.5440%\n", + "Layer 11 L2 Loss: 18.1490, Percentage of Total Loss: 4.0062%\n" + ] + } + ], + "source": [ + "# L2 (Euclidean Distance): 두 모델의 각 레이어 가중치 간의 유클리드 거리 계산\n", + "# L2 손실 계산\n", + "loss_l2 = 0\n", + "layer_losses_l2 = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산 (L2 손실)\n", + " layer_loss = F.mse_loss(cls_weights_rollout_st[i], cls_weights_rollout[i])\n", + " layer_losses_l2.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_l2 += layer_loss\n", + "\n", + "# 총 L2 손실 출력\n", + "total_loss_l2 = loss_l2.item()\n", + "print(f\"Total L2 Loss: {total_loss_l2}\")\n", + "\n", + "# 레이어별 L2 손실 비율 계산\n", + "layer_loss_percentages_l2 = [(layer_loss / total_loss_l2) * 100 for layer_loss in layer_losses_l2]\n", + "\n", + "# 각 레이어 L2 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_l2):\n", + " print(f\"Layer {i} L2 Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_l2[i]:.4f}%\")\n", + "\n", + "## Layer 2: 2번째 레이어의 L2 손실이 394.6398로, 전체 손실의 87.11% -> 레이어2에서 두 모델 간의 가중치 차이가 매우 크다\n", + "## 나머지 레이어들은 상대적으로 손실 값이 매우 작고, 전체 손실에서 차지하는 비율도 낮음.\n", + "## layer 2가 두 모델의 성능 차이에 큰 기여하고 있을 것이라 판단할 수 있음. 두 모델의 구조나 파라미터 조정시 해당 레이어 더 보기!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Cosine Loss: 11.714558601379395\n", + "Layer 0 Cosine Loss: 1.0442, Percentage of Total Loss: 8.9137%\n", + "Layer 1 Cosine Loss: 1.0548, Percentage of Total Loss: 9.0045%\n", + "Layer 2 Cosine Loss: 1.0349, Percentage of Total Loss: 8.8344%\n", + "Layer 3 Cosine Loss: 0.8728, Percentage of Total Loss: 7.4502%\n", + "Layer 4 Cosine Loss: 0.8025, Percentage of Total Loss: 6.8504%\n", + "Layer 5 Cosine Loss: 0.9400, Percentage of Total Loss: 8.0241%\n", + "Layer 6 Cosine Loss: 1.0328, Percentage of Total Loss: 8.8163%\n", + "Layer 7 Cosine Loss: 0.9947, Percentage of Total Loss: 8.4910%\n", + "Layer 8 Cosine Loss: 0.9695, Percentage of Total Loss: 8.2759%\n", + "Layer 9 Cosine Loss: 0.9745, Percentage of Total Loss: 8.3191%\n", + "Layer 10 Cosine Loss: 0.9839, Percentage of Total Loss: 8.3986%\n", + "Layer 11 Cosine Loss: 1.0100, Percentage of Total Loss: 8.6217%\n" + ] + } + ], + "source": [ + "# Cosine Similarity: 두 모델의 가중치 벡터 간의 코사인 유사도 계산\n", + "\n", + "from torch.nn.functional import cosine_similarity\n", + "\n", + "# 코사인 유사도 손실 계산\n", + "loss_cosine = 0\n", + "layer_losses_cosine = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 레이어별 가중치 차이 계산\n", + " layer_loss = 1 - cosine_similarity(cls_weights_rollout_st[i].view(1, -1), cls_weights_rollout[i].view(1, -1))\n", + " layer_losses_cosine.append(layer_loss.item()) # 손실을 리스트에 추가\n", + " loss_cosine += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_cosine = loss_cosine.item()\n", + "print(f\"Total Cosine Loss: {total_loss_cosine}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_cosine = [(layer_loss / total_loss_cosine) * 100 for layer_loss in layer_losses_cosine]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_cosine):\n", + " print(f\"Layer {i} Cosine Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_cosine[i]:.4f}%\")\n", + "\n", + "## 각 레이어의 손실은 0.8에서 1.1 사이로 분포 -> 두 모델의 가중치가 레이어별로 상당히 유사\n", + "## 레이어 0에서 레이어 11까지의 손실 값이 비슷한 수준이므로, 전체적으로 두 모델의 가중치가 유사" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total KL Divergence Loss: 31.367801666259766\n", + "Layer 0 KL Loss: 0.0042, Percentage of Total Loss: 0.0133%\n", + "Layer 1 KL Loss: 0.0067, Percentage of Total Loss: 0.0214%\n", + "Layer 2 KL Loss: 15.2732, Percentage of Total Loss: 48.6907%\n", + "Layer 3 KL Loss: 0.0010, Percentage of Total Loss: 0.0032%\n", + "Layer 4 KL Loss: 0.0025, Percentage of Total Loss: 0.0080%\n", + "Layer 5 KL Loss: 0.0023, Percentage of Total Loss: 0.0073%\n", + "Layer 6 KL Loss: 0.0020, Percentage of Total Loss: 0.0063%\n", + "Layer 7 KL Loss: 0.0030, Percentage of Total Loss: 0.0096%\n", + "Layer 8 KL Loss: 0.0287, Percentage of Total Loss: 0.0915%\n", + "Layer 9 KL Loss: 0.0246, Percentage of Total Loss: 0.0785%\n", + "Layer 10 KL Loss: 9.7367, Percentage of Total Loss: 31.0406%\n", + "Layer 11 KL Loss: 6.2829, Percentage of Total Loss: 20.0297%\n" + ] + } + ], + "source": [ + "# KL Divergence: 두 모델의 가중치를 확률 분포로 간주하고, Kullback-Leibler Divergence 계산하여 두 분포 간의 차이 측정\n", + "# KL 발산 계산\n", + "loss_kl = 0\n", + "layer_losses_kl = []\n", + "\n", + "for i in range(len(cls_weights_rollout_st)):\n", + " # 확률 분포로 정규화\n", + " p = F.softmax(cls_weights_rollout_st[i], dim=0) + 1e-10 # 작은 값 추가\n", + " q = F.softmax(cls_weights_rollout[i], dim=0) + 1e-10 # 작은 값 추가\n", + " \n", + " # KL 발산 계산\n", + " layer_loss = F.kl_div(p.log(), q, reduction='batchmean')\n", + " layer_losses_kl.append(layer_loss.item())\n", + " loss_kl += layer_loss\n", + "\n", + "# 총 손실 출력\n", + "total_loss_kl = loss_kl.item()\n", + "print(f\"Total KL Divergence Loss: {total_loss_kl}\")\n", + "\n", + "# 레이어별 손실 비율 계산\n", + "layer_loss_percentages_kl = [(layer_loss / total_loss_kl) * 100 for layer_loss in layer_losses_kl]\n", + "\n", + "# 각 레이어 손실 및 비율 출력\n", + "for i, layer_loss in enumerate(layer_losses_kl):\n", + " print(f\"Layer {i} KL Loss: {layer_loss:.4f}, Percentage of Total Loss: {layer_loss_percentages_kl[i]:.4f}%\")\n", + "## 두 모델 간의 차이가 주로 Layer 2, Layer 10, 그리고 Layer 11에서 발생" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evalutation_test.ipynb b/evalutation_test.ipynb new file mode 100644 index 0000000..b88b218 --- /dev/null +++ b/evalutation_test.ipynb @@ -0,0 +1,596 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "https://github.com/facebookresearch/deit/blob/main/README_deit.md" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#bash\n", + "\n", + "pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 torchaudio==0.8.1 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html\n", + "pip install torchtext==0.9.1\n", + "pip install timm==0.3.2\n", + "\n", + "- ImportError: cannot import name 'container_abcs' from 'torch._six' 오류 해결 위해 다운그레이드 필요!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\seonahryu\\anaconda3\\envs\\brp1\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.8.1+cu111\n", + "0.3.2\n" + ] + } + ], + "source": [ + "import torch\n", + "import timm\n", + "\n", + "print(torch.__version__) # PyTorch 버전 확인\n", + "print(timm.__version__) # timm 버전 확인\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import timm\n", + "assert timm.__version__ == \"0.3.2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluation func" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import torch.nn as nn\n", + "from thop import profile # FLOPs\n", + "\n", + "# 모델 FLOPs 계산 함수\n", + "def compute_flops(model, input_size=(1, 3, 224, 224)):\n", + " flops, params = profile(model, inputs=(torch.randn(input_size),))\n", + " return flops\n", + "\n", + "# 모델 평가 함수 정의\n", + "def evaluate_model(model, dataloader):\n", + " model.eval()\n", + " correct_top1 = 0\n", + " correct_top5 = 0\n", + " total = 0\n", + " loss_fn = nn.CrossEntropyLoss()\n", + " total_loss = 0.0\n", + "\n", + " start_time = time.time() # 평가 시작 시간 기록\n", + "\n", + " with torch.no_grad():\n", + " for images, labels in dataloader:\n", + " images, labels = images.to('cuda' if torch.cuda.is_available() else 'cpu'), labels.to('cuda' if torch.cuda.is_available() else 'cpu')\n", + " outputs = model(images)\n", + " _, predicted_top1 = torch.max(outputs.data, 1)\n", + " predicted_top5 = torch.topk(outputs.data, 5)[1] # Top-5 예측\n", + " \n", + " total += labels.size(0)\n", + " correct_top1 += (predicted_top1 == labels).sum().item()\n", + " \n", + " # Top-5 비교\n", + " correct_top5 += (predicted_top5 == labels.view(-1, 1)).sum().item()\n", + " \n", + " loss = loss_fn(outputs, labels)\n", + " total_loss += loss.item()\n", + "\n", + " end_time = time.time() # 평가 종료 시간 기록\n", + " elapsed_time = end_time - start_time # 총 소요 시간 계산\n", + " \n", + " accuracy_top1 = 100 * correct_top1 / total\n", + " accuracy_top5 = 100 * correct_top5 / total\n", + " avg_loss = total_loss / len(dataloader) # 평균 손실 계산\n", + " \n", + " # 결과 출력\n", + " print(f'Acc@1: {accuracy_top1:.3f} Acc@5: {accuracy_top5:.3f} loss: {avg_loss:.3f}')\n", + " \n", + " # 이미지 처리 속도 계산\n", + " im_per_sec = total / elapsed_time\n", + " print(f'Processing speed: {im_per_sec:.2f} images/sec')\n", + " \n", + " # FLOPs 계산\n", + " flops = compute_flops(model)\n", + " print(f'FLOPs: {flops:.2f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeiT-tiny" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + } + ], + "source": [ + "model_tiny = torch.hub.load('facebookresearch/deit:main', 'deit_tiny_patch16_224', pretrained=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): ModuleList(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_tiny.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Acc@1: 69.952 Acc@5: 90.014 loss: 1.310\n", + "Processing speed: 24.46 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "FLOPs: 1078633728.00\n" + ] + } + ], + "source": [ + "evaluate_model(model_tiny, test_loader) #34min" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which should give\n", + "\n", + "* Acc@1 72.202 Acc@5 91.124 loss 1.219\n", + "\n", + "제발..." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeiT-small" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n" + ] + } + ], + "source": [ + "model_small = torch.hub.load('facebookresearch/deit:main', 'deit_small_patch16_224', pretrained=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Acc@1: 78.846 Acc@5: 94.488 loss: 0.928\n", + "Processing speed: 11.94 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "FLOPs: 4248783360.00\n" + ] + } + ], + "source": [ + "model_small.eval()\n", + "evaluate_model(model_small, test_loader) #69min" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "giving\n", + "\n", + "* Acc@1 79.854 Acc@5 94.968 loss 0.881" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeiT-base" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in C:\\Users\\seonahryu/.cache\\torch\\hub\\facebookresearch_deit_main\n", + "Downloading: \"https://dl.fbaipublicfiles.com/deit/deit_base_patch16_224-b5f2ef4d.pth\" to C:\\Users\\seonahryu/.cache\\torch\\hub\\checkpoints\\deit_base_patch16_224-b5f2ef4d.pth\n", + "100%|██████████| 330M/330M [00:09<00:00, 36.1MB/s] \n" + ] + } + ], + "source": [ + "model_base = torch.hub.load('facebookresearch/deit:main', 'deit_base_patch16_224', pretrained=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Acc@1: 81.252 Acc@5: 95.322 loss: 0.854\n", + "Processing speed: 3.16 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "FLOPs: 16863630336.00\n" + ] + } + ], + "source": [ + "model_base.eval()\n", + "evaluate_model(model_base, test_loader) #263min" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This should give\n", + "\n", + "* Acc@1 81.846 Acc@5 95.594 loss 0.820" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/version_updated.ipynb b/version_updated.ipynb new file mode 100644 index 0000000..30975dd --- /dev/null +++ b/version_updated.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from timm import create_model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "model = create_model('deit_tiny_patch16_224', pretrained=True, num_classes=1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\seonahryu\\AppData\\Local\\Temp\\ipykernel_86428\\3522241460.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", + " model.load_state_dict(torch.load(\"C:/Users/seonahryu/Downloads/deit_tiny_patch16_224-a1311bcf.pth\"), strict=False)\n" + ] + }, + { + "data": { + "text/plain": [ + "_IncompatibleKeys(missing_keys=['cls_token', 'pos_embed', 'patch_embed.proj.weight', 'patch_embed.proj.bias', 'blocks.0.norm1.weight', 'blocks.0.norm1.bias', 'blocks.0.attn.qkv.weight', 'blocks.0.attn.qkv.bias', 'blocks.0.attn.proj.weight', 'blocks.0.attn.proj.bias', 'blocks.0.norm2.weight', 'blocks.0.norm2.bias', 'blocks.0.mlp.fc1.weight', 'blocks.0.mlp.fc1.bias', 'blocks.0.mlp.fc2.weight', 'blocks.0.mlp.fc2.bias', 'blocks.1.norm1.weight', 'blocks.1.norm1.bias', 'blocks.1.attn.qkv.weight', 'blocks.1.attn.qkv.bias', 'blocks.1.attn.proj.weight', 'blocks.1.attn.proj.bias', 'blocks.1.norm2.weight', 'blocks.1.norm2.bias', 'blocks.1.mlp.fc1.weight', 'blocks.1.mlp.fc1.bias', 'blocks.1.mlp.fc2.weight', 'blocks.1.mlp.fc2.bias', 'blocks.2.norm1.weight', 'blocks.2.norm1.bias', 'blocks.2.attn.qkv.weight', 'blocks.2.attn.qkv.bias', 'blocks.2.attn.proj.weight', 'blocks.2.attn.proj.bias', 'blocks.2.norm2.weight', 'blocks.2.norm2.bias', 'blocks.2.mlp.fc1.weight', 'blocks.2.mlp.fc1.bias', 'blocks.2.mlp.fc2.weight', 'blocks.2.mlp.fc2.bias', 'blocks.3.norm1.weight', 'blocks.3.norm1.bias', 'blocks.3.attn.qkv.weight', 'blocks.3.attn.qkv.bias', 'blocks.3.attn.proj.weight', 'blocks.3.attn.proj.bias', 'blocks.3.norm2.weight', 'blocks.3.norm2.bias', 'blocks.3.mlp.fc1.weight', 'blocks.3.mlp.fc1.bias', 'blocks.3.mlp.fc2.weight', 'blocks.3.mlp.fc2.bias', 'blocks.4.norm1.weight', 'blocks.4.norm1.bias', 'blocks.4.attn.qkv.weight', 'blocks.4.attn.qkv.bias', 'blocks.4.attn.proj.weight', 'blocks.4.attn.proj.bias', 'blocks.4.norm2.weight', 'blocks.4.norm2.bias', 'blocks.4.mlp.fc1.weight', 'blocks.4.mlp.fc1.bias', 'blocks.4.mlp.fc2.weight', 'blocks.4.mlp.fc2.bias', 'blocks.5.norm1.weight', 'blocks.5.norm1.bias', 'blocks.5.attn.qkv.weight', 'blocks.5.attn.qkv.bias', 'blocks.5.attn.proj.weight', 'blocks.5.attn.proj.bias', 'blocks.5.norm2.weight', 'blocks.5.norm2.bias', 'blocks.5.mlp.fc1.weight', 'blocks.5.mlp.fc1.bias', 'blocks.5.mlp.fc2.weight', 'blocks.5.mlp.fc2.bias', 'blocks.6.norm1.weight', 'blocks.6.norm1.bias', 'blocks.6.attn.qkv.weight', 'blocks.6.attn.qkv.bias', 'blocks.6.attn.proj.weight', 'blocks.6.attn.proj.bias', 'blocks.6.norm2.weight', 'blocks.6.norm2.bias', 'blocks.6.mlp.fc1.weight', 'blocks.6.mlp.fc1.bias', 'blocks.6.mlp.fc2.weight', 'blocks.6.mlp.fc2.bias', 'blocks.7.norm1.weight', 'blocks.7.norm1.bias', 'blocks.7.attn.qkv.weight', 'blocks.7.attn.qkv.bias', 'blocks.7.attn.proj.weight', 'blocks.7.attn.proj.bias', 'blocks.7.norm2.weight', 'blocks.7.norm2.bias', 'blocks.7.mlp.fc1.weight', 'blocks.7.mlp.fc1.bias', 'blocks.7.mlp.fc2.weight', 'blocks.7.mlp.fc2.bias', 'blocks.8.norm1.weight', 'blocks.8.norm1.bias', 'blocks.8.attn.qkv.weight', 'blocks.8.attn.qkv.bias', 'blocks.8.attn.proj.weight', 'blocks.8.attn.proj.bias', 'blocks.8.norm2.weight', 'blocks.8.norm2.bias', 'blocks.8.mlp.fc1.weight', 'blocks.8.mlp.fc1.bias', 'blocks.8.mlp.fc2.weight', 'blocks.8.mlp.fc2.bias', 'blocks.9.norm1.weight', 'blocks.9.norm1.bias', 'blocks.9.attn.qkv.weight', 'blocks.9.attn.qkv.bias', 'blocks.9.attn.proj.weight', 'blocks.9.attn.proj.bias', 'blocks.9.norm2.weight', 'blocks.9.norm2.bias', 'blocks.9.mlp.fc1.weight', 'blocks.9.mlp.fc1.bias', 'blocks.9.mlp.fc2.weight', 'blocks.9.mlp.fc2.bias', 'blocks.10.norm1.weight', 'blocks.10.norm1.bias', 'blocks.10.attn.qkv.weight', 'blocks.10.attn.qkv.bias', 'blocks.10.attn.proj.weight', 'blocks.10.attn.proj.bias', 'blocks.10.norm2.weight', 'blocks.10.norm2.bias', 'blocks.10.mlp.fc1.weight', 'blocks.10.mlp.fc1.bias', 'blocks.10.mlp.fc2.weight', 'blocks.10.mlp.fc2.bias', 'blocks.11.norm1.weight', 'blocks.11.norm1.bias', 'blocks.11.attn.qkv.weight', 'blocks.11.attn.qkv.bias', 'blocks.11.attn.proj.weight', 'blocks.11.attn.proj.bias', 'blocks.11.norm2.weight', 'blocks.11.norm2.bias', 'blocks.11.mlp.fc1.weight', 'blocks.11.mlp.fc1.bias', 'blocks.11.mlp.fc2.weight', 'blocks.11.mlp.fc2.bias', 'norm.weight', 'norm.bias', 'head.weight', 'head.bias'], unexpected_keys=['epoch', 'accuracy', 'model'])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.load_state_dict(torch.load(\"C:/Users/seonahryu/Desktop/brp1/deit_tiny_patch16_224-a1311bcf.pth\"), strict=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 192, kernel_size=(16, 16), stride=(16, 16))\n", + " (norm): Identity()\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (patch_drop): Identity()\n", + " (norm_pre): Identity()\n", + " (blocks): Sequential(\n", + " (0): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=192, out_features=576, bias=True)\n", + " (q_norm): Identity()\n", + " (k_norm): Identity()\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=192, out_features=192, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls1): Identity()\n", + " (drop_path1): Identity()\n", + " (norm2): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=192, out_features=768, bias=True)\n", + " (act): GELU(approximate='none')\n", + " (drop1): Dropout(p=0.0, inplace=False)\n", + " (norm): Identity()\n", + " (fc2): Linear(in_features=768, out_features=192, bias=True)\n", + " (drop2): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (ls2): Identity()\n", + " (drop_path2): Identity()\n", + " )\n", + " )\n", + " (norm): LayerNorm((192,), eps=1e-06, elementwise_affine=True)\n", + " (fc_norm): Identity()\n", + " (head_drop): Dropout(p=0.0, inplace=False)\n", + " (head): Linear(in_features=192, out_features=1000, bias=True)\n", + ")" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import ImageFolder\n", + "import torchvision.transforms as transforms\n", + "import torch.nn as nn\n", + "\n", + "# 데이터 전처리\n", + "transform = transforms.Compose([\n", + " transforms.Resize((224, 224)), # DeiT 모델 입력 크기에 맞춤\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", + "])\n", + "\n", + "test_dataset = ImageFolder(root=\"C:/Users/seonahryu/Desktop/brp1/ILSVRC2012_img_val\", transform=transform)\n", + "test_loader = DataLoader(dataset=test_dataset, batch_size=32, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "from thop import profile # FLOPs\n", + "\n", + "# 모델 FLOPs 계산 함수\n", + "def compute_flops(model, input_size=(1, 3, 224, 224)):\n", + " flops, params = profile(model, inputs=(torch.randn(input_size),))\n", + " return flops\n", + "\n", + "# 모델 평가 함수 정의\n", + "def evaluate_model(model, dataloader):\n", + " model.eval()\n", + " correct = 0\n", + " total = 0\n", + " loss_fn = nn.CrossEntropyLoss()\n", + "\n", + " start_time = time.time() # 평가 시작 시간 기록\n", + "\n", + " with torch.no_grad():\n", + " for images, labels in dataloader:\n", + " images, labels = images.to('cuda' if torch.cuda.is_available() else 'cpu'), labels.to('cuda' if torch.cuda.is_available() else 'cpu')\n", + " outputs = model(images)\n", + " _, predicted = torch.max(outputs.data, 1)\n", + " total += labels.size(0)\n", + " correct += (predicted == labels).sum().item()\n", + " loss = loss_fn(outputs, labels)\n", + "\n", + " end_time = time.time() # 평가 종료 시간 기록\n", + " elapsed_time = end_time - start_time # 총 소요 시간 계산\n", + "\n", + " accuracy = 100 * correct / total\n", + " print(f'Accuracy: {accuracy:.3f}%')\n", + " print(f'Loss: {loss.item():.3f}')\n", + " \n", + " # 이미지 처리 속도 계산\n", + " im_per_sec = total / elapsed_time\n", + " print(f'Processing speed: {im_per_sec:.2f} images/sec')\n", + " \n", + " # FLOPs 계산\n", + " flops = compute_flops(model)\n", + " print(f'FLOPs: {flops:.2f}')" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 69.956%\n", + "Loss: 2.464\n", + "Processing speed: 26.88 images/sec\n", + "[INFO] Register count_convNd() for .\n", + "[INFO] Register zero_ops() for .\n", + "[INFO] Register count_normalization() for .\n", + "[INFO] Register count_linear() for .\n", + "[INFO] Register zero_ops() for .\n", + "FLOPs: 1078633728.00\n" + ] + } + ], + "source": [ + "evaluate_model(model, test_loader)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "brp1", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.21" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}