ChatGPT先生に教わりながら「Transformerの肝」である「注意機構（Attention機構）」を可視化する

transformers-cli download <モデルID>

transformers-cli download bert-base-uncased

import torch
from transformers import BertTokenizer, BertModel

# BERTの事前学習済みモデルをロード
model_name = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertModel.from_pretrained(model_name, output_attentions=True)

import seaborn as sns
import matplotlib.pyplot as plt

# テキストのトークン化
text = "I love natural language processing!"
tokens = tokenizer.encode_plus(text, return_tensors='pt', add_special_tokens=True, max_length=512, truncation=True)

# Attentionの取得
outputs = model(tokens['input_ids'], attention_mask=tokens['attention_mask'])
attention = torch.mean(outputs.attentions[-1], dim=1)[0].detach().numpy()

# ヒートマップの作成
sns.heatmap(attention, cmap="YlGnBu", xticklabels=tokenizer.convert_ids_to_tokens(tokens['input_ids'][0]), yticklabels=tokenizer.convert_ids_to_tokens(tokens['input_ids'][0]))
plt.show()

attention = torch.mean(outputs.attentions[-1], dim=1)[0].detach().numpy()

import torch
import matplotlib.pyplot as plt
from transformers import BertTokenizer, BertModel

# モデルとトークナイザーの準備
model_name = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertModel.from_pretrained(model_name, output_attentions=True)

# 入力文を定義する
text = "I love natural language processing!"

# トークン化する
tokens = tokenizer.tokenize(text)
input_ids = tokenizer.encode(text, return_tensors='pt')

# Attentionを取得する
outputs = model(input_ids)
attentions = outputs.attentions
attention = torch.mean(attentions[-1], dim=1)[0].detach().numpy()

# 折れ線グラフをプロットする
plt.plot(attention)
plt.xticks(range(len(tokens)), tokens, rotation=90)
plt.show()

outputs = model(input_ids)
attentions = outputs.attentions

BertModel.from_pretrained(model_name, output_attentions=True)）

import matplotlib as mpl
plt.rcParams['font.family'] = 'MS Gothic'

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

model_name = "rinna/japanese-gpt2-medium"
tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
model = AutoModelForCausalLM.from_pretrained(model_name, output_attentions=True)

input_text = "今日はカレーライスを食べました。明日はラーメンを食べようと思います。"
input_ids = tokenizer.encode(input_text, return_tensors='pt')

# Attentionを取得する
outputs = model(input_ids)
attentions = outputs.attentions
# 言語モデルに入力された文に対するAttentionの重みを取得
attention = torch.mean(attentions[-1], dim=1)[0].detach().numpy()

import matplotlib.pyplot as plt
import numpy as np

import matplotlib as mpl
plt.rcParams['font.family'] = 'MS Gothic'

fig, ax = plt.subplots(figsize=(10, 10))
im = ax.imshow(attention)

ax.set_xticks(range(len(tokenizer.tokenize(input_text))))
ax.set_yticks(range(len(tokenizer.tokenize(input_text))))
ax.set_xticklabels(tokenizer.tokenize(input_text))
ax.set_yticklabels(tokenizer.tokenize(input_text))

plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")

for i in range(len(tokenizer.tokenize(input_text))):
    for j in range(len(tokenizer.tokenize(input_text))):
        text = ax.text(j, i, "{:.2f}".format(attention[i, j]),
                       ha="center", va="center", color="w")

plt.show()

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

# 3Dグラフを作成する
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

# ノードを配置する
x, y, z = [0], [0], [0]
for i in range(len(attention)):
    for j in range(len(attention[i])):
        x.append(j+1)
        y.append(i+1)
        z.append(attention[i][j] * 100)

# ノードをプロットする
ax.scatter(x, y, z, s=100, alpha=0.5)

# グラフを表示する
plt.show()

tokens = tokenizer.tokenize(input_text)
# 折れ線グラフをプロットする
plt.plot(attention)
plt.xticks(range(len(tokens)), tokens, rotation=90)
plt.show()