1.7 MiB
1.7 MiB
In [10]:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
from IPython.display import display
import math
def parse_log_file():
results = []
with open('logs/recall_50step.txt', 'r') as f:
for line in f:
match = re.search(r'step(\d+)_layer(\d+)_head(\d+)_window_diff', line)
if match:
step = int(match.group(1))
layer = int(match.group(2))
head = int(match.group(3))
recall_match = re.search(r's recall: ([\d.]+)', line)
recall = float(recall_match.group(1)) if recall_match else None
results.append([step, layer, head, recall])
return np.array(results)
def plot_attention_recalls(data):
steps = np.unique(data[:, 0])
n_steps = len(steps)
plots_per_row = 5
n_rows = math.ceil(n_steps / plots_per_row)
fig = plt.figure(figsize=(20, 4*n_rows))
for idx, step in enumerate(steps):
step_data = data[data[:, 0] == step]
n_layers = int(np.max(step_data[:, 1])) + 1
n_heads = int(np.max(step_data[:, 2])) + 1
recall_matrix = np.zeros((n_layers, n_heads))
for item in step_data:
layer, head, recall = int(item[1]), int(item[2]), item[3]
recall_matrix[layer, head] = recall
ax = plt.subplot(n_rows, plots_per_row, idx + 1)
sns.heatmap(recall_matrix,
cmap='rocket',
vmin=0,
vmax=1.0,
cbar_kws={'label': ''})
plt.title(f'Step {int(step)}', fontsize=10)
plt.xlabel('Head', fontsize=8)
plt.ylabel('Layer', fontsize=8)
plt.xticks(fontsize=6)
plt.yticks(fontsize=6)
plt.gca().invert_yaxis()
plt.tight_layout()
display(plt.gcf())
plt.close()
# 运行代码并检查数据
data = parse_log_file()
plot_attention_recalls(data)
In [16]:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
from IPython.display import display
import math
%matplotlib inline
def analyze_data(data):
steps = np.unique(data[:, 0])
n_layers = int(np.max(data[:, 1])) + 1
n_heads = int(np.max(data[:, 2])) + 1
# 1. Calculate maximum difference between adjacent steps
max_diff = 0
max_diff_info = None
for step_idx in range(len(steps)-1):
step = steps[step_idx]
next_step = steps[step_idx+1]
step_data = data[data[:, 0] == step]
next_step_data = data[data[:, 0] == next_step]
for layer in range(n_layers):
for head in range(n_heads):
current_recall = step_data[(step_data[:, 1] == layer) & (step_data[:, 2] == head)][0][3]
next_recall = next_step_data[(next_step_data[:, 1] == layer) & (next_step_data[:, 2] == head)][0][3]
diff = abs(next_recall - current_recall)
if diff > max_diff:
max_diff = diff
max_diff_info = (step, next_step, layer, head, current_recall, next_recall)
print("Maximum difference between adjacent steps:")
print(f"Between steps {int(max_diff_info[0])} and {int(max_diff_info[1])}")
print(f"Layer: {max_diff_info[2]}, Head: {max_diff_info[3]}")
print(f"Values: {max_diff_info[4]:.4f} -> {max_diff_info[5]:.4f}")
print(f"Difference: {max_diff:.4f}\n")
# 2. Calculate and plot average recall for each layer
print("Layer averages:")
layer_avgs = []
for layer in range(n_layers):
layer_data = data[data[:, 1] == layer]
layer_avg = np.mean(layer_data[:, 3])
layer_avgs.append(layer_avg)
print(f"Layer {layer}: {layer_avg}")
print()
plt.figure(figsize=(6, 6))
plt.plot(range(n_layers), layer_avgs, '-', linewidth=2, label='Layer Average')
plt.axhline(y=0.2825, color='r', linestyle='--', label='Baseline')
plt.xlabel('Layer')
plt.ylabel('Average Recall')
plt.title('Average Recall per Layer')
plt.legend()
plt.grid(True)
plt.show()
# 3. Calculate and plot average recall for each head
print("Head averages:")
head_avgs = []
for head in range(n_heads):
head_data = data[data[:, 2] == head]
head_avg = np.mean(head_data[:, 3])
head_avgs.append(head_avg)
print(f"Head {head}: {head_avg}")
print()
plt.figure(figsize=(6, 6))
plt.plot(range(n_heads), head_avgs, '-', linewidth=2, label='Head Average')
plt.axhline(y=0.2825, color='r', linestyle='--', label='Baseline')
plt.xlabel('Head')
plt.ylabel('Average Recall')
plt.title('Average Recall per Head')
plt.legend()
plt.grid(True)
plt.show()
# Load the data from the previous notebook's results
data = parse_log_file()
analyze_data(data)Maximum difference between adjacent steps: Between steps 7 and 8 Layer: 44, Head: 21 Values: 0.5000 -> 0.6414 Difference: 0.1414 Layer averages: Layer 0: 0.5826740657662351 Layer 1: 0.715089840516448 Layer 2: 0.7859195268278321 Layer 3: 0.6356571511055032 Layer 4: 0.7351154555504521 Layer 5: 0.6587931529308358 Layer 6: 0.6317004643132289 Layer 7: 0.6738401606927316 Layer 8: 0.6361850821847717 Layer 9: 0.6597752206151685 Layer 10: 0.7616837338606517 Layer 11: 0.6595387489845356 Layer 12: 0.5687351076429089 Layer 13: 0.6451355597873529 Layer 14: 0.582408705689013 Layer 15: 0.6924233253548543 Layer 16: 0.681581568295757 Layer 17: 0.8013674856722355 Layer 18: 0.6578779139121373 Layer 19: 0.6826423628379902 Layer 20: 0.5368523755917947 Layer 21: 0.5844278392940759 Layer 22: 0.6668876067424814 Layer 23: 0.6448011482010285 Layer 24: 0.6419519259159764 Layer 25: 0.708203005194664 Layer 26: 0.567639525110523 Layer 27: 0.623080801529189 Layer 28: 0.6187612362330158 Layer 29: 0.4842567726721366 Layer 30: 0.4185990354294578 Layer 31: 0.6471388272941112 Layer 32: 0.5902333160365622 Layer 33: 0.6613699742406607 Layer 34: 0.6553941999003291 Layer 35: 0.7131362365186215 Layer 36: 0.6908923954765002 Layer 37: 0.686744272870322 Layer 38: 0.6344761400669813 Layer 39: 0.6000020120541255 Layer 40: 0.6557533066347241 Layer 41: 0.55967210393399 Layer 42: 0.6136161725843946 Layer 43: 0.6479404142995675 Layer 44: 0.5957304690778256 Layer 45: 0.5723411684917907 Layer 46: 0.7039746219913164 Layer 47: 0.44657273807873327 Layer 48: 0.6697168015812834 Layer 49: 0.6174203474633395 Layer 50: 0.7322309567344686 Layer 51: 0.5043302394387623 Layer 52: 0.5222633640468121 Layer 53: 0.7016120676758388 Layer 54: 0.3400704135745764 Layer 55: 0.4064293996244669 Layer 56: 0.5047468397393823 Layer 57: 0.5067595624551177 Layer 58: 0.3253624338035782 Layer 59: 0.3498436001439889
Head averages: Head 0: 0.6121624366641044 Head 1: 0.600840474580725 Head 2: 0.593720748608311 Head 3: 0.6069059848984083 Head 4: 0.5974564596414565 Head 5: 0.6483907912572225 Head 6: 0.5975060957719882 Head 7: 0.5738213963111242 Head 8: 0.6072616468220949 Head 9: 0.6303625401109457 Head 10: 0.6524359939197699 Head 11: 0.6349524038756887 Head 12: 0.6262960639347632 Head 13: 0.5648046152591706 Head 14: 0.6025992100959023 Head 15: 0.6639624330550432 Head 16: 0.5980224341203769 Head 17: 0.5746792206913233 Head 18: 0.6234525771687428 Head 19: 0.5745950098410249 Head 20: 0.6045221923266848 Head 21: 0.6601326223487656 Head 22: 0.6553300572633743 Head 23: 0.6155387115478516
In [17]:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
from IPython.display import display
import math
def parse_log_file():
results = []
with open('logs/recall_50steps_61_720_720.txt', 'r') as f:
for line in f:
match = re.search(r'step(\d+)_layer(\d+)_head(\d+)_window_diff', line)
if match:
step = int(match.group(1))
layer = int(match.group(2))
head = int(match.group(3))
recall_match = re.search(r's recall: ([\d.]+)', line)
recall = float(recall_match.group(1)) if recall_match else None
results.append([step, layer, head, recall])
return np.array(results)
def plot_attention_recalls(data):
steps = np.unique(data[:, 0])
n_steps = len(steps)
plots_per_row = 5
n_rows = math.ceil(n_steps / plots_per_row)
fig = plt.figure(figsize=(20, 4*n_rows))
for idx, step in enumerate(steps):
step_data = data[data[:, 0] == step]
n_layers = int(np.max(step_data[:, 1])) + 1
n_heads = int(np.max(step_data[:, 2])) + 1
recall_matrix = np.zeros((n_layers, n_heads))
for item in step_data:
layer, head, recall = int(item[1]), int(item[2]), item[3]
recall_matrix[layer, head] = recall
ax = plt.subplot(n_rows, plots_per_row, idx + 1)
sns.heatmap(recall_matrix,
cmap='rocket',
vmin=0,
vmax=1.0,
cbar_kws={'label': ''})
plt.title(f'Step {int(step)}', fontsize=10)
plt.xlabel('Head', fontsize=8)
plt.ylabel('Layer', fontsize=8)
plt.xticks(fontsize=6)
plt.yticks(fontsize=6)
plt.gca().invert_yaxis()
plt.tight_layout()
display(plt.gcf())
plt.close()
# 运行代码并检查数据
data = parse_log_file()
plot_attention_recalls(data)
In [18]:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
from IPython.display import display
import math
%matplotlib inline
def analyze_data(data):
steps = np.unique(data[:, 0])
n_layers = int(np.max(data[:, 1])) + 1
n_heads = int(np.max(data[:, 2])) + 1
# 1. Calculate maximum difference between adjacent steps
max_diff = 0
max_diff_info = None
for step_idx in range(len(steps)-1):
step = steps[step_idx]
next_step = steps[step_idx+1]
step_data = data[data[:, 0] == step]
next_step_data = data[data[:, 0] == next_step]
for layer in range(n_layers):
for head in range(n_heads):
current_recall = step_data[(step_data[:, 1] == layer) & (step_data[:, 2] == head)][0][3]
next_recall = next_step_data[(next_step_data[:, 1] == layer) & (next_step_data[:, 2] == head)][0][3]
diff = abs(next_recall - current_recall)
if diff > max_diff:
max_diff = diff
max_diff_info = (step, next_step, layer, head, current_recall, next_recall)
print("Maximum difference between adjacent steps:")
print(f"Between steps {int(max_diff_info[0])} and {int(max_diff_info[1])}")
print(f"Layer: {max_diff_info[2]}, Head: {max_diff_info[3]}")
print(f"Values: {max_diff_info[4]:.4f} -> {max_diff_info[5]:.4f}")
print(f"Difference: {max_diff:.4f}\n")
# 2. Calculate and plot average recall for each layer
print("Layer averages:")
layer_avgs = []
for layer in range(n_layers):
layer_data = data[data[:, 1] == layer]
layer_avg = np.mean(layer_data[:, 3])
layer_avgs.append(layer_avg)
print(f"Layer {layer}: {layer_avg}")
print()
plt.figure(figsize=(6, 6))
plt.plot(range(n_layers), layer_avgs, '-', linewidth=2, label='Layer Average')
plt.axhline(y=0.1485, color='r', linestyle='--', label='Baseline')
plt.xlabel('Layer')
plt.ylabel('Average Recall')
plt.title('Average Recall per Layer')
plt.legend()
plt.grid(True)
plt.show()
# 3. Calculate and plot average recall for each head
print("Head averages:")
head_avgs = []
for head in range(n_heads):
head_data = data[data[:, 2] == head]
head_avg = np.mean(head_data[:, 3])
head_avgs.append(head_avg)
print(f"Head {head}: {head_avg}")
print()
plt.figure(figsize=(6, 6))
plt.plot(range(n_heads), head_avgs, '-', linewidth=2, label='Head Average')
plt.axhline(y=0.1485, color='r', linestyle='--', label='Baseline')
plt.xlabel('Head')
plt.ylabel('Average Recall')
plt.title('Average Recall per Head')
plt.legend()
plt.grid(True)
plt.show()
# Load the data from the previous notebook's results
data = parse_log_file()
analyze_data(data)Maximum difference between adjacent steps: Between steps 3 and 4 Layer: 41, Head: 17 Values: 0.9777 -> 0.3746 Difference: 0.6030 Layer averages: Layer 0: 0.6027111108514857 Layer 1: 0.7375502753220523 Layer 2: 0.8099431970335755 Layer 3: 0.7174270560243373 Layer 4: 0.8520716087296921 Layer 5: 0.7844819223830218 Layer 6: 0.7567682629847837 Layer 7: 0.7076818226453372 Layer 8: 0.6556781099457287 Layer 9: 0.692803903628675 Layer 10: 0.7751487468114854 Layer 11: 0.7134395380349404 Layer 12: 0.6617634386276798 Layer 13: 0.7256072581534752 Layer 14: 0.7182733753302017 Layer 15: 0.7975496623129354 Layer 16: 0.7494582831302853 Layer 17: 0.8488659605382199 Layer 18: 0.7818623669529757 Layer 19: 0.813055833722152 Layer 20: 0.7419374564104876 Layer 21: 0.7796829167001357 Layer 22: 0.7766482665610949 Layer 23: 0.7515268738347557 Layer 24: 0.82901041966918 Layer 25: 0.8077977401740234 Layer 26: 0.7595842490351408 Layer 27: 0.7069970912532331 Layer 28: 0.7148135049856196 Layer 29: 0.806903439189803 Layer 30: 0.691072097894345 Layer 31: 0.7961636811870315 Layer 32: 0.7487348708019193 Layer 33: 0.7306519327891209 Layer 34: 0.7444020360874262 Layer 35: 0.8702466995187844 Layer 36: 0.6768739120378431 Layer 37: 0.8324637117174024 Layer 38: 0.8925175252088499 Layer 39: 0.7834827655599729 Layer 40: 0.8384321862222056 Layer 41: 0.8371392665213132 Layer 42: 0.7655153957950201 Layer 43: 0.830503807544149 Layer 44: 0.7961207006591802 Layer 45: 0.8590925185539974 Layer 46: 0.8597080369755464 Layer 47: 0.5619246439986738 Layer 48: 0.6637828129204802 Layer 49: 0.659030623497436 Layer 50: 0.796580066446974 Layer 51: 0.5551761382545641 Layer 52: 0.5256997002365558 Layer 53: 0.7103276771459023 Layer 54: 0.3492981998753802 Layer 55: 0.3751112768691038 Layer 56: 0.6546684659992552 Layer 57: 0.7718023278446925 Layer 58: 0.44789506695756737 Layer 59: 0.6411244508012733
Head averages: Head 0: 0.6889132932889749 Head 1: 0.6996755154239469 Head 2: 0.7463015787472256 Head 3: 0.7433949132857464 Head 4: 0.6992309229972716 Head 5: 0.7840117977609974 Head 6: 0.7156335119020599 Head 7: 0.7264125641744904 Head 8: 0.7488031355491666 Head 9: 0.7142418678331797 Head 10: 0.7250191661592198 Head 11: 0.7072016891353844 Head 12: 0.7457247806084163 Head 13: 0.7042641562281042 Head 14: 0.7461887401756682 Head 15: 0.7759414603835987 Head 16: 0.7451361442517439 Head 17: 0.7530917539273205 Head 18: 0.7080613111665568 Head 19: 0.7195567014361254 Head 20: 0.7254043387892869 Head 21: 0.7349489967831797 Head 22: 0.7655425920550268 Head 23: 0.7143335826967014
