Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/dataloader_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:584ddc45f1addd41e01adcc6cf70f582d8774c147c912cd238caff8ebf7fb069
+size 964

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -0,0 +1,528 @@
+max_steps: 17500
+0 val loss 11.3001
+0 val perplexity 80829.3594
+0 train 11.304919 (lr=1.3750e-07) (hash(x)=153418542)
+100 val loss 9.9044
+100 val perplexity 20017.5859
+100 train 9.872492 (lr=1.3888e-05) (hash(x)=154406429)
+200 val loss 9.0083
+200 val perplexity 8170.6182
+200 train 8.930150 (lr=2.7500e-05) (hash(x)=143722001)
+300 val loss 8.0266
+300 val perplexity 3061.2393
+300 train 7.972954 (lr=2.7498e-05) (hash(x)=152421341)
+400 val loss 7.6457
+400 val perplexity 2091.6472
+400 train 7.543222 (lr=2.7492e-05) (hash(x)=152989689)
+500 val loss 7.5404
+500 val perplexity 1882.6754
+500 train 7.470467 (lr=2.7482e-05) (hash(x)=145113544)
+600 val loss 7.4701
+600 val perplexity 1754.6958
+600 train 7.464890 (lr=2.7467e-05) (hash(x)=150071018)
+700 val loss 7.4072
+700 val perplexity 1647.8019
+700 train 7.498733 (lr=2.7449e-05) (hash(x)=156594182)
+800 val loss 7.3685
+800 val perplexity 1585.1940
+800 train 7.370912 (lr=2.7427e-05) (hash(x)=153358238)
+900 val loss 7.3397
+900 val perplexity 1540.2643
+900 train 7.168626 (lr=2.7400e-05) (hash(x)=146617394)
+1000 val loss 7.3128
+1000 val perplexity 1499.4252
+1000 train 7.256218 (lr=2.7370e-05) (hash(x)=148983354)
+1100 val loss 7.2795
+1100 val perplexity 1450.2510
+1100 train 7.309369 (lr=2.7335e-05) (hash(x)=146313352)
+1200 val loss 7.2520
+1200 val perplexity 1410.9708
+1200 train 7.212644 (lr=2.7297e-05) (hash(x)=150770333)
+1300 val loss 7.2062
+1300 val perplexity 1347.7965
+1300 train 7.149687 (lr=2.7254e-05) (hash(x)=153237933)
+1400 val loss 7.1656
+1400 val perplexity 1294.1826
+1400 train 7.001445 (lr=2.7207e-05) (hash(x)=145246201)
+1500 val loss 7.1301
+1500 val perplexity 1249.0450
+1500 train 7.126824 (lr=2.7157e-05) (hash(x)=154002540)
+1600 val loss 7.0984
+1600 val perplexity 1210.0066
+1600 train 6.874021 (lr=2.7102e-05) (hash(x)=142635842)
+1700 val loss 7.0601
+1700 val perplexity 1164.6145
+1700 train 7.108422 (lr=2.7044e-05) (hash(x)=151450555)
+1800 val loss 7.0224
+1800 val perplexity 1121.4412
+1800 train 6.941932 (lr=2.6981e-05) (hash(x)=148299832)
+1900 val loss 6.9836
+1900 val perplexity 1078.8276
+1900 train 7.138231 (lr=2.6915e-05) (hash(x)=155487476)
+2000 val loss 6.9545
+2000 val perplexity 1047.8662
+2000 train 6.934770 (lr=2.6845e-05) (hash(x)=152887721)
+2100 val loss 6.9112
+2100 val perplexity 1003.4163
+2100 train 6.617202 (lr=2.6771e-05) (hash(x)=143430603)
+2200 val loss 6.8834
+2200 val perplexity 975.9449
+2200 train 6.881304 (lr=2.6693e-05) (hash(x)=153721194)
+2300 val loss 6.8433
+2300 val perplexity 937.5591
+2300 train 6.975938 (lr=2.6611e-05) (hash(x)=148702429)
+2400 val loss 6.8007
+2400 val perplexity 898.5173
+2400 train 6.848701 (lr=2.6525e-05) (hash(x)=153047184)
+2500 val loss 6.7660
+2500 val perplexity 867.8541
+2500 train 6.643397 (lr=2.6436e-05) (hash(x)=149720123)
+2600 val loss 6.7336
+2600 val perplexity 840.1345
+2600 train 6.765120 (lr=2.6343e-05) (hash(x)=152466045)
+2700 val loss 6.7126
+2700 val perplexity 822.6757
+2700 train 6.643747 (lr=2.6246e-05) (hash(x)=155796305)
+2800 val loss 6.6753
+2800 val perplexity 792.6136
+2800 train 6.515200 (lr=2.6146e-05) (hash(x)=151991552)
+2900 val loss 6.6396
+2900 val perplexity 764.7943
+2900 train 7.280913 (lr=2.6042e-05) (hash(x)=166537989)
+3000 val loss 6.6073
+3000 val perplexity 740.4809
+3000 train 6.607131 (lr=2.5934e-05) (hash(x)=147892594)
+3100 val loss 6.5759
+3100 val perplexity 717.6121
+3100 train 6.537065 (lr=2.5823e-05) (hash(x)=149536927)
+3200 val loss 6.5500
+3200 val perplexity 699.2293
+3200 train 6.262515 (lr=2.5709e-05) (hash(x)=135678663)
+3300 val loss 6.5233
+3300 val perplexity 680.8391
+3300 train 6.577153 (lr=2.5590e-05) (hash(x)=148523646)
+3400 val loss 6.5011
+3400 val perplexity 665.8766
+3400 train 6.573230 (lr=2.5469e-05) (hash(x)=150281778)
+3500 val loss 6.4749
+3500 val perplexity 648.6446
+3500 train 6.416248 (lr=2.5344e-05) (hash(x)=151527953)
+3600 val loss 6.4579
+3600 val perplexity 637.7144
+3600 train 6.692443 (lr=2.5215e-05) (hash(x)=141303739)
+3700 val loss 6.4379
+3700 val perplexity 625.0728
+3700 train 6.471801 (lr=2.5083e-05) (hash(x)=146778597)
+3800 val loss 6.4130
+3800 val perplexity 609.7248
+3800 train 6.366365 (lr=2.4948e-05) (hash(x)=146902797)
+3900 val loss 6.3946
+3900 val perplexity 598.6218
+3900 train 6.325142 (lr=2.4810e-05) (hash(x)=142022555)
+4000 val loss 6.3738
+4000 val perplexity 586.2914
+4000 train 6.296466 (lr=2.4669e-05) (hash(x)=151590309)
+4100 val loss 6.3588
+4100 val perplexity 577.5575
+4100 train 6.199865 (lr=2.4524e-05) (hash(x)=144127100)
+4200 val loss 6.3470
+4200 val perplexity 570.7855
+4200 train 6.413688 (lr=2.4376e-05) (hash(x)=149526774)
+4300 val loss 6.3291
+4300 val perplexity 560.6296
+4300 train 6.277426 (lr=2.4226e-05) (hash(x)=150924362)
+4400 val loss 6.3049
+4400 val perplexity 547.2457
+4400 train 6.464039 (lr=2.4072e-05) (hash(x)=155547282)
+4500 val loss 6.2960
+4500 val perplexity 542.4135
+4500 train 6.273605 (lr=2.3915e-05) (hash(x)=152479203)
+4600 val loss 6.2818
+4600 val perplexity 534.7637
+4600 train 6.199425 (lr=2.3755e-05) (hash(x)=146739760)
+4700 val loss 6.2702
+4700 val perplexity 528.5671
+4700 train 6.492648 (lr=2.3593e-05) (hash(x)=159977388)
+4800 val loss 6.2597
+4800 val perplexity 523.0594
+4800 train 6.143291 (lr=2.3428e-05) (hash(x)=142959739)
+4900 val loss 6.2422
+4900 val perplexity 513.9914
+4900 train 6.195841 (lr=2.3260e-05) (hash(x)=152253034)
+5000 val loss 6.2314
+5000 val perplexity 508.4516
+5000 train 6.202939 (lr=2.3089e-05) (hash(x)=140379747)
+5100 val loss 6.2158
+5100 val perplexity 500.5816
+5100 train 6.235268 (lr=2.2916e-05) (hash(x)=151826392)
+5200 val loss 6.2073
+5200 val perplexity 496.3683
+5200 train 6.160838 (lr=2.2740e-05) (hash(x)=152869594)
+5300 val loss 6.1968
+5300 val perplexity 491.1580
+5300 train 5.984524 (lr=2.2561e-05) (hash(x)=139093709)
+5400 val loss 6.1878
+5400 val perplexity 486.7791
+5400 train 6.060760 (lr=2.2381e-05) (hash(x)=145908738)
+5500 val loss 6.1784
+5500 val perplexity 482.2269
+5500 train 6.033603 (lr=2.2197e-05) (hash(x)=137913426)
+5600 val loss 6.1617
+5600 val perplexity 474.2398
+5600 train 6.011183 (lr=2.2012e-05) (hash(x)=140770390)
+5700 val loss 6.1576
+5700 val perplexity 472.2713
+5700 train 6.076570 (lr=2.1824e-05) (hash(x)=143728578)
+5800 val loss 6.1446
+5800 val perplexity 466.1837
+5800 train 6.218975 (lr=2.1634e-05) (hash(x)=154469787)
+5900 val loss 6.1377
+5900 val perplexity 462.9820
+5900 train 6.090513 (lr=2.1442e-05) (hash(x)=139316559)
+6000 val loss 6.1299
+6000 val perplexity 459.3950
+6000 train 6.138595 (lr=2.1248e-05) (hash(x)=152058488)
+6100 val loss 6.1255
+6100 val perplexity 457.3819
+6100 train 6.128792 (lr=2.1051e-05) (hash(x)=152958689)
+6200 val loss 6.1035
+6200 val perplexity 447.4265
+6200 train 6.071043 (lr=2.0853e-05) (hash(x)=150887864)
+6300 val loss 6.0974
+6300 val perplexity 444.6889
+6300 train 6.045360 (lr=2.0653e-05) (hash(x)=156290469)
+6400 val loss 6.0854
+6400 val perplexity 439.3943
+6400 train 6.386467 (lr=2.0451e-05) (hash(x)=159160159)
+6500 val loss 6.0784
+6500 val perplexity 436.3256
+6500 train 5.996751 (lr=2.0247e-05) (hash(x)=151239121)
+6600 val loss 6.0738
+6600 val perplexity 434.3249
+6600 train 6.088350 (lr=2.0042e-05) (hash(x)=151964101)
+6700 val loss 6.0651
+6700 val perplexity 430.5475
+6700 train 5.970629 (lr=1.9835e-05) (hash(x)=141291229)
+6800 val loss 6.0589
+6800 val perplexity 427.8987
+6800 train 5.985006 (lr=1.9626e-05) (hash(x)=160416800)
+6900 val loss 6.0504
+6900 val perplexity 424.2713
+6900 train 5.953206 (lr=1.9416e-05) (hash(x)=143382691)
+7000 val loss 6.0369
+7000 val perplexity 418.5984
+7000 train 5.939549 (lr=1.9205e-05) (hash(x)=147606572)
+7100 val loss 6.0279
+7100 val perplexity 414.8367
+7100 train 5.859169 (lr=1.8992e-05) (hash(x)=145901538)
+7200 val loss 6.0257
+7200 val perplexity 413.9467
+7200 train 5.854834 (lr=1.8778e-05) (hash(x)=139099153)
+7300 val loss 6.0190
+7300 val perplexity 411.1549
+7300 train 5.979025 (lr=1.8562e-05) (hash(x)=140434004)
+7400 val loss 6.0171
+7400 val perplexity 410.3832
+7400 train 6.131842 (lr=1.8346e-05) (hash(x)=155496146)
+7500 val loss 6.0081
+7500 val perplexity 406.6947
+7500 train 5.932770 (lr=1.8128e-05) (hash(x)=146940054)
+7600 val loss 5.9919
+7600 val perplexity 400.1565
+7600 train 6.220881 (lr=1.7910e-05) (hash(x)=165785723)
+7700 val loss 5.9845
+7700 val perplexity 397.2233
+7700 train 6.136736 (lr=1.7691e-05) (hash(x)=156339008)
+7800 val loss 5.9830
+7800 val perplexity 396.6223
+7800 train 5.985073 (lr=1.7470e-05) (hash(x)=148593556)
+7900 val loss 5.9865
+7900 val perplexity 398.0033
+7900 train 5.919378 (lr=1.7249e-05) (hash(x)=148079742)
+8000 val loss 5.9645
+8000 val perplexity 389.3672
+8000 train 5.894415 (lr=1.7028e-05) (hash(x)=159706718)
+8100 val loss 5.9625
+8100 val perplexity 388.5824
+8100 train 5.958558 (lr=1.6805e-05) (hash(x)=160203225)
+8200 val loss 5.9512
+8200 val perplexity 384.2123
+8200 train 5.893342 (lr=1.6582e-05) (hash(x)=146176536)
+8300 val loss 5.9434
+8300 val perplexity 381.2125
+8300 train 6.014031 (lr=1.6359e-05) (hash(x)=158883824)
+8400 val loss 5.9337
+8400 val perplexity 377.5597
+8400 train 5.745647 (lr=1.6135e-05) (hash(x)=137102572)
+8500 val loss 5.9282
+8500 val perplexity 375.4724
+8500 train 5.958540 (lr=1.5911e-05) (hash(x)=152812797)
+8600 val loss 5.9255
+8600 val perplexity 374.4656
+8600 train 5.783076 (lr=1.5687e-05) (hash(x)=148854244)
+8700 val loss 5.9364
+8700 val perplexity 378.5663
+8700 train 5.752574 (lr=1.5462e-05) (hash(x)=135590023)
+8800 val loss 5.9116
+8800 val perplexity 369.2888
+8800 train 5.844384 (lr=1.5237e-05) (hash(x)=144913509)
+8900 val loss 5.9079
+8900 val perplexity 367.9172
+8900 train 6.030931 (lr=1.5013e-05) (hash(x)=148045669)
+9000 val loss 5.9000
+9000 val perplexity 365.0257
+9000 train 5.921853 (lr=1.4788e-05) (hash(x)=146661746)
+9100 val loss 5.8940
+9100 val perplexity 362.8599
+9100 train 5.834559 (lr=1.4563e-05) (hash(x)=145354881)
+9200 val loss 5.8897
+9200 val perplexity 361.2867
+9200 train 5.755416 (lr=1.4339e-05) (hash(x)=150939310)
+9300 val loss 5.8864
+9300 val perplexity 360.0898
+9300 train 5.723167 (lr=1.4115e-05) (hash(x)=162258309)
+9400 val loss 5.8789
+9400 val perplexity 357.4210
+9400 train 5.885062 (lr=1.3891e-05) (hash(x)=154701830)
+9500 val loss 5.8721
+9500 val perplexity 355.0111
+9500 train 5.761203 (lr=1.3668e-05) (hash(x)=142241404)
+9600 val loss 5.8658
+9600 val perplexity 352.7511
+9600 train 5.832513 (lr=1.3445e-05) (hash(x)=148439858)
+9700 val loss 5.8651
+9700 val perplexity 352.5332
+9700 train 5.865323 (lr=1.3222e-05) (hash(x)=152204410)
+9800 val loss 5.8561
+9800 val perplexity 349.3512
+9800 train 5.703402 (lr=1.3001e-05) (hash(x)=141294630)
+9900 val loss 5.8539
+9900 val perplexity 348.6057
+9900 train 5.826562 (lr=1.2780e-05) (hash(x)=147390033)
+10000 val loss 5.8480
+10000 val perplexity 346.5345
+10000 train 5.846817 (lr=1.2559e-05) (hash(x)=144131604)
+10100 val loss 5.8387
+10100 val perplexity 343.3346
+10100 train 6.040196 (lr=1.2340e-05) (hash(x)=158814818)
+10200 val loss 5.8349
+10200 val perplexity 342.0135
+10200 train 5.914101 (lr=1.2122e-05) (hash(x)=153491388)
+10300 val loss 5.8303
+10300 val perplexity 340.4523
+10300 train 5.807222 (lr=1.1904e-05) (hash(x)=149600707)
+10400 val loss 5.8256
+10400 val perplexity 338.8573
+10400 train 5.740230 (lr=1.1688e-05) (hash(x)=152277064)
+10500 val loss 5.8238
+10500 val perplexity 338.2697
+10500 train 5.728241 (lr=1.1472e-05) (hash(x)=150447091)
+10600 val loss 5.8134
+10600 val perplexity 334.7462
+10600 train 5.837738 (lr=1.1258e-05) (hash(x)=151606822)
+10700 val loss 5.8070
+10700 val perplexity 332.6146
+10700 train 5.796923 (lr=1.1045e-05) (hash(x)=158730240)
+10800 val loss 5.8067
+10800 val perplexity 332.5079
+10800 train 5.856279 (lr=1.0834e-05) (hash(x)=163095518)
+10900 val loss 5.8021
+10900 val perplexity 330.9915
+10900 train 5.560940 (lr=1.0624e-05) (hash(x)=145260395)
+11000 val loss 5.7968
+11000 val perplexity 329.2379
+11000 train 5.802653 (lr=1.0415e-05) (hash(x)=155970264)
+11100 val loss 5.7948
+11100 val perplexity 328.5754
+11100 train 5.889194 (lr=1.0208e-05) (hash(x)=159871097)
+11200 val loss 5.7875
+11200 val perplexity 326.1909
+11200 train 5.876340 (lr=1.0003e-05) (hash(x)=161135244)
+11300 val loss 5.7826
+11300 val perplexity 324.6163
+11300 train 5.817194 (lr=9.7991e-06) (hash(x)=150649336)
+11400 val loss 5.7799
+11400 val perplexity 323.7270
+11400 train 5.749750 (lr=9.5972e-06) (hash(x)=152230013)
+11500 val loss 5.7759
+11500 val perplexity 322.4249
+11500 train 5.821681 (lr=9.3970e-06) (hash(x)=163300613)
+11600 val loss 5.7760
+11600 val perplexity 322.4666
+11600 train 5.755337 (lr=9.1988e-06) (hash(x)=151203838)
+11700 val loss 5.7722
+11700 val perplexity 321.2419
+11700 train 5.793112 (lr=9.0025e-06) (hash(x)=155255111)
+11800 val loss 5.7741
+11800 val perplexity 321.8658
+11800 train 5.731073 (lr=8.8082e-06) (hash(x)=139380668)
+11900 val loss 5.7614
+11900 val perplexity 317.8024
+11900 train 5.690895 (lr=8.6160e-06) (hash(x)=144033405)
+12000 val loss 5.7601
+12000 val perplexity 317.3673
+12000 train 5.683661 (lr=8.4260e-06) (hash(x)=148689574)
+12100 val loss 5.7576
+12100 val perplexity 316.5753
+12100 train 5.693658 (lr=8.2381e-06) (hash(x)=143272239)
+12200 val loss 5.7536
+12200 val perplexity 315.3213
+12200 train 5.826474 (lr=8.0526e-06) (hash(x)=157671609)
+12300 val loss 5.7511
+12300 val perplexity 314.5443
+12300 train 5.774444 (lr=7.8694e-06) (hash(x)=150363247)
+12400 val loss 5.7458
+12400 val perplexity 312.8600
+12400 train 5.633980 (lr=7.6885e-06) (hash(x)=146857787)
+12500 val loss 5.7446
+12500 val perplexity 312.5057
+12500 train 5.712029 (lr=7.5101e-06) (hash(x)=153327320)
+12600 val loss 5.7406
+12600 val perplexity 311.2592
+12600 train 5.462293 (lr=7.3342e-06) (hash(x)=143612983)
+12700 val loss 5.7379
+12700 val perplexity 310.4186
+12700 train 5.676553 (lr=7.1609e-06) (hash(x)=150739132)
+12800 val loss 5.7364
+12800 val perplexity 309.9375
+12800 train 5.584580 (lr=6.9903e-06) (hash(x)=147986204)
+12900 val loss 5.7343
+12900 val perplexity 309.3034
+12900 train 5.899726 (lr=6.8223e-06) (hash(x)=155218833)
+13000 val loss 5.7284
+13000 val perplexity 307.4839
+13000 train 5.723590 (lr=6.6570e-06) (hash(x)=148867960)
+13100 val loss 5.7259
+13100 val perplexity 306.7019
+13100 train 5.747935 (lr=6.4945e-06) (hash(x)=160101478)
+13200 val loss 5.7260
+13200 val perplexity 306.7493
+13200 train 5.585583 (lr=6.3349e-06) (hash(x)=151080097)
+13300 val loss 5.7252
+13300 val perplexity 306.4901
+13300 train 5.698304 (lr=6.1782e-06) (hash(x)=150403039)
+13400 val loss 5.7210
+13400 val perplexity 305.2061
+13400 train 5.663198 (lr=6.0244e-06) (hash(x)=129728571)
+13500 val loss 5.7166
+13500 val perplexity 303.8716
+13500 train 5.632593 (lr=5.8737e-06) (hash(x)=148988625)
+13600 val loss 5.7159
+13600 val perplexity 303.6613
+13600 train 5.503659 (lr=5.7259e-06) (hash(x)=149790861)
+13700 val loss 5.7148
+13700 val perplexity 303.3160
+13700 train 5.644720 (lr=5.5813e-06) (hash(x)=153505062)
+13800 val loss 5.7140
+13800 val perplexity 303.0839
+13800 train 5.621727 (lr=5.4398e-06) (hash(x)=143845463)
+13900 val loss 5.7126
+13900 val perplexity 302.6628
+13900 train 5.516610 (lr=5.3016e-06) (hash(x)=143378076)
+14000 val loss 5.7074
+14000 val perplexity 301.0799
+14000 train 5.666134 (lr=5.1665e-06) (hash(x)=148681755)
+14100 val loss 5.7058
+14100 val perplexity 300.6119
+14100 train 5.593862 (lr=5.0348e-06) (hash(x)=145530465)
+14200 val loss 5.7041
+14200 val perplexity 300.1066
+14200 train 5.649960 (lr=4.9063e-06) (hash(x)=162139713)
+14300 val loss 5.7019
+14300 val perplexity 299.4321
+14300 train 5.708417 (lr=4.7813e-06) (hash(x)=141997596)
+14400 val loss 5.7009
+14400 val perplexity 299.1285
+14400 train 5.672005 (lr=4.6596e-06) (hash(x)=146156313)
+14500 val loss 5.7000
+14500 val perplexity 298.8767
+14500 train 5.478105 (lr=4.5414e-06) (hash(x)=145298178)
+14600 val loss 5.6955
+14600 val perplexity 297.5303
+14600 train 5.779585 (lr=4.4267e-06) (hash(x)=148232610)
+14700 val loss 5.6937
+14700 val perplexity 296.9908
+14700 train 5.574780 (lr=4.3155e-06) (hash(x)=151079115)
+14800 val loss 5.6924
+14800 val perplexity 296.6122
+14800 train 5.674882 (lr=4.2079e-06) (hash(x)=151340983)
+14900 val loss 5.6911
+14900 val perplexity 296.2318
+14900 train 5.897525 (lr=4.1039e-06) (hash(x)=150213803)
+15000 val loss 5.6910
+15000 val perplexity 296.1945
+15000 train 5.598524 (lr=4.0035e-06) (hash(x)=157652334)
+15100 val loss 5.6897
+15100 val perplexity 295.8021
+15100 train 5.417470 (lr=3.9068e-06) (hash(x)=145078577)
+15200 val loss 5.6870
+15200 val perplexity 295.0057
+15200 train 5.639482 (lr=3.8138e-06) (hash(x)=155898006)
+15300 val loss 5.6855
+15300 val perplexity 294.5604
+15300 train 5.726047 (lr=3.7245e-06) (hash(x)=149346671)
+15400 val loss 5.6834
+15400 val perplexity 293.9359
+15400 train 5.534364 (lr=3.6390e-06) (hash(x)=145708113)
+15500 val loss 5.6821
+15500 val perplexity 293.5533
+15500 train 5.875023 (lr=3.5572e-06) (hash(x)=165102970)
+15600 val loss 5.6821
+15600 val perplexity 293.5759
+15600 train 5.470008 (lr=3.4793e-06) (hash(x)=139106517)
+15700 val loss 5.6807
+15700 val perplexity 293.1580
+15700 train 5.543227 (lr=3.4052e-06) (hash(x)=141680838)
+15800 val loss 5.6785
+15800 val perplexity 292.5013
+15800 train 5.570748 (lr=3.3350e-06) (hash(x)=136152310)
+15900 val loss 5.6766
+15900 val perplexity 291.9579
+15900 train 5.745454 (lr=3.2687e-06) (hash(x)=151104640)
+16000 val loss 5.6760
+16000 val perplexity 291.7779
+16000 train 5.618743 (lr=3.2063e-06) (hash(x)=153333281)
+16100 val loss 5.6743
+16100 val perplexity 291.2915
+16100 train 5.631518 (lr=3.1478e-06) (hash(x)=145628059)
+16200 val loss 5.6748
+16200 val perplexity 291.4199
+16200 train 5.398105 (lr=3.0932e-06) (hash(x)=137220041)
+16300 val loss 5.6739
+16300 val perplexity 291.1725
+16300 train 5.843124 (lr=3.0427e-06) (hash(x)=161215791)
+16400 val loss 5.6729
+16400 val perplexity 290.8825
+16400 train 5.388996 (lr=2.9961e-06) (hash(x)=136194680)
+16500 val loss 5.6734
+16500 val perplexity 291.0279
+16500 train 5.465913 (lr=2.9535e-06) (hash(x)=152209568)
+16600 val loss 5.6698
+16600 val perplexity 289.9884
+16600 train 5.788033 (lr=2.9149e-06) (hash(x)=157345721)
+16700 val loss 5.6678
+16700 val perplexity 289.3843
+16700 train 5.794989 (lr=2.8804e-06) (hash(x)=146883642)
+16800 val loss 5.6680
+16800 val perplexity 289.4506
+16800 train 5.638534 (lr=2.8498e-06) (hash(x)=153037972)
+16900 val loss 5.6672
+16900 val perplexity 289.2180
+16900 train 5.444636 (lr=2.8234e-06) (hash(x)=150691367)
+17000 val loss 5.6672
+17000 val perplexity 289.2126
+17000 train 5.424174 (lr=2.8010e-06) (hash(x)=148975206)
+17100 val loss 5.6681
+17100 val perplexity 289.4754
+17100 train 5.402887 (lr=2.7826e-06) (hash(x)=142026221)
+17200 val loss 5.6642
+17200 val perplexity 288.3643
+17200 train 5.721988 (lr=2.7684e-06) (hash(x)=138211880)
+17300 val loss 5.6625
+17300 val perplexity 287.8575
+17300 train 5.518986 (lr=2.7582e-06) (hash(x)=143366666)
+17400 val loss 5.6624
+17400 val perplexity 287.8243
+17400 train 5.650568 (lr=2.7520e-06) (hash(x)=150791766)
+17499 val loss 5.6616
+17499 val perplexity 287.6108

attention_kindselective_n_heads4_seed1343/model_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3803a8a02a4eef5e6266bf1ae9e90cba514f507edd246d6c5a4cfb561f938f2d
+size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1909b34facdb00da31a90a3685379e7ef706df56e3b6259af0d1102d544286dc
+size 179406214