Upload folder using huggingface_hub

12_head_baseline_lr_6e-4/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "logs/repro_selective_pattern_rankings/12_head_baseline_lr_6e-4", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "n_embd": 264, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 4375, "warmup_steps": 250, "group": "repro_selective_pattern_rankings", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 64, "total_batch_size": 131072, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.0006, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "12_head_baseline_lr_6e-4"}

12_head_baseline_lr_6e-4/dataloader_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ec6682852ed60a9d42cb4047300f65a04f87328c0ad7a4516be84d11b28f216
+size 964

12_head_baseline_lr_6e-4/log2.txt

@@ -0,0 +1,529 @@
+max_steps: 4375
+0 val loss 10.9275
+0 val perplexity 55688.3086
+0 train 10.927717 (lr=8.3916e-07) (hash(x)=93356070)
+10 train 10.157005 (lr=9.2308e-06) (hash(x)=91253010)
+20 train 9.639227 (lr=1.7622e-05) (hash(x)=74934453)
+30 train 9.478985 (lr=2.6014e-05) (hash(x)=79278034)
+40 train 9.202698 (lr=3.4406e-05) (hash(x)=80107892)
+50 train 8.920856 (lr=4.2797e-05) (hash(x)=70745428)
+60 train 8.680676 (lr=5.1189e-05) (hash(x)=80077589)
+70 train 8.218871 (lr=5.9580e-05) (hash(x)=76213766)
+80 train 7.973695 (lr=6.7972e-05) (hash(x)=83218328)
+90 train 7.729426 (lr=7.6364e-05) (hash(x)=74855845)
+100 val loss 7.5897
+100 val perplexity 1977.6705
+100 train 7.465366 (lr=8.4755e-05) (hash(x)=82814902)
+110 train 7.380744 (lr=9.3147e-05) (hash(x)=84907741)
+120 train 7.277765 (lr=1.0154e-04) (hash(x)=82613223)
+130 train 7.125362 (lr=1.0993e-04) (hash(x)=83540876)
+140 train 7.101282 (lr=1.1832e-04) (hash(x)=75095216)
+150 train 7.149079 (lr=1.2671e-04) (hash(x)=97190944)
+160 train 7.014097 (lr=1.3510e-04) (hash(x)=82117809)
+170 train 6.897202 (lr=1.4350e-04) (hash(x)=70514724)
+180 train 6.814301 (lr=1.5189e-04) (hash(x)=81029624)
+190 train 6.696559 (lr=1.6028e-04) (hash(x)=78996841)
+200 val loss 6.6417
+200 val perplexity 766.3725
+200 train 6.645527 (lr=1.6867e-04) (hash(x)=79845097)
+210 train 6.501446 (lr=1.7706e-04) (hash(x)=78997683)
+220 train 6.390601 (lr=1.8545e-04) (hash(x)=74895865)
+230 train 6.567307 (lr=1.9385e-04) (hash(x)=80933276)
+240 train 6.268052 (lr=2.0224e-04) (hash(x)=77664606)
+250 train 6.276261 (lr=2.1063e-04) (hash(x)=75903930)
+260 train 6.179594 (lr=2.1902e-04) (hash(x)=82996853)
+270 train 6.113348 (lr=2.2741e-04) (hash(x)=73269568)
+280 train 6.050037 (lr=2.3580e-04) (hash(x)=82906427)
+290 train 5.968565 (lr=2.4420e-04) (hash(x)=71797895)
+300 val loss 6.1027
+300 val perplexity 447.0765
+300 train 5.830674 (lr=2.5259e-04) (hash(x)=85232249)
+310 train 5.762503 (lr=2.6098e-04) (hash(x)=75252489)
+320 train 5.865934 (lr=2.6937e-04) (hash(x)=74394644)
+330 train 5.930887 (lr=2.7776e-04) (hash(x)=82772910)
+340 train 5.976114 (lr=2.8615e-04) (hash(x)=81627464)
+350 train 5.956862 (lr=2.9455e-04) (hash(x)=84678053)
+360 train 5.903929 (lr=3.0294e-04) (hash(x)=81884128)
+370 train 5.883206 (lr=3.1133e-04) (hash(x)=76379242)
+380 train 5.658519 (lr=3.1972e-04) (hash(x)=98182875)
+390 train 5.666385 (lr=3.2811e-04) (hash(x)=79710436)
+400 val loss 5.7099
+400 val perplexity 301.8305
+400 train 5.665086 (lr=3.3650e-04) (hash(x)=79841071)
+410 train 5.577646 (lr=3.4490e-04) (hash(x)=75844151)
+420 train 5.574399 (lr=3.5329e-04) (hash(x)=73125036)
+430 train 5.618525 (lr=3.6168e-04) (hash(x)=84214858)
+440 train 5.507903 (lr=3.7007e-04) (hash(x)=80456994)
+450 train 5.469387 (lr=3.7846e-04) (hash(x)=76735962)
+460 train 5.405846 (lr=3.8685e-04) (hash(x)=81845446)
+470 train 5.221576 (lr=3.9524e-04) (hash(x)=76094689)
+480 train 5.342197 (lr=4.0364e-04) (hash(x)=83806686)
+490 train 5.167534 (lr=4.1203e-04) (hash(x)=84690227)
+500 val loss 5.4609
+500 val perplexity 235.3170
+500 train 5.220690 (lr=4.2042e-04) (hash(x)=71851938)
+510 train 5.312815 (lr=4.2881e-04) (hash(x)=77159346)
+520 train 5.363231 (lr=4.3720e-04) (hash(x)=80755753)
+530 train 5.466526 (lr=4.4559e-04) (hash(x)=75379679)
+540 train 5.377451 (lr=4.5399e-04) (hash(x)=82458619)
+550 train 5.360423 (lr=4.6238e-04) (hash(x)=86366396)
+560 train 5.338173 (lr=4.7077e-04) (hash(x)=85095044)
+570 train 5.311355 (lr=4.7916e-04) (hash(x)=78385159)
+580 train 5.269881 (lr=4.8755e-04) (hash(x)=79342394)
+590 train 5.264031 (lr=4.9594e-04) (hash(x)=70782192)
+600 val loss 5.2656
+600 val perplexity 193.5721
+600 train 5.237235 (lr=5.0434e-04) (hash(x)=76130353)
+610 train 5.148084 (lr=5.1273e-04) (hash(x)=74778440)
+620 train 5.106846 (lr=5.2112e-04) (hash(x)=79129709)
+630 train 5.008933 (lr=5.2951e-04) (hash(x)=76469962)
+640 train 5.021534 (lr=5.3790e-04) (hash(x)=78288049)
+650 train 4.977587 (lr=5.4629e-04) (hash(x)=76641388)
+660 train 4.879727 (lr=5.5469e-04) (hash(x)=75937906)
+670 train 4.907400 (lr=5.6308e-04) (hash(x)=74807157)
+680 train 4.787249 (lr=5.7147e-04) (hash(x)=77490144)
+690 train 4.818188 (lr=5.7986e-04) (hash(x)=72900124)
+700 val loss 5.0805
+700 val perplexity 160.8561
+700 train 4.789006 (lr=5.8825e-04) (hash(x)=73424218)
+710 train 5.061962 (lr=5.9664e-04) (hash(x)=86052345)
+720 train 5.011309 (lr=6.0000e-04) (hash(x)=77462613)
+730 train 5.016435 (lr=5.9998e-04) (hash(x)=79027471)
+740 train 4.978360 (lr=5.9994e-04) (hash(x)=78149992)
+750 train 4.939659 (lr=5.9988e-04) (hash(x)=89147499)
+760 train 4.939705 (lr=5.9980e-04) (hash(x)=74931538)
+770 train 4.890773 (lr=5.9970e-04) (hash(x)=89648204)
+780 train 4.937779 (lr=5.9958e-04) (hash(x)=83708293)
+790 train 4.927411 (lr=5.9944e-04) (hash(x)=84336512)
+800 val loss 4.8783
+800 val perplexity 131.4092
+800 train 4.806079 (lr=5.9928e-04) (hash(x)=75025285)
+810 train 4.709286 (lr=5.9910e-04) (hash(x)=79436536)
+820 train 4.702584 (lr=5.9890e-04) (hash(x)=90467390)
+830 train 4.746223 (lr=5.9869e-04) (hash(x)=77292020)
+840 train 4.651864 (lr=5.9845e-04) (hash(x)=75568927)
+850 train 4.673661 (lr=5.9819e-04) (hash(x)=79671781)
+860 train 4.747439 (lr=5.9791e-04) (hash(x)=84280943)
+870 train 4.728676 (lr=5.9761e-04) (hash(x)=80901294)
+880 train 4.839090 (lr=5.9730e-04) (hash(x)=81437584)
+890 train 4.692418 (lr=5.9696e-04) (hash(x)=73893778)
+900 val loss 4.7051
+900 val perplexity 110.5109
+900 train 4.626082 (lr=5.9660e-04) (hash(x)=74987794)
+910 train 4.644361 (lr=5.9623e-04) (hash(x)=93721374)
+920 train 4.627056 (lr=5.9583e-04) (hash(x)=79149678)
+930 train 4.569266 (lr=5.9542e-04) (hash(x)=83179387)
+940 train 4.567439 (lr=5.9498e-04) (hash(x)=89009978)
+950 train 4.540099 (lr=5.9453e-04) (hash(x)=76483735)
+960 train 4.472516 (lr=5.9405e-04) (hash(x)=78638850)
+970 train 4.505652 (lr=5.9356e-04) (hash(x)=90415206)
+980 train 4.432364 (lr=5.9305e-04) (hash(x)=90819195)
+990 train 4.557418 (lr=5.9251e-04) (hash(x)=85938852)
+1000 val loss 4.5439
+1000 val perplexity 94.0544
+1000 train 4.593643 (lr=5.9196e-04) (hash(x)=82436789)
+1010 train 4.631203 (lr=5.9139e-04) (hash(x)=63022149)
+1020 train 4.533416 (lr=5.9080e-04) (hash(x)=83501199)
+1030 train 4.526703 (lr=5.9019e-04) (hash(x)=77093326)
+1040 train 4.390250 (lr=5.8956e-04) (hash(x)=73437559)
+1050 train 4.482548 (lr=5.8891e-04) (hash(x)=84550388)
+1060 train 4.478627 (lr=5.8825e-04) (hash(x)=86184566)
+1070 train 4.463887 (lr=5.8756e-04) (hash(x)=85644922)
+1080 train 4.398623 (lr=5.8686e-04) (hash(x)=86179801)
+1090 train 4.365890 (lr=5.8613e-04) (hash(x)=82868303)
+1100 val loss 4.4390
+1100 val perplexity 84.6924
+1100 train 4.466256 (lr=5.8539e-04) (hash(x)=96780388)
+1110 train 4.403344 (lr=5.8463e-04) (hash(x)=87223122)
+1120 train 4.362931 (lr=5.8385e-04) (hash(x)=77292786)
+1130 train 4.385690 (lr=5.8305e-04) (hash(x)=88761375)
+1140 train 4.295548 (lr=5.8223e-04) (hash(x)=80956468)
+1150 train 4.462841 (lr=5.8140e-04) (hash(x)=84725462)
+1160 train 4.443383 (lr=5.8054e-04) (hash(x)=76098113)
+1170 train 4.420262 (lr=5.7967e-04) (hash(x)=84387685)
+1180 train 4.357417 (lr=5.7878e-04) (hash(x)=76120321)
+1190 train 4.323272 (lr=5.7787e-04) (hash(x)=85974065)
+1200 val loss 4.3421
+1200 val perplexity 76.8714
+1200 train 4.224998 (lr=5.7694e-04) (hash(x)=79340644)
+1210 train 4.319157 (lr=5.7599e-04) (hash(x)=88345025)
+1220 train 4.219506 (lr=5.7503e-04) (hash(x)=86406230)
+1230 train 4.205584 (lr=5.7405e-04) (hash(x)=80608975)
+1240 train 4.262941 (lr=5.7305e-04) (hash(x)=81635225)
+1250 train 4.096692 (lr=5.7203e-04) (hash(x)=82126376)
+1260 train 4.269254 (lr=5.7099e-04) (hash(x)=83737972)
+1270 train 4.178236 (lr=5.6994e-04) (hash(x)=81132345)
+1280 train 4.262471 (lr=5.6887e-04) (hash(x)=85850404)
+1290 train 4.267895 (lr=5.6778e-04) (hash(x)=83490640)
+1300 val loss 4.2785
+1300 val perplexity 72.1317
+1300 train 4.247018 (lr=5.6667e-04) (hash(x)=77545187)
+1310 train 4.292161 (lr=5.6555e-04) (hash(x)=86412685)
+1320 train 4.185549 (lr=5.6440e-04) (hash(x)=86429640)
+1330 train 4.317113 (lr=5.6325e-04) (hash(x)=79645281)
+1340 train 4.271056 (lr=5.6207e-04) (hash(x)=82668541)
+1350 train 4.140747 (lr=5.6088e-04) (hash(x)=73627248)
+1360 train 4.237195 (lr=5.5967e-04) (hash(x)=74845133)
+1370 train 4.145235 (lr=5.5844e-04) (hash(x)=77850497)
+1380 train 4.166664 (lr=5.5720e-04) (hash(x)=76143954)
+1390 train 4.177283 (lr=5.5593e-04) (hash(x)=80675544)
+1400 val loss 4.2317
+1400 val perplexity 68.8322
+1400 train 4.144437 (lr=5.5466e-04) (hash(x)=76010938)
+1410 train 4.182112 (lr=5.5336e-04) (hash(x)=81260300)
+1420 train 4.298414 (lr=5.5205e-04) (hash(x)=83803629)
+1430 train 4.126324 (lr=5.5073e-04) (hash(x)=83840110)
+1440 train 4.318179 (lr=5.4938e-04) (hash(x)=97711831)
+1450 train 4.192608 (lr=5.4803e-04) (hash(x)=81110139)
+1460 train 4.119545 (lr=5.4665e-04) (hash(x)=90942839)
+1470 train 4.156369 (lr=5.4526e-04) (hash(x)=80721564)
+1480 train 4.248527 (lr=5.4385e-04) (hash(x)=77852059)
+1490 train 4.143416 (lr=5.4243e-04) (hash(x)=80011365)
+1500 val loss 4.1795
+1500 val perplexity 65.3341
+1500 train 4.172869 (lr=5.4099e-04) (hash(x)=84496142)
+1510 train 4.105510 (lr=5.3954e-04) (hash(x)=70186729)
+1520 train 4.127757 (lr=5.3807e-04) (hash(x)=74854227)
+1530 train 4.034189 (lr=5.3658e-04) (hash(x)=77468161)
+1540 train 4.044551 (lr=5.3508e-04) (hash(x)=87853059)
+1550 train 3.984467 (lr=5.3357e-04) (hash(x)=71225436)
+1560 train 4.080449 (lr=5.3204e-04) (hash(x)=84072783)
+1570 train 4.125065 (lr=5.3049e-04) (hash(x)=72723098)
+1580 train 4.152003 (lr=5.2893e-04) (hash(x)=90409866)
+1590 train 4.218709 (lr=5.2736e-04) (hash(x)=87481378)
+1600 val loss 4.1386
+1600 val perplexity 62.7133
+1600 train 4.203616 (lr=5.2577e-04) (hash(x)=77643862)
+1610 train 4.081103 (lr=5.2417e-04) (hash(x)=88862575)
+1620 train 4.097677 (lr=5.2255e-04) (hash(x)=84612581)
+1630 train 4.139013 (lr=5.2092e-04) (hash(x)=87075989)
+1640 train 4.018781 (lr=5.1927e-04) (hash(x)=88277361)
+1650 train 4.136680 (lr=5.1761e-04) (hash(x)=78750236)
+1660 train 4.050566 (lr=5.1594e-04) (hash(x)=82604581)
+1670 train 3.894072 (lr=5.1425e-04) (hash(x)=68482265)
+1680 train 4.017204 (lr=5.1255e-04) (hash(x)=75088835)
+1690 train 3.964872 (lr=5.1084e-04) (hash(x)=66060989)
+1700 val loss 4.1213
+1700 val perplexity 61.6371
+1700 train 4.119876 (lr=5.0911e-04) (hash(x)=79986754)
+1710 train 4.094191 (lr=5.0737e-04) (hash(x)=83657930)
+1720 train 4.035969 (lr=5.0562e-04) (hash(x)=81754135)
+1730 train 4.135921 (lr=5.0385e-04) (hash(x)=78904427)
+1740 train 4.030789 (lr=5.0207e-04) (hash(x)=85920177)
+1750 train 4.089312 (lr=5.0028e-04) (hash(x)=86573211)
+1760 train 4.044215 (lr=4.9847e-04) (hash(x)=81737128)
+1770 train 4.033719 (lr=4.9665e-04) (hash(x)=77400968)
+1780 train 4.044498 (lr=4.9482e-04) (hash(x)=73545497)
+1790 train 4.305316 (lr=4.9298e-04) (hash(x)=71641943)
+1800 val loss 4.0700
+1800 val perplexity 58.5541
+1800 train 4.141685 (lr=4.9113e-04) (hash(x)=87819781)
+1810 train 3.998232 (lr=4.8926e-04) (hash(x)=86870770)
+1820 train 4.010203 (lr=4.8739e-04) (hash(x)=82522211)
+1830 train 4.099488 (lr=4.8550e-04) (hash(x)=61947437)
+1840 train 3.967906 (lr=4.8360e-04) (hash(x)=79865406)
+1850 train 3.930274 (lr=4.8169e-04) (hash(x)=79828721)
+1860 train 4.070061 (lr=4.7976e-04) (hash(x)=80869571)
+1870 train 4.061483 (lr=4.7783e-04) (hash(x)=73780971)
+1880 train 4.017057 (lr=4.7588e-04) (hash(x)=79249549)
+1890 train 4.075583 (lr=4.7393e-04) (hash(x)=81041904)
+1900 val loss 4.0442
+1900 val perplexity 57.0645
+1900 train 4.147625 (lr=4.7196e-04) (hash(x)=82456430)
+1910 train 3.900007 (lr=4.6999e-04) (hash(x)=82222135)
+1920 train 3.947019 (lr=4.6800e-04) (hash(x)=70033249)
+1930 train 3.933846 (lr=4.6600e-04) (hash(x)=72887360)
+1940 train 3.982329 (lr=4.6400e-04) (hash(x)=83251100)
+1950 train 4.018911 (lr=4.6198e-04) (hash(x)=79660266)
+1960 train 3.902264 (lr=4.5995e-04) (hash(x)=78919068)
+1970 train 4.013284 (lr=4.5792e-04) (hash(x)=94690431)
+1980 train 3.827605 (lr=4.5587e-04) (hash(x)=88153756)
+1990 train 3.887743 (lr=4.5381e-04) (hash(x)=77195688)
+2000 val loss 4.0142
+2000 val perplexity 55.3801
+2000 train 3.970705 (lr=4.5175e-04) (hash(x)=81308591)
+2010 train 4.027360 (lr=4.4968e-04) (hash(x)=80205479)
+2020 train 3.944631 (lr=4.4760e-04) (hash(x)=81008704)
+2030 train 4.008580 (lr=4.4551e-04) (hash(x)=82914358)
+2040 train 4.018912 (lr=4.4341e-04) (hash(x)=88064399)
+2050 train 4.030991 (lr=4.4130e-04) (hash(x)=82889390)
+2060 train 4.097082 (lr=4.3918e-04) (hash(x)=80913554)
+2070 train 3.930981 (lr=4.3706e-04) (hash(x)=78121791)
+2080 train 4.068925 (lr=4.3493e-04) (hash(x)=75356657)
+2090 train 3.898297 (lr=4.3279e-04) (hash(x)=74048412)
+2100 val loss 3.9821
+2100 val perplexity 53.6320
+2100 train 3.815279 (lr=4.3064e-04) (hash(x)=68928225)
+2110 train 3.870517 (lr=4.2849e-04) (hash(x)=93734745)
+2120 train 3.870013 (lr=4.2633e-04) (hash(x)=72878164)
+2130 train 3.996075 (lr=4.2416e-04) (hash(x)=71508226)
+2140 train 3.948195 (lr=4.2198e-04) (hash(x)=84168671)
+2150 train 3.876794 (lr=4.1980e-04) (hash(x)=74673639)
+2160 train 4.026609 (lr=4.1761e-04) (hash(x)=75470031)
+2170 train 3.988094 (lr=4.1542e-04) (hash(x)=74307890)
+2180 train 4.108595 (lr=4.1321e-04) (hash(x)=77214245)
+2190 train 3.883413 (lr=4.1101e-04) (hash(x)=88628359)
+2200 val loss 3.9608
+2200 val perplexity 52.4977
+2200 train 3.908175 (lr=4.0879e-04) (hash(x)=74779126)
+2210 train 3.948145 (lr=4.0657e-04) (hash(x)=79817976)
+2220 train 3.891584 (lr=4.0435e-04) (hash(x)=75258996)
+2230 train 3.873952 (lr=4.0212e-04) (hash(x)=88164047)
+2240 train 3.850070 (lr=3.9988e-04) (hash(x)=78580686)
+2250 train 3.780027 (lr=3.9764e-04) (hash(x)=76510617)
+2260 train 3.862032 (lr=3.9539e-04) (hash(x)=76043966)
+2270 train 3.841270 (lr=3.9314e-04) (hash(x)=73062098)
+2280 train 4.063267 (lr=3.9089e-04) (hash(x)=81885909)
+2290 train 4.000932 (lr=3.8863e-04) (hash(x)=81481741)
+2300 val loss 3.9344
+2300 val perplexity 51.1319
+2300 train 3.856041 (lr=3.8636e-04) (hash(x)=82104275)
+2310 train 3.901805 (lr=3.8409e-04) (hash(x)=82137309)
+2320 train 3.881460 (lr=3.8182e-04) (hash(x)=86741167)
+2330 train 3.893969 (lr=3.7955e-04) (hash(x)=84796263)
+2340 train 3.890165 (lr=3.7727e-04) (hash(x)=169966529)
+2350 train 3.756200 (lr=3.7498e-04) (hash(x)=80006061)
+2360 train 3.793148 (lr=3.7270e-04) (hash(x)=71090999)
+2370 train 3.925208 (lr=3.7041e-04) (hash(x)=86641850)
+2380 train 3.824032 (lr=3.6811e-04) (hash(x)=84573253)
+2390 train 3.866942 (lr=3.6582e-04) (hash(x)=74483764)
+2400 val loss 3.9214
+2400 val perplexity 50.4720
+2400 train 3.840872 (lr=3.6352e-04) (hash(x)=78327659)
+2410 train 3.903591 (lr=3.6122e-04) (hash(x)=77273627)
+2420 train 3.769855 (lr=3.5891e-04) (hash(x)=76938049)
+2430 train 3.768020 (lr=3.5661e-04) (hash(x)=98449442)
+2440 train 3.634655 (lr=3.5430e-04) (hash(x)=78157797)
+2450 train 3.667456 (lr=3.5199e-04) (hash(x)=80637582)
+2460 train 3.648945 (lr=3.4968e-04) (hash(x)=80023854)
+2470 train 3.722162 (lr=3.4737e-04) (hash(x)=86678884)
+2480 train 3.937719 (lr=3.4506e-04) (hash(x)=84086469)
+2490 train 3.944941 (lr=3.4274e-04) (hash(x)=83131141)
+2500 val loss 3.8947
+2500 val perplexity 49.1391
+2500 train 3.866644 (lr=3.4043e-04) (hash(x)=82583497)
+2510 train 3.936357 (lr=3.3811e-04) (hash(x)=79164326)
+2520 train 3.844373 (lr=3.3579e-04) (hash(x)=76456503)
+2530 train 3.904964 (lr=3.3348e-04) (hash(x)=82001228)
+2540 train 3.934854 (lr=3.3116e-04) (hash(x)=75615595)
+2550 train 3.886881 (lr=3.2884e-04) (hash(x)=78397869)
+2560 train 3.892297 (lr=3.2652e-04) (hash(x)=75547032)
+2570 train 3.796817 (lr=3.2421e-04) (hash(x)=89201025)
+2580 train 3.783798 (lr=3.2189e-04) (hash(x)=89856704)
+2590 train 3.802163 (lr=3.1957e-04) (hash(x)=82175682)
+2600 val loss 3.8741
+2600 val perplexity 48.1387
+2600 train 3.816540 (lr=3.1726e-04) (hash(x)=78312826)
+2610 train 3.791480 (lr=3.1494e-04) (hash(x)=77066588)
+2620 train 3.626380 (lr=3.1263e-04) (hash(x)=78666061)
+2630 train 3.630448 (lr=3.1032e-04) (hash(x)=93762143)
+2640 train 3.660300 (lr=3.0801e-04) (hash(x)=83191587)
+2650 train 3.567744 (lr=3.0570e-04) (hash(x)=87169585)
+2660 train 3.774030 (lr=3.0339e-04) (hash(x)=86426388)
+2670 train 3.903484 (lr=3.0109e-04) (hash(x)=76692638)
+2680 train 3.824401 (lr=2.9878e-04) (hash(x)=77446063)
+2690 train 3.832089 (lr=2.9648e-04) (hash(x)=79809050)
+2700 val loss 3.8660
+2700 val perplexity 47.7526
+2700 train 3.756493 (lr=2.9418e-04) (hash(x)=83116823)
+2710 train 3.873871 (lr=2.9189e-04) (hash(x)=75622148)
+2720 train 3.804423 (lr=2.8959e-04) (hash(x)=80690512)
+2730 train 3.880655 (lr=2.8730e-04) (hash(x)=78009984)
+2740 train 3.828596 (lr=2.8502e-04) (hash(x)=74205488)
+2750 train 3.837606 (lr=2.8273e-04) (hash(x)=91013332)
+2760 train 3.753824 (lr=2.8045e-04) (hash(x)=76954961)
+2770 train 3.763358 (lr=2.7818e-04) (hash(x)=79390317)
+2780 train 3.636682 (lr=2.7591e-04) (hash(x)=70168783)
+2790 train 3.815640 (lr=2.7364e-04) (hash(x)=76028417)
+2800 val loss 3.8460
+2800 val perplexity 46.8056
+2800 train 3.636314 (lr=2.7137e-04) (hash(x)=77656050)
+2810 train 3.669113 (lr=2.6911e-04) (hash(x)=94248216)
+2820 train 3.734297 (lr=2.6686e-04) (hash(x)=78305078)
+2830 train 3.641318 (lr=2.6461e-04) (hash(x)=79948848)
+2840 train 3.502400 (lr=2.6236e-04) (hash(x)=85341024)
+2850 train 3.807054 (lr=2.6012e-04) (hash(x)=78735170)
+2860 train 4.066610 (lr=2.5788e-04) (hash(x)=71616419)
+2870 train 3.818464 (lr=2.5565e-04) (hash(x)=78656517)
+2880 train 3.846994 (lr=2.5343e-04) (hash(x)=80073987)
+2890 train 3.778906 (lr=2.5121e-04) (hash(x)=76894809)
+2900 val loss 3.8191
+2900 val perplexity 45.5642
+2900 train 3.758083 (lr=2.4899e-04) (hash(x)=80499838)
+2910 train 3.813251 (lr=2.4679e-04) (hash(x)=72673354)
+2920 train 3.797680 (lr=2.4458e-04) (hash(x)=84265768)
+2930 train 3.818400 (lr=2.4239e-04) (hash(x)=79612060)
+2940 train 3.672692 (lr=2.4020e-04) (hash(x)=74970087)
+2950 train 3.834738 (lr=2.3802e-04) (hash(x)=84166818)
+2960 train 3.818449 (lr=2.3584e-04) (hash(x)=89410221)
+2970 train 3.755245 (lr=2.3367e-04) (hash(x)=75672566)
+2980 train 3.614504 (lr=2.3151e-04) (hash(x)=81760314)
+2990 train 3.583991 (lr=2.2936e-04) (hash(x)=80605200)
+3000 val loss 3.8153
+3000 val perplexity 45.3921
+3000 train 3.574738 (lr=2.2721e-04) (hash(x)=83804735)
+3010 train 3.510992 (lr=2.2507e-04) (hash(x)=77015303)
+3020 train 3.582888 (lr=2.2294e-04) (hash(x)=81464523)
+3030 train 3.754169 (lr=2.2082e-04) (hash(x)=80168230)
+3040 train 3.703672 (lr=2.1870e-04) (hash(x)=84817006)
+3050 train 3.823137 (lr=2.1659e-04) (hash(x)=71601811)
+3060 train 3.747623 (lr=2.1449e-04) (hash(x)=85499733)
+3070 train 3.812778 (lr=2.1240e-04) (hash(x)=78670408)
+3080 train 3.700468 (lr=2.1032e-04) (hash(x)=77120468)
+3090 train 3.802423 (lr=2.0825e-04) (hash(x)=77927426)
+3100 val loss 3.7943
+3100 val perplexity 44.4482
+3100 train 3.821735 (lr=2.0619e-04) (hash(x)=83998606)
+3110 train 3.730011 (lr=2.0413e-04) (hash(x)=77990218)
+3120 train 3.635248 (lr=2.0208e-04) (hash(x)=81623970)
+3130 train 3.609702 (lr=2.0005e-04) (hash(x)=75014781)
+3140 train 3.663582 (lr=1.9802e-04) (hash(x)=72591250)
+3150 train 3.681089 (lr=1.9600e-04) (hash(x)=81421847)
+3160 train 3.718182 (lr=1.9400e-04) (hash(x)=75112631)
+3170 train 3.805322 (lr=1.9200e-04) (hash(x)=87518033)
+3180 train 3.678021 (lr=1.9001e-04) (hash(x)=78648348)
+3190 train 3.778646 (lr=1.8804e-04) (hash(x)=78270029)
+3200 val loss 3.7759
+3200 val perplexity 43.6360
+3200 train 3.839443 (lr=1.8607e-04) (hash(x)=83380714)
+3210 train 4.036620 (lr=1.8412e-04) (hash(x)=67804991)
+3220 train 3.760511 (lr=1.8217e-04) (hash(x)=76968804)
+3230 train 3.694715 (lr=1.8024e-04) (hash(x)=75798670)
+3240 train 3.773014 (lr=1.7831e-04) (hash(x)=83192811)
+3250 train 3.702786 (lr=1.7640e-04) (hash(x)=81659789)
+3260 train 3.760989 (lr=1.7450e-04) (hash(x)=80622502)
+3270 train 3.720280 (lr=1.7261e-04) (hash(x)=85436511)
+3280 train 3.666389 (lr=1.7074e-04) (hash(x)=80589180)
+3290 train 3.809917 (lr=1.6887e-04) (hash(x)=85747193)
+3300 val loss 3.7718
+3300 val perplexity 43.4601
+3300 train 3.673318 (lr=1.6702e-04) (hash(x)=74635692)
+3310 train 3.657753 (lr=1.6518e-04) (hash(x)=82045455)
+3320 train 3.529346 (lr=1.6335e-04) (hash(x)=73249173)
+3330 train 3.496098 (lr=1.6153e-04) (hash(x)=71492338)
+3340 train 3.820335 (lr=1.5972e-04) (hash(x)=76374071)
+3350 train 3.787876 (lr=1.5793e-04) (hash(x)=80474064)
+3360 train 3.692667 (lr=1.5615e-04) (hash(x)=84390892)
+3370 train 3.741816 (lr=1.5438e-04) (hash(x)=83399949)
+3380 train 3.797963 (lr=1.5263e-04) (hash(x)=77648059)
+3390 train 3.770819 (lr=1.5089e-04) (hash(x)=80152701)
+3400 val loss 3.7511
+3400 val perplexity 42.5687
+3400 train 3.819565 (lr=1.4916e-04) (hash(x)=81952545)
+3410 train 3.776698 (lr=1.4745e-04) (hash(x)=80908993)
+3420 train 3.828337 (lr=1.4575e-04) (hash(x)=79610037)
+3430 train 3.694364 (lr=1.4406e-04) (hash(x)=87624382)
+3440 train 3.753950 (lr=1.4239e-04) (hash(x)=82336381)
+3450 train 3.728707 (lr=1.4073e-04) (hash(x)=87687835)
+3460 train 3.725184 (lr=1.3908e-04) (hash(x)=76376135)
+3470 train 3.648288 (lr=1.3745e-04) (hash(x)=79331391)
+3480 train 3.550665 (lr=1.3583e-04) (hash(x)=107002681)
+3490 train 3.633443 (lr=1.3423e-04) (hash(x)=77465514)
+3500 val loss 3.7473
+3500 val perplexity 42.4046
+3500 train 3.587634 (lr=1.3264e-04) (hash(x)=88237229)
+3510 train 3.653171 (lr=1.3107e-04) (hash(x)=81011739)
+3520 train 3.721961 (lr=1.2951e-04) (hash(x)=64643427)
+3530 train 3.986450 (lr=1.2796e-04) (hash(x)=78029539)
+3540 train 3.812087 (lr=1.2643e-04) (hash(x)=83188968)
+3550 train 3.824206 (lr=1.2492e-04) (hash(x)=86104185)
+3560 train 3.750349 (lr=1.2342e-04) (hash(x)=81993629)
+3570 train 3.731870 (lr=1.2193e-04) (hash(x)=72719368)
+3580 train 3.730203 (lr=1.2046e-04) (hash(x)=90091487)
+3590 train 3.624060 (lr=1.1901e-04) (hash(x)=77393152)
+3600 val loss 3.7307
+3600 val perplexity 41.7097
+3600 train 3.743039 (lr=1.1757e-04) (hash(x)=73103504)
+3610 train 3.743801 (lr=1.1615e-04) (hash(x)=84429400)
+3620 train 3.716208 (lr=1.1474e-04) (hash(x)=77021795)
+3630 train 3.673646 (lr=1.1335e-04) (hash(x)=92830605)
+3640 train 3.877163 (lr=1.1197e-04) (hash(x)=78313175)
+3650 train 3.679147 (lr=1.1062e-04) (hash(x)=85395549)
+3660 train 3.466781 (lr=1.0927e-04) (hash(x)=78114459)
+3670 train 3.592271 (lr=1.0795e-04) (hash(x)=74968316)
+3680 train 3.520341 (lr=1.0664e-04) (hash(x)=73358737)
+3690 train 3.606351 (lr=1.0534e-04) (hash(x)=76399442)
+3700 val loss 3.7300
+3700 val perplexity 41.6774
+3700 train 3.623710 (lr=1.0407e-04) (hash(x)=74039273)
+3710 train 3.787826 (lr=1.0280e-04) (hash(x)=79299680)
+3720 train 3.695810 (lr=1.0156e-04) (hash(x)=94701498)
+3730 train 3.784164 (lr=1.0033e-04) (hash(x)=75352071)
+3740 train 3.664125 (lr=9.9124e-05) (hash(x)=77909487)
+3750 train 3.785180 (lr=9.7931e-05) (hash(x)=73986730)
+3760 train 3.729086 (lr=9.6755e-05) (hash(x)=79325763)
+3770 train 3.784531 (lr=9.5596e-05) (hash(x)=72457818)
+3780 train 3.905213 (lr=9.4454e-05) (hash(x)=71775590)
+3790 train 3.799231 (lr=9.3330e-05) (hash(x)=82638943)
+3800 val loss 3.7149
+3800 val perplexity 41.0546
+3800 train 3.777917 (lr=9.2224e-05) (hash(x)=79965893)
+3810 train 3.710453 (lr=9.1134e-05) (hash(x)=72598235)
+3820 train 3.692875 (lr=9.0063e-05) (hash(x)=83113889)
+3830 train 3.744063 (lr=8.9009e-05) (hash(x)=74434590)
+3840 train 3.669098 (lr=8.7973e-05) (hash(x)=82860348)
+3850 train 3.611331 (lr=8.6954e-05) (hash(x)=78067565)
+3860 train 3.664964 (lr=8.5954e-05) (hash(x)=82592498)
+3870 train 3.603543 (lr=8.4971e-05) (hash(x)=81820733)
+3880 train 3.620313 (lr=8.4007e-05) (hash(x)=87709040)
+3890 train 3.652857 (lr=8.3061e-05) (hash(x)=70379093)
+3900 val loss 3.7094
+3900 val perplexity 40.8294
+3900 train 3.774960 (lr=8.2133e-05) (hash(x)=76597431)
+3910 train 3.660633 (lr=8.1223e-05) (hash(x)=90490716)
+3920 train 3.770563 (lr=8.0331e-05) (hash(x)=81970659)
+3930 train 3.752832 (lr=7.9458e-05) (hash(x)=81496334)
+3940 train 3.742235 (lr=7.8604e-05) (hash(x)=75717605)
+3950 train 3.679429 (lr=7.7768e-05) (hash(x)=82749357)
+3960 train 3.712869 (lr=7.6950e-05) (hash(x)=89355157)
+3970 train 3.665347 (lr=7.6151e-05) (hash(x)=78980403)
+3980 train 3.639618 (lr=7.5371e-05) (hash(x)=76627217)
+3990 train 3.642324 (lr=7.4610e-05) (hash(x)=72412879)
+4000 val loss 3.7042
+4000 val perplexity 40.6167
+4000 train 3.657221 (lr=7.3867e-05) (hash(x)=83018142)
+4010 train 3.682138 (lr=7.3143e-05) (hash(x)=81272436)
+4020 train 3.605317 (lr=7.2438e-05) (hash(x)=85497482)
+4030 train 3.661194 (lr=7.1753e-05) (hash(x)=85241734)
+4040 train 3.631224 (lr=7.1086e-05) (hash(x)=77925307)
+4050 train 3.632481 (lr=7.0438e-05) (hash(x)=84826179)
+4060 train 3.689243 (lr=6.9809e-05) (hash(x)=83606764)
+4070 train 3.604437 (lr=6.9200e-05) (hash(x)=80567590)
+4080 train 3.666875 (lr=6.8610e-05) (hash(x)=76860998)
+4090 train 3.706736 (lr=6.8039e-05) (hash(x)=74902328)
+4100 val loss 3.6941
+4100 val perplexity 40.2095
+4100 train 3.743418 (lr=6.7487e-05) (hash(x)=82832041)
+4110 train 3.621112 (lr=6.6955e-05) (hash(x)=79143262)
+4120 train 3.654513 (lr=6.6442e-05) (hash(x)=77038149)
+4130 train 3.825789 (lr=6.5948e-05) (hash(x)=86339074)
+4140 train 3.908206 (lr=6.5474e-05) (hash(x)=76686216)
+4150 train 3.662502 (lr=6.5020e-05) (hash(x)=70522682)
+4160 train 3.632463 (lr=6.4585e-05) (hash(x)=90958555)
+4170 train 3.642974 (lr=6.4169e-05) (hash(x)=91463532)
+4180 train 3.642266 (lr=6.3773e-05) (hash(x)=81959329)
+4190 train 3.605531 (lr=6.3397e-05) (hash(x)=83146752)
+4200 val loss 3.6929
+4200 val perplexity 40.1599
+4200 train 3.578906 (lr=6.3040e-05) (hash(x)=78361715)
+4210 train 3.730519 (lr=6.2703e-05) (hash(x)=87364889)
+4220 train 3.591410 (lr=6.2386e-05) (hash(x)=70465156)
+4230 train 3.747005 (lr=6.2089e-05) (hash(x)=84524081)
+4240 train 3.688734 (lr=6.1811e-05) (hash(x)=77824868)
+4250 train 3.710096 (lr=6.1553e-05) (hash(x)=81710711)
+4260 train 3.693176 (lr=6.1314e-05) (hash(x)=76362728)
+4270 train 3.691400 (lr=6.1096e-05) (hash(x)=83115208)
+4280 train 3.602577 (lr=6.0897e-05) (hash(x)=87218314)
+4290 train 3.762576 (lr=6.0718e-05) (hash(x)=74582673)
+4300 val loss 3.6892
+4300 val perplexity 40.0114
+4300 train 3.626175 (lr=6.0559e-05) (hash(x)=77379615)
+4310 train 3.614428 (lr=6.0420e-05) (hash(x)=78669579)
+4320 train 3.718278 (lr=6.0301e-05) (hash(x)=83066608)
+4330 train 3.692140 (lr=6.0201e-05) (hash(x)=83037340)
+4340 train 3.718928 (lr=6.0122e-05) (hash(x)=82849771)
+4350 train 3.665056 (lr=6.0062e-05) (hash(x)=76693985)
+4360 train 3.758386 (lr=6.0022e-05) (hash(x)=77745394)
+4370 train 3.716155 (lr=6.0002e-05) (hash(x)=79954388)
+4374 val loss 3.6838
+4374 val perplexity 39.7963

12_head_baseline_lr_6e-4/model_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bbce8146b1bbcc16be5c0c94bd792bd0b55d572bb94be3f91c51b6fada48fee
+size 498607490

12_head_baseline_lr_6e-4/optimizer_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a1665da605bfe90dc8713d8d7db0fdb6aa319f9def626c3891c70a7f09951b6
+size 990934406