Instructions to use mistakeknot/wi2c-llama3-1 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use mistakeknot/wi2c-llama3-1 with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("togethercomputer/Meta-Llama-3.1-70B-Instruct-Reference__TOG__FT") model = PeftModel.from_pretrained(base_model, "mistakeknot/wi2c-llama3-1") - Notebooks
- Google Colab
- Kaggle
Download trainer_state.json from mistakeknot/wi2c-llama3-1: direct link, hf CLI and curl.
- Browser
- Download file 14.9 kB
-
https://huggingface.co/mistakeknot/wi2c-llama3-1/resolve/main/trainer_state.json
- Command line
-
hf download hf://mistakeknot/wi2c-llama3-1/trainer_state.json
-
curl -L -o trainer_state.json https://huggingface.co/mistakeknot/wi2c-llama3-1/resolve/main/trainer_state.json
14.9 kB
| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 3.0, | |
| "eval_steps": 0, | |
| "global_step": 87, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.034482758620689655, | |
| "grad_norm": 7.4375, | |
| "learning_rate": 0.0, | |
| "loss": 5.0222, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06896551724137931, | |
| "grad_norm": 7.9375, | |
| "learning_rate": 6.666666666666667e-06, | |
| "loss": 5.1147, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.10344827586206896, | |
| "grad_norm": 6.09375, | |
| "learning_rate": 1.3333333333333333e-05, | |
| "loss": 4.6125, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.13793103448275862, | |
| "grad_norm": 5.375, | |
| "learning_rate": 2e-05, | |
| "loss": 4.611, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.1724137931034483, | |
| "grad_norm": 2.5, | |
| "learning_rate": 1.9993007047883988e-05, | |
| "loss": 4.4752, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.20689655172413793, | |
| "grad_norm": 1.5703125, | |
| "learning_rate": 1.9972037971811802e-05, | |
| "loss": 4.1858, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.2413793103448276, | |
| "grad_norm": 1.2578125, | |
| "learning_rate": 1.9937122098932428e-05, | |
| "loss": 4.025, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.27586206896551724, | |
| "grad_norm": 1.1796875, | |
| "learning_rate": 1.9888308262251286e-05, | |
| "loss": 4.2741, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.3103448275862069, | |
| "grad_norm": 0.8046875, | |
| "learning_rate": 1.9825664732332886e-05, | |
| "loss": 3.9291, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.3448275862068966, | |
| "grad_norm": 0.66015625, | |
| "learning_rate": 1.9749279121818235e-05, | |
| "loss": 3.8237, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.3793103448275862, | |
| "grad_norm": 0.7890625, | |
| "learning_rate": 1.9659258262890683e-05, | |
| "loss": 3.786, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.41379310344827586, | |
| "grad_norm": 1.4609375, | |
| "learning_rate": 1.955572805786141e-05, | |
| "loss": 3.9016, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.4482758620689655, | |
| "grad_norm": 1.3984375, | |
| "learning_rate": 1.9438833303083677e-05, | |
| "loss": 4.0912, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.4827586206896552, | |
| "grad_norm": 0.53515625, | |
| "learning_rate": 1.9308737486442045e-05, | |
| "loss": 3.8579, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.5172413793103449, | |
| "grad_norm": 0.53125, | |
| "learning_rate": 1.9165622558699763e-05, | |
| "loss": 3.9025, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.5517241379310345, | |
| "grad_norm": 0.51953125, | |
| "learning_rate": 1.900968867902419e-05, | |
| "loss": 3.7475, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.5862068965517241, | |
| "grad_norm": 0.458984375, | |
| "learning_rate": 1.8841153935046098e-05, | |
| "loss": 3.8074, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.6206896551724138, | |
| "grad_norm": 0.3984375, | |
| "learning_rate": 1.866025403784439e-05, | |
| "loss": 3.8109, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.6551724137931034, | |
| "grad_norm": 0.8671875, | |
| "learning_rate": 1.8467241992282842e-05, | |
| "loss": 4.0508, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.6896551724137931, | |
| "grad_norm": 0.404296875, | |
| "learning_rate": 1.826238774315995e-05, | |
| "loss": 3.9091, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.7241379310344828, | |
| "grad_norm": 0.5859375, | |
| "learning_rate": 1.8045977797666685e-05, | |
| "loss": 3.63, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.7586206896551724, | |
| "grad_norm": 0.447265625, | |
| "learning_rate": 1.78183148246803e-05, | |
| "loss": 3.7721, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.7931034482758621, | |
| "grad_norm": 0.37890625, | |
| "learning_rate": 1.757971723145453e-05, | |
| "loss": 3.7469, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.8275862068965517, | |
| "grad_norm": 0.84765625, | |
| "learning_rate": 1.7330518718298263e-05, | |
| "loss": 3.9459, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.8620689655172413, | |
| "grad_norm": 0.345703125, | |
| "learning_rate": 1.7071067811865477e-05, | |
| "loss": 3.6702, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.896551724137931, | |
| "grad_norm": 0.3359375, | |
| "learning_rate": 1.6801727377709195e-05, | |
| "loss": 3.6578, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.9310344827586207, | |
| "grad_norm": 0.7578125, | |
| "learning_rate": 1.6522874112781213e-05, | |
| "loss": 4.0356, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.9655172413793104, | |
| "grad_norm": 0.326171875, | |
| "learning_rate": 1.6234898018587336e-05, | |
| "loss": 3.6688, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 0.486328125, | |
| "learning_rate": 1.5938201855735017e-05, | |
| "loss": 3.5728, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 1.0344827586206897, | |
| "grad_norm": 0.283203125, | |
| "learning_rate": 1.563320058063622e-05, | |
| "loss": 3.7259, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 1.0689655172413792, | |
| "grad_norm": 0.41796875, | |
| "learning_rate": 1.5320320765153367e-05, | |
| "loss": 3.6079, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 1.103448275862069, | |
| "grad_norm": 0.3203125, | |
| "learning_rate": 1.5000000000000002e-05, | |
| "loss": 3.7737, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 1.1379310344827587, | |
| "grad_norm": 0.271484375, | |
| "learning_rate": 1.4672686282730622e-05, | |
| "loss": 3.6476, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 1.1724137931034484, | |
| "grad_norm": 0.3984375, | |
| "learning_rate": 1.4338837391175582e-05, | |
| "loss": 3.7189, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 1.206896551724138, | |
| "grad_norm": 0.25390625, | |
| "learning_rate": 1.3998920243197408e-05, | |
| "loss": 3.5989, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 1.2413793103448276, | |
| "grad_norm": 0.259765625, | |
| "learning_rate": 1.3653410243663953e-05, | |
| "loss": 3.6077, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 1.2758620689655173, | |
| "grad_norm": 0.255859375, | |
| "learning_rate": 1.3302790619551673e-05, | |
| "loss": 3.6065, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 1.3103448275862069, | |
| "grad_norm": 0.3046875, | |
| "learning_rate": 1.2947551744109044e-05, | |
| "loss": 3.6663, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 1.3448275862068966, | |
| "grad_norm": 0.7734375, | |
| "learning_rate": 1.2588190451025209e-05, | |
| "loss": 3.7774, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 1.3793103448275863, | |
| "grad_norm": 0.271484375, | |
| "learning_rate": 1.2225209339563144e-05, | |
| "loss": 3.6278, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 1.4137931034482758, | |
| "grad_norm": 0.341796875, | |
| "learning_rate": 1.1859116071629148e-05, | |
| "loss": 3.7917, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 1.4482758620689655, | |
| "grad_norm": 0.29296875, | |
| "learning_rate": 1.1490422661761744e-05, | |
| "loss": 3.747, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 1.4827586206896552, | |
| "grad_norm": 0.58203125, | |
| "learning_rate": 1.1119644761033079e-05, | |
| "loss": 3.4494, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 1.5172413793103448, | |
| "grad_norm": 0.345703125, | |
| "learning_rate": 1.0747300935864245e-05, | |
| "loss": 3.6524, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 1.5517241379310345, | |
| "grad_norm": 0.4296875, | |
| "learning_rate": 1.037391194276326e-05, | |
| "loss": 3.5445, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 1.5862068965517242, | |
| "grad_norm": 0.419921875, | |
| "learning_rate": 1e-05, | |
| "loss": 3.4137, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 1.6206896551724137, | |
| "grad_norm": 0.330078125, | |
| "learning_rate": 9.626088057236745e-06, | |
| "loss": 3.4652, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 1.6551724137931034, | |
| "grad_norm": 0.2578125, | |
| "learning_rate": 9.252699064135759e-06, | |
| "loss": 3.4672, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 1.6896551724137931, | |
| "grad_norm": 0.267578125, | |
| "learning_rate": 8.880355238966923e-06, | |
| "loss": 3.4507, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 1.7241379310344827, | |
| "grad_norm": 0.306640625, | |
| "learning_rate": 8.509577338238255e-06, | |
| "loss": 3.4844, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.7586206896551724, | |
| "grad_norm": 0.66015625, | |
| "learning_rate": 8.140883928370855e-06, | |
| "loss": 3.6069, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 1.793103448275862, | |
| "grad_norm": 0.7265625, | |
| "learning_rate": 7.774790660436857e-06, | |
| "loss": 3.6394, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 1.8275862068965516, | |
| "grad_norm": 0.3515625, | |
| "learning_rate": 7.411809548974792e-06, | |
| "loss": 3.5789, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 1.8620689655172413, | |
| "grad_norm": 0.26953125, | |
| "learning_rate": 7.052448255890958e-06, | |
| "loss": 3.5094, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 1.896551724137931, | |
| "grad_norm": 0.765625, | |
| "learning_rate": 6.697209380448333e-06, | |
| "loss": 3.8513, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 1.9310344827586206, | |
| "grad_norm": 0.2490234375, | |
| "learning_rate": 6.34658975633605e-06, | |
| "loss": 3.5786, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 1.9655172413793105, | |
| "grad_norm": 0.279296875, | |
| "learning_rate": 6.001079756802592e-06, | |
| "loss": 3.5573, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "grad_norm": 0.298828125, | |
| "learning_rate": 5.66116260882442e-06, | |
| "loss": 3.5882, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 2.0344827586206895, | |
| "grad_norm": 0.390625, | |
| "learning_rate": 5.32731371726938e-06, | |
| "loss": 3.4628, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 2.0689655172413794, | |
| "grad_norm": 0.36328125, | |
| "learning_rate": 5.000000000000003e-06, | |
| "loss": 3.5358, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 2.103448275862069, | |
| "grad_norm": 0.423828125, | |
| "learning_rate": 4.679679234846636e-06, | |
| "loss": 3.4965, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 2.1379310344827585, | |
| "grad_norm": 0.302734375, | |
| "learning_rate": 4.3667994193637794e-06, | |
| "loss": 3.5483, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 2.1724137931034484, | |
| "grad_norm": 0.443359375, | |
| "learning_rate": 4.061798144264986e-06, | |
| "loss": 3.4162, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 2.206896551724138, | |
| "grad_norm": 0.345703125, | |
| "learning_rate": 3.7651019814126656e-06, | |
| "loss": 3.5583, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 2.2413793103448274, | |
| "grad_norm": 0.3125, | |
| "learning_rate": 3.4771258872187917e-06, | |
| "loss": 3.5236, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 2.2758620689655173, | |
| "grad_norm": 0.765625, | |
| "learning_rate": 3.1982726222908046e-06, | |
| "loss": 3.7135, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 2.310344827586207, | |
| "grad_norm": 0.376953125, | |
| "learning_rate": 2.9289321881345257e-06, | |
| "loss": 3.6926, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 2.344827586206897, | |
| "grad_norm": 0.263671875, | |
| "learning_rate": 2.669481281701739e-06, | |
| "loss": 3.5238, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 2.3793103448275863, | |
| "grad_norm": 0.287109375, | |
| "learning_rate": 2.420282768545469e-06, | |
| "loss": 3.4981, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 2.413793103448276, | |
| "grad_norm": 0.287109375, | |
| "learning_rate": 2.1816851753197023e-06, | |
| "loss": 3.4575, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 2.4482758620689653, | |
| "grad_norm": 0.91796875, | |
| "learning_rate": 1.9540222023333165e-06, | |
| "loss": 3.8734, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 2.4827586206896552, | |
| "grad_norm": 0.25390625, | |
| "learning_rate": 1.7376122568400533e-06, | |
| "loss": 3.4957, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 2.5172413793103448, | |
| "grad_norm": 0.26953125, | |
| "learning_rate": 1.5327580077171589e-06, | |
| "loss": 3.4607, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 2.5517241379310347, | |
| "grad_norm": 0.265625, | |
| "learning_rate": 1.339745962155613e-06, | |
| "loss": 3.4951, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 2.586206896551724, | |
| "grad_norm": 0.255859375, | |
| "learning_rate": 1.1588460649539036e-06, | |
| "loss": 3.6463, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 2.6206896551724137, | |
| "grad_norm": 0.419921875, | |
| "learning_rate": 9.903113209758098e-07, | |
| "loss": 3.7225, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 2.655172413793103, | |
| "grad_norm": 0.28125, | |
| "learning_rate": 8.343774413002382e-07, | |
| "loss": 3.5069, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 2.689655172413793, | |
| "grad_norm": 0.23828125, | |
| "learning_rate": 6.912625135579587e-07, | |
| "loss": 3.5515, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 2.7241379310344827, | |
| "grad_norm": 0.265625, | |
| "learning_rate": 5.611666969163243e-07, | |
| "loss": 3.5939, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 2.7586206896551726, | |
| "grad_norm": 0.326171875, | |
| "learning_rate": 4.4427194213859216e-07, | |
| "loss": 3.5626, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 2.793103448275862, | |
| "grad_norm": 0.2333984375, | |
| "learning_rate": 3.4074173710931804e-07, | |
| "loss": 3.5161, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 2.8275862068965516, | |
| "grad_norm": 0.2421875, | |
| "learning_rate": 2.507208781817638e-07, | |
| "loss": 3.5366, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 2.862068965517241, | |
| "grad_norm": 0.259765625, | |
| "learning_rate": 1.7433526766711727e-07, | |
| "loss": 3.5442, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 2.896551724137931, | |
| "grad_norm": 0.30859375, | |
| "learning_rate": 1.1169173774871478e-07, | |
| "loss": 3.6514, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 2.9310344827586206, | |
| "grad_norm": 0.275390625, | |
| "learning_rate": 6.287790106757396e-08, | |
| "loss": 3.4944, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 2.9655172413793105, | |
| "grad_norm": 0.341796875, | |
| "learning_rate": 2.796202818819871e-08, | |
| "loss": 3.4597, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "grad_norm": 0.255859375, | |
| "learning_rate": 6.992952116013918e-09, | |
| "loss": 3.5197, | |
| "step": 87 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 87, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 0, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 2.9935982612158874e+17, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |