add sharded checkpoint

.gitattributes

@@ -32,3 +32,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00003-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00004-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00005-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00006-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00007-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00008-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00001-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00002-of-00008.bin filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,56 @@
 ---
 license: apache-2.0
+tags:
+- generated_from_trainer
+datasets:
+- pszemraj/HC3-textgen-qa
+metrics:
+- accuracy
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# HC3-textgen-qa-pythia-6.9b-deduped-r1
+
+This model is a fine-tuned version of [EleutherAI/pythia-6.9b-deduped](https://huggingface.co/EleutherAI/pythia-6.9b-deduped) on the pszemraj/HC3-textgen-qa dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.2372
+- Accuracy: 0.6769
+
+## Model description
+
+Text generation model trained on the HC3 text data of human questions + chatGPT answers.
+
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+```yaml
+model-index:
+- name: pythia-6.9b-hc3-qa-assistant
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: pszemraj/HC3-textgen-qa
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.6768941789814655
+```
+
+
+## Training procedure
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|
+| 1.2598        | 0.99  | 79   | 1.3291          | 0.6496   |
+| 0.7446        | 1.99  | 158  | 1.2372          | 0.6769   |
+

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 1.99,
+    "eval_accuracy": 0.6768941789814655,
+    "eval_loss": 1.2372242212295532,
+    "eval_runtime": 70.9825,
+    "eval_samples": 255,
+    "eval_samples_per_second": 3.592,
+    "eval_steps_per_second": 1.803,
+    "perplexity": 3.4460347466198447,
+    "train_loss": 1.1195918215981013,
+    "train_runtime": 35849.6176,
+    "train_samples": 5097,
+    "train_samples_per_second": 0.284,
+    "train_steps_per_second": 0.004
+}

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "pythia-6.9b-hc3-qa-assistant",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.0.dev0",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50432
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 1.99,
+    "eval_accuracy": 0.6768941789814655,
+    "eval_loss": 1.2372242212295532,
+    "eval_runtime": 70.9825,
+    "eval_samples": 255,
+    "eval_samples_per_second": 3.592,
+    "eval_steps_per_second": 1.803,
+    "perplexity": 3.4460347466198447
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.27.0.dev0"
+}

pytorch_model-00001-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77ffe885e9ac70ef1a32205abfb4a423e4f6513c18f012ad2c75c8a8ec3667a3
+size 3796700725

pytorch_model-00002-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32643f73019269c0cace5bc724f07e65fdda4f8871eff213e9ab0803aa52ca1b
+size 3780092209

pytorch_model-00003-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e157695e796b416f642fd5d5a1d0e5f8ae4559eaeda4062c6d7477d0466058eb
+size 3981467953

pytorch_model-00004-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c09eb3d3698a996c50f9f47ba2e6bbaedbc94187ecc29b8539320a152d61c5a6
+size 3847217457

pytorch_model-00005-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b36caa765e45b55680c2dd1d3207759db3d51911645fd50a2a5632b936deecc
+size 3775879155

pytorch_model-00006-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f058fa4ae9ba70ccc2089f895237710d158c7f767e24336d2bd52fda6de3a07
+size 3780092273

pytorch_model-00007-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f267384c57ea6f44b91cbc6d7dc5746f563201a773a334bfde6f08b976c87fac
+size 3775862707

pytorch_model-00008-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe67a55e71986199c76ab519ec5bd759e009d8280958be08f90174fa2028226
+size 826278826

pytorch_model.bin.index.json

@@ -0,0 +1,491 @@
+{
+  "metadata": {
+    "total_size": 27563427968
+  },
+  "weight_map": {
+    "embed_out.weight": "pytorch_model-00008-of-00008.bin",
+    "gpt_neox.embed_in.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.final_layer_norm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.final_layer_norm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.0.attention.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.attention.dense.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.attention.dense.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.attention.masked_bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.attention.query_key_value.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.attention.query_key_value.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.input_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.dense.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.dense.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.masked_bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.query_key_value.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.query_key_value.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.input_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.10.attention.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.attention.dense.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.attention.dense.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.attention.masked_bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.attention.query_key_value.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.attention.query_key_value.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.input_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.input_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.post_attention_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.10.post_attention_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.dense.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.dense.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.masked_bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.query_key_value.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.query_key_value.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.input_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.input_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.post_attention_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.11.post_attention_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.dense.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.dense.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.masked_bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.query_key_value.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.query_key_value.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.input_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.input_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.post_attention_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.12.post_attention_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.attention.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.attention.dense.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.13.attention.dense.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.13.attention.masked_bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.attention.query_key_value.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.attention.query_key_value.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.input_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.input_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.13.post_attention_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.13.post_attention_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.14.attention.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.attention.dense.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.attention.dense.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.attention.masked_bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.attention.query_key_value.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.attention.query_key_value.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.input_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.input_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.post_attention_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.14.post_attention_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.dense.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.dense.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.masked_bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.query_key_value.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.query_key_value.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.input_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.input_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.post_attention_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.15.post_attention_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.dense.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.dense.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.masked_bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.query_key_value.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.query_key_value.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.input_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.input_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.post_attention_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.16.post_attention_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.dense.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.dense.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.masked_bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.query_key_value.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.query_key_value.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.input_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.input_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.post_attention_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.17.post_attention_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.attention.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.attention.dense.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.attention.dense.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.attention.masked_bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.attention.query_key_value.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.attention.query_key_value.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.input_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.input_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.18.post_attention_layernorm.bias": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.18.post_attention_layernorm.weight": "pytorch_model-00004-of-00008.bin",
+    "gpt_neox.layers.19.attention.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.attention.dense.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.attention.dense.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.attention.masked_bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.attention.query_key_value.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.attention.query_key_value.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.input_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.input_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.post_attention_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.19.post_attention_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.2.attention.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.attention.dense.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.attention.dense.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.attention.masked_bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.attention.query_key_value.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.attention.query_key_value.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.input_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.20.attention.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.attention.dense.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.attention.dense.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.attention.masked_bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.attention.query_key_value.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.attention.query_key_value.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.input_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.input_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.post_attention_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.20.post_attention_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.dense.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.dense.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.masked_bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.query_key_value.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.query_key_value.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.input_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.input_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.post_attention_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.21.post_attention_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.dense.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.dense.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.masked_bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.query_key_value.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.query_key_value.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.input_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.input_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.post_attention_layernorm.bias": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.22.post_attention_layernorm.weight": "pytorch_model-00005-of-00008.bin",
+    "gpt_neox.layers.23.attention.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.attention.dense.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.attention.dense.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.attention.masked_bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.attention.query_key_value.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.attention.query_key_value.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.input_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.input_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.post_attention_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.23.post_attention_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.dense.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.dense.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.masked_bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.query_key_value.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.query_key_value.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.input_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.input_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.post_attention_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.24.post_attention_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.dense.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.dense.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.masked_bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.query_key_value.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.query_key_value.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.input_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.input_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.post_attention_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.25.post_attention_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.dense.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.dense.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.masked_bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.query_key_value.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.query_key_value.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.input_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.input_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.post_attention_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.26.post_attention_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.dense.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.dense.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.masked_bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.query_key_value.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.query_key_value.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.input_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.input_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.27.post_attention_layernorm.bias": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.27.post_attention_layernorm.weight": "pytorch_model-00006-of-00008.bin",
+    "gpt_neox.layers.28.attention.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.attention.dense.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.attention.dense.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.attention.masked_bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.attention.query_key_value.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.attention.query_key_value.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.input_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.input_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.post_attention_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.28.post_attention_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.dense.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.dense.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.masked_bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.query_key_value.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.query_key_value.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.input_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.input_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.post_attention_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.29.post_attention_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.3.attention.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.attention.dense.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.attention.dense.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.attention.masked_bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.attention.query_key_value.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.attention.query_key_value.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.input_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.post_attention_layernorm.bias": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00008.bin",
+    "gpt_neox.layers.30.attention.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.attention.dense.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.attention.dense.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.attention.masked_bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.attention.query_key_value.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.attention.query_key_value.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.input_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.input_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.post_attention_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.30.post_attention_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.dense.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.dense.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.masked_bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.query_key_value.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.query_key_value.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.input_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.input_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.post_attention_layernorm.bias": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.31.post_attention_layernorm.weight": "pytorch_model-00007-of-00008.bin",
+    "gpt_neox.layers.4.attention.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.attention.dense.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.attention.dense.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.attention.masked_bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.attention.query_key_value.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.attention.query_key_value.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.input_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.input_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.post_attention_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.4.post_attention_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.dense.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.dense.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.masked_bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.query_key_value.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.query_key_value.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.input_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.input_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.post_attention_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.5.post_attention_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.dense.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.dense.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.masked_bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.query_key_value.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.query_key_value.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.input_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.post_attention_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.6.post_attention_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.dense.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.dense.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.masked_bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.query_key_value.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.query_key_value.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.input_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.post_attention_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.7.post_attention_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.dense.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.dense.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.masked_bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.query_key_value.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.query_key_value.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.input_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.8.post_attention_layernorm.bias": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00008.bin",
+    "gpt_neox.layers.9.attention.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.attention.dense.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.attention.dense.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.attention.masked_bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.attention.query_key_value.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.attention.query_key_value.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.input_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.input_layernorm.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.post_attention_layernorm.bias": "pytorch_model-00003-of-00008.bin",
+    "gpt_neox.layers.9.post_attention_layernorm.weight": "pytorch_model-00003-of-00008.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "name_or_path": "EleutherAI/pythia-6.9b-deduped",
+  "special_tokens_map_file": "/fsx/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/3523781c8df75f7741687a4284f6f70e1afa12f4/special_tokens_map.json",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.99,
+    "train_loss": 1.1195918215981013,
+    "train_runtime": 35849.6176,
+    "train_samples": 5097,
+    "train_samples_per_second": 0.284,
+    "train_steps_per_second": 0.004
+}

trainer_state.json

@@ -0,0 +1,517 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9905956112852663,
+  "global_step": 158,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03,
+      "learning_rate": 2.5e-05,
+      "loss": 1.5684,
+      "step": 2
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 5e-05,
+      "loss": 1.7603,
+      "step": 4
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 1.7578,
+      "step": 6
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0001,
+      "loss": 1.5327,
+      "step": 8
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 9.995614150494293e-05,
+      "loss": 1.5737,
+      "step": 10
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 9.982464296247522e-05,
+      "loss": 1.4917,
+      "step": 12
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 9.96057350657239e-05,
+      "loss": 1.4312,
+      "step": 14
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 9.929980185352526e-05,
+      "loss": 1.4136,
+      "step": 16
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 9.890738003669029e-05,
+      "loss": 1.3789,
+      "step": 18
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 9.842915805643155e-05,
+      "loss": 1.5537,
+      "step": 20
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 9.786597487660337e-05,
+      "loss": 1.6504,
+      "step": 22
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 9.721881851187406e-05,
+      "loss": 1.4834,
+      "step": 24
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 9.648882429441257e-05,
+      "loss": 1.4912,
+      "step": 26
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 9.567727288213005e-05,
+      "loss": 1.4409,
+      "step": 28
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 9.478558801197065e-05,
+      "loss": 1.3872,
+      "step": 30
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 9.381533400219318e-05,
+      "loss": 1.4736,
+      "step": 32
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 9.276821300802534e-05,
+      "loss": 1.4048,
+      "step": 34
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 9.164606203550497e-05,
+      "loss": 1.4502,
+      "step": 36
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 9.045084971874738e-05,
+      "loss": 1.4141,
+      "step": 38
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 8.9184672866292e-05,
+      "loss": 1.3911,
+      "step": 40
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 8.784975278258783e-05,
+      "loss": 1.3379,
+      "step": 42
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.644843137107059e-05,
+      "loss": 1.3379,
+      "step": 44
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.498316702566828e-05,
+      "loss": 1.3242,
+      "step": 46
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 8.345653031794292e-05,
+      "loss": 1.3091,
+      "step": 48
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 8.18711994874345e-05,
+      "loss": 1.312,
+      "step": 50
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 8.022995574311876e-05,
+      "loss": 1.3018,
+      "step": 52
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 7.85356783842216e-05,
+      "loss": 1.3115,
+      "step": 54
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 7.679133974894983e-05,
+      "loss": 1.3188,
+      "step": 56
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 1.314,
+      "step": 58
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 7.316480175599309e-05,
+      "loss": 1.3062,
+      "step": 60
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.128896457825364e-05,
+      "loss": 1.3076,
+      "step": 62
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.937577932260515e-05,
+      "loss": 1.2847,
+      "step": 64
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 6.742860236609077e-05,
+      "loss": 1.2837,
+      "step": 66
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 1.2798,
+      "step": 68
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.344599103076329e-05,
+      "loss": 1.3247,
+      "step": 70
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 6.141754350553279e-05,
+      "loss": 1.2842,
+      "step": 72
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 5.9369065729286245e-05,
+      "loss": 1.2969,
+      "step": 74
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.730415142812059e-05,
+      "loss": 1.2866,
+      "step": 76
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.522642316338268e-05,
+      "loss": 1.2598,
+      "step": 78
+    },
+    {
+      "epoch": 0.99,
+      "eval_accuracy": 0.6496387020106186,
+      "eval_loss": 1.329074740409851,
+      "eval_runtime": 71.5884,
+      "eval_samples_per_second": 3.562,
+      "eval_steps_per_second": 1.788,
+      "step": 79
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 5.313952597646568e-05,
+      "loss": 1.6475,
+      "step": 80
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 5.104712099416785e-05,
+      "loss": 0.9348,
+      "step": 82
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 4.895287900583216e-05,
+      "loss": 0.8953,
+      "step": 84
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 4.6860474023534335e-05,
+      "loss": 0.8994,
+      "step": 86
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 4.477357683661734e-05,
+      "loss": 0.8591,
+      "step": 88
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 4.269584857187943e-05,
+      "loss": 0.8354,
+      "step": 90
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 4.063093427071376e-05,
+      "loss": 0.863,
+      "step": 92
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 3.858245649446721e-05,
+      "loss": 0.8845,
+      "step": 94
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 3.655400896923672e-05,
+      "loss": 0.8325,
+      "step": 96
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.8989,
+      "step": 98
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 3.257139763390925e-05,
+      "loss": 0.8782,
+      "step": 100
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 3.062422067739485e-05,
+      "loss": 0.8052,
+      "step": 102
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 2.8711035421746367e-05,
+      "loss": 0.8518,
+      "step": 104
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 2.6835198244006927e-05,
+      "loss": 0.8516,
+      "step": 106
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 2.500000000000001e-05,
+      "loss": 0.7979,
+      "step": 108
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 2.3208660251050158e-05,
+      "loss": 0.8108,
+      "step": 110
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 2.1464321615778422e-05,
+      "loss": 0.8547,
+      "step": 112
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 1.977004425688126e-05,
+      "loss": 0.8684,
+      "step": 114
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 1.8128800512565513e-05,
+      "loss": 0.8428,
+      "step": 116
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 1.6543469682057106e-05,
+      "loss": 0.7878,
+      "step": 118
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 1.5016832974331724e-05,
+      "loss": 0.822,
+      "step": 120
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 1.3551568628929434e-05,
+      "loss": 0.79,
+      "step": 122
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 1.2150247217412186e-05,
+      "loss": 0.7979,
+      "step": 124
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 1.0815327133708015e-05,
+      "loss": 0.8081,
+      "step": 126
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.7751,
+      "step": 128
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 8.353937964495029e-06,
+      "loss": 0.7871,
+      "step": 130
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 7.2317869919746705e-06,
+      "loss": 0.7722,
+      "step": 132
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 6.184665997806832e-06,
+      "loss": 0.8198,
+      "step": 134
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 5.214411988029355e-06,
+      "loss": 0.7859,
+      "step": 136
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 4.322727117869951e-06,
+      "loss": 0.7458,
+      "step": 138
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 3.511175705587433e-06,
+      "loss": 0.7554,
+      "step": 140
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 2.7811814881259503e-06,
+      "loss": 0.7783,
+      "step": 142
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 2.134025123396638e-06,
+      "loss": 0.7783,
+      "step": 144
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 1.5708419435684462e-06,
+      "loss": 0.7793,
+      "step": 146
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 1.0926199633097157e-06,
+      "loss": 0.8064,
+      "step": 148
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 7.001981464747565e-07,
+      "loss": 0.8042,
+      "step": 150
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 3.9426493427611177e-07,
+      "loss": 0.791,
+      "step": 152
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.753570375247815e-07,
+      "loss": 0.8147,
+      "step": 154
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 4.385849505708084e-08,
+      "loss": 0.7617,
+      "step": 156
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 0.0,
+      "loss": 0.7446,
+      "step": 158
+    },
+    {
+      "epoch": 1.99,
+      "eval_accuracy": 0.6768941789814655,
+      "eval_loss": 1.2372242212295532,
+      "eval_runtime": 71.61,
+      "eval_samples_per_second": 3.561,
+      "eval_steps_per_second": 1.787,
+      "step": 158
+    },
+    {
+      "epoch": 1.99,
+      "step": 158,
+      "total_flos": 4.14872908733612e+17,
+      "train_loss": 1.1195918215981013,
+      "train_runtime": 35849.6176,
+      "train_samples_per_second": 0.284,
+      "train_steps_per_second": 0.004
+    }
+  ],
+  "max_steps": 158,
+  "num_train_epochs": 2,
+  "total_flos": 4.14872908733612e+17,
+  "trial_name": null,
+  "trial_params": null
+}