Model save

README.md

@@ -1,10 +1,11 @@
 ---
 license: apache-2.0
-base_model: mistralai/Mistral-7B-v0.1
+library_name: peft
 tags:
 - generated_from_trainer
 datasets:
 - generator
+base_model: mistralai/Mistral-7B-v0.1
 model-index:
 - name: zephyr-7b-sft-lora
   results: []
@@ -17,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0927
+- Loss: 1.0922
 
 ## Model description
 
@@ -47,19 +48,25 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.0931        | 1.0   | 272  | 1.0927          |
+| 1.3681        | 1.0   | 272  | 1.0922          |
 
 
 ### Framework versions
 
-- Transformers 4.35.0
+- Transformers 4.36.0
 - Pytorch 2.1.1+cu121
 - Datasets 2.14.6
-- Tokenizers 0.14.1
+- Tokenizers 0.15.0
+## Training procedure
+
+
+### Framework versions
+
+
+- PEFT 0.6.1

adapter_config.json

@@ -16,9 +16,9 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "q_proj",
     "v_proj",
     "o_proj",
-    "q_proj",
     "k_proj"
   ],
   "task_type": "CAUSAL_LM"

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eaa5158d98175f4ed59693761cec1820cae65c52813b72136dc84fe785a420b
+oid sha256:46933e34b2e227ff195e628e2c8b61bf212c9ef7309bda18a4978b0225e175c2
 size 109086672

all_results.json

@@ -1,12 +1,12 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.0926721096038818,
-    "eval_runtime": 2381.0415,
+    "eval_loss": 1.0922234058380127,
+    "eval_runtime": 2385.5237,
     "eval_samples": 23110,
-    "eval_samples_per_second": 6.481,
-    "eval_steps_per_second": 0.102,
-    "train_loss": 1.1944836974143982,
-    "train_runtime": 72472.9134,
+    "eval_samples_per_second": 6.469,
+    "eval_steps_per_second": 0.101,
+    "train_loss": 1.4444872824584736,
+    "train_runtime": 72477.8005,
     "train_samples": 207865,
     "train_samples_per_second": 1.924,
     "train_steps_per_second": 0.004

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.0926721096038818,
-    "eval_runtime": 2381.0415,
+    "eval_loss": 1.0922234058380127,
+    "eval_runtime": 2385.5237,
     "eval_samples": 23110,
-    "eval_samples_per_second": 6.481,
-    "eval_steps_per_second": 0.102
+    "eval_samples_per_second": 6.469,
+    "eval_steps_per_second": 0.101
 }

runs/Dec12_14-33-55_vista-trx/events.out.tfevents.1702384616.vista-trx.76389.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d671b24ef781b7c435904a85193eddc752de83ce5b7096e702d3afcf99aeab69
+size 4426

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -37,5 +39,5 @@
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": true
+  "use_default_system_prompt": false
 }

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 1.1944836974143982,
-    "train_runtime": 72472.9134,
+    "train_loss": 1.4444872824584736,
+    "train_runtime": 72477.8005,
     "train_samples": 207865,
     "train_samples_per_second": 1.924,
     "train_steps_per_second": 0.004

trainer_state.json

@@ -10,357 +10,359 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 7.142857142857143e-07,
-      "loss": 1.6842,
+      "learning_rate": 1.9999332998034515e-05,
+      "loss": 1.9596,
       "step": 1
     },
     {
       "epoch": 0.02,
-      "learning_rate": 3.5714285714285718e-06,
-      "loss": 1.6847,
+      "learning_rate": 1.99833293993636e-05,
+      "loss": 1.9397,
       "step": 5
     },
     {
       "epoch": 0.04,
-      "learning_rate": 7.1428571428571436e-06,
-      "loss": 1.6885,
+      "learning_rate": 1.99333731792395e-05,
+      "loss": 1.8704,
       "step": 10
     },
     {
       "epoch": 0.06,
-      "learning_rate": 1.0714285714285714e-05,
-      "loss": 1.6795,
+      "learning_rate": 1.985029789966671e-05,
+      "loss": 1.7903,
       "step": 15
     },
     {
       "epoch": 0.07,
-      "learning_rate": 1.4285714285714287e-05,
-      "loss": 1.6085,
+      "learning_rate": 1.9734380543606932e-05,
+      "loss": 1.6992,
       "step": 20
     },
     {
       "epoch": 0.09,
-      "learning_rate": 1.785714285714286e-05,
-      "loss": 1.5545,
+      "learning_rate": 1.9586007593450098e-05,
+      "loss": 1.6678,
       "step": 25
     },
     {
       "epoch": 0.11,
-      "learning_rate": 1.9996684675143132e-05,
-      "loss": 1.465,
+      "learning_rate": 1.9405673742435677e-05,
+      "loss": 1.6189,
       "step": 30
     },
     {
       "epoch": 0.13,
-      "learning_rate": 1.995941251172356e-05,
-      "loss": 1.4256,
+      "learning_rate": 1.9193980245285967e-05,
+      "loss": 1.6122,
       "step": 35
     },
     {
       "epoch": 0.15,
-      "learning_rate": 1.9880878960910772e-05,
-      "loss": 1.3637,
+      "learning_rate": 1.8951632913550625e-05,
+      "loss": 1.5687,
       "step": 40
     },
     {
       "epoch": 0.17,
-      "learning_rate": 1.9761409383589216e-05,
-      "loss": 1.3306,
+      "learning_rate": 1.8679439762346186e-05,
+      "loss": 1.5472,
       "step": 45
     },
     {
       "epoch": 0.18,
-      "learning_rate": 1.960149873671602e-05,
-      "loss": 1.307,
+      "learning_rate": 1.8378308316336585e-05,
+      "loss": 1.5302,
       "step": 50
     },
     {
       "epoch": 0.2,
-      "learning_rate": 1.94018095227371e-05,
-      "loss": 1.2769,
+      "learning_rate": 1.8049242583936923e-05,
+      "loss": 1.5045,
       "step": 55
     },
     {
       "epoch": 0.22,
-      "learning_rate": 1.916316904487005e-05,
-      "loss": 1.2581,
+      "learning_rate": 1.769333970982879e-05,
+      "loss": 1.49,
       "step": 60
     },
     {
       "epoch": 0.24,
-      "learning_rate": 1.888656597962499e-05,
-      "loss": 1.2415,
+      "learning_rate": 1.7311786316948112e-05,
+      "loss": 1.4782,
       "step": 65
     },
     {
       "epoch": 0.26,
-      "learning_rate": 1.8573146280763327e-05,
-      "loss": 1.2178,
+      "learning_rate": 1.6905854550141717e-05,
+      "loss": 1.4596,
       "step": 70
     },
     {
       "epoch": 0.28,
-      "learning_rate": 1.8224208431664093e-05,
-      "loss": 1.2036,
+      "learning_rate": 1.647689783468362e-05,
+      "loss": 1.4509,
       "step": 75
     },
     {
       "epoch": 0.29,
-      "learning_rate": 1.7841198065767107e-05,
-      "loss": 1.1768,
+      "learning_rate": 1.6026346363792565e-05,
+      "loss": 1.4295,
       "step": 80
     },
     {
       "epoch": 0.31,
-      "learning_rate": 1.742570197738019e-05,
-      "loss": 1.1738,
+      "learning_rate": 1.5555702330196024e-05,
+      "loss": 1.4301,
       "step": 85
     },
     {
       "epoch": 0.33,
-      "learning_rate": 1.6979441547663434e-05,
-      "loss": 1.1708,
+      "learning_rate": 1.5066534917639195e-05,
+      "loss": 1.4297,
       "step": 90
     },
     {
       "epoch": 0.35,
-      "learning_rate": 1.650426561302634e-05,
-      "loss": 1.1551,
+      "learning_rate": 1.4560475069037895e-05,
+      "loss": 1.4161,
       "step": 95
     },
     {
       "epoch": 0.37,
-      "learning_rate": 1.6002142805483686e-05,
-      "loss": 1.1515,
+      "learning_rate": 1.403921004871895e-05,
+      "loss": 1.4144,
       "step": 100
     },
     {
       "epoch": 0.39,
-      "learning_rate": 1.5475153396703756e-05,
-      "loss": 1.1406,
+      "learning_rate": 1.350447781687826e-05,
+      "loss": 1.4051,
       "step": 105
     },
     {
       "epoch": 0.4,
-      "learning_rate": 1.4925480679538646e-05,
-      "loss": 1.1402,
+      "learning_rate": 1.2958061235012707e-05,
+      "loss": 1.4061,
       "step": 110
     },
     {
       "epoch": 0.42,
-      "learning_rate": 1.4355401922742478e-05,
-      "loss": 1.1297,
+      "learning_rate": 1.2401782121645767e-05,
+      "loss": 1.3968,
       "step": 115
     },
     {
       "epoch": 0.44,
-      "learning_rate": 1.3767278936351853e-05,
-      "loss": 1.1265,
+      "learning_rate": 1.1837495178165706e-05,
+      "loss": 1.3946,
       "step": 120
     },
     {
       "epoch": 0.46,
-      "learning_rate": 1.316354828681562e-05,
-      "loss": 1.113,
+      "learning_rate": 1.126708180502834e-05,
+      "loss": 1.382,
       "step": 125
     },
     {
       "epoch": 0.48,
-      "learning_rate": 1.2546711202412287e-05,
-      "loss": 1.1146,
+      "learning_rate": 1.0692443828941918e-05,
+      "loss": 1.3844,
       "step": 130
     },
     {
       "epoch": 0.5,
-      "learning_rate": 1.1919323210776459e-05,
-      "loss": 1.1153,
+      "learning_rate": 1.0115497161948409e-05,
+      "loss": 1.3857,
       "step": 135
     },
     {
       "epoch": 0.51,
-      "learning_rate": 1.1283983551465512e-05,
-      "loss": 1.1102,
+      "learning_rate": 9.538165413542607e-06,
+      "loss": 1.3812,
       "step": 140
     },
     {
       "epoch": 0.53,
-      "learning_rate": 1.0643324407429818e-05,
-      "loss": 1.0941,
+      "learning_rate": 8.962373477126983e-06,
+      "loss": 1.3657,
       "step": 145
     },
     {
       "epoch": 0.55,
-      "learning_rate": 1e-05,
-      "loss": 1.1037,
+      "learning_rate": 8.39004111218587e-06,
+      "loss": 1.3757,
       "step": 150
     },
     {
       "epoch": 0.57,
-      "learning_rate": 9.356675592570185e-06,
-      "loss": 1.099,
+      "learning_rate": 7.823076543576718e-06,
+      "loss": 1.3713,
       "step": 155
     },
     {
       "epoch": 0.59,
-      "learning_rate": 8.71601644853449e-06,
-      "loss": 1.0926,
+      "learning_rate": 7.263370099279173e-06,
+      "loss": 1.3653,
       "step": 160
     },
     {
       "epoch": 0.61,
-      "learning_rate": 8.080676789223545e-06,
-      "loss": 1.1019,
+      "learning_rate": 6.712787907814542e-06,
+      "loss": 1.3749,
       "step": 165
     },
     {
       "epoch": 0.62,
-      "learning_rate": 7.453288797587714e-06,
-      "loss": 1.0985,
+      "learning_rate": 6.173165676349103e-06,
+      "loss": 1.3718,
       "step": 170
     },
     {
       "epoch": 0.64,
-      "learning_rate": 6.8364517131843865e-06,
-      "loss": 1.0993,
+      "learning_rate": 5.646302570225919e-06,
+      "loss": 1.3728,
       "step": 175
     },
     {
       "epoch": 0.66,
-      "learning_rate": 6.232721063648148e-06,
-      "loss": 1.0935,
+      "learning_rate": 5.133955214331439e-06,
+      "loss": 1.3672,
       "step": 180
     },
     {
       "epoch": 0.68,
-      "learning_rate": 5.644598077257524e-06,
-      "loss": 1.0921,
+      "learning_rate": 4.637831836297103e-06,
+      "loss": 1.366,
       "step": 185
     },
     {
       "epoch": 0.7,
-      "learning_rate": 5.074519320461358e-06,
-      "loss": 1.0967,
+      "learning_rate": 4.1595865710632366e-06,
+      "loss": 1.3708,
       "step": 190
     },
     {
       "epoch": 0.72,
-      "learning_rate": 4.524846603296245e-06,
-      "loss": 1.0958,
+      "learning_rate": 3.700813945794425e-06,
+      "loss": 1.37,
       "step": 195
     },
     {
       "epoch": 0.73,
-      "learning_rate": 3.997857194516319e-06,
-      "loss": 1.0936,
+      "learning_rate": 3.2630435635344283e-06,
+      "loss": 1.3679,
       "step": 200
     },
     {
       "epoch": 0.75,
-      "learning_rate": 3.4957343869736604e-06,
-      "loss": 1.0927,
+      "learning_rate": 2.847735003325868e-06,
+      "loss": 1.3671,
       "step": 205
     },
     {
       "epoch": 0.77,
-      "learning_rate": 3.0205584523365626e-06,
-      "loss": 1.0931,
+      "learning_rate": 2.456272953798361e-06,
+      "loss": 1.3677,
       "step": 210
     },
     {
       "epoch": 0.79,
-      "learning_rate": 2.5742980226198133e-06,
-      "loss": 1.0928,
+      "learning_rate": 2.0899625964503113e-06,
+      "loss": 1.3675,
       "step": 215
     },
     {
       "epoch": 0.81,
-      "learning_rate": 2.158801934232897e-06,
-      "loss": 1.09,
+      "learning_rate": 1.7500252540169782e-06,
+      "loss": 1.3647,
       "step": 220
     },
     {
       "epoch": 0.83,
-      "learning_rate": 1.7757915683359084e-06,
-      "loss": 1.0963,
+      "learning_rate": 1.4375943184337871e-06,
+      "loss": 1.371,
       "step": 225
     },
     {
       "epoch": 0.84,
-      "learning_rate": 1.426853719236676e-06,
-      "loss": 1.0874,
+      "learning_rate": 1.1537114719714482e-06,
+      "loss": 1.3622,
       "step": 230
     },
     {
       "epoch": 0.86,
-      "learning_rate": 1.113434020375015e-06,
-      "loss": 1.0848,
+      "learning_rate": 8.993232141421415e-07,
+      "loss": 1.3596,
       "step": 235
     },
     {
       "epoch": 0.88,
-      "learning_rate": 8.368309551299536e-07,
-      "loss": 1.0941,
+      "learning_rate": 6.752777059564431e-07,
+      "loss": 1.369,
       "step": 240
     },
     {
       "epoch": 0.9,
-      "learning_rate": 5.98190477262901e-07,
-      "loss": 1.0872,
+      "learning_rate": 4.823219420526182e-07,
+      "loss": 1.3621,
       "step": 245
     },
     {
       "epoch": 0.92,
-      "learning_rate": 3.985012632839824e-07,
-      "loss": 1.09,
+      "learning_rate": 3.2109926012677484e-07,
+      "loss": 1.3649,
       "step": 250
     },
     {
       "epoch": 0.94,
-      "learning_rate": 2.385906164107854e-07,
-      "loss": 1.0896,
+      "learning_rate": 1.921471959676957e-07,
+      "loss": 1.3646,
       "step": 255
     },
     {
       "epoch": 0.95,
-      "learning_rate": 1.1912103908922945e-07,
-      "loss": 1.0803,
+      "learning_rate": 9.589569124794918e-08,
+      "loss": 1.3552,
       "step": 260
     },
     {
       "epoch": 0.97,
-      "learning_rate": 4.058748827643899e-08,
-      "loss": 1.0919,
+      "learning_rate": 3.266566004670013e-08,
+      "loss": 1.3669,
       "step": 265
     },
     {
       "epoch": 0.99,
-      "learning_rate": 3.315324856869584e-09,
-      "loss": 1.0931,
+      "learning_rate": 2.667918883627607e-09,
+      "loss": 1.3681,
       "step": 270
     },
     {
       "epoch": 1.0,
-      "eval_loss": 1.0926721096038818,
-      "eval_runtime": 2381.3881,
-      "eval_samples_per_second": 6.48,
-      "eval_steps_per_second": 0.102,
+      "eval_loss": 1.0922234058380127,
+      "eval_runtime": 2386.6161,
+      "eval_samples_per_second": 6.466,
+      "eval_steps_per_second": 0.101,
       "step": 272
     },
     {
       "epoch": 1.0,
       "step": 272,
       "total_flos": 7631468079611904.0,
-      "train_loss": 1.1944836974143982,
-      "train_runtime": 72472.9134,
+      "train_loss": 1.4444872824584736,
+      "train_runtime": 72477.8005,
       "train_samples_per_second": 1.924,
       "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 5,
   "max_steps": 272,
+  "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
   "total_flos": 7631468079611904.0,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db90f8ee5898ac57b8ff11416193b63fff88243010bb512f35030b2ccadbeff0
-size 5624
+oid sha256:4c87469285d4d700c58756e9373aa90df9e87013e22d4b14969e26b47131d2b4
+size 5752