Add quantization config

config.json

@@ -23,5 +23,13 @@
   "torch_dtype": "float16",
   "transformers_version": "4.34.0",
   "use_cache": true,
-  "vocab_size": 32002
+  "vocab_size": 32002,
+  "quantization_config" : {
+    "quant_method": "awq",
+    "zero_point": true,
+    "group_size": 128,
+    "bits": 4,
+    "version": "gemm",
+    "backend" : "autoawq",
+  }
 }