Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-405B-FP8

Text Generation
Transformers
Safetensors
llama
fp8
vllm
text-generation-inference
Inference Endpoints
Model card Files Community
Lin-K76 commited on
Commit
ff4cae8
•
1 Parent(s): 3a77955

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +75 -0
  2. generation_config.json +9 -0
  3. model-00001-of-00086.safetensors +3 -0
  4. model-00002-of-00086.safetensors +3 -0
  5. model-00003-of-00086.safetensors +3 -0
  6. model-00004-of-00086.safetensors +3 -0
  7. model-00005-of-00086.safetensors +3 -0
  8. model-00006-of-00086.safetensors +3 -0
  9. model-00007-of-00086.safetensors +3 -0
  10. model-00008-of-00086.safetensors +3 -0
  11. model-00009-of-00086.safetensors +3 -0
  12. model-00010-of-00086.safetensors +3 -0
  13. model-00011-of-00086.safetensors +3 -0
  14. model-00012-of-00086.safetensors +3 -0
  15. model-00013-of-00086.safetensors +3 -0
  16. model-00014-of-00086.safetensors +3 -0
  17. model-00015-of-00086.safetensors +3 -0
  18. model-00016-of-00086.safetensors +3 -0
  19. model-00017-of-00086.safetensors +3 -0
  20. model-00018-of-00086.safetensors +3 -0
  21. model-00019-of-00086.safetensors +3 -0
  22. model-00020-of-00086.safetensors +3 -0
  23. model-00021-of-00086.safetensors +3 -0
  24. model-00022-of-00086.safetensors +3 -0
  25. model-00023-of-00086.safetensors +3 -0
  26. model-00024-of-00086.safetensors +3 -0
  27. model-00025-of-00086.safetensors +3 -0
  28. model-00026-of-00086.safetensors +3 -0
  29. model-00027-of-00086.safetensors +3 -0
  30. model-00028-of-00086.safetensors +3 -0
  31. model-00029-of-00086.safetensors +3 -0
  32. model-00030-of-00086.safetensors +3 -0
  33. model-00031-of-00086.safetensors +3 -0
  34. model-00032-of-00086.safetensors +3 -0
  35. model-00033-of-00086.safetensors +3 -0
  36. model-00034-of-00086.safetensors +3 -0
  37. model-00035-of-00086.safetensors +3 -0
  38. model-00036-of-00086.safetensors +3 -0
  39. model-00037-of-00086.safetensors +3 -0
  40. model-00038-of-00086.safetensors +3 -0
  41. model-00039-of-00086.safetensors +3 -0
  42. model-00040-of-00086.safetensors +3 -0
  43. model-00041-of-00086.safetensors +3 -0
  44. model-00042-of-00086.safetensors +3 -0
  45. model-00043-of-00086.safetensors +3 -0
  46. model-00044-of-00086.safetensors +3 -0
  47. model-00045-of-00086.safetensors +3 -0
  48. model-00046-of-00086.safetensors +3 -0
  49. model-00047-of-00086.safetensors +3 -0
  50. model-00048-of-00086.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/mnt/nvme1/meta-llama/Meta-Llama-3.1-405B/",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "compression_config": {
10
+ "config_groups": {
11
+ "group_0": {
12
+ "input_activations": {
13
+ "block_structure": null,
14
+ "dynamic": false,
15
+ "group_size": null,
16
+ "num_bits": 8,
17
+ "observer": "minmax",
18
+ "observer_kwargs": {},
19
+ "strategy": "tensor",
20
+ "symmetric": true,
21
+ "type": "float"
22
+ },
23
+ "output_activations": null,
24
+ "targets": [
25
+ "Linear"
26
+ ],
27
+ "weights": {
28
+ "block_structure": null,
29
+ "dynamic": false,
30
+ "group_size": null,
31
+ "num_bits": 8,
32
+ "observer": "minmax",
33
+ "observer_kwargs": {},
34
+ "strategy": "tensor",
35
+ "symmetric": true,
36
+ "type": "float"
37
+ }
38
+ }
39
+ },
40
+ "format": "naive-quantized",
41
+ "global_compression_ratio": 1.4646996824138383,
42
+ "ignore": [
43
+ "lm_head"
44
+ ],
45
+ "kv_cache_scheme": null,
46
+ "quant_method": "compressed-tensors",
47
+ "quantization_status": "frozen"
48
+ },
49
+ "eos_token_id": 128001,
50
+ "hidden_act": "silu",
51
+ "hidden_size": 16384,
52
+ "initializer_range": 0.02,
53
+ "intermediate_size": 53248,
54
+ "max_position_embeddings": 131072,
55
+ "mlp_bias": false,
56
+ "model_type": "llama",
57
+ "num_attention_heads": 128,
58
+ "num_hidden_layers": 126,
59
+ "num_key_value_heads": 16,
60
+ "pretraining_tp": 1,
61
+ "rms_norm_eps": 1e-05,
62
+ "rope_scaling": {
63
+ "factor": 8.0,
64
+ "high_freq_factor": 4.0,
65
+ "low_freq_factor": 1.0,
66
+ "original_max_position_embeddings": 8192,
67
+ "rope_type": "llama3"
68
+ },
69
+ "rope_theta": 500000.0,
70
+ "tie_word_embeddings": false,
71
+ "torch_dtype": "float16",
72
+ "transformers_version": "4.44.0.dev0",
73
+ "use_cache": true,
74
+ "vocab_size": 128256
75
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "do_sample": true,
5
+ "eos_token_id": 128001,
6
+ "temperature": 0.6,
7
+ "top_p": 0.9,
8
+ "transformers_version": "4.44.0.dev0"
9
+ }
model-00001-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3ee69ec3222371783fe4677f152b5389e4d0b67cc778a60ef399f1bf0e5f032
3
+ size 4806673936
model-00002-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:082637138b70414e4b1f36145d088e48eec320f84adbf26ff85c7525413205c1
3
+ size 4966124652
model-00003-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01b8a3c94b239ad22668421096b25657ef81a3202936825001a4755342bf3f5c
3
+ size 4697755992
model-00004-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05cce7b14215f1132de2c31de37a69ea5e429e818260bd10331276720001fa09
3
+ size 4966124652
model-00005-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e32aa7b3fe2936e0631d5ecb7a6e8e901e8758440d373d1be70f478b2d3e68ce
3
+ size 4697755992
model-00006-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5242dd75de02aa2f400fdbd8a3521bc933ae7ec4cd35b3a0b7bed1b5dd77c9b3
3
+ size 4966124652
model-00007-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5837cca28c9cfc79ee38a71e0719ba9be42de788e9d052a9bbe09ddd4eecbb5
3
+ size 4697755992
model-00008-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:174778c3e56299f23ba6f7124b772b5e38daaecae5bd5d9c8a1f227e15b00f2e
3
+ size 4966124588
model-00009-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fe67c483f53b650932ec26a3968d59e00ca46512eb49c28a5034052927d77ae
3
+ size 4697756032
model-00010-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51b48b4d3764673fe9e500725ad172f8e5add87476849b03d1aef7f037e5a053
3
+ size 4966124684
model-00011-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c99f110a87eed4af57167d7b2c779496d0c5cbbd25e7c65435d116ad8ffe1ecc
3
+ size 4697756032
model-00012-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d43c83bfcd70a45d7291621abc70a0be0597ab70d27af862c75e52c33285a498
3
+ size 4966124684
model-00013-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6e5391366f3848d8392cfb2b55b264024b6a752170e7546b09ad4d8cf741748
3
+ size 4697756032
model-00014-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf5c31df8fe6691392cf625ddd3c1c298f6fc6b62c03bf373a6caca3d49f9db5
3
+ size 4966124684
model-00015-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd8190f3f217f4795ed374cf3daac61b2806f712b782a2634c9efa10abb6972b
3
+ size 4697756032
model-00016-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9adf503596ec02abecaf1e7e0d7e68f822f1823ab8e8d46e3636704af6c63d3d
3
+ size 4966124684
model-00017-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4cdb70afe33df164fcd8612172f1d3021b53caaae55fcfc66f0344d3bb4e36d2
3
+ size 4697756032
model-00018-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c40246d196d725e0327320ab1f9c807379a0983425010dca55caf676a6072155
3
+ size 4966124684
model-00019-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c229907c3ead05fd3fb54646275f87e5234137508d350f77d92c2700f049127
3
+ size 4697756032
model-00020-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d604bedd805abebbccbf20800c8e090e4fba0b26a39d80792a43fcde5f3bd992
3
+ size 4966124684
model-00021-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d85808f0514de1dfdc640e6635e060b93c8163207cabfffccbe2ea539c29dae8
3
+ size 4697756032
model-00022-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44b57daaf24f2473dc7273bc9ced0f3d2699188797aba665fd79e5f8ab0b4ac4
3
+ size 4966124684
model-00023-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5957b48835ce95f67dbbaa904a7c649d80acc47e8bd6d77e7e23a94ed6c3a10
3
+ size 4697756032
model-00024-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b3e8a89f2c4d00b1a6bd5fdbd3993c78f19dd06b2008e9885a40e5ebd6c8c5b
3
+ size 4966124684
model-00025-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75bfdc8aaa7c5850ed6b06fc29a61addc4d7b412147a3dca327d3c1c1a6da6d9
3
+ size 4697756032
model-00026-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca01d94a32c0be06bb852c7f5dc779a48ed48fc04db0d777b3780af032d78b0a
3
+ size 4966124684
model-00027-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b65578f1f447d6be2243b93debbc5f40ca2686c55abd14e313d3eb3be3a0f2ee
3
+ size 4697756032
model-00028-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcdeb9f81d569cc226ab909fea7d39927c1a4ac8fd6dfec9b7ae745c555e2bd7
3
+ size 4966124684
model-00029-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c3fb2e5a596ecc83cf8610a352fc8d4679fe800760cfa3b5922538168de13b6
3
+ size 4697756032
model-00030-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:918c1f0ca400791b5363e901dbfec73d03020dee4c287e14a09f0af8ed264eb2
3
+ size 4966124684
model-00031-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:654d070a517a79b56f8ecf35826508e7c02594ca9fabe71cc0174ab78ba09af3
3
+ size 4697756032
model-00032-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c029aca150156fd91e344fb0e3794b0d74d8a8e002c2a1f4bec71fe5b508b4e6
3
+ size 4966124684
model-00033-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:385ce6e1626a20daf5e85fcb59e3b72982d9e23a76451b043487283ca05ae0d1
3
+ size 4697756032
model-00034-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f950a58e12b40001d6a1b16ffb43af2d40d27a00066bec7ea85ba639577aefa8
3
+ size 4966124684
model-00035-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9692222809ade9ef0de400ad4ce303acc0a9f031ecb1e0b14e6f6589ae494cad
3
+ size 4697756032
model-00036-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2e4091b1c9ae0c0260270f6e45e50885dc9b1ad3a269a68c78cfe02033cb9a2
3
+ size 4966124684
model-00037-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87e99d436dc7d291d6e8c34fa61d5c6a121a6e820a2ca0e9eb9f533c690b32b2
3
+ size 4697756032
model-00038-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4548d7926cbadcba4c29f9934f47fce719a8e59d45276348fdf0bdf6f35951da
3
+ size 4966124684
model-00039-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1436f3f4c7ecf1330e9775ac60175c179d240059f0060285d2781fe081794aa8
3
+ size 4697756032
model-00040-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e3aaa09a9dd4ad0650505816b1bd0d8307570eaef5d848f3bd470781a2107c4
3
+ size 4966124684
model-00041-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:317e4b541e06ba5052d14492294308527cdcc11c81cad6c77f48b6c8f9532d5b
3
+ size 4697756032
model-00042-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:492cc3cf108294cf2b05b0b10e31bcf9e8e59255c687ad435ac831c917b0b58e
3
+ size 4966124684
model-00043-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78dbe0f33783397ec660346239aa9d0be814d8b5be25c433f727bf58c81eacef
3
+ size 4697756032
model-00044-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43bd605808844ec11855b88b7c3b25c0d9514ee2651420c75181b227e2a9c55f
3
+ size 4966124684
model-00045-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc461f9d0c1d81d17daf90e4c95f36a8f606a6b3416fd52b59fb44809fb83f3d
3
+ size 4697756032
model-00046-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:184fae3f9ca9048cd12f5f0f83a14cbb0baff2cbca008255319824d6bfbdcbb3
3
+ size 4966124684
model-00047-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21ba180f2f9a12e40eb4cc2893ba68e4ebf8c60c07bde6506fbe9ee084b2ed9c
3
+ size 4697756032
model-00048-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c46221dc7a489e1675cf44af15e140e8c7d3347ce4d4141ab3e1557a8f024a98
3
+ size 4966124684