shunxing1234
commited on
Commit
•
3d19935
1
Parent(s):
72802bf
Update modeling_aquila.py
Browse files- modeling_aquila.py +3 -3
modeling_aquila.py
CHANGED
@@ -127,7 +127,7 @@ class AquilaRotaryEmbedding(torch.nn.Module):
|
|
127 |
|
128 |
# Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Aquila
|
129 |
class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
|
130 |
-
"""
|
131 |
|
132 |
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
|
133 |
self.scaling_factor = scaling_factor
|
@@ -145,8 +145,8 @@ class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
|
|
145 |
self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
|
146 |
|
147 |
# Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Aquila
|
148 |
-
class AquilaDynamicNTKScalingRotaryEmbedding(
|
149 |
-
"""
|
150 |
|
151 |
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
|
152 |
self.scaling_factor = scaling_factor
|
|
|
127 |
|
128 |
# Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Aquila
|
129 |
class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
|
130 |
+
"""AquilaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
|
131 |
|
132 |
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
|
133 |
self.scaling_factor = scaling_factor
|
|
|
145 |
self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
|
146 |
|
147 |
# Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Aquila
|
148 |
+
class AquilaDynamicNTKScalingRotaryEmbedding(AquilaRotaryEmbedding):
|
149 |
+
"""AquilaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
|
150 |
|
151 |
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
|
152 |
self.scaling_factor = scaling_factor
|