File size: 896 Bytes
d0ecd4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | from transformers import PretrainedConfig
class BinaryLLMConfig(PretrainedConfig):
model_type = "binaryllm"
def __init__(
self,
vocab_size: int = 4,
hidden_size: int = 384,
num_hidden_layers: int = 6,
num_attention_heads: int = 6,
intermediate_size: int = 1536,
max_position_embeddings: int = 4096,
dropout: float = 0.1,
activation: str = "gelu",
**kwargs,
):
self.vocab_size = int(vocab_size)
self.hidden_size = int(hidden_size)
self.num_hidden_layers = int(num_hidden_layers)
self.num_attention_heads = int(num_attention_heads)
self.intermediate_size = int(intermediate_size)
self.max_position_embeddings = int(max_position_embeddings)
self.dropout = float(dropout)
self.activation = str(activation)
super().__init__(**kwargs)
|