We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 7c69c25 commit 697890bCopy full SHA for 697890b
1 file changed
transformer/config.py
@@ -84,6 +84,7 @@ def __init__(
84
attn_dropout: Optional[float] = 0.0,
85
tied_weights: bool = False,
86
seq_len: int = 1024,
87
+ pos_encoding: str = "RoPE",
88
rope_base: float = 10000.0,
89
max_seq_len: int = 4096,
90
**kwargs: Dict,
0 commit comments