Skip to content

Commit

Permalink
Changing XLNet default from not using memories to 512 context size fo…
Browse files Browse the repository at this point in the history
…llowing paper (#8417)

* Move XLNet memory length FutureWarning

* isort

* style

* Changed default XLNet memory length
  • Loading branch information
TevenLeScao authored Nov 10, 2020
1 parent 190df58 commit 4185b11
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 16 deletions.
15 changes: 1 addition & 14 deletions src/transformers/configuration_xlnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
# limitations under the License.
""" XLNet configuration """

import warnings

from .configuration_utils import PretrainedConfig
from .utils import logging

Expand Down Expand Up @@ -144,7 +142,7 @@ def __init__(
initializer_range=0.02,
layer_norm_eps=1e-12,
dropout=0.1,
mem_len=None,
mem_len=512,
reuse_len=None,
bi_data=False,
clamp_len=-1,
Expand Down Expand Up @@ -198,17 +196,6 @@ def __init__(
self.pad_token_id = pad_token_id
self.eos_token_id = eos_token_id

if mem_len is None or mem_len == 0:
warnings.warn(
"This config doesn't use attention memories, a core feature of XLNet."
" Consider setting `mem_len` to a non-zero value, for example "
"`xlnet = XLNetLMHeadModel.from_pretrained('xlnet-base-cased'', mem_len=1024)`,"
" for accurate training performance as well as an order of magnitude faster inference."
" Starting from version 3.5.0, the default parameter will be 1024, following"
" the implementation in https://arxiv.org/abs/1906.08237",
FutureWarning,
)

@property
def max_position_embeddings(self):
return -1
Expand Down
3 changes: 1 addition & 2 deletions src/transformers/modeling_xlnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
"""
PyTorch XLNet model.
"""


from dataclasses import dataclass
from typing import List, Optional, Tuple

Expand Down Expand Up @@ -1087,6 +1085,7 @@ def forward(
output_hidden_states=None,
return_dict=None,
):

output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
Expand Down

0 comments on commit 4185b11

Please sign in to comment.