default use xentropy and use only last logit in sampling

2023-09-14 21:31:48 -05:00
parent 2c4eec6aa8
commit 6327b9fb1a
2 changed files with 2 additions and 1 deletions
@@ -325,6 +325,7 @@ class SampleGenerateCallback(transformers.TrainerCallback):
                    input_ids=input_ids,
                    max_new_tokens=15,
                    do_sample = False,
+                    only_last_logit=True,  # to save memory
                )


@@ -782,7 +782,7 @@ class LlamaForCausalLM(LlamaPreTrainedModel):
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        only_last_logit: Optional[bool] = None,
-        xentropy: Optional[bool] = None,
+        xentropy: Optional[bool] = True,
        is_padded_inputs: Optional[bool] = None,
    ) -> Union[Tuple, CausalLMOutputWithPast]:
        r"""