diff options
Diffstat (limited to 'continuedev')
| -rw-r--r-- | continuedev/src/continuedev/libs/llm/hf_inference_api.py | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/continuedev/src/continuedev/libs/llm/hf_inference_api.py b/continuedev/src/continuedev/libs/llm/hf_inference_api.py index 9664fec2..8945250c 100644 --- a/continuedev/src/continuedev/libs/llm/hf_inference_api.py +++ b/continuedev/src/continuedev/libs/llm/hf_inference_api.py @@ -52,7 +52,7 @@ class HuggingFaceInferenceAPI(LLM): response = requests.post(API_URL, headers=headers, json={ "inputs": prompt, "parameters": { - "max_new_tokens": self.max_context_length - self.count_tokens(prompt), + "max_new_tokens": min(250, self.max_context_length - self.count_tokens(prompt)), "max_time": DEFAULT_MAX_TIME, "return_full_text": False, } |