From a16ba7a0166dbf9062ee4616e3ccfbff377e9f4b Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Fri, 15 Sep 2023 23:22:37 -0700 Subject: feat: :sparkles: add stop_tokens option to LLM --- continuedev/src/continuedev/libs/llm/__init__.py | 10 +++++++--- docs/docs/reference/Models/anthropic.md | 1 + docs/docs/reference/Models/ggml.md | 1 + docs/docs/reference/Models/hf_inference_api.md | 1 + docs/docs/reference/Models/hf_tgi.md | 1 + docs/docs/reference/Models/llamacpp.md | 1 + docs/docs/reference/Models/maybe_proxy_openai.md | 1 + docs/docs/reference/Models/ollama.md | 1 + docs/docs/reference/Models/openai.md | 1 + docs/docs/reference/Models/queued.md | 1 + docs/docs/reference/Models/replicate.md | 1 + docs/docs/reference/Models/text_gen_interface.md | 1 + docs/docs/reference/Models/together.md | 1 + docs/docs/reference/config.md | 2 +- 14 files changed, 20 insertions(+), 4 deletions(-) diff --git a/continuedev/src/continuedev/libs/llm/__init__.py b/continuedev/src/continuedev/libs/llm/__init__.py index baeb9d1a..b2eecab6 100644 --- a/continuedev/src/continuedev/libs/llm/__init__.py +++ b/continuedev/src/continuedev/libs/llm/__init__.py @@ -68,6 +68,10 @@ class LLM(ContinueBaseModel): ..., description="The name of the model to be used (e.g. gpt-4, codellama)" ) + stop_tokens: Optional[List[str]] = Field( + None, description="Tokens that will stop the completion." + ) + timeout: Optional[int] = Field( 300, description="Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", @@ -204,7 +208,7 @@ class LLM(ContinueBaseModel): top_k=top_k, presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, - stop=stop, + stop=stop or self.stop_tokens, max_tokens=max_tokens, functions=functions, ) @@ -251,7 +255,7 @@ class LLM(ContinueBaseModel): top_k=top_k, presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, - stop=stop, + stop=stop or self.stop_tokens, max_tokens=max_tokens, functions=functions, ) @@ -296,7 +300,7 @@ class LLM(ContinueBaseModel): top_k=top_k, presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, - stop=stop, + stop=stop or self.stop_tokens, max_tokens=max_tokens, functions=functions, ) diff --git a/docs/docs/reference/Models/anthropic.md b/docs/docs/reference/Models/anthropic.md index 3f405d39..e2c6f683 100644 --- a/docs/docs/reference/Models/anthropic.md +++ b/docs/docs/reference/Models/anthropic.md @@ -31,6 +31,7 @@ Claude 2 is not yet publicly released. You can request early access [here](https + diff --git a/docs/docs/reference/Models/ggml.md b/docs/docs/reference/Models/ggml.md index 3369df6f..d02f6d05 100644 --- a/docs/docs/reference/Models/ggml.md +++ b/docs/docs/reference/Models/ggml.md @@ -34,6 +34,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/Models/hf_inference_api.md b/docs/docs/reference/Models/hf_inference_api.md index 6e5aeeca..e7857b21 100644 --- a/docs/docs/reference/Models/hf_inference_api.md +++ b/docs/docs/reference/Models/hf_inference_api.md @@ -33,6 +33,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/Models/hf_tgi.md b/docs/docs/reference/Models/hf_tgi.md index 3ea28730..ab3f4d61 100644 --- a/docs/docs/reference/Models/hf_tgi.md +++ b/docs/docs/reference/Models/hf_tgi.md @@ -18,6 +18,7 @@ import ClassPropertyRef from '@site/src/components/ClassPropertyRef.tsx'; + diff --git a/docs/docs/reference/Models/llamacpp.md b/docs/docs/reference/Models/llamacpp.md index c1c5e4f9..ae4b6e62 100644 --- a/docs/docs/reference/Models/llamacpp.md +++ b/docs/docs/reference/Models/llamacpp.md @@ -38,6 +38,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/Models/maybe_proxy_openai.md b/docs/docs/reference/Models/maybe_proxy_openai.md index 651cbdba..c080b54d 100644 --- a/docs/docs/reference/Models/maybe_proxy_openai.md +++ b/docs/docs/reference/Models/maybe_proxy_openai.md @@ -39,6 +39,7 @@ These classes support any models available through the OpenAI API, assuming your + diff --git a/docs/docs/reference/Models/ollama.md b/docs/docs/reference/Models/ollama.md index 9f92c850..f0370b45 100644 --- a/docs/docs/reference/Models/ollama.md +++ b/docs/docs/reference/Models/ollama.md @@ -29,6 +29,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/Models/openai.md b/docs/docs/reference/Models/openai.md index d9c440b7..f28e0598 100644 --- a/docs/docs/reference/Models/openai.md +++ b/docs/docs/reference/Models/openai.md @@ -47,6 +47,7 @@ Options for serving models locally with an OpenAI-compatible server include: + diff --git a/docs/docs/reference/Models/queued.md b/docs/docs/reference/Models/queued.md index 2499e03b..231aa4dc 100644 --- a/docs/docs/reference/Models/queued.md +++ b/docs/docs/reference/Models/queued.md @@ -31,6 +31,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/Models/replicate.md b/docs/docs/reference/Models/replicate.md index 041f4c2b..83bfd383 100644 --- a/docs/docs/reference/Models/replicate.md +++ b/docs/docs/reference/Models/replicate.md @@ -34,6 +34,7 @@ If you don't specify the `model` parameter, it will default to `replicate/llama- + diff --git a/docs/docs/reference/Models/text_gen_interface.md b/docs/docs/reference/Models/text_gen_interface.md index 7ca38b36..d910bee2 100644 --- a/docs/docs/reference/Models/text_gen_interface.md +++ b/docs/docs/reference/Models/text_gen_interface.md @@ -32,6 +32,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/Models/together.md b/docs/docs/reference/Models/together.md index 1a13cbbc..6838ba36 100644 --- a/docs/docs/reference/Models/together.md +++ b/docs/docs/reference/Models/together.md @@ -34,6 +34,7 @@ config = ContinueConfig( + diff --git a/docs/docs/reference/config.md b/docs/docs/reference/config.md index a96dc2ac..f867ee1e 100644 --- a/docs/docs/reference/config.md +++ b/docs/docs/reference/config.md @@ -11,7 +11,7 @@ Continue can be deeply customized by editing the `ContinueConfig` object in `~/. - + -- cgit v1.2.3-70-g09d2