diff options
22 files changed, 835 insertions, 206 deletions
diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index 9ebf288b..9f2338ff 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -2,6 +2,7 @@ import json import os import time import traceback +import uuid from functools import cached_property from typing import Callable, Coroutine, Dict, List, Optional @@ -380,11 +381,27 @@ class Autopilot(ContinueBaseModel): # last_depth = self.history.timeline[i].depth # i -= 1 + # Log the context and step to dev data + context_used = await self.context_manager.get_selected_items() posthog_logger.capture_event( "step run", {"step_name": step.name, "params": step.dict()} ) + step_id = uuid.uuid4().hex dev_data_logger.capture( - "step_run", {"step_name": step.name, "params": step.dict()} + "step_run", + {"step_name": step.name, "params": step.dict(), "step_id": step_id}, + ) + dev_data_logger.capture( + "context_used", + { + "context": list( + map( + lambda item: item.dict(), + context_used, + ) + ), + "step_id": step_id, + }, ) if not is_future_step: @@ -402,7 +419,7 @@ class Autopilot(ContinueBaseModel): step=step, observation=None, depth=self._step_depth, - context_used=await self.context_manager.get_selected_items(), + context_used=context_used, ) ) diff --git a/continuedev/src/continuedev/libs/llm/__init__.py b/continuedev/src/continuedev/libs/llm/__init__.py index 28f614c7..e6a90ef7 100644 --- a/continuedev/src/continuedev/libs/llm/__init__.py +++ b/continuedev/src/continuedev/libs/llm/__init__.py @@ -71,6 +71,10 @@ class LLM(ContinueBaseModel): ..., description="The name of the model to be used (e.g. gpt-4, codellama)" ) + max_tokens: int = Field( + DEFAULT_MAX_TOKENS, description="The maximum number of tokens to generate." + ) + stop_tokens: Optional[List[str]] = Field( None, description="Tokens that will stop the completion." ) @@ -237,7 +241,7 @@ class LLM(ContinueBaseModel): presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, stop=stop or self.stop_tokens, - max_tokens=max_tokens, + max_tokens=max_tokens or self.max_tokens, functions=functions, ) @@ -288,7 +292,7 @@ class LLM(ContinueBaseModel): presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, stop=stop or self.stop_tokens, - max_tokens=max_tokens, + max_tokens=max_tokens or self.max_tokens, functions=functions, ) @@ -337,7 +341,7 @@ class LLM(ContinueBaseModel): presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, stop=stop or self.stop_tokens, - max_tokens=max_tokens, + max_tokens=max_tokens or self.max_tokens, functions=functions, ) diff --git a/continuedev/src/continuedev/libs/util/edit_config.py b/continuedev/src/continuedev/libs/util/edit_config.py index f4285bc9..c77eb2e3 100644 --- a/continuedev/src/continuedev/libs/util/edit_config.py +++ b/continuedev/src/continuedev/libs/util/edit_config.py @@ -98,7 +98,9 @@ def display_llm_class(llm, new: bool = False): [ f"{k}={display_val(v)}" for k, v in llm.dict().items() - if k not in filtered_attrs and v is not None + if k not in filtered_attrs + and v is not None + and not v == llm.__fields__[k].default ] ) return f"{llm.__class__.__name__}(\n\t\t\t{args}\n\t\t)" diff --git a/continuedev/src/continuedev/server/gui.py b/continuedev/src/continuedev/server/gui.py index 10f6974f..cc6bc911 100644 --- a/continuedev/src/continuedev/server/gui.py +++ b/continuedev/src/continuedev/server/gui.py @@ -10,6 +10,7 @@ from uvicorn.main import Server from ..core.main import ContextItem from ..core.models import ALL_MODEL_ROLES, MODEL_CLASSES, MODEL_MODULE_NAMES +from ..libs.llm.prompts.chat import llama2_template_messages, template_alpaca_messages from ..libs.util.create_async_task import create_async_task from ..libs.util.edit_config import ( add_config_import, @@ -323,7 +324,22 @@ class GUIProtocolServer: existing_saved_models.add(display_llm_class(val)) models.__setattr__(role, None) + # Add the requisite import to config.py + add_config_import( + f"from continuedev.src.continuedev.libs.llm.{MODEL_MODULE_NAMES[model_class]} import {model_class}" + ) + if "template_messages" in model: + add_config_import( + f"from continuedev.src.continuedev.libs.llm.prompts.chat import {model['template_messages']}" + ) + # Set and start the new default model + + if "template_messages" in model: + model["template_messages"] = { + "llama2_template_messages": llama2_template_messages, + "template_alpaca_messages": template_alpaca_messages, + }[model["template_messages"]] new_model = MODEL_CLASSES[model_class](**model) models.default = new_model await self.session.autopilot.continue_sdk.start_model(models.default) @@ -343,11 +359,6 @@ class GUIProtocolServer: create_obj_node("Models", models_args), ) - # Add the requisite import to config.py - add_config_import( - f"from continuedev.src.continuedev.libs.llm.{MODEL_MODULE_NAMES[model_class]} import {model_class}" - ) - # Set all roles (in-memory) to the new default model for role in ALL_MODEL_ROLES: if role != "default": diff --git a/docs/docs/reference/Models/anthropicllm.md b/docs/docs/reference/Models/anthropicllm.md index 128b706d..1ff17ce7 100644 --- a/docs/docs/reference/Models/anthropicllm.md +++ b/docs/docs/reference/Models/anthropicllm.md @@ -31,6 +31,7 @@ Claude 2 is not yet publicly released. You can request early access [here](https <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "claude-2", "type": "string"}' required={false} default="claude-2"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/ggml.md b/docs/docs/reference/Models/ggml.md index 7bdb5441..aa2af17f 100644 --- a/docs/docs/reference/Models/ggml.md +++ b/docs/docs/reference/Models/ggml.md @@ -33,6 +33,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to use (optional for the GGML class)", "default": "ggml", "type": "string"}' required={false} default="ggml"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/huggingfaceinferenceapi.md b/docs/docs/reference/Models/huggingfaceinferenceapi.md index 560309f2..ca85522c 100644 --- a/docs/docs/reference/Models/huggingfaceinferenceapi.md +++ b/docs/docs/reference/Models/huggingfaceinferenceapi.md @@ -33,6 +33,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to use (optional for the HuggingFaceInferenceAPI class)", "default": "Hugging Face Inference API", "type": "string"}' required={false} default="Hugging Face Inference API"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/huggingfacetgi.md b/docs/docs/reference/Models/huggingfacetgi.md index 2cee9fe1..1275c13f 100644 --- a/docs/docs/reference/Models/huggingfacetgi.md +++ b/docs/docs/reference/Models/huggingfacetgi.md @@ -18,6 +18,7 @@ import ClassPropertyRef from '@site/src/components/ClassPropertyRef.tsx'; <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "huggingface-tgi", "type": "string"}' required={false} default="huggingface-tgi"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/llamacpp.md b/docs/docs/reference/Models/llamacpp.md index 8a6be11e..69b528bd 100644 --- a/docs/docs/reference/Models/llamacpp.md +++ b/docs/docs/reference/Models/llamacpp.md @@ -38,6 +38,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "llamacpp", "type": "string"}' required={false} default="llamacpp"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/ollama.md b/docs/docs/reference/Models/ollama.md index 6388e8cc..2a5fcff7 100644 --- a/docs/docs/reference/Models/ollama.md +++ b/docs/docs/reference/Models/ollama.md @@ -21,6 +21,7 @@ config = ContinueConfig( <ClassPropertyRef name='server_url' details='{"title": "Server Url", "description": "URL of the Ollama server", "default": "http://localhost:11434", "type": "string"}' required={false} default="http://localhost:11434"/> + ### Inherited Properties <ClassPropertyRef name='title' details='{"title": "Title", "description": "A title that will identify this model in the model selection dropdown", "type": "string"}' required={false} default=""/> @@ -28,6 +29,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "llama2", "type": "string"}' required={false} default="llama2"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/openai.md b/docs/docs/reference/Models/openai.md index e78dd404..5287e61d 100644 --- a/docs/docs/reference/Models/openai.md +++ b/docs/docs/reference/Models/openai.md @@ -46,6 +46,7 @@ Options for serving models locally with an OpenAI-compatible server include: <ClassPropertyRef name='system_message' details='{"title": "System Message", "description": "A system message that will always be followed by the LLM", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/openaifreetrial.md b/docs/docs/reference/Models/openaifreetrial.md index 99c21689..5175273b 100644 --- a/docs/docs/reference/Models/openaifreetrial.md +++ b/docs/docs/reference/Models/openaifreetrial.md @@ -39,6 +39,7 @@ These classes support any models available through the OpenAI API, assuming your <ClassPropertyRef name='system_message' details='{"title": "System Message", "description": "A system message that will always be followed by the LLM", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/queuedllm.md b/docs/docs/reference/Models/queuedllm.md index 06942e3e..edb980ab 100644 --- a/docs/docs/reference/Models/queuedllm.md +++ b/docs/docs/reference/Models/queuedllm.md @@ -31,6 +31,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "queued", "type": "string"}' required={false} default="queued"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/replicatellm.md b/docs/docs/reference/Models/replicatellm.md index 879459e0..5a474f71 100644 --- a/docs/docs/reference/Models/replicatellm.md +++ b/docs/docs/reference/Models/replicatellm.md @@ -34,6 +34,7 @@ If you don't specify the `model` parameter, it will default to `replicate/llama- <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781", "type": "string"}' required={false} default="replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/textgenui.md b/docs/docs/reference/Models/textgenui.md index bb8dce1d..daede8eb 100644 --- a/docs/docs/reference/Models/textgenui.md +++ b/docs/docs/reference/Models/textgenui.md @@ -32,6 +32,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "text-gen-ui", "type": "string"}' required={false} default="text-gen-ui"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/Models/togetherllm.md b/docs/docs/reference/Models/togetherllm.md index 3718f046..6ddde9dd 100644 --- a/docs/docs/reference/Models/togetherllm.md +++ b/docs/docs/reference/Models/togetherllm.md @@ -34,6 +34,7 @@ config = ContinueConfig( <ClassPropertyRef name='context_length' details='{"title": "Context Length", "description": "The maximum context length of the LLM in tokens, as counted by count_tokens.", "default": 2048, "type": "integer"}' required={false} default="2048"/> <ClassPropertyRef name='unique_id' details='{"title": "Unique Id", "description": "The unique ID of the user.", "type": "string"}' required={false} default=""/> <ClassPropertyRef name='model' details='{"title": "Model", "description": "The name of the model to be used (e.g. gpt-4, codellama)", "default": "togethercomputer/RedPajama-INCITE-7B-Instruct", "type": "string"}' required={false} default="togethercomputer/RedPajama-INCITE-7B-Instruct"/> +<ClassPropertyRef name='max_tokens' details='{"title": "Max Tokens", "description": "The maximum number of tokens to generate.", "default": 1024, "type": "integer"}' required={false} default="1024"/> <ClassPropertyRef name='stop_tokens' details='{"title": "Stop Tokens", "description": "Tokens that will stop the completion.", "type": "array", "items": {"type": "string"}}' required={false} default=""/> <ClassPropertyRef name='timeout' details='{"title": "Timeout", "description": "Set the timeout for each request to the LLM. If you are running a local LLM that takes a while to respond, you might want to set this to avoid timeouts.", "default": 300, "type": "integer"}' required={false} default="300"/> <ClassPropertyRef name='verify_ssl' details='{"title": "Verify Ssl", "description": "Whether to verify SSL certificates for requests.", "type": "boolean"}' required={false} default=""/> diff --git a/docs/docs/reference/config.md b/docs/docs/reference/config.md index 60d5b73e..a160a5c2 100644 --- a/docs/docs/reference/config.md +++ b/docs/docs/reference/config.md @@ -11,7 +11,7 @@ Continue can be deeply customized by editing the `ContinueConfig` object in `~/. <ClassPropertyRef name='steps_on_startup' details='{"title": "Steps On Startup", "description": "Steps that will be automatically run at the beginning of a new session", "default": [], "type": "array", "items": {"$ref": "#/definitions/Step"}}' required={false} default="[]"/> <ClassPropertyRef name='disallowed_steps' details='{"title": "Disallowed Steps", "description": "Steps that are not allowed to be run, and will be skipped if attempted", "default": [], "type": "array", "items": {"type": "string"}}' required={false} default="[]"/> <ClassPropertyRef name='allow_anonymous_telemetry' details='{"title": "Allow Anonymous Telemetry", "description": "If this field is set to True, we will collect anonymous telemetry as described in the documentation page on telemetry. If set to False, we will not collect any data.", "default": true, "type": "boolean"}' required={false} default="True"/> -<ClassPropertyRef name='models' details='{"title": "Models", "description": "Configuration for the models used by Continue. Read more about how to configure models in the documentation.", "default": {"default": {"title": null, "system_message": null, "context_length": 2048, "model": "gpt-4", "stop_tokens": null, "timeout": 300, "verify_ssl": null, "ca_bundle_path": null, "proxy": null, "prompt_templates": {}, "api_key": null, "llm": null, "class_name": "OpenAIFreeTrial"}, "summarize": {"title": null, "system_message": null, "context_length": 2048, "model": "gpt-3.5-turbo", "stop_tokens": null, "timeout": 300, "verify_ssl": null, "ca_bundle_path": null, "proxy": null, "prompt_templates": {}, "api_key": null, "llm": null, "class_name": "OpenAIFreeTrial"}, "edit": null, "chat": null, "saved": []}, "allOf": [{"$ref": "#/definitions/Models"}]}' required={false} default="{'default': {'title': None, 'system_message': None, 'context_length': 2048, 'model': 'gpt-4', 'stop_tokens': None, 'timeout': 300, 'verify_ssl': None, 'ca_bundle_path': None, 'proxy': None, 'prompt_templates': {}, 'api_key': None, 'llm': None, 'class_name': 'OpenAIFreeTrial'}, 'summarize': {'title': None, 'system_message': None, 'context_length': 2048, 'model': 'gpt-3.5-turbo', 'stop_tokens': None, 'timeout': 300, 'verify_ssl': None, 'ca_bundle_path': None, 'proxy': None, 'prompt_templates': {}, 'api_key': None, 'llm': None, 'class_name': 'OpenAIFreeTrial'}, 'edit': None, 'chat': None, 'saved': []}"/> +<ClassPropertyRef name='models' details='{"title": "Models", "description": "Configuration for the models used by Continue. Read more about how to configure models in the documentation.", "default": {"default": {"title": null, "system_message": null, "context_length": 2048, "model": "gpt-4", "max_tokens": 1024, "stop_tokens": null, "timeout": 300, "verify_ssl": null, "ca_bundle_path": null, "proxy": null, "prompt_templates": {}, "api_key": null, "llm": null, "class_name": "OpenAIFreeTrial"}, "summarize": {"title": null, "system_message": null, "context_length": 2048, "model": "gpt-3.5-turbo", "max_tokens": 1024, "stop_tokens": null, "timeout": 300, "verify_ssl": null, "ca_bundle_path": null, "proxy": null, "prompt_templates": {}, "api_key": null, "llm": null, "class_name": "OpenAIFreeTrial"}, "edit": null, "chat": null, "saved": []}, "allOf": [{"$ref": "#/definitions/Models"}]}' required={false} default="{'default': {'title': None, 'system_message': None, 'context_length': 2048, 'model': 'gpt-4', 'max_tokens': 1024, 'stop_tokens': None, 'timeout': 300, 'verify_ssl': None, 'ca_bundle_path': None, 'proxy': None, 'prompt_templates': {}, 'api_key': None, 'llm': None, 'class_name': 'OpenAIFreeTrial'}, 'summarize': {'title': None, 'system_message': None, 'context_length': 2048, 'model': 'gpt-3.5-turbo', 'max_tokens': 1024, 'stop_tokens': None, 'timeout': 300, 'verify_ssl': None, 'ca_bundle_path': None, 'proxy': None, 'prompt_templates': {}, 'api_key': None, 'llm': None, 'class_name': 'OpenAIFreeTrial'}, 'edit': None, 'chat': None, 'saved': []}"/> <ClassPropertyRef name='temperature' details='{"title": "Temperature", "description": "The temperature parameter for sampling from the LLM. Higher temperatures will result in more random output, while lower temperatures will result in more predictable output. This value ranges from 0 to 1.", "default": 0.5, "type": "number"}' required={false} default="0.5"/> <ClassPropertyRef name='custom_commands' details='{"title": "Custom Commands", "description": "An array of custom commands that allow you to reuse prompts. Each has name, description, and prompt properties. When you enter /<name> in the text input, it will act as a shortcut to the prompt.", "default": [{"name": "test", "prompt": "Write a comprehensive set of unit tests for the selected code. It should setup, run tests that check for correctness including important edge cases, and teardown. Ensure that the tests are complete and sophisticated. Give the tests just as chat output, don't edit any file.", "description": "This is an example custom command. Use /config to edit it and create more"}], "type": "array", "items": {"$ref": "#/definitions/CustomCommand"}}' required={false} default="[{'name': 'test', 'prompt': "Write a comprehensive set of unit tests for the selected code. It should setup, run tests that check for correctness including important edge cases, and teardown. Ensure that the tests are complete and sophisticated. Give the tests just as chat output, don't edit any file.", 'description': 'This is an example custom command. Use /config to edit it and create more'}]"/> <ClassPropertyRef name='slash_commands' details='{"title": "Slash Commands", "description": "An array of slash commands that let you map custom Steps to a shortcut.", "default": [], "type": "array", "items": {"$ref": "#/definitions/SlashCommand"}}' required={false} default="[]"/> diff --git a/extension/react-app/src/App.tsx b/extension/react-app/src/App.tsx index bbb1a952..eb5d0164 100644 --- a/extension/react-app/src/App.tsx +++ b/extension/react-app/src/App.tsx @@ -21,6 +21,7 @@ import ErrorPage from "./pages/error"; import SettingsPage from "./pages/settings"; import Models from "./pages/models"; import HelpPage from "./pages/help"; +import ModelConfig from "./pages/modelconfig"; const router = createMemoryRouter([ { @@ -56,6 +57,10 @@ const router = createMemoryRouter([ path: "/help", element: <HelpPage />, }, + { + path: "/modelconfig/:modelName", + element: <ModelConfig />, + }, ], }, ]); diff --git a/extension/react-app/src/components/ModelCard.tsx b/extension/react-app/src/components/ModelCard.tsx index a537c5f4..d1cb3165 100644 --- a/extension/react-app/src/components/ModelCard.tsx +++ b/extension/react-app/src/components/ModelCard.tsx @@ -1,95 +1,70 @@ import React, { useContext } from "react"; import styled from "styled-components"; -import { buttonColor, defaultBorderRadius, lightGray, vscForeground } from "."; -import { setShowDialog } from "../redux/slices/uiStateSlice"; -import { GUIClientContext } from "../App"; -import { useDispatch, useSelector } from "react-redux"; -import { useNavigate } from "react-router-dom"; +import { buttonColor, defaultBorderRadius, lightGray } from "."; +import { useSelector } from "react-redux"; import { RootStore } from "../redux/store"; import { BookOpenIcon } from "@heroicons/react/24/outline"; import HeaderButtonWithText from "./HeaderButtonWithText"; -import ReactDOM from "react-dom"; +import { MODEL_PROVIDER_TAG_COLORS } from "../util/modelData"; -export enum ModelTag { - "Requires API Key" = "Requires API Key", - "Local" = "Local", - "Free" = "Free", - "Open-Source" = "Open-Source", -} - -const MODEL_TAG_COLORS: any = {}; -MODEL_TAG_COLORS[ModelTag["Requires API Key"]] = "#FF0000"; -MODEL_TAG_COLORS[ModelTag["Local"]] = "#00bb00"; -MODEL_TAG_COLORS[ModelTag["Open-Source"]] = "#0033FF"; -MODEL_TAG_COLORS[ModelTag["Free"]] = "#ffff00"; - -export interface ModelInfo { - title: string; - class: string; - args: any; - description: string; - icon?: string; - tags?: ModelTag[]; -} - -const Div = styled.div<{ color: string }>` +const Div = styled.div<{ color: string; disabled: boolean }>` border: 1px solid ${lightGray}; border-radius: ${defaultBorderRadius}; - cursor: pointer; padding: 4px 8px; position: relative; width: 100%; transition: all 0.5s; + ${(props) => + props.disabled + ? ` + opacity: 0.5; + ` + : ` &:hover { - border: 1px solid ${(props) => props.color}; - background-color: ${(props) => props.color}22; + border: 1px solid ${props.color}; + background-color: ${props.color}22; + cursor: pointer; } + `} `; interface ModelCardProps { - modelInfo: ModelInfo; + title: string; + description: string; + tags?: string[]; + refUrl?: string; + icon?: string; + onClick: (e: React.MouseEvent<HTMLDivElement, MouseEvent>) => void; + disabled?: boolean; } function ModelCard(props: ModelCardProps) { - const client = useContext(GUIClientContext); - const dispatch = useDispatch(); - const navigate = useNavigate(); const vscMediaUrl = useSelector( (state: RootStore) => state.config.vscMediaUrl ); return ( <Div + disabled={props.disabled || false} color={buttonColor} - onClick={(e) => { - if ((e.target as any).closest("a")) { - return; - } - client?.addModelForRole( - "*", - props.modelInfo.class, - props.modelInfo.args - ); - dispatch(setShowDialog(false)); - navigate("/"); - }} + onClick={props.disabled ? undefined : (e) => props.onClick(e)} > <div style={{ display: "flex", alignItems: "center" }}> - {vscMediaUrl && ( + {vscMediaUrl && props.icon && ( <img - src={`${vscMediaUrl}/logos/${props.modelInfo.icon}`} + src={`${vscMediaUrl}/logos/${props.icon}`} height="24px" style={{ marginRight: "10px" }} /> )} - <h3>{props.modelInfo.title}</h3> + <h3>{props.title}</h3> </div> - {props.modelInfo.tags?.map((tag) => { + {props.tags?.map((tag) => { return ( <span style={{ - backgroundColor: `${MODEL_TAG_COLORS[tag]}55`, + backgroundColor: `${MODEL_PROVIDER_TAG_COLORS[tag]}55`, color: "white", padding: "2px 4px", borderRadius: defaultBorderRadius, @@ -100,21 +75,23 @@ function ModelCard(props: ModelCardProps) { </span> ); })} - <p>{props.modelInfo.description}</p> + <p>{props.description}</p> - <a - style={{ - position: "absolute", - right: "8px", - top: "8px", - }} - href={`https://continue.dev/docs/reference/Models/${props.modelInfo.class.toLowerCase()}`} - target="_blank" - > - <HeaderButtonWithText text="Read the docs"> - <BookOpenIcon width="1.6em" height="1.6em" /> - </HeaderButtonWithText> - </a> + {props.refUrl && ( + <a + style={{ + position: "absolute", + right: "8px", + top: "8px", + }} + href={props.refUrl} + target="_blank" + > + <HeaderButtonWithText text="Read the docs"> + <BookOpenIcon width="1.6em" height="1.6em" /> + </HeaderButtonWithText> + </a> + )} </Div> ); } diff --git a/extension/react-app/src/pages/modelconfig.tsx b/extension/react-app/src/pages/modelconfig.tsx new file mode 100644 index 00000000..97e2d76c --- /dev/null +++ b/extension/react-app/src/pages/modelconfig.tsx @@ -0,0 +1,261 @@ +import React, { useCallback, useContext, useEffect, useState } from "react"; +import ModelCard from "../components/ModelCard"; +import styled from "styled-components"; +import { ArrowLeftIcon } from "@heroicons/react/24/outline"; +import { + TextInput, + defaultBorderRadius, + lightGray, + vscBackground, +} from "../components"; +import { Form, useNavigate } from "react-router-dom"; +import { useDispatch, useSelector } from "react-redux"; +import { GUIClientContext } from "../App"; +import { setShowDialog } from "../redux/slices/uiStateSlice"; +import { useParams } from "react-router-dom"; +import { + MODEL_INFO, + MODEL_PROVIDER_TAG_COLORS, + ModelInfo, +} from "../util/modelData"; +import { RootStore } from "../redux/store"; +import StyledMarkdownPreview from "../components/StyledMarkdownPreview"; +import { getFontSize } from "../util"; +import { FormProvider, useForm } from "react-hook-form"; + +const GridDiv = styled.div` + display: grid; + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + grid-gap: 2rem; + padding: 1rem; + justify-items: center; + align-items: center; +`; + +const CustomModelButton = styled.div<{ disabled: boolean }>` + border: 1px solid ${lightGray}; + border-radius: ${defaultBorderRadius}; + padding: 4px 8px; + display: flex; + justify-content: center; + align-items: center; + width: 100%; + transition: all 0.5s; + + ${(props) => + props.disabled + ? ` + opacity: 0.5; + ` + : ` + &:hover { + border: 1px solid #be1b55; + background-color: #be1b5522; + cursor: pointer; + } + `} +`; + +function ModelConfig() { + const formMethods = useForm(); + const { modelName } = useParams(); + + const [modelInfo, setModelInfo] = useState<ModelInfo | undefined>(undefined); + + useEffect(() => { + if (modelName) { + setModelInfo(MODEL_INFO[modelName]); + } + }, [modelName]); + + const client = useContext(GUIClientContext); + const dispatch = useDispatch(); + const navigate = useNavigate(); + const vscMediaUrl = useSelector( + (state: RootStore) => state.config.vscMediaUrl + ); + + const disableModelCards = useCallback(() => { + return ( + modelInfo?.collectInputFor?.some((d) => { + if (!d.required) return false; + const val = formMethods.watch(d.key); + return ( + typeof val === "undefined" || (typeof val === "string" && val === "") + ); + }) || false + ); + }, [modelInfo, formMethods]); + + return ( + <FormProvider {...formMethods}> + <div className="overflow-y-scroll"> + <div + className="items-center flex m-0 p-0 sticky top-0" + style={{ + borderBottom: `0.5px solid ${lightGray}`, + backgroundColor: vscBackground, + zIndex: 2, + }} + > + <ArrowLeftIcon + width="1.2em" + height="1.2em" + onClick={() => navigate("/models")} + className="inline-block ml-4 cursor-pointer" + /> + <h3 className="text-lg font-bold m-2 inline-block"> + Configure Model + </h3> + </div> + + <div className="px-2"> + <div style={{ display: "flex", alignItems: "center" }}> + {vscMediaUrl && ( + <img + src={`${vscMediaUrl}/logos/${modelInfo?.icon}`} + height="24px" + style={{ marginRight: "10px" }} + /> + )} + <h2>{modelInfo?.title}</h2> + </div> + {modelInfo?.tags?.map((tag) => { + return ( + <span + style={{ + backgroundColor: `${MODEL_PROVIDER_TAG_COLORS[tag]}55`, + color: "white", + padding: "2px 4px", + borderRadius: defaultBorderRadius, + marginRight: "4px", + }} + > + {tag} + </span> + ); + })} + <StyledMarkdownPreview + className="mt-2" + fontSize={getFontSize()} + source={modelInfo?.longDescription || modelInfo?.description || ""} + wrapperElement={{ + "data-color-mode": "dark", + }} + maxHeight={200} + /> + <br /> + + {(modelInfo?.collectInputFor?.filter((d) => d.required).length || 0) > + 0 && ( + <> + <h3 className="mb-2">Enter required parameters</h3> + + {modelInfo?.collectInputFor?.map((d) => { + return ( + <div> + <label htmlFor={d.key}>{d.key}</label> + <TextInput + id={d.key} + className="border-2 border-gray-200 rounded-md p-2 m-2" + placeholder={d.key} + defaultValue={d.defaultValue} + {...formMethods.register(d.key, { + required: true, + })} + /> + </div> + ); + })} + </> + )} + + {(modelInfo?.collectInputFor?.filter((d) => !d.required).length || + 0) > 0 && ( + <details> + <summary className="mb-2"> + <b>Advanced (optional)</b> + </summary> + + {modelInfo?.collectInputFor?.map((d) => { + if (d.required) return null; + return ( + <div> + <label htmlFor={d.key}>{d.key}</label> + <TextInput + id={d.key} + className="border-2 border-gray-200 rounded-md p-2 m-2" + placeholder={d.key} + defaultValue={d.defaultValue} + {...formMethods.register(d.key, { + required: false, + })} + /> + </div> + ); + })} + </details> + )} + + <h3 className="mb-2">Select a model preset</h3> + </div> + <GridDiv> + {modelInfo?.packages.map((pkg) => { + return ( + <ModelCard + disabled={disableModelCards()} + title={pkg.title} + description={pkg.description} + tags={pkg.tags} + refUrl={pkg.refUrl} + icon={pkg.icon || modelInfo.icon} + onClick={(e) => { + if (disableModelCards()) return; + const formParams: any = {}; + for (const d of modelInfo.collectInputFor || []) { + formParams[d.key] = + d.inputType === "text" + ? formMethods.watch(d.key) + : parseInt(formMethods.watch(d.key)); + } + + client?.addModelForRole("*", modelInfo.class, { + ...pkg.params, + ...modelInfo.params, + ...formParams, + }); + navigate("/"); + }} + /> + ); + })} + + <CustomModelButton + disabled={disableModelCards()} + onClick={(e) => { + if (!modelInfo || disableModelCards()) return; + const formParams: any = {}; + for (const d of modelInfo.collectInputFor || []) { + formParams[d.key] = + d.inputType === "text" + ? formMethods.watch(d.key) + : parseInt(formMethods.watch(d.key)); + } + + client?.addModelForRole("*", modelInfo.class, { + ...modelInfo.packages[0]?.params, + ...modelInfo.params, + ...formParams, + }); + navigate("/"); + }} + > + <h3 className="text-center my-2">Configure Model in config.py</h3> + </CustomModelButton> + </GridDiv> + </div> + </FormProvider> + ); +} + +export default ModelConfig; diff --git a/extension/react-app/src/pages/models.tsx b/extension/react-app/src/pages/models.tsx index c20d820c..a9a97a13 100644 --- a/extension/react-app/src/pages/models.tsx +++ b/extension/react-app/src/pages/models.tsx @@ -1,131 +1,13 @@ -import React from "react"; -import ModelCard, { ModelInfo, ModelTag } from "../components/ModelCard"; +import React, { useContext } from "react"; +import ModelCard from "../components/ModelCard"; import styled from "styled-components"; import { ArrowLeftIcon } from "@heroicons/react/24/outline"; import { lightGray, vscBackground } from "../components"; import { useNavigate } from "react-router-dom"; - -const MODEL_INFO: ModelInfo[] = [ - { - title: "OpenAI", - class: "OpenAI", - description: "Use gpt-4, gpt-3.5-turbo, or any other OpenAI model", - args: { - model: "gpt-4", - api_key: "", - title: "OpenAI", - }, - icon: "openai.svg", - tags: [ModelTag["Requires API Key"]], - }, - { - title: "Anthropic", - class: "AnthropicLLM", - description: - "Claude-2 is a highly capable model with a 100k context length", - args: { - model: "claude-2", - api_key: "<ANTHROPIC_API_KEY>", - title: "Anthropic", - }, - icon: "anthropic.png", - tags: [ModelTag["Requires API Key"]], - }, - { - title: "Ollama", - class: "Ollama", - description: - "One of the fastest ways to get started with local models on Mac or Linux", - args: { - model: "codellama", - title: "Ollama", - }, - icon: "ollama.png", - tags: [ModelTag["Local"], ModelTag["Open-Source"]], - }, - { - title: "TogetherAI", - class: "TogetherLLM", - description: - "Use the TogetherAI API for extremely fast streaming of open-source models", - args: { - model: "togethercomputer/CodeLlama-13b-Instruct", - api_key: "<TOGETHER_API_KEY>", - title: "TogetherAI", - }, - icon: "together.png", - tags: [ModelTag["Requires API Key"], ModelTag["Open-Source"]], - }, - { - title: "LM Studio", - class: "GGML", - description: - "One of the fastest ways to get started with local models on Mac or Windows", - args: { - server_url: "http://localhost:1234", - title: "LM Studio", - }, - icon: "lmstudio.png", - tags: [ModelTag["Local"], ModelTag["Open-Source"]], - }, - { - title: "Replicate", - class: "ReplicateLLM", - description: "Use the Replicate API to run open-source models", - args: { - model: - "replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781", - api_key: "<REPLICATE_API_KEY>", - title: "Replicate", - }, - icon: "replicate.png", - tags: [ModelTag["Requires API Key"], ModelTag["Open-Source"]], - }, - { - title: "llama.cpp", - class: "LlamaCpp", - description: "If you are running the llama.cpp server from source", - args: { - title: "llama.cpp", - }, - icon: "llamacpp.png", - tags: [ModelTag.Local, ModelTag["Open-Source"]], - }, - { - title: "HuggingFace TGI", - class: "HuggingFaceTGI", - description: - "HuggingFace Text Generation Inference is an advanced, highly performant option for serving open-source models to multiple people", - args: { - title: "HuggingFace TGI", - }, - icon: "hf.png", - tags: [ModelTag.Local, ModelTag["Open-Source"]], - }, - { - title: "Other OpenAI-compatible API", - class: "GGML", - description: - "If you are using any other OpenAI-compatible API, for example text-gen-webui, FastChat, LocalAI, or llama-cpp-python, you can simply enter your server URL", - args: { - server_url: "<SERVER_URL>", - }, - icon: "openai.svg", - tags: [ModelTag.Local, ModelTag["Open-Source"]], - }, - { - title: "GPT-4 limited free trial", - class: "OpenAIFreeTrial", - description: - "New users can try out Continue with GPT-4 using a proxy server that securely makes calls to OpenAI using our API key", - args: { - model: "gpt-4", - title: "GPT-4 Free Trial", - }, - icon: "openai.svg", - tags: [ModelTag.Free], - }, -]; +import { useDispatch } from "react-redux"; +import { GUIClientContext } from "../App"; +import { setShowDialog } from "../redux/slices/uiStateSlice"; +import { MODEL_INFO } from "../util/modelData"; const GridDiv = styled.div` display: grid; @@ -138,6 +20,8 @@ const GridDiv = styled.div` function Models() { const navigate = useNavigate(); + const client = useContext(GUIClientContext); + const dispatch = useDispatch(); return ( <div className="overflow-y-scroll"> <div @@ -154,11 +38,25 @@ function Models() { onClick={() => navigate("/")} className="inline-block ml-4 cursor-pointer" /> - <h3 className="text-lg font-bold m-2 inline-block">Add a new model</h3> + <h3 className="text-lg font-bold m-2 inline-block"> + Select LLM Provider + </h3> </div> <GridDiv> - {MODEL_INFO.map((model) => ( - <ModelCard modelInfo={model} /> + {Object.entries(MODEL_INFO).map(([name, modelInfo]) => ( + <ModelCard + title={modelInfo.title} + description={modelInfo.description} + tags={modelInfo.tags} + icon={modelInfo.icon} + refUrl={`https://continue.dev/docs/reference/Models/${modelInfo.class.toLowerCase()}`} + onClick={(e) => { + if ((e.target as any).closest("a")) { + return; + } + navigate(`/modelconfig/${name}`); + }} + /> ))} </GridDiv> </div> diff --git a/extension/react-app/src/util/modelData.ts b/extension/react-app/src/util/modelData.ts new file mode 100644 index 00000000..615cbb79 --- /dev/null +++ b/extension/react-app/src/util/modelData.ts @@ -0,0 +1,441 @@ +export enum ModelProviderTag { + "Requires API Key" = "Requires API Key", + "Local" = "Local", + "Free" = "Free", + "Open-Source" = "Open-Source", +} + +export const MODEL_PROVIDER_TAG_COLORS: any = {}; +MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Requires API Key"]] = "#FF0000"; +MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Local"]] = "#00bb00"; +MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Open-Source"]] = "#0033FF"; +MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Free"]] = "#ffff00"; + +export enum CollectInputType { + "text" = "text", + "number" = "number", +} + +export interface InputDescriptor { + inputType: CollectInputType; + key: string; + label: string; + placeholder?: string; + defaultValue?: string | number; + min?: number; + max?: number; + step?: number; + options?: string[]; + required?: boolean; + description?: string; + [key: string]: any; +} + +const contextLengthInput: InputDescriptor = { + inputType: CollectInputType.number, + key: "context_length", + label: "Context Length", + defaultValue: 2048, + required: false, +}; + +const serverUrlInput = { + inputType: CollectInputType.text, + key: "server_url", + label: "Server URL", + placeholder: "e.g. http://localhost:8080", + required: false, +}; + +export interface ModelInfo { + title: string; + class: string; + description: string; + longDescription?: string; + icon?: string; + tags?: ModelProviderTag[]; + packages: ModelPackage[]; + params?: any; + collectInputFor?: InputDescriptor[]; +} + +export interface ModelPackage { + collectInputFor?: InputDescriptor[]; + description: string; + title: string; + refUrl?: string; + tags?: ModelProviderTag[]; + icon?: string; + params: { + model: string; + template_messages?: string; + context_length: number; + stop_tokens?: string[]; + prompt_templates?: any; + replace?: [string, string][]; + [key: string]: any; + }; +} + +const codeLlama7bInstruct: ModelPackage = { + title: "CodeLlama-7b-Instruct", + description: "A 7b parameter model tuned for code generation", + refUrl: "", + params: { + title: "CodeLlama-7b-Instruct", + model: "codellama:7b-instruct", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; +const codeLlama13bInstruct: ModelPackage = { + title: "CodeLlama-13b-Instruct", + description: "A 13b parameter model tuned for code generation", + refUrl: "", + params: { + title: "CodeLlama13b-Instruct", + model: "codellama13b-instruct", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; +const codeLlama34bInstruct: ModelPackage = { + title: "CodeLlama-34b-Instruct", + description: "A 34b parameter model tuned for code generation", + refUrl: "", + params: { + title: "CodeLlama-34b-Instruct", + model: "codellama:34b-instruct", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; + +const llama2Chat7b: ModelPackage = { + title: "Llama2-7b-Chat", + description: "A 7b parameter model fine-tuned for chat", + refUrl: "", + params: { + title: "Llama2-7b-Chat", + model: "llama2:7b-chat", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; +const llama2Chat13b: ModelPackage = { + title: "Llama2-13b-Chat", + description: "A 13b parameter model fine-tuned for chat", + refUrl: "", + params: { + title: "Llama2-13b-Chat", + model: "llama2:13b-chat", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; +const llama2Chat34b: ModelPackage = { + title: "Llama2-34b-Chat", + description: "A 34b parameter model fine-tuned for chat", + refUrl: "", + params: { + title: "Llama2-34b-Chat", + model: "llama2:34b-chat", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; + +const codeLlamaPackages = [ + codeLlama7bInstruct, + codeLlama13bInstruct, + codeLlama34bInstruct, +]; + +const llama2Packages = [llama2Chat7b, llama2Chat13b, llama2Chat34b]; +const llama2FamilyPackage = { + title: "Llama2 or CodeLlama", + description: "Any model using the Llama2 or CodeLlama chat template", + params: { + model: "llama2", + context_length: 2048, + template_messages: "llama2_template_messages", + }, + icon: "meta.svg", +}; + +const gpt4: ModelPackage = { + title: "GPT-4", + description: "The latest model from OpenAI", + params: { + model: "gpt-4", + context_length: 8096, + api_key: "", + title: "GPT-4", + }, +}; + +const gpt35turbo: ModelPackage = { + title: "GPT-3.5-Turbo", + description: + "A faster, cheaper OpenAI model with slightly lower capabilities", + params: { + model: "gpt-3.5-turbo", + context_length: 8096, + title: "GPT-3.5-Turbo", + api_key: "", + }, +}; + +export const MODEL_INFO: { [key: string]: ModelInfo } = { + openai: { + title: "OpenAI", + class: "OpenAI", + description: "Use gpt-4, gpt-3.5-turbo, or any other OpenAI model", + longDescription: + "Use gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.", + icon: "openai.svg", + tags: [ModelProviderTag["Requires API Key"]], + packages: [gpt4, gpt35turbo], + collectInputFor: [ + { + inputType: CollectInputType.text, + key: "api_key", + label: "API Key", + placeholder: "Enter your OpenAI API key", + required: true, + }, + ], + }, + anthropic: { + title: "Anthropic", + class: "AnthropicLLM", + description: + "Claude-2 is a highly capable model with a 100k context length", + icon: "anthropic.png", + tags: [ModelProviderTag["Requires API Key"]], + longDescription: + "To get started with Anthropic models, you first need to sign up for the open beta [here](https://claude.ai/login) to obtain an API key.", + collectInputFor: [ + { + inputType: CollectInputType.text, + key: "api_key", + label: "API Key", + placeholder: "Enter your Anthropic API key", + required: true, + }, + ], + packages: [ + { + title: "Claude-2", + description: "A highly capable model with a 100k context length", + params: { + model: "claude-2", + context_length: 100000, + title: "Claude-2", + }, + }, + ], + }, + ollama: { + title: "Ollama", + class: "Ollama", + description: + "One of the fastest ways to get started with local models on Mac or Linux", + longDescription: + 'To get started with Ollama, follow these steps:\n1. Download from [ollama.ai](https://ollama.ai/) and open the application\n2. Open a terminal and run `ollama pull <MODEL_NAME>`. Example model names are `codellama:7b-instruct` or `llama2:7b-text`. You can find the full list [here](https://ollama.ai/library).\n3. Make sure that the model name used in step 2 is the same as the one in config.py (e.g. `model="codellama:7b-instruct"`)\n4. Once the model has finished downloading, you can start asking questions through Continue.', + icon: "ollama.png", + tags: [ModelProviderTag["Local"], ModelProviderTag["Open-Source"]], + packages: [ + ...codeLlamaPackages.map((p) => ({ + ...p, + refUrl: "https://ollama.ai/library/codellama", + })), + ...llama2Packages.map((p) => ({ + ...p, + refUrl: "https://ollama.ai/library/llama2", + })), + ], + collectInputFor: [contextLengthInput], + }, + together: { + title: "TogetherAI", + class: "TogetherLLM", + description: + "Use the TogetherAI API for extremely fast streaming of open-source models", + icon: "together.png", + longDescription: `Together is a hosted service that provides extremely fast streaming of open-source language models. To get started with Together:\n1. Obtain an API key from [here](https://together.ai)\n2. Paste below\n3. Select a model preset`, + tags: [ + ModelProviderTag["Requires API Key"], + ModelProviderTag["Open-Source"], + ], + params: { + api_key: "", + }, + collectInputFor: [ + { + inputType: CollectInputType.text, + key: "api_key", + label: "API Key", + placeholder: "Enter your TogetherAI API key", + required: true, + }, + ], + packages: [ + ...codeLlamaPackages.map((p) => { + return { + ...p, + params: { + ...p.params, + model: + "togethercomputer/" + + p.params.model.replace("llama2", "llama-2").replace(":", "-"), + }, + }; + }), + ...llama2Packages.map((p) => { + return { + ...p, + params: { + ...p.params, + model: + "togethercomputer/" + + p.params.model + .replace("codellama", "CodeLlama") + .replace(":", "-") + .replace("instruct", "Instruct"), + }, + }; + }), + ].map((p) => { + p.params.context_length = 4096; + return p; + }), + }, + lmstudio: { + title: "LM Studio", + class: "GGML", + description: + "One of the fastest ways to get started with local models on Mac or Windows", + longDescription: + "LMStudio provides a professional and well-designed GUI for exploring, configuring, and serving LLMs. It is available on both Mac and Windows. To get started:\n1. Download from [lmstudio.ai](https://lmstudio.ai/) and open the application\n2. Search for and download the desired model from the home screen of LMStudio.\n3. In the left-bar, click the '<->' icon to open the Local Inference Server and press 'Start Server'.\n4. Once your model is loaded and the server has started, you can begin using Continue.", + icon: "lmstudio.png", + tags: [ModelProviderTag["Local"], ModelProviderTag["Open-Source"]], + params: { + server_url: "http://localhost:1234", + }, + packages: [llama2FamilyPackage], + collectInputFor: [contextLengthInput], + }, + replicate: { + title: "Replicate", + class: "ReplicateLLM", + description: "Use the Replicate API to run open-source models", + longDescription: `Replicate is a hosted service that makes it easy to run ML models. To get started with Replicate:\n1. Obtain an API key from [here](https://replicate.com)\n2. Paste below\n3. Select a model preset`, + params: { + api_key: "", + }, + collectInputFor: [ + { + inputType: CollectInputType.text, + key: "api_key", + label: "API Key", + placeholder: "Enter your Replicate API key", + required: true, + }, + ], + icon: "replicate.png", + tags: [ + ModelProviderTag["Requires API Key"], + ModelProviderTag["Open-Source"], + ], + packages: [...codeLlamaPackages, ...llama2Packages].map((p) => { + return { + ...p, + params: { + ...p.params, + model: + "meta/" + + p.params.model.replace(":", "-").replace("llama2", "llama-2"), + }, + }; + }), + }, + llamacpp: { + title: "llama.cpp", + class: "LlamaCpp", + description: "If you are running the llama.cpp server from source", + longDescription: `llama.cpp comes with a [built-in server](https://github.com/ggerganov/llama.cpp/tree/master/examples/server#llamacppexampleserver) that can be run from source. To do this: + +1. Clone the repository with \`git clone https://github.com/ggerganov/llama.cpp\`. +2. \`cd llama.cpp\` +3. Download the model you'd like to use and place it in the \`llama.cpp/models\` directory (the best place to find models is [The Bloke on HuggingFace](https://huggingface.co/TheBloke)) +4. Run the llama.cpp server with the command below (replacing with the model you downloaded): + +\`\`\`shell +.\\server.exe -c 4096 --host 0.0.0.0 -t 16 --mlock -m models/codellama-7b-instruct.Q8_0.gguf +\`\`\` + +After it's up and running, you can start using Continue.`, + icon: "llamacpp.png", + tags: [ModelProviderTag.Local, ModelProviderTag["Open-Source"]], + packages: [llama2FamilyPackage], + collectInputFor: [contextLengthInput], + }, + hftgi: { + title: "HuggingFace TGI", + class: "HuggingFaceTGI", + description: + "HuggingFace Text Generation Inference is an advanced, highly-performant option for serving open-source models to multiple people", + longDescription: + "HuggingFace Text Generation Inference is an advanced, highly-performant option for serving open-source models to multiple people. To get started, follow the [Quick Tour](https://huggingface.co/docs/text-generation-inference/quicktour) on their website to set up the Docker container. Make sure to enter the server URL below that corresponds to the host and port you set up for the Docker container.", + icon: "hf.png", + tags: [ModelProviderTag.Local, ModelProviderTag["Open-Source"]], + packages: [llama2FamilyPackage], + collectInputFor: [ + contextLengthInput, + { ...serverUrlInput, defaultValue: "http://localhost:8080" }, + ], + }, + ggml: { + title: "Other OpenAI-compatible API", + class: "GGML", + description: + "If you are using any other OpenAI-compatible API, for example text-gen-webui, FastChat, LocalAI, or llama-cpp-python, you can simply enter your server URL", + longDescription: `If you are using any other OpenAI-compatible API, you can simply enter your server URL. If you still need to set up your model server, you can follow a guide below: + +- [text-gen-webui](https://github.com/oobabooga/text-generation-webui/tree/main/extensions/openai#setup--installation) +- [LocalAI](https://localai.io/basics/getting_started/) +- [llama-cpp-python](https://github.com/continuedev/ggml-server-example) +- [FastChat](https://github.com/lm-sys/FastChat/blob/main/docs/openai_api.md)`, + params: { + server_url: "", + }, + collectInputFor: [ + { + ...serverUrlInput, + defaultValue: "http://localhost:8000", + }, + contextLengthInput, + ], + icon: "openai.svg", + tags: [ModelProviderTag.Local, ModelProviderTag["Open-Source"]], + packages: [llama2FamilyPackage], + }, + freetrial: { + title: "GPT-4 limited free trial", + class: "OpenAIFreeTrial", + description: + "New users can try out Continue for free using a proxy server that securely makes calls to OpenAI using our API key", + longDescription: + 'New users can try out Continue for free using a proxy server that securely makes calls to OpenAI using our API key. If you are ready to use your own API key or have used all 250 free uses, you can enter your API key in config.py where it says `api_key=""` or select another model provider.', + icon: "openai.svg", + tags: [ModelProviderTag.Free], + packages: [gpt4, gpt35turbo], + }, +}; |