import _ from "lodash"; function updatedObj(old: any, pathToValue: { [key: string]: any }) { const newObject = _.cloneDeep(old); for (const key in pathToValue) { if (typeof pathToValue[key] === "function") { _.updateWith(newObject, key, pathToValue[key]); } else { _.updateWith(newObject, key, (__) => pathToValue[key]); } } return newObject; } export enum ModelProviderTag { "Requires API Key" = "Requires API Key", "Local" = "Local", "Free" = "Free", "Open-Source" = "Open-Source", } export const MODEL_PROVIDER_TAG_COLORS: any = {}; MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Requires API Key"]] = "#FF0000"; MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Local"]] = "#00bb00"; MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Open-Source"]] = "#0033FF"; MODEL_PROVIDER_TAG_COLORS[ModelProviderTag["Free"]] = "#ffff00"; export enum CollectInputType { "text" = "text", "number" = "number", "range" = "range", } export interface InputDescriptor { inputType: CollectInputType; key: string; label: string; placeholder?: string; defaultValue?: string | number; min?: number; max?: number; step?: number; options?: string[]; required?: boolean; description?: string; [key: string]: any; } const contextLengthInput: InputDescriptor = { inputType: CollectInputType.number, key: "context_length", label: "Context Length", defaultValue: 2048, required: false, }; const temperatureInput: InputDescriptor = { inputType: CollectInputType.number, key: "temperature", label: "Temperature", defaultValue: undefined, required: false, min: 0.0, max: 1.0, step: 0.01, }; const topPInput: InputDescriptor = { inputType: CollectInputType.number, key: "top_p", label: "Top-P", defaultValue: undefined, required: false, min: 0, max: 1, step: 0.01, }; const topKInput: InputDescriptor = { inputType: CollectInputType.number, key: "top_k", label: "Top-K", defaultValue: undefined, required: false, min: 0, max: 1, step: 0.01, }; const presencePenaltyInput: InputDescriptor = { inputType: CollectInputType.number, key: "presence_penalty", label: "Presence Penalty", defaultValue: undefined, required: false, min: 0, max: 1, step: 0.01, }; const FrequencyPenaltyInput: InputDescriptor = { inputType: CollectInputType.number, key: "frequency_penalty", label: "Frequency Penalty", defaultValue: undefined, required: false, min: 0, max: 1, step: 0.01, }; const completionParamsInputs = [ contextLengthInput, temperatureInput, topKInput, topPInput, presencePenaltyInput, FrequencyPenaltyInput, ]; const serverUrlInput = { inputType: CollectInputType.text, key: "server_url", label: "Server URL", placeholder: "e.g. http://localhost:8080", required: false, }; export interface ModelInfo { title: string; class: string; description: string; longDescription?: string; icon?: string; tags?: ModelProviderTag[]; packages: ModelPackage[]; params?: any; collectInputFor?: InputDescriptor[]; } // A dimension is like parameter count - 7b, 13b, 34b, etc. // You would set options to the field that should be changed for that option in the params field of ModelPackage export interface PackageDimension { name: string; description: string; options: { [key: string]: { [key: string]: any } }; } export interface ModelPackage { collectInputFor?: InputDescriptor[]; description: string; title: string; refUrl?: string; tags?: ModelProviderTag[]; icon?: string; params: { model: string; template_messages?: string; context_length: number; stop_tokens?: string[]; prompt_templates?: any; replace?: [string, string][]; [key: string]: any; }; dimensions?: PackageDimension[]; } enum ChatTemplates { "alpaca" = "template_alpaca_messages", "llama2" = "llama2_template_messages", "sqlcoder" = "sqlcoder_template_messages", } const codeLlamaInstruct: ModelPackage = { title: "CodeLlama Instruct", description: "A model from Meta, fine-tuned for code generation and conversation", refUrl: "", params: { title: "CodeLlama-7b-Instruct", model: "codellama:7b-instruct", context_length: 2048, template_messages: ChatTemplates.llama2, }, icon: "meta.svg", dimensions: [ { name: "Parameter Count", description: "The number of parameters in the model", options: { "7b": { model: "codellama:7b-instruct", title: "CodeLlama-7b-Instruct", }, "13b": { model: "codellama:13b-instruct", title: "CodeLlama-13b-Instruct", }, "34b": { model: "codellama:34b-instruct", title: "CodeLlama-34b-Instruct", }, }, }, ], }; const llama2Chat: ModelPackage = { title: "Llama2 Chat", description: "The latest Llama model from Meta, fine-tuned for chat", refUrl: "", params: { title: "Llama2-7b-Chat", model: "llama2:7b-chat", context_length: 2048, template_messages: ChatTemplates.llama2, }, icon: "meta.svg", dimensions: [ { name: "Parameter Count", description: "The number of parameters in the model", options: { "7b": { model: "llama2:7b-chat", title: "Llama2-7b-Chat", }, "13b": { model: "llama2:13b-chat", title: "Llama2-13b-Chat", }, "34b": { model: "llama2:34b-chat", title: "Llama2-34b-Chat", }, }, }, ], }; const wizardCoder: ModelPackage = { title: "WizardCoder", description: "A CodeLlama-based code generation model from WizardLM, focused on Python", refUrl: "", params: { title: "WizardCoder-7b-Python", model: "wizardcoder:7b-python", context_length: 2048, template_messages: ChatTemplates.alpaca, }, icon: "wizardlm.png", dimensions: [ { name: "Parameter Count", description: "The number of parameters in the model", options: { "7b": { model: "wizardcoder:7b-python", title: "WizardCoder-7b-Python", }, "13b": { model: "wizardcoder:13b-python", title: "WizardCoder-13b-Python", }, "34b": { model: "wizardcoder:34b-python", title: "WizardCoder-34b-Python", }, }, }, ], }; const phindCodeLlama: ModelPackage = { title: "Phind CodeLlama (34b)", description: "A finetune of CodeLlama by Phind", params: { title: "Phind CodeLlama", model: "phind-codellama", context_length: 2048, template_messages: ChatTemplates.llama2, }, }; const mistral: ModelPackage = { title: "Mistral (7b)", description: "A 7b parameter base model created by Mistral AI, very competent for code generation and other tasks", params: { title: "Mistral", model: "mistral", context_length: 2048, template_messages: ChatTemplates.llama2, }, icon: "mistral.png", }; const sqlCoder: ModelPackage = { title: "SQLCoder", description: "A finetune of StarCoder by Defog.ai, focused specifically on SQL", params: { title: "SQLCoder", model: "sqlcoder", context_length: 2048, template_messages: ChatTemplates.sqlcoder, }, dimensions: [ { name: "Parameter Count", description: "The number of parameters in the model", options: { "7b": { model: "sqlcoder:7b", title: "SQLCoder-7b", }, "13b": { model: "sqlcoder:15b", title: "SQLCoder-15b", }, }, }, ], }; const codeup: ModelPackage = { title: "CodeUp (13b)", description: "An open-source coding model based on Llama2", params: { title: "CodeUp", model: "codeup", context_length: 2048, template_messages: ChatTemplates.llama2, }, }; const osModels = [ codeLlamaInstruct, llama2Chat, wizardCoder, phindCodeLlama, sqlCoder, mistral, codeup, ]; export const MODEL_INFO: { [key: string]: ModelInfo } = { ollama: { title: "Ollama", class: "Ollama", description: "One of the fastest ways to get started with local models on Mac or Linux", longDescription: 'To get started with Ollama, follow these steps:\n1. Download from [ollama.ai](https://ollama.ai/) and open the application\n2. Open a terminal and run `ollama pull `. Example model names are `codellama:7b-instruct` or `llama2:7b-text`. You can find the full list [here](https://ollama.ai/library).\n3. Make sure that the model name used in step 2 is the same as the one in config.py (e.g. `model="codellama:7b-instruct"`)\n4. Once the model has finished downloading, you can start asking questions through Continue.', icon: "ollama.png", tags: [ModelProviderTag["Local"], ModelProviderTag["Open-Source"]], packages: osModels, collectInputFor: [...completionParamsInputs], }, llamacpp: { title: "llama.cpp", class: "LlamaCpp", description: "If you are running the llama.cpp server from source", longDescription: `llama.cpp comes with a [built-in server](https://github.com/ggerganov/llama.cpp/tree/master/examples/server#llamacppexampleserver) that can be run from source. To do this: 1. Clone the repository with \`git clone https://github.com/ggerganov/llama.cpp\`. 2. \`cd llama.cpp\` 3. Download the model you'd like to use and place it in the \`llama.cpp/models\` directory (the best place to find models is [The Bloke on HuggingFace](https://huggingface.co/TheBloke)) 4. Run the llama.cpp server with the command below (replacing with the model you downloaded): \`\`\`shell .\\server.exe -c 4096 --host 0.0.0.0 -t 16 --mlock -m models/codellama-7b-instruct.Q8_0.gguf \`\`\` After it's up and running, you can start using Continue.`, icon: "llamacpp.png", tags: [ModelProviderTag.Local, ModelProviderTag["Open-Source"]], packages: osModels, collectInputFor: [...completionParamsInputs], } };