diff options
| author | Ty Dunn <ty@tydunn.com> | 2023-07-17 19:25:32 -0500 |
|---|---|---|
| committer | Ty Dunn <ty@tydunn.com> | 2023-07-17 19:25:32 -0500 |
| commit | 0ec688123abb8dd63fe5ae8c191a69746829884a (patch) | |
| tree | d98a78480e3a34b43f67cd1fffd5d5d5fdd09406 /continuedev/src/continuedev/libs/util/count_tokens.py | |
| parent | 043bdade5605a1c509a9f1927ebbe54db7d900f4 (diff) | |
| parent | 8498ab7fd2945703f4ad59dabf51cb851db4f64d (diff) | |
| download | sncontinue-0ec688123abb8dd63fe5ae8c191a69746829884a.tar.gz sncontinue-0ec688123abb8dd63fe5ae8c191a69746829884a.tar.bz2 sncontinue-0ec688123abb8dd63fe5ae8c191a69746829884a.zip | |
Merge branch 'main' of github.com:continuedev/continue
Diffstat (limited to 'continuedev/src/continuedev/libs/util/count_tokens.py')
| -rw-r--r-- | continuedev/src/continuedev/libs/util/count_tokens.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/continuedev/src/continuedev/libs/util/count_tokens.py b/continuedev/src/continuedev/libs/util/count_tokens.py index 1ca98fe6..1d5d6729 100644 --- a/continuedev/src/continuedev/libs/util/count_tokens.py +++ b/continuedev/src/continuedev/libs/util/count_tokens.py @@ -6,6 +6,7 @@ import tiktoken aliases = { "ggml": "gpt-3.5-turbo", + "claude-2": "gpt-3.5-turbo", } DEFAULT_MAX_TOKENS = 2048 MAX_TOKENS_FOR_MODEL = { @@ -13,7 +14,8 @@ MAX_TOKENS_FOR_MODEL = { "gpt-3.5-turbo-0613": 4096, "gpt-3.5-turbo-16k": 16384, "gpt-4": 8192, - "ggml": 2048 + "ggml": 2048, + "claude-2": 100000 } CHAT_MODELS = { "gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4", "gpt-3.5-turbo-0613" |