diff options
| author | Nate Sesti <sestinj@gmail.com> | 2023-09-02 12:15:34 -0700 |
|---|---|---|
| committer | Nate Sesti <sestinj@gmail.com> | 2023-09-02 12:15:34 -0700 |
| commit | d04eec7ee97319a6bcc48d289cd6eb3e0d9b8e19 (patch) | |
| tree | 6b848dfbfb09f8c6bf8c777607f1f085845fb9f4 /continuedev | |
| parent | aadaf4fbac9d37c5cd2249030e90de054f7c832f (diff) | |
| download | sncontinue-d04eec7ee97319a6bcc48d289cd6eb3e0d9b8e19.tar.gz sncontinue-d04eec7ee97319a6bcc48d289cd6eb3e0d9b8e19.tar.bz2 sncontinue-d04eec7ee97319a6bcc48d289cd6eb3e0d9b8e19.zip | |
feat: :sparkles: set session timeout on GGML requests
Diffstat (limited to 'continuedev')
| -rw-r--r-- | continuedev/src/continuedev/libs/llm/ggml.py | 11 |
1 files changed, 8 insertions, 3 deletions
diff --git a/continuedev/src/continuedev/libs/llm/ggml.py b/continuedev/src/continuedev/libs/llm/ggml.py index 06fb2658..b2ebaf7e 100644 --- a/continuedev/src/continuedev/libs/llm/ggml.py +++ b/continuedev/src/continuedev/libs/llm/ggml.py @@ -13,6 +13,8 @@ class GGML(LLM): verify_ssl: Optional[bool] = None model: str = "ggml" + timeout: int = 300 + class Config: arbitrary_types_allowed = True @@ -35,7 +37,8 @@ class GGML(LLM): self.write_log(f"Prompt: \n\n{format_chat_messages(messages)}") completion = "" async with aiohttp.ClientSession( - connector=aiohttp.TCPConnector(verify_ssl=self.verify_ssl) + connector=aiohttp.TCPConnector(verify_ssl=self.verify_ssl), + timeout=self.timeout ) as client_session: async with client_session.post( f"{self.server_url}/v1/completions", json={"messages": messages, **args} @@ -68,7 +71,8 @@ class GGML(LLM): async def generator(): async with aiohttp.ClientSession( - connector=aiohttp.TCPConnector(verify_ssl=self.verify_ssl) + connector=aiohttp.TCPConnector(verify_ssl=self.verify_ssl), + timeout=self.timeout ) as client_session: async with client_session.post( f"{self.server_url}/v1/chat/completions", @@ -116,7 +120,8 @@ class GGML(LLM): self.write_log(f"Prompt: \n\n{prompt}") async with aiohttp.ClientSession( - connector=aiohttp.TCPConnector(verify_ssl=self.verify_ssl) + connector=aiohttp.TCPConnector(verify_ssl=self.verify_ssl), + timeout=self.timeout ) as client_session: async with client_session.post( f"{self.server_url}/v1/completions", |