summaryrefslogtreecommitdiff
path: root/continuedev/src
diff options
context:
space:
mode:
authorNate Sesti <sestinj@gmail.com>2023-08-21 15:56:25 -0700
committerNate Sesti <sestinj@gmail.com>2023-08-21 15:56:25 -0700
commit87409c31832ccb707abbf134843323c9eb6e1183 (patch)
treede8fd14a9437539907abf34e79f3f5ab610287af /continuedev/src
parent64a318041e68be1cd5cca7a88ef1f87fa15c9c28 (diff)
downloadsncontinue-87409c31832ccb707abbf134843323c9eb6e1183.tar.gz
sncontinue-87409c31832ccb707abbf134843323c9eb6e1183.tar.bz2
sncontinue-87409c31832ccb707abbf134843323c9eb6e1183.zip
fix: :bug: fixing bugs with ggml
Diffstat (limited to 'continuedev/src')
-rw-r--r--continuedev/src/continuedev/libs/llm/ggml.py69
-rw-r--r--continuedev/src/continuedev/plugins/steps/chat.py2
2 files changed, 40 insertions, 31 deletions
diff --git a/continuedev/src/continuedev/libs/llm/ggml.py b/continuedev/src/continuedev/libs/llm/ggml.py
index d1d6d941..f2c53e7b 100644
--- a/continuedev/src/continuedev/libs/llm/ggml.py
+++ b/continuedev/src/continuedev/libs/llm/ggml.py
@@ -85,45 +85,54 @@ class GGML(LLM):
)
args["stream"] = True
- async with self._client_session.post(
- f"{self.server_url}/v1/chat/completions",
- json={"messages": messages, **args},
- ) as resp:
- # This is streaming application/json instaed of text/event-stream
- async for line, end in resp.content.iter_chunks():
- if end:
- continue
-
- json_chunk = line.decode("utf-8")
- if json_chunk.startswith(": ping - ") or json_chunk.startswith(
- "data: [DONE]"
- ):
- continue
- chunks = json_chunk.split("\n")
- for chunk in chunks:
- if chunk.strip() != "":
- yield json.loads(chunk[6:])["choices"][0][
- "delta"
- ] # {"role": "assistant", "content": "..."}
+ async def generator():
+ async with self._client_session.post(
+ f"{self.server_url}/v1/chat/completions",
+ json={"messages": messages, **args},
+ ) as resp:
+ # This is streaming application/json instaed of text/event-stream
+ async for line, end in resp.content.iter_chunks():
+ json_chunk = line.decode("utf-8")
+ if json_chunk.startswith(": ping - ") or json_chunk.startswith(
+ "data: [DONE]"
+ ):
+ continue
+ chunks = json_chunk.split("\n")
+ for chunk in chunks:
+ if chunk.strip() != "":
+ yield json.loads(chunk[6:])["choices"][0][
+ "delta"
+ ] # {"role": "assistant", "content": "..."}
+
+ # Because quite often the first attempt fails, and it works thereafter
+ try:
+ async for chunk in generator():
+ yield chunk
+ except:
+ async for chunk in generator():
+ yield chunk
async def complete(
self, prompt: str, with_history: List[ChatMessage] = None, **kwargs
) -> Coroutine[Any, Any, str]:
args = {**self.default_args, **kwargs}
+ # messages = compile_chat_messages(
+ # args["model"],
+ # with_history,
+ # self.context_length,
+ # args["max_tokens"],
+ # prompt,
+ # functions=None,
+ # system_message=self.system_message,
+ # )
+
async with self._client_session.post(
f"{self.server_url}/v1/completions",
json={
- "messages": compile_chat_messages(
- args["model"],
- with_history,
- self.context_length,
- args["max_tokens"],
- prompt,
- functions=None,
- system_message=self.system_message,
- ),
+ "prompt": prompt,
**args,
},
) as resp:
- return json.loads(await resp.text())["choices"][0]["text"]
+ text = await resp.text()
+ return json.loads(text)["choices"][0]["text"]
diff --git a/continuedev/src/continuedev/plugins/steps/chat.py b/continuedev/src/continuedev/plugins/steps/chat.py
index 7e674272..7f2ebef1 100644
--- a/continuedev/src/continuedev/plugins/steps/chat.py
+++ b/continuedev/src/continuedev/plugins/steps/chat.py
@@ -95,7 +95,7 @@ class SimpleChatStep(Step):
self.name = remove_quotes_and_escapes(
await sdk.models.medium.complete(
- f"Write a short title for the following chat message: {self.description}"
+ f"{self.description}\n\nHere is a short title for the above chat message:"
)
)