From 861a873f7ecf455b9c7833040b2a8163e369e062 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Tue, 25 Jul 2023 22:26:09 -0700 Subject: checkpoint. Somewhat working, just a bit slow, probably some blocking meilisearch calls still happening --- continuedev/src/continuedev/server/gui.py | 4 ++-- continuedev/src/continuedev/server/ide.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'continuedev/src') diff --git a/continuedev/src/continuedev/server/gui.py b/continuedev/src/continuedev/server/gui.py index fa203c28..c0957395 100644 --- a/continuedev/src/continuedev/server/gui.py +++ b/continuedev/src/continuedev/server/gui.py @@ -61,12 +61,12 @@ class GUIProtocolServer(AbstractGUIProtocolServer): "data": data }) - async def _receive_json(self, message_type: str, timeout: int = 5) -> Any: + async def _receive_json(self, message_type: str, timeout: int = 20) -> Any: try: return await asyncio.wait_for(self.sub_queue.get(message_type), timeout=timeout) except asyncio.TimeoutError: raise Exception( - "GUI Protocol _receive_json timed out after 5 seconds") + "GUI Protocol _receive_json timed out after 20 seconds") async def _send_and_receive_json(self, data: Any, resp_model: Type[T], message_type: str) -> T: await self._send_json(message_type, data) diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index d6a28c92..3401cbac 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -139,6 +139,7 @@ class IdeProtocolServer(AbstractIdeProtocolServer): continue message_type = message["messageType"] data = message["data"] + print("Received message while initializing", message_type) if message_type == "workspaceDirectory": self.workspace_directory = data["workspaceDirectory"] elif message_type == "uniqueId": @@ -153,17 +154,18 @@ class IdeProtocolServer(AbstractIdeProtocolServer): async def _send_json(self, message_type: str, data: Any): if self.websocket.application_state == WebSocketState.DISCONNECTED: return + print("Sending IDE message: ", message_type) await self.websocket.send_json({ "messageType": message_type, "data": data }) - async def _receive_json(self, message_type: str, timeout: int = 5) -> Any: + async def _receive_json(self, message_type: str, timeout: int = 20) -> Any: try: return await asyncio.wait_for(self.sub_queue.get(message_type), timeout=timeout) except asyncio.TimeoutError: raise Exception( - "IDE Protocol _receive_json timed out after 5 seconds") + "IDE Protocol _receive_json timed out after 20 seconds", message_type) async def _send_and_receive_json(self, data: Any, resp_model: Type[T], message_type: str) -> T: await self._send_json(message_type, data) -- cgit v1.2.3-70-g09d2 From d2afe5ff258eb2443e0f2178da523150fdb5eb0d Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Tue, 25 Jul 2023 23:09:21 -0700 Subject: meilisearch async client --- continuedev/poetry.lock | 45 +++++++++++----------- continuedev/pyproject.toml | 3 +- continuedev/src/continuedev/core/autopilot.py | 2 +- continuedev/src/continuedev/core/context.py | 40 ++++++++++--------- .../plugins/context_providers/highlighted_code.py | 4 +- continuedev/src/continuedev/server/main.py | 14 +++---- .../src/continuedev/server/meilisearch_server.py | 12 +++--- extension/package-lock.json | 4 +- extension/package.json | 2 +- 9 files changed, 65 insertions(+), 61 deletions(-) (limited to 'continuedev/src') diff --git a/continuedev/poetry.lock b/continuedev/poetry.lock index 6644bf6f..d1b5f3d6 100644 --- a/continuedev/poetry.lock +++ b/continuedev/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +[[package]] +name = "aiofiles" +version = "23.1.0" +description = "File support for asyncio." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "aiofiles-23.1.0-py3-none-any.whl", hash = "sha256:9312414ae06472eb6f1d163f555e466a23aed1c8f60c30cccf7121dba2e53eb2"}, + {file = "aiofiles-23.1.0.tar.gz", hash = "sha256:edd247df9a19e0db16534d4baaf536d6609a43e1de5401d7a4c1c148753a1635"}, +] + [[package]] name = "aiohttp" version = "3.8.4" @@ -223,9 +234,6 @@ files = [ {file = "camel_converter-3.0.2.tar.gz", hash = "sha256:3b3d076e824ae979b271b4d497c90514c2b218811f76b0c368fb69da2556fe07"}, ] -[package.dependencies] -pydantic = {version = ">=1.8.2", optional = true, markers = "extra == \"pydantic\""} - [package.extras] pydantic = ["pydantic (>=1.8.2)"] @@ -660,16 +668,6 @@ files = [ {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] -[[package]] -name = "func-timeout" -version = "4.3.5" -description = "Python module which allows you to specify timeouts when calling any existing function. Also provides support for stoppable-threads" -optional = false -python-versions = "*" -files = [ - {file = "func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd"}, -] - [[package]] name = "gpt-index" version = "0.6.8" @@ -951,19 +949,22 @@ files = [ marshmallow = ">=2.0.0" [[package]] -name = "meilisearch" -version = "0.28.1" -description = "The python client for Meilisearch API." +name = "meilisearch-python-async" +version = "1.4.8" +description = "A Python async client for the Meilisearch API" optional = false -python-versions = ">=3.8" +python-versions = ">=3.8,<4.0" files = [ - {file = "meilisearch-0.28.1-py3-none-any.whl", hash = "sha256:b4cfaf140fc323c429a4e174b801bcc6dddef88bf901560ebf2fc5c0014099b5"}, - {file = "meilisearch-0.28.1.tar.gz", hash = "sha256:4e8627abe1394f04125c99e60f9f4356ba200effe5735ce9b8a44b09e3c6403c"}, + {file = "meilisearch_python_async-1.4.8-py3-none-any.whl", hash = "sha256:dea8da89ea254cd2de7e3c0f0883e98486fc724ba8212e4fe19b2b44d9ca6aa1"}, + {file = "meilisearch_python_async-1.4.8.tar.gz", hash = "sha256:ebcea0ed800dce291809ec599384f103da55362b4485997ff4daa0010c935fc6"}, ] [package.dependencies] -camel-converter = {version = "*", extras = ["pydantic"]} -requests = "*" +aiofiles = ">=0.7" +camel-converter = ">=1.0.0" +httpx = ">=0.17" +pydantic = ">=1.8" +PyJWT = ">=2.3.0" [[package]] name = "monotonic" @@ -2311,4 +2312,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "53ad0b6fa45f00fe53fbd2a90845ae1647e0be22b0b52dfe55996f2c9a6faa38" +content-hash = "126119ec6b94f1da4af9f8f2de4352df20522652cd58726546bd2fb46b40a9ef" diff --git a/continuedev/pyproject.toml b/continuedev/pyproject.toml index ad5657b1..d7505e2b 100644 --- a/continuedev/pyproject.toml +++ b/continuedev/pyproject.toml @@ -27,9 +27,8 @@ directory-tree = "^0.0.3.1" anthropic = "^0.3.4" chevron = "^0.14.0" psutil = "^5.9.5" -meilisearch = "^0.28.1" pygithub = "^1.59.0" -func-timeout = "^4.3.5" +meilisearch-python-async = "^1.4.8" [tool.poetry.scripts] typegen = "src.continuedev.models.generate_json_schema:main" diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index d018c29e..42a58423 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -69,7 +69,7 @@ class Autopilot(ContinueBaseModel): autopilot.continue_sdk = await ContinueSDK.create(autopilot) # Load documents into the search index - autopilot.context_manager = ContextManager( + autopilot.context_manager = await ContextManager.create( autopilot.continue_sdk.config.context_providers + [ HighlightedCodeContextProvider(ide=ide), FileContextProvider(workspace_dir=ide.workspace_directory) diff --git a/continuedev/src/continuedev/core/context.py b/continuedev/src/continuedev/core/context.py index 67bba651..7d302656 100644 --- a/continuedev/src/continuedev/core/context.py +++ b/continuedev/src/continuedev/core/context.py @@ -1,7 +1,7 @@ from abc import abstractmethod from typing import Dict, List -import meilisearch +from meilisearch_python_async import Client from pydantic import BaseModel @@ -50,21 +50,21 @@ class ContextProvider(BaseModel): """ return [ChatMessage(role="user", content=f"{item.description.name}: {item.description.description}\n\n{item.content}", summary=item.description.description) for item in await self.get_selected_items()] - async def get_item(self, id: ContextItemId, query: str, search_client: meilisearch.Client) -> ContextItem: + async def get_item(self, id: ContextItemId, query: str, search_client: Client) -> ContextItem: """ Returns the ContextItem with the given id. Default implementation uses the search index to get the item. """ - result = search_client.index( + result = await search_client.index( SEARCH_INDEX_NAME).get_document(id.to_string()) return ContextItem( description=ContextItemDescription( - name=result.name, - description=result.description, + name=result["name"], + description=result["description"], id=id ), - content=result.content + content=result["content"] ) async def delete_context_with_ids(self, ids: List[ContextItemId]): @@ -85,7 +85,7 @@ class ContextProvider(BaseModel): """ self.selected_items = [] - async def add_context_item(self, id: ContextItemId, query: str, search_client: meilisearch.Client): + async def add_context_item(self, id: ContextItemId, query: str, search_client: Client): """ Adds the given ContextItem to the list of ContextItems. @@ -126,21 +126,26 @@ class ContextManager: """ return sum([await provider.get_chat_messages() for provider in self.context_providers.values()], []) - search_client: meilisearch.Client - - def __init__(self, context_providers: List[ContextProvider]): - self.search_client = meilisearch.Client('http://localhost:7700') - - # If meilisearch isn't running, don't use any ContextProviders that might depend on it - if not check_meilisearch_running(): - context_providers = list( - filter(lambda cp: cp.title == "code", context_providers)) + search_client: Client + def __init__(self, context_providers: List[ContextProvider], search_client: Client): + self.search_client = search_client self.context_providers = { prov.title: prov for prov in context_providers} self.provider_titles = { provider.title for provider in context_providers} + @classmethod + async def create(cls, context_providers: List[ContextProvider]): + search_client = Client('http://localhost:7700') + health = await search_client.health() + if not health.status == "available": + print("MeiliSearch not running, avoiding any dependent context providers") + context_providers = list( + filter(lambda cp: cp.title == "code", context_providers)) + + return cls(context_providers, search_client) + async def load_index(self): for _, provider in self.context_providers.items(): context_items = await provider.provide_context_items() @@ -154,8 +159,7 @@ class ContextManager: for item in context_items ] if len(documents) > 0: - self.search_client.index( - SEARCH_INDEX_NAME).add_documents(documents) + await self.search_client.index(SEARCH_INDEX_NAME).add_documents(documents) # def compile_chat_messages(self, max_tokens: int) -> List[Dict]: # """ diff --git a/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py b/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py index 23d4fc86..426c0804 100644 --- a/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py +++ b/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py @@ -1,7 +1,7 @@ import os from typing import Any, Dict, List -import meilisearch +from meilisearch_python_async import Client from ...core.main import ChatMessage from ...models.filesystem import RangeInFile, RangeInFileWithContents from ...core.context import ContextItem, ContextItemDescription, ContextItemId @@ -187,5 +187,5 @@ class HighlightedCodeContextProvider(BaseModel): for hr in self.highlighted_ranges: hr.item.editing = hr.item.description.id.to_string() in ids - async def add_context_item(self, id: ContextItemId, query: str, search_client: meilisearch.Client, prev: List[ContextItem] = None) -> List[ContextItem]: + async def add_context_item(self, id: ContextItemId, query: str, search_client: Client, prev: List[ContextItem] = None) -> List[ContextItem]: raise NotImplementedError() diff --git a/continuedev/src/continuedev/server/main.py b/continuedev/src/continuedev/server/main.py index 7ee64041..0398c3b0 100644 --- a/continuedev/src/continuedev/server/main.py +++ b/continuedev/src/continuedev/server/main.py @@ -1,7 +1,6 @@ import asyncio -import subprocess import time -import meilisearch +from meilisearch_python_async import Client import psutil import os from fastapi import FastAPI @@ -87,13 +86,14 @@ if __name__ == "__main__": # cpu_thread = threading.Thread(target=cpu_usage_loop) # cpu_thread.start() - try: - start_meilisearch() - except Exception as e: - print("Failed to start MeiliSearch") - print(e) + # try: + # await start_meilisearch() + # except Exception as e: + # print("Failed to start MeiliSearch") + # print(e) run_server() except Exception as e: + print("Error starting Continue server: ", e) cleanup() raise e diff --git a/continuedev/src/continuedev/server/meilisearch_server.py b/continuedev/src/continuedev/server/meilisearch_server.py index 232b6243..286019e1 100644 --- a/continuedev/src/continuedev/server/meilisearch_server.py +++ b/continuedev/src/continuedev/server/meilisearch_server.py @@ -2,7 +2,7 @@ import os import shutil import subprocess -import meilisearch +from meilisearch_python_async import Client from ..libs.util.paths import getServerFolderPath @@ -41,14 +41,14 @@ def ensure_meilisearch_installed(): f"curl -L https://install.meilisearch.com | sh", shell=True, check=True, cwd=serverPath) -def check_meilisearch_running() -> bool: +async def check_meilisearch_running() -> bool: """ Checks if MeiliSearch is running. """ try: - client = meilisearch.Client('http://localhost:7700') - resp = client.health() + client = Client('http://localhost:7700') + resp = await client.health() if resp["status"] != "available": return False return True @@ -56,7 +56,7 @@ def check_meilisearch_running() -> bool: return False -def start_meilisearch(): +async def start_meilisearch(): """ Starts the MeiliSearch server, wait for it. """ @@ -71,7 +71,7 @@ def start_meilisearch(): ensure_meilisearch_installed() # Check if MeiliSearch is running - if not check_meilisearch_running(): + if not await check_meilisearch_running(): print("Starting MeiliSearch...") subprocess.Popen(["./meilisearch"], cwd=serverPath, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, close_fds=True, start_new_session=True) diff --git a/extension/package-lock.json b/extension/package-lock.json index 71c489c6..e694f2e2 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.195", + "version": "0.0.199", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.195", + "version": "0.0.199", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index 1e9314e5..bd26dd16 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "The open-source coding autopilot", - "version": "0.0.195", + "version": "0.0.199", "publisher": "Continue", "engines": { "vscode": "^1.67.0" -- cgit v1.2.3-70-g09d2 From 6e0c1dece5d9331ebb558d6ae85a99f8db379177 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Tue, 25 Jul 2023 23:18:19 -0700 Subject: start meilisearch in ide.py --- continuedev/src/continuedev/server/ide.py | 8 ++++++++ continuedev/src/continuedev/server/main.py | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'continuedev/src') diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index 3401cbac..cf8b32a1 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -10,6 +10,7 @@ from pydantic import BaseModel import traceback import asyncio +from .meilisearch_server import start_meilisearch from ..libs.util.telemetry import posthog_logger from ..libs.util.queue import AsyncSubscriptionQueue from ..models.filesystem import FileSystem, RangeInFile, EditDiff, RangeInFileWithContents, RealFileSystem @@ -434,6 +435,13 @@ class IdeProtocolServer(AbstractIdeProtocolServer): @router.websocket("/ws") async def websocket_endpoint(websocket: WebSocket, session_id: str = None): try: + # Start meilisearch + try: + await start_meilisearch() + except Exception as e: + print("Failed to start MeiliSearch") + print(e) + await websocket.accept() print("Accepted websocket connection from, ", websocket.client) await websocket.send_json({"messageType": "connected", "data": {}}) diff --git a/continuedev/src/continuedev/server/main.py b/continuedev/src/continuedev/server/main.py index 0398c3b0..0b59d4fe 100644 --- a/continuedev/src/continuedev/server/main.py +++ b/continuedev/src/continuedev/server/main.py @@ -1,6 +1,5 @@ import asyncio import time -from meilisearch_python_async import Client import psutil import os from fastapi import FastAPI @@ -12,7 +11,6 @@ import argparse from .ide import router as ide_router from .gui import router as gui_router from .session_manager import session_manager -from .meilisearch_server import start_meilisearch app = FastAPI() @@ -86,12 +84,6 @@ if __name__ == "__main__": # cpu_thread = threading.Thread(target=cpu_usage_loop) # cpu_thread.start() - # try: - # await start_meilisearch() - # except Exception as e: - # print("Failed to start MeiliSearch") - # print(e) - run_server() except Exception as e: print("Error starting Continue server: ", e) -- cgit v1.2.3-70-g09d2 From 932925ccc90096c7869797faa00b95c8ac1eff0e Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Tue, 25 Jul 2023 23:46:34 -0700 Subject: correctly ignore directories in file context prov --- continuedev/src/continuedev/plugins/context_providers/file.py | 7 +++++-- extension/src/util/messenger.ts | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'continuedev/src') diff --git a/continuedev/src/continuedev/plugins/context_providers/file.py b/continuedev/src/continuedev/plugins/context_providers/file.py index fc0af7ba..6222ec6a 100644 --- a/continuedev/src/continuedev/plugins/context_providers/file.py +++ b/continuedev/src/continuedev/plugins/context_providers/file.py @@ -21,7 +21,7 @@ class FileContextProvider(ContextProvider): title = "file" workspace_dir: str - ignore_patterns: List[str] = list(map(lambda folder: f"**/{folder}", [ + ignore_patterns: List[str] = [ ".git", ".vscode", ".idea", @@ -35,7 +35,10 @@ class FileContextProvider(ContextProvider): "target", "out", "bin", - ])) + ".pytest_cache", + ".vscode-test", + ".continue", + ] async def provide_context_items(self) -> List[ContextItem]: filepaths = [] diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts index e28040f5..be4233b2 100644 --- a/extension/src/util/messenger.ts +++ b/extension/src/util/messenger.ts @@ -106,14 +106,11 @@ export class WebsocketMessenger extends Messenger { send(messageType: string, data: object) { const payload = JSON.stringify({ messageType, data }); if (this.websocket.readyState === this.websocket.OPEN) { - console.log("websocket is open, sending message: ", messageType); this.websocket.send(payload); } else { - console.log("websocket is not open, creating new websocket", messageType); this.websocket = this._newWebsocket(); this.websocket.addEventListener("open", () => { - console.log("websocket is open, resending message: ", messageType); this.websocket.send(payload); }); } -- cgit v1.2.3-70-g09d2