From 861a873f7ecf455b9c7833040b2a8163e369e062 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Tue, 25 Jul 2023 22:26:09 -0700
Subject: checkpoint. Somewhat working, just a bit slow, probably some blocking
meilisearch calls still happening
---
extension/react-app/src/main.tsx | 7 +++----
extension/src/activation/activate.ts | 3 ++-
extension/src/continueIdeClient.ts | 3 ++-
extension/src/debugPanel.ts | 2 ++
extension/src/util/messenger.ts | 9 ++++++---
5 files changed, 15 insertions(+), 9 deletions(-)
(limited to 'extension')
diff --git a/extension/react-app/src/main.tsx b/extension/react-app/src/main.tsx
index a76bced6..1776490c 100644
--- a/extension/react-app/src/main.tsx
+++ b/extension/react-app/src/main.tsx
@@ -8,12 +8,11 @@ import "./index.css";
import posthog from "posthog-js";
import { PostHogProvider } from "posthog-js/react";
+console.log("Starting React");
+
posthog.init("phc_JS6XFROuNbhJtVCEdTSYk6gl5ArRrTNMpCcguAXlSPs", {
api_host: "https://app.posthog.com",
- session_recording: {
- // WARNING: Only enable this if you understand the security implications
- recordCrossOriginIframes: true,
- } as any,
+ disable_session_recording: true,
});
ReactDOM.createRoot(document.getElementById("root") as HTMLElement).render(
diff --git a/extension/src/activation/activate.ts b/extension/src/activation/activate.ts
index a1d88a31..356d0256 100644
--- a/extension/src/activation/activate.ts
+++ b/extension/src/activation/activate.ts
@@ -17,7 +17,7 @@ export let ideProtocolClient: IdeProtocolClient;
export async function activateExtension(context: vscode.ExtensionContext) {
extensionContext = context;
-
+ console.log("Using Continue version: ", getExtensionVersion());
// Before anything else, check whether this is an out-of-date version of the extension
// Do so by grabbing the package.json off of the GitHub respository for now.
fetch(PACKAGE_JSON_RAW_GITHUB_URL)
@@ -67,6 +67,7 @@ export async function activateExtension(context: vscode.ExtensionContext) {
}, 2000);
});
+ console.log("Continue server started");
// Initialize IDE Protocol Client
const serverUrl = getContinueServerUrl();
ideProtocolClient = new IdeProtocolClient(
diff --git a/extension/src/continueIdeClient.ts b/extension/src/continueIdeClient.ts
index 802afc1d..d92a829d 100644
--- a/extension/src/continueIdeClient.ts
+++ b/extension/src/continueIdeClient.ts
@@ -399,8 +399,9 @@ class IdeProtocolClient {
}
}, 1000);
});
+ console.log("Getting session ID");
const resp = await this.messenger?.sendAndReceive("getSessionId", {});
- // console.log("New Continue session with ID: ", sessionId);
+ console.log("New Continue session with ID: ", resp.sessionId);
this._sessionId = resp.sessionId;
return resp.sessionId;
}
diff --git a/extension/src/debugPanel.ts b/extension/src/debugPanel.ts
index 6da79cdc..6dcb588a 100644
--- a/extension/src/debugPanel.ts
+++ b/extension/src/debugPanel.ts
@@ -166,11 +166,13 @@ export function setupDebugPanel(
switch (data.type) {
case "onLoad": {
let sessionId: string;
+ console.log("Running onLoad");
if (typeof sessionIdPromise === "string") {
sessionId = sessionIdPromise;
} else {
sessionId = await sessionIdPromise;
}
+ console.log("Done with onLoad: ", sessionId);
panel.webview.postMessage({
type: "onLoad",
vscMachineId: vscode.env.machineId,
diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts
index 3044898e..e28040f5 100644
--- a/extension/src/util/messenger.ts
+++ b/extension/src/util/messenger.ts
@@ -39,6 +39,7 @@ export class WebsocketMessenger extends Messenger {
// var WebSocket = require("ws");
// }
+ console.log("Creating websocket at: ", this.serverUrl);
const newWebsocket = new WebSocket(this.serverUrl);
for (const listener of this.onOpenListeners) {
this.onOpen(listener);
@@ -105,12 +106,14 @@ export class WebsocketMessenger extends Messenger {
send(messageType: string, data: object) {
const payload = JSON.stringify({ messageType, data });
if (this.websocket.readyState === this.websocket.OPEN) {
+ console.log("websocket is open, sending message: ", messageType);
this.websocket.send(payload);
} else {
- if (this.websocket.readyState !== this.websocket.CONNECTING) {
- this.websocket = this._newWebsocket();
- }
+ console.log("websocket is not open, creating new websocket", messageType);
+ this.websocket = this._newWebsocket();
+
this.websocket.addEventListener("open", () => {
+ console.log("websocket is open, resending message: ", messageType);
this.websocket.send(payload);
});
}
--
cgit v1.2.3-70-g09d2
From d2afe5ff258eb2443e0f2178da523150fdb5eb0d Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Tue, 25 Jul 2023 23:09:21 -0700
Subject: meilisearch async client
---
continuedev/poetry.lock | 45 +++++++++++-----------
continuedev/pyproject.toml | 3 +-
continuedev/src/continuedev/core/autopilot.py | 2 +-
continuedev/src/continuedev/core/context.py | 40 ++++++++++---------
.../plugins/context_providers/highlighted_code.py | 4 +-
continuedev/src/continuedev/server/main.py | 14 +++----
.../src/continuedev/server/meilisearch_server.py | 12 +++---
extension/package-lock.json | 4 +-
extension/package.json | 2 +-
9 files changed, 65 insertions(+), 61 deletions(-)
(limited to 'extension')
diff --git a/continuedev/poetry.lock b/continuedev/poetry.lock
index 6644bf6f..d1b5f3d6 100644
--- a/continuedev/poetry.lock
+++ b/continuedev/poetry.lock
@@ -1,5 +1,16 @@
# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
+[[package]]
+name = "aiofiles"
+version = "23.1.0"
+description = "File support for asyncio."
+optional = false
+python-versions = ">=3.7,<4.0"
+files = [
+ {file = "aiofiles-23.1.0-py3-none-any.whl", hash = "sha256:9312414ae06472eb6f1d163f555e466a23aed1c8f60c30cccf7121dba2e53eb2"},
+ {file = "aiofiles-23.1.0.tar.gz", hash = "sha256:edd247df9a19e0db16534d4baaf536d6609a43e1de5401d7a4c1c148753a1635"},
+]
+
[[package]]
name = "aiohttp"
version = "3.8.4"
@@ -223,9 +234,6 @@ files = [
{file = "camel_converter-3.0.2.tar.gz", hash = "sha256:3b3d076e824ae979b271b4d497c90514c2b218811f76b0c368fb69da2556fe07"},
]
-[package.dependencies]
-pydantic = {version = ">=1.8.2", optional = true, markers = "extra == \"pydantic\""}
-
[package.extras]
pydantic = ["pydantic (>=1.8.2)"]
@@ -660,16 +668,6 @@ files = [
{file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"},
]
-[[package]]
-name = "func-timeout"
-version = "4.3.5"
-description = "Python module which allows you to specify timeouts when calling any existing function. Also provides support for stoppable-threads"
-optional = false
-python-versions = "*"
-files = [
- {file = "func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd"},
-]
-
[[package]]
name = "gpt-index"
version = "0.6.8"
@@ -951,19 +949,22 @@ files = [
marshmallow = ">=2.0.0"
[[package]]
-name = "meilisearch"
-version = "0.28.1"
-description = "The python client for Meilisearch API."
+name = "meilisearch-python-async"
+version = "1.4.8"
+description = "A Python async client for the Meilisearch API"
optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.8,<4.0"
files = [
- {file = "meilisearch-0.28.1-py3-none-any.whl", hash = "sha256:b4cfaf140fc323c429a4e174b801bcc6dddef88bf901560ebf2fc5c0014099b5"},
- {file = "meilisearch-0.28.1.tar.gz", hash = "sha256:4e8627abe1394f04125c99e60f9f4356ba200effe5735ce9b8a44b09e3c6403c"},
+ {file = "meilisearch_python_async-1.4.8-py3-none-any.whl", hash = "sha256:dea8da89ea254cd2de7e3c0f0883e98486fc724ba8212e4fe19b2b44d9ca6aa1"},
+ {file = "meilisearch_python_async-1.4.8.tar.gz", hash = "sha256:ebcea0ed800dce291809ec599384f103da55362b4485997ff4daa0010c935fc6"},
]
[package.dependencies]
-camel-converter = {version = "*", extras = ["pydantic"]}
-requests = "*"
+aiofiles = ">=0.7"
+camel-converter = ">=1.0.0"
+httpx = ">=0.17"
+pydantic = ">=1.8"
+PyJWT = ">=2.3.0"
[[package]]
name = "monotonic"
@@ -2311,4 +2312,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = "^3.8.1"
-content-hash = "53ad0b6fa45f00fe53fbd2a90845ae1647e0be22b0b52dfe55996f2c9a6faa38"
+content-hash = "126119ec6b94f1da4af9f8f2de4352df20522652cd58726546bd2fb46b40a9ef"
diff --git a/continuedev/pyproject.toml b/continuedev/pyproject.toml
index ad5657b1..d7505e2b 100644
--- a/continuedev/pyproject.toml
+++ b/continuedev/pyproject.toml
@@ -27,9 +27,8 @@ directory-tree = "^0.0.3.1"
anthropic = "^0.3.4"
chevron = "^0.14.0"
psutil = "^5.9.5"
-meilisearch = "^0.28.1"
pygithub = "^1.59.0"
-func-timeout = "^4.3.5"
+meilisearch-python-async = "^1.4.8"
[tool.poetry.scripts]
typegen = "src.continuedev.models.generate_json_schema:main"
diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py
index d018c29e..42a58423 100644
--- a/continuedev/src/continuedev/core/autopilot.py
+++ b/continuedev/src/continuedev/core/autopilot.py
@@ -69,7 +69,7 @@ class Autopilot(ContinueBaseModel):
autopilot.continue_sdk = await ContinueSDK.create(autopilot)
# Load documents into the search index
- autopilot.context_manager = ContextManager(
+ autopilot.context_manager = await ContextManager.create(
autopilot.continue_sdk.config.context_providers + [
HighlightedCodeContextProvider(ide=ide),
FileContextProvider(workspace_dir=ide.workspace_directory)
diff --git a/continuedev/src/continuedev/core/context.py b/continuedev/src/continuedev/core/context.py
index 67bba651..7d302656 100644
--- a/continuedev/src/continuedev/core/context.py
+++ b/continuedev/src/continuedev/core/context.py
@@ -1,7 +1,7 @@
from abc import abstractmethod
from typing import Dict, List
-import meilisearch
+from meilisearch_python_async import Client
from pydantic import BaseModel
@@ -50,21 +50,21 @@ class ContextProvider(BaseModel):
"""
return [ChatMessage(role="user", content=f"{item.description.name}: {item.description.description}\n\n{item.content}", summary=item.description.description) for item in await self.get_selected_items()]
- async def get_item(self, id: ContextItemId, query: str, search_client: meilisearch.Client) -> ContextItem:
+ async def get_item(self, id: ContextItemId, query: str, search_client: Client) -> ContextItem:
"""
Returns the ContextItem with the given id.
Default implementation uses the search index to get the item.
"""
- result = search_client.index(
+ result = await search_client.index(
SEARCH_INDEX_NAME).get_document(id.to_string())
return ContextItem(
description=ContextItemDescription(
- name=result.name,
- description=result.description,
+ name=result["name"],
+ description=result["description"],
id=id
),
- content=result.content
+ content=result["content"]
)
async def delete_context_with_ids(self, ids: List[ContextItemId]):
@@ -85,7 +85,7 @@ class ContextProvider(BaseModel):
"""
self.selected_items = []
- async def add_context_item(self, id: ContextItemId, query: str, search_client: meilisearch.Client):
+ async def add_context_item(self, id: ContextItemId, query: str, search_client: Client):
"""
Adds the given ContextItem to the list of ContextItems.
@@ -126,21 +126,26 @@ class ContextManager:
"""
return sum([await provider.get_chat_messages() for provider in self.context_providers.values()], [])
- search_client: meilisearch.Client
-
- def __init__(self, context_providers: List[ContextProvider]):
- self.search_client = meilisearch.Client('http://localhost:7700')
-
- # If meilisearch isn't running, don't use any ContextProviders that might depend on it
- if not check_meilisearch_running():
- context_providers = list(
- filter(lambda cp: cp.title == "code", context_providers))
+ search_client: Client
+ def __init__(self, context_providers: List[ContextProvider], search_client: Client):
+ self.search_client = search_client
self.context_providers = {
prov.title: prov for prov in context_providers}
self.provider_titles = {
provider.title for provider in context_providers}
+ @classmethod
+ async def create(cls, context_providers: List[ContextProvider]):
+ search_client = Client('http://localhost:7700')
+ health = await search_client.health()
+ if not health.status == "available":
+ print("MeiliSearch not running, avoiding any dependent context providers")
+ context_providers = list(
+ filter(lambda cp: cp.title == "code", context_providers))
+
+ return cls(context_providers, search_client)
+
async def load_index(self):
for _, provider in self.context_providers.items():
context_items = await provider.provide_context_items()
@@ -154,8 +159,7 @@ class ContextManager:
for item in context_items
]
if len(documents) > 0:
- self.search_client.index(
- SEARCH_INDEX_NAME).add_documents(documents)
+ await self.search_client.index(SEARCH_INDEX_NAME).add_documents(documents)
# def compile_chat_messages(self, max_tokens: int) -> List[Dict]:
# """
diff --git a/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py b/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py
index 23d4fc86..426c0804 100644
--- a/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py
+++ b/continuedev/src/continuedev/plugins/context_providers/highlighted_code.py
@@ -1,7 +1,7 @@
import os
from typing import Any, Dict, List
-import meilisearch
+from meilisearch_python_async import Client
from ...core.main import ChatMessage
from ...models.filesystem import RangeInFile, RangeInFileWithContents
from ...core.context import ContextItem, ContextItemDescription, ContextItemId
@@ -187,5 +187,5 @@ class HighlightedCodeContextProvider(BaseModel):
for hr in self.highlighted_ranges:
hr.item.editing = hr.item.description.id.to_string() in ids
- async def add_context_item(self, id: ContextItemId, query: str, search_client: meilisearch.Client, prev: List[ContextItem] = None) -> List[ContextItem]:
+ async def add_context_item(self, id: ContextItemId, query: str, search_client: Client, prev: List[ContextItem] = None) -> List[ContextItem]:
raise NotImplementedError()
diff --git a/continuedev/src/continuedev/server/main.py b/continuedev/src/continuedev/server/main.py
index 7ee64041..0398c3b0 100644
--- a/continuedev/src/continuedev/server/main.py
+++ b/continuedev/src/continuedev/server/main.py
@@ -1,7 +1,6 @@
import asyncio
-import subprocess
import time
-import meilisearch
+from meilisearch_python_async import Client
import psutil
import os
from fastapi import FastAPI
@@ -87,13 +86,14 @@ if __name__ == "__main__":
# cpu_thread = threading.Thread(target=cpu_usage_loop)
# cpu_thread.start()
- try:
- start_meilisearch()
- except Exception as e:
- print("Failed to start MeiliSearch")
- print(e)
+ # try:
+ # await start_meilisearch()
+ # except Exception as e:
+ # print("Failed to start MeiliSearch")
+ # print(e)
run_server()
except Exception as e:
+ print("Error starting Continue server: ", e)
cleanup()
raise e
diff --git a/continuedev/src/continuedev/server/meilisearch_server.py b/continuedev/src/continuedev/server/meilisearch_server.py
index 232b6243..286019e1 100644
--- a/continuedev/src/continuedev/server/meilisearch_server.py
+++ b/continuedev/src/continuedev/server/meilisearch_server.py
@@ -2,7 +2,7 @@ import os
import shutil
import subprocess
-import meilisearch
+from meilisearch_python_async import Client
from ..libs.util.paths import getServerFolderPath
@@ -41,14 +41,14 @@ def ensure_meilisearch_installed():
f"curl -L https://install.meilisearch.com | sh", shell=True, check=True, cwd=serverPath)
-def check_meilisearch_running() -> bool:
+async def check_meilisearch_running() -> bool:
"""
Checks if MeiliSearch is running.
"""
try:
- client = meilisearch.Client('http://localhost:7700')
- resp = client.health()
+ client = Client('http://localhost:7700')
+ resp = await client.health()
if resp["status"] != "available":
return False
return True
@@ -56,7 +56,7 @@ def check_meilisearch_running() -> bool:
return False
-def start_meilisearch():
+async def start_meilisearch():
"""
Starts the MeiliSearch server, wait for it.
"""
@@ -71,7 +71,7 @@ def start_meilisearch():
ensure_meilisearch_installed()
# Check if MeiliSearch is running
- if not check_meilisearch_running():
+ if not await check_meilisearch_running():
print("Starting MeiliSearch...")
subprocess.Popen(["./meilisearch"], cwd=serverPath, stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT, close_fds=True, start_new_session=True)
diff --git a/extension/package-lock.json b/extension/package-lock.json
index 71c489c6..e694f2e2 100644
--- a/extension/package-lock.json
+++ b/extension/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "continue",
- "version": "0.0.195",
+ "version": "0.0.199",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "continue",
- "version": "0.0.195",
+ "version": "0.0.199",
"license": "Apache-2.0",
"dependencies": {
"@electron/rebuild": "^3.2.10",
diff --git a/extension/package.json b/extension/package.json
index 1e9314e5..bd26dd16 100644
--- a/extension/package.json
+++ b/extension/package.json
@@ -14,7 +14,7 @@
"displayName": "Continue",
"pricing": "Free",
"description": "The open-source coding autopilot",
- "version": "0.0.195",
+ "version": "0.0.199",
"publisher": "Continue",
"engines": {
"vscode": "^1.67.0"
--
cgit v1.2.3-70-g09d2
From 932925ccc90096c7869797faa00b95c8ac1eff0e Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Tue, 25 Jul 2023 23:46:34 -0700
Subject: correctly ignore directories in file context prov
---
continuedev/src/continuedev/plugins/context_providers/file.py | 7 +++++--
extension/src/util/messenger.ts | 3 ---
2 files changed, 5 insertions(+), 5 deletions(-)
(limited to 'extension')
diff --git a/continuedev/src/continuedev/plugins/context_providers/file.py b/continuedev/src/continuedev/plugins/context_providers/file.py
index fc0af7ba..6222ec6a 100644
--- a/continuedev/src/continuedev/plugins/context_providers/file.py
+++ b/continuedev/src/continuedev/plugins/context_providers/file.py
@@ -21,7 +21,7 @@ class FileContextProvider(ContextProvider):
title = "file"
workspace_dir: str
- ignore_patterns: List[str] = list(map(lambda folder: f"**/{folder}", [
+ ignore_patterns: List[str] = [
".git",
".vscode",
".idea",
@@ -35,7 +35,10 @@ class FileContextProvider(ContextProvider):
"target",
"out",
"bin",
- ]))
+ ".pytest_cache",
+ ".vscode-test",
+ ".continue",
+ ]
async def provide_context_items(self) -> List[ContextItem]:
filepaths = []
diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts
index e28040f5..be4233b2 100644
--- a/extension/src/util/messenger.ts
+++ b/extension/src/util/messenger.ts
@@ -106,14 +106,11 @@ export class WebsocketMessenger extends Messenger {
send(messageType: string, data: object) {
const payload = JSON.stringify({ messageType, data });
if (this.websocket.readyState === this.websocket.OPEN) {
- console.log("websocket is open, sending message: ", messageType);
this.websocket.send(payload);
} else {
- console.log("websocket is not open, creating new websocket", messageType);
this.websocket = this._newWebsocket();
this.websocket.addEventListener("open", () => {
- console.log("websocket is open, resending message: ", messageType);
this.websocket.send(payload);
});
}
--
cgit v1.2.3-70-g09d2
From fc66d0d8cc33183b0b075485803e29f8f376dcc7 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Tue, 25 Jul 2023 23:48:30 -0700
Subject: patch
---
extension/package-lock.json | 4 ++--
extension/package.json | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
(limited to 'extension')
diff --git a/extension/package-lock.json b/extension/package-lock.json
index e694f2e2..c965f8be 100644
--- a/extension/package-lock.json
+++ b/extension/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "continue",
- "version": "0.0.199",
+ "version": "0.0.200",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "continue",
- "version": "0.0.199",
+ "version": "0.0.200",
"license": "Apache-2.0",
"dependencies": {
"@electron/rebuild": "^3.2.10",
diff --git a/extension/package.json b/extension/package.json
index bd26dd16..5459299d 100644
--- a/extension/package.json
+++ b/extension/package.json
@@ -14,7 +14,7 @@
"displayName": "Continue",
"pricing": "Free",
"description": "The open-source coding autopilot",
- "version": "0.0.199",
+ "version": "0.0.200",
"publisher": "Continue",
"engines": {
"vscode": "^1.67.0"
--
cgit v1.2.3-70-g09d2
From 03da423e5abdf32c5bf9755ffd2e1c7be98e6454 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Tue, 25 Jul 2023 23:51:55 -0700
Subject: messenger don't create new websocket if connecting
---
extension/src/util/messenger.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
(limited to 'extension')
diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts
index be4233b2..dc59a593 100644
--- a/extension/src/util/messenger.ts
+++ b/extension/src/util/messenger.ts
@@ -108,7 +108,9 @@ export class WebsocketMessenger extends Messenger {
if (this.websocket.readyState === this.websocket.OPEN) {
this.websocket.send(payload);
} else {
- this.websocket = this._newWebsocket();
+ if (this.websocket.readyState !== this.websocket.CONNECTING) {
+ this.websocket = this._newWebsocket();
+ }
this.websocket.addEventListener("open", () => {
this.websocket.send(payload);
--
cgit v1.2.3-70-g09d2