From f53768612b1e2268697b5444e502032ef9f3fb3c Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Tue, 23 May 2023 23:45:12 -0400 Subject: copying from old repo --- extension/src/util/util.ts | 115 ++++++++++++++++++++++++++++++++ extension/src/util/vscode.ts | 152 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 extension/src/util/util.ts create mode 100644 extension/src/util/vscode.ts (limited to 'extension/src/util') diff --git a/extension/src/util/util.ts b/extension/src/util/util.ts new file mode 100644 index 00000000..d33593e1 --- /dev/null +++ b/extension/src/util/util.ts @@ -0,0 +1,115 @@ +import { RangeInFile, SerializedDebugContext } from "../client"; +import * as fs from "fs"; + +function charIsEscapedAtIndex(index: number, str: string): boolean { + if (index === 0) return false; + if (str[index - 1] !== "\\") return false; + return !charIsEscapedAtIndex(index - 1, str); +} + +export function convertSingleToDoubleQuoteJSON(json: string): string { + const singleQuote = "'"; + const doubleQuote = '"'; + const isQuote = (char: string) => + char === doubleQuote || char === singleQuote; + + let newJson = ""; + let insideString = false; + let enclosingQuoteType = doubleQuote; + for (let i = 0; i < json.length; i++) { + if (insideString) { + if (json[i] === enclosingQuoteType && !charIsEscapedAtIndex(i, json)) { + // Close string with a double quote + insideString = false; + newJson += doubleQuote; + } else if (json[i] === singleQuote) { + if (charIsEscapedAtIndex(i, json)) { + // Unescape single quote + newJson = newJson.slice(0, -1); + } + newJson += singleQuote; + } else if (json[i] === doubleQuote) { + if (!charIsEscapedAtIndex(i, json)) { + // Escape double quote + newJson += "\\"; + } + newJson += doubleQuote; + } else { + newJson += json[i]; + } + } else { + if (isQuote(json[i])) { + insideString = true; + enclosingQuoteType = json[i]; + newJson += doubleQuote; + } else { + newJson += json[i]; + } + } + } + + return newJson; +} + +export async function readRangeInFile( + rangeInFile: RangeInFile +): Promise { + const range = rangeInFile.range; + return new Promise((resolve, reject) => { + fs.readFile(rangeInFile.filepath, (err, data) => { + if (err) { + reject(err); + } else { + let lines = data.toString().split("\n"); + if (range.start.line === range.end.line) { + resolve( + lines[rangeInFile.range.start.line].slice( + rangeInFile.range.start.character, + rangeInFile.range.end.character + ) + ); + } else { + let firstLine = lines[range.start.line].slice(range.start.character); + let lastLine = lines[range.end.line].slice(0, range.end.character); + let middleLines = lines.slice(range.start.line + 1, range.end.line); + resolve([firstLine, ...middleLines, lastLine].join("\n")); + } + } + }); + }); +} + +export function codeSelectionsToVirtualFileSystem( + codeSelections: RangeInFile[] +): { + [filepath: string]: string; +} { + let virtualFileSystem: { [filepath: string]: string } = {}; + for (let cs of codeSelections) { + if (!cs.filepath) continue; + if (cs.filepath in virtualFileSystem) continue; + let content = fs.readFileSync(cs.filepath, "utf8"); + virtualFileSystem[cs.filepath] = content; + } + return virtualFileSystem; +} + +export function addFileSystemToDebugContext( + ctx: SerializedDebugContext +): SerializedDebugContext { + ctx.filesystem = codeSelectionsToVirtualFileSystem(ctx.rangesInFiles); + return ctx; +} + +export function debounced(delay: number, fn: Function) { + let timerId: NodeJS.Timeout | null; + return function (...args: any[]) { + if (timerId) { + clearTimeout(timerId); + } + timerId = setTimeout(() => { + fn(...args); + timerId = null; + }, delay); + }; +} diff --git a/extension/src/util/vscode.ts b/extension/src/util/vscode.ts new file mode 100644 index 00000000..4eab98a7 --- /dev/null +++ b/extension/src/util/vscode.ts @@ -0,0 +1,152 @@ +import * as vscode from "vscode"; +import * as path from "path"; +import * as fs from "fs"; + +export function translate(range: vscode.Range, lines: number): vscode.Range { + return new vscode.Range( + range.start.line + lines, + range.start.character, + range.end.line + lines, + range.end.character + ); +} + +export function getNonce() { + let text = ""; + const possible = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + for (let i = 0; i < 32; i++) { + text += possible.charAt(Math.floor(Math.random() * possible.length)); + } + return text; +} + +export function getTestFile( + filename: string, + createFile: boolean = false +): string { + let basename = path.basename(filename).split(".")[0]; + switch (path.extname(filename)) { + case ".py": + basename += "_test"; + break; + case ".js": + case ".jsx": + case ".ts": + case ".tsx": + basename += ".test"; + break; + default: + basename += "_test"; + } + + const directory = path.join(path.dirname(filename), "tests"); + const testFilename = path.join(directory, basename + path.extname(filename)); + + // Optionally, create the file if it doesn't exist + if (createFile && !fs.existsSync(testFilename)) { + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory); + } + fs.writeFileSync(testFilename, ""); + } + + return testFilename; +} + +export function getExtensionUri(): vscode.Uri { + return vscode.extensions.getExtension("Continue.continue")!.extensionUri; +} + +export function getViewColumnOfFile( + filepath: string +): vscode.ViewColumn | undefined { + for (let tabGroup of vscode.window.tabGroups.all) { + for (let tab of tabGroup.tabs) { + if ( + (tab?.input as any)?.uri && + (tab.input as any).uri.fsPath === filepath + ) { + return tabGroup.viewColumn; + } + } + } + return undefined; +} + +export function getRightViewColumn(): vscode.ViewColumn { + // When you want to place in the rightmost panel if there is already more than one, otherwise use Beside + let column = vscode.ViewColumn.Beside; + let columnOrdering = [ + vscode.ViewColumn.One, + vscode.ViewColumn.Beside, + vscode.ViewColumn.Two, + vscode.ViewColumn.Three, + vscode.ViewColumn.Four, + vscode.ViewColumn.Five, + vscode.ViewColumn.Six, + vscode.ViewColumn.Seven, + vscode.ViewColumn.Eight, + vscode.ViewColumn.Nine, + ]; + for (let tabGroup of vscode.window.tabGroups.all) { + if ( + columnOrdering.indexOf(tabGroup.viewColumn) > + columnOrdering.indexOf(column) + ) { + column = tabGroup.viewColumn; + } + } + return column; +} + +export async function readFileAtRange( + range: vscode.Range, + filepath: string +): Promise { + return new Promise((resolve, reject) => { + fs.readFile(filepath, (err, data) => { + if (err) { + reject(err); + } else { + let lines = data.toString().split("\n"); + if (range.isSingleLine) { + resolve( + lines[range.start.line].slice( + range.start.character, + range.end.character + ) + ); + } else { + let firstLine = lines[range.start.line].slice(range.start.character); + let lastLine = lines[range.end.line].slice(0, range.end.character); + let middleLines = lines.slice(range.start.line + 1, range.end.line); + resolve([firstLine, ...middleLines, lastLine].join("\n")); + } + } + }); + }); +} + +export function openEditorAndRevealRange( + editorFilename: string, + range?: vscode.Range, + viewColumn?: vscode.ViewColumn +): Promise { + return new Promise((resolve, _) => { + // Check if the editor is already open + vscode.workspace.openTextDocument(editorFilename).then((doc) => { + vscode.window + .showTextDocument( + doc, + getViewColumnOfFile(editorFilename) || viewColumn + ) + .then((editor) => { + if (range) { + editor.revealRange(range); + } + resolve(editor); + }); + }); + }); +} -- cgit v1.2.3-70-g09d2 From dd5b9f6b7f08f178d6034a57f97faea38442eb0a Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Wed, 31 May 2023 16:13:01 -0400 Subject: checkpoint! protocol reform and it works now --- continuedev/src/continuedev/core/agent.py | 34 ++-- continuedev/src/continuedev/core/env.py | 4 + continuedev/src/continuedev/core/main.py | 14 +- continuedev/src/continuedev/core/policy.py | 17 +- continuedev/src/continuedev/core/sdk.py | 13 +- continuedev/src/continuedev/libs/steps/chroma.py | 2 +- .../src/continuedev/libs/steps/core/core.py | 33 ++-- .../src/continuedev/libs/steps/draft/dlt.py | 2 +- continuedev/src/continuedev/libs/steps/main.py | 20 +- .../src/continuedev/libs/steps/migration.py | 2 +- continuedev/src/continuedev/libs/steps/nate.py | 6 +- continuedev/src/continuedev/libs/steps/pytest.py | 2 +- continuedev/src/continuedev/libs/steps/ty.py | 2 +- continuedev/src/continuedev/server/ide.py | 50 ++--- continuedev/src/continuedev/server/main.py | 2 +- continuedev/src/continuedev/server/notebook.py | 207 +++++++-------------- .../src/continuedev/server/notebook_protocol.py | 28 +++ .../src/continuedev/server/session_manager.py | 101 ++++++++++ extension/package-lock.json | 19 ++ extension/package.json | 7 +- .../src/hooks/ContinueNotebookClientProtocol.ts | 13 ++ extension/react-app/src/hooks/messenger.ts | 108 +++++++++++ .../src/hooks/useContinueNotebookProtocol.ts | 49 +++++ extension/react-app/src/hooks/useWebsocket.ts | 171 +++-------------- extension/react-app/src/hooks/vscodeMessenger.ts | 68 +++++++ extension/react-app/src/tabs/notebook.tsx | 70 +++---- extension/react-app/src/vscode/index.ts | 1 + extension/react-app/tsconfig.json | 2 +- .../scripts/continuedev-0.1.0-py3-none-any.whl | Bin 53104 -> 56070 bytes extension/src/activation/activate.ts | 4 +- extension/src/activation/environmentSetup.ts | 104 +++++++---- extension/src/commands.ts | 4 +- extension/src/continueIdeClient.ts | 146 ++++----------- extension/src/debugPanel.ts | 26 ++- extension/src/extension.ts | 7 +- extension/src/test/runTest.ts | 30 +-- extension/src/util/messenger.ts | 108 +++++++++++ 37 files changed, 864 insertions(+), 612 deletions(-) create mode 100644 continuedev/src/continuedev/server/notebook_protocol.py create mode 100644 continuedev/src/continuedev/server/session_manager.py create mode 100644 extension/react-app/src/hooks/ContinueNotebookClientProtocol.ts create mode 100644 extension/react-app/src/hooks/messenger.ts create mode 100644 extension/react-app/src/hooks/useContinueNotebookProtocol.ts create mode 100644 extension/react-app/src/hooks/vscodeMessenger.ts create mode 100644 extension/src/util/messenger.ts (limited to 'extension/src/util') diff --git a/continuedev/src/continuedev/core/agent.py b/continuedev/src/continuedev/core/agent.py index 329e3d4c..7f7466a2 100644 --- a/continuedev/src/continuedev/core/agent.py +++ b/continuedev/src/continuedev/core/agent.py @@ -13,7 +13,6 @@ from .sdk import ContinueSDK class Agent(ContinueBaseModel): - llm: LLM policy: Policy ide: AbstractIdeProtocolServer history: History = History.from_empty() @@ -31,27 +30,24 @@ class Agent(ContinueBaseModel): def get_full_state(self) -> FullState: return FullState(history=self.history, active=self._active, user_input_queue=self._main_user_input_queue) - def on_update(self, callback: Callable[["FullState"], None]): + def on_update(self, callback: Coroutine["FullState", None, None]): """Subscribe to changes to state""" self._on_update_callbacks.append(callback) - def update_subscribers(self): + async def update_subscribers(self): full_state = self.get_full_state() for callback in self._on_update_callbacks: - callback(full_state) - - def __get_step_params(self, step: "Step"): - return ContinueSDK(agent=self, llm=self.llm.with_system_message(step.system_message)) + await callback(full_state) def give_user_input(self, input: str, index: int): - self._user_input_queue.post(index, input) + self._user_input_queue.post(str(index), input) async def wait_for_user_input(self) -> str: self._active = False - self.update_subscribers() - user_input = await self._user_input_queue.get(self.history.current_index) + await self.update_subscribers() + user_input = await self._user_input_queue.get(str(self.history.current_index)) self._active = True - self.update_subscribers() + await self.update_subscribers() return user_input _manual_edits_buffer: List[FileEditWithFullContents] = [] @@ -62,9 +58,9 @@ class Agent(ContinueBaseModel): current_step = self.history.get_current().step self.history.step_back() if issubclass(current_step.__class__, ReversibleStep): - await current_step.reverse(self.__get_step_params(current_step)) + await current_step.reverse(ContinueSDK(self)) - self.update_subscribers() + await self.update_subscribers() except Exception as e: print(e) @@ -94,17 +90,17 @@ class Agent(ContinueBaseModel): # Run step self._step_depth += 1 - observation = await step(self.__get_step_params(step)) + observation = await step(ContinueSDK(self)) self._step_depth -= 1 # Add observation to history self.history.get_current().observation = observation # Update its description - step._set_description(await step.describe(self.llm)) + step._set_description(await step.describe(ContinueSDK(self))) # Call all subscribed callbacks - self.update_subscribers() + await self.update_subscribers() return observation @@ -138,7 +134,7 @@ class Agent(ContinueBaseModel): # Doing this so active can make it to the frontend after steps are done. But want better state syncing tools for callback in self._on_update_callbacks: - callback(None) + await callback(None) async def run_from_observation(self, observation: Observation): next_step = self.policy.next(self.history) @@ -158,7 +154,7 @@ class Agent(ContinueBaseModel): async def accept_user_input(self, user_input: str): self._main_user_input_queue.append(user_input) - self.update_subscribers() + await self.update_subscribers() if len(self._main_user_input_queue) > 1: return @@ -167,7 +163,7 @@ class Agent(ContinueBaseModel): # Just run the step that takes user input, and # then up to the policy to decide how to deal with it. self._main_user_input_queue.pop(0) - self.update_subscribers() + await self.update_subscribers() await self.run_from_step(UserInputStep(user_input=user_input)) while len(self._main_user_input_queue) > 0: diff --git a/continuedev/src/continuedev/core/env.py b/continuedev/src/continuedev/core/env.py index 6267ed60..edd3297c 100644 --- a/continuedev/src/continuedev/core/env.py +++ b/continuedev/src/continuedev/core/env.py @@ -8,6 +8,10 @@ def get_env_var(var_name: str): def save_env_var(var_name: str, var_value: str): + if not os.path.exists('.env'): + with open('.env', 'w') as f: + f.write(f'{var_name}="{var_value}"\n') + return with open('.env', 'r') as f: lines = f.readlines() with open('.env', 'w') as f: diff --git a/continuedev/src/continuedev/core/main.py b/continuedev/src/continuedev/core/main.py index 51fcd299..6be5139b 100644 --- a/continuedev/src/continuedev/core/main.py +++ b/continuedev/src/continuedev/core/main.py @@ -72,7 +72,7 @@ class ContinueSDK: pass -class SequentialStep: +class Models: pass @@ -94,7 +94,7 @@ class Step(ContinueBaseModel): class Config: copy_on_model_validation = False - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: if self._description is not None: return self._description return "Running step: " + self.name @@ -135,6 +135,16 @@ class Step(ContinueBaseModel): return SequentialStep(steps=steps) +class SequentialStep(Step): + steps: list[Step] + hide: bool = True + + async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: + for step in self.steps: + observation = await sdk.run_step(step) + return observation + + class ValidatorObservation(Observation): passed: bool observation: Observation diff --git a/continuedev/src/continuedev/core/policy.py b/continuedev/src/continuedev/core/policy.py index 07101576..c0ba0f4f 100644 --- a/continuedev/src/continuedev/core/policy.py +++ b/continuedev/src/continuedev/core/policy.py @@ -1,20 +1,16 @@ from typing import List, Tuple, Type - -from ..models.main import ContinueBaseModel - from ..libs.steps.ty import CreatePipelineStep from .main import Step, Validator, History, Policy from .observation import Observation, TracebackObservation, UserInputObservation from ..libs.steps.main import EditHighlightedCodeStep, SolveTracebackStep, RunCodeStep, FasterEditHighlightedCodeStep, StarCoderEditHighlightedCodeStep from ..libs.steps.nate import WritePytestsStep, CreateTableStep -from ..libs.steps.chroma import AnswerQuestionChroma, EditFileChroma +# from ..libs.steps.chroma import AnswerQuestionChroma, EditFileChroma from ..libs.steps.continue_step import ContinueStepStep class DemoPolicy(Policy): ran_code_last: bool = False - cmd: str def next(self, history: History) -> Step: observation = history.last_observation() @@ -26,18 +22,15 @@ class DemoPolicy(Policy): return CreatePipelineStep() elif "/table" in observation.user_input: return CreateTableStep(sql_str=" ".join(observation.user_input.split(" ")[1:])) - elif "/ask" in observation.user_input: - return AnswerQuestionChroma(question=" ".join(observation.user_input.split(" ")[1:])) - elif "/edit" in observation.user_input: - return EditFileChroma(request=" ".join(observation.user_input.split(" ")[1:])) + # elif "/ask" in observation.user_input: + # return AnswerQuestionChroma(question=" ".join(observation.user_input.split(" ")[1:])) + # elif "/edit" in observation.user_input: + # return EditFileChroma(request=" ".join(observation.user_input.split(" ")[1:])) elif "/step" in observation.user_input: return ContinueStepStep(prompt=" ".join(observation.user_input.split(" ")[1:])) return StarCoderEditHighlightedCodeStep(user_input=observation.user_input) state = history.get_current() - if state is None or not self.ran_code_last: - self.ran_code_last = True - return RunCodeStep(cmd=self.cmd) if observation is not None and isinstance(observation, TracebackObservation): self.ran_code_last = False diff --git a/continuedev/src/continuedev/core/sdk.py b/continuedev/src/continuedev/core/sdk.py index 750b335d..6ae0be04 100644 --- a/continuedev/src/continuedev/core/sdk.py +++ b/continuedev/src/continuedev/core/sdk.py @@ -4,6 +4,7 @@ from ..models.filesystem_edit import FileSystemEdit, AddFile, DeleteFile, AddDir from ..models.filesystem import RangeInFile from ..libs.llm import LLM from ..libs.llm.hf_inference_api import HuggingFaceInferenceAPI +from ..libs.llm.openai import OpenAI from .observation import Observation from ..server.ide_protocol import AbstractIdeProtocolServer from .main import History, Step @@ -29,20 +30,20 @@ class Models: 'HUGGING_FACE_TOKEN', 'Please enter your Hugging Face token') return HuggingFaceInferenceAPI(api_key=api_key) + async def gpt35(self): + api_key = await self.sdk.get_user_secret( + 'OPENAI_API_KEY', 'Please enter your OpenAI API key') + return OpenAI(api_key=api_key, default_model="gpt-3.5-turbo") + class ContinueSDK: """The SDK provided as parameters to a step""" - llm: LLM ide: AbstractIdeProtocolServer steps: ContinueSDKSteps models: Models __agent: Agent - def __init__(self, agent: Agent, llm: Union[LLM, None] = None): - if llm is None: - self.llm = agent.llm - else: - self.llm = llm + def __init__(self, agent: Agent): self.ide = agent.ide self.__agent = agent self.steps = ContinueSDKSteps(self) diff --git a/continuedev/src/continuedev/libs/steps/chroma.py b/continuedev/src/continuedev/libs/steps/chroma.py index f13a2bab..39424c5c 100644 --- a/continuedev/src/continuedev/libs/steps/chroma.py +++ b/continuedev/src/continuedev/libs/steps/chroma.py @@ -40,7 +40,7 @@ class AnswerQuestionChroma(Step): Here is the answer:""") - answer = sdk.llm.complete(prompt) + answer = (await sdk.models.gpt35()).complete(prompt) print(answer) self._answer = answer diff --git a/continuedev/src/continuedev/libs/steps/core/core.py b/continuedev/src/continuedev/libs/steps/core/core.py index 0338d635..14b3cb80 100644 --- a/continuedev/src/continuedev/libs/steps/core/core.py +++ b/continuedev/src/continuedev/libs/steps/core/core.py @@ -4,27 +4,18 @@ from textwrap import dedent from typing import Coroutine, List, Union from ...llm.prompt_utils import MarkdownStyleEncoderDecoder -from ...util.traceback_parsers import parse_python_traceback - from ....models.filesystem_edit import EditDiff, FileEditWithFullContents, FileSystemEdit from ....models.filesystem import FileSystem, RangeInFile, RangeInFileWithContents -from ...llm import LLM from ....core.observation import Observation, TextObservation, TracebackObservation, UserInputObservation -from ....core.main import Step +from ....core.main import Step, SequentialStep class ContinueSDK: pass -class SequentialStep(Step): - steps: list[Step] - hide: bool = True - - async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: - for step in self.steps: - observation = await sdk.run_step(step) - return observation +class Models: + pass class ReversibleStep(Step): @@ -52,7 +43,7 @@ def ShellCommandsStep(Step): cwd: str | None = None name: str = "Run Shell Commands" - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "\n".join(self.cmds) async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -81,13 +72,13 @@ class EditCodeStep(Step): _prompt: Union[str, None] = None _completion: Union[str, None] = None - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: if self._edit_diffs is None: return "Editing files: " + ", ".join(map(lambda rif: rif.filepath, self.range_in_files)) elif len(self._edit_diffs) == 0: return "No edits made" else: - return llm.complete(dedent(f"""{self._prompt}{self._completion} + return (await models.gpt35()).complete(dedent(f"""{self._prompt}{self._completion} Maximally concise summary of changes in bullet points (can use markdown): """)) @@ -102,7 +93,7 @@ class EditCodeStep(Step): code_string = enc_dec.encode() prompt = self.prompt.format(code=code_string) - completion = sdk.llm.complete(prompt) + completion = (await sdk.models.gpt35()).complete(prompt) # Temporarily doing this to generate description. self._prompt = prompt @@ -127,7 +118,7 @@ class EditFileStep(Step): prompt: str hide: bool = True - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "Editing file: " + self.filepath async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -145,7 +136,7 @@ class ManualEditStep(ReversibleStep): hide: bool = True - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "Manual edit step" # TODO - only handling FileEdit here, but need all other types of FileSystemEdits # Also requires the merge_file_edit function @@ -181,7 +172,7 @@ class UserInputStep(Step): name: str = "User Input" hide: bool = True - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return self.user_input async def run(self, sdk: ContinueSDK) -> Coroutine[UserInputObservation, None, None]: @@ -194,7 +185,7 @@ class WaitForUserInputStep(Step): _description: Union[str, None] = None - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return self.prompt async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -207,7 +198,7 @@ class WaitForUserConfirmationStep(Step): prompt: str name: str = "Waiting for user confirmation" - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return self.prompt async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: diff --git a/continuedev/src/continuedev/libs/steps/draft/dlt.py b/continuedev/src/continuedev/libs/steps/draft/dlt.py index 5ba5692a..460aa0cc 100644 --- a/continuedev/src/continuedev/libs/steps/draft/dlt.py +++ b/continuedev/src/continuedev/libs/steps/draft/dlt.py @@ -10,7 +10,7 @@ class SetupPipelineStep(Step): api_description: str # e.g. "I want to load data from the weatherapi.com API" async def run(self, sdk: ContinueSDK): - source_name = sdk.llm.complete( + source_name = (await sdk.models.gpt35()).complete( f"Write a snake_case name for the data source described by {self.api_description}: ").strip() filename = f'{source_name}.py' diff --git a/continuedev/src/continuedev/libs/steps/main.py b/continuedev/src/continuedev/libs/steps/main.py index c8a85800..70c0d4b8 100644 --- a/continuedev/src/continuedev/libs/steps/main.py +++ b/continuedev/src/continuedev/libs/steps/main.py @@ -11,7 +11,7 @@ from ...core.observation import Observation, TextObservation, TracebackObservati from ..llm.prompt_utils import MarkdownStyleEncoderDecoder from textwrap import dedent from ...core.main import Step -from ...core.sdk import ContinueSDK +from ...core.sdk import ContinueSDK, Models from ...core.observation import Observation import subprocess from .core.core import EditCodeStep @@ -20,7 +20,7 @@ from .core.core import EditCodeStep class RunCodeStep(Step): cmd: str - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return f"Ran command: `{self.cmd}`" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -59,7 +59,7 @@ class RunCommandStep(Step): name: str = "Run command" _description: str = None - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: if self._description is not None: return self._description return self.cmd @@ -125,7 +125,7 @@ class FasterEditHighlightedCodeStep(Step): Here is the description of changes to make: """) - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "Editing highlighted code" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -154,7 +154,7 @@ class FasterEditHighlightedCodeStep(Step): for rif in rif_with_contents: rif_dict[rif.filepath] = rif.contents - completion = sdk.llm.complete(prompt) + completion = (await sdk.models.gpt35()).complete(prompt) # Temporarily doing this to generate description. self._prompt = prompt @@ -215,10 +215,10 @@ class FasterEditHighlightedCodeStep(Step): class StarCoderEditHighlightedCodeStep(Step): user_input: str - hide = True + hide = False _prompt: str = "{code}{user_request}" - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "Editing highlighted code" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -271,7 +271,7 @@ This is the user request: This is the code after being changed to perfectly satisfy the user request: """) - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "Editing highlighted code" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -293,7 +293,7 @@ This is the code after being changed to perfectly satisfy the user request: class FindCodeStep(Step): prompt: str - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return "Finding code" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: @@ -307,7 +307,7 @@ class UserInputStep(Step): class SolveTracebackStep(Step): traceback: Traceback - async def describe(self, llm: LLM) -> Coroutine[str, None, None]: + async def describe(self, models: Models) -> Coroutine[str, None, None]: return f"```\n{self.traceback.full_traceback}\n```" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: diff --git a/continuedev/src/continuedev/libs/steps/migration.py b/continuedev/src/continuedev/libs/steps/migration.py index f044a60f..7b70422d 100644 --- a/continuedev/src/continuedev/libs/steps/migration.py +++ b/continuedev/src/continuedev/libs/steps/migration.py @@ -15,7 +15,7 @@ class MigrationStep(Step): recent_edits = await sdk.ide.get_recent_edits(self.edited_file) recent_edits_string = "\n\n".join( map(lambda x: x.to_string(), recent_edits)) - description = await sdk.llm.complete(f"{recent_edits_string}\n\nGenerate a short description of the migration made in the above changes:\n") + description = await (await sdk.models.gpt35()).complete(f"{recent_edits_string}\n\nGenerate a short description of the migration made in the above changes:\n") await sdk.run_step(RunCommandStep(cmd=f"cd libs && poetry run alembic revision --autogenerate -m {description}")) migration_file = f"libs/alembic/versions/{?}.py" contents = await sdk.ide.readFile(migration_file) diff --git a/continuedev/src/continuedev/libs/steps/nate.py b/continuedev/src/continuedev/libs/steps/nate.py index a0e728e5..2f84e9d7 100644 --- a/continuedev/src/continuedev/libs/steps/nate.py +++ b/continuedev/src/continuedev/libs/steps/nate.py @@ -45,7 +45,7 @@ Here are additional instructions: Here is a complete set of pytest unit tests: """) - # tests = sdk.llm.complete(prompt) + # tests = (await sdk.models.gpt35()).complete(prompt) tests = ''' import pytest @@ -169,9 +169,9 @@ export class Order { tracking_number: string; }''' time.sleep(2) - # orm_entity = sdk.llm.complete( + # orm_entity = (await sdk.models.gpt35()).complete( # f"{self.sql_str}\n\nWrite a TypeORM entity called {entity_name} for this table, importing as necessary:") - # sdk.llm.complete("What is the name of the entity?") + # (await sdk.models.gpt35()).complete("What is the name of the entity?") await sdk.apply_filesystem_edit(AddFile(filepath=f"/Users/natesesti/Desktop/continue/extension/examples/python/MyProject/src/entity/{entity_name}.ts", content=orm_entity)) await sdk.ide.setFileOpen(f"/Users/natesesti/Desktop/continue/extension/examples/python/MyProject/src/entity/{entity_name}.ts", True) diff --git a/continuedev/src/continuedev/libs/steps/pytest.py b/continuedev/src/continuedev/libs/steps/pytest.py index b4e6dfd2..2e83ae2d 100644 --- a/continuedev/src/continuedev/libs/steps/pytest.py +++ b/continuedev/src/continuedev/libs/steps/pytest.py @@ -33,5 +33,5 @@ class WritePytestsStep(Step): Here is a complete set of pytest unit tests: """) - tests = sdk.llm.complete(prompt) + tests = (await sdk.models.gpt35()).complete(prompt) await sdk.apply_filesystem_edit(AddFile(filepath=path, content=tests)) diff --git a/continuedev/src/continuedev/libs/steps/ty.py b/continuedev/src/continuedev/libs/steps/ty.py index 5ff03f04..9dde7c86 100644 --- a/continuedev/src/continuedev/libs/steps/ty.py +++ b/continuedev/src/continuedev/libs/steps/ty.py @@ -18,7 +18,7 @@ class SetupPipelineStep(Step): api_description: str # e.g. "I want to load data from the weatherapi.com API" async def run(self, sdk: ContinueSDK): - # source_name = sdk.llm.complete( + # source_name = (await sdk.models.gpt35()).complete( # f"Write a snake_case name for the data source described by {self.api_description}: ").strip() filename = f'/Users/natesesti/Desktop/continue/extension/examples/python/{source_name}.py' diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index dd1dc463..50296841 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -1,5 +1,6 @@ # This is a separate server from server/main.py import asyncio +import json import os from typing import Any, Dict, List, Type, TypeVar, Union import uuid @@ -90,31 +91,33 @@ class IdeProtocolServer(AbstractIdeProtocolServer): def __init__(self, session_manager: SessionManager): self.session_manager = session_manager - async def _send_json(self, data: Any): - await self.websocket.send_json(data) + async def _send_json(self, message_type: str, data: Any): + await self.websocket.send_json({ + "messageType": message_type, + "data": data + }) async def _receive_json(self, message_type: str) -> Any: return await self.sub_queue.get(message_type) async def _send_and_receive_json(self, data: Any, resp_model: Type[T], message_type: str) -> T: - await self._send_json(data) + await self._send_json(message_type, data) resp = await self._receive_json(message_type) return resp_model.parse_obj(resp) - async def handle_json(self, data: Any): - t = data["messageType"] - if t == "openNotebook": + async def handle_json(self, message_type: str, data: Any): + if message_type == "openNotebook": await self.openNotebook() - elif t == "setFileOpen": + elif message_type == "setFileOpen": await self.setFileOpen(data["filepath"], data["open"]) - elif t == "fileEdits": + elif message_type == "fileEdits": fileEdits = list( map(lambda d: FileEditWithFullContents.parse_obj(d), data["fileEdits"])) self.onFileEdits(fileEdits) - elif t in ["highlightedCode", "openFiles", "readFile", "editFile", "workspaceDirectory"]: - self.sub_queue.post(t, data) + elif message_type in ["highlightedCode", "openFiles", "readFile", "editFile", "workspaceDirectory"]: + self.sub_queue.post(message_type, data) else: - raise ValueError("Unknown message type", t) + raise ValueError("Unknown message type", message_type) # ------------------------------- # # Request actions in IDE, doesn't matter which Session @@ -123,24 +126,21 @@ class IdeProtocolServer(AbstractIdeProtocolServer): async def setFileOpen(self, filepath: str, open: bool = True): # Agent needs access to this. - await self.websocket.send_json({ - "messageType": "setFileOpen", + await self._send_json("setFileOpen", { "filepath": filepath, "open": open }) async def openNotebook(self): session_id = self.session_manager.new_session(self) - await self._send_json({ - "messageType": "openNotebook", + await self._send_json("openNotebook", { "sessionId": session_id }) async def showSuggestionsAndWait(self, suggestions: List[FileEdit]) -> bool: ids = [str(uuid.uuid4()) for _ in suggestions] for i in range(len(suggestions)): - self._send_json({ - "messageType": "showSuggestion", + self._send_json("showSuggestion", { "suggestion": suggestions[i], "suggestionId": ids[i] }) @@ -210,8 +210,7 @@ class IdeProtocolServer(AbstractIdeProtocolServer): async def saveFile(self, filepath: str): """Save a file""" - await self._send_json({ - "messageType": "saveFile", + await self._send_json("saveFile", { "filepath": filepath }) @@ -293,10 +292,17 @@ ideProtocolServer = IdeProtocolServer(session_manager) async def websocket_endpoint(websocket: WebSocket): await websocket.accept() print("Accepted websocket connection from, ", websocket.client) - await websocket.send_json({"messageType": "connected"}) + await websocket.send_json({"messageType": "connected", "data": {}}) ideProtocolServer.websocket = websocket while True: - data = await websocket.receive_json() - await ideProtocolServer.handle_json(data) + message = await websocket.receive_text() + message = json.loads(message) + + if "messageType" not in message or "data" not in message: + continue + message_type = message["messageType"] + data = message["data"] + + await ideProtocolServer.handle_json(message_type, data) await websocket.close() diff --git a/continuedev/src/continuedev/server/main.py b/continuedev/src/continuedev/server/main.py index 11ad1d8f..e87d5fa9 100644 --- a/continuedev/src/continuedev/server/main.py +++ b/continuedev/src/continuedev/server/main.py @@ -32,7 +32,7 @@ args = parser.parse_args() def run_server(): - uvicorn.run(app, host="0.0.0.0", port=args.port, log_config="logging.ini") + uvicorn.run(app, host="0.0.0.0", port=args.port) if __name__ == "__main__": diff --git a/continuedev/src/continuedev/server/notebook.py b/continuedev/src/continuedev/server/notebook.py index c5dcea31..edb61a45 100644 --- a/continuedev/src/continuedev/server/notebook.py +++ b/continuedev/src/continuedev/server/notebook.py @@ -1,18 +1,12 @@ -from fastapi import FastAPI, Depends, Header, WebSocket, APIRouter -from typing import Any, Dict, List, Union -from uuid import uuid4 +import json +from fastapi import Depends, Header, WebSocket, APIRouter +from typing import Any, Type, TypeVar, Union from pydantic import BaseModel from uvicorn.main import Server -from ..models.filesystem_edit import FileEditWithFullContents -from ..core.policy import DemoPolicy -from ..core.main import FullState, History, Step -from ..core.agent import Agent -from ..libs.steps.nate import ImplementAbstractMethodStep -from ..core.observation import Observation -from ..libs.llm.openai import OpenAI -from .ide_protocol import AbstractIdeProtocolServer -from ..core.env import get_env_var +from .session_manager import SessionManager, session_manager, Session +from .notebook_protocol import AbstractNotebookProtocolServer +from ..libs.util.queue import AsyncSubscriptionQueue import asyncio import nest_asyncio nest_asyncio.apply() @@ -36,160 +30,99 @@ class AppStatus: Server.handle_exit = AppStatus.handle_exit -class Session: - session_id: str - agent: Agent - ws: Union[WebSocket, None] - - def __init__(self, session_id: str, agent: Agent): - self.session_id = session_id - self.agent = agent - self.ws = None - - -class DemoAgent(Agent): - first_seen: bool = False - cumulative_edit_string = "" - - def handle_manual_edits(self, edits: List[FileEditWithFullContents]): - for edit in edits: - self.cumulative_edit_string += edit.fileEdit.replacement - self._manual_edits_buffer.append(edit) - # Note that you're storing a lot of unecessary data here. Can compress into EditDiffs on the spot, and merge. - # self._manual_edits_buffer = merge_file_edit(self._manual_edits_buffer, edit) - # FOR DEMO PURPOSES - if edit.fileEdit.filepath.endswith("filesystem.py") and "List" in self.cumulative_edit_string and ":" in edit.fileEdit.replacement: - self.cumulative_edit_string = "" - asyncio.create_task(self.run_from_step( - ImplementAbstractMethodStep())) - - -class SessionManager: - sessions: Dict[str, Session] = {} - _event_loop: Union[asyncio.BaseEventLoop, None] = None - - def get_session(self, session_id: str) -> Session: - if session_id not in self.sessions: - raise KeyError("Session ID not recognized") - return self.sessions[session_id] - - def new_session(self, ide: AbstractIdeProtocolServer) -> str: - cmd = "python3 /Users/natesesti/Desktop/continue/extension/examples/python/main.py" - agent = DemoAgent(llm=OpenAI(api_key=get_env_var("OPENAI_API_KEY")), - policy=DemoPolicy(cmd=cmd), ide=ide) - session_id = str(uuid4()) - session = Session(session_id=session_id, agent=agent) - self.sessions[session_id] = session +def session(x_continue_session_id: str = Header("anonymous")) -> Session: + return session_manager.get_session(x_continue_session_id) - def on_update(state: FullState): - session_manager.send_ws_data(session_id, { - "messageType": "state", - "state": agent.get_full_state().dict() - }) - agent.on_update(on_update) - asyncio.create_task(agent.run_policy()) - return session_id +def websocket_session(session_id: str) -> Session: + return session_manager.get_session(session_id) - def remove_session(self, session_id: str): - del self.sessions[session_id] - def register_websocket(self, session_id: str, ws: WebSocket): - self.sessions[session_id].ws = ws - print("Registered websocket for session", session_id) +T = TypeVar("T", bound=BaseModel) - def send_ws_data(self, session_id: str, data: Any): - if self.sessions[session_id].ws is None: - print(f"Session {session_id} has no websocket") - return +# You should probably abstract away the websocket stuff into a separate class - async def a(): - await self.sessions[session_id].ws.send_json(data) - # Run coroutine in background - if self._event_loop is None or self._event_loop.is_closed(): - self._event_loop = asyncio.new_event_loop() - self._event_loop.run_until_complete(a()) - self._event_loop.close() - else: - self._event_loop.run_until_complete(a()) - self._event_loop.close() +class NotebookProtocolServer(AbstractNotebookProtocolServer): + websocket: WebSocket + session: Session + sub_queue: AsyncSubscriptionQueue = AsyncSubscriptionQueue() + def __init__(self, session: Session): + self.session = session -session_manager = SessionManager() + async def _send_json(self, data: Any): + await self.websocket.send_json(data) + async def _receive_json(self, message_type: str) -> Any: + return await self.sub_queue.get(message_type) -def session(x_continue_session_id: str = Header("anonymous")) -> Session: - return session_manager.get_session(x_continue_session_id) + async def _send_and_receive_json(self, data: Any, resp_model: Type[T], message_type: str) -> T: + await self._send_json(data) + resp = await self._receive_json(message_type) + return resp_model.parse_obj(resp) + def handle_json(self, message_type: str, data: Any): + try: + if message_type == "main_input": + self.on_main_input(data["input"]) + elif message_type == "step_user_input": + self.on_step_user_input(data["input"], data["index"]) + elif message_type == "refinement_input": + self.on_refinement_input(data["input"], data["index"]) + elif message_type == "reverse_to_index": + self.on_reverse_to_index(data["index"]) + except Exception as e: + print(e) -def websocket_session(session_id: str) -> Session: - return session_manager.get_session(session_id) + async def send_state_update(self): + state = self.session.agent.get_full_state().dict() + await self._send_json({ + "messageType": "state_update", + "state": state + }) + def on_main_input(self, input: str): + # Do something with user input + asyncio.create_task(self.session.agent.accept_user_input(input)) -class StartSessionBody(BaseModel): - config_file_path: Union[str, None] + def on_reverse_to_index(self, index: int): + # Reverse the history to the given index + asyncio.create_task(self.session.agent.reverse_to_index(index)) + def on_step_user_input(self, input: str, index: int): + asyncio.create_task( + self.session.agent.give_user_input(input, index)) -class StartSessionResp(BaseModel): - session_id: str + def on_refinement_input(self, input: str, index: int): + asyncio.create_task( + self.session.agent.accept_refinement_input(input, index)) @router.websocket("/ws") async def websocket_endpoint(websocket: WebSocket, session: Session = Depends(websocket_session)): await websocket.accept() + print("Session started") session_manager.register_websocket(session.session_id, websocket) - data = await websocket.receive_text() + protocol = NotebookProtocolServer(session) + protocol.websocket = websocket + # Update any history that may have happened before connection - await websocket.send_json({ - "messageType": "state", - "state": session_manager.get_session(session.session_id).agent.get_full_state().dict() - }) - print("Session started", data) + await protocol.send_state_update() + while AppStatus.should_exit is False: - data = await websocket.receive_json() - print("Received data", data) + message = await websocket.receive_json() + print("Received message", message) + if type(message) is str: + message = json.loads(message) - if "messageType" not in data: + if "messageType" not in message or "data" not in message: continue - messageType = data["messageType"] + message_type = message["messageType"] + data = message["data"] - try: - if messageType == "main_input": - # Do something with user input - asyncio.create_task( - session.agent.accept_user_input(data["value"])) - elif messageType == "step_user_input": - asyncio.create_task( - session.agent.give_user_input(data["value"], data["index"])) - elif messageType == "refinement_input": - asyncio.create_task( - session.agent.accept_refinement_input(data["value"], data["index"])) - elif messageType == "reverse": - # Reverse the history to the given index - asyncio.create_task( - session.agent.reverse_to_index(data["index"])) - except Exception as e: - print(e) + protocol.handle_json(message_type, data) print("Closing websocket") await websocket.close() - - -@router.post("/run") -def request_run(step: Step, session=Depends(session)): - """Tell an agent to take a specific action.""" - asyncio.create_task(session.agent.run_from_step(step)) - return "Success" - - -@router.get("/history") -def get_history(session=Depends(session)) -> History: - return session.agent.history - - -@router.post("/observation") -def post_observation(observation: Observation, session=Depends(session)): - asyncio.create_task(session.agent.run_from_observation(observation)) - return "Success" diff --git a/continuedev/src/continuedev/server/notebook_protocol.py b/continuedev/src/continuedev/server/notebook_protocol.py new file mode 100644 index 00000000..c2be82e0 --- /dev/null +++ b/continuedev/src/continuedev/server/notebook_protocol.py @@ -0,0 +1,28 @@ +from typing import Any +from abc import ABC, abstractmethod + + +class AbstractNotebookProtocolServer(ABC): + @abstractmethod + async def handle_json(self, data: Any): + """Handle a json message""" + + @abstractmethod + def on_main_input(self, input: str): + """Called when the user inputs something""" + + @abstractmethod + def on_reverse_to_index(self, index: int): + """Called when the user requests reverse to a previous index""" + + @abstractmethod + def on_refinement_input(self, input: str, index: int): + """Called when the user inputs a refinement""" + + @abstractmethod + def on_step_user_input(self, input: str, index: int): + """Called when the user inputs a step""" + + @abstractmethod + async def send_state_update(self, state: dict): + """Send a state update to the client""" diff --git a/continuedev/src/continuedev/server/session_manager.py b/continuedev/src/continuedev/server/session_manager.py new file mode 100644 index 00000000..b48c21b7 --- /dev/null +++ b/continuedev/src/continuedev/server/session_manager.py @@ -0,0 +1,101 @@ +from fastapi import WebSocket +from typing import Any, Dict, List, Union +from uuid import uuid4 + +from ..models.filesystem_edit import FileEditWithFullContents +from ..core.policy import DemoPolicy +from ..core.main import FullState +from ..core.agent import Agent +from ..libs.steps.nate import ImplementAbstractMethodStep +from .ide_protocol import AbstractIdeProtocolServer +import asyncio +import nest_asyncio +nest_asyncio.apply() + + +class Session: + session_id: str + agent: Agent + ws: Union[WebSocket, None] + + def __init__(self, session_id: str, agent: Agent): + self.session_id = session_id + self.agent = agent + self.ws = None + + +class DemoAgent(Agent): + first_seen: bool = False + cumulative_edit_string = "" + + def handle_manual_edits(self, edits: List[FileEditWithFullContents]): + for edit in edits: + self.cumulative_edit_string += edit.fileEdit.replacement + self._manual_edits_buffer.append(edit) + # Note that you're storing a lot of unecessary data here. Can compress into EditDiffs on the spot, and merge. + # self._manual_edits_buffer = merge_file_edit(self._manual_edits_buffer, edit) + # FOR DEMO PURPOSES + if edit.fileEdit.filepath.endswith("filesystem.py") and "List" in self.cumulative_edit_string and ":" in edit.fileEdit.replacement: + self.cumulative_edit_string = "" + asyncio.create_task(self.run_from_step( + ImplementAbstractMethodStep())) + + +class SessionManager: + sessions: Dict[str, Session] = {} + _event_loop: Union[asyncio.BaseEventLoop, None] = None + + def get_session(self, session_id: str) -> Session: + if session_id not in self.sessions: + raise KeyError("Session ID not recognized") + return self.sessions[session_id] + + def new_session(self, ide: AbstractIdeProtocolServer) -> str: + agent = DemoAgent(policy=DemoPolicy(), ide=ide) + session_id = str(uuid4()) + session = Session(session_id=session_id, agent=agent) + self.sessions[session_id] = session + + async def on_update(state: FullState): + await session_manager.send_ws_data(session_id, "state_update", { + "state": agent.get_full_state().dict() + }) + + agent.on_update(on_update) + asyncio.create_task(agent.run_policy()) + return session_id + + def remove_session(self, session_id: str): + del self.sessions[session_id] + + def register_websocket(self, session_id: str, ws: WebSocket): + self.sessions[session_id].ws = ws + print("Registered websocket for session", session_id) + + async def send_ws_data(self, session_id: str, message_type: str, data: Any): + if self.sessions[session_id].ws is None: + print(f"Session {session_id} has no websocket") + return + + async def a(): + await self.sessions[session_id].ws.send_json({ + "messageType": message_type, + "data": data + }) + + # Run coroutine in background + await self.sessions[session_id].ws.send_json({ + "messageType": message_type, + "data": data + }) + return + if self._event_loop is None or self._event_loop.is_closed(): + self._event_loop = asyncio.new_event_loop() + self._event_loop.run_until_complete(a()) + self._event_loop.close() + else: + self._event_loop.run_until_complete(a()) + self._event_loop.close() + + +session_manager = SessionManager() diff --git a/extension/package-lock.json b/extension/package-lock.json index 20ac24be..04af09d3 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -28,6 +28,7 @@ "@types/node": "16.x", "@types/node-fetch": "^2.6.2", "@types/vscode": "^1.74.0", + "@types/ws": "^8.5.4", "@typescript-eslint/eslint-plugin": "^5.45.0", "@typescript-eslint/parser": "^5.45.0", "@vscode/test-electron": "^2.2.0", @@ -2027,6 +2028,15 @@ "integrity": "sha512-LyeCIU3jb9d38w0MXFwta9r0Jx23ugujkAxdwLTNCyspdZTKUc43t7ppPbCiPoQ/Ivd/pnDFZrb4hWd45wrsgA==", "dev": true }, + "node_modules/@types/ws": { + "version": "8.5.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", + "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.48.2", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.48.2.tgz", @@ -9246,6 +9256,15 @@ "integrity": "sha512-LyeCIU3jb9d38w0MXFwta9r0Jx23ugujkAxdwLTNCyspdZTKUc43t7ppPbCiPoQ/Ivd/pnDFZrb4hWd45wrsgA==", "dev": true }, + "@types/ws": { + "version": "8.5.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", + "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@typescript-eslint/eslint-plugin": { "version": "5.48.2", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.48.2.tgz", diff --git a/extension/package.json b/extension/package.json index dc0192c3..c96655a9 100644 --- a/extension/package.json +++ b/extension/package.json @@ -148,7 +148,7 @@ }, "scripts": { "vscode:prepublish": "npm run esbuild-base -- --minify", - "esbuild-base": "esbuild ./src/extension.ts --bundle --outfile=out/extension.js --external:vscode --format=cjs --platform=node", + "esbuild-base": "rm -rf ./out && esbuild ./src/extension.ts --bundle --outfile=out/extension.js --external:vscode --format=cjs --platform=node", "esbuild": "rm -rf ./out && npm run esbuild-base -- --sourcemap", "esbuild-watch": "npm run esbuild-base -- --sourcemap --watch", "test-compile": "tsc -p ./", @@ -160,9 +160,9 @@ "pretest": "npm run compile && npm run lint", "lint": "eslint src --ext ts", "test": "node ./out/test/runTest.js", - "package": "cp ./config/prod_config.json ./config/config.json && mkdir -p ./build && vsce package --out ./build && chmod 777 ./build/continue-0.0.2.vsix && cp ./config/dev_config.json ./config/config.json", + "package": "cp ./config/prod_config.json ./config/config.json && mkdir -p ./build && vsce package --out ./build && chmod 777 ./build/continue-0.0.5.vsix && cp ./config/dev_config.json ./config/config.json", "full-package": "cd ../continuedev && poetry build && cp ./dist/continuedev-0.1.0-py3-none-any.whl ../extension/scripts/continuedev-0.1.0-py3-none-any.whl && cd ../extension && npm run typegen && npm run clientgen && cd react-app && npm run build && cd .. && npm run package", - "install-extension": "code --install-extension ./build/continue-0.0.1.vsix", + "install-extension": "code --install-extension ./build/continue-0.0.5.vsix", "uninstall": "code --uninstall-extension .continue", "reinstall": "rm -rf ./build && npm run package && npm run uninstall && npm run install-extension" }, @@ -173,6 +173,7 @@ "@types/node": "16.x", "@types/node-fetch": "^2.6.2", "@types/vscode": "^1.74.0", + "@types/ws": "^8.5.4", "@typescript-eslint/eslint-plugin": "^5.45.0", "@typescript-eslint/parser": "^5.45.0", "@vscode/test-electron": "^2.2.0", diff --git a/extension/react-app/src/hooks/ContinueNotebookClientProtocol.ts b/extension/react-app/src/hooks/ContinueNotebookClientProtocol.ts new file mode 100644 index 00000000..75fd7373 --- /dev/null +++ b/extension/react-app/src/hooks/ContinueNotebookClientProtocol.ts @@ -0,0 +1,13 @@ +abstract class AbstractContinueNotebookClientProtocol { + abstract sendMainInput(input: string): void; + + abstract reverseToIndex(index: number): void; + + abstract sendRefinementInput(input: string, index: number): void; + + abstract sendStepUserInput(input: string, index: number): void; + + abstract onStateUpdate(state: any): void; +} + +export default AbstractContinueNotebookClientProtocol; diff --git a/extension/react-app/src/hooks/messenger.ts b/extension/react-app/src/hooks/messenger.ts new file mode 100644 index 00000000..e2a0bab8 --- /dev/null +++ b/extension/react-app/src/hooks/messenger.ts @@ -0,0 +1,108 @@ +// console.log("Websocket import"); +// const WebSocket = require("ws"); + +export abstract class Messenger { + abstract send(messageType: string, data: object): void; + + abstract onMessageType( + messageType: string, + callback: (data: object) => void + ): void; + + abstract onMessage(callback: (messageType: string, data: any) => void): void; + + abstract onOpen(callback: () => void): void; + + abstract onClose(callback: () => void): void; + + abstract sendAndReceive(messageType: string, data: any): Promise; +} + +export class WebsocketMessenger extends Messenger { + websocket: WebSocket; + private onMessageListeners: { + [messageType: string]: ((data: object) => void)[]; + } = {}; + private onOpenListeners: (() => void)[] = []; + private onCloseListeners: (() => void)[] = []; + private serverUrl: string; + + _newWebsocket(): WebSocket { + // // Dynamic import, because WebSocket is builtin with browser, but not with node. And can't use require in browser. + // if (typeof process === "object") { + // console.log("Using node"); + // // process is only available in Node + // var WebSocket = require("ws"); + // } + + const newWebsocket = new WebSocket(this.serverUrl); + for (const listener of this.onOpenListeners) { + this.onOpen(listener); + } + for (const listener of this.onCloseListeners) { + this.onClose(listener); + } + for (const messageType in this.onMessageListeners) { + for (const listener of this.onMessageListeners[messageType]) { + this.onMessageType(messageType, listener); + } + } + return newWebsocket; + } + + constructor(serverUrl: string) { + super(); + this.serverUrl = serverUrl; + this.websocket = this._newWebsocket(); + } + + send(messageType: string, data: object) { + const payload = JSON.stringify({ messageType, data }); + if (this.websocket.readyState === this.websocket.OPEN) { + this.websocket.send(payload); + } else { + if (this.websocket.readyState !== this.websocket.CONNECTING) { + this.websocket = this._newWebsocket(); + } + this.websocket.addEventListener("open", () => { + this.websocket.send(payload); + }); + } + } + + sendAndReceive(messageType: string, data: any): Promise { + return new Promise((resolve, reject) => { + const eventListener = (data: any) => { + // THIS ISN"T GETTING CALLED + resolve(data); + this.websocket.removeEventListener("message", eventListener); + }; + this.onMessageType(messageType, eventListener); + this.send(messageType, data); + }); + } + + onMessageType(messageType: string, callback: (data: any) => void): void { + this.websocket.addEventListener("message", (event: any) => { + const msg = JSON.parse(event.data); + if (msg.messageType === messageType) { + callback(msg.data); + } + }); + } + + onMessage(callback: (messageType: string, data: any) => void): void { + this.websocket.addEventListener("message", (event) => { + const msg = JSON.parse(event.data); + callback(msg.messageType, msg.data); + }); + } + + onOpen(callback: () => void): void { + this.websocket.addEventListener("open", callback); + } + + onClose(callback: () => void): void { + this.websocket.addEventListener("close", callback); + } +} diff --git a/extension/react-app/src/hooks/useContinueNotebookProtocol.ts b/extension/react-app/src/hooks/useContinueNotebookProtocol.ts new file mode 100644 index 00000000..d5ffbf09 --- /dev/null +++ b/extension/react-app/src/hooks/useContinueNotebookProtocol.ts @@ -0,0 +1,49 @@ +import AbstractContinueNotebookClientProtocol from "./ContinueNotebookClientProtocol"; +// import { Messenger, WebsocketMessenger } from "../../../src/util/messenger"; +import { Messenger, WebsocketMessenger } from "./messenger"; +import { VscodeMessenger } from "./vscodeMessenger"; + +class ContinueNotebookClientProtocol extends AbstractContinueNotebookClientProtocol { + messenger: Messenger; + // Server URL must contain the session ID param + serverUrlWithSessionId: string; + + constructor( + serverUrlWithSessionId: string, + useVscodeMessagePassing: boolean = false + ) { + super(); + this.serverUrlWithSessionId = serverUrlWithSessionId; + if (useVscodeMessagePassing) { + this.messenger = new VscodeMessenger(serverUrlWithSessionId); + } else { + this.messenger = new WebsocketMessenger(serverUrlWithSessionId); + } + } + + sendMainInput(input: string) { + this.messenger.send("main_input", { input }); + } + + reverseToIndex(index: number) { + this.messenger.send("reverse_to_index", { index }); + } + + sendRefinementInput(input: string, index: number) { + this.messenger.send("refinement_input", { input, index }); + } + + sendStepUserInput(input: string, index: number) { + this.messenger.send("step_user_input", { input, index }); + } + + onStateUpdate(callback: (state: any) => void) { + this.messenger.onMessageType("state_update", (data: any) => { + if (data.state) { + callback(data.state); + } + }); + } +} + +export default ContinueNotebookClientProtocol; diff --git a/extension/react-app/src/hooks/useWebsocket.ts b/extension/react-app/src/hooks/useWebsocket.ts index 6e8e68fa..b98be577 100644 --- a/extension/react-app/src/hooks/useWebsocket.ts +++ b/extension/react-app/src/hooks/useWebsocket.ts @@ -1,158 +1,39 @@ import React, { useEffect, useState } from "react"; import { RootStore } from "../redux/store"; import { useSelector } from "react-redux"; +import ContinueNotebookClientProtocol from "./useContinueNotebookProtocol"; import { postVscMessage } from "../vscode"; -abstract class Messenger { - abstract send(data: string): void; -} - -class VscodeMessenger extends Messenger { - url: string; - - constructor( - url: string, - onMessage: (message: { data: any }) => void, - onOpen: (messenger: Messenger) => void, - onClose: (messenger: Messenger) => void - ) { - super(); - this.url = url; - window.addEventListener("message", (event: any) => { - switch (event.data.type) { - case "websocketForwardingMessage": - onMessage(event.data); - break; - case "websocketForwardingOpen": - onOpen(this); - break; - case "websocketForwardingClose": - onClose(this); - break; - } - }); - - postVscMessage("websocketForwardingOpen", { url: this.url }); - } - - send(data: string) { - postVscMessage("websocketForwardingMessage", { - message: data, - url: this.url, - }); - } -} - -class WebsocketMessenger extends Messenger { - websocket: WebSocket; - constructor( - websocket: WebSocket, - onMessage: (message: { data: any }) => void, - onOpen: (messenger: Messenger) => void, - onClose: (messenger: Messenger) => void - ) { - super(); - this.websocket = websocket; - - websocket.addEventListener("close", () => { - onClose(this); - }); - - websocket.addEventListener("open", () => { - onOpen(this); - }); - - websocket.addEventListener("message", (event) => { - onMessage(event.data); - }); - } - - static async connect( - url: string, - sessionId: string, - onMessage: (message: { data: any }) => void, - onOpen: (messenger: Messenger) => void, - onClose: (messenger: Messenger) => void - ): Promise { - const ws = new WebSocket(url); - - return new Promise((resolve, reject) => { - ws.addEventListener("open", () => { - resolve(new WebsocketMessenger(ws, onMessage, onOpen, onClose)); - }); - }); - } - - send(data: string) { - this.websocket.send(JSON.stringify(data)); - } -} - -function useContinueWebsocket( - serverUrl: string, - onMessage: (message: { data: any }) => void, - useVscodeMessagePassing: boolean = true -) { +function useContinueNotebookProtocol(useVscodeMessagePassing: boolean = false) { const sessionId = useSelector((state: RootStore) => state.config.sessionId); - const [websocket, setWebsocket] = useState(undefined); + const serverHttpUrl = useSelector((state: RootStore) => state.config.apiUrl); + const [client, setClient] = useState< + ContinueNotebookClientProtocol | undefined + >(undefined); - async function connect() { - while (!sessionId) { - await new Promise((resolve) => setTimeout(resolve, 300)); + useEffect(() => { + if (!sessionId || !serverHttpUrl) { + if (useVscodeMessagePassing) { + postVscMessage("onLoad", {}); + } + setClient(undefined); + return; } - console.log("Creating websocket", sessionId); - console.log("Using vscode message passing", useVscodeMessagePassing); - - const onClose = (messenger: Messenger) => { - console.log("Websocket closed"); - setWebsocket(undefined); - }; - - const onOpen = (messenger: Messenger) => { - console.log("Websocket opened"); - messenger.send(JSON.stringify({ sessionId })); - }; - - const url = - serverUrl.replace("http", "ws") + + const serverUrlWithSessionId = + serverHttpUrl.replace("http", "ws") + "/notebook/ws?session_id=" + encodeURIComponent(sessionId); - const messenger: Messenger = useVscodeMessagePassing - ? new VscodeMessenger(url, onMessage, onOpen, onClose) - : await WebsocketMessenger.connect( - url, - sessionId, - onMessage, - onOpen, - onClose - ); - - setWebsocket(messenger); - - return messenger; - } - - async function getConnection() { - if (!websocket) { - return await connect(); - } - return websocket; - } - - async function send(message: object) { - let ws = await getConnection(); - ws.send(JSON.stringify(message)); - } - - useEffect(() => { - if (!sessionId) { - return; - } - connect(); - }, [sessionId]); - - return { send }; + console.log("Creating websocket", serverUrlWithSessionId); + console.log("Using vscode message passing", useVscodeMessagePassing); + const newClient = new ContinueNotebookClientProtocol( + serverUrlWithSessionId, + useVscodeMessagePassing + ); + setClient(newClient); + }, [sessionId, serverHttpUrl]); + + return client; } -export default useContinueWebsocket; +export default useContinueNotebookProtocol; diff --git a/extension/react-app/src/hooks/vscodeMessenger.ts b/extension/react-app/src/hooks/vscodeMessenger.ts new file mode 100644 index 00000000..746c4302 --- /dev/null +++ b/extension/react-app/src/hooks/vscodeMessenger.ts @@ -0,0 +1,68 @@ +import { postVscMessage } from "../vscode"; +// import { Messenger } from "../../../src/util/messenger"; +import { Messenger } from "./messenger"; + +export class VscodeMessenger extends Messenger { + serverUrl: string; + + constructor(serverUrl: string) { + super(); + this.serverUrl = serverUrl; + postVscMessage("websocketForwardingOpen", { url: this.serverUrl }); + } + + send(messageType: string, data: object) { + postVscMessage("websocketForwardingMessage", { + message: { messageType, data }, + url: this.serverUrl, + }); + } + + onMessageType(messageType: string, callback: (data: object) => void): void { + window.addEventListener("message", (event: any) => { + if (event.data.type === "websocketForwardingMessage") { + if (event.data.message.messageType === messageType) { + callback(event.data.message.data); + } + } + }); + } + + onMessage(callback: (messageType: string, data: any) => void): void { + window.addEventListener("message", (event: any) => { + if (event.data.type === "websocketForwardingMessage") { + callback(event.data.message.messageType, event.data.message.data); + } + }); + } + + sendAndReceive(messageType: string, data: any): Promise { + return new Promise((resolve) => { + const handler = (event: any) => { + if (event.data.type === "websocketForwardingMessage") { + if (event.data.message.messageType === messageType) { + window.removeEventListener("message", handler); + resolve(event.data.message.data); + } + } + }; + window.addEventListener("message", handler); + this.send(messageType, data); + }); + } + + onOpen(callback: () => void): void { + window.addEventListener("message", (event: any) => { + if (event.data.type === "websocketForwardingOpen") { + callback(); + } + }); + } + onClose(callback: () => void): void { + window.addEventListener("message", (event: any) => { + if (event.data.type === "websocketForwardingClose") { + callback(); + } + }); + } +} diff --git a/extension/react-app/src/tabs/notebook.tsx b/extension/react-app/src/tabs/notebook.tsx index a9c69c5b..02c9ff31 100644 --- a/extension/react-app/src/tabs/notebook.tsx +++ b/extension/react-app/src/tabs/notebook.tsx @@ -14,6 +14,7 @@ import StepContainer from "../components/StepContainer"; import { useSelector } from "react-redux"; import { RootStore } from "../redux/store"; import useContinueWebsocket from "../hooks/useWebsocket"; +import useContinueNotebookProtocol from "../hooks/useWebsocket"; let TopNotebookDiv = styled.div` display: grid; @@ -33,8 +34,6 @@ interface NotebookProps { } function Notebook(props: NotebookProps) { - const serverUrl = useSelector((state: RootStore) => state.config.apiUrl); - const [waitingForSteps, setWaitingForSteps] = useState(false); const [userInputQueue, setUserInputQueue] = useState([]); const [history, setHistory] = useState(); @@ -157,30 +156,17 @@ function Notebook(props: NotebookProps) { // } as any // ); - const { send: websocketSend } = useContinueWebsocket(serverUrl, (msg) => { - let data = JSON.parse(msg.data); - if (data.messageType === "state") { - setWaitingForSteps(data.state.active); - setHistory(data.state.history); - setUserInputQueue(data.state.user_input_queue); - } - }); + const client = useContinueNotebookProtocol(); - // useEffect(() => { - // (async () => { - // if (sessionId && props.firstObservation) { - // let resp = await fetch(serverUrl + "/observation", { - // method: "POST", - // headers: new Headers({ - // "x-continue-session-id": sessionId, - // }), - // body: JSON.stringify({ - // observation: props.firstObservation, - // }), - // }); - // } - // })(); - // }, [props.firstObservation]); + useEffect(() => { + console.log("CLIENT ON STATE UPDATE: ", client, client?.onStateUpdate); + client?.onStateUpdate((state) => { + console.log("Received state update: ", state); + setWaitingForSteps(state.active); + setHistory(state.history); + setUserInputQueue(state.user_input_queue); + }); + }, [client]); const mainTextInputRef = useRef(null); @@ -201,14 +187,12 @@ function Notebook(props: NotebookProps) { const onMainTextInput = () => { if (mainTextInputRef.current) { - let value = mainTextInputRef.current.value; + if (!client) return; + let input = mainTextInputRef.current.value; setWaitingForSteps(true); - websocketSend({ - messageType: "main_input", - value: value, - }); + client.sendMainInput(input); setUserInputQueue((queue) => { - return [...queue, value]; + return [...queue, input]; }); mainTextInputRef.current.value = ""; mainTextInputRef.current.style.height = ""; @@ -216,17 +200,20 @@ function Notebook(props: NotebookProps) { }; const onStepUserInput = (input: string, index: number) => { + if (!client) return; console.log("Sending step user input", input, index); - websocketSend({ - messageType: "step_user_input", - value: input, - index, - }); + client.sendStepUserInput(input, index); }; // const iterations = useSelector(selectIterations); return ( + {typeof client === "undefined" && ( + <> + +

Server disconnected

+ + )} {history?.timeline.map((node: HistoryNode, index: number) => { return ( history?.current_index} historyNode={node} onRefinement={(input: string) => { - websocketSend({ - messageType: "refinement_input", - value: input, - index, - }); + client?.sendRefinementInput(input, index); }} onReverse={() => { - websocketSend({ - messageType: "reverse", - index, - }); + client?.reverseToIndex(index); }} /> ); diff --git a/extension/react-app/src/vscode/index.ts b/extension/react-app/src/vscode/index.ts index 7e373cd9..0785aa4d 100644 --- a/extension/react-app/src/vscode/index.ts +++ b/extension/react-app/src/vscode/index.ts @@ -5,6 +5,7 @@ declare const vscode: any; export function postVscMessage(type: string, data: any) { if (typeof vscode === "undefined") { + console.log("Unable to send message: vscode is undefined"); return; } vscode.postMessage({ diff --git a/extension/react-app/tsconfig.json b/extension/react-app/tsconfig.json index 3d0a51a8..940a9359 100644 --- a/extension/react-app/tsconfig.json +++ b/extension/react-app/tsconfig.json @@ -16,6 +16,6 @@ "noEmit": true, "jsx": "react-jsx" }, - "include": ["src"], + "include": ["src", "../src/util/messenger.ts"], "references": [{ "path": "./tsconfig.node.json" }] } diff --git a/extension/scripts/continuedev-0.1.0-py3-none-any.whl b/extension/scripts/continuedev-0.1.0-py3-none-any.whl index d1483db9..2019c904 100644 Binary files a/extension/scripts/continuedev-0.1.0-py3-none-any.whl and b/extension/scripts/continuedev-0.1.0-py3-none-any.whl differ diff --git a/extension/src/activation/activate.ts b/extension/src/activation/activate.ts index a0aa560b..712ffe13 100644 --- a/extension/src/activation/activate.ts +++ b/extension/src/activation/activate.ts @@ -10,7 +10,7 @@ import { getContinueServerUrl } from "../bridge"; export let extensionContext: vscode.ExtensionContext | undefined = undefined; -export let ideProtocolClient: IdeProtocolClient | undefined = undefined; +export let ideProtocolClient: IdeProtocolClient; export function activateExtension( context: vscode.ExtensionContext, @@ -24,7 +24,7 @@ export function activateExtension( let serverUrl = getContinueServerUrl(); ideProtocolClient = new IdeProtocolClient( - serverUrl.replace("http", "ws") + "/ide/ws", + `${serverUrl.replace("http", "ws")}/ide/ws`, context ); diff --git a/extension/src/activation/environmentSetup.ts b/extension/src/activation/environmentSetup.ts index 93a471ff..ad6ac71b 100644 --- a/extension/src/activation/environmentSetup.ts +++ b/extension/src/activation/environmentSetup.ts @@ -28,18 +28,7 @@ async function runCommand(cmd: string): Promise<[string, string | undefined]> { return [stdout, stderr]; } -async function getPythonCmdAssumingInstalled() { - const [, stderr] = await runCommand("python3 --version"); - if (stderr) { - return "python"; - } - return "python3"; -} - -async function setupPythonEnv() { - console.log("Setting up python env for Continue extension..."); - // First check that python3 is installed - +async function getPythonPipCommands() { var [stdout, stderr] = await runCommand("python3 --version"); let pythonCmd = "python3"; if (stderr) { @@ -58,28 +47,77 @@ async function setupPythonEnv() { } } let pipCmd = pythonCmd.endsWith("3") ? "pip3" : "pip"; + return [pythonCmd, pipCmd]; +} +function getActivateUpgradeCommands(pythonCmd: string, pipCmd: string) { let activateCmd = ". env/bin/activate"; let pipUpgradeCmd = `${pipCmd} install --upgrade pip`; if (process.platform == "win32") { activateCmd = ".\\env\\Scripts\\activate"; pipUpgradeCmd = `${pythonCmd} -m pip install --upgrade pip`; } + return [activateCmd, pipUpgradeCmd]; +} - let command = `cd ${path.join( +function checkEnvExists() { + const envBinActivatePath = path.join( getExtensionUri().fsPath, - "scripts" - )} && ${pythonCmd} -m venv env && ${activateCmd} && ${pipUpgradeCmd} && ${pipCmd} install -r requirements.txt`; - var [stdout, stderr] = await runCommand(command); - if (stderr) { - throw new Error(stderr); + "scripts", + "env", + "bin", + "activate" + ); + return fs.existsSync(envBinActivatePath); +} + +async function setupPythonEnv() { + console.log("Setting up python env for Continue extension..."); + + // Assemble the command to create the env + const [pythonCmd, pipCmd] = await getPythonPipCommands(); + const [activateCmd, pipUpgradeCmd] = getActivateUpgradeCommands( + pythonCmd, + pipCmd + ); + const createEnvCommand = [ + `cd ${path.join(getExtensionUri().fsPath, "scripts")}`, + `${pythonCmd} -m venv env`, + ].join(" && "); + + // Repeat until it is successfully created (sometimes it fails to generate the bin, need to try again) + while (true) { + const [, stderr] = await runCommand(createEnvCommand); + if (stderr) { + throw new Error(stderr); + } + if (checkEnvExists()) { + break; + } else { + // Remove the env and try again + const removeCommand = `rm -rf ${path.join( + getExtensionUri().fsPath, + "scripts", + "env" + )}`; + await runCommand(removeCommand); + } } console.log( "Successfully set up python env at ", getExtensionUri().fsPath + "/scripts/env" ); - await startContinuePythonServer(); + const installRequirementsCommand = [ + `cd ${path.join(getExtensionUri().fsPath, "scripts")}`, + activateCmd, + pipUpgradeCmd, + `${pipCmd} install -r requirements.txt`, + ].join(" && "); + const [, stderr] = await runCommand(installRequirementsCommand); + if (stderr) { + throw new Error(stderr); + } } function readEnvFile(path: string) { @@ -116,29 +154,19 @@ function writeEnvFile(path: string, key: string, value: string) { } export async function startContinuePythonServer() { + await setupPythonEnv(); + // Check vscode settings let serverUrl = getContinueServerUrl(); if (serverUrl !== "http://localhost:8000") { return; } - let envFile = path.join(getExtensionUri().fsPath, "scripts", ".env"); - let openai_api_key: string | undefined = - readEnvFile(envFile)["OPENAI_API_KEY"]; - while (typeof openai_api_key === "undefined" || openai_api_key === "") { - openai_api_key = await vscode.window.showInputBox({ - prompt: "Enter your OpenAI API key", - placeHolder: "Enter your OpenAI API key", - }); - // Write to .env file - } - writeEnvFile(envFile, "OPENAI_API_KEY", openai_api_key); - console.log("Starting Continue python server..."); // Check if already running by calling /health try { - let response = await fetch(serverUrl + "/health"); + const response = await fetch(serverUrl + "/health"); if (response.status === 200) { console.log("Continue python server already running"); return; @@ -152,15 +180,18 @@ export async function startContinuePythonServer() { pythonCmd = "python"; } + // let command = `cd ${path.join( + // getExtensionUri().fsPath, + // "scripts" + // )} && ${activateCmd} && cd env/lib/python3.11/site-packages && ${pythonCmd} -m continuedev.server.main`; let command = `cd ${path.join( getExtensionUri().fsPath, "scripts" )} && ${activateCmd} && cd .. && ${pythonCmd} -m scripts.run_continue_server`; try { // exec(command); - let child = spawn(command, { + const child = spawn(command, { shell: true, - detached: true, }); child.stdout.on("data", (data: any) => { console.log(`stdout: ${data}`); @@ -194,11 +225,6 @@ export function isPythonEnvSetup(): boolean { return fs.existsSync(path.join(pathToEnvCfg)); } -export async function setupExtensionEnvironment() { - console.log("Setting up environment for Continue extension..."); - await Promise.all([setupPythonEnv()]); -} - export async function downloadPython3() { // Download python3 and return the command to run it (python or python3) let os = process.platform; diff --git a/extension/src/commands.ts b/extension/src/commands.ts index 18f08e31..aeeb4b4f 100644 --- a/extension/src/commands.ts +++ b/extension/src/commands.ts @@ -62,11 +62,11 @@ const commandsMap: { [command: string]: (...args: any) => any } = { "continue.acceptSuggestion": acceptSuggestionCommand, "continue.rejectSuggestion": rejectSuggestionCommand, "continue.openDebugPanel": () => { - ideProtocolClient?.openNotebook(); + ideProtocolClient.openNotebook(); }, "continue.focusContinueInput": async () => { if (!debugPanelWebview) { - await ideProtocolClient?.openNotebook(); + await ideProtocolClient.openNotebook(); } debugPanelWebview?.postMessage({ type: "focusContinueInput", diff --git a/extension/src/continueIdeClient.ts b/extension/src/continueIdeClient.ts index 35eb668d..477d1420 100644 --- a/extension/src/continueIdeClient.ts +++ b/extension/src/continueIdeClient.ts @@ -10,30 +10,28 @@ import { } from "./suggestions"; import { debugPanelWebview, setupDebugPanel } from "./debugPanel"; import { FileEditWithFullContents } from "../schema/FileEditWithFullContents"; -const util = require("util"); -const exec = util.promisify(require("child_process").exec); -const WebSocket = require("ws"); import fs = require("fs"); +import { WebsocketMessenger } from "./util/messenger"; class IdeProtocolClient { - private _ws: WebSocket | null = null; - private _panels: Map = new Map(); - private readonly _serverUrl: string; - private readonly _context: vscode.ExtensionContext; + private messenger: WebsocketMessenger | null = null; + private panels: Map = new Map(); + private readonly context: vscode.ExtensionContext; private _makingEdit = 0; constructor(serverUrl: string, context: vscode.ExtensionContext) { - this._context = context; - this._serverUrl = serverUrl; - let ws = new WebSocket(serverUrl); - this._ws = ws; - ws.onclose = () => { - this._ws = null; - }; - ws.on("message", (data: any) => { - this.handleMessage(JSON.parse(data)); + this.context = context; + + let messenger = new WebsocketMessenger(serverUrl); + this.messenger = messenger; + messenger.onClose(() => { + this.messenger = null; + }); + messenger.onMessage((messageType, data) => { + this.handleMessage(messageType, data); }); + // Setup listeners for any file changes in open editors vscode.workspace.onDidChangeTextDocument((event) => { if (this._makingEdit === 0) { @@ -58,125 +56,52 @@ class IdeProtocolClient { }; } ); - this.send("fileEdits", { fileEdits }); + this.messenger?.send("fileEdits", { fileEdits }); } else { this._makingEdit--; } }); } - async isConnected() { - if (this._ws === null || this._ws.readyState !== WebSocket.OPEN) { - let ws = new WebSocket(this._serverUrl); - ws.onclose = () => { - this._ws = null; - }; - ws.on("message", (data: any) => { - this.handleMessage(JSON.parse(data)); - }); - this._ws = ws; - - return new Promise((resolve, reject) => { - ws.addEventListener("open", () => { - resolve(null); - }); - }); - } - } - - async startCore() { - var { stdout, stderr } = await exec( - "cd /Users/natesesti/Desktop/continue/continue && poetry shell" - ); - if (stderr) { - throw new Error(stderr); - } - var { stdout, stderr } = await exec( - "cd .. && uvicorn continue.src.server.main:app --reload --reload-dir continue" - ); - if (stderr) { - throw new Error(stderr); - } - var { stdout, stderr } = await exec("python3 -m continue.src.libs.ide"); - if (stderr) { - throw new Error(stderr); - } - } - - async send(messageType: string, data: object) { - await this.isConnected(); - let msg = JSON.stringify({ messageType, ...data }); - this._ws!.send(msg); - console.log("Sent message", msg); - } - - async receiveMessage(messageType: string): Promise { - await this.isConnected(); - console.log("Connected to websocket"); - return await new Promise((resolve, reject) => { - if (!this._ws) { - reject("Not connected to websocket"); - } - this._ws!.onmessage = (event: any) => { - let message = JSON.parse(event.data); - console.log("RECEIVED MESSAGE", message); - if (message.messageType === messageType) { - resolve(message); - } - }; - }); - } - - async sendAndReceive(message: any, messageType: string): Promise { - try { - await this.send(messageType, message); - let msg = await this.receiveMessage(messageType); - console.log("Received message", msg); - return msg; - } catch (e) { - console.log("Error sending message", e); - } - } - - async handleMessage(message: any) { - switch (message.messageType) { + async handleMessage(messageType: string, data: any) { + switch (messageType) { case "highlightedCode": - this.send("highlightedCode", { + this.messenger?.send("highlightedCode", { highlightedCode: this.getHighlightedCode(), }); break; case "workspaceDirectory": - this.send("workspaceDirectory", { + this.messenger?.send("workspaceDirectory", { workspaceDirectory: this.getWorkspaceDirectory(), }); case "openFiles": - this.send("openFiles", { + this.messenger?.send("openFiles", { openFiles: this.getOpenFiles(), }); break; case "readFile": - this.send("readFile", { - contents: this.readFile(message.filepath), + this.messenger?.send("readFile", { + contents: this.readFile(data.filepath), }); break; case "editFile": - let fileEdit = await this.editFile(message.edit); - this.send("editFile", { + const fileEdit = await this.editFile(data.edit); + this.messenger?.send("editFile", { fileEdit, }); break; case "saveFile": - this.saveFile(message.filepath); + this.saveFile(data.filepath); break; case "setFileOpen": - this.openFile(message.filepath); + this.openFile(data.filepath); // TODO: Close file break; case "openNotebook": case "connected": break; default: - throw Error("Unknown message type:" + message.messageType); + throw Error("Unknown message type:" + messageType); } } getWorkspaceDirectory() { @@ -209,17 +134,20 @@ class IdeProtocolClient { // Initiate Request closeNotebook(sessionId: string) { - this._panels.get(sessionId)?.dispose(); - this._panels.delete(sessionId); + this.panels.get(sessionId)?.dispose(); + this.panels.delete(sessionId); } async openNotebook() { console.log("OPENING NOTEBOOK"); - let resp = await this.sendAndReceive({}, "openNotebook"); - let sessionId = resp.sessionId; + if (this.messenger === null) { + console.log("MESSENGER IS NULL"); + } + const resp = await this.messenger?.sendAndReceive("openNotebook", {}); + const sessionId = resp.sessionId; console.log("SESSION ID", sessionId); - let column = getRightViewColumn(); + const column = getRightViewColumn(); const panel = vscode.window.createWebviewPanel( "continue.debugPanelView", "Continue", @@ -231,9 +159,9 @@ class IdeProtocolClient { ); // And set its HTML content - panel.webview.html = setupDebugPanel(panel, this._context, sessionId); + panel.webview.html = setupDebugPanel(panel, this.context, sessionId); - this._panels.set(sessionId, panel); + this.panels.set(sessionId, panel); } acceptRejectSuggestion(accept: boolean, key: SuggestionRanges) { diff --git a/extension/src/debugPanel.ts b/extension/src/debugPanel.ts index 4192595c..a295085f 100644 --- a/extension/src/debugPanel.ts +++ b/extension/src/debugPanel.ts @@ -16,7 +16,6 @@ import { import { sendTelemetryEvent, TelemetryEvent } from "./telemetry"; import { RangeInFile, SerializedDebugContext } from "./client"; import { addFileSystemToDebugContext } from "./util/util"; -const WebSocket = require("ws"); class StreamManager { private _fullText: string = ""; @@ -108,15 +107,15 @@ class WebsocketConnection { this._onOpen = onOpen; this._onClose = onClose; - this._ws.onmessage = (event) => { + this._ws.addEventListener("message", (event) => { this._onMessage(event.data); - }; - this._ws.onclose = () => { + }); + this._ws.addEventListener("close", () => { this._onClose(); - }; - this._ws.onopen = () => { + }); + this._ws.addEventListener("open", () => { this._onOpen(); - }; + }); } public send(message: string) { @@ -230,6 +229,19 @@ export function setupDebugPanel( apiUrl: getContinueServerUrl(), sessionId, }); + + // // Listen for changes to server URL in settings + // vscode.workspace.onDidChangeConfiguration((event) => { + // if (event.affectsConfiguration("continue.serverUrl")) { + // debugPanelWebview?.postMessage({ + // type: "onLoad", + // vscMachineId: vscode.env.machineId, + // apiUrl: getContinueServerUrl(), + // sessionId, + // }); + // } + // }); + break; } diff --git a/extension/src/extension.ts b/extension/src/extension.ts index e0b94278..88af0d19 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -4,7 +4,6 @@ import * as vscode from "vscode"; import { - setupExtensionEnvironment, isPythonEnvSetup, startContinuePythonServer, } from "./activation/environmentSetup"; @@ -26,11 +25,7 @@ export function activate(context: vscode.ExtensionContext) { cancellable: false, }, async () => { - if (isPythonEnvSetup()) { - await startContinuePythonServer(); - } else { - await setupExtensionEnvironment(); - } + await startContinuePythonServer(); dynamicImportAndActivate(context, true); } ); diff --git a/extension/src/test/runTest.ts b/extension/src/test/runTest.ts index 27b3ceb2..e810ed5b 100644 --- a/extension/src/test/runTest.ts +++ b/extension/src/test/runTest.ts @@ -1,23 +1,23 @@ -import * as path from 'path'; +import * as path from "path"; -import { runTests } from '@vscode/test-electron'; +import { runTests } from "@vscode/test-electron"; async function main() { - try { - // The folder containing the Extension Manifest package.json - // Passed to `--extensionDevelopmentPath` - const extensionDevelopmentPath = path.resolve(__dirname, '../../'); + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionDevelopmentPath = path.resolve(__dirname, "../../"); - // The path to test runner - // Passed to --extensionTestsPath - const extensionTestsPath = path.resolve(__dirname, './suite/index'); + // The path to test runner + // Passed to --extensionTestsPath + const extensionTestsPath = path.resolve(__dirname, "./suite/index"); - // Download VS Code, unzip it and run the integration test - await runTests({ extensionDevelopmentPath, extensionTestsPath }); - } catch (err) { - console.error('Failed to run tests'); - process.exit(1); - } + // Download VS Code, unzip it and run the integration test + await runTests({ extensionDevelopmentPath, extensionTestsPath }); + } catch (err) { + console.error("Failed to run tests"); + process.exit(1); + } } main(); diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts new file mode 100644 index 00000000..6f8bb29d --- /dev/null +++ b/extension/src/util/messenger.ts @@ -0,0 +1,108 @@ +console.log("Websocket import"); +const WebSocket = require("ws"); + +export abstract class Messenger { + abstract send(messageType: string, data: object): void; + + abstract onMessageType( + messageType: string, + callback: (data: object) => void + ): void; + + abstract onMessage(callback: (messageType: string, data: any) => void): void; + + abstract onOpen(callback: () => void): void; + + abstract onClose(callback: () => void): void; + + abstract sendAndReceive(messageType: string, data: any): Promise; +} + +export class WebsocketMessenger extends Messenger { + websocket: WebSocket; + private onMessageListeners: { + [messageType: string]: ((data: object) => void)[]; + } = {}; + private onOpenListeners: (() => void)[] = []; + private onCloseListeners: (() => void)[] = []; + private serverUrl: string; + + _newWebsocket(): WebSocket { + // // Dynamic import, because WebSocket is builtin with browser, but not with node. And can't use require in browser. + // if (typeof process === "object") { + // console.log("Using node"); + // // process is only available in Node + // var WebSocket = require("ws"); + // } + + const newWebsocket = new WebSocket(this.serverUrl); + for (const listener of this.onOpenListeners) { + this.onOpen(listener); + } + for (const listener of this.onCloseListeners) { + this.onClose(listener); + } + for (const messageType in this.onMessageListeners) { + for (const listener of this.onMessageListeners[messageType]) { + this.onMessageType(messageType, listener); + } + } + return newWebsocket; + } + + constructor(serverUrl: string) { + super(); + this.serverUrl = serverUrl; + this.websocket = this._newWebsocket(); + } + + send(messageType: string, data: object) { + const payload = JSON.stringify({ messageType, data }); + if (this.websocket.readyState === this.websocket.OPEN) { + this.websocket.send(payload); + } else { + if (this.websocket.readyState !== this.websocket.CONNECTING) { + this.websocket = this._newWebsocket(); + } + this.websocket.addEventListener("open", () => { + this.websocket.send(payload); + }); + } + } + + sendAndReceive(messageType: string, data: any): Promise { + return new Promise((resolve, reject) => { + const eventListener = (data: any) => { + // THIS ISN"T GETTING CALLED + resolve(data); + this.websocket.removeEventListener("message", eventListener); + }; + this.onMessageType(messageType, eventListener); + this.send(messageType, data); + }); + } + + onMessageType(messageType: string, callback: (data: any) => void): void { + this.websocket.addEventListener("message", (event: any) => { + const msg = JSON.parse(event.data); + if (msg.messageType === messageType) { + callback(msg.data); + } + }); + } + + onMessage(callback: (messageType: string, data: any) => void): void { + this.websocket.addEventListener("message", (event) => { + const msg = JSON.parse(event.data); + callback(msg.messageType, msg.data); + }); + } + + onOpen(callback: () => void): void { + this.websocket.addEventListener("open", callback); + } + + onClose(callback: () => void): void { + this.websocket.addEventListener("close", callback); + } +} -- cgit v1.2.3-70-g09d2 From 20a388db56d380d5520bf90ccb596fbbb2eae468 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Fri, 2 Jun 2023 17:54:17 -0400 Subject: openFile fix for codespaces --- extension/package-lock.json | 4 ++-- extension/package.json | 2 +- extension/src/util/vscode.ts | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'extension/src/util') diff --git a/extension/package-lock.json b/extension/package-lock.json index 647e3aa2..15ea6428 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.13", + "version": "0.0.14", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.13", + "version": "0.0.14", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index 1219ca8e..2d83a58c 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "Refine code 10x faster", - "version": "0.0.13", + "version": "0.0.14", "publisher": "Continue", "engines": { "vscode": "^1.74.0" diff --git a/extension/src/util/vscode.ts b/extension/src/util/vscode.ts index 4eab98a7..59e1ae8e 100644 --- a/extension/src/util/vscode.ts +++ b/extension/src/util/vscode.ts @@ -135,6 +135,9 @@ export function openEditorAndRevealRange( ): Promise { return new Promise((resolve, _) => { // Check if the editor is already open + if (editorFilename.startsWith("file://")) { + editorFilename = editorFilename.slice(7); + } vscode.workspace.openTextDocument(editorFilename).then((doc) => { vscode.window .showTextDocument( -- cgit v1.2.3-70-g09d2 From fcc3249323a30839d13c7fb256a8d967a8417bbf Mon Sep 17 00:00:00 2001 From: Nate Sesti <33237525+sestinj@users.noreply.github.com> Date: Fri, 2 Jun 2023 23:08:14 +0000 Subject: Fixed codespaces showTextDocument error --- extension/src/util/vscode.ts | 46 ++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 17 deletions(-) (limited to 'extension/src/util') diff --git a/extension/src/util/vscode.ts b/extension/src/util/vscode.ts index 59e1ae8e..a76b53c7 100644 --- a/extension/src/util/vscode.ts +++ b/extension/src/util/vscode.ts @@ -128,28 +128,40 @@ export async function readFileAtRange( }); } +let showTextDocumentInProcess = false; + export function openEditorAndRevealRange( editorFilename: string, range?: vscode.Range, viewColumn?: vscode.ViewColumn ): Promise { return new Promise((resolve, _) => { - // Check if the editor is already open - if (editorFilename.startsWith("file://")) { - editorFilename = editorFilename.slice(7); - } - vscode.workspace.openTextDocument(editorFilename).then((doc) => { - vscode.window - .showTextDocument( - doc, - getViewColumnOfFile(editorFilename) || viewColumn - ) - .then((editor) => { - if (range) { - editor.revealRange(range); - } - resolve(editor); - }); - }); + vscode.workspace.openTextDocument(editorFilename).then(async (doc) => { + try { + // An error is thrown mysteriously if you open two documents in parallel, hence this + while (showTextDocumentInProcess) { + await new Promise((resolve) => { + setInterval(() => { + resolve(null); + }, 200); + }) + } + showTextDocumentInProcess = true; + vscode.window + .showTextDocument( + doc, + getViewColumnOfFile(editorFilename) || viewColumn + ) + .then((editor) => { + if (range) { + editor.revealRange(range); + } + resolve(editor); + showTextDocumentInProcess = false; + }) + } catch (err) { + console.log(err); + } + }); }); } -- cgit v1.2.3-70-g09d2 From b740c5bf216b63dbb2ffa88f8385e9795aa57403 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Fri, 16 Jun 2023 14:34:30 -0700 Subject: Capturing command line prompt --- extension/package-lock.json | 4 +- extension/package.json | 2 +- extension/src/terminal/terminalEmulator.ts | 97 ++++++++++++++++++++++++++---- extension/src/util/lcs.ts | 30 +++++++++ 4 files changed, 117 insertions(+), 16 deletions(-) create mode 100644 extension/src/util/lcs.ts (limited to 'extension/src/util') diff --git a/extension/package-lock.json b/extension/package-lock.json index eac8edf2..86c816e0 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.45", + "version": "0.0.47", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.45", + "version": "0.0.47", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index 8f7da94d..3a3927f3 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "Refine code 10x faster", - "version": "0.0.45", + "version": "0.0.47", "publisher": "Continue", "engines": { "vscode": "^1.74.0" diff --git a/extension/src/terminal/terminalEmulator.ts b/extension/src/terminal/terminalEmulator.ts index ff2060f8..8e49737e 100644 --- a/extension/src/terminal/terminalEmulator.ts +++ b/extension/src/terminal/terminalEmulator.ts @@ -3,6 +3,7 @@ import * as vscode from "vscode"; import os = require("os"); import stripAnsi from "strip-ansi"; +import { longestCommonSubsequence } from "../util/lcs"; function loadNativeModule(id: string): T | null { try { @@ -71,13 +72,20 @@ export class CapturedTerminal { private dataEndsInPrompt(strippedData: string): boolean { const lines = strippedData.split("\n"); - const last_line = lines[lines.length - 1]; + const lastLine = lines[lines.length - 1]; + return ( - (lines.length > 0 && - (last_line.includes("bash-") || last_line.includes(") $ ")) && - last_line.includes("$")) || - (last_line.includes("]> ") && last_line.includes(") [")) || - (last_line.includes(" (") && last_line.includes(")>")) + lines.length > 0 && + (((lastLine.includes("bash-") || lastLine.includes(") $ ")) && + lastLine.includes("$")) || + (lastLine.includes("]> ") && lastLine.includes(") [")) || + (lastLine.includes(" (") && lastLine.includes(")>")) || + (typeof this.commandPromptString !== "undefined" && + (lastLine.includes(this.commandPromptString) || + this.commandPromptString.length - + longestCommonSubsequence(lastLine, this.commandPromptString) + .length < + 3))) ); } @@ -96,12 +104,6 @@ export class CapturedTerminal { } async runCommand(command: string): Promise { - if (!this.hasRunCommand) { - this.hasRunCommand = true; - // Let the first bash- prompt appear and let python env be opened - // await this.waitForCommandToFinish(); - } - if (this.commandQueue.length === 0) { return new Promise(async (resolve, reject) => { this.commandQueue.push([command, resolve]); @@ -109,8 +111,12 @@ export class CapturedTerminal { while (this.commandQueue.length > 0) { const [command, resolve] = this.commandQueue.shift()!; + // Refresh the command prompt string every time in case it changes + await this.refreshCommandPromptString(); + this.terminal.sendText(command); - resolve(await this.waitForCommandToFinish()); + const output = await this.waitForCommandToFinish(); + resolve(output); } }); } else { @@ -138,6 +144,24 @@ export class CapturedTerminal { } } + private runningClearToGetPrompt: boolean = false; + private seenClear: boolean = false; + private commandPromptString: string | undefined = undefined; + private resolveMeWhenCommandPromptStringFound: + | ((_: unknown) => void) + | undefined = undefined; + + private async refreshCommandPromptString(): Promise { + // Sends a message that will be received by the terminal to get the command prompt string, see the onData method below in constructor. + this.runningClearToGetPrompt = true; + this.terminal.sendText("echo"); + const promise = new Promise((resolve, reject) => { + this.resolveMeWhenCommandPromptStringFound = resolve; + }); + await promise; + return this.commandPromptString; + } + constructor( options: { name: string } & Partial, onCommandOutput?: (output: string) => void @@ -165,7 +189,54 @@ export class CapturedTerminal { this.writeEmitter = new vscode.EventEmitter(); this.ptyProcess.onData((data: any) => { + if (this.runningClearToGetPrompt) { + if ( + stripAnsi(data) + .split("\n") + .flatMap((line) => line.split("\r")) + .find((line) => line.trim() === "echo") !== undefined + ) { + this.seenClear = true; + return; + } else if (this.seenClear) { + const strippedLines = stripAnsi(data) + .split("\r") + .filter( + (line) => + line.trim().length > 0 && + line.trim() !== "%" && + line.trim() !== "⏎" + ); + const lastLine = strippedLines[strippedLines.length - 1] || ""; + const lines = lastLine + .split("\n") + .filter( + (line) => + line.trim().length > 0 && + line.trim() !== "%" && + line.trim() !== "⏎" + ); + const commandPromptString = (lines[lines.length - 1] || "").trim(); + if ( + commandPromptString.length > 0 && + !commandPromptString.includes("echo") + ) { + this.runningClearToGetPrompt = false; + this.seenClear = false; + this.commandPromptString = commandPromptString; + console.log( + "Found command prompt string: " + this.commandPromptString + ); + if (this.resolveMeWhenCommandPromptStringFound) { + this.resolveMeWhenCommandPromptStringFound(undefined); + } + } + return; + } + } + // Pass data through to terminal + data = data.replace("⏎", ""); this.writeEmitter.fire(data); this.splitByCommandsListener(data); diff --git a/extension/src/util/lcs.ts b/extension/src/util/lcs.ts new file mode 100644 index 00000000..17ea63f9 --- /dev/null +++ b/extension/src/util/lcs.ts @@ -0,0 +1,30 @@ +export function longestCommonSubsequence(a: string, b: string) { + const lengths: number[][] = []; + for (let i = 0; i <= a.length; i++) { + lengths[i] = []; + for (let j = 0; j <= b.length; j++) { + if (i === 0 || j === 0) { + lengths[i][j] = 0; + } else if (a[i - 1] === b[j - 1]) { + lengths[i][j] = lengths[i - 1][j - 1] + 1; + } else { + lengths[i][j] = Math.max(lengths[i - 1][j], lengths[i][j - 1]); + } + } + } + let result = ""; + let x = a.length; + let y = b.length; + while (x !== 0 && y !== 0) { + if (lengths[x][y] === lengths[x - 1][y]) { + x--; + } else if (lengths[x][y] === lengths[x][y - 1]) { + y--; + } else { + result = a[x - 1] + result; + x--; + y--; + } + } + return result; +} -- cgit v1.2.3-70-g09d2 From 287ec48055b86afad2fdd62fdd3f3987077f448b Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Wed, 21 Jun 2023 14:02:56 -0700 Subject: first slightly working version of suggestions --- continuedev/src/continuedev/server/ide.py | 6 +- continuedev/src/continuedev/server/ide_protocol.py | 2 +- continuedev/src/continuedev/steps/core/core.py | 448 ++++++++++----------- extension/src/continueIdeClient.ts | 25 +- extension/src/lang-server/codeLens.ts | 19 +- extension/src/suggestions.ts | 60 +-- extension/src/util/vscode.ts | 20 +- 7 files changed, 286 insertions(+), 294 deletions(-) (limited to 'extension/src/util') diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index c83fbc8a..c2ebaccf 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -146,8 +146,10 @@ class IdeProtocolServer(AbstractIdeProtocolServer): # ------------------------------- # # Request actions in IDE, doesn't matter which Session - def showSuggestion(): - pass + async def showSuggestion(self, file_edit: FileEdit): + await self._send_json("showSuggestion", { + "edit": file_edit.dict() + }) async def setFileOpen(self, filepath: str, open: bool = True): # Autopilot needs access to this. diff --git a/continuedev/src/continuedev/server/ide_protocol.py b/continuedev/src/continuedev/server/ide_protocol.py index 2dcedc30..79820c36 100644 --- a/continuedev/src/continuedev/server/ide_protocol.py +++ b/continuedev/src/continuedev/server/ide_protocol.py @@ -12,7 +12,7 @@ class AbstractIdeProtocolServer(ABC): """Handle a json message""" @abstractmethod - def showSuggestion(): + def showSuggestion(self, file_edit: FileEdit): """Show a suggestion to the user""" @abstractmethod diff --git a/continuedev/src/continuedev/steps/core/core.py b/continuedev/src/continuedev/steps/core/core.py index 71a5b5b2..eb6a00c6 100644 --- a/continuedev/src/continuedev/steps/core/core.py +++ b/continuedev/src/continuedev/steps/core/core.py @@ -159,253 +159,247 @@ class DefaultModelEditCodeStep(Step): self.name = await models.gpt35.complete(f"Write a very short title to describe this requested change: '{self.user_input}'. This is the title:") return f"`{self.user_input}`\n\n" + description - async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: - self.description = f"`{self.user_input}`" - await sdk.update_ui() - - rif_with_contents = [] - for range_in_file in map(lambda x: RangeInFile( - filepath=x.filepath, - # Only consider the range line-by-line. Maybe later don't if it's only a single line. - range=x.range.to_full_lines() - ), self.range_in_files): - file_contents = await sdk.ide.readRangeInFile(range_in_file) - rif_with_contents.append( - RangeInFileWithContents.from_range_in_file(range_in_file, file_contents)) - - rif_dict = {} - for rif in rif_with_contents: - rif_dict[rif.filepath] = rif.contents - - for rif in rif_with_contents: - await sdk.ide.setFileOpen(rif.filepath) - - model_to_use = sdk.models.default - - full_file_contents = await sdk.ide.readFile(rif.filepath) - - full_file_contents_lst = full_file_contents.split("\n") - - max_start_line = rif.range.start.line - min_end_line = rif.range.end.line - cur_start_line = 0 - cur_end_line = len(full_file_contents_lst) - 1 - - def cut_context(model_to_use, total_tokens, cur_start_line, cur_end_line): - - if total_tokens > MAX_TOKENS_FOR_MODEL[model_to_use.name]: - while cur_end_line > min_end_line: - total_tokens -= model_to_use.count_tokens( - full_file_contents_lst[cur_end_line]) - cur_end_line -= 1 - if total_tokens < MAX_TOKENS_FOR_MODEL[model_to_use.name]: - return cur_start_line, cur_end_line - - if total_tokens > MAX_TOKENS_FOR_MODEL[model_to_use.name]: - while cur_start_line < max_start_line: - cur_start_line += 1 - total_tokens -= model_to_use.count_tokens( - full_file_contents_lst[cur_end_line]) - if total_tokens < MAX_TOKENS_FOR_MODEL[model_to_use.name]: - return cur_start_line, cur_end_line - - return cur_start_line, cur_end_line - - if model_to_use.name == "gpt-4": - - total_tokens = model_to_use.count_tokens(full_file_contents + self._prompt) - cur_start_line, cur_end_line = cut_context( - model_to_use, total_tokens, cur_start_line, cur_end_line) - - elif model_to_use.name == "gpt-3.5-turbo" or model_to_use.name == "gpt-3.5-turbo-16k": - - if sdk.models.gpt35.count_tokens(full_file_contents) > MAX_TOKENS_FOR_MODEL["gpt-3.5-turbo"]: - - model_to_use = sdk.models.gpt3516k - total_tokens = model_to_use.count_tokens( - full_file_contents + self._prompt) - cur_start_line, cur_end_line = cut_context( - model_to_use, total_tokens, cur_start_line, cur_end_line) - - else: - - raise Exception("Unknown default model") + async def get_prompt_parts(self, rif: RangeInFileWithContents, sdk: ContinueSDK, full_file_contents: str): + # If using 3.5 and overflows, upgrade to 3.5.16k + model_to_use = sdk.models.default + if model_to_use.name == "gpt-3.5-turbo": + if sdk.models.gpt35.count_tokens(full_file_contents) > MAX_TOKENS_FOR_MODEL["gpt-3.5-turbo"]: + model_to_use = sdk.models.gpt3516k + + # Remove tokens from the end first, and then the start to clear space + # This part finds the start and end lines + full_file_contents_lst = full_file_contents.split("\n") + max_start_line = rif.range.start.line + min_end_line = rif.range.end.line + cur_start_line = 0 + cur_end_line = len(full_file_contents_lst) - 1 + + total_tokens = model_to_use.count_tokens( + full_file_contents + self._prompt) + + if total_tokens > MAX_TOKENS_FOR_MODEL[model_to_use.name]: + while cur_end_line > min_end_line: + total_tokens -= model_to_use.count_tokens( + full_file_contents_lst[cur_end_line]) + cur_end_line -= 1 + if total_tokens < MAX_TOKENS_FOR_MODEL[model_to_use.name]: + return cur_start_line, cur_end_line + + if total_tokens > MAX_TOKENS_FOR_MODEL[model_to_use.name]: + while cur_start_line < max_start_line: + cur_start_line += 1 + total_tokens -= model_to_use.count_tokens( + full_file_contents_lst[cur_end_line]) + if total_tokens < MAX_TOKENS_FOR_MODEL[model_to_use.name]: + return cur_start_line, cur_end_line + + # Now use the found start/end lines to get the prefix and suffix strings + file_prefix = "\n".join( + full_file_contents_lst[cur_start_line:max_start_line]) + file_suffix = "\n".join( + full_file_contents_lst[min_end_line:cur_end_line - 1]) + + # Move any surrounding blank line in rif.contents to the prefix/suffix + # TODO: Keep track of start line of the range, because it's needed below for offset stuff + rif_start_line = rif.range.start.line + if len(rif.contents) > 0: + first_line = rif.contents.splitlines(keepends=True)[0] + while first_line.strip() == "": + file_prefix += first_line + rif.contents = rif.contents[len(first_line):] + first_line = rif.contents.splitlines(keepends=True)[0] - code_before = "\n".join( - full_file_contents_lst[cur_start_line:max_start_line]) - code_after = "\n".join( - full_file_contents_lst[min_end_line:cur_end_line - 1]) + last_line = rif.contents.splitlines(keepends=True)[-1] + while last_line.strip() == "": + file_suffix = last_line + file_suffix + rif.contents = rif.contents[:len( + rif.contents) - len(last_line)] + last_line = rif.contents.splitlines(keepends=True)[-1] - segs = [code_before, code_after] - if segs[0].strip() == "": - segs[0] = segs[0].strip() - if segs[1].strip() == "": - segs[1] = segs[1].strip() + while rif.contents.startswith("\n"): + file_prefix += "\n" + rif.contents = rif.contents[1:] + while rif.contents.endswith("\n"): + file_suffix = "\n" + file_suffix + rif.contents = rif.contents[:-1] - # Move any surrounding blank line in rif.contents to the prefix/suffix - if len(rif.contents) > 0: - first_line = rif.contents.splitlines(keepends=True)[0] - while first_line.strip() == "": - segs[0] += first_line - rif.contents = rif.contents[len(first_line):] - first_line = rif.contents.splitlines(keepends=True)[0] + return file_prefix, rif.contents, file_suffix, model_to_use - last_line = rif.contents.splitlines(keepends=True)[-1] - while last_line.strip() == "": - segs[1] = last_line + segs[1] - rif.contents = rif.contents[:len( - rif.contents) - len(last_line)] - last_line = rif.contents.splitlines(keepends=True)[-1] - - while rif.contents.startswith("\n"): - segs[0] += "\n" - rif.contents = rif.contents[1:] - while rif.contents.endswith("\n"): - segs[1] = "\n" + segs[1] - rif.contents = rif.contents[:-1] - - # .format(code=rif.contents, user_request=self.user_input, file_prefix=segs[0], file_suffix=segs[1]) - prompt = self._prompt - if segs[0].strip() != "": - prompt += dedent(f""" + def compile_prompt(self, file_prefix: str, contents: str, file_suffix: str, sdk: ContinueSDK) -> str: + prompt = self._prompt + if file_prefix.strip() != "": + prompt += dedent(f""" -{segs[0]} +{file_prefix} """) - prompt += dedent(f""" + prompt += dedent(f""" -{rif.contents} +{contents} """) - if segs[1].strip() != "": - prompt += dedent(f""" + if file_suffix.strip() != "": + prompt += dedent(f""" -{segs[1]} +{file_suffix} """) - prompt += dedent(f""" + prompt += dedent(f""" {self.user_input} """) - lines = [] - unfinished_line = "" - i = 0 - original_lines = rif.contents.split("\n") - - async def add_line(i: int, line: str): - if i == 0: - # First line indentation, because the model will assume that it is replacing in this way - line = original_lines[0].replace( - original_lines[0].strip(), "") + line - - if i < len(original_lines): - # Replace original line - range = Range.from_shorthand( - rif.range.start.line + i, rif.range.start.character if i == 0 else 0, rif.range.start.line + i + 1, 0) - else: - # Insert a line - range = Range.from_shorthand( - rif.range.start.line + i, 0, rif.range.start.line + i, 0) - - await sdk.ide.applyFileSystemEdit(FileEdit( - filepath=rif.filepath, - range=range, - replacement=line + "\n" - )) - - lines_of_prefix_copied = 0 - line_below_highlighted_range = segs[1].lstrip().split("\n")[0] - should_stop = False - async for chunk in model_to_use.stream_chat(prompt, with_history=await sdk.get_chat_context(), temperature=0): - if should_stop: + return prompt + + def is_end_line(self, line: str) -> bool: + return "" in line + + def line_to_be_ignored(self, line: str) -> bool: + return "```" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line + + async def stream_rif(self, rif: RangeInFileWithContents, sdk: ContinueSDK): + full_file_contents = await sdk.ide.readFile(rif.filepath) + + file_prefix, contents, file_suffix, model_to_use = await self.get_prompt_parts( + rif, sdk, full_file_contents) + prompt = self.compile_prompt(file_prefix, contents, file_suffix, sdk) + + full_file_contents_lines = full_file_contents.split("\n") + original_lines = rif.contents.split("\n") + i = 0 + lines = [] + unfinished_line = "" + + current_block = [] + offset_from_blocks = 0 + + async def insert_line(line: str, line_no: int): + nonlocal current_block + # Insert line, highlight green, highlight corresponding line red + range = Range.from_shorthand( + line_no, 0, line_no, 0) + red_range = Range.from_shorthand( + line_no + len(current_block), 0, line_no + len(current_block), 0) + + await sdk.ide.applyFileSystemEdit(FileEdit( + filepath=rif.filepath, + range=range, + replacement=line + "\n" + )) + await sdk.ide.highlightCode(RangeInFile(filepath=rif.filepath, range=range), "#00FF0022") + await sdk.ide.highlightCode(RangeInFile(filepath=rif.filepath, range=red_range), "#FF000022") + + async def show_block_as_suggestion(): + nonlocal i, offset_from_blocks, current_block + await sdk.ide.showSuggestion(FileEdit( + filepath=rif.filepath, + range=Range.from_shorthand( + i + offset_from_blocks - len(current_block) + rif.range.start.line, 0, i + offset_from_blocks + rif.range.start.line, 0), + replacement="\n".join(current_block) + "\n" + )) + offset_from_blocks += len(current_block) + current_block.clear() + + async def add_to_block(line: str): + current_block.append(line) + # TODO: This start line might have changed above + # await insert_line(line, i + offset_from_blocks + + # rif.range.start.line) + + async def handle_generated_line(line: str): + nonlocal i, lines, current_block, offset_from_blocks, original_lines + # diff = list(difflib.ndiff(rif.contents.splitlines( + # keepends=True), completion.splitlines(keepends=True))) + if i < len(original_lines) and line == original_lines[i]: + # Line is the same as the original. Start a new block + await show_block_as_suggestion() + else: + # Add to the current block + await add_to_block(line) + + lines_of_prefix_copied = 0 + repeating_file_suffix = False + line_below_highlighted_range = file_suffix.lstrip().split("\n")[0] + async for chunk in model_to_use.stream_chat(prompt, with_history=await sdk.get_chat_context(), temperature=0): + # Stop early if it is repeating the file_suffix + if repeating_file_suffix: + break + + # Accumulate lines + chunk_lines = chunk.split("\n") + chunk_lines[0] = unfinished_line + chunk_lines[0] + if chunk.endswith("\n"): + unfinished_line = "" + chunk_lines.pop() # because this will be an empty string + else: + unfinished_line = chunk_lines.pop() + lines.extend(chunk_lines) + + # Deal with newly accumulated lines + for line in chunk_lines: + # Lines that should signify the end of generation + if self.is_end_line(line): + break + # Lines that should be ignored, like the <> tags + elif self.line_to_be_ignored(line): + continue + # Check if we are currently just copying the prefix + elif (lines_of_prefix_copied > 0 or i == 0) and lines_of_prefix_copied < len(file_prefix.splitlines()) and line == full_file_contents_lines[lines_of_prefix_copied]: + # This is a sketchy way of stopping it from repeating the file_prefix. Is a bug if output happens to have a matching line + lines_of_prefix_copied += 1 + continue + # Because really short lines might be expected to be repeated, this is only a !heuristic! + # Stop when it starts copying the file_suffix + elif line.strip() == line_below_highlighted_range.strip() and len(line.strip()) > 4: + repeating_file_suffix = True break - chunk_lines = chunk.split("\n") - chunk_lines[0] = unfinished_line + chunk_lines[0] - if chunk.endswith("\n"): - unfinished_line = "" - chunk_lines.pop() # because this will be an empty string - else: - unfinished_line = chunk_lines.pop() - lines.extend(chunk_lines) - - for line in chunk_lines: - if "" in line: - break - elif "```" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line or "" in line: - continue - elif (lines_of_prefix_copied > 0 or i == 0) and lines_of_prefix_copied < len(segs[0].splitlines()) and line == full_file_contents_lst[lines_of_prefix_copied]: - # This is a sketchy way of stopping it from repeating the file_prefix. Is a bug if output happens to have a matching line - lines_of_prefix_copied += 1 - continue - elif i < len(original_lines) and line == original_lines[i]: - i += 1 - continue - # Because really short lines might be expected to be repeated !heuristic! - elif line.strip() == line_below_highlighted_range.strip() and len(line.strip()) > 4: - should_stop = True - break - await add_line(i, line) - i += 1 - - # Add the unfinished line - if unfinished_line != "": - unfinished_line = unfinished_line.replace( - "", "").replace("", "").replace("```", "").replace("", "").replace("", "").replace("", "").replace("", "").replace(" 0: + await show_block_as_suggestion() + + # Record the completion + completion = "\n".join(lines) + self._prompt_and_completion += prompt + completion + + async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: + self.description = f"`{self.user_input}`" + await sdk.update_ui() + + rif_with_contents = [] + for range_in_file in map(lambda x: RangeInFile( + filepath=x.filepath, + # Only consider the range line-by-line. Maybe later don't if it's only a single line. + range=x.range.to_full_lines() + ), self.range_in_files): + file_contents = await sdk.ide.readRangeInFile(range_in_file) + rif_with_contents.append( + RangeInFileWithContents.from_range_in_file(range_in_file, file_contents)) + + rif_dict = {} + for rif in rif_with_contents: + rif_dict[rif.filepath] = rif.contents + + for rif in rif_with_contents: + await sdk.ide.setFileOpen(rif.filepath) + await self.stream_rif(rif, sdk) + # await sdk.ide.saveFile(rif.filepath) class EditFileStep(Step): diff --git a/extension/src/continueIdeClient.ts b/extension/src/continueIdeClient.ts index fbad5f5d..fbd00c6e 100644 --- a/extension/src/continueIdeClient.ts +++ b/extension/src/continueIdeClient.ts @@ -1,5 +1,8 @@ // import { ShowSuggestionRequest } from "../schema/ShowSuggestionRequest"; -import { showSuggestion, SuggestionRanges } from "./suggestions"; +import { + showSuggestion as showSuggestionInEditor, + SuggestionRanges, +} from "./suggestions"; import { openEditorAndRevealRange, getRightViewColumn } from "./util/vscode"; import { FileEdit } from "../schema/FileEdit"; import { RangeInFile } from "../schema/RangeInFile"; @@ -115,7 +118,10 @@ class IdeProtocolClient { break; case "setFileOpen": this.openFile(data.filepath); - // TODO: Close file + // TODO: Close file if False + break; + case "showSuggestion": + this.showSuggestion(data.edit); break; case "openGUI": case "connected": @@ -157,25 +163,12 @@ class IdeProtocolClient { isWholeLine: true, }); editor.setDecorations(decorationType, [range]); - - // Listen for changes to cursor position and then remove the decoration (but keep for at least 2 seconds) - const allowRemoveHighlight = () => { - const cursorDisposable = vscode.window.onDidChangeTextEditorSelection( - (event) => { - if (event.textEditor.document.uri.fsPath === rangeInFile.filepath) { - cursorDisposable.dispose(); - editor.setDecorations(decorationType, []); - } - } - ); - }; - setTimeout(allowRemoveHighlight, 2000); } } showSuggestion(edit: FileEdit) { // showSuggestion already exists - showSuggestion( + showSuggestionInEditor( edit.filepath, new vscode.Range( edit.range.start.line, diff --git a/extension/src/lang-server/codeLens.ts b/extension/src/lang-server/codeLens.ts index 2a362b62..e8766c3c 100644 --- a/extension/src/lang-server/codeLens.ts +++ b/extension/src/lang-server/codeLens.ts @@ -13,8 +13,8 @@ class SuggestionsCodeLensProvider implements vscode.CodeLensProvider { } let codeLenses: vscode.CodeLens[] = []; - for (let suggestion of suggestions) { - let range = new vscode.Range( + for (const suggestion of suggestions) { + const range = new vscode.Range( suggestion.oldRange.start, suggestion.newRange.end ); @@ -57,16 +57,16 @@ class PytestCodeLensProvider implements vscode.CodeLensProvider { document: vscode.TextDocument, token: vscode.CancellationToken ): vscode.CodeLens[] | Thenable { - let codeLenses: vscode.CodeLens[] = []; + const codeLenses: vscode.CodeLens[] = []; let lineno = 1; - let languageLibrary = getLanguageLibrary(document.fileName); + const languageLibrary = getLanguageLibrary(document.fileName); for (let line of document.getText().split("\n")) { if ( languageLibrary.lineIsFunctionDef(line) && languageLibrary.parseFunctionDefForName(line).startsWith("test_") ) { - let functionToTest = languageLibrary.parseFunctionDefForName(line); - let fileAndFunctionNameSpecifier = + const functionToTest = languageLibrary.parseFunctionDefForName(line); + const fileAndFunctionNameSpecifier = document.fileName + "::" + functionToTest; codeLenses.push( new vscode.CodeLens(new vscode.Range(lineno, 0, lineno, 1), { @@ -85,12 +85,13 @@ class PytestCodeLensProvider implements vscode.CodeLensProvider { const allCodeLensProviders: { [langauge: string]: vscode.CodeLensProvider[] } = { - python: [new SuggestionsCodeLensProvider(), new PytestCodeLensProvider()], + // python: [new SuggestionsCodeLensProvider(), new PytestCodeLensProvider()], + "*": [new SuggestionsCodeLensProvider()], }; export function registerAllCodeLensProviders(context: vscode.ExtensionContext) { - for (let language in allCodeLensProviders) { - for (let codeLensProvider of allCodeLensProviders[language]) { + for (const language in allCodeLensProviders) { + for (const codeLensProvider of allCodeLensProviders[language]) { context.subscriptions.push( vscode.languages.registerCodeLensProvider(language, codeLensProvider) ); diff --git a/extension/src/suggestions.ts b/extension/src/suggestions.ts index c66fad86..5ac6e095 100644 --- a/extension/src/suggestions.ts +++ b/extension/src/suggestions.ts @@ -14,7 +14,7 @@ export const editorToSuggestions: Map< string, // URI of file SuggestionRanges[] > = new Map(); -export let currentSuggestion: Map = new Map(); // Map from editor URI to index of current SuggestionRanges in editorToSuggestions +export const currentSuggestion: Map = new Map(); // Map from editor URI to index of current SuggestionRanges in editorToSuggestions // When tab is reopened, rerender the decorations: vscode.window.onDidChangeActiveTextEditor((editor) => { @@ -25,16 +25,16 @@ vscode.workspace.onDidOpenTextDocument((doc) => { rerenderDecorations(doc.uri.toString()); }); -let newDecorationType = vscode.window.createTextEditorDecorationType({ +const newDecorationType = vscode.window.createTextEditorDecorationType({ backgroundColor: "rgb(0, 255, 0, 0.1)", isWholeLine: true, }); -let oldDecorationType = vscode.window.createTextEditorDecorationType({ +const oldDecorationType = vscode.window.createTextEditorDecorationType({ backgroundColor: "rgb(255, 0, 0, 0.1)", isWholeLine: true, cursor: "pointer", }); -let newSelDecorationType = vscode.window.createTextEditorDecorationType({ +const newSelDecorationType = vscode.window.createTextEditorDecorationType({ backgroundColor: "rgb(0, 255, 0, 0.25)", isWholeLine: true, after: { @@ -42,7 +42,7 @@ let newSelDecorationType = vscode.window.createTextEditorDecorationType({ margin: "0 0 0 1em", }, }); -let oldSelDecorationType = vscode.window.createTextEditorDecorationType({ +const oldSelDecorationType = vscode.window.createTextEditorDecorationType({ backgroundColor: "rgb(255, 0, 0, 0.25)", isWholeLine: true, after: { @@ -52,19 +52,19 @@ let oldSelDecorationType = vscode.window.createTextEditorDecorationType({ }); export function rerenderDecorations(editorUri: string) { - let suggestions = editorToSuggestions.get(editorUri); - let idx = currentSuggestion.get(editorUri); - let editor = vscode.window.visibleTextEditors.find( + const suggestions = editorToSuggestions.get(editorUri); + const idx = currentSuggestion.get(editorUri); + const editor = vscode.window.visibleTextEditors.find( (editor) => editor.document.uri.toString() === editorUri ); if (!suggestions || !editor) return; - let olds: vscode.Range[] = [], - news: vscode.Range[] = [], - oldSels: vscode.Range[] = [], - newSels: vscode.Range[] = []; + const olds: vscode.Range[] = []; + const news: vscode.Range[] = []; + const oldSels: vscode.Range[] = []; + const newSels: vscode.Range[] = []; for (let i = 0; i < suggestions.length; i++) { - let suggestion = suggestions[i]; + const suggestion = suggestions[i]; if (typeof idx != "undefined" && idx === i) { if (suggestion.newSelected) { olds.push(suggestion.oldRange); @@ -92,14 +92,14 @@ export function rerenderDecorations(editorUri: string) { } export function suggestionDownCommand() { - let editor = vscode.window.activeTextEditor; + const editor = vscode.window.activeTextEditor; if (!editor) return; - let editorUri = editor.document.uri.toString(); - let suggestions = editorToSuggestions.get(editorUri); - let idx = currentSuggestion.get(editorUri); + const editorUri = editor.document.uri.toString(); + const suggestions = editorToSuggestions.get(editorUri); + const idx = currentSuggestion.get(editorUri); if (!suggestions || idx === undefined) return; - let suggestion = suggestions[idx]; + const suggestion = suggestions[idx]; if (!suggestion.newSelected) { suggestion.newSelected = true; } else if (idx + 1 < suggestions.length) { @@ -109,14 +109,14 @@ export function suggestionDownCommand() { } export function suggestionUpCommand() { - let editor = vscode.window.activeTextEditor; + const editor = vscode.window.activeTextEditor; if (!editor) return; - let editorUri = editor.document.uri.toString(); - let suggestions = editorToSuggestions.get(editorUri); - let idx = currentSuggestion.get(editorUri); + const editorUri = editor.document.uri.toString(); + const suggestions = editorToSuggestions.get(editorUri); + const idx = currentSuggestion.get(editorUri); if (!suggestions || idx === undefined) return; - let suggestion = suggestions[idx]; + const suggestion = suggestions[idx]; if (suggestion.newSelected) { suggestion.newSelected = false; } else if (idx > 0) { @@ -130,10 +130,10 @@ function selectSuggestion( accept: SuggestionSelectionOption, key: SuggestionRanges | null = null ) { - let editor = vscode.window.activeTextEditor; + const editor = vscode.window.activeTextEditor; if (!editor) return; - let editorUri = editor.document.uri.toString(); - let suggestions = editorToSuggestions.get(editorUri); + const editorUri = editor.document.uri.toString(); + const suggestions = editorToSuggestions.get(editorUri); if (!suggestions) return; @@ -218,14 +218,14 @@ export async function showSuggestion( range: vscode.Range, suggestion: string ): Promise { - let existingCode = await readFileAtRange( + const existingCode = await readFileAtRange( new vscode.Range(range.start, range.end), editorFilename ); // If any of the outside lines are the same, don't repeat them in the suggestion - let slines = suggestion.split("\n"); - let elines = existingCode.split("\n"); + const slines = suggestion.split("\n"); + const elines = existingCode.split("\n"); let linesRemovedBefore = 0; let linesRemovedAfter = 0; while (slines.length > 0 && elines.length > 0 && slines[0] === elines[0]) { @@ -255,7 +255,7 @@ export async function showSuggestion( ) ); - let editor = await openEditorAndRevealRange(editorFilename, range); + const editor = await openEditorAndRevealRange(editorFilename, range); if (!editor) return Promise.resolve(false); return new Promise((resolve, reject) => { diff --git a/extension/src/util/vscode.ts b/extension/src/util/vscode.ts index a76b53c7..3110d589 100644 --- a/extension/src/util/vscode.ts +++ b/extension/src/util/vscode.ts @@ -118,9 +118,11 @@ export async function readFileAtRange( ) ); } else { - let firstLine = lines[range.start.line].slice(range.start.character); - let lastLine = lines[range.end.line].slice(0, range.end.character); - let middleLines = lines.slice(range.start.line + 1, range.end.line); + const firstLine = lines[range.start.line].slice( + range.start.character + ); + const lastLine = lines[range.end.line].slice(0, range.end.character); + const middleLines = lines.slice(range.start.line + 1, range.end.line); resolve([firstLine, ...middleLines, lastLine].join("\n")); } } @@ -144,7 +146,7 @@ export function openEditorAndRevealRange( setInterval(() => { resolve(null); }, 200); - }) + }); } showTextDocumentInProcess = true; vscode.window @@ -158,10 +160,10 @@ export function openEditorAndRevealRange( } resolve(editor); showTextDocumentInProcess = false; - }) - } catch (err) { - console.log(err); - } - }); + }); + } catch (err) { + console.log(err); + } + }); }); } -- cgit v1.2.3-70-g09d2 From 43dc3459997d79d112a2776a8e95546580112460 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Thu, 29 Jun 2023 09:51:24 -0700 Subject: messaging of welcome step --- extension/media/walkthrough.md | 1 + extension/package-lock.json | 4 ++-- extension/package.json | 2 +- extension/react-app/src/tabs/gui.tsx | 28 ++++++++++++++++++++++------ extension/src/activation/environmentSetup.ts | 9 ++++----- extension/src/continueIdeClient.ts | 15 ++++++++++++--- extension/src/util/messenger.ts | 8 ++++++++ 7 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 extension/media/walkthrough.md (limited to 'extension/src/util') diff --git a/extension/media/walkthrough.md b/extension/media/walkthrough.md new file mode 100644 index 00000000..fe77a69a --- /dev/null +++ b/extension/media/walkthrough.md @@ -0,0 +1 @@ +# Welcome to Continue diff --git a/extension/package-lock.json b/extension/package-lock.json index 241ea5ca..c33567c4 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.86", + "version": "0.0.88", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.86", + "version": "0.0.88", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index 7eca6042..e34af438 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "Accelerating software development with language models", - "version": "0.0.86", + "version": "0.0.88", "publisher": "Continue", "engines": { "vscode": "^1.74.0" diff --git a/extension/react-app/src/tabs/gui.tsx b/extension/react-app/src/tabs/gui.tsx index 249d9785..8d21a457 100644 --- a/extension/react-app/src/tabs/gui.tsx +++ b/extension/react-app/src/tabs/gui.tsx @@ -63,7 +63,12 @@ function GUI(props: GUIProps) { >([]); const [dataSwitchChecked, setDataSwitchChecked] = useState(false); const [showDataSharingInfo, setShowDataSharingInfo] = useState(false); - const [stepsOpen, setStepsOpen] = useState([]); + const [stepsOpen, setStepsOpen] = useState([ + true, + true, + true, + true, + ]); const [history, setHistory] = useState({ timeline: [ { @@ -78,13 +83,24 @@ function GUI(props: GUIProps) { { name: "Welcome to Continue", hide: false, - description: - "Type '/' to see the list of available slash commands. If you highlight code, edits and explanations will be localized to the highlighted range. Otherwise, the currently open file is used. In both cases, the code is combined with the previous steps to construct the context.", + description: `Welcome to Continue`, system_message: null, chat_context: [], manage_own_chat_context: false, - message: - "Type '/' to see the list of available slash commands. If you highlight code, edits and explanations will be localized to the highlighted range. Otherwise, the currently open file is used. In both cases, the code is combined with the previous steps to construct the context.", + message: `# Welcome to Continue + + _If it's your first time using Continue, it can take up to a minute for the server to install._ + + Continue is not perfect, but a great tool to add to your toolbox. These are the tasks that Continue is currently best at: + + - Highlight a section of code and instruct Continue to refactor it, e.g. \`"/edit make this use more descriptive variable names"\` + - Ask questions of the open file, e.g. \`"/explain what is the purpose of each of these if statements?"\` + - Ask Continue to build the scaffolding of a new file from scratch, e.g. \`"add a React component for syntax highlighted code"\` + + You can use "slash commands" to directly instruct Continue what to do, or just enter a request and it will automatically decide next steps. To see the list of available slash commands, type '/'. + + If you highlight code, edits and explanations will be localized to the highlighted range. Otherwise, the currently open file is used. In both cases, the code is combined with the previous steps to construct the context. + `, }, { name: "Welcome to Continue!", @@ -585,7 +601,7 @@ function GUI(props: GUIProps) { onClick={() => { client?.sendClear(); }} - text="Clear All" + text="Clear" > diff --git a/extension/src/activation/environmentSetup.ts b/extension/src/activation/environmentSetup.ts index 823670fd..b8c23733 100644 --- a/extension/src/activation/environmentSetup.ts +++ b/extension/src/activation/environmentSetup.ts @@ -150,11 +150,10 @@ async function setupPythonEnv() { // Repeat until it is successfully created (sometimes it fails to generate the bin, need to try again) while (true) { const [, stderr] = await runCommand(createEnvCommand); - if (stderr) { - throw new Error(stderr); - } if (checkEnvExists()) { break; + } else if (stderr) { + throw new Error(stderr); } else { // Remove the env and try again const removeCommand = `rm -rf "${path.join( @@ -180,7 +179,6 @@ async function setupPythonEnv() { activateCmd, pipUpgradeCmd, `${pipCmd} install -r requirements.txt`, - touchCmd, ].join(" ; "); const [, stderr] = await runCommand(installRequirementsCommand); if (stderr) { @@ -273,13 +271,14 @@ export async function startContinuePythonServer() { console.log(`stdout: ${data}`); }); child.stderr.on("data", (data: any) => { - console.log(`stderr: ${data}`); if ( data.includes("Uvicorn running on") || // Successfully started the server data.includes("address already in use") // The server is already running (probably a simultaneously opened VS Code window) ) { console.log("Successfully started Continue python server"); resolve(null); + } else { + console.log(`stderr: ${data}`); } }); child.on("error", (error: any) => { diff --git a/extension/src/continueIdeClient.ts b/extension/src/continueIdeClient.ts index 8ab3e075..d983d93a 100644 --- a/extension/src/continueIdeClient.ts +++ b/extension/src/continueIdeClient.ts @@ -226,9 +226,18 @@ class IdeProtocolClient { } async getSessionId(): Promise { - if (this.messenger === null) { - console.log("MESSENGER IS NULL"); - } + await new Promise((resolve, reject) => { + // Repeatedly try to connect to the server + const interval = setInterval(() => { + if ( + this.messenger && + this.messenger.websocket.readyState === 1 // 1 => OPEN + ) { + clearInterval(interval); + resolve(null); + } + }, 1000); + }); const resp = await this.messenger?.sendAndReceive("openGUI", {}); const sessionId = resp.sessionId; console.log("New Continue session with ID: ", sessionId); diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts index 6f8bb29d..e4133230 100644 --- a/extension/src/util/messenger.ts +++ b/extension/src/util/messenger.ts @@ -54,6 +54,14 @@ export class WebsocketMessenger extends Messenger { super(); this.serverUrl = serverUrl; this.websocket = this._newWebsocket(); + + const interval = setInterval(() => { + if (this.websocket.readyState === this.websocket.OPEN) { + clearInterval(interval); + } else if (this.websocket.readyState !== this.websocket.CONNECTING) { + this.websocket = this._newWebsocket(); + } + }, 1000); } send(messageType: string, data: object) { -- cgit v1.2.3-70-g09d2 From 81a5ed86119b81a07daf31144e8f09fe6b66246d Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Fri, 30 Jun 2023 17:25:35 -0700 Subject: the fix. and others. --- continuedev/pyproject.toml | 2 +- continuedev/src/continuedev/server/ide.py | 6 +-- continuedev/src/continuedev/steps/chat.py | 5 ++- continuedev/src/continuedev/steps/core/core.py | 41 ++++++++++++++++--- extension/.gitignore | 3 +- extension/package-lock.json | 4 +- extension/package.json | 4 +- extension/scripts/requirements.txt | 2 +- extension/src/activation/activate.ts | 32 +++++++-------- extension/src/continueIdeClient.ts | 28 ++++++++----- extension/src/debugPanel.ts | 13 ++++-- extension/src/lang-server/codeLens.ts | 19 ++++----- extension/src/suggestions.ts | 50 +++++++---------------- extension/src/util/messenger.ts | 56 +++++++++++++++++++++----- 14 files changed, 161 insertions(+), 104 deletions(-) (limited to 'extension/src/util') diff --git a/continuedev/pyproject.toml b/continuedev/pyproject.toml index 64d88b8c..e33627e7 100644 --- a/continuedev/pyproject.toml +++ b/continuedev/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "continuedev" -version = "0.1.1" +version = "0.1.2" description = "" authors = ["Nate Sesti "] readme = "README.md" diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index 3c3555f1..e1f19447 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -110,7 +110,8 @@ class IdeProtocolServer(AbstractIdeProtocolServer): session_manager: SessionManager sub_queue: AsyncSubscriptionQueue = AsyncSubscriptionQueue() - def __init__(self, session_manager: SessionManager): + def __init__(self, session_manager: SessionManager, websocket: WebSocket): + self.websocket = websocket self.session_manager = session_manager async def _send_json(self, message_type: str, data: Any): @@ -354,8 +355,7 @@ async def websocket_endpoint(websocket: WebSocket): print("Accepted websocket connection from, ", websocket.client) await websocket.send_json({"messageType": "connected", "data": {}}) - ideProtocolServer = IdeProtocolServer(session_manager) - ideProtocolServer.websocket = websocket + ideProtocolServer = IdeProtocolServer(session_manager, websocket) while AppStatus.should_exit is False: message = await websocket.receive_text() diff --git a/continuedev/src/continuedev/steps/chat.py b/continuedev/src/continuedev/steps/chat.py index 6a2c136e..8494563b 100644 --- a/continuedev/src/continuedev/steps/chat.py +++ b/continuedev/src/continuedev/steps/chat.py @@ -23,7 +23,10 @@ class SimpleChatStep(Step): name: str = "Chat" async def run(self, sdk: ContinueSDK): - self.description = f"```{self.user_input}```\n\n" + self.description = f"`{self.user_input}`\n\n" + if self.user_input.strip() == "": + self.user_input = "Explain this code's function is a concise list of markdown bullets." + self.description = "" await sdk.update_ui() async for chunk in sdk.models.default.stream_complete(self.user_input, with_history=await sdk.get_chat_context()): diff --git a/continuedev/src/continuedev/steps/core/core.py b/continuedev/src/continuedev/steps/core/core.py index dfc7d309..f81b3f6d 100644 --- a/continuedev/src/continuedev/steps/core/core.py +++ b/continuedev/src/continuedev/steps/core/core.py @@ -158,6 +158,14 @@ class DefaultModelEditCodeStep(Step): description = await models.gpt3516k.complete( f"{self._prompt_and_completion}\n\nPlease give brief a description of the changes made above using markdown bullet points. Be concise and only mention changes made to the commit before, not prefix or suffix:") self.name = await models.gpt3516k.complete(f"Write a very short title to describe this requested change: '{self.user_input}'. This is the title:") + + # Remove quotes from title and description if they are wrapped + if description.startswith('"') and description.endswith('"'): + description = description[1:-1] + + if self.name.startswith('"') and self.name.endswith('"'): + self.name = self.name[1:-1] + return f"`{self.user_input}`\n\n" + description async def get_prompt_parts(self, rif: RangeInFileWithContents, sdk: ContinueSDK, full_file_contents: str): @@ -170,6 +178,10 @@ class DefaultModelEditCodeStep(Step): total_tokens = model_to_use.count_tokens( full_file_contents + self._prompt + self.user_input) + BUFFER_FOR_FUNCTIONS + DEFAULT_MAX_TOKENS + TOKENS_TO_BE_CONSIDERED_LARGE_RANGE = 1000 + if model_to_use.count_tokens(rif.contents) > TOKENS_TO_BE_CONSIDERED_LARGE_RANGE: + self.description += "\n\n**It looks like you've selected a large range to edit, which may take a while to complete. If you'd like to cancel, click the 'X' button above. If you highlight a more specific range, Continue will only edit within it.**" + # If using 3.5 and overflows, upgrade to 3.5.16k if model_to_use.name == "gpt-3.5-turbo": if total_tokens > MAX_TOKENS_FOR_MODEL["gpt-3.5-turbo"]: @@ -267,8 +279,8 @@ class DefaultModelEditCodeStep(Step): file_prefix, contents, file_suffix, model_to_use = await self.get_prompt_parts( rif, sdk, full_file_contents) - # contents, common_whitespace = dedent_and_get_common_whitespace( - # contents) + contents, common_whitespace = dedent_and_get_common_whitespace( + contents) prompt = self.compile_prompt(file_prefix, contents, file_suffix, sdk) full_file_contents_lines = full_file_contents.split("\n") @@ -304,7 +316,7 @@ class DefaultModelEditCodeStep(Step): if len(current_block_lines) == 0: # Set this as the start of the next block current_block_start = rif.range.start.line + len(original_lines) - len( - original_lines_below_previous_blocks) + offset_from_blocks # current_line_in_file + original_lines_below_previous_blocks) + offset_from_blocks if len(original_lines_below_previous_blocks) > 0 and line == original_lines_below_previous_blocks[0]: # Line is equal to the next line in file, move past this line original_lines_below_previous_blocks = original_lines_below_previous_blocks[ @@ -335,12 +347,23 @@ class DefaultModelEditCodeStep(Step): lines_stripped.append(current_block_lines.pop()) index_of_last_line_in_block -= 1 + # It's also possible that some lines match at the beginning of the block + # lines_stripped_at_beginning = [] + # j = 0 + # while len(current_block_lines) > 0 and current_block_lines[0] == original_lines_below_previous_blocks[first_valid_match[0] - first_valid_match[1] + j]: + # lines_stripped_at_beginning.append( + # current_block_lines.pop(0)) + # j += 1 + # # current_block_start += 1 + # Insert the suggestion replacement = "\n".join(current_block_lines) + start_line = current_block_start + 1 + end_line = current_block_start + index_of_last_line_in_block await sdk.ide.showSuggestion(FileEdit( filepath=rif.filepath, range=Range.from_shorthand( - current_block_start + 1, 0, current_block_start + index_of_last_line_in_block, 0), + start_line, 0, end_line, 0), replacement=replacement )) if replacement == "": @@ -411,7 +434,7 @@ class DefaultModelEditCodeStep(Step): line = line.rstrip() # Add the common whitespace that was removed before prompting - # line = common_whitespace + line + line = common_whitespace + line # Lines that should signify the end of generation if self.is_end_line(line): @@ -437,7 +460,7 @@ class DefaultModelEditCodeStep(Step): # Add the unfinished line if unfinished_line != "" and not self.line_to_be_ignored(unfinished_line, completion_lines_covered == 0) and not self.is_end_line(unfinished_line): - # unfinished_line = common_whitespace + unfinished_line + unfinished_line = common_whitespace + unfinished_line lines.append(unfinished_line) await handle_generated_line(unfinished_line) completion_lines_covered += 1 @@ -459,6 +482,12 @@ class DefaultModelEditCodeStep(Step): current_block_lines = current_block_lines[:- num_to_remove] if num_to_remove > 0 else current_block_lines + # It's also possible that some lines match at the beginning of the block + # while len(current_block_lines) > 0 and len(original_lines_below_previous_blocks) > 0 and current_block_lines[0] == original_lines_below_previous_blocks[0]: + # current_block_lines.pop(0) + # original_lines_below_previous_blocks.pop(0) + # current_block_start += 1 + await sdk.ide.showSuggestion(FileEdit( filepath=rif.filepath, range=Range.from_shorthand( diff --git a/extension/.gitignore b/extension/.gitignore index 6d1b35bf..43b10889 100644 --- a/extension/.gitignore +++ b/extension/.gitignore @@ -3,4 +3,5 @@ node_modules/ out/ .vscode-test/ data/ -src/client \ No newline at end of file +src/client +.continue diff --git a/extension/package-lock.json b/extension/package-lock.json index 26e1a631..7565f480 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.91", + "version": "0.0.97", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.91", + "version": "0.0.97", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index 314e540f..08f5f081 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "The open-source coding autopilot", - "version": "0.0.91", + "version": "0.0.97", "publisher": "Continue", "engines": { "vscode": "^1.74.0" @@ -210,7 +210,7 @@ "lint": "eslint src --ext ts", "test": "node ./out/test/runTest.js", "package": "cp ./config/prod_config.json ./config/config.json && mkdir -p ./build && vsce package --out ./build && cp ./config/dev_config.json ./config/config.json", - "full-package": "cd ../continuedev && poetry build && cp ./dist/continuedev-0.1.1-py3-none-any.whl ../extension/scripts/continuedev-0.1.1-py3-none-any.whl && cd ../extension && npm install && npm run typegen && npm run clientgen && cd react-app && npm install && npm run build && cd .. && npm run package", + "full-package": "cd ../continuedev && poetry build && cp ./dist/continuedev-0.1.2-py3-none-any.whl ../extension/scripts/continuedev-0.1.2-py3-none-any.whl && cd ../extension && npm install && npm run typegen && npm run clientgen && cd react-app && npm install && npm run build && cd .. && npm run package", "install-extension": "code --install-extension ./build/continue-0.0.8.vsix", "uninstall": "code --uninstall-extension .continue", "reinstall": "rm -rf ./build && npm run package && npm run uninstall && npm run install-extension" diff --git a/extension/scripts/requirements.txt b/extension/scripts/requirements.txt index e6ba1e0a..c51c9d73 100644 --- a/extension/scripts/requirements.txt +++ b/extension/scripts/requirements.txt @@ -3,4 +3,4 @@ # typer==0.7.0 # pydantic # pytest -./continuedev-0.1.1-py3-none-any.whl \ No newline at end of file +./continuedev-0.1.2-py3-none-any.whl \ No newline at end of file diff --git a/extension/src/activation/activate.ts b/extension/src/activation/activate.ts index cd8f0cf3..18650561 100644 --- a/extension/src/activation/activate.ts +++ b/extension/src/activation/activate.ts @@ -20,6 +20,21 @@ export async function activateExtension( ) { extensionContext = context; + await new Promise((resolve, reject) => { + vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: + "Starting Continue Server... (it may take a minute to download Python packages)", + cancellable: false, + }, + async (progress, token) => { + await startContinuePythonServer(); + resolve(null); + } + ); + }); + sendTelemetryEvent(TelemetryEvent.ExtensionActivated); registerAllCodeLensProviders(context); registerAllCommands(context); @@ -33,7 +48,7 @@ export async function activateExtension( // Setup the left panel (async () => { - const sessionIdPromise = ideProtocolClient.getSessionId(); + const sessionIdPromise = await ideProtocolClient.getSessionId(); const provider = new ContinueGUIWebviewViewProvider(sessionIdPromise); context.subscriptions.push( @@ -46,21 +61,6 @@ export async function activateExtension( ) ); })(); - - await new Promise((resolve, reject) => { - vscode.window.withProgress( - { - location: vscode.ProgressLocation.Notification, - title: - "Starting Continue Server... (it may take a minute to download Python packages)", - cancellable: false, - }, - async (progress, token) => { - await startContinuePythonServer(); - resolve(null); - } - ); - }); // All opened terminals should be replaced by our own terminal // vscode.window.onDidOpenTerminal((terminal) => {}); diff --git a/extension/src/continueIdeClient.ts b/extension/src/continueIdeClient.ts index b179cbf3..21104abe 100644 --- a/extension/src/continueIdeClient.ts +++ b/extension/src/continueIdeClient.ts @@ -32,8 +32,8 @@ class IdeProtocolClient { messenger.onClose(() => { this.messenger = null; }); - messenger.onMessage((messageType, data) => { - this.handleMessage(messageType, data); + messenger.onMessage((messageType, data, messenger) => { + this.handleMessage(messageType, data, messenger); }); // Setup listeners for any file changes in open editors @@ -67,41 +67,45 @@ class IdeProtocolClient { // }); } - async handleMessage(messageType: string, data: any) { + async handleMessage( + messageType: string, + data: any, + messenger: WebsocketMessenger + ) { switch (messageType) { case "highlightedCode": - this.messenger?.send("highlightedCode", { + messenger.send("highlightedCode", { highlightedCode: this.getHighlightedCode(), }); break; case "workspaceDirectory": - this.messenger?.send("workspaceDirectory", { + messenger.send("workspaceDirectory", { workspaceDirectory: this.getWorkspaceDirectory(), }); break; case "uniqueId": - this.messenger?.send("uniqueId", { + messenger.send("uniqueId", { uniqueId: this.getUniqueId(), }); break; case "getUserSecret": - this.messenger?.send("getUserSecret", { + messenger.send("getUserSecret", { value: await this.getUserSecret(data.key), }); break; case "openFiles": - this.messenger?.send("openFiles", { + messenger.send("openFiles", { openFiles: this.getOpenFiles(), }); break; case "readFile": - this.messenger?.send("readFile", { + messenger.send("readFile", { contents: this.readFile(data.filepath), }); break; case "editFile": const fileEdit = await this.editFile(data.edit); - this.messenger?.send("editFile", { + messenger.send("editFile", { fileEdit, }); break; @@ -109,7 +113,7 @@ class IdeProtocolClient { this.highlightCode(data.rangeInFile, data.color); break; case "runCommand": - this.messenger?.send("runCommand", { + messenger.send("runCommand", { output: await this.runCommand(data.command), }); break; @@ -249,6 +253,8 @@ class IdeProtocolClient { ) { clearInterval(interval); resolve(null); + } else { + console.log("Websocket not yet open, trying again..."); } }, 1000); }); diff --git a/extension/src/debugPanel.ts b/extension/src/debugPanel.ts index b176eee7..b88c86f3 100644 --- a/extension/src/debugPanel.ts +++ b/extension/src/debugPanel.ts @@ -132,7 +132,7 @@ let streamManager = new StreamManager(); export let debugPanelWebview: vscode.Webview | undefined; export function setupDebugPanel( panel: vscode.WebviewPanel | vscode.WebviewView, - sessionIdPromise: Promise + sessionIdPromise: Promise | string ): string { debugPanelWebview = panel.webview; panel.onDidDispose(() => { @@ -230,7 +230,12 @@ export function setupDebugPanel( panel.webview.onDidReceiveMessage(async (data) => { switch (data.type) { case "onLoad": { - const sessionId = await sessionIdPromise; + let sessionId: string; + if (typeof sessionIdPromise === "string") { + sessionId = sessionIdPromise; + } else { + sessionId = await sessionIdPromise; + } panel.webview.postMessage({ type: "onLoad", vscMachineId: vscode.env.machineId, @@ -350,9 +355,9 @@ export class ContinueGUIWebviewViewProvider implements vscode.WebviewViewProvider { public static readonly viewType = "continue.continueGUIView"; - private readonly sessionIdPromise: Promise; + private readonly sessionIdPromise: Promise | string; - constructor(sessionIdPromise: Promise) { + constructor(sessionIdPromise: Promise | string) { this.sessionIdPromise = sessionIdPromise; } diff --git a/extension/src/lang-server/codeLens.ts b/extension/src/lang-server/codeLens.ts index 21448e31..3bd4f153 100644 --- a/extension/src/lang-server/codeLens.ts +++ b/extension/src/lang-server/codeLens.ts @@ -60,18 +60,15 @@ class SuggestionsCodeLensProvider implements vscode.CodeLensProvider { } } -const allCodeLensProviders: { [langauge: string]: vscode.CodeLensProvider[] } = - { - // python: [new SuggestionsCodeLensProvider(), new PytestCodeLensProvider()], - "*": [new SuggestionsCodeLensProvider()], - }; +let suggestionsCodeLensDisposable: vscode.Disposable | undefined = undefined; export function registerAllCodeLensProviders(context: vscode.ExtensionContext) { - for (const language in allCodeLensProviders) { - for (const codeLensProvider of allCodeLensProviders[language]) { - context.subscriptions.push( - vscode.languages.registerCodeLensProvider(language, codeLensProvider) - ); - } + if (suggestionsCodeLensDisposable) { + suggestionsCodeLensDisposable.dispose(); } + suggestionsCodeLensDisposable = vscode.languages.registerCodeLensProvider( + "*", + new SuggestionsCodeLensProvider() + ); + context.subscriptions.push(suggestionsCodeLensDisposable); } diff --git a/extension/src/suggestions.ts b/extension/src/suggestions.ts index e269f38a..baa49711 100644 --- a/extension/src/suggestions.ts +++ b/extension/src/suggestions.ts @@ -4,6 +4,8 @@ import { openEditorAndRevealRange } from "./util/vscode"; import { translate, readFileAtRange } from "./util/vscode"; import * as fs from "fs"; import * as path from "path"; +import { registerAllCodeLensProviders } from "./lang-server/codeLens"; +import { extensionContext } from "./activation/activate"; export interface SuggestionRanges { oldRange: vscode.Range; @@ -125,6 +127,10 @@ export function rerenderDecorations(editorUri: string) { suggestions[idx].newRange, vscode.TextEditorRevealType.Default ); + + if (extensionContext) { + registerAllCodeLensProviders(extensionContext); + } } export function suggestionDownCommand() { @@ -337,42 +343,14 @@ export async function showSuggestion( range: vscode.Range, suggestion: string ): Promise { - // const existingCode = await readFileAtRange( - // new vscode.Range(range.start, range.end), - // editorFilename - // ); - - // If any of the outside lines are the same, don't repeat them in the suggestion - // const slines = suggestion.split("\n"); - // const elines = existingCode.split("\n"); - // let linesRemovedBefore = 0; - // let linesRemovedAfter = 0; - // while (slines.length > 0 && elines.length > 0 && slines[0] === elines[0]) { - // slines.shift(); - // elines.shift(); - // linesRemovedBefore++; - // } - - // while ( - // slines.length > 0 && - // elines.length > 0 && - // slines[slines.length - 1] === elines[elines.length - 1] - // ) { - // slines.pop(); - // elines.pop(); - // linesRemovedAfter++; - // } - - // suggestion = slines.join("\n"); - // if (suggestion === "") return Promise.resolve(false); // Don't even make a suggestion if they are exactly the same - - // range = new vscode.Range( - // new vscode.Position(range.start.line + linesRemovedBefore, 0), - // new vscode.Position( - // range.end.line - linesRemovedAfter, - // elines.at(-1)?.length || 0 - // ) - // ); + // Check for empty suggestions: + if ( + suggestion === "" && + range.start.line === range.end.line && + range.start.character === range.end.character + ) { + return Promise.resolve(false); + } const editor = await openEditorAndRevealRange(editorFilename, range); if (!editor) return Promise.resolve(false); diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts index e4133230..b1df161b 100644 --- a/extension/src/util/messenger.ts +++ b/extension/src/util/messenger.ts @@ -1,5 +1,6 @@ console.log("Websocket import"); const WebSocket = require("ws"); +import fetch from "node-fetch"; export abstract class Messenger { abstract send(messageType: string, data: object): void; @@ -50,18 +51,49 @@ export class WebsocketMessenger extends Messenger { return newWebsocket; } + async checkServerRunning(serverUrl: string): Promise { + // Check if already running by calling /health + try { + const response = await fetch(serverUrl + "/health"); + if (response.status === 200) { + console.log("Continue python server already running"); + return true; + } else { + return false; + } + } catch (e) { + return false; + } + } + constructor(serverUrl: string) { super(); this.serverUrl = serverUrl; this.websocket = this._newWebsocket(); - const interval = setInterval(() => { - if (this.websocket.readyState === this.websocket.OPEN) { - clearInterval(interval); - } else if (this.websocket.readyState !== this.websocket.CONNECTING) { - this.websocket = this._newWebsocket(); - } - }, 1000); + // Wait until the server is running + // const interval = setInterval(async () => { + // if ( + // await this.checkServerRunning( + // serverUrl.replace("/ide/ws", "").replace("ws://", "http://") + // ) + // ) { + // this.websocket = this._newWebsocket(); + // clearInterval(interval); + // } else { + // console.log( + // "Waiting for python server to start-----------------------" + // ); + // } + // }, 1000); + + // const interval = setInterval(() => { + // if (this.websocket.readyState === this.websocket.OPEN) { + // clearInterval(interval); + // } else if (this.websocket.readyState !== this.websocket.CONNECTING) { + // this.websocket = this._newWebsocket(); + // } + // }, 1000); } send(messageType: string, data: object) { @@ -99,10 +131,16 @@ export class WebsocketMessenger extends Messenger { }); } - onMessage(callback: (messageType: string, data: any) => void): void { + onMessage( + callback: ( + messageType: string, + data: any, + messenger: WebsocketMessenger + ) => void + ): void { this.websocket.addEventListener("message", (event) => { const msg = JSON.parse(event.data); - callback(msg.messageType, msg.data); + callback(msg.messageType, msg.data, this); }); } -- cgit v1.2.3-70-g09d2 From 391764f1371dab06af30a29e10a826a516b69bb3 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Wed, 12 Jul 2023 21:53:06 -0700 Subject: persist state and reconnect automatically --- continuedev/src/continuedev/core/autopilot.py | 17 +++- continuedev/src/continuedev/libs/constants/main.py | 6 ++ continuedev/src/continuedev/libs/util/paths.py | 17 ++++ continuedev/src/continuedev/server/gui.py | 10 +- continuedev/src/continuedev/server/ide.py | 25 +++-- continuedev/src/continuedev/server/ide_protocol.py | 10 +- continuedev/src/continuedev/server/main.py | 16 ++- .../src/continuedev/server/session_manager.py | 41 ++++++-- docs/docs/concepts/ide.md | 4 +- extension/react-app/src/components/ComboBox.tsx | 1 - extension/react-app/src/hooks/messenger.ts | 10 -- extension/react-app/src/pages/gui.tsx | 1 - extension/src/activation/activate.ts | 2 +- extension/src/continueIdeClient.ts | 62 ++++++++--- extension/src/debugPanel.ts | 113 ++++----------------- extension/src/util/messenger.ts | 10 ++ 16 files changed, 192 insertions(+), 153 deletions(-) create mode 100644 continuedev/src/continuedev/libs/constants/main.py create mode 100644 continuedev/src/continuedev/libs/util/paths.py (limited to 'extension/src/util') diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index 1b074435..e1c8a076 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -1,13 +1,13 @@ from functools import cached_property import traceback import time -from typing import Any, Callable, Coroutine, Dict, List +from typing import Any, Callable, Coroutine, Dict, List, Union import os from aiohttp import ClientPayloadError +from pydantic import root_validator from ..models.filesystem import RangeInFileWithContents from ..models.filesystem_edit import FileEditWithFullContents -from ..libs.llm import LLM from .observation import Observation, InternalErrorObservation from ..server.ide_protocol import AbstractIdeProtocolServer from ..libs.util.queue import AsyncSubscriptionQueue @@ -16,7 +16,6 @@ from .main import Context, ContinueCustomException, HighlightedRangeContext, Pol from ..steps.core.core import ReversibleStep, ManualEditStep, UserInputStep from ..libs.util.telemetry import capture_event from .sdk import ContinueSDK -import asyncio from ..libs.util.step_name_to_steps import get_step_from_name from ..libs.util.traceback_parsers import get_python_traceback, get_javascript_traceback from openai import error as openai_errors @@ -46,6 +45,7 @@ class Autopilot(ContinueBaseModel): ide: AbstractIdeProtocolServer history: History = History.from_empty() context: Context = Context() + full_state: Union[FullState, None] = None _on_update_callbacks: List[Callable[[FullState], None]] = [] _active: bool = False @@ -63,8 +63,15 @@ class Autopilot(ContinueBaseModel): arbitrary_types_allowed = True keep_untouched = (cached_property,) + @root_validator(pre=True) + def fill_in_values(cls, values): + full_state: FullState = values.get('full_state') + if full_state is not None: + values['history'] = full_state.history + return values + def get_full_state(self) -> FullState: - return FullState( + full_state = FullState( history=self.history, active=self._active, user_input_queue=self._main_user_input_queue, @@ -73,6 +80,8 @@ class Autopilot(ContinueBaseModel): slash_commands=self.get_available_slash_commands(), adding_highlighted_code=self._adding_highlighted_code, ) + self.full_state = full_state + return full_state def get_available_slash_commands(self) -> List[Dict]: custom_commands = list(map(lambda x: { diff --git a/continuedev/src/continuedev/libs/constants/main.py b/continuedev/src/continuedev/libs/constants/main.py new file mode 100644 index 00000000..96eb6e69 --- /dev/null +++ b/continuedev/src/continuedev/libs/constants/main.py @@ -0,0 +1,6 @@ +## PATHS ## + +CONTINUE_GLOBAL_FOLDER = ".continue" +CONTINUE_SESSIONS_FOLDER = "sessions" +CONTINUE_SERVER_FOLDER = "server" + diff --git a/continuedev/src/continuedev/libs/util/paths.py b/continuedev/src/continuedev/libs/util/paths.py new file mode 100644 index 00000000..fddef887 --- /dev/null +++ b/continuedev/src/continuedev/libs/util/paths.py @@ -0,0 +1,17 @@ +import os + +from ..constants.main import CONTINUE_SESSIONS_FOLDER, CONTINUE_GLOBAL_FOLDER, CONTINUE_SERVER_FOLDER + +def getGlobalFolderPath(): + return os.path.join(os.path.expanduser("~"), CONTINUE_GLOBAL_FOLDER) + + + +def getSessionsFolderPath(): + return os.path.join(getGlobalFolderPath(), CONTINUE_SESSIONS_FOLDER) + +def getServerFolderPath(): + return os.path.join(getGlobalFolderPath(), CONTINUE_SERVER_FOLDER) + +def getSessionFilePath(session_id: str): + return os.path.join(getSessionsFolderPath(), f"{session_id}.json") \ No newline at end of file diff --git a/continuedev/src/continuedev/server/gui.py b/continuedev/src/continuedev/server/gui.py index 21089f30..8f6f68f6 100644 --- a/continuedev/src/continuedev/server/gui.py +++ b/continuedev/src/continuedev/server/gui.py @@ -31,12 +31,12 @@ class AppStatus: Server.handle_exit = AppStatus.handle_exit -def session(x_continue_session_id: str = Header("anonymous")) -> Session: - return session_manager.get_session(x_continue_session_id) +async def session(x_continue_session_id: str = Header("anonymous")) -> Session: + return await session_manager.get_session(x_continue_session_id) -def websocket_session(session_id: str) -> Session: - return session_manager.get_session(session_id) +async def websocket_session(session_id: str) -> Session: + return await session_manager.get_session(session_id) T = TypeVar("T", bound=BaseModel) @@ -199,4 +199,6 @@ async def websocket_endpoint(websocket: WebSocket, session: Session = Depends(we print("Closing gui websocket") if websocket.client_state != WebSocketState.DISCONNECTED: await websocket.close() + + session_manager.persist_session(session.session_id) session_manager.remove_session(session.session_id) diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index 4645b49e..12a21f19 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -52,9 +52,11 @@ class FileEditsUpdate(BaseModel): class OpenFilesResponse(BaseModel): openFiles: List[str] + class VisibleFilesResponse(BaseModel): visibleFiles: List[str] + class HighlightedCodeResponse(BaseModel): highlightedCode: List[RangeInFile] @@ -115,6 +117,7 @@ class IdeProtocolServer(AbstractIdeProtocolServer): websocket: WebSocket session_manager: SessionManager sub_queue: AsyncSubscriptionQueue = AsyncSubscriptionQueue() + session_id: Union[str, None] = None def __init__(self, session_manager: SessionManager, websocket: WebSocket): self.websocket = websocket @@ -132,8 +135,6 @@ class IdeProtocolServer(AbstractIdeProtocolServer): continue message_type = message["messageType"] data = message["data"] - # if message_type == "openGUI": - # await self.openGUI() if message_type == "workspaceDirectory": self.workspace_directory = data["workspaceDirectory"] break @@ -158,8 +159,8 @@ class IdeProtocolServer(AbstractIdeProtocolServer): return resp_model.parse_obj(resp) async def handle_json(self, message_type: str, data: Any): - if message_type == "openGUI": - await self.openGUI() + if message_type == "getSessionId": + await self.getSessionId() elif message_type == "setFileOpen": await self.setFileOpen(data["filepath"], data["open"]) elif message_type == "setSuggestionsLocked": @@ -217,9 +218,10 @@ class IdeProtocolServer(AbstractIdeProtocolServer): "locked": locked }) - async def openGUI(self): - session_id = self.session_manager.new_session(self) - await self._send_json("openGUI", { + async def getSessionId(self): + session_id = self.session_manager.new_session( + self, self.session_id).session_id + await self._send_json("getSessionId", { "sessionId": session_id }) @@ -304,7 +306,7 @@ class IdeProtocolServer(AbstractIdeProtocolServer): async def getOpenFiles(self) -> List[str]: resp = await self._send_and_receive_json({}, OpenFilesResponse, "openFiles") return resp.openFiles - + async def getVisibleFiles(self) -> List[str]: resp = await self._send_and_receive_json({}, VisibleFilesResponse, "visibleFiles") return resp.visibleFiles @@ -416,7 +418,7 @@ class IdeProtocolServer(AbstractIdeProtocolServer): @router.websocket("/ws") -async def websocket_endpoint(websocket: WebSocket): +async def websocket_endpoint(websocket: WebSocket, session_id: str = None): try: await websocket.accept() print("Accepted websocket connection from, ", websocket.client) @@ -434,6 +436,9 @@ async def websocket_endpoint(websocket: WebSocket): ideProtocolServer.handle_json(message_type, data)) ideProtocolServer = IdeProtocolServer(session_manager, websocket) + ideProtocolServer.session_id = session_id + if session_id is not None: + session_manager.registered_ides[session_id] = ideProtocolServer other_msgs = await ideProtocolServer.initialize() for other_msg in other_msgs: @@ -454,3 +459,5 @@ async def websocket_endpoint(websocket: WebSocket): finally: if websocket.client_state != WebSocketState.DISCONNECTED: await websocket.close() + + session_manager.registered_ides.pop(ideProtocolServer.session_id) diff --git a/continuedev/src/continuedev/server/ide_protocol.py b/continuedev/src/continuedev/server/ide_protocol.py index 2783dc61..2f78cf0e 100644 --- a/continuedev/src/continuedev/server/ide_protocol.py +++ b/continuedev/src/continuedev/server/ide_protocol.py @@ -1,5 +1,6 @@ -from typing import Any, List +from typing import Any, List, Union from abc import ABC, abstractmethod, abstractproperty +from fastapi import WebSocket from ..models.main import Traceback from ..models.filesystem_edit import FileEdit, FileSystemEdit, EditDiff @@ -7,6 +8,9 @@ from ..models.filesystem import RangeInFile, RangeInFileWithContents class AbstractIdeProtocolServer(ABC): + websocket: WebSocket + session_id: Union[str, None] + @abstractmethod async def handle_json(self, data: Any): """Handle a json message""" @@ -24,8 +28,8 @@ class AbstractIdeProtocolServer(ABC): """Set whether suggestions are locked""" @abstractmethod - async def openGUI(self): - """Open a GUI""" + async def getSessionId(self): + """Get a new session ID""" @abstractmethod async def showSuggestionsAndWait(self, suggestions: List[FileEdit]) -> bool: diff --git a/continuedev/src/continuedev/server/main.py b/continuedev/src/continuedev/server/main.py index f4d82903..aa093853 100644 --- a/continuedev/src/continuedev/server/main.py +++ b/continuedev/src/continuedev/server/main.py @@ -4,7 +4,8 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from .ide import router as ide_router from .gui import router as gui_router -import logging +from .session_manager import session_manager +import atexit import uvicorn import argparse @@ -44,5 +45,16 @@ def run_server(): uvicorn.run(app, host="0.0.0.0", port=args.port) +def cleanup(): + print("Cleaning up sessions") + for session_id in session_manager.sessions: + session_manager.persist_session(session_id) + + +atexit.register(cleanup) if __name__ == "__main__": - run_server() + try: + run_server() + except Exception as e: + cleanup() + raise e diff --git a/continuedev/src/continuedev/server/session_manager.py b/continuedev/src/continuedev/server/session_manager.py index 7147dcfa..fb8ac386 100644 --- a/continuedev/src/continuedev/server/session_manager.py +++ b/continuedev/src/continuedev/server/session_manager.py @@ -1,9 +1,12 @@ -from asyncio import BaseEventLoop +import os from fastapi import WebSocket from typing import Any, Dict, List, Union from uuid import uuid4 +import json +from ..libs.util.paths import getSessionFilePath, getSessionsFolderPath from ..models.filesystem_edit import FileEditWithFullContents +from ..libs.constants.main import CONTINUE_SESSIONS_FOLDER from ..core.policy import DemoPolicy from ..core.main import FullState from ..core.autopilot import Autopilot @@ -39,17 +42,35 @@ class DemoAutopilot(Autopilot): class SessionManager: sessions: Dict[str, Session] = {} + # Mapping of session_id to IDE, where the IDE is still alive + registered_ides: Dict[str, AbstractIdeProtocolServer] = {} - def get_session(self, session_id: str) -> Session: + async def get_session(self, session_id: str) -> Session: if session_id not in self.sessions: + # Check then whether it is persisted by listing all files in the sessions folder + # And only if the IDE is still alive + sessions_folder = getSessionsFolderPath() + session_files = os.listdir(sessions_folder) + if f"{session_id}.json" in session_files and session_id in self.registered_ides: + if self.registered_ides[session_id].session_id is not None: + return self.new_session(self.registered_ides[session_id], session_id=session_id) + raise KeyError("Session ID not recognized", session_id) return self.sessions[session_id] - def new_session(self, ide: AbstractIdeProtocolServer) -> str: - autopilot = DemoAutopilot(policy=DemoPolicy(), ide=ide) - session_id = str(uuid4()) + def new_session(self, ide: AbstractIdeProtocolServer, session_id: Union[str, None] = None) -> Session: + full_state = None + if session_id is not None and os.path.exists(getSessionFilePath(session_id)): + with open(getSessionFilePath(session_id), "r") as f: + full_state = FullState(**json.load(f)) + + autopilot = DemoAutopilot( + policy=DemoPolicy(), ide=ide, full_state=full_state) + session_id = session_id or str(uuid4()) + ide.session_id = session_id session = Session(session_id=session_id, autopilot=autopilot) self.sessions[session_id] = session + self.registered_ides[session_id] = ide async def on_update(state: FullState): await session_manager.send_ws_data(session_id, "state_update", { @@ -58,11 +79,19 @@ class SessionManager: autopilot.on_update(on_update) create_async_task(autopilot.run_policy()) - return session_id + return session def remove_session(self, session_id: str): del self.sessions[session_id] + def persist_session(self, session_id: str): + """Save the session's FullState as a json file""" + full_state = self.sessions[session_id].autopilot.get_full_state() + if not os.path.exists(getSessionsFolderPath()): + os.mkdir(getSessionsFolderPath()) + with open(getSessionFilePath(session_id), "w") as f: + json.dump(full_state.dict(), f) + def register_websocket(self, session_id: str, ws: WebSocket): self.sessions[session_id].ws = ws print("Registered websocket for session", session_id) diff --git a/docs/docs/concepts/ide.md b/docs/docs/concepts/ide.md index dc7b9e23..bd31481b 100644 --- a/docs/docs/concepts/ide.md +++ b/docs/docs/concepts/ide.md @@ -41,9 +41,9 @@ Get the workspace directory Set whether a file is open -### openGUI +### getSessionId -Open a gui +Get a new session ID ### showSuggestionsAndWait diff --git a/extension/react-app/src/components/ComboBox.tsx b/extension/react-app/src/components/ComboBox.tsx index ac994b0a..7d6541c7 100644 --- a/extension/react-app/src/components/ComboBox.tsx +++ b/extension/react-app/src/components/ComboBox.tsx @@ -331,7 +331,6 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => { ) { (event.nativeEvent as any).preventDownshiftDefault = true; } else if (event.key === "ArrowUp") { - console.log("OWJFOIJO"); if (positionInHistory == 0) return; else if ( positionInHistory == history.length && diff --git a/extension/react-app/src/hooks/messenger.ts b/extension/react-app/src/hooks/messenger.ts index e2a0bab8..00ce1fbb 100644 --- a/extension/react-app/src/hooks/messenger.ts +++ b/extension/react-app/src/hooks/messenger.ts @@ -1,6 +1,3 @@ -// console.log("Websocket import"); -// const WebSocket = require("ws"); - export abstract class Messenger { abstract send(messageType: string, data: object): void; @@ -28,13 +25,6 @@ export class WebsocketMessenger extends Messenger { private serverUrl: string; _newWebsocket(): WebSocket { - // // Dynamic import, because WebSocket is builtin with browser, but not with node. And can't use require in browser. - // if (typeof process === "object") { - // console.log("Using node"); - // // process is only available in Node - // var WebSocket = require("ws"); - // } - const newWebsocket = new WebSocket(this.serverUrl); for (const listener of this.onOpenListeners) { this.onOpen(listener); diff --git a/extension/react-app/src/pages/gui.tsx b/extension/react-app/src/pages/gui.tsx index b9382bd1..4ff260fa 100644 --- a/extension/react-app/src/pages/gui.tsx +++ b/extension/react-app/src/pages/gui.tsx @@ -262,7 +262,6 @@ function GUI(props: GUIProps) { const onStepUserInput = (input: string, index: number) => { if (!client) return; - console.log("Sending step user input", input, index); client.sendStepUserInput(input, index); }; diff --git a/extension/src/activation/activate.ts b/extension/src/activation/activate.ts index 559caf44..cd885b12 100644 --- a/extension/src/activation/activate.ts +++ b/extension/src/activation/activate.ts @@ -56,7 +56,7 @@ export async function activateExtension(context: vscode.ExtensionContext) { registerAllCodeLensProviders(context); registerAllCommands(context); - // Initialize IDE Protocol Client, then call "openGUI" + // Initialize IDE Protocol Client const serverUrl = getContinueServerUrl(); ideProtocolClient = new IdeProtocolClient( `${serverUrl.replace("http", "ws")}/ide/ws`, diff --git a/extension/src/continueIdeClient.ts b/extension/src/continueIdeClient.ts index b728833f..4c1fdf1e 100644 --- a/extension/src/continueIdeClient.ts +++ b/extension/src/continueIdeClient.ts @@ -1,10 +1,9 @@ -// import { ShowSuggestionRequest } from "../schema/ShowSuggestionRequest"; import { editorSuggestionsLocked, showSuggestion as showSuggestionInEditor, SuggestionRanges, } from "./suggestions"; -import { openEditorAndRevealRange, getRightViewColumn } from "./util/vscode"; +import { openEditorAndRevealRange } from "./util/vscode"; import { FileEdit } from "../schema/FileEdit"; import { RangeInFile } from "../schema/RangeInFile"; import * as vscode from "vscode"; @@ -15,8 +14,6 @@ import { import { FileEditWithFullContents } from "../schema/FileEditWithFullContents"; import fs = require("fs"); import { WebsocketMessenger } from "./util/messenger"; -import * as path from "path"; -import * as os from "os"; import { diffManager } from "./diffs"; class IdeProtocolClient { @@ -27,17 +24,54 @@ class IdeProtocolClient { private _highlightDebounce: NodeJS.Timeout | null = null; - constructor(serverUrl: string, context: vscode.ExtensionContext) { - this.context = context; + private _lastReloadTime: number = 16; + private _reconnectionTimeouts: NodeJS.Timeout[] = []; + + private _sessionId: string | null = null; + private _serverUrl: string; - let messenger = new WebsocketMessenger(serverUrl); + private _newWebsocketMessenger() { + const requestUrl = + this._serverUrl + + (this._sessionId ? `?session_id=${this._sessionId}` : ""); + const messenger = new WebsocketMessenger(requestUrl); this.messenger = messenger; - messenger.onClose(() => { + + const reconnect = () => { + console.log("Trying to reconnect IDE protocol websocket..."); this.messenger = null; + + // Exponential backoff to reconnect + this._reconnectionTimeouts.forEach((to) => clearTimeout(to)); + + const timeout = setTimeout(() => { + if (this.messenger?.websocket?.readyState === 1) { + return; + } + this._newWebsocketMessenger(); + }, this._lastReloadTime); + + this._reconnectionTimeouts.push(timeout); + this._lastReloadTime = Math.min(2 * this._lastReloadTime, 5000); + }; + messenger.onOpen(() => { + this._reconnectionTimeouts.forEach((to) => clearTimeout(to)); + }); + messenger.onClose(() => { + reconnect(); + }); + messenger.onError(() => { + reconnect(); }); messenger.onMessage((messageType, data, messenger) => { this.handleMessage(messageType, data, messenger); }); + } + + constructor(serverUrl: string, context: vscode.ExtensionContext) { + this.context = context; + this._serverUrl = serverUrl; + this._newWebsocketMessenger(); // Setup listeners for any file changes in open editors // vscode.workspace.onDidChangeTextDocument((event) => { @@ -171,7 +205,7 @@ class IdeProtocolClient { case "showDiff": this.showDiff(data.filepath, data.replacement, data.step_index); break; - case "openGUI": + case "getSessionId": case "connected": break; default: @@ -284,10 +318,6 @@ class IdeProtocolClient { // ------------------------------------ // // Initiate Request - async openGUI(asRightWebviewPanel: boolean = false) { - // Open the webview panel - } - async getSessionId(): Promise { await new Promise((resolve, reject) => { // Repeatedly try to connect to the server @@ -303,10 +333,10 @@ class IdeProtocolClient { } }, 1000); }); - const resp = await this.messenger?.sendAndReceive("openGUI", {}); - const sessionId = resp.sessionId; + const resp = await this.messenger?.sendAndReceive("getSessionId", {}); // console.log("New Continue session with ID: ", sessionId); - return sessionId; + this._sessionId = resp.sessionId; + return resp.sessionId; } acceptRejectSuggestion(accept: boolean, key: SuggestionRanges) { diff --git a/extension/src/debugPanel.ts b/extension/src/debugPanel.ts index 487bbedf..5e1689d1 100644 --- a/extension/src/debugPanel.ts +++ b/extension/src/debugPanel.ts @@ -8,76 +8,6 @@ import { import { RangeInFile } from "./client"; const WebSocket = require("ws"); -class StreamManager { - private _fullText: string = ""; - private _insertionPoint: vscode.Position | undefined; - - private _addToEditor(update: string) { - let editor = - vscode.window.activeTextEditor || vscode.window.visibleTextEditors[0]; - - if (typeof this._insertionPoint === "undefined") { - if (editor?.selection.isEmpty) { - this._insertionPoint = editor?.selection.active; - } else { - this._insertionPoint = editor?.selection.end; - } - } - editor?.edit((editBuilder) => { - if (this._insertionPoint) { - editBuilder.insert(this._insertionPoint, update); - this._insertionPoint = this._insertionPoint.translate( - Array.from(update.matchAll(/\n/g)).length, - update.length - ); - } - }); - } - - public closeStream() { - this._fullText = ""; - this._insertionPoint = undefined; - this._codeBlockStatus = "closed"; - this._pendingBackticks = 0; - } - - private _codeBlockStatus: "open" | "closed" | "language-descriptor" = - "closed"; - private _pendingBackticks: number = 0; - public onStreamUpdate(update: string) { - let textToInsert = ""; - for (let i = 0; i < update.length; i++) { - switch (this._codeBlockStatus) { - case "closed": - if (update[i] === "`" && this._fullText.endsWith("``")) { - this._codeBlockStatus = "language-descriptor"; - } - break; - case "language-descriptor": - if (update[i] === " " || update[i] === "\n") { - this._codeBlockStatus = "open"; - } - break; - case "open": - if (update[i] === "`") { - if (this._fullText.endsWith("``")) { - this._codeBlockStatus = "closed"; - this._pendingBackticks = 0; - } else { - this._pendingBackticks += 1; - } - } else { - textToInsert += "`".repeat(this._pendingBackticks) + update[i]; - this._pendingBackticks = 0; - } - break; - } - this._fullText += update[i]; - } - this._addToEditor(textToInsert); - } -} - let websocketConnections: { [url: string]: WebsocketConnection | undefined } = {}; @@ -127,8 +57,6 @@ class WebsocketConnection { } } -let streamManager = new StreamManager(); - export let debugPanelWebview: vscode.Webview | undefined; export function setupDebugPanel( panel: vscode.WebviewPanel | vscode.WebviewView, @@ -147,10 +75,7 @@ export function setupDebugPanel( .toString(); const isProduction = true; // context?.extensionMode === vscode.ExtensionMode.Development; - if (!isProduction) { - scriptUri = "http://localhost:5173/src/main.tsx"; - styleMainUri = "http://localhost:5173/src/main.css"; - } else { + if (isProduction) { scriptUri = debugPanelWebview .asWebviewUri( vscode.Uri.joinPath(extensionUri, "react-app/dist/assets/index.js") @@ -161,6 +86,9 @@ export function setupDebugPanel( vscode.Uri.joinPath(extensionUri, "react-app/dist/assets/index.css") ) .toString(); + } else { + scriptUri = "http://localhost:5173/src/main.tsx"; + styleMainUri = "http://localhost:5173/src/main.css"; } panel.webview.options = { @@ -175,11 +103,11 @@ export function setupDebugPanel( return; } - let rangeInFile: RangeInFile = { + const rangeInFile: RangeInFile = { range: e.selections[0], filepath: e.textEditor.document.fileName, }; - let filesystem = { + const filesystem = { [rangeInFile.filepath]: e.textEditor.document.getText(), }; panel.webview.postMessage({ @@ -217,13 +145,19 @@ export function setupDebugPanel( url, }); }; - const connection = new WebsocketConnection( - url, - onMessage, - onOpen, - onClose - ); - websocketConnections[url] = connection; + try { + const connection = new WebsocketConnection( + url, + onMessage, + onOpen, + onClose + ); + websocketConnections[url] = connection; + resolve(null); + } catch (e) { + console.log("Caught it!: ", e); + reject(e); + } }); } @@ -292,15 +226,6 @@ export function setupDebugPanel( openEditorAndRevealRange(data.path, undefined, vscode.ViewColumn.One); break; } - case "streamUpdate": { - // Write code at the position of the cursor - streamManager.onStreamUpdate(data.update); - break; - } - case "closeStream": { - streamManager.closeStream(); - break; - } case "withProgress": { // This message allows withProgress to be used in the webview if (data.done) { diff --git a/extension/src/util/messenger.ts b/extension/src/util/messenger.ts index b1df161b..7fd71ddd 100644 --- a/extension/src/util/messenger.ts +++ b/extension/src/util/messenger.ts @@ -15,6 +15,8 @@ export abstract class Messenger { abstract onOpen(callback: () => void): void; abstract onClose(callback: () => void): void; + + abstract onError(callback: () => void): void; abstract sendAndReceive(messageType: string, data: any): Promise; } @@ -26,6 +28,7 @@ export class WebsocketMessenger extends Messenger { } = {}; private onOpenListeners: (() => void)[] = []; private onCloseListeners: (() => void)[] = []; + private onErrorListeners: (() => void)[] = []; private serverUrl: string; _newWebsocket(): WebSocket { @@ -43,6 +46,9 @@ export class WebsocketMessenger extends Messenger { for (const listener of this.onCloseListeners) { this.onClose(listener); } + for (const listener of this.onErrorListeners) { + this.onError(listener); + } for (const messageType in this.onMessageListeners) { for (const listener of this.onMessageListeners[messageType]) { this.onMessageType(messageType, listener); @@ -151,4 +157,8 @@ export class WebsocketMessenger extends Messenger { onClose(callback: () => void): void { this.websocket.addEventListener("close", callback); } + + onError(callback: () => void): void { + this.websocket.addEventListener("error", callback); + } } -- cgit v1.2.3-70-g09d2 From c5102b0997baa81ce544514d6b5b4d5a2eae804f Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Fri, 14 Jul 2023 13:45:10 -0700 Subject: insidious client_state vs application_state err --- continuedev/src/continuedev/core/autopilot.py | 2 ++ continuedev/src/continuedev/server/gui.py | 2 +- continuedev/src/continuedev/server/ide.py | 2 +- continuedev/src/continuedev/steps/core/core.py | 9 ++++++++- extension/react-app/src/components/ComboBox.tsx | 9 ++++++++- extension/react-app/src/components/StepContainer.tsx | 9 ++++++--- extension/src/continueIdeClient.ts | 8 ++++++-- extension/src/diffs.ts | 2 +- extension/src/util/messenger.ts | 2 +- 9 files changed, 34 insertions(+), 11 deletions(-) (limited to 'extension/src/util') diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index e1c8a076..82439f49 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -37,6 +37,8 @@ def get_error_title(e: Exception) -> str: return "The request failed. Please check your internet connection and try again. If this issue persists, you can use our API key for free by going to VS Code settings and changing the value of continue.OPENAI_API_KEY to \"\"" elif isinstance(e, openai_errors.InvalidRequestError): return 'Your API key does not have access to GPT-4. You can use ours for free by going to VS Code settings and changing the value of continue.OPENAI_API_KEY to ""' + elif e.__str__().startswith("Cannot connect to host"): + return "The request failed. Please check your internet connection and try again." return e.__str__() or e.__repr__() diff --git a/continuedev/src/continuedev/server/gui.py b/continuedev/src/continuedev/server/gui.py index 238273b2..9a411fbe 100644 --- a/continuedev/src/continuedev/server/gui.py +++ b/continuedev/src/continuedev/server/gui.py @@ -53,7 +53,7 @@ class GUIProtocolServer(AbstractGUIProtocolServer): self.session = session async def _send_json(self, message_type: str, data: Any): - if self.websocket.client_state == WebSocketState.DISCONNECTED: + if self.websocket.application_state == WebSocketState.DISCONNECTED: return await self.websocket.send_json({ "messageType": message_type, diff --git a/continuedev/src/continuedev/server/ide.py b/continuedev/src/continuedev/server/ide.py index 73cce201..7875c94d 100644 --- a/continuedev/src/continuedev/server/ide.py +++ b/continuedev/src/continuedev/server/ide.py @@ -149,7 +149,7 @@ class IdeProtocolServer(AbstractIdeProtocolServer): return other_msgs async def _send_json(self, message_type: str, data: Any): - if self.websocket.client_state == WebSocketState.DISCONNECTED: + if self.websocket.application_state == WebSocketState.DISCONNECTED: return await self.websocket.send_json({ "messageType": message_type, diff --git a/continuedev/src/continuedev/steps/core/core.py b/continuedev/src/continuedev/steps/core/core.py index 787da316..75f8e460 100644 --- a/continuedev/src/continuedev/steps/core/core.py +++ b/continuedev/src/continuedev/steps/core/core.py @@ -9,7 +9,7 @@ from ...libs.llm.prompt_utils import MarkdownStyleEncoderDecoder from ...models.filesystem_edit import EditDiff, FileEdit, FileEditWithFullContents, FileSystemEdit from ...models.filesystem import FileSystem, RangeInFile, RangeInFileWithContents from ...core.observation import Observation, TextObservation, TracebackObservation, UserInputObservation -from ...core.main import ChatMessage, Step, SequentialStep +from ...core.main import ChatMessage, ContinueCustomException, Step, SequentialStep from ...libs.util.count_tokens import MAX_TOKENS_FOR_MODEL, DEFAULT_MAX_TOKENS from ...libs.util.dedent import dedent_and_get_common_whitespace import difflib @@ -608,6 +608,13 @@ Please output the code to be inserted at the cursor in order to fulfill the user rif_dict[rif.filepath] = rif.contents for rif in rif_with_contents: + # If the file doesn't exist, ask them to save it first + if not os.path.exists(rif.filepath): + message = f"The file {rif.filepath} does not exist. Please save it first." + raise ContinueCustomException( + title=message, message=message + ) + await sdk.ide.setFileOpen(rif.filepath) await sdk.ide.setSuggestionsLocked(rif.filepath, True) await self.stream_rif(rif, sdk) diff --git a/extension/react-app/src/components/ComboBox.tsx b/extension/react-app/src/components/ComboBox.tsx index 5d9b5109..754c9445 100644 --- a/extension/react-app/src/components/ComboBox.tsx +++ b/extension/react-app/src/components/ComboBox.tsx @@ -169,6 +169,7 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => { useImperativeHandle(ref, () => downshiftProps, [downshiftProps]); const [metaKeyPressed, setMetaKeyPressed] = useState(false); + const [focused, setFocused] = useState(false); useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if (e.key === "Meta") { @@ -298,7 +299,11 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => { // setShowContextDropdown(target.value.endsWith("@")); }, + onFocus: (e) => { + setFocused(true); + }, onBlur: (e) => { + setFocused(false); postVscMessage("blurContinueInput", {}); }, onKeyDown: (event) => { @@ -374,7 +379,9 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => { {highlightedCodeSections.length === 0 && (downshiftProps.inputValue?.startsWith("/edit") || - (metaKeyPressed && downshiftProps.inputValue?.length > 0)) && ( + (focused && + metaKeyPressed && + downshiftProps.inputValue?.length > 0)) && (
Inserting at cursor
diff --git a/extension/react-app/src/components/StepContainer.tsx b/extension/react-app/src/components/StepContainer.tsx index 6fa4ba13..14e9b854 100644 --- a/extension/react-app/src/components/StepContainer.tsx +++ b/extension/react-app/src/components/StepContainer.tsx @@ -253,9 +253,12 @@ function StepContainer(props: StepContainerProps) { )} {props.historyNode.observation?.error ? ( - 
-              {props.historyNode.observation.error as string}
-
+
+ View Traceback + 
+                {props.historyNode.observation.error as string}
+
+
) : ( void): void; abstract onClose(callback: () => void): void; - + abstract onError(callback: () => void): void; abstract sendAndReceive(messageType: string, data: any): Promise; -- cgit v1.2.3-70-g09d2 From 45ee33f7fd84c0bc49d35d9d1a7a3a8e9f6addd7 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sat, 15 Jul 2023 15:06:32 -0700 Subject: use correct label for meta key --- extension/react-app/src/components/ComboBox.tsx | 3 ++- .../react-app/src/components/StepContainer.tsx | 7 +++++- extension/react-app/src/util/index.ts | 17 +++++++++++-- extension/src/diffs.ts | 3 ++- extension/src/lang-server/codeLens.ts | 7 +++--- extension/src/util/util.ts | 29 ++++++++++++++++++++++ 6 files changed, 58 insertions(+), 8 deletions(-) (limited to 'extension/src/util') diff --git a/extension/react-app/src/components/ComboBox.tsx b/extension/react-app/src/components/ComboBox.tsx index 754c9445..f11e07af 100644 --- a/extension/react-app/src/components/ComboBox.tsx +++ b/extension/react-app/src/components/ComboBox.tsx @@ -13,6 +13,7 @@ import HeaderButtonWithText from "./HeaderButtonWithText"; import { DocumentPlus } from "@styled-icons/heroicons-outline"; import { HighlightedRangeContext } from "../../../schema/FullState"; import { postVscMessage } from "../vscode"; +import { getMetaKeyLabel } from "../util"; // #region styled components const mainInputFontSize = 13; @@ -286,7 +287,7 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => {