From d342c55ed2b8b229ccd40cc45f7af05cb73e7ad0 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sun, 9 Jul 2023 13:09:34 -0700 Subject: expand max_tokens for large highlighted ranges --- continuedev/src/continuedev/libs/llm/openai.py | 6 +++--- .../src/continuedev/libs/llm/proxy_server.py | 6 +++--- .../src/continuedev/libs/util/count_tokens.py | 4 ++-- continuedev/src/continuedev/steps/core/core.py | 23 ++++++++++++++-------- extension/package-lock.json | 4 ++-- extension/package.json | 2 +- extension/src/activation/environmentSetup.ts | 9 ++++++++- 7 files changed, 34 insertions(+), 20 deletions(-) diff --git a/continuedev/src/continuedev/libs/llm/openai.py b/continuedev/src/continuedev/libs/llm/openai.py index c4e4139f..f0877d90 100644 --- a/continuedev/src/continuedev/libs/llm/openai.py +++ b/continuedev/src/continuedev/libs/llm/openai.py @@ -37,7 +37,7 @@ class OpenAI(LLM): if args["model"] in CHAT_MODELS: async for chunk in await openai.ChatCompletion.acreate( messages=compile_chat_messages( - args["model"], with_history, prompt, functions=None), + args["model"], with_history, args["max_tokens"], prompt, functions=None), **args, ): if "content" in chunk.choices[0].delta: @@ -58,7 +58,7 @@ class OpenAI(LLM): async for chunk in await openai.ChatCompletion.acreate( messages=compile_chat_messages( - args["model"], messages, functions=args.get("functions", None)), + args["model"], messages, args["max_tokens"], functions=args.get("functions", None)), **args, ): yield chunk.choices[0].delta @@ -69,7 +69,7 @@ class OpenAI(LLM): if args["model"] in CHAT_MODELS: resp = (await openai.ChatCompletion.acreate( messages=compile_chat_messages( - args["model"], with_history, prompt, functions=None), + args["model"], with_history, args["max_tokens"], prompt, functions=None), **args, )).choices[0].message.content else: diff --git a/continuedev/src/continuedev/libs/llm/proxy_server.py b/continuedev/src/continuedev/libs/llm/proxy_server.py index 05ece394..eab6e441 100644 --- a/continuedev/src/continuedev/libs/llm/proxy_server.py +++ b/continuedev/src/continuedev/libs/llm/proxy_server.py @@ -38,7 +38,7 @@ class ProxyServer(LLM): async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl_context=ssl_context)) as session: async with session.post(f"{SERVER_URL}/complete", json={ - "messages": compile_chat_messages(args["model"], with_history, prompt, functions=None), + "messages": compile_chat_messages(args["model"], with_history, args["max_tokens"], prompt, functions=None), "unique_id": self.unique_id, **args }) as resp: @@ -50,7 +50,7 @@ class ProxyServer(LLM): async def stream_chat(self, messages: List[ChatMessage] = [], **kwargs) -> Coroutine[Any, Any, Generator[Union[Any, List, Dict], None, None]]: args = {**self.default_args, **kwargs} messages = compile_chat_messages( - self.default_model, messages, None, functions=args.get("functions", None)) + self.default_model, messages, args["max_tokens"], None, functions=args.get("functions", None)) async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl_context=ssl_context)) as session: async with session.post(f"{SERVER_URL}/stream_chat", json={ @@ -74,7 +74,7 @@ class ProxyServer(LLM): async def stream_complete(self, prompt, with_history: List[ChatMessage] = [], **kwargs) -> Generator[Union[Any, List, Dict], None, None]: args = {**self.default_args, **kwargs} messages = compile_chat_messages( - self.default_model, with_history, prompt, functions=args.get("functions", None)) + self.default_model, with_history, args["max_tokens"], prompt, functions=args.get("functions", None)) async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl_context=ssl_context)) as session: async with session.post(f"{SERVER_URL}/stream_complete", json={ diff --git a/continuedev/src/continuedev/libs/util/count_tokens.py b/continuedev/src/continuedev/libs/util/count_tokens.py index 8b06fef9..73be0717 100644 --- a/continuedev/src/continuedev/libs/util/count_tokens.py +++ b/continuedev/src/continuedev/libs/util/count_tokens.py @@ -76,14 +76,14 @@ def prune_chat_history(model: str, chat_history: List[ChatMessage], max_tokens: return chat_history -def compile_chat_messages(model: str, msgs: List[ChatMessage], prompt: Union[str, None] = None, functions: Union[List, None] = None, system_message: Union[str, None] = None) -> List[Dict]: +def compile_chat_messages(model: str, msgs: List[ChatMessage], max_tokens: int, prompt: Union[str, None] = None, functions: Union[List, None] = None, system_message: Union[str, None] = None) -> List[Dict]: prompt_tokens = count_tokens(model, prompt) if functions is not None: for function in functions: prompt_tokens += count_tokens(model, json.dumps(function)) msgs = prune_chat_history(model, - msgs, MAX_TOKENS_FOR_MODEL[model], prompt_tokens + DEFAULT_MAX_TOKENS + count_tokens(model, system_message)) + msgs, MAX_TOKENS_FOR_MODEL[model], prompt_tokens + max_tokens + count_tokens(model, system_message)) history = [] if system_message: history.append({ diff --git a/continuedev/src/continuedev/steps/core/core.py b/continuedev/src/continuedev/steps/core/core.py index 10853828..4b35a758 100644 --- a/continuedev/src/continuedev/steps/core/core.py +++ b/continuedev/src/continuedev/steps/core/core.py @@ -181,15 +181,22 @@ class DefaultModelEditCodeStep(Step): # We care because if this prompt itself goes over the limit, then the entire message will have to be cut from the completion. # Overflow won't happen, but prune_chat_messages in count_tokens.py will cut out this whole thing, instead of us cutting out only as many lines as we need. model_to_use = sdk.models.gpt4 + max_tokens = DEFAULT_MAX_TOKENS - BUFFER_FOR_FUNCTIONS = 400 - total_tokens = model_to_use.count_tokens( - full_file_contents + self._prompt + self.user_input) + BUFFER_FOR_FUNCTIONS + DEFAULT_MAX_TOKENS - - TOKENS_TO_BE_CONSIDERED_LARGE_RANGE = 1000 + TOKENS_TO_BE_CONSIDERED_LARGE_RANGE = 1200 if model_to_use.count_tokens(rif.contents) > TOKENS_TO_BE_CONSIDERED_LARGE_RANGE: self.description += "\n\n**It looks like you've selected a large range to edit, which may take a while to complete. If you'd like to cancel, click the 'X' button above. If you highlight a more specific range, Continue will only edit within it.**" + # At this point, we also increase the max_tokens parameter so it doesn't stop in the middle of generation + # Increase max_tokens to be double the size of the range + # But don't exceed twice default max tokens + max_tokens = int(min(model_to_use.count_tokens( + rif.contents), DEFAULT_MAX_TOKENS) * 2.5) + + BUFFER_FOR_FUNCTIONS = 400 + total_tokens = model_to_use.count_tokens( + full_file_contents + self._prompt + self.user_input) + BUFFER_FOR_FUNCTIONS + max_tokens + # If using 3.5 and overflows, upgrade to 3.5.16k if model_to_use.name == "gpt-3.5-turbo": if total_tokens > MAX_TOKENS_FOR_MODEL["gpt-3.5-turbo"]: @@ -252,7 +259,7 @@ class DefaultModelEditCodeStep(Step): file_suffix = "\n" + file_suffix rif.contents = rif.contents[:-1] - return file_prefix, rif.contents, file_suffix, model_to_use + return file_prefix, rif.contents, file_suffix, model_to_use, max_tokens def compile_prompt(self, file_prefix: str, contents: str, file_suffix: str, sdk: ContinueSDK) -> str: prompt = self._prompt @@ -289,7 +296,7 @@ class DefaultModelEditCodeStep(Step): await sdk.ide.saveFile(rif.filepath) full_file_contents = await sdk.ide.readFile(rif.filepath) - file_prefix, contents, file_suffix, model_to_use = await self.get_prompt_parts( + file_prefix, contents, file_suffix, model_to_use, max_tokens = await self.get_prompt_parts( rif, sdk, full_file_contents) contents, common_whitespace = dedent_and_get_common_whitespace( contents) @@ -435,7 +442,7 @@ class DefaultModelEditCodeStep(Step): completion_lines_covered = 0 repeating_file_suffix = False line_below_highlighted_range = file_suffix.lstrip().split("\n")[0] - async for chunk in model_to_use.stream_chat(messages, temperature=0): + async for chunk in model_to_use.stream_chat(messages, temperature=0, max_tokens=max_tokens): # Stop early if it is repeating the file_suffix or the step was deleted if repeating_file_suffix: break diff --git a/extension/package-lock.json b/extension/package-lock.json index 3c0d6e3e..22f8b492 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.139", + "version": "0.0.141", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.139", + "version": "0.0.141", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index e1de3b73..beb675b3 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "The open-source coding autopilot", - "version": "0.0.139", + "version": "0.0.141", "publisher": "Continue", "engines": { "vscode": "^1.67.0" diff --git a/extension/src/activation/environmentSetup.ts b/extension/src/activation/environmentSetup.ts index 90ec9259..714080e3 100644 --- a/extension/src/activation/environmentSetup.ts +++ b/extension/src/activation/environmentSetup.ts @@ -324,7 +324,14 @@ export async function startContinuePythonServer() { } } console.log("Killing old server..."); - await fkill(":65432"); + try { + await fkill(":65432"); + } catch (e) { + console.log( + "Failed to kill old server, likely because it didn't exist:", + e + ); + } } // Do this after above check so we don't have to waste time setting up the env -- cgit v1.2.3-70-g09d2 From 2146529ff2f73356102f67f5011ef2437f334ad6 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sun, 9 Jul 2023 13:21:02 -0700 Subject: cleaning up context adding logic --- continuedev/src/continuedev/core/autopilot.py | 32 +++++++++++++-------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index 8b3fb97d..b583d682 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -151,31 +151,29 @@ class Autopilot(ContinueBaseModel): self._highlighted_ranges[0].editing = True async def handle_highlighted_code(self, range_in_files: List[RangeInFileWithContents]): - - # If un-highlighting, then remove the range - if len(self._highlighted_ranges) == 1 and len(range_in_files) <= 1 and (len(range_in_files) == 0 or range_in_files[0].range.start == range_in_files[0].range.end) and not self._adding_highlighted_code: - self._highlighted_ranges = [] - await self.update_subscribers() - return - - # If not toggled to be adding context, only edit or add the first range - if not self._adding_highlighted_code and len(self._highlighted_ranges) > 0: - if len(range_in_files) == 0: - return - if range_in_files[0].range.overlaps_with(self._highlighted_ranges[0].range.range) and range_in_files[0].filepath == self._highlighted_ranges[0].range.filepath: - self._highlighted_ranges = [HighlightedRangeContext( - range=range_in_files[0].range, editing=True, pinned=False)] - await self.update_subscribers() - return - # Filter out rifs from ~/.continue/diffs folder range_in_files = [ rif for rif in range_in_files if not os.path.dirname(rif.filepath) == os.path.expanduser("~/.continue/diffs")] + # Make sure all filepaths are relative to workspace workspace_path = self.continue_sdk.ide.workspace_directory for rif in range_in_files: rif.filepath = os.path.basename(rif.filepath) + # If not adding highlighted code + if not self._adding_highlighted_code: + if len(self._highlighted_ranges) == 1 and len(range_in_files) <= 1 and (len(range_in_files) == 0 or range_in_files[0].range.start == range_in_files[0].range.end): + # If un-highlighting the range to edit, then remove the range + self._highlighted_ranges = [] + await self.update_subscribers() + elif len(range_in_files) > 0: + # Otherwise, replace the current range with the new one + # This is the first range to be highlighted + self._highlighted_ranges = [HighlightedRangeContext( + range=range_in_files[0], editing=True, pinned=False)] + await self.update_subscribers() + return + # If current range overlaps with any others, delete them and only keep the new range new_ranges = [] for i, rif in enumerate(self._highlighted_ranges): -- cgit v1.2.3-70-g09d2 From 5709c9ae003ededb520ee641cae8c8570d4c93af Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sun, 9 Jul 2023 14:12:36 -0700 Subject: sdk.get_code_context --- continuedev/src/continuedev/core/autopilot.py | 6 ++-- continuedev/src/continuedev/core/main.py | 1 + continuedev/src/continuedev/core/sdk.py | 7 ++++- continuedev/src/continuedev/steps/main.py | 38 +++++++++++-------------- extension/react-app/src/components/ComboBox.tsx | 6 ++-- extension/schema/FullState.d.ts | 2 ++ schema/json/FullState.json | 7 ++++- 7 files changed, 36 insertions(+), 31 deletions(-) diff --git a/continuedev/src/continuedev/core/autopilot.py b/continuedev/src/continuedev/core/autopilot.py index b583d682..5c3baafd 100644 --- a/continuedev/src/continuedev/core/autopilot.py +++ b/continuedev/src/continuedev/core/autopilot.py @@ -157,8 +157,6 @@ class Autopilot(ContinueBaseModel): # Make sure all filepaths are relative to workspace workspace_path = self.continue_sdk.ide.workspace_directory - for rif in range_in_files: - rif.filepath = os.path.basename(rif.filepath) # If not adding highlighted code if not self._adding_highlighted_code: @@ -170,7 +168,7 @@ class Autopilot(ContinueBaseModel): # Otherwise, replace the current range with the new one # This is the first range to be highlighted self._highlighted_ranges = [HighlightedRangeContext( - range=range_in_files[0], editing=True, pinned=False)] + range=range_in_files[0], editing=True, pinned=False, display_name=os.path.basename(range_in_files[0].filepath))] await self.update_subscribers() return @@ -193,7 +191,7 @@ class Autopilot(ContinueBaseModel): new_ranges.append(rif) self._highlighted_ranges = new_ranges + [HighlightedRangeContext( - range=rif, editing=False, pinned=False + range=rif, editing=False, pinned=False, display_name=os.path.basename(rif.filepath) ) for rif in range_in_files] self._make_sure_is_editing_range() diff --git a/continuedev/src/continuedev/core/main.py b/continuedev/src/continuedev/core/main.py index 4ea17f20..88690c83 100644 --- a/continuedev/src/continuedev/core/main.py +++ b/continuedev/src/continuedev/core/main.py @@ -205,6 +205,7 @@ class HighlightedRangeContext(ContinueBaseModel): range: RangeInFileWithContents editing: bool pinned: bool + display_name: str class FullState(ContinueBaseModel): diff --git a/continuedev/src/continuedev/core/sdk.py b/continuedev/src/continuedev/core/sdk.py index ed670799..8649cd58 100644 --- a/continuedev/src/continuedev/core/sdk.py +++ b/continuedev/src/continuedev/core/sdk.py @@ -13,7 +13,7 @@ from ..libs.llm.hf_inference_api import HuggingFaceInferenceAPI from ..libs.llm.openai import OpenAI from .observation import Observation from ..server.ide_protocol import AbstractIdeProtocolServer -from .main import Context, ContinueCustomException, History, Step, ChatMessage, ChatMessageRole +from .main import Context, ContinueCustomException, HighlightedRangeContext, History, Step, ChatMessage, ChatMessageRole from ..steps.core.core import * from ..libs.llm.proxy_server import ProxyServer @@ -178,6 +178,11 @@ class ContinueSDK(AbstractContinueSDK): else: return load_global_config() + def get_code_context(self, only_editing: bool = False) -> List[RangeInFileWithContents]: + context = list(filter(lambda x: x.editing, self.__autopilot._highlighted_ranges) + ) if only_editing else self.__autopilot._highlighted_ranges + return [c.range for c in context] + def update_default_model(self, model: str): config = self.config config.default_model = model diff --git a/continuedev/src/continuedev/steps/main.py b/continuedev/src/continuedev/steps/main.py index 5ccffbfe..0a20ddd7 100644 --- a/continuedev/src/continuedev/steps/main.py +++ b/continuedev/src/continuedev/steps/main.py @@ -97,7 +97,7 @@ class FasterEditHighlightedCodeStep(Step): return "Editing highlighted code" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: - range_in_files = await sdk.ide.getHighlightedCode() + range_in_files = await sdk.get_code_context(only_editing=True) if len(range_in_files) == 0: # Get the full contents of all open files files = await sdk.ide.getOpenFiles() @@ -105,21 +105,16 @@ class FasterEditHighlightedCodeStep(Step): for file in files: contents[file] = await sdk.ide.readFile(file) - range_in_files = [RangeInFile.from_entire_file( + range_in_files = [RangeInFileWithContents.from_entire_file( filepath, content) for filepath, content in contents.items()] - rif_with_contents = [] - for range_in_file in range_in_files: - file_contents = await sdk.ide.readRangeInFile(range_in_file) - rif_with_contents.append( - RangeInFileWithContents.from_range_in_file(range_in_file, file_contents)) - enc_dec = MarkdownStyleEncoderDecoder(rif_with_contents) + enc_dec = MarkdownStyleEncoderDecoder(range_in_files) code_string = enc_dec.encode() prompt = self._prompt.format( code=code_string, user_input=self.user_input) rif_dict = {} - for rif in rif_with_contents: + for rif in range_in_files: rif_dict[rif.filepath] = rif.contents completion = await sdk.models.gpt35.complete(prompt) @@ -193,7 +188,7 @@ class StarCoderEditHighlightedCodeStep(Step): return await models.gpt35.complete(f"{self._prompt_and_completion}\n\nPlease give brief a description of the changes made above using markdown bullet points:") async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: - range_in_files = await sdk.ide.getHighlightedCode() + range_in_files = await sdk.get_code_context(only_editing=True) found_highlighted_code = len(range_in_files) > 0 if not found_highlighted_code: # Get the full contents of all open files @@ -202,20 +197,14 @@ class StarCoderEditHighlightedCodeStep(Step): for file in files: contents[file] = await sdk.ide.readFile(file) - range_in_files = [RangeInFile.from_entire_file( + range_in_files = [RangeInFileWithContents.from_entire_file( filepath, content) for filepath, content in contents.items()] - rif_with_contents = [] - for range_in_file in range_in_files: - file_contents = await sdk.ide.readRangeInFile(range_in_file) - rif_with_contents.append( - RangeInFileWithContents.from_range_in_file(range_in_file, file_contents)) - rif_dict = {} - for rif in rif_with_contents: + for rif in range_in_files: rif_dict[rif.filepath] = rif.contents - for rif in rif_with_contents: + for rif in range_in_files: prompt = self._prompt.format( code=rif.contents, user_request=self.user_input) @@ -255,7 +244,7 @@ class EditHighlightedCodeStep(Step): return "Editing code" async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: - range_in_files = await sdk.ide.getHighlightedCode() + range_in_files = sdk.get_code_context(only_editing=True) if len(range_in_files) == 0: # Get the full contents of all open files files = await sdk.ide.getOpenFiles() @@ -263,7 +252,7 @@ class EditHighlightedCodeStep(Step): for file in files: contents[file] = await sdk.ide.readFile(file) - range_in_files = [RangeInFile.from_entire_file( + range_in_files = [RangeInFileWithContents.from_entire_file( filepath, content) for filepath, content in contents.items()] # If still no highlighted code, create a new file and edit there @@ -271,7 +260,12 @@ class EditHighlightedCodeStep(Step): # Create a new file new_file_path = "new_file.txt" await sdk.add_file(new_file_path, "") - range_in_files = [RangeInFile.from_entire_file(new_file_path, "")] + range_in_files = [ + RangeInFileWithContents.from_entire_file(new_file_path, "")] + + range_in_files = list(map(lambda x: RangeInFile( + filepath=x.filepath, range=x.range + ), range_in_files)) await sdk.run_step(DefaultModelEditCodeStep(user_input=self.user_input, range_in_files=range_in_files)) diff --git a/extension/react-app/src/components/ComboBox.tsx b/extension/react-app/src/components/ComboBox.tsx index e6632360..801c3a03 100644 --- a/extension/react-app/src/components/ComboBox.tsx +++ b/extension/react-app/src/components/ComboBox.tsx @@ -224,8 +224,8 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => { editing={section.editing} pinned={section.pinned} index={idx} - key={`${section.filepath}${idx}`} - title={`${section.range.filepath} (${ + key={`${section.display_name}${idx}`} + title={`${section.display_name} (${ section.range.range.start.line + 1 }-${section.range.range.end.line + 1})`} onDelete={() => { @@ -372,7 +372,7 @@ const ComboBox = React.forwardRef((props: ComboBoxProps, ref) => { > {highlightedCodeSections.map((section, idx) => ( <> -

{section.range.filepath}

+

{section.display_name}

{section.range.contents} diff --git a/extension/schema/FullState.d.ts b/extension/schema/FullState.d.ts index 981e772e..abb0832d 100644 --- a/extension/schema/FullState.d.ts +++ b/extension/schema/FullState.d.ts @@ -32,6 +32,7 @@ export type Character = number; export type Contents = string; export type Editing = boolean; export type Pinned = boolean; +export type DisplayName = string; export type HighlightedRanges = HighlightedRangeContext[]; export type Name3 = string; export type Description1 = string; @@ -102,6 +103,7 @@ export interface HighlightedRangeContext { range: RangeInFileWithContents; editing: Editing; pinned: Pinned; + display_name: DisplayName; [k: string]: unknown; } /** diff --git a/schema/json/FullState.json b/schema/json/FullState.json index af0f25e1..5a7e9d10 100644 --- a/schema/json/FullState.json +++ b/schema/json/FullState.json @@ -222,12 +222,17 @@ "pinned": { "title": "Pinned", "type": "boolean" + }, + "display_name": { + "title": "Display Name", + "type": "string" } }, "required": [ "range", "editing", - "pinned" + "pinned", + "display_name" ] }, "SlashCommandDescription": { -- cgit v1.2.3-70-g09d2 From 20f4d07eb1d584569752e67c754951b7892e3e6b Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sun, 9 Jul 2023 14:31:54 -0700 Subject: edit at cursor --- continuedev/src/continuedev/steps/main.py | 11 +++++++++++ extension/package-lock.json | 4 ++-- extension/package.json | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/continuedev/src/continuedev/steps/main.py b/continuedev/src/continuedev/steps/main.py index 0a20ddd7..4f543022 100644 --- a/continuedev/src/continuedev/steps/main.py +++ b/continuedev/src/continuedev/steps/main.py @@ -245,6 +245,17 @@ class EditHighlightedCodeStep(Step): async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]: range_in_files = sdk.get_code_context(only_editing=True) + + # If nothing highlighted, insert at the cursor if possible + if len(range_in_files) == 0: + highlighted_code = await sdk.ide.getHighlightedCode() + if highlighted_code is not None: + for rif in highlighted_code: + if rif.range.start == rif.range.end: + range_in_files.append( + RangeInFileWithContents.from_range_in_file(rif, "")) + + # If nothing highlighted, edit the first open file if len(range_in_files) == 0: # Get the full contents of all open files files = await sdk.ide.getOpenFiles() diff --git a/extension/package-lock.json b/extension/package-lock.json index 22f8b492..5733c2dd 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.141", + "version": "0.0.143", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.141", + "version": "0.0.143", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index beb675b3..444372f8 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "The open-source coding autopilot", - "version": "0.0.141", + "version": "0.0.143", "publisher": "Continue", "engines": { "vscode": "^1.67.0" -- cgit v1.2.3-70-g09d2 From d566dc90a32db926393e1d10f983c79bc3feb56a Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sun, 9 Jul 2023 16:19:16 -0700 Subject: cache server env in .continue folder --- extension/DEV_README.md | 2 +- extension/package.json | 2 +- extension/react-app/src/tabs/gui.tsx | 2 +- extension/scripts/.gitignore | 5 - extension/scripts/README.md | 5 - extension/scripts/chroma.py | 152 ---------------------- extension/scripts/index.py | 52 -------- extension/scripts/query.py | 63 --------- extension/scripts/replace.py | 17 --- extension/scripts/requirements.txt | 6 - extension/scripts/run_continue_server.py | 4 - extension/scripts/update.py | 185 --------------------------- extension/server/.gitignore | 1 + extension/server/README.md | 3 + extension/server/requirements.txt | 1 + extension/server/run_continue_server.py | 4 + extension/src/activation/environmentSetup.ts | 97 ++++++++++---- 17 files changed, 85 insertions(+), 516 deletions(-) delete mode 100644 extension/scripts/.gitignore delete mode 100644 extension/scripts/README.md delete mode 100644 extension/scripts/chroma.py delete mode 100644 extension/scripts/index.py delete mode 100644 extension/scripts/query.py delete mode 100644 extension/scripts/replace.py delete mode 100644 extension/scripts/requirements.txt delete mode 100644 extension/scripts/run_continue_server.py delete mode 100644 extension/scripts/update.py create mode 100644 extension/server/.gitignore create mode 100644 extension/server/README.md create mode 100644 extension/server/requirements.txt create mode 100644 extension/server/run_continue_server.py diff --git a/extension/DEV_README.md b/extension/DEV_README.md index c247b82c..87ed9334 100644 --- a/extension/DEV_README.md +++ b/extension/DEV_README.md @@ -4,7 +4,7 @@ This is the Continue VS Code Extension. Its primary jobs are 1. Implement the IDE side of the Continue IDE protocol, allowing a Continue server to interact natively in an IDE. This happens in `src/continueIdeClient.ts`. 2. Open the Continue React app in a side panel. The React app's source code lives in the `react-app` directory. The panel is opened by the `continue.openContinueGUI` command, as defined in `src/commands.ts`. -3. Run a Continue server in the background, which connects to both the IDE protocol and the React app. The server is launched in `src/activation/environmentSetup.ts` by calling Python code that lives in `scripts/` (unless extension settings define a server URL other than localhost:65432, in which case the extension will just connect to that). +3. Run a Continue server in the background, which connects to both the IDE protocol and the React app. The server is launched in `src/activation/environmentSetup.ts` by calling Python code that lives in `server/` (unless extension settings define a server URL other than localhost:65432, in which case the extension will just connect to that). 4. Open Continue diff --git a/extension/package.json b/extension/package.json index 444372f8..3b23ff19 100644 --- a/extension/package.json +++ b/extension/package.json @@ -227,7 +227,7 @@ "test": "node ./out/test/runTest.js", "jest": "jest --config ./jest.config.js", "package": "cp ./config/prod_config.json ./config/config.json && mkdir -p ./build && vsce package --out ./build && cp ./config/dev_config.json ./config/config.json", - "full-package": "cd ../continuedev && poetry build && cp ./dist/continuedev-0.1.2-py3-none-any.whl ../extension/scripts/continuedev-0.1.2-py3-none-any.whl && cd ../extension && npm install && npm run typegen && npm run clientgen && cd react-app && npm install && npm run build && cd .. && npm run package", + "full-package": "cd ../continuedev && poetry build && cp ./dist/continuedev-0.1.2-py3-none-any.whl ../extension/server/continuedev-0.1.2-py3-none-any.whl && cd ../extension && npm install && npm run typegen && npm run clientgen && cd react-app && npm install && npm run build && cd .. && npm run package", "install-extension": "code --install-extension ./build/continue-0.0.8.vsix", "uninstall": "code --uninstall-extension .continue", "reinstall": "rm -rf ./build && npm run package && npm run uninstall && npm run install-extension" diff --git a/extension/react-app/src/tabs/gui.tsx b/extension/react-app/src/tabs/gui.tsx index e1ecec9e..646aef50 100644 --- a/extension/react-app/src/tabs/gui.tsx +++ b/extension/react-app/src/tabs/gui.tsx @@ -438,7 +438,7 @@ function GUI(props: GUIProps) { if (!usingFastModel) { // Show the dialog setFeedbackDialogMessage( - "We don't yet support local models, but we're working on it! If privacy is a concern of yours, please use the feedback button in the bottom right to let us know." + "We don't yet support local models, but we're working on it! If privacy is a concern of yours, please write a short note to let us know." ); setShowFeedbackDialog(true); } diff --git a/extension/scripts/.gitignore b/extension/scripts/.gitignore deleted file mode 100644 index fbb3bf9f..00000000 --- a/extension/scripts/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -testdb -env -stdout.txt -.continue_env_installed -**.whl \ No newline at end of file diff --git a/extension/scripts/README.md b/extension/scripts/README.md deleted file mode 100644 index da1ad493..00000000 --- a/extension/scripts/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Scripts - -Whenever we need python to run on the client side, we include a script file at the top level of this folder. All other files that are not to be run directly as a script (utility files) should be in a subfolder of `scripts`. You can call one of these scripts from the VS Code extension using the `runPythonScript` function in `bridge.ts`. - -When the extension is activated (`activate` function in `src/extension.ts`), we call `setupPythonEnv`, which makes the virtual environment and downloads all the necessary requirements as given in `requirements.txt`. With this in mind, be sure to run `pip freeze > requirements.txt` whenever you add a new requirement. diff --git a/extension/scripts/chroma.py b/extension/scripts/chroma.py deleted file mode 100644 index 7425394e..00000000 --- a/extension/scripts/chroma.py +++ /dev/null @@ -1,152 +0,0 @@ -import chromadb -import os -import json -import subprocess - -from typing import List, Tuple - -from chromadb.config import Settings - -client = chromadb.Client(Settings( - chroma_db_impl="duckdb+parquet", - persist_directory="./data/" -)) - -FILE_TYPES_TO_IGNORE = [ - '.pyc', - '.png', - '.jpg', - '.jpeg', - '.gif', - '.svg', - '.ico' -] - -def further_filter(files: List[str], root_dir: str): - """Further filter files before indexing.""" - for file in files: - if file.endswith(tuple(FILE_TYPES_TO_IGNORE)) or file.startswith('.git') or file.startswith('archive'): - continue - yield root_dir + "/" + file - -def get_git_root_dir(path: str): - """Get the root directory of a Git repository.""" - try: - return subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=path).strip().decode() - except subprocess.CalledProcessError: - return None - -def get_git_ignored_files(root_dir: str): - """Get the list of ignored files in a Git repository.""" - try: - output = subprocess.check_output(['git', 'ls-files', '--ignored', '--others', '--exclude-standard'], cwd=root_dir).strip().decode() - return output.split('\n') - except subprocess.CalledProcessError: - return [] - -def get_all_files(root_dir: str): - """Get a list of all files in a directory.""" - for dir_path, _, file_names in os.walk(root_dir): - for file_name in file_names: - yield os.path.join(os.path.relpath(dir_path, root_dir), file_name) - -def get_input_files(root_dir: str): - """Get a list of all files in a Git repository that are not ignored.""" - ignored_files = set(get_git_ignored_files(root_dir)) - all_files = set(get_all_files(root_dir)) - nonignored_files = all_files - ignored_files - return further_filter(nonignored_files, root_dir) - -def get_git_root_dir(cwd: str): - """Get the root directory of a Git repository.""" - result = subprocess.run(['git', 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) - return result.stdout.decode().strip() - -def get_current_branch(cwd: str) -> str: - """Get the current Git branch.""" - try: - return subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd).decode("utf-8").strip() - except: - return "main" - -def get_current_commit(cwd: str) -> str: - try: - return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd).decode("utf-8").strip() - except: - return "NO_COMMITS" - -def get_modified_deleted_files(cwd: str) -> Tuple[List[str], List[str]]: - """Get a list of all files that have been modified since the last commit.""" - branch = get_current_branch(cwd) - current_commit = get_current_commit(cwd) - - with open(f"./data/{branch}.json", 'r') as f: - previous_commit = json.load(f)["commit"] - - modified_deleted_files = subprocess.check_output(["git", "diff", "--name-only", previous_commit, current_commit], cwd=cwd).decode("utf-8").strip() - modified_deleted_files = modified_deleted_files.split("\n") - modified_deleted_files = [f for f in modified_deleted_files if f] - - root = get_git_root_dir(cwd) - deleted_files = [f for f in modified_deleted_files if not os.path.exists(root + "/" + f)] - modified_files = [f for f in modified_deleted_files if os.path.exists(root + "/" + f)] - - return further_filter(modified_files, root), further_filter(deleted_files, root) - -def create_collection(branch: str, cwd: str): - """Create a new collection, returning whether it already existed.""" - try: - collection = client.create_collection(name=branch) - except Exception as e: - print(e) - return - - files = get_input_files(get_git_root_dir(cwd)) - for file in files: - with open(file, 'r') as f: - collection.add(documents=[f.read()], ids=[file]) - print(f"Added {file}") - with open(f"./data/{branch}.json", 'w') as f: - json.dump({"commit": get_current_commit(cwd)}, f) - -def collection_exists(cwd: str): - """Check if a collection exists.""" - branch = get_current_branch(cwd) - return branch in client.list_collections() - -def update_collection(cwd: str): - """Update the collection.""" - branch = get_current_branch(cwd) - - try: - - collection = client.get_collection(branch) - - modified_files, deleted_files = get_modified_deleted_files(cwd) - - for file in deleted_files: - collection.delete(ids=[file]) - print(f"Deleted {file}") - - for file in modified_files: - with open(file, 'r') as f: - collection.update(documents=[f.read()], ids=[file]) - print(f"Updated {file}") - - with open(f"./data/{branch}.json", 'w') as f: - json.dump({"commit": get_current_commit(cwd)}, f) - - except: - - create_collection(branch, cwd) - -def query_collection(query: str, n_results: int, cwd: str): - """Query the collection.""" - branch = get_current_branch(cwd) - try: - collection = client.get_collection(branch) - except: - create_collection(branch, cwd) - collection = client.get_collection(branch) - results = collection.query(query_texts=[query], n_results=n_results) - return results \ No newline at end of file diff --git a/extension/scripts/index.py b/extension/scripts/index.py deleted file mode 100644 index 3afc9131..00000000 --- a/extension/scripts/index.py +++ /dev/null @@ -1,52 +0,0 @@ -import sys -import os -from typing import TextIO -from chroma import update_collection, query_collection, create_collection, collection_exists, get_current_branch -from typer import Typer - -app = Typer() - -class SilenceStdoutContextManager: - saved_stdout: TextIO - - def __enter__(self): - self._original_stdout = sys.stdout - sys.stdout = open(os.devnull, 'w') - - def __exit__(self, exc_type, exc_val, exc_tb): - sys.stdout.close() - sys.stdout = self._original_stdout - -silence = SilenceStdoutContextManager() - -@app.command("exists") -def exists(cwd: str): - with silence: - exists = collection_exists(cwd) - print({"exists": exists}) - -@app.command("create") -def create(cwd: str): - with silence: - branch = get_current_branch(cwd) - create_collection(branch, cwd) - print({"success": True}) - -@app.command("update") -def update(cwd: str): - with silence: - update_collection(cwd) - print({"success": True}) - -@app.command("query") -def query(query: str, n_results: int, cwd: str): - with silence: - resp = query_collection(query, n_results, cwd) - results = [{ - "id": resp["ids"][0][i], - "document": resp["documents"][0][i] - } for i in range(len(resp["ids"][0]))] - print({"results": results}) - -if __name__ == "__main__": - app() \ No newline at end of file diff --git a/extension/scripts/query.py b/extension/scripts/query.py deleted file mode 100644 index f2e44413..00000000 --- a/extension/scripts/query.py +++ /dev/null @@ -1,63 +0,0 @@ -import subprocess -import sys -from gpt_index import GPTSimpleVectorIndex, GPTFaissIndex -import os -from typer import Typer -from enum import Enum -from update import update_codebase_index, create_codebase_index, index_dir_for, get_current_branch -from replace import replace_additional_index - -app = Typer() - -def query_codebase_index(query: str) -> str: - """Query the codebase index.""" - branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip() - path = 'data/{branch}/index.json' - if not os.path.exists(path): - print("No index found for the codebase") - return "" - index = GPTFaissIndex.load_from_disk(path) - return index.query(query) - -def query_additional_index(query: str) -> str: - """Query the additional index.""" - index = GPTSimpleVectorIndex.load_from_disk('data/additional_index.json') - return index.query(query) - -class IndexTypeOption(str, Enum): - codebase = "codebase" - additional = "additional" - -@app.command() -def query(context: IndexTypeOption, query: str): - if context == IndexTypeOption.additional: - response = query_additional_index(query) - elif context == IndexTypeOption.codebase: - response = query_codebase_index(query) - else: - print("Error: unknown context") - print({ "response": response }) - -@app.command() -def check_index_exists(root_path: str): - branch = get_current_branch() - exists = os.path.exists(index_dir_for(branch)) - print({ "exists": exists }) - -@app.command() -def update(): - update_codebase_index() - print("Updated codebase index") - -@app.command() -def create_index(path: str): - create_codebase_index() - print("Created file index") - -@app.command() -def replace_additional_index(info: str): - replace_additional_index() - print("Replaced additional index") - -if __name__ == '__main__': - app() \ No newline at end of file diff --git a/extension/scripts/replace.py b/extension/scripts/replace.py deleted file mode 100644 index 08810243..00000000 --- a/extension/scripts/replace.py +++ /dev/null @@ -1,17 +0,0 @@ -import sys -from gpt_index import GPTSimpleVectorIndex, Document - -def replace_additional_index(info: str): - """Replace the additional index.""" - with open('data/additional_context.txt', 'w') as f: - f.write(info) - documents = [Document(info)] - index = GPTSimpleVectorIndex(documents) - index.save_to_disk('data/additional_index.json') - print("Additional index replaced") - -if __name__ == "__main__": - """python3 replace.py """ - info = sys.argv[1] if len(sys.argv) > 1 else None - if info: - replace_additional_index(info) \ No newline at end of file diff --git a/extension/scripts/requirements.txt b/extension/scripts/requirements.txt deleted file mode 100644 index c51c9d73..00000000 --- a/extension/scripts/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# chromadb==0.3.10 -# pathspec==0.11.0 -# typer==0.7.0 -# pydantic -# pytest -./continuedev-0.1.2-py3-none-any.whl \ No newline at end of file diff --git a/extension/scripts/run_continue_server.py b/extension/scripts/run_continue_server.py deleted file mode 100644 index 089cc54d..00000000 --- a/extension/scripts/run_continue_server.py +++ /dev/null @@ -1,4 +0,0 @@ -from continuedev.server.main import run_server - -if __name__ == "__main__": - run_server() diff --git a/extension/scripts/update.py b/extension/scripts/update.py deleted file mode 100644 index 15ad6ac0..00000000 --- a/extension/scripts/update.py +++ /dev/null @@ -1,185 +0,0 @@ -# import faiss -import json -import os -import subprocess - -from gpt_index.langchain_helpers.text_splitter import TokenTextSplitter -from gpt_index import GPTSimpleVectorIndex, SimpleDirectoryReader, Document, GPTFaissIndex -from typing import List, Generator, Tuple - -FILE_TYPES_TO_IGNORE = [ - '.pyc', - '.png', - '.jpg', - '.jpeg', - '.gif', - '.svg', - '.ico' -] - -def further_filter(files: List[str], root_dir: str): - """Further filter files before indexing.""" - for file in files: - if file.endswith(tuple(FILE_TYPES_TO_IGNORE)) or file.startswith('.git') or file.startswith('archive'): - continue - yield root_dir + "/" + file - -def get_git_root_dir(path: str): - """Get the root directory of a Git repository.""" - try: - return subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=path).strip().decode() - except subprocess.CalledProcessError: - return None - -def get_git_ignored_files(root_dir: str): - """Get the list of ignored files in a Git repository.""" - try: - output = subprocess.check_output(['git', 'ls-files', '--ignored', '--others', '--exclude-standard'], cwd=root_dir).strip().decode() - return output.split('\n') - except subprocess.CalledProcessError: - return [] - -def get_all_files(root_dir: str): - """Get a list of all files in a directory.""" - for dir_path, _, file_names in os.walk(root_dir): - for file_name in file_names: - yield os.path.join(os.path.relpath(dir_path, root_dir), file_name) - -def get_input_files(root_dir: str): - """Get a list of all files in a Git repository that are not ignored.""" - ignored_files = set(get_git_ignored_files(root_dir)) - all_files = set(get_all_files(root_dir)) - nonignored_files = all_files - ignored_files - return further_filter(nonignored_files, root_dir) - -def load_gpt_index_documents(root: str) -> List[Document]: - """Loads a list of GPTIndex Documents, respecting .gitignore files.""" - # Get input files - input_files = get_input_files(root) - # Use SimpleDirectoryReader to load the files into Documents - return SimpleDirectoryReader(root, input_files=input_files, file_metadata=lambda filename: {"filename": filename}).load_data() - -def index_dir_for(branch: str) -> str: - return f"data/{branch}" - -def get_git_root_dir(): - result = subprocess.run(['git', 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return result.stdout.decode().strip() - -def get_current_branch() -> str: - return subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip() - -def get_current_commit() -> str: - return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf-8").strip() - -def create_codebase_index(): - """Create a new index for the current branch.""" - branch = get_current_branch() - if not os.path.exists(index_dir_for(branch)): - os.makedirs(index_dir_for(branch)) - - documents = load_gpt_index_documents(get_git_root_dir()) - - chunks = {} - doc_chunks = [] - for doc in documents: - text_splitter = TokenTextSplitter() - text_chunks = text_splitter.split_text(doc.text) - filename = doc.extra_info["filename"] - chunks[filename] = len(text_chunks) - for i, text in enumerate(text_chunks): - doc_chunks.append(Document(text, doc_id=f"{filename}::{i}")) - - with open(f"{index_dir_for(branch)}/metadata.json", "w") as f: - json.dump({"commit": get_current_commit(), "chunks" : chunks}, f, indent=4) - - index = GPTSimpleVectorIndex([]) - for chunk in doc_chunks: - index.insert(chunk) - - # d = 1536 # Dimension of text-ada-embedding-002 - # faiss_index = faiss.IndexFlatL2(d) - # index = GPTFaissIndex(documents, faiss_index=faiss_index) - # index.save_to_disk(f"{index_dir_for(branch)}/index.json", faiss_index_save_path=f"{index_dir_for(branch)}/index_faiss_core.index") - - index.save_to_disk(f"{index_dir_for(branch)}/index.json") - - print("Codebase index created") - -def get_modified_deleted_files() -> Tuple[List[str], List[str]]: - """Get a list of all files that have been modified since the last commit.""" - branch = get_current_branch() - current_commit = get_current_commit() - - metadata = f"{index_dir_for(branch)}/metadata.json" - with open(metadata, "r") as f: - previous_commit = json.load(f)["commit"] - - modified_deleted_files = subprocess.check_output(["git", "diff", "--name-only", previous_commit, current_commit]).decode("utf-8").strip() - modified_deleted_files = modified_deleted_files.split("\n") - modified_deleted_files = [f for f in modified_deleted_files if f] - - root = get_git_root_dir() - deleted_files = [f for f in modified_deleted_files if not os.path.exists(root + "/" + f)] - modified_files = [f for f in modified_deleted_files if os.path.exists(root + "/" + f)] - - return further_filter(modified_files, index_dir_for(branch)), further_filter(deleted_files, index_dir_for(branch)) - -def update_codebase_index(): - """Update the index with a list of files.""" - branch = get_current_branch() - - if not os.path.exists(index_dir_for(branch)): - create_codebase_index() - else: - # index = GPTFaissIndex.load_from_disk(f"{index_dir_for(branch)}/index.json", faiss_index_save_path=f"{index_dir_for(branch)}/index_faiss_core.index") - index = GPTSimpleVectorIndex.load_from_disk(f"{index_dir_for(branch)}/index.json") - modified_files, deleted_files = get_modified_deleted_files() - - with open(f"{index_dir_for(branch)}/metadata.json", "r") as f: - metadata = json.load(f) - - for file in deleted_files: - - num_chunks = metadata["chunks"][file] - for i in range(num_chunks): - index.delete(f"{file}::{i}") - - del metadata["chunks"][file] - - print(f"Deleted {file}") - - for file in modified_files: - - if file in metadata["chunks"]: - - num_chunks = metadata["chunks"][file] - - for i in range(num_chunks): - index.delete(f"{file}::{i}") - - print(f"Deleted old version of {file}") - - with open(file, "r") as f: - text = f.read() - - text_splitter = TokenTextSplitter() - text_chunks = text_splitter.split_text(text) - - for i, text in enumerate(text_chunks): - index.insert(Document(text, doc_id=f"{file}::{i}")) - - metadata["chunks"][file] = len(text_chunks) - - print(f"Inserted new version of {file}") - - metadata["commit"] = get_current_commit() - - with open(f"{index_dir_for(branch)}/metadata.json", "w") as f: - json.dump(metadata, f, indent=4) - - print("Codebase index updated") - -if __name__ == "__main__": - """python3 update.py""" - update_codebase_index() \ No newline at end of file diff --git a/extension/server/.gitignore b/extension/server/.gitignore new file mode 100644 index 00000000..0b6e11dd --- /dev/null +++ b/extension/server/.gitignore @@ -0,0 +1 @@ +**.whl \ No newline at end of file diff --git a/extension/server/README.md b/extension/server/README.md new file mode 100644 index 00000000..56208b5a --- /dev/null +++ b/extension/server/README.md @@ -0,0 +1,3 @@ +# server + +These are the files that get copied over to `~/.continue/server` in order to create the Python environment and start the Continue server. diff --git a/extension/server/requirements.txt b/extension/server/requirements.txt new file mode 100644 index 00000000..eb1f3738 --- /dev/null +++ b/extension/server/requirements.txt @@ -0,0 +1 @@ +./continuedev-0.1.2-py3-none-any.whl \ No newline at end of file diff --git a/extension/server/run_continue_server.py b/extension/server/run_continue_server.py new file mode 100644 index 00000000..089cc54d --- /dev/null +++ b/extension/server/run_continue_server.py @@ -0,0 +1,4 @@ +from continuedev.server.main import run_server + +if __name__ == "__main__": + run_server() diff --git a/extension/src/activation/environmentSetup.ts b/extension/src/activation/environmentSetup.ts index 714080e3..02118501 100644 --- a/extension/src/activation/environmentSetup.ts +++ b/extension/src/activation/environmentSetup.ts @@ -7,6 +7,7 @@ import * as fs from "fs"; import { getContinueServerUrl } from "../bridge"; import fetch from "node-fetch"; import * as vscode from "vscode"; +import * as os from "os"; import fkill from "fkill"; import { sendTelemetryEvent, TelemetryEvent } from "../telemetry"; @@ -127,8 +128,7 @@ function getActivateUpgradeCommands(pythonCmd: string, pipCmd: string) { function checkEnvExists() { const envBinPath = path.join( - getExtensionUri().fsPath, - "scripts", + serverPath(), "env", process.platform == "win32" ? "Scripts" : "bin" ); @@ -140,10 +140,32 @@ function checkEnvExists() { ); } -function checkRequirementsInstalled() { +async function checkRequirementsInstalled() { + // First, check if the requirements have been installed most recently for a later version of the extension + if (fs.existsSync(requirementsVersionPath())) { + const requirementsVersion = fs.readFileSync( + requirementsVersionPath(), + "utf8" + ); + if (requirementsVersion !== getExtensionVersion()) { + // Remove the old version of continuedev from site-packages + const [pythonCmd, pipCmd] = await getPythonPipCommands(); + const [activateCmd] = getActivateUpgradeCommands(pythonCmd, pipCmd); + const removeOldVersionCommand = [ + `cd "${serverPath()}"`, + activateCmd, + `${pipCmd} uninstall -y continuedev`, + ].join(" ; "); + const [, stderr] = await runCommand(removeOldVersionCommand); + if (stderr) { + throw new Error(stderr); + } + return false; + } + } + let envLibsPath = path.join( - getExtensionUri().fsPath, - "scripts", + serverPath(), "env", process.platform == "win32" ? "Lib" : "lib" ); @@ -165,10 +187,6 @@ function checkRequirementsInstalled() { const continuePath = path.join(envLibsPath, "continuedev"); return fs.existsSync(continuePath); - - // return fs.existsSync( - // path.join(getExtensionUri().fsPath, "scripts", ".continue_env_installed") - // ); } async function setupPythonEnv() { @@ -180,12 +198,13 @@ async function setupPythonEnv() { pipCmd ); + // First, create the virtual environment if (checkEnvExists()) { console.log("Python env already exists, skipping..."); } else { // Assemble the command to create the env const createEnvCommand = [ - `cd "${path.join(getExtensionUri().fsPath, "scripts")}"`, + `cd "${serverPath()}"`, `${pythonCmd} -m venv env`, ].join(" ; "); @@ -216,10 +235,7 @@ async function setupPythonEnv() { console.log(msg); await vscode.window.showErrorMessage(msg); } else if (checkEnvExists()) { - console.log( - "Successfully set up python env at ", - getExtensionUri().fsPath + "/scripts/env" - ); + console.log("Successfully set up python env at ", `${serverPath()}/env`); } else { const msg = [ "Python environment not successfully created. Trying again. Here was the stdout + stderr: ", @@ -231,12 +247,13 @@ async function setupPythonEnv() { } } + // Install the requirements await retryThenFail(async () => { - if (checkRequirementsInstalled()) { + if (await checkRequirementsInstalled()) { console.log("Python requirements already installed, skipping..."); } else { const installRequirementsCommand = [ - `cd "${path.join(getExtensionUri().fsPath, "scripts")}"`, + `cd "${serverPath()}"`, activateCmd, pipUpgradeCmd, `${pipCmd} install -r requirements.txt`, @@ -245,6 +262,8 @@ async function setupPythonEnv() { if (stderr) { throw new Error(stderr); } + // Write the version number for which requirements were installed + fs.writeFileSync(requirementsVersionPath(), getExtensionVersion()); } }); } @@ -297,9 +316,39 @@ async function checkServerRunning(serverUrl: string): Promise { } } +export function getContinueGlobalPath(): string { + // This is ~/.continue on mac/linux + const continuePath = path.join(os.homedir(), ".continue"); + if (!fs.existsSync(continuePath)) { + fs.mkdirSync(continuePath); + } + return continuePath; +} + +function setupServerPath() { + const sPath = serverPath(); + const extensionServerPath = path.join(getExtensionUri().fsPath, "server"); + const files = fs.readdirSync(extensionServerPath); + files.forEach((file) => { + const filePath = path.join(extensionServerPath, file); + fs.copyFileSync(filePath, path.join(sPath, file)); + }); +} + +function serverPath(): string { + const sPath = path.join(getContinueGlobalPath(), "server"); + if (!fs.existsSync(sPath)) { + fs.mkdirSync(sPath); + } + return sPath; +} + function serverVersionPath(): string { - const extensionPath = getExtensionUri().fsPath; - return path.join(extensionPath, "server_version.txt"); + return path.join(serverPath(), "server_version.txt"); +} + +function requirementsVersionPath(): string { + return path.join(serverPath(), "requirements_version.txt"); } function getExtensionVersion() { @@ -314,6 +363,8 @@ export async function startContinuePythonServer() { return; } + setupServerPath(); + return await retryThenFail(async () => { if (await checkServerRunning(serverUrl)) { // Kill the server if it is running an old version @@ -337,16 +388,14 @@ export async function startContinuePythonServer() { // Do this after above check so we don't have to waste time setting up the env await setupPythonEnv(); + // Spawn the server process on port 65432 const [pythonCmd] = await getPythonPipCommands(); const activateCmd = process.platform == "win32" ? ".\\env\\Scripts\\activate" : ". env/bin/activate"; - const command = `cd "${path.join( - getExtensionUri().fsPath, - "scripts" - )}" && ${activateCmd} && cd .. && ${pythonCmd} -m scripts.run_continue_server`; + const command = `cd "${serverPath()}" && ${activateCmd} && cd .. && ${pythonCmd} -m server.run_continue_server`; console.log("Starting Continue python server..."); @@ -392,8 +441,8 @@ export async function startContinuePythonServer() { } export function isPythonEnvSetup(): boolean { - let pathToEnvCfg = getExtensionUri().fsPath + "/scripts/env/pyvenv.cfg"; - return fs.existsSync(path.join(pathToEnvCfg)); + const pathToEnvCfg = path.join(serverPath(), "env", "pyvenv.cfg"); + return fs.existsSync(pathToEnvCfg); } export async function downloadPython3() { -- cgit v1.2.3-70-g09d2 From 13848f99e12f69d9d58c3cb31e9e54e386eb3af0 Mon Sep 17 00:00:00 2001 From: Nate Sesti Date: Sun, 9 Jul 2023 17:43:37 -0700 Subject: say "No edits were made" if true --- continuedev/src/continuedev/steps/core/core.py | 11 +++++++---- extension/package-lock.json | 4 ++-- extension/package.json | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/continuedev/src/continuedev/steps/core/core.py b/continuedev/src/continuedev/steps/core/core.py index 4b35a758..b0d9d719 100644 --- a/continuedev/src/continuedev/steps/core/core.py +++ b/continuedev/src/continuedev/steps/core/core.py @@ -167,10 +167,13 @@ class DefaultModelEditCodeStep(Step): return output async def describe(self, models: Models) -> Coroutine[str, None, None]: - description = await models.gpt3516k.complete(dedent(f"""\ - {self._prompt_and_completion} - - Please give brief a description of the changes made above using markdown bullet points. Be concise and only mention changes made to the commit before, not prefix or suffix:""")) + if self._prompt_and_completion == "": + description = "No edits were made" + else: + description = await models.gpt3516k.complete(dedent(f"""\ + {self._prompt_and_completion} + + Please give brief a description of the changes made above using markdown bullet points. Be concise and only mention changes made to the commit before, not prefix or suffix:""")) name = await models.gpt3516k.complete(f"Write a very short title to describe this requested change (no quotes): '{self.user_input}'. This is the title:") self.name = self._cleanup_output(name) diff --git a/extension/package-lock.json b/extension/package-lock.json index 5733c2dd..a2ac0a04 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.0.143", + "version": "0.0.145", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "continue", - "version": "0.0.143", + "version": "0.0.145", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", diff --git a/extension/package.json b/extension/package.json index 3b23ff19..0464ba55 100644 --- a/extension/package.json +++ b/extension/package.json @@ -14,7 +14,7 @@ "displayName": "Continue", "pricing": "Free", "description": "The open-source coding autopilot", - "version": "0.0.143", + "version": "0.0.145", "publisher": "Continue", "engines": { "vscode": "^1.67.0" -- cgit v1.2.3-70-g09d2 From b9525d5f0851d13b3d4936868f064810f3376dbd Mon Sep 17 00:00:00 2001 From: Ty Dunn Date: Mon, 10 Jul 2023 15:25:29 -0700 Subject: adding onboarding --- extension/react-app/public/edit.gif | Bin 0 -> 41667733 bytes extension/react-app/public/explain.gif | Bin 0 -> 56475028 bytes extension/react-app/public/generate.gif | Bin 0 -> 32532380 bytes extension/react-app/public/intro.gif | Bin 0 -> 3976676 bytes extension/react-app/src/components/Onboarding.tsx | 51 ++++++++++++++++++++++ extension/react-app/src/tabs/gui.tsx | 2 + 6 files changed, 53 insertions(+) create mode 100644 extension/react-app/public/edit.gif create mode 100644 extension/react-app/public/explain.gif create mode 100644 extension/react-app/public/generate.gif create mode 100644 extension/react-app/public/intro.gif create mode 100644 extension/react-app/src/components/Onboarding.tsx diff --git a/extension/react-app/public/edit.gif b/extension/react-app/public/edit.gif new file mode 100644 index 00000000..6780cdf7 Binary files /dev/null and b/extension/react-app/public/edit.gif differ diff --git a/extension/react-app/public/explain.gif b/extension/react-app/public/explain.gif new file mode 100644 index 00000000..e74803dc Binary files /dev/null and b/extension/react-app/public/explain.gif differ diff --git a/extension/react-app/public/generate.gif b/extension/react-app/public/generate.gif new file mode 100644 index 00000000..5c1d112b Binary files /dev/null and b/extension/react-app/public/generate.gif differ diff --git a/extension/react-app/public/intro.gif b/extension/react-app/public/intro.gif new file mode 100644 index 00000000..f872dc91 Binary files /dev/null and b/extension/react-app/public/intro.gif differ diff --git a/extension/react-app/src/components/Onboarding.tsx b/extension/react-app/src/components/Onboarding.tsx new file mode 100644 index 00000000..373f7bea --- /dev/null +++ b/extension/react-app/src/components/Onboarding.tsx @@ -0,0 +1,51 @@ +import { useSelector } from "react-redux"; +import { RootStore } from "../redux/store"; +import React, { useState } from 'react'; + + +const Onboarding = () => { + const [counter, setCounter] = useState(0); + + const handleClick = () => { + setCounter(counter + 1); + } + + const vscMediaUrl = useSelector( + (state: RootStore) => state.config.vscMediaUrl + ); + + return ( +
+ {counter === 0 && ( +
+

Welcome to Continue!

+ Intro +
+ )} + {counter === 1 && ( +
+

Answer coding questions

+ Explain +

Ask Continue about a part of your code to get another perspective

+
+ )} + {counter === 2 && ( +
+

Edit in natural language

+ Edit +

Highlight a section of code and instruct Continue to refactor it

+
+ )} + {counter === 3 && ( +
+

Generate files from scratch

+ Generate +

Let Continue build the scaffolding of Python scripts, React components, and more

+
+ )} +

Click to learn how to use Continue...

+
+ ); +} + +export default Onboarding; diff --git a/extension/react-app/src/tabs/gui.tsx b/extension/react-app/src/tabs/gui.tsx index e1ecec9e..14556829 100644 --- a/extension/react-app/src/tabs/gui.tsx +++ b/extension/react-app/src/tabs/gui.tsx @@ -23,6 +23,7 @@ import { RootStore } from "../redux/store"; import LoadingCover from "../components/LoadingCover"; import { postVscMessage } from "../vscode"; import UserInputContainer from "../components/UserInputContainer"; +import Onboarding from "../components/Onboarding"; const TopGUIDiv = styled.div` overflow: hidden; @@ -267,6 +268,7 @@ function GUI(props: GUIProps) { // const iterations = useSelector(selectIterations); return ( <> +