Merge branch 'main' into docs

1 files changed, 182 insertions, 169 deletions

diff --git a/continuedev/src/continuedev/libs/steps/main.py b/continuedev/src/continuedev/libs/steps/main.py
index 70953e95..73ad3352 100644
--- a/continuedev/src/continuedev/libs/steps/main.py
+++ b/continuedev/src/continuedev/libs/steps/main.py
@@ -1,18 +1,46 @@
-import time
-from typing import Callable, Coroutine, List, Union
+from typing import Coroutine, List, Union
 
+from pydantic import BaseModel
+
+from ..util.traceback_parsers import parse_python_traceback
 from ..llm import LLM
 from ...models.main import Traceback, Range
 from ...models.filesystem_edit import EditDiff, FileEdit
 from ...models.filesystem import RangeInFile, RangeInFileWithContents
-from ..observation import Observation, TextObservation
+from ...core.observation import Observation, TextObservation, TracebackObservation
 from ..llm.prompt_utils import MarkdownStyleEncoderDecoder
 from textwrap import dedent
-from ..core import History, Policy, Step, ContinueSDK, Observation
+from ...core.main import Step
+from ...core.sdk import ContinueSDK, Models
+from ...core.observation import Observation
 import subprocess
-from ..util.traceback_parsers import parse_python_traceback
-from ..observation import TracebackObservation
-import json
+from .core.core import EditCodeStep
+
+
+class RunCodeStep(Step):
+    cmd: str
+
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
+        return f"Ran command: `{self.cmd}`"
+
+    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
+        result = subprocess.run(
+            self.cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout = result.stdout.decode("utf-8")
+        stderr = result.stderr.decode("utf-8")
+        print(stdout, stderr)
+
+        # If it fails, return the error
+        tb = parse_python_traceback(stdout) or parse_python_traceback(stderr)
+        if tb:
+            return TracebackObservation(traceback=tb)
+        else:
+            self.hide = True
+            return None
+
+
+class Policy(BaseModel):
+    pass
 
 
 class RunPolicyUntilDoneStep(Step):
@@ -31,7 +59,7 @@ class RunCommandStep(Step):
     name: str = "Run command"
     _description: str = None
 
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
         if self._description is not None:
             return self._description
         return self.cmd
@@ -51,167 +79,53 @@ class RunCommandStep(Step):
             return TextObservation(text=stdout)
 
 
-class WaitForUserInputStep(Step):
-    prompt: str
-    name: str = "Waiting for user input"
-
-    _description: Union[str, None] = None
-
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
-        return self.prompt
-
-    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
-        self._description = self.prompt
-        resp = await sdk.wait_for_user_input()
-        return TextObservation(text=resp)
-
-
-class WaitForUserConfirmationStep(Step):
-    prompt: str
-    name: str = "Waiting for user confirmation"
-
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
-        return self.prompt
-
-    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
-        self._description = self.prompt
-        resp = await sdk.wait_for_user_input()
-        return TextObservation(text=resp)
-
-
-class RunCodeStep(Step):
-    cmd: str
-
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
-        return f"Ran command: `{self.cmd}`"
-
-    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
-        result = subprocess.run(
-            self.cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout = result.stdout.decode("utf-8")
-        stderr = result.stderr.decode("utf-8")
-        print(stdout, stderr)
-
-        # If it fails, return the error
-        tb = parse_python_traceback(stdout) or parse_python_traceback(stderr)
-        if tb:
-            return TracebackObservation(traceback=tb)
-        else:
-            self.hide = True
-            return None
-
-
-class EditCodeStep(Step):
-    # Might make an even more specific atomic step, which is "apply file edit"
-    range_in_files: List[RangeInFile]
-    prompt: str  # String with {code} somewhere
-    name: str = "Edit code"
-
-    _edit_diffs: Union[List[EditDiff], None] = None
-    _prompt: Union[str, None] = None
-    _completion: Union[str, None] = None
-
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
-        if self._edit_diffs is None:
-            return "Editing files: " + ", ".join(map(lambda rif: rif.filepath, self.range_in_files))
-        elif len(self._edit_diffs) == 0:
-            return "No edits made"
-        else:
-            return llm.complete(dedent(f"""{self._prompt}{self._completion}
-
-                Maximally concise summary of changes in bullet points (can use markdown):
-            """))
-
-    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
-        rif_with_contents = []
-        for range_in_file in self.range_in_files:
-            file_contents = await sdk.ide.readRangeInFile(range_in_file)
-            rif_with_contents.append(
-                RangeInFileWithContents.from_range_in_file(range_in_file, file_contents))
-        enc_dec = MarkdownStyleEncoderDecoder(rif_with_contents)
-        code_string = enc_dec.encode()
-        prompt = self.prompt.format(code=code_string)
-
-        completion = sdk.llm.complete(prompt)
-
-        # Temporarily doing this to generate description.
-        self._prompt = prompt
-        self._completion = completion
-
-        file_edits = enc_dec.decode(completion)
-
-        self._edit_diffs = []
-        for file_edit in file_edits:
-            diff = await sdk.apply_filesystem_edit(file_edit)
-            self._edit_diffs.append(diff)
-
-        for filepath in set([file_edit.filepath for file_edit in file_edits]):
-            await sdk.ide.saveFile(filepath)
-            await sdk.ide.setFileOpen(filepath)
-
-        return None
-
-
-class EditFileStep(Step):
-    filepath: str
-    prompt: str
-    hide: bool = True
-
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
-        return "Editing file: " + self.filepath
-
-    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
-        file_contents = await sdk.ide.readFile(self.filepath)
-        await sdk.run_step(EditCodeStep(
-            range_in_files=[RangeInFile.from_entire_file(
-                self.filepath, file_contents)],
-            prompt=self.prompt
-        ))
-
-
 class FasterEditHighlightedCodeStep(Step):
     user_input: str
     hide = True
     _completion: str = "Edit Code"
     _edit_diffs: Union[List[EditDiff], None] = None
-    _prompt: str = dedent("""Below is the code before changes:
-
-{code}
-
-This is the user request:
-
-{user_input}
-
-Edit the code to perfectly satifsfy the user request. Format the changes you want to make as a comma-separated array of JSON objects of the form:
-{{
-    "edits": [{{
-        "filepath": <FILEPATH>,
-        "replace_me": <CODE_TO_REPLACE>,
-        "replace_with": <CODE_TO_REPLACE_WITH>
-    }}]
-}}
-
-For example, if you want to replace the code `x = 1` with `x = 2` in main.py, you would write:
-{{
-    "edits": [{{
-        "filepath": "main.py",
-        "replace_me": "x = 1",
-        "replace_with": "x = 2"
-    }}]
-}}
-If you wanted to delete the code `def sum(a, b):\\n    return a + b` in main.py, you would write:
-{{
-    "edits": [{{
-        "filepath": "main.py",
-        "replace_me": "def sum(a, b):\\n    return a + b",
-        "replace_with": ""
-    }}]
-}}
-
-Respond with only as many edits as needed, and output only the list of json objects, no other text.
+    _prompt: str = dedent("""\
+        You will be given code to edit in order to perfectly satisfy the user request. All the changes you make must be described as replacements, which you should format in the following way:
+        FILEPATH
+        <FILE_TO_EDIT>
+        REPLACE_ME
+        <CODE_TO_REPLACE>
+        REPLACE_WITH
+        <CODE_TO_REPLACE_WITH>
+
+        where <CODE_TO_REPLACE> and <CODE_TO_REPLACE_WITH> can be multiple lines, but should be the mininum needed to make the edit. Be sure to maintain existing whitespace at the start of lines.
+
+        For example, if you want to replace the code `x = 1` with `x = 2` in main.py, you would write:
+        FILEPATH
+        main.py
+        REPLACE_ME
+        x = 1
+        REPLACE_WITH
+        x = 2
+        If you wanted to delete the code
+        ```
+        def sum(a, b):
+            return a + b
+        ```
+        in main.py, you would write:
+        FILEPATH
+        main.py
+        REPLACE_ME
+        def sum(a, b):
+            return a + b
+        REPLACE_WITH
+
+        You may need to make multiple edits; respond with exactly as many as needed.
+
+        Below is the code before changes:
+
+        {code}
+
+        This is the user request: "{user_input}"
+        Here is the description of changes to make:
 """)
 
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
         return "Editing highlighted code"
 
     async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
@@ -240,21 +154,51 @@ Respond with only as many edits as needed, and output only the list of json obje
         for rif in rif_with_contents:
             rif_dict[rif.filepath] = rif.contents
 
-        completion = sdk.llm.complete(prompt)
+        completion = (await sdk.models.gpt35()).complete(prompt)
 
         # Temporarily doing this to generate description.
         self._prompt = prompt
         self._completion = completion
+        print(completion)
 
         # ALTERNATIVE DECODING STEP HERE
+        raw_file_edits = []
+        lines = completion.split("\n")
+        current_edit = {}
+        status = "FILEPATH"
+        for i in range(0, len(lines)):
+            line = lines[i]
+            if line == "FILEPATH":
+                if "FILEPATH" in current_edit:
+                    raw_file_edits.append(current_edit)
+                current_edit = {}
+                status = "FILEPATH"
+            elif line == "REPLACE_ME":
+                status = "REPLACE_ME"
+            elif line == "REPLACE_WITH":
+                status = "REPLACE_WITH"
+            elif status == "FILEPATH":
+                current_edit["filepath"] = line
+            elif status == "REPLACE_ME":
+                if "replace_me" in current_edit:
+                    current_edit["replace_me"] += "\n" + line
+                else:
+                    current_edit["replace_me"] = line
+            elif status == "REPLACE_WITH":
+                if "replace_with" in current_edit:
+                    current_edit["replace_with"] += "\n" + line
+                else:
+                    current_edit["replace_with"] = line
+        if "filepath" in current_edit:
+            raw_file_edits.append(current_edit)
+
         file_edits = []
-        obj = json.loads(completion.strip())
-        for edit in obj["edits"]:
+        for edit in raw_file_edits:
             filepath = edit["filepath"]
             replace_me = edit["replace_me"]
             replace_with = edit["replace_with"]
             file_edits.append(
-                FileEdit(filepath=filepath, range=Range.from_snippet_in_file(content=rif_dict[filepath], snippet=replace_me), replacement=replace_with))
+                FileEdit(filepath=filepath, range=Range.from_lines_snippet_in_file(content=rif_dict[filepath], snippet=replace_me), replacement=replace_with))
         # ------------------------------
 
         self._edit_diffs = []
@@ -269,6 +213,54 @@ Respond with only as many edits as needed, and output only the list of json obje
         return None
 
 
+class StarCoderEditHighlightedCodeStep(Step):
+    user_input: str
+    hide = False
+    _prompt: str = "<commit_before>{code}<commit_msg>{user_request}<commit_after>"
+
+    _prompt_and_completion: str = ""
+
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
+        return (await models.gpt35()).complete(f"{self._prompt_and_completion}\n\nPlease give brief a description of the changes made above using markdown bullet points:")
+
+    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
+        range_in_files = await sdk.ide.getHighlightedCode()
+        if len(range_in_files) == 0:
+            # Get the full contents of all open files
+            files = await sdk.ide.getOpenFiles()
+            contents = {}
+            for file in files:
+                contents[file] = await sdk.ide.readFile(file)
+
+            range_in_files = [RangeInFile.from_entire_file(
+                filepath, content) for filepath, content in contents.items()]
+
+        rif_with_contents = []
+        for range_in_file in range_in_files:
+            file_contents = await sdk.ide.readRangeInFile(range_in_file)
+            rif_with_contents.append(
+                RangeInFileWithContents.from_range_in_file(range_in_file, file_contents))
+
+        rif_dict = {}
+        for rif in rif_with_contents:
+            rif_dict[rif.filepath] = rif.contents
+
+        for rif in rif_with_contents:
+            prompt = self._prompt.format(
+                code=rif.contents, user_request=self.user_input)
+            completion = str((await sdk.models.starcoder()).complete(prompt))
+            eot_token = "<|endoftext|>"
+            if completion.endswith(eot_token):
+                completion = completion[:completion.rindex(eot_token)]
+
+            self._prompt_and_completion += prompt + completion
+
+            await sdk.ide.applyFileSystemEdit(
+                FileEdit(filepath=rif.filepath, range=rif.range, replacement=completion))
+            await sdk.ide.saveFile(rif.filepath)
+            await sdk.ide.setFileOpen(rif.filepath)
+
+
 class EditHighlightedCodeStep(Step):
     user_input: str
     hide = True
@@ -283,7 +275,7 @@ This is the user request:
 This is the code after being changed to perfectly satisfy the user request:
     """)
 
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
         return "Editing highlighted code"
 
     async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
@@ -305,7 +297,7 @@ This is the code after being changed to perfectly satisfy the user request:
 class FindCodeStep(Step):
     prompt: str
 
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
         return "Finding code"
 
     async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
@@ -319,7 +311,7 @@ class UserInputStep(Step):
 class SolveTracebackStep(Step):
     traceback: Traceback
 
-    async def describe(self, llm: LLM) -> Coroutine[str, None, None]:
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
         return f"```\n{self.traceback.full_traceback}\n```"
 
     async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
@@ -343,3 +335,24 @@ class SolveTracebackStep(Step):
         await sdk.run_step(EditCodeStep(
             range_in_files=range_in_files, prompt=prompt))
         return None
+
+
+class MessageStep(Step):
+    name: str = "Message"
+    message: str
+
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
+        return self.message
+
+    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
+        return TextObservation(text=self.message)
+
+
+class EmptyStep(Step):
+    hide: bool = True
+
+    async def describe(self, models: Models) -> Coroutine[str, None, None]:
+        return ""
+
+    async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
+        pass