From 8d59100b3194cc8d122708523226968899efb5e1 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Mon, 29 May 2023 18:31:25 -0400
Subject: (much!) faster inference with starcoder

---
 .../react-app/src/components/StepContainer.tsx      | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

(limited to 'extension/react-app/src/components/StepContainer.tsx')

diff --git a/extension/react-app/src/components/StepContainer.tsx b/extension/react-app/src/components/StepContainer.tsx
index 03649b66..36b3d99a 100644
--- a/extension/react-app/src/components/StepContainer.tsx
+++ b/extension/react-app/src/components/StepContainer.tsx
@@ -144,6 +144,9 @@ function StepContainer(props: StepContainerProps) {
               onSubmit={(ev) => {
                 props.onUserInput(ev.currentTarget.value);
               }}
+              onClick={(e) => {
+                e.stopPropagation();
+              }}
             />
           )}
           {props.historyNode.step.name === "Waiting for user confirmation" && (
@@ -165,24 +168,6 @@ function StepContainer(props: StepContainerProps) {
               />
             </>
           )}
-
-          {open && (
-            <>
-              {/* {props.historyNode.observation && (
-                <SubContainer title="Error">
-                  <CodeBlock>Error Here</CodeBlock>
-                </SubContainer>
-              )} */}
-              {/* {props.iterationContext.suggestedChanges.map((sc) => {
-              return (
-                <SubContainer title="Suggested Change">
-                  {sc.filepath}
-                  <CodeBlock>{sc.replacement}</CodeBlock>
-                </SubContainer>
-              );
-            })} */}
-            </>
-          )}
         </StepContainerDiv>
       </GradientBorder>
 
-- 
cgit v1.2.3-70-g09d2


From 4d4d82084bdd45039eb72ef9c6c4582cd770b4d5 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Thu, 1 Jun 2023 19:03:10 -0400
Subject: polishing dlt recipe

---
 continuedev/src/continuedev/core/agent.py          | 13 ++++++++----
 .../src/continuedev/libs/steps/core/core.py        |  3 ++-
 .../src/continuedev/libs/steps/draft/dlt.py        | 24 ++++++++++++++++++++--
 continuedev/src/continuedev/libs/steps/main.py     |  1 +
 .../react-app/src/components/StepContainer.tsx     |  2 ++
 extension/react-app/src/components/index.ts        |  2 +-
 extension/src/activation/environmentSetup.ts       |  6 +++---
 7 files changed, 40 insertions(+), 11 deletions(-)

(limited to 'extension/react-app/src/components/StepContainer.tsx')

diff --git a/continuedev/src/continuedev/core/agent.py b/continuedev/src/continuedev/core/agent.py
index 0dba6122..cf5c9781 100644
--- a/continuedev/src/continuedev/core/agent.py
+++ b/continuedev/src/continuedev/core/agent.py
@@ -10,6 +10,7 @@ from ..models.main import ContinueBaseModel
 from .main import Policy, History, FullState, Step, HistoryNode
 from ..libs.steps.core.core import ReversibleStep, ManualEditStep, UserInputStep
 from .sdk import ContinueSDK
+import asyncio
 
 
 class Agent(ContinueBaseModel):
@@ -88,6 +89,9 @@ class Agent(ContinueBaseModel):
         self.history.add_node(HistoryNode(
             step=step, observation=None, depth=self._step_depth))
 
+        # Call all subscribed callbacks
+        await self.update_subscribers()
+
         # Run step
         self._step_depth += 1
         observation = await step(ContinueSDK(self))
@@ -98,10 +102,11 @@ class Agent(ContinueBaseModel):
             self._step_depth, include_current=True).observation = observation
 
         # Update its description
-        step._set_description(await step.describe(ContinueSDK(self).models))
-
-        # Call all subscribed callbacks
-        await self.update_subscribers()
+        async def update_description():
+            step._set_description(await step.describe(ContinueSDK(self).models))
+            # Update subscribers with new description
+            await self.update_subscribers()
+        asyncio.create_task(update_description())
 
         return observation
 
diff --git a/continuedev/src/continuedev/libs/steps/core/core.py b/continuedev/src/continuedev/libs/steps/core/core.py
index 1761cbfd..9a5d54f0 100644
--- a/continuedev/src/continuedev/libs/steps/core/core.py
+++ b/continuedev/src/continuedev/libs/steps/core/core.py
@@ -44,7 +44,8 @@ class ShellCommandsStep(Step):
     name: str = "Run Shell Commands"
 
     async def describe(self, models: Models) -> Coroutine[str, None, None]:
-        return "\n".join(self.cmds)
+        cmds_str = "\n".join(self.cmds)
+        return (await models.gpt35()).complete(f"{cmds_str}\n\nSummarize what was done in these shell commands, using markdown bullet points:")
 
     async def run(self, sdk: ContinueSDK) -> Coroutine[Observation, None, None]:
         cwd = await sdk.ide.getWorkspaceDirectory() if self.cwd is None else self.cwd
diff --git a/continuedev/src/continuedev/libs/steps/draft/dlt.py b/continuedev/src/continuedev/libs/steps/draft/dlt.py
index 94ca2323..73762327 100644
--- a/continuedev/src/continuedev/libs/steps/draft/dlt.py
+++ b/continuedev/src/continuedev/libs/steps/draft/dlt.py
@@ -1,16 +1,27 @@
 from textwrap import dedent
 
+from ....core.sdk import Models
+
 from ....core.observation import DictObservation
 from ....models.filesystem_edit import AddFile
 from ....core.main import Step
 from ....core.sdk import ContinueSDK
 from ..core.core import WaitForUserInputStep
+from ..main import MessageStep
 
 
 class SetupPipelineStep(Step):
+    hide: bool = True
+    name: str = "Setup dlt Pipeline"
 
     api_description: str  # e.g. "I want to load data from the weatherapi.com API"
 
+    async def describe(self, models: Models):
+        return dedent(f"""\
+        This step will create a new dlt pipeline that loads data from an API, as per your request:
+        {self.api_description}
+        """)
+
     async def run(self, sdk: ContinueSDK):
         source_name = (await sdk.models.gpt35()).complete(
             f"Write a snake_case name for the data source described by {self.api_description}: ").strip()
@@ -34,12 +45,11 @@ class SetupPipelineStep(Step):
 
         # wait for user to put API key in secrets.toml
         await sdk.ide.setFileOpen(await sdk.ide.getWorkspaceDirectory() + "/.dlt/secrets.toml")
-        await sdk.wait_for_user_confirmation("Please add the API key to the `secrets.toml` file and then press `Continue`")
+        await sdk.wait_for_user_confirmation("If this service requires an API key, please add it to the `secrets.toml` file and then press `Continue`")
         return DictObservation(values={"source_name": source_name})
 
 
 class ValidatePipelineStep(Step):
-
     async def run(self, sdk: ContinueSDK):
         source_name = sdk.history.last_observation().values["source_name"]
         filename = f'{source_name}.py'
@@ -77,9 +87,19 @@ class ValidatePipelineStep(Step):
 
 
 class CreatePipelineStep(Step):
+    hide: bool = True
 
     async def run(self, sdk: ContinueSDK):
         await sdk.run_step(
+            MessageStep(message=dedent("""\
+                This recipe will walk you through the process of creating a dlt pipeline for your chosen data source. With the help of Continue, you will:
+                - Create a Python virtual environment with dlt installed
+                - Run `dlt init` to generate a pipeline template
+                - Write the code to call the API
+                - Add any required API keys to the `secrets.toml` file
+                - Test that the API call works
+                - Load the data into a local DuckDB instance
+                - Write a query to view the data""")) >>
             WaitForUserInputStep(prompt="What API do you want to load data from?") >>
             SetupPipelineStep(api_description="WeatherAPI.com API") >>
             ValidatePipelineStep()
diff --git a/continuedev/src/continuedev/libs/steps/main.py b/continuedev/src/continuedev/libs/steps/main.py
index 6a7f14c7..c70d5c2c 100644
--- a/continuedev/src/continuedev/libs/steps/main.py
+++ b/continuedev/src/continuedev/libs/steps/main.py
@@ -334,6 +334,7 @@ class SolveTracebackStep(Step):
 
 
 class MessageStep(Step):
+    name: str = "Message"
     message: str
 
     async def describe(self, models: Models) -> Coroutine[str, None, None]:
diff --git a/extension/react-app/src/components/StepContainer.tsx b/extension/react-app/src/components/StepContainer.tsx
index 36b3d99a..ec601ea7 100644
--- a/extension/react-app/src/components/StepContainer.tsx
+++ b/extension/react-app/src/components/StepContainer.tsx
@@ -36,6 +36,8 @@ const MainDiv = styled.div<{ stepDepth: number; inFuture: boolean }>`
   animation: ${appear} 0.3s ease-in-out;
   /* padding-left: ${(props) => props.stepDepth * 20}px; */
   overflow: hidden;
+  margin-left: 0px;
+  margin-right: 0px;
 `;
 
 const StepContainerDiv = styled.div<{ open: boolean }>`
diff --git a/extension/react-app/src/components/index.ts b/extension/react-app/src/components/index.ts
index e37c97f3..7ba60467 100644
--- a/extension/react-app/src/components/index.ts
+++ b/extension/react-app/src/components/index.ts
@@ -45,7 +45,7 @@ export const Pre = styled.pre`
   border-radius: ${defaultBorderRadius};
   padding: 8px;
   max-height: 150px;
-  overflow: scroll;
+  overflow-y: scroll;
   margin: 0;
   background-color: ${secondaryDark};
   border: none;
diff --git a/extension/src/activation/environmentSetup.ts b/extension/src/activation/environmentSetup.ts
index 419d32ed..170426e1 100644
--- a/extension/src/activation/environmentSetup.ts
+++ b/extension/src/activation/environmentSetup.ts
@@ -66,11 +66,11 @@ function checkEnvExists() {
     getExtensionUri().fsPath,
     "scripts",
     "env",
-    "bin"
+    process.platform == "win32" ? "Scripts" : "bin"
   );
   return (
-    fs.existsSync(envBinPath + "/activate") &&
-    fs.existsSync(envBinPath + "/pip")
+    fs.existsSync(path.join(envBinPath, "activate")) &&
+    fs.existsSync(path.join(envBinPath, "pip"))
   );
 }
 
-- 
cgit v1.2.3-70-g09d2


From 75af2ad952e1137ca3e1d52f92330468bb3f0281 Mon Sep 17 00:00:00 2001
From: Nate Sesti <sestinj@gmail.com>
Date: Thu, 1 Jun 2023 20:59:10 -0400
Subject: enter secrets directly into .env file

---
 continuedev/src/continuedev/core/env.py            | 10 ++++--
 continuedev/src/continuedev/core/sdk.py            | 37 +++++++++++++++-------
 .../src/continuedev/models/filesystem_edit.py      |  4 +++
 continuedev/src/continuedev/models/main.py         |  8 +++++
 .../react-app/src/components/StepContainer.tsx     |  8 +++++
 5 files changed, 53 insertions(+), 14 deletions(-)

(limited to 'extension/react-app/src/components/StepContainer.tsx')

diff --git a/continuedev/src/continuedev/core/env.py b/continuedev/src/continuedev/core/env.py
index edd3297c..2692c348 100644
--- a/continuedev/src/continuedev/core/env.py
+++ b/continuedev/src/continuedev/core/env.py
@@ -7,11 +7,15 @@ def get_env_var(var_name: str):
     return os.getenv(var_name)
 
 
-def save_env_var(var_name: str, var_value: str):
+def make_sure_env_exists():
     if not os.path.exists('.env'):
         with open('.env', 'w') as f:
-            f.write(f'{var_name}="{var_value}"\n')
-        return
+            f.write('')
+
+
+def save_env_var(var_name: str, var_value: str):
+    make_sure_env_exists()
+
     with open('.env', 'r') as f:
         lines = f.readlines()
     with open('.env', 'w') as f:
diff --git a/continuedev/src/continuedev/core/sdk.py b/continuedev/src/continuedev/core/sdk.py
index b9b20422..ce0c53fd 100644
--- a/continuedev/src/continuedev/core/sdk.py
+++ b/continuedev/src/continuedev/core/sdk.py
@@ -2,7 +2,7 @@ import os
 from typing import Coroutine, Union
 
 from .config import ContinueConfig, load_config
-from ..models.filesystem_edit import FileSystemEdit, AddFile, DeleteFile, AddDirectory, DeleteDirectory
+from ..models.filesystem_edit import FileEdit, FileSystemEdit, AddFile, DeleteFile, AddDirectory, DeleteDirectory
 from ..models.filesystem import RangeInFile
 from ..libs.llm import LLM
 from ..libs.llm.hf_inference_api import HuggingFaceInferenceAPI
@@ -11,7 +11,7 @@ from .observation import Observation
 from ..server.ide_protocol import AbstractIdeProtocolServer
 from .main import History, Step
 from ..libs.steps.core.core import *
-from .env import get_env_var, save_env_var
+from .env import get_env_var, make_sure_env_exists
 
 
 class Agent:
@@ -29,12 +29,12 @@ class Models:
 
     async def starcoder(self):
         api_key = await self.sdk.get_user_secret(
-            'HUGGING_FACE_TOKEN', 'Please enter your Hugging Face token')
+            'HUGGING_FACE_TOKEN', 'Please add your Hugging Face token to the .env file')
         return HuggingFaceInferenceAPI(api_key=api_key)
 
     async def gpt35(self):
         api_key = await self.sdk.get_user_secret(
-            'OPENAI_API_KEY', 'Please enter your OpenAI API key')
+            'OPENAI_API_KEY', 'Please add your OpenAI API key to the .env file')
         return OpenAI(api_key=api_key, default_model="gpt-3.5-turbo")
 
 
@@ -86,6 +86,12 @@ class ContinueSDK:
             prompt=f'Here is the code before:\n\n{{code}}\n\nHere is the user request:\n\n{prompt}\n\nHere is the code edited to perfectly solve the user request:\n\n'
         ))
 
+    async def append_to_file(self, filename: str, content: str):
+        filepath = await self._ensure_absolute_path(filename)
+        previous_content = await self.ide.readFile(filepath)
+        file_edit = FileEdit.from_append(filepath, previous_content, content)
+        await self.ide.applyFileSystemEdit(file_edit)
+
     async def add_file(self, filename: str, content: str | None):
         return await self.run_step(FileSystemEditStep(edit=AddFile(filename=filename, content=content)))
 
@@ -99,14 +105,23 @@ class ContinueSDK:
         return await self.run_step(FileSystemEditStep(edit=DeleteDirectory(path=path)))
 
     async def get_user_secret(self, env_var: str, prompt: str) -> str:
-        try:
+        make_sure_env_exists()
+
+        val = None
+        while val is None:
+            try:
+                val = get_env_var(env_var)
+                if val is not None:
+                    return val
+            except:
+                pass
+            server_dir = os.getcwd()
+            env_path = os.path.join(server_dir, ".env")
+            await self.ide.setFileOpen(env_path)
+            await self.append_to_file(env_path, f'\n{env_var}="<ENTER SECRET HERE>"')
+            await self.run_step(WaitForUserConfirmationStep(prompt=prompt))
             val = get_env_var(env_var)
-            if val is not None:
-                return val
-        except:
-            pass
-        val = (await self.run_step(WaitForUserInputStep(prompt=prompt))).text
-        save_env_var(env_var, val)
+
         return val
 
     async def get_config(self) -> ContinueConfig:
diff --git a/continuedev/src/continuedev/models/filesystem_edit.py b/continuedev/src/continuedev/models/filesystem_edit.py
index 7526d4c9..8e74b819 100644
--- a/continuedev/src/continuedev/models/filesystem_edit.py
+++ b/continuedev/src/continuedev/models/filesystem_edit.py
@@ -37,6 +37,10 @@ class FileEdit(AtomicFileSystemEdit):
     def from_insertion(filepath: str, position: Position, content: str) -> "FileEdit":
         return FileEdit(filepath=filepath, range=Range.from_shorthand(position.line, position.character, position.line, position.character), replacement=content)
 
+    @staticmethod
+    def from_append(filepath: str, previous_content: str, appended_content: str) -> "FileEdit":
+        return FileEdit(filepath=filepath, range=Range.from_position(Position.from_end_of_file(previous_content)), replacement=appended_content)
+
 
 class FileEditWithFullContents(BaseModel):
     fileEdit: FileEdit
diff --git a/continuedev/src/continuedev/models/main.py b/continuedev/src/continuedev/models/main.py
index 1bc51ff1..02c44aae 100644
--- a/continuedev/src/continuedev/models/main.py
+++ b/continuedev/src/continuedev/models/main.py
@@ -39,6 +39,10 @@ class Position(BaseModel):
 
         return Position(line=line, character=character)
 
+    @staticmethod
+    def from_end_of_file(contents: str) -> "Position":
+        return Position.from_index(contents, len(contents))
+
 
 class Range(BaseModel):
     """A range in a file. 0-indexed."""
@@ -117,6 +121,10 @@ class Range(BaseModel):
 
         return Range.from_shorthand(start_line, 0, end_line, len(content_lines[end_line]) - 1)
 
+    @staticmethod
+    def from_position(position: Position) -> "Range":
+        return Range(start=position, end=position)
+
 
 class AbstractModel(ABC, BaseModel):
     @root_validator(pre=True)
diff --git a/extension/react-app/src/components/StepContainer.tsx b/extension/react-app/src/components/StepContainer.tsx
index ec601ea7..5e979b34 100644
--- a/extension/react-app/src/components/StepContainer.tsx
+++ b/extension/react-app/src/components/StepContainer.tsx
@@ -80,6 +80,13 @@ function StepContainer(props: StepContainerProps) {
   const [open, setOpen] = useState(false);
   const [isHovered, setIsHovered] = useState(false);
   const naturalLanguageInputRef = useRef<HTMLTextAreaElement>(null);
+  const userInputRef = useRef<HTMLInputElement>(null);
+
+  useEffect(() => {
+    if (userInputRef?.current) {
+      userInputRef.current.focus();
+    }
+  }, [userInputRef]);
 
   useEffect(() => {
     if (isHovered) {
@@ -136,6 +143,7 @@ function StepContainer(props: StepContainerProps) {
 
           {props.historyNode.step.name === "Waiting for user input" && (
             <input
+              ref={userInputRef}
               className="m-auto p-2 rounded-md border-1 border-solid text-white w-3/4 border-gray-200 bg-vsc-background"
               onKeyDown={(e) => {
                 if (e.key === "Enter") {
-- 
cgit v1.2.3-70-g09d2

