Preview (#541)

* Strong typing (#533)

* refactor: :recycle: get rid of continuedev.src.continuedev structure

* refactor: :recycle: switching back to server folder

* feat: :sparkles: make config.py imports shorter

* feat: :bookmark: publish as pre-release vscode extension

* refactor: :recycle: refactor and add more completion params to ui

* build: :building_construction: download from preview S3

* fix: :bug: fix paths

* fix: :green_heart: package:pre-release

* ci: :green_heart: more time for tests

* fix: :green_heart: fix build scripts

* fix: :bug: fix import in run.py

* fix: :bookmark: update version to try again

* ci: 💚 Update package.json version [skip ci]

* refactor: :fire: don't check for old extensions version

* fix: :bug: small bug fixes

* fix: :bug: fix config.py import paths

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: platform-specific builds test #1

* feat: :green_heart: ship with binary

* fix: :green_heart: fix copy statement to include.exe for windows

* fix: :green_heart: cd extension before packaging

* chore: :loud_sound: count tokens generated

* fix: :green_heart: remove npm_config_arch

* fix: :green_heart: publish as pre-release!

* chore: :bookmark: update version

* perf: :green_heart: hardcode distro paths

* fix: :bug: fix yaml syntax error

* chore: :bookmark: update version

* fix: :green_heart: update permissions and version

* feat: :bug: kill old server if needed

* feat: :lipstick: update marketplace icon for pre-release

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: auto-reload for config.py

* feat: :wrench: update default config.py imports

* feat: :sparkles: codelens in config.py

* feat: :sparkles: select model param count from UI

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: more model options, ollama error handling

* perf: :zap: don't show server loading immediately

* fix: :bug: fixing small UI details

* ci: 💚 Update package.json version [skip ci]

* feat: :rocket: headers param on LLM class

* fix: :bug: fix headers for openai.;y

* feat: :sparkles: highlight code on cmd+shift+L

* ci: 💚 Update package.json version [skip ci]

* feat: :lipstick: sticky top bar in gui.tsx

* fix: :loud_sound: websocket logging and horizontal scrollbar

* ci: 💚 Update package.json version [skip ci]

* feat: :sparkles: allow AzureOpenAI Service through GGML

* ci: 💚 Update package.json version [skip ci]

* fix: :bug: fix automigration

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: upload binaries in ci, download apple silicon

* chore: :fire: remove notes

* fix: :green_heart: use curl to download binary

* fix: :green_heart: set permissions on apple silicon binary

* fix: :green_heart: testing

* fix: :green_heart: cleanup file

* fix: :green_heart: fix preview.yaml

* fix: :green_heart: only upload once per binary

* fix: :green_heart: install rosetta

* ci: :green_heart: download binary after tests

* ci: 💚 Update package.json version [skip ci]

* ci: :green_heart: prepare ci for merge to main

---------

Co-authored-by: GitHub Action <action@github.com>

1 files changed, 125 insertions, 0 deletions

diff --git a/server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py b/server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py
new file mode 100644
index 00000000..e4a932af
--- /dev/null
+++ b/server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py
@@ -0,0 +1,125 @@
+import os
+from textwrap import dedent
+
+from ....core.main import Step
+from ....core.sdk import ContinueSDK, Models
+from ....core.steps import MessageStep
+from ....plugins.steps.find_and_replace import FindAndReplaceStep
+
+AI_ASSISTED_STRING = "(✨ AI-Assisted ✨)"
+
+
+class SetupPipelineStep(Step):
+    hide: bool = True
+    name: str = "Setup dlt Pipeline"
+
+    source_name: str
+
+    async def describe(self, models: Models):
+        pass
+
+    async def run(self, sdk: ContinueSDK):
+        await sdk.run(
+            [
+                "python3 -m venv .env",
+                "source .env/bin/activate",
+                "pip install dlt",
+                f"dlt --non-interactive init {self.source_name} duckdb",
+                "pip install -r requirements.txt",
+            ],
+            description=dedent(
+                f"""\
+            Running the following commands:
+            - `python3 -m venv .env`: Create a Python virtual environment
+            - `source .env/bin/activate`: Activate the virtual environment
+            - `pip install dlt`: Install dlt
+            - `dlt init {self.source_name} duckdb`: Create a new dlt pipeline called {self.source_name} that loads data into a local DuckDB instance
+            - `pip install -r requirements.txt`: Install the Python dependencies for the pipeline"""
+            ),
+            name="Setup Python environment",
+        )
+
+
+class RunPipelineStep(Step):
+    hide: bool = True
+    name: str = "Run dlt Pipeline"
+
+    source_name: str
+
+    async def describe(self, models: Models):
+        pass
+
+    async def run(self, sdk: ContinueSDK):
+        await sdk.run(
+            [
+                f"python3 {self.source_name}_pipeline.py",
+            ],
+            description=dedent(
+                f"""\
+            Running the command `python3 {self.source_name}_pipeline.py to run the pipeline: """
+            ),
+            name="Run dlt pipeline",
+        )
+
+
+class DeployAirflowStep(Step):
+    hide: bool = True
+    source_name: str
+
+    async def run(self, sdk: ContinueSDK):
+        # Run dlt command to deploy pipeline to Airflow
+        await sdk.run(
+            [
+                "git init",
+                f"dlt --non-interactive deploy {self.source_name}_pipeline.py airflow-composer",
+            ],
+            description="Running `dlt deploy airflow` to deploy the dlt pipeline to Airflow",
+            name="Deploy dlt pipeline to Airflow",
+        )
+
+        # Get filepaths, open the DAG file
+        directory = await sdk.ide.getWorkspaceDirectory()
+        pipeline_filepath = os.path.join(directory, f"{self.source_name}_pipeline.py")
+        dag_filepath = os.path.join(
+            directory, f"dags/dag_{self.source_name}_pipeline.py"
+        )
+
+        await sdk.ide.setFileOpen(dag_filepath)
+
+        # Replace the pipeline name and dataset name
+        await sdk.run_step(
+            FindAndReplaceStep(
+                filepath=pipeline_filepath,
+                pattern="'pipeline_name'",
+                replacement=f"'{self.source_name}_pipeline'",
+            )
+        )
+        await sdk.run_step(
+            FindAndReplaceStep(
+                filepath=pipeline_filepath,
+                pattern="'dataset_name'",
+                replacement=f"'{self.source_name}_data'",
+            )
+        )
+        await sdk.run_step(
+            FindAndReplaceStep(
+                filepath=pipeline_filepath,
+                pattern="pipeline_or_source_script",
+                replacement=f"{self.source_name}_pipeline",
+            )
+        )
+
+        # Prompt the user for the DAG schedule
+        # edit_dag_range = Range.from_shorthand(18, 0, 23, 0)
+        # await sdk.ide.highlightCode(range_in_file=RangeInFile(filepath=dag_filepath, range=edit_dag_range), color="#33993333")
+        # response = await sdk.run_step(WaitForUserInputStep(prompt="When would you like this Airflow DAG to run? (e.g. every day, every Monday, every 1st of the month, etc.)"))
+        # await sdk.edit_file(dag_filepath, prompt=f"Edit the DAG so that it runs at the following schedule: '{response.text}'",
+        #                     range=edit_dag_range)
+
+        # Tell the user to check the schedule and fill in owner, email, other default_args
+        await sdk.run_step(
+            MessageStep(
+                message="Fill in the owner, email, and other default_args in the DAG file with your own personal information. Then the DAG will be ready to run!",
+                name="Fill in default_args",
+            )
+        )