From f09150617ed2454f3074bcf93f53aae5ae637d40 Mon Sep 17 00:00:00 2001 From: Nate Sesti <33237525+sestinj@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:37:27 -0700 Subject: Preview (#541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Strong typing (#533) * refactor: :recycle: get rid of continuedev.src.continuedev structure * refactor: :recycle: switching back to server folder * feat: :sparkles: make config.py imports shorter * feat: :bookmark: publish as pre-release vscode extension * refactor: :recycle: refactor and add more completion params to ui * build: :building_construction: download from preview S3 * fix: :bug: fix paths * fix: :green_heart: package:pre-release * ci: :green_heart: more time for tests * fix: :green_heart: fix build scripts * fix: :bug: fix import in run.py * fix: :bookmark: update version to try again * ci: 💚 Update package.json version [skip ci] * refactor: :fire: don't check for old extensions version * fix: :bug: small bug fixes * fix: :bug: fix config.py import paths * ci: 💚 Update package.json version [skip ci] * ci: :green_heart: platform-specific builds test #1 * feat: :green_heart: ship with binary * fix: :green_heart: fix copy statement to include.exe for windows * fix: :green_heart: cd extension before packaging * chore: :loud_sound: count tokens generated * fix: :green_heart: remove npm_config_arch * fix: :green_heart: publish as pre-release! * chore: :bookmark: update version * perf: :green_heart: hardcode distro paths * fix: :bug: fix yaml syntax error * chore: :bookmark: update version * fix: :green_heart: update permissions and version * feat: :bug: kill old server if needed * feat: :lipstick: update marketplace icon for pre-release * ci: 💚 Update package.json version [skip ci] * feat: :sparkles: auto-reload for config.py * feat: :wrench: update default config.py imports * feat: :sparkles: codelens in config.py * feat: :sparkles: select model param count from UI * ci: 💚 Update package.json version [skip ci] * feat: :sparkles: more model options, ollama error handling * perf: :zap: don't show server loading immediately * fix: :bug: fixing small UI details * ci: 💚 Update package.json version [skip ci] * feat: :rocket: headers param on LLM class * fix: :bug: fix headers for openai.;y * feat: :sparkles: highlight code on cmd+shift+L * ci: 💚 Update package.json version [skip ci] * feat: :lipstick: sticky top bar in gui.tsx * fix: :loud_sound: websocket logging and horizontal scrollbar * ci: 💚 Update package.json version [skip ci] * feat: :sparkles: allow AzureOpenAI Service through GGML * ci: 💚 Update package.json version [skip ci] * fix: :bug: fix automigration * ci: 💚 Update package.json version [skip ci] * ci: :green_heart: upload binaries in ci, download apple silicon * chore: :fire: remove notes * fix: :green_heart: use curl to download binary * fix: :green_heart: set permissions on apple silicon binary * fix: :green_heart: testing * fix: :green_heart: cleanup file * fix: :green_heart: fix preview.yaml * fix: :green_heart: only upload once per binary * fix: :green_heart: install rosetta * ci: :green_heart: download binary after tests * ci: 💚 Update package.json version [skip ci] * ci: :green_heart: prepare ci for merge to main --------- Co-authored-by: GitHub Action --- .../recipes/DeployPipelineAirflowRecipe/steps.py | 125 +++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py (limited to 'server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py') diff --git a/server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py b/server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py new file mode 100644 index 00000000..e4a932af --- /dev/null +++ b/server/continuedev/plugins/recipes/DeployPipelineAirflowRecipe/steps.py @@ -0,0 +1,125 @@ +import os +from textwrap import dedent + +from ....core.main import Step +from ....core.sdk import ContinueSDK, Models +from ....core.steps import MessageStep +from ....plugins.steps.find_and_replace import FindAndReplaceStep + +AI_ASSISTED_STRING = "(✨ AI-Assisted ✨)" + + +class SetupPipelineStep(Step): + hide: bool = True + name: str = "Setup dlt Pipeline" + + source_name: str + + async def describe(self, models: Models): + pass + + async def run(self, sdk: ContinueSDK): + await sdk.run( + [ + "python3 -m venv .env", + "source .env/bin/activate", + "pip install dlt", + f"dlt --non-interactive init {self.source_name} duckdb", + "pip install -r requirements.txt", + ], + description=dedent( + f"""\ + Running the following commands: + - `python3 -m venv .env`: Create a Python virtual environment + - `source .env/bin/activate`: Activate the virtual environment + - `pip install dlt`: Install dlt + - `dlt init {self.source_name} duckdb`: Create a new dlt pipeline called {self.source_name} that loads data into a local DuckDB instance + - `pip install -r requirements.txt`: Install the Python dependencies for the pipeline""" + ), + name="Setup Python environment", + ) + + +class RunPipelineStep(Step): + hide: bool = True + name: str = "Run dlt Pipeline" + + source_name: str + + async def describe(self, models: Models): + pass + + async def run(self, sdk: ContinueSDK): + await sdk.run( + [ + f"python3 {self.source_name}_pipeline.py", + ], + description=dedent( + f"""\ + Running the command `python3 {self.source_name}_pipeline.py to run the pipeline: """ + ), + name="Run dlt pipeline", + ) + + +class DeployAirflowStep(Step): + hide: bool = True + source_name: str + + async def run(self, sdk: ContinueSDK): + # Run dlt command to deploy pipeline to Airflow + await sdk.run( + [ + "git init", + f"dlt --non-interactive deploy {self.source_name}_pipeline.py airflow-composer", + ], + description="Running `dlt deploy airflow` to deploy the dlt pipeline to Airflow", + name="Deploy dlt pipeline to Airflow", + ) + + # Get filepaths, open the DAG file + directory = await sdk.ide.getWorkspaceDirectory() + pipeline_filepath = os.path.join(directory, f"{self.source_name}_pipeline.py") + dag_filepath = os.path.join( + directory, f"dags/dag_{self.source_name}_pipeline.py" + ) + + await sdk.ide.setFileOpen(dag_filepath) + + # Replace the pipeline name and dataset name + await sdk.run_step( + FindAndReplaceStep( + filepath=pipeline_filepath, + pattern="'pipeline_name'", + replacement=f"'{self.source_name}_pipeline'", + ) + ) + await sdk.run_step( + FindAndReplaceStep( + filepath=pipeline_filepath, + pattern="'dataset_name'", + replacement=f"'{self.source_name}_data'", + ) + ) + await sdk.run_step( + FindAndReplaceStep( + filepath=pipeline_filepath, + pattern="pipeline_or_source_script", + replacement=f"{self.source_name}_pipeline", + ) + ) + + # Prompt the user for the DAG schedule + # edit_dag_range = Range.from_shorthand(18, 0, 23, 0) + # await sdk.ide.highlightCode(range_in_file=RangeInFile(filepath=dag_filepath, range=edit_dag_range), color="#33993333") + # response = await sdk.run_step(WaitForUserInputStep(prompt="When would you like this Airflow DAG to run? (e.g. every day, every Monday, every 1st of the month, etc.)")) + # await sdk.edit_file(dag_filepath, prompt=f"Edit the DAG so that it runs at the following schedule: '{response.text}'", + # range=edit_dag_range) + + # Tell the user to check the schedule and fill in owner, email, other default_args + await sdk.run_step( + MessageStep( + message="Fill in the owner, email, and other default_args in the DAG file with your own personal information. Then the DAG will be ready to run!", + name="Fill in default_args", + ) + ) -- cgit v1.2.3-70-g09d2