diff options
3 files changed, 18 insertions, 16 deletions
diff --git a/continuedev/src/continuedev/recipes/DDtoBQRecipe/main.py b/continuedev/src/continuedev/recipes/DDtoBQRecipe/main.py index 1cb12ff3..cd1ff1b9 100644 --- a/continuedev/src/continuedev/recipes/DDtoBQRecipe/main.py +++ b/continuedev/src/continuedev/recipes/DDtoBQRecipe/main.py @@ -4,24 +4,24 @@ from ...core.main import Step from ...core.sdk import ContinueSDK from ...steps.core.core import WaitForUserInputStep from ...steps.main import MessageStep -from .steps import SetUpChessPipelineStep, SwitchDestinationStep +from .steps import SetUpChessPipelineStep, SwitchDestinationStep, LoadDataStep # Based on the following guide: # https://github.com/dlt-hub/dlt/pull/392 + class DDtoBQRecipeRecipe(Step): hide: bool = True async def run(self, sdk: ContinueSDK): - text_observation = await sdk.run_step( + await sdk.run_step( MessageStep(name="Move from using DuckDB to Google BigQuery as the destination", message=dedent("""\ This recipe will walk you through the process of moving from using DuckDB to Google BigQuery as the destination for your dlt pipeline. With the help of Continue, you will: - Set up a dlt pipeline for the chess.com API - Switch destination from DuckDB to Google BigQuery - Add BigQuery credentials to your secrets.toml file - - Run the pipeline again to load data to BigQuery""")) - ) - await sdk.run_step( + - Run the pipeline again to load data to BigQuery""")) >> SetUpChessPipelineStep() >> - SwitchDestinationStep() - )
\ No newline at end of file + SwitchDestinationStep() >> + LoadDataStep() + ) diff --git a/continuedev/src/continuedev/recipes/DDtoBQRecipe/steps.py b/continuedev/src/continuedev/recipes/DDtoBQRecipe/steps.py index 395cbbc8..c7e5d095 100644 --- a/continuedev/src/continuedev/recipes/DDtoBQRecipe/steps.py +++ b/continuedev/src/continuedev/recipes/DDtoBQRecipe/steps.py @@ -3,6 +3,7 @@ import subprocess from textwrap import dedent import time +from ...steps.find_and_replace import FindAndReplaceStep from ...models.main import Range from ...models.filesystem import RangeInFile from ...steps.main import MessageStep @@ -14,6 +15,7 @@ from ...core.sdk import ContinueSDK AI_ASSISTED_STRING = "(✨ AI-Assisted ✨)" + class SetUpChessPipelineStep(Step): hide: bool = True name: str = "Setup Chess.com API dlt Pipeline" @@ -45,7 +47,8 @@ class SwitchDestinationStep(Step): async def run(self, sdk: ContinueSDK): # Switch destination from DuckDB to Google BigQuery - filepath = os.path.join(sdk.ide.workspace_directory, 'chess.py') + filepath = os.path.join( + sdk.ide.workspace_directory, 'chess_pipeline.py') await sdk.run_step(FindAndReplaceStep(filepath=filepath, pattern="destination='duckdb'", replacement="destination='bigquery'")) # Add BigQuery credentials to your secrets.toml file @@ -58,7 +61,7 @@ class SwitchDestinationStep(Step): # wait for user to put API key in secrets.toml secrets_path = os.path.join( - sdk.ide.workspace_directory, "/.dlt/secrets.toml") + sdk.ide.workspace_directory, ".dlt/secrets.toml") await sdk.ide.setFileOpen(secrets_path) await sdk.append_to_file(secrets_path, template) @@ -72,17 +75,15 @@ class LoadDataStep(Step): async def run(self, sdk: ContinueSDK): # Run the pipeline again to load data to BigQuery - output = await sdk.run('env/bin/python3 chess.py', name="Load data to BigQuery", description="Running `env/bin/python3 chess.py` to load data to Google BigQuery") + output = await sdk.run('env/bin/python3 chess_pipeline.py', name="Load data to BigQuery", description="Running `env/bin/python3 chess_pipeline.py` to load data to Google BigQuery") if "Traceback" in output or "SyntaxError" in output: - with open(os.path.join(__file__, "dlt_duckdb_to_bigquery_docs.md"), "r") as f: + with open(os.path.join(os.path.dirname(__file__), "dlt_duckdb_to_bigquery_docs.md"), "r") as f: docs = f.read() + output = "Traceback" + output.split("Traceback")[-1] suggestion = sdk.models.gpt35.complete(dedent(f"""\ - ```python - {await sdk.ide.readFile(os.path.join(sdk.ide.workspace_directory, "query.py"))} - ``` - This above code is a query that runs on the DuckDB instance. While attempting to run the query, the following error occurred: + When trying to load data into BigQuery, the following error occurred: ```ascii {output} diff --git a/continuedev/src/continuedev/steps/find_and_replace.py b/continuedev/src/continuedev/steps/find_and_replace.py index 78511b27..c9654867 100644 --- a/continuedev/src/continuedev/steps/find_and_replace.py +++ b/continuedev/src/continuedev/steps/find_and_replace.py @@ -19,7 +19,8 @@ class FindAndReplaceStep(Step): end_index = start_index + len(self.pattern) await sdk.ide.applyFileSystemEdit(FileEdit( filepath=self.filepath, - range=Range.from_indices(file_content, start_index, end_index), + range=Range.from_indices( + file_content, start_index, end_index - 1), replacement=self.replacement )) file_content = file_content[:start_index] + \ |