1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
|
from textwrap import dedent
from ....core.main import Step
from ....core.sdk import ContinueSDK
from ....core.steps import MessageStep
from ....plugins.steps.input.nl_multiselect import NLMultiselectStep
from .steps import DeployAirflowStep, RunPipelineStep, SetupPipelineStep
# https://github.com/dlt-hub/dlt-deploy-template/blob/master/airflow-composer/dag_template.py
# https://www.notion.so/dlthub/Deploy-a-pipeline-with-Airflow-245fd1058652479494307ead0b5565f3
# 1. What verified pipeline do you want to deploy with Airflow?
# 2. Set up selected verified pipeline
# 3. Deploy selected verified pipeline with Airflow
# 4. Set up Airflow locally?
class DeployPipelineAirflowRecipe(Step):
hide: bool = True
async def run(self, sdk: ContinueSDK):
source_name = await sdk.run_step(
MessageStep(
name="Deploying a pipeline to Airflow",
message=dedent(
"""\
This recipe will show you how to deploy a pipeline to Airflow. With the help of Continue, you will:
- Select a dlt-verified pipeline
- Setup the pipeline
- Deploy it to Airflow
- Optionally, setup Airflow locally"""
),
)
>> NLMultiselectStep(
prompt=dedent(
"""\
Which verified pipeline do you want to deploy with Airflow? The options are:
- Asana
- Chess.com
- Facebook Ads
- GitHub
- Google Analytics
- Google Sheets
- HubSpot
- Jira
- Matomo
- Mux
- Notion
- Pipedrive
- Pokemon
- Salesforce
- Shopify
- Strapi
- Stripe
- SQL Database
- Workable
- Zendesk"""
),
options=[
"asana_dlt",
"chess",
"github",
"google_analytics",
"google_sheets",
"hubspot",
"matomo",
"pipedrive",
"shopify_dlt",
"strapi",
"zendesk",
"facebook_ads",
"jira",
"mux",
"notion",
"pokemon",
"salesforce",
"stripe_analytics",
"sql_database",
"workable",
],
)
)
await sdk.run_step(
SetupPipelineStep(source_name=source_name)
>> RunPipelineStep(source_name=source_name)
>> DeployAirflowStep(source_name=source_name)
)
|
