Airflow Dag for Vertex AI

Google Cloud Build with AI Custom ML & MLOps
1

Please advice on how to create airflow dag for vertex ai.

from datetime import datetime
from airflow import DAG
from airflow.decorators import task
from google.cloud import aiplatform
from airflow.operators import CreateDatasetOperator

YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)

default_dag_args = {

‘start_date’: YESTERDAY,
}

with models.DAG(
‘composer_sample_simple_greeting’,
schedule_interval=datetime.timedelta(weeks=2),
default_args=default_dag_args) as dag:

def create_entity_type_sample(
project: str,
location: str,
entity_type_id: str,
vertexai: str,
service_account_id: str
task_id: str,
project_id: str,

aiplatform.init(project=project, location=location)

my_entity_type = aiplatform.EntityType.create(
entity_type_id=entity_type_id, vertexai=vertexai
)

my_entity_type.wait()

return my_entity_type

create_image_dataset_job = CreateDatasetOperator(
task_id=“image_dataset”,
dataset=IMAGE_DATASET,
region=REGION,
project_id=PROJECT_ID,
)
create_tabular_dataset_job = CreateDatasetOperator(
task_id=“tabular_dataset”,
dataset=TABULAR_DATASET,
region=REGION,
project_id=PROJECT_ID,
)
create_text_dataset_job = CreateDatasetOperator(
task_id=“text_dataset”,
dataset=TEXT_DATASET,
region=REGION,
project_id=PROJECT_ID,
)
create_video_dataset_job = CreateDatasetOperator(
task_id=“video_dataset”,
dataset=VIDEO_DATASET,
region=REGION,
project_id=PROJECT_ID,
)
create_time_series_dataset_job = CreateDatasetOperator(
task_id=“time_series_dataset”,
dataset=TIME_SERIES_DATASET,
region=REGION,
project_id=PROJECT_ID,
)

create_image_dataset_job >> create_tabular_dataset_job >> create_text_dataset_job >> create_video_dataset_job >> create_time_series_dataset_job

2

You can check in this link for available [Google Cloud VertexAI Operators](http://Google%20Cloud VertexAI Operators). There are available operators for creating Datasets (in which you already used in your code), training jobs, batch prediction jobs, endpoint service and more.

You may also check this documentation on how to run an apache airflow DAG in Cloud Composer.