refactor: restructure monorepo for clean portfolio layout
- Move timesfm-forecast into apps/ directory - Flatten Udacity portfolio projects from deep URL-encoded paths into data-engineering/01-XX numbered directories - Remove old My-Data-Engineering-Portifolio/ parent directory - Rewrite root README.md: professional overview with badges, project table, and repo structure diagram - Create data-engineering/README.md with per-project descriptions - Add README.md for 02-cassandra-modeling (was missing) - Add README.md for 05-airflow-pipelines (was missing) - Normalize capstone readme.md -> README.md - Update .gitignore: add *.cfg, *.env, *.zip, *.sas7bdat, Jupyter checkpoints, IDE dirs; remove uv.lock exclusion - Add dwh.cfg.example and dl.cfg.example credential templates - Untrack real credential files (dwh.cfg, dl.cfg) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
from airflow import DAG
|
||||
from airflow.operators.dummy_operator import DummyOperator
|
||||
from airflow.operators import (StageToRedshiftOperator, LoadFactOperator,
|
||||
LoadDimensionOperator, DataQualityOperator)
|
||||
from helpers import SqlQueries
|
||||
|
||||
# AWS_KEY = os.environ.get('AWS_KEY')
|
||||
# AWS_SECRET = os.environ.get('AWS_SECRET')
|
||||
|
||||
default_args = {
|
||||
'owner': 'udacity',
|
||||
'start_date': datetime(2019, 1, 12),
|
||||
}
|
||||
|
||||
dag = DAG('udac_example_dag',
|
||||
default_args=default_args,
|
||||
description='Load and transform data in Redshift with Airflow',
|
||||
schedule_interval='0 * * * *'
|
||||
)
|
||||
|
||||
start_operator = DummyOperator(task_id='Begin_execution', dag=dag)
|
||||
|
||||
stage_events_to_redshift = StageToRedshiftOperator(
|
||||
task_id='Stage_events',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
stage_songs_to_redshift = StageToRedshiftOperator(
|
||||
task_id='Stage_songs',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
load_songplays_table = LoadFactOperator(
|
||||
task_id='Load_songplays_fact_table',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
load_user_dimension_table = LoadDimensionOperator(
|
||||
task_id='Load_user_dim_table',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
load_song_dimension_table = LoadDimensionOperator(
|
||||
task_id='Load_song_dim_table',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
load_artist_dimension_table = LoadDimensionOperator(
|
||||
task_id='Load_artist_dim_table',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
load_time_dimension_table = LoadDimensionOperator(
|
||||
task_id='Load_time_dim_table',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
run_quality_checks = DataQualityOperator(
|
||||
task_id='Run_data_quality_checks',
|
||||
dag=dag
|
||||
)
|
||||
|
||||
end_operator = DummyOperator(task_id='Stop_execution', dag=dag)
|
||||
Reference in New Issue
Block a user