refactor: restructure monorepo for clean portfolio layout

- Move timesfm-forecast into apps/ directory
- Flatten Udacity portfolio projects from deep URL-encoded paths
  into data-engineering/01-XX numbered directories
- Remove old My-Data-Engineering-Portifolio/ parent directory
- Rewrite root README.md: professional overview with badges,
  project table, and repo structure diagram
- Create data-engineering/README.md with per-project descriptions
- Add README.md for 02-cassandra-modeling (was missing)
- Add README.md for 05-airflow-pipelines (was missing)
- Normalize capstone readme.md -> README.md
- Update .gitignore: add *.cfg, *.env, *.zip, *.sas7bdat,
  Jupyter checkpoints, IDE dirs; remove uv.lock exclusion
- Add dwh.cfg.example and dl.cfg.example credential templates
- Untrack real credential files (dwh.cfg, dl.cfg)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

data-engineering/05-airflow-pipelines/README.md

@@ -0,0 +1,83 @@
+# Automate Pipelines with Apache Airflow
+
+**Udacity Data Engineering Nanodegree — Project 5**
+
+## Overview
+
+Build a production-grade, orchestrated ETL pipeline using **Apache Airflow**. The pipeline runs on an hourly schedule, loading data from AWS S3 into AWS Redshift staging tables, transforming it into a star schema, and validating data quality — all using modular, reusable custom operators.
+
+## Architecture
+
+```
+S3 (raw JSON)
+    │
+    ▼
+[StageToRedshiftOperator]    ← Stage events & songs
+    │
+    ▼
+[LoadFactOperator]           ← Load songplays fact table
+    │
+    ▼
+[LoadDimensionOperator] ×4   ← Load users, songs, artists, time
+    │
+    ▼
+[DataQualityOperator]        ← Assert tables are non-empty
+```
+
+## DAG
+
+| Property | Value |
+|---|---|
+| **Schedule** | Hourly (`0 * * * *`) |
+| **Start date** | 2019-01-12 |
+| **Catchup** | Disabled |
+
+## Custom Operators
+
+| Operator | File | Purpose |
+|---|---|---|
+| `StageToRedshiftOperator` | `plugins/operators/stage_redshift.py` | COPY JSON from S3 to Redshift staging tables |
+| `LoadFactOperator` | `plugins/operators/load_fact.py` | INSERT into fact table from staging |
+| `LoadDimensionOperator` | `plugins/operators/load_dimension.py` | INSERT into dimension tables (supports truncate-insert or append) |
+| `DataQualityOperator` | `plugins/operators/data_quality.py` | Assert row counts > 0 for all tables |
+
+## Project Structure
+
+```
+05-airflow-pipelines/
+├── create_tables.sql          # DDL for Redshift tables
+├── dags/
+│   └── udac_example_dag.py    # Main DAG definition
+└── plugins/
+    ├── helpers/
+    │   └── sql_queries.py     # Shared SQL INSERT statements
+    └── operators/
+        ├── stage_redshift.py
+        ├── load_fact.py
+        ├── load_dimension.py
+        └── data_quality.py
+```
+
+## Key Concepts
+
+- **DAG design** — directed acyclic graphs for workflow orchestration
+- **Custom operators** — reusable, parameterized Airflow tasks
+- **Modular SQL helpers** — shared query library via `SqlQueries` class
+- **Data quality checks** — automated validation at end of every run
+- **AWS integration** — S3 `COPY` + Redshift connections via Airflow Connections
+
+## How to Run
+
+1. Set up an Airflow environment with AWS connections configured:
+   - `aws_credentials` — IAM access key & secret
+   - `redshift` — Redshift cluster connection string
+
+2. Copy DAG and plugins into your Airflow home:
+   ```bash
+   cp -r dags/ $AIRFLOW_HOME/dags/
+   cp -r plugins/ $AIRFLOW_HOME/plugins/
+   ```
+
+3. Run `create_tables.sql` against your Redshift cluster to create staging and DW tables.
+
+4. Enable the DAG in the Airflow UI — it will trigger hourly.

data-engineering/05-airflow-pipelines/create_tables.sql

@@ -0,0 +1,83 @@
+CREATE TABLE public.artists (
+	artistid varchar(256) NOT NULL,
+	name varchar(256),
+	location varchar(256),
+	lattitude numeric(18,0),
+	longitude numeric(18,0)
+);
+
+CREATE TABLE public.songplays (
+	playid varchar(32) NOT NULL,
+	start_time timestamp NOT NULL,
+	userid int4 NOT NULL,
+	"level" varchar(256),
+	songid varchar(256),
+	artistid varchar(256),
+	sessionid int4,
+	location varchar(256),
+	user_agent varchar(256),
+	CONSTRAINT songplays_pkey PRIMARY KEY (playid)
+);
+
+CREATE TABLE public.songs (
+	songid varchar(256) NOT NULL,
+	title varchar(256),
+	artistid varchar(256),
+	"year" int4,
+	duration numeric(18,0),
+	CONSTRAINT songs_pkey PRIMARY KEY (songid)
+);
+
+CREATE TABLE public.staging_events (
+	artist varchar(256),
+	auth varchar(256),
+	firstname varchar(256),
+	gender varchar(256),
+	iteminsession int4,
+	lastname varchar(256),
+	length numeric(18,0),
+	"level" varchar(256),
+	location varchar(256),
+	"method" varchar(256),
+	page varchar(256),
+	registration numeric(18,0),
+	sessionid int4,
+	song varchar(256),
+	status int4,
+	ts int8,
+	useragent varchar(256),
+	userid int4
+);
+
+CREATE TABLE public.staging_songs (
+	num_songs int4,
+	artist_id varchar(256),
+	artist_name varchar(256),
+	artist_latitude numeric(18,0),
+	artist_longitude numeric(18,0),
+	artist_location varchar(256),
+	song_id varchar(256),
+	title varchar(256),
+	duration numeric(18,0),
+	"year" int4
+);
+
+CREATE TABLE public."time" (
+	start_time timestamp NOT NULL,
+	"hour" int4,
+	"day" int4,
+	week int4,
+	"month" varchar(256),
+	"year" int4,
+	weekday varchar(256),
+	CONSTRAINT time_pkey PRIMARY KEY (start_time)
+) ;
+
+CREATE TABLE public.users (
+	userid int4 NOT NULL,
+	first_name varchar(256),
+	last_name varchar(256),
+	gender varchar(256),
+	"level" varchar(256),
+	CONSTRAINT users_pkey PRIMARY KEY (userid)
+);

data-engineering/05-airflow-pipelines/dags/udac_example_dag.py

@@ -0,0 +1,65 @@
+from datetime import datetime, timedelta
+import os
+from airflow import DAG
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.operators import (StageToRedshiftOperator, LoadFactOperator,
+                                LoadDimensionOperator, DataQualityOperator)
+from helpers import SqlQueries
+
+# AWS_KEY = os.environ.get('AWS_KEY')
+# AWS_SECRET = os.environ.get('AWS_SECRET')
+
+default_args = {
+    'owner': 'udacity',
+    'start_date': datetime(2019, 1, 12),
+}
+
+dag = DAG('udac_example_dag',
+          default_args=default_args,
+          description='Load and transform data in Redshift with Airflow',
+          schedule_interval='0 * * * *'
+        )
+
+start_operator = DummyOperator(task_id='Begin_execution',  dag=dag)
+
+stage_events_to_redshift = StageToRedshiftOperator(
+    task_id='Stage_events',
+    dag=dag
+)
+
+stage_songs_to_redshift = StageToRedshiftOperator(
+    task_id='Stage_songs',
+    dag=dag
+)
+
+load_songplays_table = LoadFactOperator(
+    task_id='Load_songplays_fact_table',
+    dag=dag
+)
+
+load_user_dimension_table = LoadDimensionOperator(
+    task_id='Load_user_dim_table',
+    dag=dag
+)
+
+load_song_dimension_table = LoadDimensionOperator(
+    task_id='Load_song_dim_table',
+    dag=dag
+)
+
+load_artist_dimension_table = LoadDimensionOperator(
+    task_id='Load_artist_dim_table',
+    dag=dag
+)
+
+load_time_dimension_table = LoadDimensionOperator(
+    task_id='Load_time_dim_table',
+    dag=dag
+)
+
+run_quality_checks = DataQualityOperator(
+    task_id='Run_data_quality_checks',
+    dag=dag
+)
+
+end_operator = DummyOperator(task_id='Stop_execution',  dag=dag)

data-engineering/05-airflow-pipelines/plugins/__init__.py

@@ -0,0 +1,19 @@
+from __future__ import division, absolute_import, print_function
+
+from airflow.plugins_manager import AirflowPlugin
+
+import operators
+import helpers
+
+# Defining the plugin class
+class UdacityPlugin(AirflowPlugin):
+    name = "udacity_plugin"
+    operators = [
+        operators.StageToRedshiftOperator,
+        operators.LoadFactOperator,
+        operators.LoadDimensionOperator,
+        operators.DataQualityOperator
+    ]
+    helpers = [
+        helpers.SqlQueries
+    ]

data-engineering/05-airflow-pipelines/plugins/helpers/__init__.py

@@ -0,0 +1,5 @@
+from helpers.sql_queries import SqlQueries
+
+__all__ = [
+    'SqlQueries',
+]

data-engineering/05-airflow-pipelines/plugins/helpers/sql_queries.py

@@ -0,0 +1,42 @@
+class SqlQueries:
+    songplay_table_insert = ("""
+        SELECT
+                md5(events.sessionid || events.start_time) songplay_id,
+                events.start_time, 
+                events.userid, 
+                events.level, 
+                songs.song_id, 
+                songs.artist_id, 
+                events.sessionid, 
+                events.location, 
+                events.useragent
+                FROM (SELECT TIMESTAMP 'epoch' + ts/1000 * interval '1 second' AS start_time, *
+            FROM staging_events
+            WHERE page='NextSong') events
+            LEFT JOIN staging_songs songs
+            ON events.song = songs.title
+                AND events.artist = songs.artist_name
+                AND events.length = songs.duration
+    """)
+
+    user_table_insert = ("""
+        SELECT distinct userid, firstname, lastname, gender, level
+        FROM staging_events
+        WHERE page='NextSong'
+    """)
+
+    song_table_insert = ("""
+        SELECT distinct song_id, title, artist_id, year, duration
+        FROM staging_songs
+    """)
+
+    artist_table_insert = ("""
+        SELECT distinct artist_id, artist_name, artist_location, artist_latitude, artist_longitude
+        FROM staging_songs
+    """)
+
+    time_table_insert = ("""
+        SELECT start_time, extract(hour from start_time), extract(day from start_time), extract(week from start_time), 
+               extract(month from start_time), extract(year from start_time), extract(dayofweek from start_time)
+        FROM songplays
+    """)

data-engineering/05-airflow-pipelines/plugins/operators/__init__.py

@@ -0,0 +1,11 @@
+from operators.stage_redshift import StageToRedshiftOperator
+from operators.load_fact import LoadFactOperator
+from operators.load_dimension import LoadDimensionOperator
+from operators.data_quality import DataQualityOperator
+
+__all__ = [
+    'StageToRedshiftOperator',
+    'LoadFactOperator',
+    'LoadDimensionOperator',
+    'DataQualityOperator'
+]

data-engineering/05-airflow-pipelines/plugins/operators/data_quality.py

@@ -0,0 +1,22 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class DataQualityOperator(BaseOperator):
+
+    ui_color = '#89DA59'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # conn_id = your-connection-name
+                 *args, **kwargs):
+
+        super(DataQualityOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('DataQualityOperator not implemented yet')

data-engineering/05-airflow-pipelines/plugins/operators/load_dimension.py

@@ -0,0 +1,22 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class LoadDimensionOperator(BaseOperator):
+
+    ui_color = '#80BD9E'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # conn_id = your-connection-name
+                 *args, **kwargs):
+
+        super(LoadDimensionOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('LoadDimensionOperator not implemented yet')

data-engineering/05-airflow-pipelines/plugins/operators/load_fact.py

@@ -0,0 +1,22 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class LoadFactOperator(BaseOperator):
+
+    ui_color = '#F98866'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # conn_id = your-connection-name
+                 *args, **kwargs):
+
+        super(LoadFactOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('LoadFactOperator not implemented yet')

data-engineering/05-airflow-pipelines/plugins/operators/stage_redshift.py

@@ -0,0 +1,26 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class StageToRedshiftOperator(BaseOperator):
+    ui_color = '#358140'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # redshift_conn_id=your-connection-name
+                 *args, **kwargs):
+
+        super(StageToRedshiftOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('StageToRedshiftOperator not implemented yet')
+
+
+
+
+