refactor: restructure monorepo for clean portfolio layout

- Move timesfm-forecast into apps/ directory
- Flatten Udacity portfolio projects from deep URL-encoded paths
  into data-engineering/01-XX numbered directories
- Remove old My-Data-Engineering-Portifolio/ parent directory
- Rewrite root README.md: professional overview with badges,
  project table, and repo structure diagram
- Create data-engineering/README.md with per-project descriptions
- Add README.md for 02-cassandra-modeling (was missing)
- Add README.md for 05-airflow-pipelines (was missing)
- Normalize capstone readme.md -> README.md
- Update .gitignore: add *.cfg, *.env, *.zip, *.sas7bdat,
  Jupyter checkpoints, IDE dirs; remove uv.lock exclusion
- Add dwh.cfg.example and dl.cfg.example credential templates
- Untrack real credential files (dwh.cfg, dl.cfg)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@gabriel.pereira
2026-03-26 16:48:50 -03:00
parent 5c4e6075e1
commit 6796398924
160 changed files with 308 additions and 34 deletions

Binary file not shown.

View File

@@ -0,0 +1,11 @@
from operators.stage_redshift import StageToRedshiftOperator
from operators.load_fact import LoadFactOperator
from operators.load_dimension import LoadDimensionOperator
from operators.data_quality import DataQualityOperator
__all__ = [
'StageToRedshiftOperator',
'LoadFactOperator',
'LoadDimensionOperator',
'DataQualityOperator'
]

View File

@@ -0,0 +1,22 @@
from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
class DataQualityOperator(BaseOperator):
ui_color = '#89DA59'
@apply_defaults
def __init__(self,
# Define your operators params (with defaults) here
# Example:
# conn_id = your-connection-name
*args, **kwargs):
super(DataQualityOperator, self).__init__(*args, **kwargs)
# Map params here
# Example:
# self.conn_id = conn_id
def execute(self, context):
self.log.info('DataQualityOperator not implemented yet')

View File

@@ -0,0 +1,22 @@
from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
class LoadDimensionOperator(BaseOperator):
ui_color = '#80BD9E'
@apply_defaults
def __init__(self,
# Define your operators params (with defaults) here
# Example:
# conn_id = your-connection-name
*args, **kwargs):
super(LoadDimensionOperator, self).__init__(*args, **kwargs)
# Map params here
# Example:
# self.conn_id = conn_id
def execute(self, context):
self.log.info('LoadDimensionOperator not implemented yet')

View File

@@ -0,0 +1,22 @@
from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
class LoadFactOperator(BaseOperator):
ui_color = '#F98866'
@apply_defaults
def __init__(self,
# Define your operators params (with defaults) here
# Example:
# conn_id = your-connection-name
*args, **kwargs):
super(LoadFactOperator, self).__init__(*args, **kwargs)
# Map params here
# Example:
# self.conn_id = conn_id
def execute(self, context):
self.log.info('LoadFactOperator not implemented yet')

View File

@@ -0,0 +1,26 @@
from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
class StageToRedshiftOperator(BaseOperator):
ui_color = '#358140'
@apply_defaults
def __init__(self,
# Define your operators params (with defaults) here
# Example:
# redshift_conn_id=your-connection-name
*args, **kwargs):
super(StageToRedshiftOperator, self).__init__(*args, **kwargs)
# Map params here
# Example:
# self.conn_id = conn_id
def execute(self, context):
self.log.info('StageToRedshiftOperator not implemented yet')