refactor: restructure monorepo for clean portfolio layout
- Move timesfm-forecast into apps/ directory - Flatten Udacity portfolio projects from deep URL-encoded paths into data-engineering/01-XX numbered directories - Remove old My-Data-Engineering-Portifolio/ parent directory - Rewrite root README.md: professional overview with badges, project table, and repo structure diagram - Create data-engineering/README.md with per-project descriptions - Add README.md for 02-cassandra-modeling (was missing) - Add README.md for 05-airflow-pipelines (was missing) - Normalize capstone readme.md -> README.md - Update .gitignore: add *.cfg, *.env, *.zip, *.sas7bdat, Jupyter checkpoints, IDE dirs; remove uv.lock exclusion - Add dwh.cfg.example and dl.cfg.example credential templates - Untrack real credential files (dwh.cfg, dl.cfg) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
BIN
data-engineering/05-airflow-pipelines/plugins/operators/.DS_Store
vendored
Normal file
BIN
data-engineering/05-airflow-pipelines/plugins/operators/.DS_Store
vendored
Normal file
Binary file not shown.
@@ -0,0 +1,11 @@
|
||||
from operators.stage_redshift import StageToRedshiftOperator
|
||||
from operators.load_fact import LoadFactOperator
|
||||
from operators.load_dimension import LoadDimensionOperator
|
||||
from operators.data_quality import DataQualityOperator
|
||||
|
||||
__all__ = [
|
||||
'StageToRedshiftOperator',
|
||||
'LoadFactOperator',
|
||||
'LoadDimensionOperator',
|
||||
'DataQualityOperator'
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
from airflow.hooks.postgres_hook import PostgresHook
|
||||
from airflow.models import BaseOperator
|
||||
from airflow.utils.decorators import apply_defaults
|
||||
|
||||
class DataQualityOperator(BaseOperator):
|
||||
|
||||
ui_color = '#89DA59'
|
||||
|
||||
@apply_defaults
|
||||
def __init__(self,
|
||||
# Define your operators params (with defaults) here
|
||||
# Example:
|
||||
# conn_id = your-connection-name
|
||||
*args, **kwargs):
|
||||
|
||||
super(DataQualityOperator, self).__init__(*args, **kwargs)
|
||||
# Map params here
|
||||
# Example:
|
||||
# self.conn_id = conn_id
|
||||
|
||||
def execute(self, context):
|
||||
self.log.info('DataQualityOperator not implemented yet')
|
||||
@@ -0,0 +1,22 @@
|
||||
from airflow.hooks.postgres_hook import PostgresHook
|
||||
from airflow.models import BaseOperator
|
||||
from airflow.utils.decorators import apply_defaults
|
||||
|
||||
class LoadDimensionOperator(BaseOperator):
|
||||
|
||||
ui_color = '#80BD9E'
|
||||
|
||||
@apply_defaults
|
||||
def __init__(self,
|
||||
# Define your operators params (with defaults) here
|
||||
# Example:
|
||||
# conn_id = your-connection-name
|
||||
*args, **kwargs):
|
||||
|
||||
super(LoadDimensionOperator, self).__init__(*args, **kwargs)
|
||||
# Map params here
|
||||
# Example:
|
||||
# self.conn_id = conn_id
|
||||
|
||||
def execute(self, context):
|
||||
self.log.info('LoadDimensionOperator not implemented yet')
|
||||
@@ -0,0 +1,22 @@
|
||||
from airflow.hooks.postgres_hook import PostgresHook
|
||||
from airflow.models import BaseOperator
|
||||
from airflow.utils.decorators import apply_defaults
|
||||
|
||||
class LoadFactOperator(BaseOperator):
|
||||
|
||||
ui_color = '#F98866'
|
||||
|
||||
@apply_defaults
|
||||
def __init__(self,
|
||||
# Define your operators params (with defaults) here
|
||||
# Example:
|
||||
# conn_id = your-connection-name
|
||||
*args, **kwargs):
|
||||
|
||||
super(LoadFactOperator, self).__init__(*args, **kwargs)
|
||||
# Map params here
|
||||
# Example:
|
||||
# self.conn_id = conn_id
|
||||
|
||||
def execute(self, context):
|
||||
self.log.info('LoadFactOperator not implemented yet')
|
||||
@@ -0,0 +1,26 @@
|
||||
from airflow.hooks.postgres_hook import PostgresHook
|
||||
from airflow.models import BaseOperator
|
||||
from airflow.utils.decorators import apply_defaults
|
||||
|
||||
class StageToRedshiftOperator(BaseOperator):
|
||||
ui_color = '#358140'
|
||||
|
||||
@apply_defaults
|
||||
def __init__(self,
|
||||
# Define your operators params (with defaults) here
|
||||
# Example:
|
||||
# redshift_conn_id=your-connection-name
|
||||
*args, **kwargs):
|
||||
|
||||
super(StageToRedshiftOperator, self).__init__(*args, **kwargs)
|
||||
# Map params here
|
||||
# Example:
|
||||
# self.conn_id = conn_id
|
||||
|
||||
def execute(self, context):
|
||||
self.log.info('StageToRedshiftOperator not implemented yet')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user