Add 'My-Data-Engineering-Portifolio/' from commit 'af9dfac5db38895dd0aacc6ea42ddede38a11ca5'

git-subtree-dir: My-Data-Engineering-Portifolio
git-subtree-mainline: d606a91bd5
git-subtree-split: af9dfac5db

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/__init__.py

@@ -0,0 +1,19 @@
+from __future__ import division, absolute_import, print_function
+
+from airflow.plugins_manager import AirflowPlugin
+
+import operators
+import helpers
+
+# Defining the plugin class
+class UdacityPlugin(AirflowPlugin):
+    name = "udacity_plugin"
+    operators = [
+        operators.StageToRedshiftOperator,
+        operators.LoadFactOperator,
+        operators.LoadDimensionOperator,
+        operators.DataQualityOperator
+    ]
+    helpers = [
+        helpers.SqlQueries
+    ]

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/helpers/__init__.py

@@ -0,0 +1,5 @@
+from helpers.sql_queries import SqlQueries
+
+__all__ = [
+    'SqlQueries',
+]

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/helpers/sql_queries.py

@@ -0,0 +1,42 @@
+class SqlQueries:
+    songplay_table_insert = ("""
+        SELECT
+                md5(events.sessionid || events.start_time) songplay_id,
+                events.start_time, 
+                events.userid, 
+                events.level, 
+                songs.song_id, 
+                songs.artist_id, 
+                events.sessionid, 
+                events.location, 
+                events.useragent
+                FROM (SELECT TIMESTAMP 'epoch' + ts/1000 * interval '1 second' AS start_time, *
+            FROM staging_events
+            WHERE page='NextSong') events
+            LEFT JOIN staging_songs songs
+            ON events.song = songs.title
+                AND events.artist = songs.artist_name
+                AND events.length = songs.duration
+    """)
+
+    user_table_insert = ("""
+        SELECT distinct userid, firstname, lastname, gender, level
+        FROM staging_events
+        WHERE page='NextSong'
+    """)
+
+    song_table_insert = ("""
+        SELECT distinct song_id, title, artist_id, year, duration
+        FROM staging_songs
+    """)
+
+    artist_table_insert = ("""
+        SELECT distinct artist_id, artist_name, artist_location, artist_latitude, artist_longitude
+        FROM staging_songs
+    """)
+
+    time_table_insert = ("""
+        SELECT start_time, extract(hour from start_time), extract(day from start_time), extract(week from start_time), 
+               extract(month from start_time), extract(year from start_time), extract(dayofweek from start_time)
+        FROM songplays
+    """)

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/operators/__init__.py

@@ -0,0 +1,11 @@
+from operators.stage_redshift import StageToRedshiftOperator
+from operators.load_fact import LoadFactOperator
+from operators.load_dimension import LoadDimensionOperator
+from operators.data_quality import DataQualityOperator
+
+__all__ = [
+    'StageToRedshiftOperator',
+    'LoadFactOperator',
+    'LoadDimensionOperator',
+    'DataQualityOperator'
+]

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/operators/data_quality.py

@@ -0,0 +1,22 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class DataQualityOperator(BaseOperator):
+
+    ui_color = '#89DA59'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # conn_id = your-connection-name
+                 *args, **kwargs):
+
+        super(DataQualityOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('DataQualityOperator not implemented yet')

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/operators/load_dimension.py

@@ -0,0 +1,22 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class LoadDimensionOperator(BaseOperator):
+
+    ui_color = '#80BD9E'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # conn_id = your-connection-name
+                 *args, **kwargs):
+
+        super(LoadDimensionOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('LoadDimensionOperator not implemented yet')

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/operators/load_fact.py

@@ -0,0 +1,22 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class LoadFactOperator(BaseOperator):
+
+    ui_color = '#F98866'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # conn_id = your-connection-name
+                 *args, **kwargs):
+
+        super(LoadFactOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('LoadFactOperator not implemented yet')

My-Data-Engineering-Portifolio/Automate+Pipelines/plugins/operators/stage_redshift.py

@@ -0,0 +1,26 @@
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+class StageToRedshiftOperator(BaseOperator):
+    ui_color = '#358140'
+
+    @apply_defaults
+    def __init__(self,
+                 # Define your operators params (with defaults) here
+                 # Example:
+                 # redshift_conn_id=your-connection-name
+                 *args, **kwargs):
+
+        super(StageToRedshiftOperator, self).__init__(*args, **kwargs)
+        # Map params here
+        # Example:
+        # self.conn_id = conn_id
+
+    def execute(self, context):
+        self.log.info('StageToRedshiftOperator not implemented yet')
+
+
+
+
+