217 lines
6.1 KiB
Plaintext
217 lines
6.1 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import configparser\n",
|
|
"import os\n",
|
|
"from pathlib import Path\n",
|
|
"from pyspark.sql import SparkSession"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# CONFIG\n",
|
|
"config = configparser.ConfigParser()\n",
|
|
"config.read('dl.cfg')\n",
|
|
"\n",
|
|
"KEY = config.get('AWS', 'AWS_ACCESS_KEY_ID')\n",
|
|
"SECRET = config.get('AWS', 'AWS_SECRET_ACCESS_KEY')\n",
|
|
"output_data = './data/outputs'# config.get('S3', 'DEST_S3_BUCKET')\n",
|
|
"\n",
|
|
"\n",
|
|
"os.environ['AWS_ACCESS_KEY_ID']=KEY\n",
|
|
"os.environ['AWS_SECRET_ACCESS_KEY']=SECRET"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"spark = SparkSession.builder\\\n",
|
|
" .config(\"spark.jars.packages\", \"org.apache.hadoop:hadoop-aws:2.7.0\")\\\n",
|
|
" .enableHiveSupport().getOrCreate()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"s3_bucket = Path(output_data)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Table: city_code\n",
|
|
"root\n",
|
|
" |-- city_code: string (nullable = true)\n",
|
|
" |-- city: string (nullable = true)\n",
|
|
"\n",
|
|
"Table: d_temperature\n",
|
|
"root\n",
|
|
" |-- dt: date (nullable = true)\n",
|
|
" |-- avg_temp: string (nullable = true)\n",
|
|
" |-- avg_temp_uncertnty: string (nullable = true)\n",
|
|
" |-- city: string (nullable = true)\n",
|
|
" |-- country: string (nullable = true)\n",
|
|
" |-- year: integer (nullable = true)\n",
|
|
" |-- month: integer (nullable = true)\n",
|
|
"\n",
|
|
"Table: d_citizen\n",
|
|
"root\n",
|
|
" |-- cic_id: double (nullable = true)\n",
|
|
" |-- citizen_country: double (nullable = true)\n",
|
|
" |-- residence_country: double (nullable = true)\n",
|
|
" |-- birth_year: double (nullable = true)\n",
|
|
" |-- gender: string (nullable = true)\n",
|
|
" |-- ins_num: string (nullable = true)\n",
|
|
" |-- immi_citizen_id: long (nullable = true)\n",
|
|
"\n",
|
|
"Table: d_airline\n",
|
|
"root\n",
|
|
" |-- cic_id: double (nullable = true)\n",
|
|
" |-- airline: string (nullable = true)\n",
|
|
" |-- admin_num: double (nullable = true)\n",
|
|
" |-- flight_number: string (nullable = true)\n",
|
|
" |-- visa_type: string (nullable = true)\n",
|
|
" |-- immi_airline_id: long (nullable = true)\n",
|
|
"\n",
|
|
"Table: f_immigration\n",
|
|
"root\n",
|
|
" |-- cic_id: double (nullable = true)\n",
|
|
" |-- year: double (nullable = true)\n",
|
|
" |-- month: double (nullable = true)\n",
|
|
" |-- city_code: string (nullable = true)\n",
|
|
" |-- arrive_date: date (nullable = true)\n",
|
|
" |-- departure_date: date (nullable = true)\n",
|
|
" |-- mode: double (nullable = true)\n",
|
|
" |-- visa: double (nullable = true)\n",
|
|
" |-- immigration_id: long (nullable = true)\n",
|
|
" |-- country: string (nullable = true)\n",
|
|
" |-- state_code: string (nullable = true)\n",
|
|
"\n",
|
|
"Table: d_demog_statistics\n",
|
|
"root\n",
|
|
" |-- city: string (nullable = true)\n",
|
|
" |-- state: string (nullable = true)\n",
|
|
" |-- median_age: string (nullable = true)\n",
|
|
" |-- avg_household_size: string (nullable = true)\n",
|
|
" |-- d_demog_statistics: long (nullable = true)\n",
|
|
"\n",
|
|
"Table: country_code\n",
|
|
"root\n",
|
|
" |-- country_code: string (nullable = true)\n",
|
|
" |-- country: string (nullable = true)\n",
|
|
"\n",
|
|
"Table: state_code\n",
|
|
"root\n",
|
|
" |-- state_code: string (nullable = true)\n",
|
|
" |-- state: string (nullable = true)\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for file_dir in s3_bucket.iterdir():\n",
|
|
" if file_dir.is_dir():\n",
|
|
" path = str(file_dir)\n",
|
|
" df = spark.read.parquet(path)\n",
|
|
" print(\"Table: \" + path.split('/')[-1])\n",
|
|
" schema = df.printSchema()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Table: city_code is not empty: total 660 records.\n",
|
|
"Table: d_temperature is not empty: total 687004 records.\n",
|
|
"Table: d_citizen is not empty: total 3096313 records.\n",
|
|
"Table: d_airline is not empty: total 3096313 records.\n",
|
|
"Table: f_immigration is not empty: total 3096313 records.\n",
|
|
"Table: d_demog_statistics is not empty: total 596 records.\n",
|
|
"Table: country_code is not empty: total 235 records.\n",
|
|
"Table: state_code is not empty: total 55 records.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for file_dir in s3_bucket.iterdir():\n",
|
|
" if file_dir.is_dir():\n",
|
|
" path = str(file_dir)\n",
|
|
" df = spark.read.parquet(path)\n",
|
|
" record_num = df.count()\n",
|
|
" if record_num <= 0:\n",
|
|
" raise ValueError(\"This table is empty!\")\n",
|
|
" else:\n",
|
|
" print(\"Table: \" + path.split('/')[-1] + f\" is not empty: total {record_num} records.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"editable": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|