refactor: restructure monorepo for clean portfolio layout

- Move timesfm-forecast into apps/ directory
- Flatten Udacity portfolio projects from deep URL-encoded paths
  into data-engineering/01-XX numbered directories
- Remove old My-Data-Engineering-Portifolio/ parent directory
- Rewrite root README.md: professional overview with badges,
  project table, and repo structure diagram
- Create data-engineering/README.md with per-project descriptions
- Add README.md for 02-cassandra-modeling (was missing)
- Add README.md for 05-airflow-pipelines (was missing)
- Normalize capstone readme.md -> README.md
- Update .gitignore: add *.cfg, *.env, *.zip, *.sas7bdat,
  Jupyter checkpoints, IDE dirs; remove uv.lock exclusion
- Add dwh.cfg.example and dl.cfg.example credential templates
- Untrack real credential files (dwh.cfg, dl.cfg)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@gabriel.pereira
2026-03-26 16:48:50 -03:00
parent 5c4e6075e1
commit 6796398924
160 changed files with 308 additions and 34 deletions

View File

@@ -0,0 +1,178 @@
import io
import pandas as pd
import streamlit as st
import torch
from timesfm_app.core import (
ForecastConfig,
TimesFMService,
ForecastPipeline,
DataValidator,
)
st.set_page_config(page_title="TimesFM Forecast", layout="wide")
def horizon_range_slider(label: str, default_min: int = 1, default_max: int = 12):
return st.slider(
label,
min_value=1,
max_value=120,
value=(default_min, default_max),
step=1,
help="Select an inclusive range of horizons. The app outputs each step separately.",
)
@st.cache_resource(show_spinner="Preparing TimesFM service…")
def get_service(model_name: str, device: str) -> TimesFMService:
return TimesFMService(model_name=model_name, device=device)
def main():
st.title("📈 TimesFM Forecast")
st.markdown(
"""
Upload a CSV with columns **`date, value, key`**.
**Features**
- Uses **all rows** as training (no holdout)
- Choose a **range of horizons** (e.g., 112)
- Multi-series (**by key**) batching
- **Progress bar** during inference
- **Download** forecasts as CSV
"""
)
with st.sidebar:
st.header("Settings")
model_name = st.text_input(
"TimesFM model",
value="google/timesfm-2.0-500m-pytorch",
)
cuda_available = torch.cuda.is_available()
device = st.selectbox(
"Device",
options=(["cuda", "cpu"] if cuda_available else ["cpu"]),
index=(0 if cuda_available else 0),
)
timesfm_freq_code = st.number_input(
"TimesFM frequency code",
min_value=0,
max_value=99,
value=2, # monthly by default
step=1,
help="Default 2 = monthly (matches your draft).",
)
pandas_freq_fallback = st.selectbox(
"Fallback pandas frequency for future dates",
options=["D", "W", "M", "Q", "Y"],
index=2, # "M"
help="Used when per-key frequency inference fails.",
)
batch_size = st.slider("Batch size", 1, 64, 16, 1)
h_min, h_max = horizon_range_slider("Horizon range", 1, 12)
uploaded = st.file_uploader("Upload CSV", type=["csv"])
if "forecast" not in st.session_state:
st.session_state["forecast"] = None
if "history" not in st.session_state:
st.session_state["history"] = None
if uploaded is None:
st.info("👆 Upload a CSV to begin.")
return
raw_df = pd.read_csv(uploaded)
st.dataframe(raw_df.head(10), use_container_width=True)
run = st.button("▶️ Run forecast", type="primary")
if run:
cfg = ForecastConfig(
model_name=model_name,
device=device,
batch_size=batch_size,
timesfm_freq_code=timesfm_freq_code,
pandas_freq_fallback=pandas_freq_fallback,
horizon_min=h_min,
horizon_max=h_max,
)
svc = get_service(model_name=cfg.model_name, device=cfg.device)
pipe = ForecastPipeline(svc=svc, config=cfg)
progress = st.progress(0.0)
info = st.empty()
def progress_cb(batch_idx: int, total_batches: int, message: str):
pct = min(1.0, batch_idx / max(1, total_batches))
progress.progress(pct)
info.write(f"{message} ({batch_idx}/{total_batches})")
st.info("Running TimesFM… This may take a while for large files.")
try:
forecast_df = pipe.run(raw_df, progress_cb=progress_cb)
finally:
info.empty()
progress.empty()
hist = DataValidator.standardize_columns(raw_df)
hist = DataValidator.parse_dates(hist)
hist = DataValidator.sanitize(hist)
st.session_state["forecast"] = forecast_df
st.session_state["history"] = hist
st.success("✅ Forecast complete!")
forecast_df = st.session_state["forecast"]
hist = st.session_state["history"]
if forecast_df is None or hist is None:
return
st.subheader("Forecast preview (long format)")
st.dataframe(forecast_df.head(20), use_container_width=True)
st.subheader("Visualize one key")
keys_sorted = sorted(hist["key"].unique().tolist())
sel_key = st.selectbox("Select key", options=keys_sorted)
hist_k = hist[hist["key"] == sel_key][["date", "value"]].rename(
columns={"value": "y"}
)
hist_k = hist_k.assign(series="history")
fcast_k = forecast_df[forecast_df["key"] == sel_key][["date", "forecast"]]
fcast_k = fcast_k.rename(columns={"forecast": "y"}).assign(series="forecast")
display_df = pd.concat([hist_k, fcast_k], ignore_index=True).sort_values("date")
st.line_chart(data=display_df, x="date", y="y", color="series", height=340)
st.subheader("Download")
buf = io.StringIO()
forecast_df.to_csv(buf, index=False)
st.download_button(
label="💾 Download forecasts (CSV)",
data=buf.getvalue().encode("utf-8"),
file_name="timesfm_forecasts_long.csv",
mime="text/csv",
)
st.caption(
"Output columns: key, date, step, forecast. Each 'step' is the horizon (1..H)."
)
if __name__ == "__main__":
main()