refactor: restructure monorepo for clean portfolio layout
- Move timesfm-forecast into apps/ directory - Flatten Udacity portfolio projects from deep URL-encoded paths into data-engineering/01-XX numbered directories - Remove old My-Data-Engineering-Portifolio/ parent directory - Rewrite root README.md: professional overview with badges, project table, and repo structure diagram - Create data-engineering/README.md with per-project descriptions - Add README.md for 02-cassandra-modeling (was missing) - Add README.md for 05-airflow-pipelines (was missing) - Normalize capstone readme.md -> README.md - Update .gitignore: add *.cfg, *.env, *.zip, *.sas7bdat, Jupyter checkpoints, IDE dirs; remove uv.lock exclusion - Add dwh.cfg.example and dl.cfg.example credential templates - Untrack real credential files (dwh.cfg, dl.cfg) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@gabriel.pereira
16
apps/timesfm-forecast/src/timesfm_app/__init__.py
Executable file
16
apps/timesfm-forecast/src/timesfm_app/__init__.py
Executable file
@@ -0,0 +1,16 @@
|
||||
|
||||
__all__ = [
|
||||
"ForecastConfig",
|
||||
"TimesFMService",
|
||||
"ForecastPipeline",
|
||||
"DataValidator",
|
||||
"FrequencyHelper",
|
||||
]
|
||||
|
||||
from .core import (
|
||||
ForecastConfig,
|
||||
TimesFMService,
|
||||
ForecastPipeline,
|
||||
DataValidator,
|
||||
FrequencyHelper,
|
||||
)
|
||||
13
apps/timesfm-forecast/src/timesfm_app/cli.py
Executable file
13
apps/timesfm-forecast/src/timesfm_app/cli.py
Executable file
@@ -0,0 +1,13 @@
|
||||
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
def main() -> int:
|
||||
"""
|
||||
Launch the Streamlit UI via CLI:
|
||||
$ timesfm-app
|
||||
"""
|
||||
app_path = pathlib.Path(__file__).with_name("ui").joinpath("app.py")
|
||||
cmd = [sys.executable, "-m", "streamlit", "run", str(app_path)]
|
||||
return subprocess.call(cmd)
|
||||
198
apps/timesfm-forecast/src/timesfm_app/core.py
Executable file
198
apps/timesfm-forecast/src/timesfm_app/core.py
Executable file
@@ -0,0 +1,198 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, List, Optional, Tuple
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from transformers import TimesFmModelForPrediction
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ForecastConfig:
|
||||
model_name: str = "google/timesfm-2.0-500m-pytorch"
|
||||
device: str = "cpu" # "cpu" or "cuda"
|
||||
batch_size: int = 16
|
||||
timesfm_freq_code: int = 2 # 2 = Monthly (as per your draft)
|
||||
pandas_freq_fallback: str = "M" # for building future index
|
||||
horizon_min: int = 1
|
||||
horizon_max: int = 12 # inclusive
|
||||
|
||||
def validate(self) -> None:
|
||||
if self.horizon_min < 1:
|
||||
raise ValueError("horizon_min must be >= 1")
|
||||
if self.horizon_max < self.horizon_min:
|
||||
raise ValueError("horizon_max must be >= horizon_min")
|
||||
if self.batch_size < 1:
|
||||
raise ValueError("batch_size must be >= 1")
|
||||
|
||||
|
||||
class DataValidator:
|
||||
REQUIRED = {"date", "value", "key"}
|
||||
|
||||
@staticmethod
|
||||
def standardize_columns(df: pd.DataFrame) -> pd.DataFrame:
|
||||
out = df.copy()
|
||||
out.columns = out.columns.str.lower()
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def validate_schema(df: pd.DataFrame) -> None:
|
||||
cols = set(df.columns)
|
||||
if not DataValidator.REQUIRED.issubset(cols):
|
||||
raise ValueError(
|
||||
f"CSV must contain columns {sorted(DataValidator.REQUIRED)}, "
|
||||
f"found {sorted(cols)}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def parse_dates(df: pd.DataFrame) -> pd.DataFrame:
|
||||
out = df.copy()
|
||||
sample = str(out["date"].iloc[0])
|
||||
if "/" in sample and sample.count("/") == 2:
|
||||
out["date"] = pd.to_datetime(out["date"], dayfirst=True, errors="coerce")
|
||||
else:
|
||||
out["date"] = pd.to_datetime(out["date"], dayfirst=False, errors="coerce")
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def sanitize(df: pd.DataFrame) -> pd.DataFrame:
|
||||
out = df.copy()
|
||||
out = out.dropna(subset=["date", "value", "key"])
|
||||
out["value"] = pd.to_numeric(out["value"], errors="coerce")
|
||||
out = out.dropna(subset=["value"])
|
||||
out = out.sort_values(["key", "date"]).reset_index(drop=True)
|
||||
return out
|
||||
|
||||
|
||||
class FrequencyHelper:
|
||||
@staticmethod
|
||||
def infer_pandas_freq(dates: pd.Series) -> str:
|
||||
if dates.is_monotonic_increasing and dates.notna().all():
|
||||
try:
|
||||
inferred = pd.infer_freq(pd.DatetimeIndex(dates))
|
||||
return inferred or ""
|
||||
except Exception:
|
||||
return ""
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def future_dates(
|
||||
last_date: pd.Timestamp, periods: int, pandas_freq: str
|
||||
) -> pd.DatetimeIndex:
|
||||
if periods <= 0:
|
||||
return pd.DatetimeIndex([])
|
||||
try:
|
||||
offset = pd.tseries.frequencies.to_offset(pandas_freq)
|
||||
except Exception:
|
||||
offset = pd.tseries.frequencies.to_offset("M") # safe fallback
|
||||
start = last_date + offset
|
||||
return pd.date_range(start=start, periods=periods, freq=offset)
|
||||
|
||||
|
||||
class TimesFMService:
|
||||
def __init__(self, model_name: str, device: str = "cpu"):
|
||||
self.model_name = model_name
|
||||
self.device = device
|
||||
self._model: Optional[TimesFmModelForPrediction] = None
|
||||
|
||||
@property
|
||||
def model(self) -> TimesFmModelForPrediction:
|
||||
if self._model is None:
|
||||
self._model = TimesFmModelForPrediction.from_pretrained(
|
||||
self.model_name,
|
||||
dtype=torch.float32,
|
||||
device_map=self.device, # "cpu" or "cuda"
|
||||
)
|
||||
return self._model
|
||||
|
||||
@torch.no_grad()
|
||||
def forecast_batch(
|
||||
self, past_values: List[torch.Tensor], freq_code: int, max_h: int
|
||||
) -> np.ndarray:
|
||||
if max_h < 1:
|
||||
raise ValueError("max_h must be >= 1")
|
||||
freq_tensor = torch.tensor([freq_code] * len(past_values), dtype=torch.long)
|
||||
out = self.model(
|
||||
past_values=past_values,
|
||||
freq=freq_tensor,
|
||||
return_dict=True,
|
||||
)
|
||||
preds = out.mean_predictions[:, :max_h].detach().cpu().numpy()
|
||||
return preds
|
||||
|
||||
|
||||
class ForecastPipeline:
|
||||
def __init__(self, svc: TimesFMService, config: ForecastConfig):
|
||||
config.validate()
|
||||
self.svc = svc
|
||||
self.cfg = config
|
||||
|
||||
def _chunked(self, seq: List, n: int) -> Iterable[List]:
|
||||
for i in range(0, len(seq), n):
|
||||
yield seq[i : i + n]
|
||||
|
||||
def run(
|
||||
self, raw_df: pd.DataFrame, progress_cb: Optional[callable] = None
|
||||
) -> pd.DataFrame:
|
||||
df = DataValidator.standardize_columns(raw_df)
|
||||
DataValidator.validate_schema(df)
|
||||
df = DataValidator.parse_dates(df)
|
||||
df = DataValidator.sanitize(df)
|
||||
|
||||
keys = sorted(df["key"].unique().tolist())
|
||||
n_keys = len(keys)
|
||||
if n_keys == 0:
|
||||
return pd.DataFrame(columns=["key", "date", "step", "forecast"])
|
||||
|
||||
series_payload: List[torch.Tensor] = []
|
||||
meta: List[Tuple[str, pd.Timestamp, str]] = []
|
||||
|
||||
for k in keys:
|
||||
sub = df[df["key"] == k]
|
||||
values = sub["value"].astype(float).values
|
||||
series_payload.append(torch.tensor(values, dtype=torch.float32))
|
||||
last_dt = sub["date"].iloc[-1]
|
||||
inferred = FrequencyHelper.infer_pandas_freq(sub["date"])
|
||||
meta.append((k, last_dt, inferred))
|
||||
|
||||
bsz = self.cfg.batch_size
|
||||
total_batches = math.ceil(n_keys / bsz)
|
||||
results: List[pd.DataFrame] = []
|
||||
|
||||
horizon_min, horizon_max = self.cfg.horizon_min, self.cfg.horizon_max
|
||||
|
||||
for batch_idx, (payload_chunk, meta_chunk) in enumerate(
|
||||
zip(self._chunked(series_payload, bsz), self._chunked(meta, bsz)), start=1
|
||||
):
|
||||
if progress_cb:
|
||||
progress_cb(batch_idx, total_batches, "Forecasting…")
|
||||
|
||||
preds = self.svc.forecast_batch(
|
||||
past_values=payload_chunk,
|
||||
freq_code=self.cfg.timesfm_freq_code,
|
||||
max_h=horizon_max,
|
||||
)
|
||||
|
||||
for i, (k, last_dt, inferred) in enumerate(meta_chunk):
|
||||
pfreq = inferred or self.cfg.pandas_freq_fallback
|
||||
future_idx = FrequencyHelper.future_dates(last_dt, horizon_max, pfreq)
|
||||
|
||||
steps = np.arange(horizon_min, horizon_max + 1, dtype=int)
|
||||
dates = future_idx[steps - 1]
|
||||
fc_vals = preds[i, steps - 1]
|
||||
|
||||
res = pd.DataFrame(
|
||||
{
|
||||
"key": k,
|
||||
"date": dates,
|
||||
"step": steps,
|
||||
"forecast": fc_vals,
|
||||
}
|
||||
)
|
||||
results.append(res)
|
||||
|
||||
out_df = pd.concat(results, ignore_index=True)
|
||||
return out_df
|
||||
178
apps/timesfm-forecast/src/timesfm_app/ui/app.py
Executable file
178
apps/timesfm-forecast/src/timesfm_app/ui/app.py
Executable file
@@ -0,0 +1,178 @@
|
||||
import io
|
||||
|
||||
import pandas as pd
|
||||
import streamlit as st
|
||||
import torch
|
||||
|
||||
from timesfm_app.core import (
|
||||
ForecastConfig,
|
||||
TimesFMService,
|
||||
ForecastPipeline,
|
||||
DataValidator,
|
||||
)
|
||||
|
||||
st.set_page_config(page_title="TimesFM Forecast", layout="wide")
|
||||
|
||||
|
||||
def horizon_range_slider(label: str, default_min: int = 1, default_max: int = 12):
|
||||
return st.slider(
|
||||
label,
|
||||
min_value=1,
|
||||
max_value=120,
|
||||
value=(default_min, default_max),
|
||||
step=1,
|
||||
help="Select an inclusive range of horizons. The app outputs each step separately.",
|
||||
)
|
||||
|
||||
|
||||
@st.cache_resource(show_spinner="Preparing TimesFM service…")
|
||||
def get_service(model_name: str, device: str) -> TimesFMService:
|
||||
return TimesFMService(model_name=model_name, device=device)
|
||||
|
||||
|
||||
def main():
|
||||
st.title("📈 TimesFM Forecast")
|
||||
|
||||
st.markdown(
|
||||
"""
|
||||
Upload a CSV with columns **`date, value, key`**.
|
||||
|
||||
**Features**
|
||||
- Uses **all rows** as training (no holdout)
|
||||
- Choose a **range of horizons** (e.g., 1–12)
|
||||
- Multi-series (**by key**) batching
|
||||
- **Progress bar** during inference
|
||||
- **Download** forecasts as CSV
|
||||
"""
|
||||
)
|
||||
|
||||
with st.sidebar:
|
||||
st.header("Settings")
|
||||
|
||||
model_name = st.text_input(
|
||||
"TimesFM model",
|
||||
value="google/timesfm-2.0-500m-pytorch",
|
||||
)
|
||||
|
||||
cuda_available = torch.cuda.is_available()
|
||||
device = st.selectbox(
|
||||
"Device",
|
||||
options=(["cuda", "cpu"] if cuda_available else ["cpu"]),
|
||||
index=(0 if cuda_available else 0),
|
||||
)
|
||||
|
||||
timesfm_freq_code = st.number_input(
|
||||
"TimesFM frequency code",
|
||||
min_value=0,
|
||||
max_value=99,
|
||||
value=2, # monthly by default
|
||||
step=1,
|
||||
help="Default 2 = monthly (matches your draft).",
|
||||
)
|
||||
|
||||
pandas_freq_fallback = st.selectbox(
|
||||
"Fallback pandas frequency for future dates",
|
||||
options=["D", "W", "M", "Q", "Y"],
|
||||
index=2, # "M"
|
||||
help="Used when per-key frequency inference fails.",
|
||||
)
|
||||
|
||||
batch_size = st.slider("Batch size", 1, 64, 16, 1)
|
||||
h_min, h_max = horizon_range_slider("Horizon range", 1, 12)
|
||||
|
||||
uploaded = st.file_uploader("Upload CSV", type=["csv"])
|
||||
|
||||
if "forecast" not in st.session_state:
|
||||
st.session_state["forecast"] = None
|
||||
if "history" not in st.session_state:
|
||||
st.session_state["history"] = None
|
||||
|
||||
if uploaded is None:
|
||||
st.info("👆 Upload a CSV to begin.")
|
||||
return
|
||||
|
||||
raw_df = pd.read_csv(uploaded)
|
||||
st.dataframe(raw_df.head(10), use_container_width=True)
|
||||
|
||||
run = st.button("▶️ Run forecast", type="primary")
|
||||
|
||||
if run:
|
||||
cfg = ForecastConfig(
|
||||
model_name=model_name,
|
||||
device=device,
|
||||
batch_size=batch_size,
|
||||
timesfm_freq_code=timesfm_freq_code,
|
||||
pandas_freq_fallback=pandas_freq_fallback,
|
||||
horizon_min=h_min,
|
||||
horizon_max=h_max,
|
||||
)
|
||||
|
||||
svc = get_service(model_name=cfg.model_name, device=cfg.device)
|
||||
pipe = ForecastPipeline(svc=svc, config=cfg)
|
||||
|
||||
progress = st.progress(0.0)
|
||||
info = st.empty()
|
||||
|
||||
def progress_cb(batch_idx: int, total_batches: int, message: str):
|
||||
pct = min(1.0, batch_idx / max(1, total_batches))
|
||||
progress.progress(pct)
|
||||
info.write(f"{message} ({batch_idx}/{total_batches})")
|
||||
|
||||
st.info("Running TimesFM… This may take a while for large files.")
|
||||
|
||||
try:
|
||||
forecast_df = pipe.run(raw_df, progress_cb=progress_cb)
|
||||
finally:
|
||||
info.empty()
|
||||
progress.empty()
|
||||
|
||||
hist = DataValidator.standardize_columns(raw_df)
|
||||
hist = DataValidator.parse_dates(hist)
|
||||
hist = DataValidator.sanitize(hist)
|
||||
|
||||
st.session_state["forecast"] = forecast_df
|
||||
st.session_state["history"] = hist
|
||||
|
||||
st.success("✅ Forecast complete!")
|
||||
|
||||
forecast_df = st.session_state["forecast"]
|
||||
hist = st.session_state["history"]
|
||||
|
||||
if forecast_df is None or hist is None:
|
||||
return
|
||||
|
||||
st.subheader("Forecast preview (long format)")
|
||||
st.dataframe(forecast_df.head(20), use_container_width=True)
|
||||
|
||||
st.subheader("Visualize one key")
|
||||
keys_sorted = sorted(hist["key"].unique().tolist())
|
||||
sel_key = st.selectbox("Select key", options=keys_sorted)
|
||||
|
||||
hist_k = hist[hist["key"] == sel_key][["date", "value"]].rename(
|
||||
columns={"value": "y"}
|
||||
)
|
||||
hist_k = hist_k.assign(series="history")
|
||||
|
||||
fcast_k = forecast_df[forecast_df["key"] == sel_key][["date", "forecast"]]
|
||||
fcast_k = fcast_k.rename(columns={"forecast": "y"}).assign(series="forecast")
|
||||
|
||||
display_df = pd.concat([hist_k, fcast_k], ignore_index=True).sort_values("date")
|
||||
st.line_chart(data=display_df, x="date", y="y", color="series", height=340)
|
||||
|
||||
st.subheader("Download")
|
||||
buf = io.StringIO()
|
||||
forecast_df.to_csv(buf, index=False)
|
||||
st.download_button(
|
||||
label="💾 Download forecasts (CSV)",
|
||||
data=buf.getvalue().encode("utf-8"),
|
||||
file_name="timesfm_forecasts_long.csv",
|
||||
mime="text/csv",
|
||||
)
|
||||
|
||||
st.caption(
|
||||
"Output columns: key, date, step, forecast. Each 'step' is the horizon (1..H)."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user