# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os

# Input data files (if any are attached) are available in the read-only "../input/" directory
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to /kaggle/working/, which is preserved as output
# when you create a version using "Save & Run All"

# scikit-learn, pandas, numpy, matplotlib, and joblib are already installed.
# optuna, autogluon.tabular, wandb, and trackio are NOT preinstalled on Kaggle
# - each is installed right where it's first used, not here, so you only pay
# the install cost for the sections you actually run.

import json, time, os, platform, subprocess
from pathlib import Path
from datetime import datetime

RUNS_DIR = Path("/kaggle/working/runs"); RUNS_DIR.mkdir(parents=True, exist_ok=True)

from sklearn.datasets import load_breast_cancer

data = load_breast_cancer(as_frame=True)
df = data.frame.copy()
df["diagnosis"] = df["target"].map({0: "malignant", 1: "benign"})
df = df.drop(columns=["target"])
print(df.shape)
df.head()

(569, 31)

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

X = df[data.feature_names]
y = df["diagnosis"]

# Train the SAME model three times, with no seed set anywhere
for i in range(3):
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2)
    m = RandomForestClassifier(n_estimators=50).fit(Xtr, ytr)
    print(f"Run {i+1}: accuracy = {m.score(Xte, yte):.4f}")

Run 1: accuracy = 0.9474
Run 2: accuracy = 0.9649
Run 3: accuracy = 0.9561

import random

def set_seed(seed=42):
    """Set the seed for every library that might use randomness."""
    random.seed(seed)                    # Python's own random
    np.random.seed(seed)                 # NumPy
    os.environ["PYTHONHASHSEED"] = str(seed)
    try:
        import torch
        torch.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    except Exception:
        # Not installed, or installed but broken (e.g. a numpy/torch binary
        # ABI mismatch on this particular environment) - this lab never
        # actually needs torch, so skip it either way rather than crash.
        pass
    print(f"All seeds set to {seed}")

set_seed(42)

All seeds set to 42

# TODO

# 1. Rerun the Section 3 experiment with set_seed(42) + random_state=42
#    everywhere, three times, and confirm the accuracies are identical.

# 2. Rerun with seeds 0, 1, 2, 3, 4. Print the five accuracies and their
#    mean and standard deviation.

def run_experiment(seed):
    set_seed(seed)
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, random_state=seed)
    m = RandomForestClassifier(n_estimators=50, random_state=seed).fit(Xtr, ytr)
    return m.score(Xte, yte)

# 1. Same seed three times -> identical accuracy
for i in range(3):
    print(f"Run {i+1}: accuracy = {run_experiment(42):.4f}")

# 2. Five different seeds -> mean and std
accuracies = np.array([run_experiment(s) for s in [0, 1, 2, 3, 4]])
print("Accuracies:", accuracies)
print(f"Mean = {accuracies.mean():.4f}, Std = {accuracies.std():.4f}")
# Std is roughly 2 accuracy points here (the 20% test split is only ~114 rows,
# so a handful of flipped predictions swings the score a lot) - a 96% vs 97%
# result between two models is well within that noise band.

All seeds set to 42
Run 1: accuracy = 0.9649
All seeds set to 42
Run 2: accuracy = 0.9649
All seeds set to 42
Run 3: accuracy = 0.9649
All seeds set to 0
All seeds set to 1
All seeds set to 2
All seeds set to 3
All seeds set to 4
Accuracies: [0.97368421 0.95614035 0.94736842 0.93859649 0.9122807 ]
Mean = 0.9456, Std = 0.0203

# TODO (Extension) - which source of randomness matters most?

# 1. Split only: fix the model's random_state=42, vary the split 15 times.

# 2. Model only: fix the split's random_state=42, vary the model 15 times.

# 3. Both unseeded, 15 times.

# Print a table: source | mean | std

N = 15

split_scores = []
for _ in range(N):
    Xtr_, Xte_, ytr_, yte_ = train_test_split(X, y, test_size=0.2)  # unseeded split
    m = RandomForestClassifier(n_estimators=50, random_state=42).fit(Xtr_, ytr_)
    split_scores.append(m.score(Xte_, yte_))

Xtr_fixed, Xte_fixed, ytr_fixed, yte_fixed = train_test_split(X, y, test_size=0.2, random_state=42)
model_scores = []
for _ in range(N):
    m = RandomForestClassifier(n_estimators=50).fit(Xtr_fixed, ytr_fixed)  # unseeded model
    model_scores.append(m.score(Xte_fixed, yte_fixed))

both_scores = []
for _ in range(N):
    Xtr_, Xte_, ytr_, yte_ = train_test_split(X, y, test_size=0.2)
    m = RandomForestClassifier(n_estimators=50).fit(Xtr_, ytr_)
    both_scores.append(m.score(Xte_, yte_))

print(f"{'Source':<15} {'Mean':>8} {'Std':>8}")
print("-" * 33)
for name, scores in [("Split only", split_scores), ("Model only", model_scores), ("Both", both_scores)]:
    print(f"{name:<15} {np.mean(scores):>8.4f} {np.std(scores):>8.4f}")
# On this dataset, the split usually contributes more variance than the model:
# RandomForest's own internal randomness averages out over 50 trees, but which
# ~114 rows land in the test set changes the score directly.

Source              Mean      Std
---------------------------------
Split only        0.9526   0.0146
Model only        0.9643   0.0039
Both              0.9637   0.0132

# TODO (Extension) - the KFold shuffle trap: KFold(shuffle=True) without a
# random_state reshuffles differently every call, even inside the same script.

# 1. KFold(shuffle=True), no random_state, 5 repeats - print the scores.

# 2. Same but with random_state=42 - confirm the scores are now identical.

from sklearn.model_selection import KFold, cross_val_score

model = RandomForestClassifier(n_estimators=50, random_state=42)

# 1. WRONG - shuffles differently every call
scores_unseeded = []
for _ in range(5):
    kf = KFold(n_splits=5, shuffle=True)  # no random_state!
    scores_unseeded.append(cross_val_score(model, X, y, cv=kf).mean())
print("Unseeded CV scores:", [f"{s:.4f}" for s in scores_unseeded])

# 2. Fixed
scores_seeded = []
for _ in range(5):
    kf = KFold(n_splits=5, shuffle=True, random_state=42)
    scores_seeded.append(cross_val_score(model, X, y, cv=kf).mean())
print("Seeded CV scores:  ", [f"{s:.4f}" for s in scores_seeded])

Unseeded CV scores: ['0.9666', '0.9578', '0.9578', '0.9667', '0.9543']
Seeded CV scores:   ['0.9578', '0.9578', '0.9578', '0.9578', '0.9578']

import sklearn
print("Python :", platform.python_version())
print("OS     :", platform.system(), platform.release())
print("NumPy  :", np.__version__)
print("Pandas :", pd.__version__)
print("Sklearn:", sklearn.__version__)

Python : 3.12.13
OS     : Linux 6.12.90+
NumPy  : 2.0.2
Pandas : 2.3.3
Sklearn: 1.6.1

# TODO

# 1. Print a requirements.txt (package==version) and an environment.yml
#    using the versions above.

# 2. Write capture_environment() -> dict with python/os/library versions,
#    git commit hash (or "not a git repo"), and a timestamp.
#    Tip: subprocess.run(["git","rev-parse","HEAD"], capture_output=True)

versions = {
    "python": platform.python_version(),
    "numpy": np.__version__,
    "pandas": pd.__version__,
    "scikit-learn": sklearn.__version__,
}

requirements_txt = "\n".join(f"{pkg}=={ver}" for pkg, ver in versions.items() if pkg != "python")
print("requirements.txt:\n" + requirements_txt)

environment_yml = f"""name: sttai-env
channels:
  - conda-forge
dependencies:
  - python={versions['python']}
  - numpy={versions['numpy']}
  - pandas={versions['pandas']}
  - scikit-learn={versions['scikit-learn']}
"""
print("\nenvironment.yml:\n" + environment_yml)

def capture_environment():
    try:
        commit = subprocess.run(["git", "rev-parse", "HEAD"],
                                 capture_output=True, text=True).stdout.strip()
    except Exception:
        commit = None
    return {
        "python": platform.python_version(),
        "os": f"{platform.system()} {platform.release()}",
        "numpy": np.__version__,
        "pandas": pd.__version__,
        "scikit-learn": sklearn.__version__,
        "git_commit": commit or "not a git repo",
        "timestamp": datetime.now().isoformat(timespec="seconds"),
    }

capture_environment()

requirements.txt:
numpy==2.0.2
pandas==2.3.3
scikit-learn==1.6.1

environment.yml:
name: sttai-env
channels:
  - conda-forge
dependencies:
  - python=3.12.13
  - numpy=2.0.2
  - pandas=2.3.3
  - scikit-learn=1.6.1

{'python': '3.12.13',
 'os': 'Linux 6.12.90+',
 'numpy': '2.0.2',
 'pandas': '2.3.3',
 'scikit-learn': '1.6.1',
 'git_commit': 'not a git repo',
 'timestamp': '2026-08-10T09:24:26'}

def log_run(name, hyperparams, metrics, notes=""):
    """Save one experiment as a JSON file in runs/."""
    run = {
        "name": name,
        "time": datetime.now().isoformat(timespec="seconds"),
        "hyperparams": hyperparams,
        "metrics": metrics,
        "environment": {"python": platform.python_version(),
                        "numpy": np.__version__},
        "notes": notes,
    }
    path = RUNS_DIR / f"{name}_{run['time'].replace(':','-')}.json"
    path.write_text(json.dumps(run, indent=2))
    return path

log_run("baseline_rf",
        {"n_estimators": 50, "max_depth": None, "seed": 42},
        {"val_accuracy": 0.94, "train_time_s": 0.4},
        notes="First try, no tuning")

PosixPath('/kaggle/working/runs/baseline_rf_2026-08-10T09-24-26.json')

# TODO

# 1. Train a RandomForestClassifier (n_estimators=100, max_depth=8,
#    min_samples_split=2, seed=42) and log it with log_run(), including
#    train accuracy, validation accuracy, training time, and
#    capture_environment().

set_seed(42)
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, random_state=42)

hyperparams = {"n_estimators": 100, "max_depth": 8, "min_samples_split": 2, "seed": 42}

t0 = time.time()
model = RandomForestClassifier(n_estimators=hyperparams["n_estimators"],
                               max_depth=hyperparams["max_depth"],
                               min_samples_split=hyperparams["min_samples_split"],
                               random_state=hyperparams["seed"]).fit(Xtr, ytr)
train_time = time.time() - t0

metrics = {
    "train_accuracy": model.score(Xtr, ytr),
    "val_accuracy": model.score(Xte, yte),
    "train_time_s": round(train_time, 3),
}

record = {
    "name": "rf_full_record",
    "time": datetime.now().isoformat(timespec="seconds"),
    "hyperparams": hyperparams,
    "metrics": metrics,
    "environment": capture_environment(),
}
path = RUNS_DIR / f"rf_full_record_{record['time'].replace(':','-')}.json"
path.write_text(json.dumps(record, indent=2))
print(path)

All seeds set to 42
/kaggle/working/runs/rf_full_record_2026-08-10T09-24-27.json

# TODO

# 2. Train a SGDClassifier for 20 rounds with partial_fit, logging
#    train/val accuracy each round. Plot both curves.

from sklearn.linear_model import SGDClassifier
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler().fit(Xtr)
Xtr_s, Xte_s = scaler.transform(Xtr), scaler.transform(Xte)
classes = np.unique(y)

sgd = SGDClassifier(random_state=42)
history = []
for round_ in range(1, 21):
    sgd.partial_fit(Xtr_s, ytr, classes=classes)
    history.append({
        "round": round_,
        "train_acc": sgd.score(Xtr_s, ytr),
        "val_acc": sgd.score(Xte_s, yte),
    })

hist_df = pd.DataFrame(history)
log_run("sgd_partial_fit", {"rounds": 20},
        {"final_val_acc": hist_df["val_acc"].iloc[-1]},
        notes=json.dumps(history))

hist_df.plot(x="round", y=["train_acc", "val_acc"])
plt.show()

# TODO

# 3. Write load_all_runs() -> DataFrame of every run in runs/, sorted by
#    val_accuracy (highest first).

def load_all_runs():
    rows = []
    for f in RUNS_DIR.glob("*.json"):
        run = json.loads(f.read_text())
        row = {"name": run["name"], "time": run["time"]}
        row.update(run.get("hyperparams", {}))
        row.update(run.get("metrics", {}))
        rows.append(row)
    return pd.DataFrame(rows)

runs_df = load_all_runs()
runs_df.sort_values("val_accuracy", ascending=False).head(3)

# TODO

# 4. Save one run's model, config, and metrics into its own folder instead
#    of one flat JSON file, e.g. runs/2026-08-09_rf-baseline_seed42/.

def make_run_dir(name, seed):
    stamp = datetime.now().strftime("%Y-%m-%d")
    run_dir = RUNS_DIR / f"{stamp}_{name}_seed{seed}"
    run_dir.mkdir(parents=True, exist_ok=True)
    return run_dir

run_dir = make_run_dir("rf-baseline", 42)
(run_dir / "config.json").write_text(json.dumps(hyperparams, indent=2))
(run_dir / "metrics.json").write_text(json.dumps(metrics, indent=2))

import joblib
joblib.dump(model, run_dir / "model.joblib")
print(run_dir)

/kaggle/working/runs/2026-08-10_rf-baseline_seed42

!pip install -q trackio

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.0/2.0 MB 38.8 MB/s eta 0:00:0000:01
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.7/61.7 kB 3.1 MB/s eta 0:00:00
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
gradio 5.50.0 requires gradio-client==1.14.0, but you have gradio-client 2.6.0 which is incompatible.

import trackio

trackio.init(project="stt-ai-breast-cancer", name="rf-baseline",
             config={"n_estimators": 100, "max_depth": 8, "seed": 42})

model = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42).fit(Xtr, ytr)

trackio.log({"train_acc": model.score(Xtr, ytr), "val_acc": model.score(Xte, yte)})
trackio.finish()
print("Run logged.")

* Trackio project initialized: stt-ai-breast-cancer
* Trackio metrics logged to: /root/.cache/huggingface/trackio
* psutil detected, enabling automatic CPU/system metrics logging
* Created new run: rf-baseline

* Run finished. Uploading logs to Trackio (please wait...)
Run logged.

# TODO

# 1. ONE TrackIO run (trackio.init once): train a GradientBoostingClassifier,
#    sweeping n_estimators from 10 to 100 in steps of 10. At each step, call
#    trackio.log({...}, step=n_estimators) with train_accuracy and
#    test_accuracy. trackio.finish() once at the end. This is the difference
#    between Section 6's "log the final number" and "log a curve".

from sklearn.ensemble import GradientBoostingClassifier

trackio.init(project="stt-ai-breast-cancer", name="gb-curve", config={"model": "gb"})
for n_est in range(10, 110, 10):
    gb = GradientBoostingClassifier(n_estimators=n_est, random_state=42).fit(Xtr, ytr)
    trackio.log({
        "train_accuracy": gb.score(Xtr, ytr),
        "test_accuracy": gb.score(Xte, yte),
    }, step=n_est)
trackio.finish()
print("Curve logged - open the dashboard (below) to see it plotted.")

* Created new run: gb-curve
* Run finished. Uploading logs to Trackio (please wait...)
Curve logged - open the dashboard (below) to see it plotted.

# TODO

# 2. Sweep max_depth 1-20 for a DecisionTreeClassifier, log the overfit gap,
#    alert if gap > 0.05, then plot train vs test accuracy vs depth.

from sklearn.tree import DecisionTreeClassifier

trackio.init(project="stt-ai-breast-cancer", name="dt-overfit-sweep", config={"model": "dt"})

depths, train_accs, test_accs, gaps = [], [], [], []
for depth in range(1, 21):
    dt = DecisionTreeClassifier(max_depth=depth, random_state=42).fit(Xtr, ytr)
    train_acc, test_acc = dt.score(Xtr, ytr), dt.score(Xte, yte)
    gap = train_acc - test_acc

    trackio.log({"max_depth": depth, "train_accuracy": train_acc,
                "test_accuracy": test_acc, "overfit_gap": gap}, step=depth)
    if gap > 0.05:
        trackio.alert(title="Overfitting detected!", text=f"gap={gap:.3f} at depth={depth}",
                      level=trackio.AlertLevel.ERROR)

    depths.append(depth); train_accs.append(train_acc); test_accs.append(test_acc); gaps.append(gap)

trackio.finish()

plt.plot(depths, train_accs, marker="o", label="train")
plt.plot(depths, test_accs, marker="o", label="test")
plt.xlabel("max_depth"); plt.ylabel("accuracy"); plt.legend()
plt.show()

* Created new run: dt-overfit-sweep
[TRACKIO ERROR] Overfitting detected!: gap=0.059 at depth=6 (step 6)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=7 (step 7)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=8 (step 8)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=9 (step 9)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=10 (step 10)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=11 (step 11)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=12 (step 12)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=13 (step 13)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=14 (step 14)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=15 (step 15)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=16 (step 16)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=17 (step 17)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=18 (step 18)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=19 (step 19)
[TRACKIO ERROR] Overfitting detected!: gap=0.053 at depth=20 (step 20)
* Run finished. Uploading logs to Trackio (please wait...)

# TODO

# 3. Log 5 classifiers (LogisticRegression, DecisionTreeClassifier,
#    RandomForestClassifier, GradientBoostingClassifier, and an SVM inside a
#    Pipeline with StandardScaler) as 5 SEPARATE TrackIO runs. For each, log
#    cv_accuracy (5-fold), test_accuracy, and fit_time_s. Print a leaderboard
#    DataFrame sorted by cv_accuracy - build it from values computed locally
#    in Python, not by reading them back from TrackIO (its API is for
#    browsing/managing runs, not a metrics database you query from code).

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score

model_zoo = {
    "LogisticRegression": LogisticRegression(max_iter=5000, random_state=42),
    "DecisionTree":       DecisionTreeClassifier(max_depth=5, random_state=42),
    "RandomForest":       RandomForestClassifier(n_estimators=100, random_state=42),
    "GradientBoosting":   GradientBoostingClassifier(n_estimators=100, random_state=42),
    "SVM":                Pipeline([("scaler", StandardScaler()), ("svm", SVC(random_state=42))]),
}

leaderboard = []
for name, clf in model_zoo.items():
    trackio.init(project="stt-ai-breast-cancer", name=f"leaderboard-{name}", config={"model": name})
    t0 = time.time()
    clf.fit(Xtr, ytr)
    fit_time = time.time() - t0
    cv_acc = cross_val_score(clf, Xtr, ytr, cv=5).mean()
    test_acc = clf.score(Xte, yte)
    trackio.log({"cv_accuracy": cv_acc, "test_accuracy": test_acc, "fit_time_s": fit_time})
    trackio.finish()
    leaderboard.append({"model": name, "cv_accuracy": cv_acc,
                        "test_accuracy": test_acc, "fit_time_s": fit_time})

pd.DataFrame(leaderboard).sort_values("cv_accuracy", ascending=False)

* Created new run: leaderboard-LogisticRegression
* Run finished. Uploading logs to Trackio (please wait...)
* Created new run: leaderboard-DecisionTree
* Run finished. Uploading logs to Trackio (please wait...)
* Created new run: leaderboard-RandomForest
* Run finished. Uploading logs to Trackio (please wait...)
* Created new run: leaderboard-GradientBoosting
* Run finished. Uploading logs to Trackio (please wait...)
* Created new run: leaderboard-SVM
* Run finished. Uploading logs to Trackio (please wait...)

trackio.show(project="stt-ai-breast-cancer", open_browser=False, block_thread=False)
print("Dashboard starting - open the printed URL above in your browser.")

Dashboard starting - open the printed URL above in your browser.

m = RandomForestClassifier(n_estimators=10, max_depth=4, random_state=0).fit(X, y)

# Hyperparameters: what YOU chose
print("Hyperparameters:", {k: m.get_params()[k]
                           for k in ["n_estimators", "max_depth", "criterion"]})

# Parameters: what TRAINING learned - here, the tree splits
total_nodes = sum(t.tree_.node_count for t in m.estimators_)
print("Learned parameters (total tree nodes):", total_nodes)

Hyperparameters: {'n_estimators': 10, 'max_depth': 4, 'criterion': 'gini'}
Learned parameters (total tree nodes): 228

# TODO

# 1. Sort each of these into "parameter" or "hyperparameter": learning rate,
#    weights of a neural network layer, number of epochs, the split value at
#    a decision tree node, batch size, dropout rate, the bias term, k in
#    k-nearest-neighbours, the centroids found by k-means.
parameters = ["..."]
hyperparameters = ["..."]

# 2. Train a DecisionTreeClassifier at max_depth = 1, 3, 5, 10, and None.
#    Plot train and validation accuracy against depth, and mark where
#    validation accuracy peaks and where overfitting begins.

parameters = [
    "weights of a neural network layer",
    "the split value at a decision tree node",
    "the bias term",
    "the centroids found by k-means",
]
hyperparameters = [
    "learning rate",
    "number of epochs",
    "batch size",
    "dropout rate",
    "k in k-nearest-neighbours",
]
print(parameters, hyperparameters)

from sklearn.tree import DecisionTreeClassifier

depths = [1, 3, 5, 10, None]
train_accs, val_accs = [], []
for d in depths:
    dt = DecisionTreeClassifier(max_depth=d, random_state=42).fit(Xtr, ytr)
    train_accs.append(dt.score(Xtr, ytr))
    val_accs.append(dt.score(Xte, yte))

depth_labels = [str(d) for d in depths]
plt.plot(depth_labels, train_accs, marker="o", label="train")
plt.plot(depth_labels, val_accs, marker="o", label="validation")
plt.xlabel("max_depth"); plt.ylabel("accuracy"); plt.legend()
plt.show()

best_depth = depths[int(np.argmax(val_accs))]
print("Best validation depth:", best_depth)

['weights of a neural network layer', 'the split value at a decision tree node', 'the bias term', 'the centroids found by k-means'] ['learning rate', 'number of epochs', 'batch size', 'dropout rate', 'k in k-nearest-neighbours']

Best validation depth: 3

from sklearn.model_selection import GridSearchCV, KFold

grid = {
    "n_estimators": [50, 100],
    "max_depth": [4, 8, None],
}
gs = GridSearchCV(RandomForestClassifier(random_state=42),
                  grid, cv=3, scoring="accuracy", n_jobs=-1)
gs.fit(Xtr, ytr)
print("Best settings:", gs.best_params_)
print("Best CV score:", round(gs.best_score_, 4))
print("Total fits:", len(gs.cv_results_["params"]) * 3)

Best settings: {'max_depth': 8, 'n_estimators': 50}
Best CV score: 0.9582
Total fits: 18

# TODO

# 1. Combinations, total fits, total time for 5x4x3x3 values, 5-fold CV.

# 2. Real grid search over n_estimators/max_depth: top 5 settings + heatmap.

# 3. Recompute the combination count for 8 hyperparameters, 3 values each.

# 1.
n_estimators_vals, max_depth_vals = 5, 4
min_samples_split_vals, max_features_vals = 3, 3
cv_folds = 5

combinations = n_estimators_vals * max_depth_vals * min_samples_split_vals * max_features_vals
total_fits = combinations * cv_folds
seconds_per_fit = 2
total_time_s = total_fits * seconds_per_fit

print(f"Combinations: {combinations}")
print(f"Total fits: {total_fits}")
print(f"Total time: {total_time_s} s = {total_time_s/60:.1f} min")

# 2.
grid2 = {"n_estimators": [50, 100, 150], "max_depth": [4, 8, 12, None]}
gs2 = GridSearchCV(RandomForestClassifier(random_state=42), grid2, cv=3,
                   scoring="accuracy", n_jobs=-1)
gs2.fit(Xtr, ytr)

results = pd.DataFrame(gs2.cv_results_)
print(results.sort_values("mean_test_score", ascending=False)[
    ["params", "mean_test_score"]].head(5))

pivot = results.pivot_table(values="mean_test_score",
                            index="param_max_depth", columns="param_n_estimators")
plt.imshow(pivot, cmap="viridis"); plt.colorbar()
plt.xticks(range(len(pivot.columns)), pivot.columns)
plt.yticks(range(len(pivot.index)), pivot.index)
plt.xlabel("n_estimators"); plt.ylabel("max_depth")
plt.show()

# 3. Curse of dimensionality: 8 hyperparameters, 3 values each
print("Combinations for 8 hyperparameters x 3 values:", 3 ** 8)
# Adding a hyperparameter multiplies the total; adding values to one existing
# hyperparameter only adds a linear factor - many hyperparameters is worse.

Combinations: 180
Total fits: 900
Total time: 1800 s = 30.0 min
                                      params  mean_test_score
3       {'max_depth': 8, 'n_estimators': 50}         0.958217
6      {'max_depth': 12, 'n_estimators': 50}         0.958217
9    {'max_depth': None, 'n_estimators': 50}         0.958217
0       {'max_depth': 4, 'n_estimators': 50}         0.956024
11  {'max_depth': None, 'n_estimators': 150}         0.951624

Combinations for 8 hyperparameters x 3 values: 6561

# TODO (Extension) - build MyGridSearchCV from scratch. GridSearchCV feels
# like magic until you build it yourself - fill in the three methods below,
# then test on breast cancer with a small DecisionTreeClassifier grid and
# compare against sklearn's GridSearchCV on the same grid.
import itertools

class MyGridSearchCV:
    def __init__(self, estimator, param_grid, cv=5, scoring="accuracy"):
        self.estimator = estimator
        self.param_grid = param_grid
        self.cv = cv
        self.scoring = scoring
        self.best_params_ = None
        self.best_score_ = -np.inf
        self.cv_results_ = []

    def _get_all_combinations(self):
        """Return a list of dicts, one per combination of self.param_grid."""
        # YOUR CODE HERE
        pass

    def _score_single_combination(self, params, X, y):
        """Run k-fold CV for one param combination using KFold directly.
        Return the mean accuracy across folds."""
        # YOUR CODE HERE
        pass

    def fit(self, X, y):
        """Run the grid search. Populate best_params_/best_score_/cv_results_."""
        # YOUR CODE HERE
        pass

    def get_results_df(self):
        return pd.DataFrame(self.cv_results_).sort_values("mean_test_score", ascending=False)


# Test on breast cancer + compare with sklearn's GridSearchCV

import itertools

class MyGridSearchCV:
    def __init__(self, estimator, param_grid, cv=5, scoring="accuracy"):
        self.estimator = estimator
        self.param_grid = param_grid
        self.cv = cv
        self.scoring = scoring
        self.best_params_ = None
        self.best_score_ = -np.inf
        self.cv_results_ = []

    def _get_all_combinations(self):
        keys = list(self.param_grid.keys())
        value_lists = list(self.param_grid.values())
        return [dict(zip(keys, combo)) for combo in itertools.product(*value_lists)]

    def _score_single_combination(self, params, X, y):
        est = self.estimator.__class__(**{**self.estimator.get_params(), **params})
        kf = KFold(n_splits=self.cv, shuffle=True, random_state=42)
        fold_scores = []
        for train_idx, val_idx in kf.split(X):
            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
            est.fit(X_train, y_train)
            fold_scores.append(est.score(X_val, y_val))
        return np.mean(fold_scores)

    def fit(self, X, y):
        for params in self._get_all_combinations():
            mean_score = self._score_single_combination(params, X, y)
            self.cv_results_.append({"params": params, "mean_test_score": mean_score})
            if mean_score > self.best_score_:
                self.best_score_ = mean_score
                self.best_params_ = params
        return self

    def get_results_df(self):
        return pd.DataFrame(self.cv_results_).sort_values("mean_test_score", ascending=False)


param_grid_dt = {"max_depth": [2, 4, 6, 8], "min_samples_leaf": [1, 2, 4]}

my_gs = MyGridSearchCV(DecisionTreeClassifier(random_state=42), param_grid_dt, cv=5)
my_gs.fit(Xtr, ytr)
print(f"[mine]    best={my_gs.best_score_:.4f}  params={my_gs.best_params_}")

# Pass sklearn the SAME KFold object mine uses internally, rather than a plain
# cv=5. Without this, sklearn silently swaps in StratifiedKFold for
# classifiers, which is a genuinely different split - not a bug in either
# implementation, just two different (valid) CV schemes that will disagree.
same_folds = KFold(n_splits=5, shuffle=True, random_state=42)
sk_gs = GridSearchCV(DecisionTreeClassifier(random_state=42), param_grid_dt, cv=same_folds)
sk_gs.fit(Xtr, ytr)
print(f"[sklearn] best={sk_gs.best_score_:.4f}  params={sk_gs.best_params_}")
# With identical folds, both should land on the same or a very close
# best_score_ - this is the real test that MyGridSearchCV's fold loop and
# scoring logic match sklearn's, not just that both "get a good score."

[mine]    best=0.9275  params={'max_depth': 4, 'min_samples_leaf': 1}
[sklearn] best=0.9275  params={'max_depth': 4, 'min_samples_leaf': 1}

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint

dist = {
    "n_estimators": randint(20, 300),
    "max_depth": randint(2, 20),
    "min_samples_split": randint(2, 20),
}
rs = RandomizedSearchCV(RandomForestClassifier(random_state=42),
                        dist, n_iter=20, cv=3, random_state=42, n_jobs=-1)
rs.fit(Xtr, ytr)
print("Best settings:", rs.best_params_)
print("Best CV score:", round(rs.best_score_, 4))

Best settings: {'max_depth': 10, 'min_samples_split': 3, 'n_estimators': 111}
Best CV score: 0.9582

# TODO

# 1. Grid search vs random search, both exactly 20 fits. Compare best score
#    and time in a small table.

# 2. Draw 2000 samples from np.random.uniform(0.0001, 0.1) and 2000 from
#    scipy.stats.loguniform(0.0001, 0.1). Print what share of each falls
#    below 0.001 - this is why log-uniform is the better choice for
#    something like a learning rate.

# 1.
from itertools import product

grid_values = {"n_estimators": [50, 100, 150, 200], "max_depth": [4, 8, 12, None, 16]}
combos = list(product(grid_values["n_estimators"], grid_values["max_depth"]))[:20]

t0 = time.time()
grid_scores = []
for n_est, depth in combos:
    m = RandomForestClassifier(n_estimators=n_est, max_depth=depth,
                               random_state=42).fit(Xtr, ytr)
    grid_scores.append(m.score(Xte, yte))
grid_time = time.time() - t0
grid_best = max(grid_scores)

t0 = time.time()
rs20 = RandomizedSearchCV(RandomForestClassifier(random_state=42), dist, n_iter=20,
                          cv=3, random_state=42, n_jobs=-1)
rs20.fit(Xtr, ytr)
random_time = time.time() - t0

print(pd.DataFrame({
    "method": ["Grid (20 fits)", "Random (20 iter x 3-fold CV)"],
    "best_score": [grid_best, rs20.best_score_],
    "time_s": [grid_time, random_time],
}))

# 2.
from scipy.stats import loguniform

uniform_samples = np.random.uniform(0.0001, 0.1, 2000)
log_samples = loguniform.rvs(0.0001, 0.1, size=2000)

print("Uniform draws below 0.001:", (uniform_samples < 0.001).mean())
print("Log-uniform draws below 0.001:", (log_samples < 0.001).mean())
# Plain uniform draws almost never land below 0.001, so small-but-important
# learning rates are barely ever tried; log-uniform spreads draws evenly
# across orders of magnitude.

                         method  best_score    time_s
0                Grid (20 fits)    0.964912  6.813295
1  Random (20 iter x 3-fold CV)    0.958203  9.715124
Uniform draws below 0.001: 0.005
Log-uniform draws below 0.001: 0.3365

# TODO (Extension) - build MyRandomSearchCV from scratch. Same interface as
# MyGridSearchCV, plus n_iter and random_state. Each param_distributions
# value is either a list (sample uniformly) or a scipy distribution (anything
# with .rvs(random_state=...), like randint(2, 20)). Fill in the two methods
# below, then test on breast cancer and compare against sklearn's
# RandomizedSearchCV on the same distributions.
class MyRandomSearchCV:
    def __init__(self, estimator, param_distributions, n_iter=20, cv=5,
                 scoring="accuracy", random_state=None):
        self.estimator = estimator
        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.cv = cv
        self.scoring = scoring
        self.random_state = random_state
        self.best_params_ = None
        self.best_score_ = -np.inf
        self.cv_results_ = []

    def _sample_params(self, rng):
        """One value per hyperparameter: rng.choice() for a list,
        .rvs(random_state=rng) for a scipy distribution."""
        # YOUR CODE HERE
        pass

    def fit(self, X, y):
        """Run n_iter random trials with a manual KFold loop
        (reuse the same CV logic as MyGridSearchCV)."""
        # YOUR CODE HERE
        pass

    def get_results_df(self):
        return pd.DataFrame(self.cv_results_).sort_values("mean_test_score", ascending=False)


# Test on breast cancer + compare with sklearn's RandomizedSearchCV

class MyRandomSearchCV:
    def __init__(self, estimator, param_distributions, n_iter=20, cv=5,
                 scoring="accuracy", random_state=None):
        self.estimator = estimator
        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.cv = cv
        self.scoring = scoring
        self.random_state = random_state
        self.best_params_ = None
        self.best_score_ = -np.inf
        self.cv_results_ = []

    def _sample_params(self, rng):
        params = {}
        for name, spec in self.param_distributions.items():
            if hasattr(spec, "rvs"):
                params[name] = int(spec.rvs(random_state=rng))
            else:
                params[name] = spec[rng.integers(0, len(spec))]
        return params

    def _score_single_combination(self, params, X, y):
        est = self.estimator.__class__(**{**self.estimator.get_params(), **params})
        kf = KFold(n_splits=self.cv, shuffle=True, random_state=42)
        fold_scores = []
        for train_idx, val_idx in kf.split(X):
            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
            est.fit(X_train, y_train)
            fold_scores.append(est.score(X_val, y_val))
        return np.mean(fold_scores)

    def fit(self, X, y):
        rng = np.random.default_rng(self.random_state)
        for _ in range(self.n_iter):
            params = self._sample_params(rng)
            mean_score = self._score_single_combination(params, X, y)
            self.cv_results_.append({"params": params, "mean_test_score": mean_score})
            if mean_score > self.best_score_:
                self.best_score_ = mean_score
                self.best_params_ = params
        return self

    def get_results_df(self):
        return pd.DataFrame(self.cv_results_).sort_values("mean_test_score", ascending=False)


param_dist_dt = {
    "max_depth": [2, 4, 6, 8, 10, 15],
    "min_samples_leaf": randint(1, 10),
}

my_rs = MyRandomSearchCV(DecisionTreeClassifier(random_state=42), param_dist_dt,
                         n_iter=20, cv=5, random_state=42)
my_rs.fit(Xtr, ytr)
print(f"[mine]    best={my_rs.best_score_:.4f}  params={my_rs.best_params_}")

# Same KFold object as the Grid Search comparison above, so any remaining gap
# is purely about the sampling path (mine vs sklearn's), not the CV scheme.
sk_rs = RandomizedSearchCV(DecisionTreeClassifier(random_state=42), param_dist_dt,
                           n_iter=20, cv=same_folds, random_state=42)
sk_rs.fit(Xtr, ytr)
print(f"[sklearn] best={sk_rs.best_score_:.4f}  params={sk_rs.best_params_}")
# Unlike grid search, an exact match isn't expected here even with matched
# folds - MyRandomSearchCV and RandomizedSearchCV use different internal
# sampling implementations, so the 20 random points they draw are different
# points from the same distribution. Both scores should be in the same
# ballpark; the params found won't usually be identical.

[mine]    best=0.9275  params={'max_depth': 4, 'min_samples_leaf': 1}
[sklearn] best=0.9143  params={'max_depth': 4, 'min_samples_leaf': 3}

# TODO (Extension) - coverage plot. Grid and random search don't just score
# differently, they land in different places in parameter space. Scatter the
# 20 grid points and the 20 random-search points from the first TODO above,
# colour-coded, on the same (n_estimators, max_depth) axes.

grid_n_est = [c[0] for c in combos]
grid_depth = [c[1] if c[1] is not None else 25 for c in combos]  # None -> plot at 25 for visibility

random_n_est = [p["n_estimators"] for p in rs20.cv_results_["params"]]
random_depth = [p["max_depth"] for p in rs20.cv_results_["params"]]

plt.scatter(grid_n_est, grid_depth, label="Grid", marker="s", alpha=0.7)
plt.scatter(random_n_est, random_depth, label="Random", marker="o", alpha=0.7)
plt.xlabel("n_estimators"); plt.ylabel("max_depth (None plotted as 25)"); plt.legend()
plt.show()
# Grid points fall on a rigid lattice - only 5 distinct max_depth values, ever.
# Random points spread continuously across the whole max_depth range, so if
# the "sweet spot" for max_depth isn't exactly one of the grid's 5 values,
# only random search can actually find it.

!pip install -q optuna

import optuna
from sklearn.model_selection import cross_val_score

optuna.logging.set_verbosity(optuna.logging.WARNING)

def objective(trial):
    params = {
        "n_estimators":      trial.suggest_int("n_estimators", 20, 300),
        "max_depth":         trial.suggest_int("max_depth", 2, 20),
        "min_samples_split": trial.suggest_int("min_samples_split", 2, 20),
    }
    model = RandomForestClassifier(**params, random_state=42)
    return cross_val_score(model, Xtr, ytr, cv=3).mean()

study = optuna.create_study(direction="maximize",
                            sampler=optuna.samplers.TPESampler(seed=42))
study.optimize(objective, n_trials=20)
print("Best settings:", study.best_params)
print("Best score:", round(study.best_value, 4))

Best settings: {'n_estimators': 253, 'max_depth': 6, 'min_samples_split': 5}
Best score: 0.9538

# TODO

# 1. Grid vs random vs Optuna, 20 trials each, one comparison table.

# 2. Plot best-score-so-far for random search and Optuna on the same graph.

# 1.
results_82 = {}

t0 = time.time()
grid20 = list(product([50, 100, 150, 200], [4, 8, 12, None, 16]))[:20]
scores20 = [RandomForestClassifier(n_estimators=n, max_depth=d, random_state=42)
            .fit(Xtr, ytr).score(Xte, yte) for n, d in grid20]
results_82["Grid search"] = (max(scores20), time.time() - t0)

t0 = time.time()
rs = RandomizedSearchCV(RandomForestClassifier(random_state=42), dist, n_iter=20, cv=3,
                        random_state=42, n_jobs=-1).fit(Xtr, ytr)
results_82["Random search"] = (rs.best_score_, time.time() - t0)

t0 = time.time()
study20 = optuna.create_study(direction="maximize",
                              sampler=optuna.samplers.TPESampler(seed=42))
study20.optimize(objective, n_trials=20)
results_82["Bayesian (Optuna)"] = (study20.best_value, time.time() - t0)

print(pd.DataFrame(results_82, index=["best_score", "time_s"]).T)

# 2.
random_param_draws = [
    dict(n_estimators=int(a), max_depth=int(b), min_samples_split=int(c))
    for a, b, c in zip(np.random.randint(20, 300, 20),
                       np.random.randint(2, 20, 20),
                       np.random.randint(2, 20, 20))
]
random_trial_scores = [
    cross_val_score(RandomForestClassifier(**p, random_state=42), Xtr, ytr, cv=3).mean()
    for p in random_param_draws
]
random_best_so_far = np.maximum.accumulate(random_trial_scores)
optuna_best_so_far = np.maximum.accumulate([t.value for t in study20.trials])

plt.plot(range(1, 21), random_best_so_far, marker="o", label="Random search")
plt.plot(range(1, 21), optuna_best_so_far, marker="o", label="Optuna (Bayesian)")
plt.xlabel("trial"); plt.ylabel("best score so far"); plt.legend()
plt.show()

                   best_score     time_s
Grid search          0.964912   6.850107
Random search        0.958203   9.671659
Bayesian (Optuna)    0.953817  23.671229

# TODO (Extension) - parameter importance. Optuna can compute how much each
# hyperparameter contributed to the score variance across your `study` from
# above. Print optuna.importance.get_param_importances(study) and plot a
# horizontal bar chart (param name on y-axis, importance on x-axis).

importances = optuna.importance.get_param_importances(study)
print(importances)

plt.barh(list(importances.keys()), list(importances.values()))
plt.xlabel("importance")
plt.show()

{'max_depth': np.float64(0.5575685746312921), 'min_samples_split': np.float64(0.3274357242530612), 'n_estimators': np.float64(0.11499570111564669)}

# TODO (Extension) - is Optuna itself reproducible? Run two fresh studies
# with the exact same objective, n_trials=10, and TPESampler(seed=42).
# Extract [t.value for t in study.trials] from each and assert they are
# equal. What does this prove - and where would it break?

study_a = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42))
study_a.optimize(objective, n_trials=10)

study_b = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42))
study_b.optimize(objective, n_trials=10)

values_a = [t.value for t in study_a.trials]
values_b = [t.value for t in study_b.trials]
assert values_a == values_b, "Studies diverged!"
print("Identical trial sequences:", values_a == values_b)
# This proves Optuna's own trial-selection process is seeded and deterministic
# - same sampler seed, same objective, same 10 trials in the same order. It
# would break if the objective itself is non-deterministic (e.g. an unseeded
# model inside it), or if trials ran in parallel with n_jobs>1, where trial
# completion order (and therefore what the sampler has seen so far) can vary.

Identical trial sequences: True

# TODO (Extension) - a regression task. Everything so far has been
# classification; switch to regression. Tune GradientBoostingRegressor on the
# California Housing dataset (built into scikit-learn, downloaded on first
# use - needs Internet On). Minimise RMSE, 20 trials. n_estimators 50-300,
# learning_rate log-uniform 1e-3 to 1.0, max_depth 2-8.

from sklearn.datasets import fetch_california_housing
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error

housing = fetch_california_housing(as_frame=True)
Xh, yh = housing.data, housing.target
Xh_tr, Xh_te, yh_tr, yh_te = train_test_split(Xh, yh, test_size=0.2, random_state=42)

def objective_housing(trial):
    params = {
        "n_estimators":  trial.suggest_int("n_estimators", 50, 300),
        "learning_rate": trial.suggest_float("learning_rate", 1e-3, 1.0, log=True),
        "max_depth":     trial.suggest_int("max_depth", 2, 8),
    }
    model = GradientBoostingRegressor(**params, random_state=42)
    model.fit(Xh_tr, yh_tr)
    return mean_squared_error(yh_te, model.predict(Xh_te)) ** 0.5  # RMSE

study_housing = optuna.create_study(direction="minimize",
                                    sampler=optuna.samplers.TPESampler(seed=42))
study_housing.optimize(objective_housing, n_trials=20)
print("Best RMSE  :", round(study_housing.best_value, 4))
print("Best params:", study_housing.best_params)

Best RMSE  : 0.4503
Best params: {'n_estimators': 252, 'learning_rate': 0.12356971289592052, 'max_depth': 7}

# TODO (Extension) - let Optuna pick the model family. Use
# trial.suggest_categorical to let Optuna choose both the model family AND
# its hyperparameters in a single study - a tiny hand-rolled AutoML. Include
# DecisionTree, RandomForest, and KNN. Run 30 trials on the breast cancer
# data, then count how many trials picked each family.
def objective_automl(trial):
    classifier = trial.suggest_categorical("classifier", ["DT", "RF", "KNN"])
    # if classifier == "DT": suggest DT params, build DecisionTreeClassifier
    # elif classifier == "RF": suggest RF params, build RandomForestClassifier
    # elif classifier == "KNN": suggest KNN params, build KNeighborsClassifier
    # return 5-fold CV accuracy on Xtr, ytr
    pass

# Run 30 trials, print best, then count how often each family was chosen.

from sklearn.neighbors import KNeighborsClassifier

def objective_automl(trial):
    classifier = trial.suggest_categorical("classifier", ["DT", "RF", "KNN"])
    if classifier == "DT":
        model = DecisionTreeClassifier(max_depth=trial.suggest_int("dt_max_depth", 2, 20),
                                       random_state=42)
    elif classifier == "RF":
        model = RandomForestClassifier(n_estimators=trial.suggest_int("rf_n_estimators", 20, 200),
                                       max_depth=trial.suggest_int("rf_max_depth", 2, 20),
                                       random_state=42)
    else:
        model = KNeighborsClassifier(n_neighbors=trial.suggest_int("knn_n_neighbors", 1, 20))
    return cross_val_score(model, Xtr, ytr, cv=5).mean()

study_automl = optuna.create_study(direction="maximize",
                                   sampler=optuna.samplers.TPESampler(seed=42))
study_automl.optimize(objective_automl, n_trials=30)
print("Best value :", round(study_automl.best_value, 4))
print("Best params:", study_automl.best_params)

chosen = pd.Series([t.params["classifier"] for t in study_automl.trials]).value_counts()
print("\nTimes each family was chosen:")
print(chosen)

Best value : 0.9626
Best params: {'classifier': 'RF', 'rf_n_estimators': 155, 'rf_max_depth': 17}

Times each family was chosen:
RF     20
KNN     5
DT      5
Name: count, dtype: int64

from sklearn.svm import SVC
from sklearn.pipeline import Pipeline

# --- WRONG: scale before CV (leaky) ---
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)  # sees ALL data, including future validation folds!

leaky_score = cross_val_score(
    SVC(C=1, kernel="rbf", random_state=42), X_scaled, y, cv=5
).mean()

# --- RIGHT: scale inside a Pipeline ---
pipe = Pipeline([
    ("scaler", StandardScaler()),
    ("svm", SVC(C=1, kernel="rbf", random_state=42)),
])
correct_score = cross_val_score(pipe, X, y, cv=5).mean()

print(f"Leaky  (wrong)    CV accuracy: {leaky_score:.4f}")
print(f"Correct (Pipeline) CV accuracy: {correct_score:.4f}")
print(f"Difference: {leaky_score - correct_score:+.4f}")
if abs(leaky_score - correct_score) < 0.005:
    print("(Small or zero gap here - breast cancer has 569 fairly stable rows, so a "
          "per-fold scaler and a whole-dataset scaler end up with nearly identical "
          "mean/std anyway. The leak is still real; it just doesn't move the needle "
          "much on this particular dataset. The Pipeline fix costs nothing, so use it "
          "regardless of whether the gap happens to be visible - on a smaller dataset, "
          "or with a leakier step like selecting features by correlation with the "
          "label, the same mistake can inflate scores by many points.)")

Leaky  (wrong)    CV accuracy: 0.9736
Correct (Pipeline) CV accuracy: 0.9736
Difference: +0.0000
(Small or zero gap here - breast cancer has 569 fairly stable rows, so a per-fold scaler and a whole-dataset scaler end up with nearly identical mean/std anyway. The leak is still real; it just doesn't move the needle much on this particular dataset. The Pipeline fix costs nothing, so use it regardless of whether the gap happens to be visible - on a smaller dataset, or with a leakier step like selecting features by correlation with the label, the same mistake can inflate scores by many points.)

# TODO

# 1. One sentence: why does fitting StandardScaler on the whole dataset
#    before cross_val_score leak information from the validation folds?
answer = "..."

# 2. Build Pipeline(StandardScaler -> SVC) and tune it with GridSearchCV on
#    the breast cancer data. svm__C: [0.01, 0.1, 1, 10, 100],
#    svm__kernel: ["linear", "rbf"] - note the svm__ prefix, needed because
#    "svm" is the pipeline step's name.

answer = (
    "The scaler's mean and standard deviation are computed from every row, "
    "including the ones that later become a validation fold - so each fold's "
    "'unseen' data already shaped the scaling applied to it, inflating the score."
)
print(answer)

param_grid_pipe = {
    "svm__C": [0.01, 0.1, 1, 10, 100],
    "svm__kernel": ["linear", "rbf"],
}

pipe2 = Pipeline([("scaler", StandardScaler()), ("svm", SVC(random_state=42))])
gs_pipe = GridSearchCV(pipe2, param_grid_pipe, cv=5)
gs_pipe.fit(Xtr, ytr)

print(f"Best params : {gs_pipe.best_params_}")
print(f"Best CV score: {gs_pipe.best_score_:.4f}")

The scaler's mean and standard deviation are computed from every row, including the ones that later become a validation fold - so each fold's 'unseen' data already shaped the scaling applied to it, inflating the score.
Best params : {'svm__C': 0.1, 'svm__kernel': 'linear'}
Best CV score: 0.9736

!pip install -q autogluon.tabular

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 514.8/514.8 kB 17.4 MB/s eta 0:00:00
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 248.4/248.4 kB 13.0 MB/s eta 0:00:00
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 112.3/112.3 kB 6.1 MB/s eta 0:00:00
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 90.7/90.7 kB 5.2 MB/s eta 0:00:00

from autogluon.tabular import TabularPredictor

train_df = pd.concat([Xtr, ytr], axis=1)
test_df  = pd.concat([Xte, yte], axis=1)

predictor = TabularPredictor(label="diagnosis", path="ag_models").fit(
    train_df,
    time_limit=60,                # stop after 1 minute - this is a small dataset
    presets="medium_quality",     # use "best_quality" if you have more time
)
print(predictor.evaluate(test_df))

Verbosity: 2 (Standard Logging)
=================== System Info ===================
AutoGluon Version:  1.6.1
Python Version:     3.12.13
Operating System:   Linux
Platform Machine:   x86_64
Platform Version:   #1 SMP Sat May 30 15:40:53 UTC 2026
CPU Count:          4
Pytorch Version:    2.10.0+cpu
CUDA Version:       CUDA is not available
Memory Avail:       29.83 GB / 31.35 GB (95.1%)
Disk Space Avail:   19.50 GB / 19.52 GB (99.9%)
===================================================
Presets specified: ['medium_quality']
Using hyperparameters preset: hyperparameters='default'
Beginning AutoGluon training ... Time limit = 60s
AutoGluon will save models to "/kaggle/working/ag_models"
Train Data Rows:    455
Train Data Columns: 30
Label Column:       diagnosis
AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).
	2 unique label values:  ['benign', 'malignant']
	If 'binary' is not the correct problem_type, please manually specify the problem_type parameter during Predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression', 'quantile'])
Problem Type:       binary
Preprocessing data...
Selected class <--> label mapping:  class 1 = malignant, class 0 = benign
	Note: For your binary classification, AutoGluon arbitrarily selected which label-value represents positive (malignant) vs negative (benign) class.
	To explicitly set the positive_class, either rename classes to 1 and 0, or specify positive_class in Predictor init.
Using Feature Generators to preprocess the data ...
Fitting AutoMLPipelineFeatureGenerator...
	Available Memory:                    30542.66 MB
	Train Data (Original)  Memory Usage: 0.10 MB (0.0% of available memory)
	Inferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.
	Stage 1 Generators:
		Fitting AsTypeFeatureGenerator...
	Stage 2 Generators:
		Fitting FillNaFeatureGenerator...
	Stage 3 Generators:
		Fitting IdentityFeatureGenerator...
	Stage 4 Generators:
		Fitting DropUniqueFeatureGenerator...
	Stage 5 Generators:
		Fitting DropDuplicatesFeatureGenerator...
	Types of features in original data (raw dtype, special dtypes):
		('float', []) : 30 | ['mean radius', 'mean texture', 'mean perimeter', 'mean area', 'mean smoothness', ...]
	Types of features in processed data (raw dtype, special dtypes):
		('float', []) : 30 | ['mean radius', 'mean texture', 'mean perimeter', 'mean area', 'mean smoothness', ...]
	0.1s = Fit runtime
	30 features in original data used to generate 30 features in processed data.
	Train Data (Processed) Memory Usage: 0.10 MB (0.0% of available memory)
Data preprocessing and feature engineering runtime = 0.07s ...
AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'
	To change this, specify the eval_metric parameter of Predictor()
Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 364, Val Rows: 91
User-specified model hyperparameters to be fit:
{
	'NN_TORCH': [{}],
	'GBM': [{'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}}, {}, {'learning_rate': 0.03, 'num_leaves': 128, 'feature_fraction': 0.9, 'min_data_in_leaf': 3, 'ag_args': {'name_suffix': 'Large', 'priority': 0, 'hyperparameter_tune_kwargs': None}}],
	'CAT': [{}],
	'XGB': [{}],
	'FASTAI': [{}],
	'RF': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],
	'XT': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],
}
Fitting 11 L1 models, fit_strategy="sequential" ...
Fitting model: LightGBMXT ... Training model for up to 59.93s of the 59.92s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.1/29.8 GB
	0.978	 = Validation score   (accuracy)
	4.13s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: LightGBM ... Training model for up to 55.78s of the 55.77s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.1/29.8 GB
	0.989	 = Validation score   (accuracy)
	0.45s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: RandomForestGini ... Training model for up to 55.32s of the 55.31s of remaining time.
	Fitting with cpus=4, gpus=0
	0.978	 = Validation score   (accuracy)
	0.87s	 = Training   runtime
	0.08s	 = Validation runtime
Fitting model: RandomForestEntr ... Training model for up to 54.35s of the 54.35s of remaining time.
	Fitting with cpus=4, gpus=0
	0.978	 = Validation score   (accuracy)
	0.83s	 = Training   runtime
	0.08s	 = Validation runtime
Fitting model: CatBoost ... Training model for up to 53.42s of the 53.42s of remaining time.
	Fitting with cpus=2, gpus=0
	0.978	 = Validation score   (accuracy)
	3.78s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: ExtraTreesGini ... Training model for up to 49.63s of the 49.63s of remaining time.
	Fitting with cpus=4, gpus=0
	0.956	 = Validation score   (accuracy)
	0.83s	 = Training   runtime
	0.08s	 = Validation runtime
Fitting model: ExtraTreesEntr ... Training model for up to 48.70s of the 48.70s of remaining time.
	Fitting with cpus=4, gpus=0
	0.956	 = Validation score   (accuracy)
	0.81s	 = Training   runtime
	0.08s	 = Validation runtime
Fitting model: NeuralNetFastAI ... Training model for up to 47.79s of the 47.79s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.0/29.8 GB
	0.967	 = Validation score   (accuracy)
	2.84s	 = Training   runtime
	0.02s	 = Validation runtime
Fitting model: XGBoost ... Training model for up to 44.92s of the 44.92s of remaining time.
	Fitting with cpus=2, gpus=0
	0.978	 = Validation score   (accuracy)
	0.28s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: NeuralNetTorch ... Training model for up to 44.62s of the 44.62s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.0/29.7 GB
	0.9451	 = Validation score   (accuracy)
	4.86s	 = Training   runtime
	0.02s	 = Validation runtime
Fitting model: LightGBMLarge ... Training model for up to 39.74s of the 39.74s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.1/29.6 GB
	0.978	 = Validation score   (accuracy)
	1.15s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: WeightedEnsemble_L2 ... Training model for up to 59.93s of the 38.57s of remaining time.
	Fitting 1 model on all data | Fitting with cpus=4, gpus=0, mem=0.0/29.7 GB
	Ensemble Weights: {'LightGBM': 1.0}
	0.989	 = Validation score   (accuracy)
	0.1s	 = Training   runtime
	0.0s	 = Validation runtime
AutoGluon training complete, total runtime = 21.82s ... Best model: WeightedEnsemble_L2 | Estimated inference throughput: 32195.8 rows/s (91 batch size)
Disabling decision threshold calibration for metric `accuracy` due to having fewer than 10000 rows of validation data for calibration, to avoid overfitting (91 rows).
	`accuracy` is generally not improved through threshold calibration. Force calibration via specifying `calibrate_decision_threshold=True`.
TabularPredictor saved. To load, use: predictor = TabularPredictor.load("/kaggle/working/ag_models")

{'accuracy': 0.9649122807017544, 'balanced_accuracy': np.float64(0.9580740255486406), 'mcc': np.float64(0.9252853920667758), 'roc_auc': np.float64(0.9895185063871601), 'f1': 0.9523809523809523, 'precision': 0.975609756097561, 'recall': 0.9302325581395349}

# TODO

# 1. Leaderboard: models trained, winner, is it a WeightedEnsemble blend?

# 2. Feature importance ranking - does the top feature match intuition?

# 3. Compare Optuna (Section 11) vs AutoGluon: score, wall time, human time.

# 1.
lb = predictor.leaderboard(test_df, silent=True)
print(lb)
print("Models trained:", len(lb))
print("Winning model:", lb.iloc[0]["model"])
print("Is a blend:", "WeightedEnsemble" in lb.iloc[0]["model"])

# 2.
importance = predictor.feature_importance(test_df)
print(importance)
# The "worst_*" features (the most extreme per-nucleus measurement in each
# image, e.g. worst texture, worst radius, worst concave points) tend to rank
# at or near the top - large, irregular nuclei are a classic visual sign of
# malignancy. Expect most other features to show low/insignificant importance
# (high p-value) once accuracy is already near-ceiling on this dataset.

# 3.
comparison = pd.DataFrame({
    "method": ["Hand-tuned (Optuna, Section 11)", "AutoGluon"],
    "best_score": [study20.best_value, lb.iloc[0]["score_val"]],
    "wall_time_s": [results_82["Bayesian (Optuna)"][1], 60],
    "human_time_min": [45, 5],
})
print(comparison)
# AutoGluon typically matches or slightly beats the hand-tuned model with far
# less setup time. For this small, well-separated dataset the accuracy gap is
# tiny, so AutoML wins on time-to-result; on a dataset 100x bigger, AutoGluon's
# automatic ensembling and longer training budget would likely widen that gap.

Computing feature importance via permutation shuffling for 30 features using 114 rows with 5 shuffle sets...
	1.26s	= Expected runtime (0.25s per shuffle set)

                  model  score_test  score_val eval_metric  pred_time_test  \
0        NeuralNetTorch    0.982456   0.945055    accuracy        0.023309   
1              CatBoost    0.973684   0.978022    accuracy        0.005536   
2            LightGBMXT    0.973684   0.978022    accuracy        0.007405   
3        ExtraTreesGini    0.973684   0.956044    accuracy        0.101346   
4              LightGBM    0.964912   0.989011    accuracy        0.005107   
5   WeightedEnsemble_L2    0.964912   0.989011    accuracy        0.007136   
6       NeuralNetFastAI    0.964912   0.967033    accuracy        0.017529   
7        ExtraTreesEntr    0.964912   0.956044    accuracy        0.092512   
8      RandomForestEntr    0.964912   0.978022    accuracy        0.094451   
9      RandomForestGini    0.964912   0.978022    accuracy        0.095569   
10              XGBoost    0.956140   0.978022    accuracy        0.011258   
11        LightGBMLarge    0.947368   0.978022    accuracy        0.003166   

    pred_time_val  fit_time  pred_time_test_marginal  pred_time_val_marginal  \
0        0.018548  4.855072                 0.023309                0.018548   
1        0.002865  3.781012                 0.005536                0.002865   
2        0.002361  4.134031                 0.007405                0.002361   
3        0.079099  0.826705                 0.101346                0.079099   
4        0.001880  0.448317                 0.005107                0.001880   
5        0.002826  0.548856                 0.002029                0.000947   
6        0.017754  2.840411                 0.017529                0.017754   
7        0.080442  0.808881                 0.092512                0.080442   
8        0.078155  0.834161                 0.094451                0.078155   
9        0.079860  0.870968                 0.095569                0.079860   
10       0.003258  0.278810                 0.011258                0.003258   
11       0.001723  1.150648                 0.003166                0.001723   

    fit_time_marginal  stack_level  can_infer  fit_order  
0            4.855072            1       True         10  
1            3.781012            1       True          5  
2            4.134031            1       True          1  
3            0.826705            1       True          6  
4            0.448317            1       True          2  
5            0.100539            2       True         12  
6            2.840411            1       True          8  
7            0.808881            1       True          7  
8            0.834161            1       True          4  
9            0.870968            1       True          3  
10           0.278810            1       True          9  
11           1.150648            1       True         11  
Models trained: 12
Winning model: NeuralNetTorch
Is a blend: False

	0.26s	= Actual runtime (Completed 5 of 5 shuffle sets)

                         importance    stddev   p_value  n  p99_high   p99_low
worst area                 0.019298  0.003923  0.000194  5  0.027376  0.011221
worst perimeter            0.017544  0.006203  0.001599  5  0.030315  0.004772
worst radius               0.008772  0.000000  0.000000  5  0.008772  0.008772
mean texture               0.005263  0.004805  0.035242  5  0.015156 -0.004630
worst texture              0.003509  0.004805  0.088904  5  0.013401 -0.006384
mean radius                0.000000  0.000000  0.500000  5  0.000000  0.000000
mean concavity             0.000000  0.000000  0.500000  5  0.000000  0.000000
mean symmetry              0.000000  0.000000  0.500000  5  0.000000  0.000000
mean fractal dimension     0.000000  0.000000  0.500000  5  0.000000  0.000000
radius error               0.000000  0.000000  0.500000  5  0.000000  0.000000
mean perimeter             0.000000  0.000000  0.500000  5  0.000000  0.000000
mean area                  0.000000  0.000000  0.500000  5  0.000000  0.000000
mean smoothness            0.000000  0.000000  0.500000  5  0.000000  0.000000
mean compactness           0.000000  0.000000  0.500000  5  0.000000  0.000000
worst fractal dimension    0.000000  0.000000  0.500000  5  0.000000  0.000000
area error                 0.000000  0.000000  0.500000  5  0.000000  0.000000
texture error              0.000000  0.000000  0.500000  5  0.000000  0.000000
perimeter error            0.000000  0.000000  0.500000  5  0.000000  0.000000
fractal dimension error    0.000000  0.000000  0.500000  5  0.000000  0.000000
concave points error       0.000000  0.000000  0.500000  5  0.000000  0.000000
smoothness error           0.000000  0.000000  0.500000  5  0.000000  0.000000
concavity error            0.000000  0.000000  0.500000  5  0.000000  0.000000
worst smoothness           0.000000  0.000000  0.500000  5  0.000000  0.000000
symmetry error             0.000000  0.000000  0.500000  5  0.000000  0.000000
worst compactness          0.000000  0.000000  0.500000  5  0.000000  0.000000
worst symmetry            -0.001754  0.003923  0.813050  5  0.006323 -0.009832
worst concave points      -0.001754  0.003923  0.813050  5  0.006323 -0.009832
mean concave points       -0.007018  0.003923  0.991935  5  0.001060 -0.015095
compactness error         -0.007018  0.003923  0.991935  5  0.001060 -0.015095
worst concavity           -0.008772  0.000000  1.000000  5 -0.008772 -0.008772
                            method  best_score  wall_time_s  human_time_min
0  Hand-tuned (Optuna, Section 11)    0.953817    23.671229              45
1                        AutoGluon    0.945055    60.000000               5

!pip install -q wandb

# Run once per session
from kaggle_secrets import UserSecretsClient

# Initialize the secrets client
user_secrets = UserSecretsClient()

# Retrieve your secret value using its label
secret_value = user_secrets.get_secret("WANDB_API_KEY")

import os
os.environ["WANDB_API_KEY"] = secret_value

import wandb
wandb.login()
print("Logged in to W&B.")

/usr/local/lib/python3.12/dist-packages/notebook/notebookapp.py:191: SyntaxWarning: invalid escape sequence '\/'
  | |_| | '_ \/ _` / _` |  _/ -_)
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
wandb: Currently logged in as: sankalpturankar567 (sankalpturankar567-indian-institute-of-technology-gandhi) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin

Logged in to W&B.

wandb.init(project="stt-ai-breast-cancer-hpo", name="rf-baseline",
           config={"n_estimators": 100, "max_depth": 8, "seed": 42})

cfg = wandb.config
model = RandomForestClassifier(n_estimators=cfg.n_estimators,
                               max_depth=cfg.max_depth,
                               random_state=cfg.seed).fit(Xtr, ytr)

wandb.log({"train_acc": model.score(Xtr, ytr),
           "val_acc":   model.score(Xte, yte)})
wandb.finish()

# All three parameters use discrete "values" lists, not "min"/"max" ranges.
# This matters specifically for method="grid": grid search needs a finite set
# of values for every parameter. Give it a "min"/"max" range instead and it
# will silently enumerate EVERY integer in that range - for n_estimators
# (20-300) x max_depth (2-20) x 4 values that's ~21,000 grid points, which
# looks like an infinite loop (it's not infinite, just enormous and slow).
# Using "values" everywhere also means grid/random/bayes all search the exact
# same space, making the count=8 comparison in the next TODO a fair one.
sweep_config = {
    "method": "bayes",                                   # or "grid" / "random"
    "metric": {"name": "val_acc", "goal": "maximize"},
    "parameters": {
        "n_estimators":      {"values": [20, 60, 100, 150, 200, 300]},
        "max_depth":         {"values": [2, 4, 8, 12, 16, 20]},
        "min_samples_split": {"values": [2, 5, 10, 20]},
    },
}

def train_one():
    wandb.init()
    cfg = wandb.config
    model = RandomForestClassifier(n_estimators=cfg.n_estimators,
                                   max_depth=cfg.max_depth,
                                   min_samples_split=cfg.min_samples_split,
                                   random_state=42).fit(Xtr, ytr)
    wandb.log({"val_acc": model.score(Xte, yte)})

# sweep_id = wandb.sweep(sweep_config, project="stt-ai-breast-cancer-hpo")
# wandb.agent(sweep_id, function=train_one, count=8)

# TODO

# 1. Run the sweep 3 times (method = grid, random, bayes), 8 runs each.
#    Print the best val_acc from each.

# 2. (On wandb.ai) note which hyperparameter matters most in the
#    parallel-coordinates chart - write your observation as a comment.

best_scores = {}
for method in ["grid", "random", "bayes"]:
    sweep_config["method"] = method
    sweep_id = wandb.sweep(sweep_config, project="stt-ai-breast-cancer-hpo")
    wandb.agent(sweep_id, function=train_one, count=8)

    api = wandb.Api()
    sweep = api.sweep(f"{api.default_entity}/stt-ai-breast-cancer-hpo/{sweep_id}")
    best_scores[method] = max(run.summary.get("val_acc", 0) for run in sweep.runs)

print(pd.Series(best_scores))
# On wandb.ai's parallel-coordinates chart: n_estimators and max_depth usually
# separate good from bad runs most clearly; min_samples_split usually matters
# least across a wide range.

Create sweep with ID: wtd15hjl
Sweep URL: https://wandb.ai/sankalpturankar567-indian-institute-of-technology-gandhi/stt-ai-breast-cancer-hpo/sweeps/wtd15hjl

wandb: Agent Starting Run: dxvrnx65 with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 2
wandb: 	n_estimators: 20
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Agent Starting Run: n5xqyxaa with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 2
wandb: 	n_estimators: 60
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Agent Starting Run: m5vc8ko6 with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 2
wandb: 	n_estimators: 100
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Agent Starting Run: vm2tuese with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 2
wandb: 	n_estimators: 150
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Agent Starting Run: v55k324w with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 2
wandb: 	n_estimators: 200
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Sweep Agent: Waiting for job.
wandb: Job received.
wandb: Agent Starting Run: 5tkieihi with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 2
wandb: 	n_estimators: 300
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Agent Starting Run: n8c7muos with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 5
wandb: 	n_estimators: 20
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

wandb: Agent Starting Run: d6p96f4c with config:
wandb: 	max_depth: 2
wandb: 	min_samples_split: 5
wandb: 	n_estimators: 60
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.

Create sweep with ID: 96fosq5h
Sweep URL: https://wandb.ai/sankalpturankar567-indian-institute-of-technology-gandhi/stt-ai-breast-cancer-hpo/sweeps/96fosq5h

# ==========================================================
# 🏆 Lab Exercise: One Reproducible Training Script
# ==========================================================
#
# Objective
# ---------
# Combine everything from Sections 1-6 into one script that a classmate can
# run to get your exact validation accuracy - the real test of reproducibility.

# ✅ Task 1: Write train.py that reads settings from config.json.
#
# ✅ Task 2: Set every seed inside the script.
#
# ✅ Task 3: Load the data and print its file hash.
#
# ✅ Task 4: Train the model and print the validation accuracy.
#
# ✅ Task 5: Save the model, metrics, and environment info into one run folder.
#
# ✅ Task 6: Swap train.py, config.json, and requirements.txt with a
#            classmate. Run theirs. Do you get their exact number? If not,
#            find out why and write down the cause.

# ==========================================================
# 🏆 Lab Exercise Solution
# ==========================================================
# Task 6's swap test is something you'll do with an actual classmate, not
# something this cell can simulate.

%%writefile train.py
import json, time, platform, hashlib, random, os
from pathlib import Path
import numpy as np, pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split


def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)


def main():
    config = json.loads(Path("config.json").read_text())
    set_seed(config["seed"])

    data_path = Path(config["data_path"])
    df = pd.read_csv(data_path)
    data_hash = hashlib.md5(data_path.read_bytes()).hexdigest()
    print("Data file hash:", data_hash)

    feature_cols = [c for c in df.columns if c != "diagnosis"]
    X = df[feature_cols]
    y = df["diagnosis"]
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, random_state=config["seed"])

    model = RandomForestClassifier(n_estimators=config["n_estimators"],
                                   max_depth=config["max_depth"],
                                   random_state=config["seed"]).fit(Xtr, ytr)
    val_acc = model.score(Xte, yte)

    run_dir = Path("runs") / f"run_seed{config['seed']}"
    run_dir.mkdir(parents=True, exist_ok=True)
    import joblib
    joblib.dump(model, run_dir / "model.joblib")
    (run_dir / "metrics.json").write_text(json.dumps(
        {"val_accuracy": val_acc, "data_hash": data_hash}, indent=2))
    (run_dir / "environment.json").write_text(json.dumps(
        {"python": platform.python_version()}, indent=2))

    print(f"Validation accuracy: {val_acc:.4f}")


if __name__ == "__main__":
    main()

Writing train.py

df.to_csv("breast_cancer.csv", index=False)
print("Saved breast_cancer.csv")

Saved breast_cancer.csv

config = {
    "seed": 42,
    "n_estimators": 100,
    "max_depth": 8,
    "data_path": "breast_cancer.csv",
}
Path("config.json").write_text(json.dumps(config, indent=2))
print(Path("config.json").read_text())

{
  "seed": 42,
  "n_estimators": 100,
  "max_depth": 8,
  "data_path": "breast_cancer.csv"
}

!python train.py

Data file hash: 0d1690bbd5734538a9c8b3b1dc902760
Validation accuracy: 0.9649

# TODO

# On the Wine dataset, build a leaderboard with:
# 1. DummyClassifier baseline (most-frequent-class strategy) - what does a
#    classifier that isn't even trying score?
# 2. LogisticRegression (default)
# 3. RandomForest tuned with Optuna (30 trials)
# 4. DIY AutoML: >=4 model families, each with its own small GridSearchCV
# Print one leaderboard sorted by CV accuracy.

from sklearn.datasets import load_wine
from sklearn.dummy import DummyClassifier
from sklearn.neighbors import KNeighborsClassifier

Xw, yw = load_wine(return_X_y=True)
Xw_tr, Xw_te, yw_tr, yw_te = train_test_split(Xw, yw, test_size=0.2, random_state=42)

leaderboard = {}

# 1. Dummy baseline
dummy = DummyClassifier(strategy="most_frequent").fit(Xw_tr, yw_tr)
leaderboard["DummyClassifier"] = cross_val_score(dummy, Xw_tr, yw_tr, cv=5).mean()

# 2. Logistic Regression (default)
logreg = LogisticRegression(max_iter=5000, random_state=42)
leaderboard["LogisticRegression (default)"] = cross_val_score(logreg, Xw_tr, yw_tr, cv=5).mean()

# 3. Random Forest tuned with Optuna
def objective_wine(trial):
    params = {
        "n_estimators": trial.suggest_int("n_estimators", 20, 300),
        "max_depth": trial.suggest_int("max_depth", 2, 20),
    }
    return cross_val_score(RandomForestClassifier(**params, random_state=42), Xw_tr, yw_tr, cv=5).mean()

study_wine = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42))
study_wine.optimize(objective_wine, n_trials=30)
leaderboard["RandomForest (Optuna 30 trials)"] = study_wine.best_value

# 4. DIY AutoML - 4 model families, each with a small grid
model_configs = {
    "DecisionTree": (DecisionTreeClassifier(random_state=42), {"max_depth": [2, 4, 6, 8]}),
    "RandomForest": (RandomForestClassifier(random_state=42), {"n_estimators": [50, 100], "max_depth": [4, 8]}),
    "KNN":          (KNeighborsClassifier(), {"n_neighbors": [3, 5, 7, 9]}),
    "SVM":          (Pipeline([("scaler", StandardScaler()), ("svm", SVC(random_state=42))]),
                     {"svm__C": [0.1, 1, 10], "svm__kernel": ["linear", "rbf"]}),
}
for name, (clf, params) in model_configs.items():
    gs = GridSearchCV(clf, params, cv=5, n_jobs=-1)
    gs.fit(Xw_tr, yw_tr)
    leaderboard[f"AutoML-{name}"] = gs.best_score_

print(f"{'Method':<35} {'CV Score':>10}")
print("-" * 46)
for method, score in sorted(leaderboard.items(), key=lambda x: x[1], reverse=True):
    print(f"{method:<35} {score:>10.4f}")

/usr/local/lib/python3.12/dist-packages/sklearn/linear_model/_logistic.py:465: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

Method                                CV Score
----------------------------------------------
RandomForest (Optuna 30 trials)         0.9786
AutoML-RandomForest                     0.9786
AutoML-SVM                              0.9719
LogisticRegression (default)            0.9576
AutoML-DecisionTree                     0.9224
AutoML-KNN                              0.6973
DummyClassifier                         0.4012

# TODO

# Create a reusable function compare_search_methods(X, y, n_trials=20) that
# runs grid search, random search, and Bayesian search (Optuna) on any
# (X, y) with the same fit budget, and returns one DataFrame comparing best
# score and time taken for all three - so you never have to copy-paste the
# Section 9-11 code again for a new dataset. Test it on the Wine data
# from Part A.

def compare_search_methods(X, y, n_trials=20):
    Xtr_, Xte_, ytr_, yte_ = train_test_split(X, y, test_size=0.2, random_state=42)
    results = {}

    # Grid search: first n_trials combinations from a small fixed grid
    grid_vals = list(product([50, 100, 150, 200], [4, 8, 12, None, 16]))[:n_trials]
    t0 = time.time()
    grid_scores = [RandomForestClassifier(n_estimators=n, max_depth=d, random_state=42)
                   .fit(Xtr_, ytr_).score(Xte_, yte_) for n, d in grid_vals]
    results["Grid"] = (max(grid_scores), time.time() - t0)

    # Random search
    dist_local = {
        "n_estimators": randint(20, 300),
        "max_depth": randint(2, 20),
        "min_samples_split": randint(2, 20),
    }
    t0 = time.time()
    rs_local = RandomizedSearchCV(RandomForestClassifier(random_state=42), dist_local,
                                  n_iter=n_trials, cv=3, random_state=42,
                                  n_jobs=-1).fit(Xtr_, ytr_)
    results["Random"] = (rs_local.best_score_, time.time() - t0)

    # Bayesian search
    def local_objective(trial):
        params = {
            "n_estimators": trial.suggest_int("n_estimators", 20, 300),
            "max_depth": trial.suggest_int("max_depth", 2, 20),
            "min_samples_split": trial.suggest_int("min_samples_split", 2, 20),
        }
        return cross_val_score(RandomForestClassifier(**params, random_state=42),
                               Xtr_, ytr_, cv=3).mean()

    t0 = time.time()
    study_local = optuna.create_study(direction="maximize",
                                      sampler=optuna.samplers.TPESampler(seed=42))
    study_local.optimize(local_objective, n_trials=n_trials)
    results["Bayesian"] = (study_local.best_value, time.time() - t0)

    return pd.DataFrame(results, index=["best_score", "time_s"]).T

compare_search_methods(Xw, yw, n_trials=20)

# TODO - write your answers as a dict of short strings, no need to run code
reflection_answers = {
    1: "...",
    2: "...",
    3: "...",
    4: "...",
    5: "...",
}

reflection_answers = {
    1: "Small differences usually come from KFold's own shuffle randomness (a "
       "different random_state or fold order) or from ties in mean_test_score "
       "being broken differently between the two implementations.",
    2: "Random search covered the space more evenly - grid search only ever "
       "tests 5 fixed max_depth values no matter how many n_estimators values "
       "surround them, while random search's points spread continuously, so it "
       "can land near a 'sweet spot' that isn't exactly one of the grid's fixed "
       "values.",
    3: "(1) Diminishing returns - after the surrogate model has enough trials "
       "to find the good region, extra trials mostly refine a score that's "
       "already close to optimal. (2) Wasted compute/time - 500 trials on a "
       "slow-to-train model could take hours for a gain of a fraction of a "
       "percent, when that budget might be better spent trying a different "
       "model family or getting more data.",
    4: "A Pipeline refits StandardScaler on only the training fold at every "
       "step of cross-validation, so the validation fold's statistics never "
       "influence the scaling applied to it - unlike scaling the whole dataset "
       "once before CV, which lets every fold 'see' data it's supposed to be "
       "held out from.",
    5: "Typically n_estimators or max_depth dominate on this dataset, since "
       "they most directly control model capacity; min_samples_split usually "
       "contributes the least. This generally matches intuition - capacity "
       "controls tend to matter more than fine-grained regularization knobs "
       "on a dataset this size.",
}
reflection_answers

{1: "Small differences usually come from KFold's own shuffle randomness (a different random_state or fold order) or from ties in mean_test_score being broken differently between the two implementations.",
 2: "Random search covered the space more evenly - grid search only ever tests 5 fixed max_depth values no matter how many n_estimators values surround them, while random search's points spread continuously, so it can land near a 'sweet spot' that isn't exactly one of the grid's fixed values.",
 3: "(1) Diminishing returns - after the surrogate model has enough trials to find the good region, extra trials mostly refine a score that's already close to optimal. (2) Wasted compute/time - 500 trials on a slow-to-train model could take hours for a gain of a fraction of a percent, when that budget might be better spent trying a different model family or getting more data.",
 4: "A Pipeline refits StandardScaler on only the training fold at every step of cross-validation, so the validation fold's statistics never influence the scaling applied to it - unlike scaling the whole dataset once before CV, which lets every fold 'see' data it's supposed to be held out from.",
 5: 'Typically n_estimators or max_depth dominate on this dataset, since they most directly control model capacity; min_samples_split usually contributes the least. This generally matches intuition - capacity controls tend to matter more than fine-grained regularization knobs on a dataset this size.'}

WARNING:  Invalid HTTP request received.

	mean radius	mean texture	mean perimeter	mean area	mean smoothness	mean compactness	mean concavity	mean concave points	mean symmetry	mean fractal dimension	...	worst texture	worst perimeter	worst area	worst smoothness	worst compactness	worst concavity	worst concave points	worst symmetry	worst fractal dimension	diagnosis
0	17.99	10.38	122.80	1001.0	0.11840	0.27760	0.3001	0.14710	0.2419	0.07871	...	17.33	184.60	2019.0	0.1622	0.6656	0.7119	0.2654	0.4601	0.11890	malignant
1	20.57	17.77	132.90	1326.0	0.08474	0.07864	0.0869	0.07017	0.1812	0.05667	...	23.41	158.80	1956.0	0.1238	0.1866	0.2416	0.1860	0.2750	0.08902	malignant
2	19.69	21.25	130.00	1203.0	0.10960	0.15990	0.1974	0.12790	0.2069	0.05999	...	25.53	152.50	1709.0	0.1444	0.4245	0.4504	0.2430	0.3613	0.08758	malignant
3	11.42	20.38	77.58	386.1	0.14250	0.28390	0.2414	0.10520	0.2597	0.09744	...	26.50	98.87	567.7	0.2098	0.8663	0.6869	0.2575	0.6638	0.17300	malignant
4	20.29	14.34	135.10	1297.0	0.10030	0.13280	0.1980	0.10430	0.1809	0.05883	...	16.67	152.20	1575.0	0.1374	0.2050	0.4000	0.1625	0.2364	0.07678	malignant

Word	Meaning
Reproducibility	Same code + same data → same result. Anyone can rerun your work.
Replicability	New code or new data → same conclusion. Someone else confirms your finding.

	name	time	n_estimators	max_depth	min_samples_split	seed	train_accuracy	val_accuracy	train_time_s	rounds	final_val_acc
0	rf_full_record	2026-08-10T09:24:27	100.0	8.0	2.0	42.0	1.0	0.964912	0.273	NaN	NaN
2	baseline_rf	2026-08-10T09:24:26	50.0	NaN	NaN	42.0	NaN	0.940000	0.400	NaN	NaN
1	sgd_partial_fit	2026-08-10T09:24:27	NaN	NaN	NaN	NaN	NaN	NaN	NaN	20.0	0.95614

	Parameters	Hyperparameters
Who sets them	the training process	you, before training
Examples	weights, biases, split points in a tree	learning rate, `max_depth`, batch size, number of trees
When fixed	after training	before training

	model	cv_accuracy	test_accuracy	fit_time_s
4	SVM	0.973626	0.982456	0.011417
0	LogisticRegression	0.956044	0.956140	0.628755
2	RandomForest	0.956044	0.964912	0.261352
3	GradientBoosting	0.951648	0.956140	0.823639
1	DecisionTree	0.925275	0.947368	0.017197

	best_score	time_s
Grid	1.000000	4.251302
Random	0.985816	7.014925
Bayesian	0.978723	15.666713

train_acc	▁
val_acc	▁

Lab 3: Reproducibility, Experiment Management, and Hyperparameter Search¶

1. Import Libraries¶

Why these libraries?¶

Learning Outcomes¶

2. Load Dataset¶

Why a built-in dataset for this lab?¶

Learning Outcomes¶

3. Why Reproducibility Matters¶

Learning Outcomes¶

4. Random Seeds¶

A Function That Seeds Everything¶

Learning Outcomes¶

5. Environment Management¶

Learning Outcomes¶

6. Recording Experiments¶

Learning Outcomes¶

7. Experiment Tracking with TrackIO¶

Learning Outcomes¶

Launch the dashboard¶

8. Parameters vs Hyperparameters¶

Learning Outcomes¶

9. Grid Search¶

Learning Outcomes¶

10. Random Search¶

Learning Outcomes¶

11. Bayesian Optimization¶

Learning Outcomes¶

12. Data Leakage & Pipelines¶

Learning Outcomes¶

13. AutoML and AutoGluon¶

Learning Outcomes¶

14. Weights & Biases Sweeps¶

One-time setup: create your account and connect it to Kaggle¶

Learning Outcomes¶

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary: