# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os

# Input data files (if any are attached) are available in the read-only "../input/" directory
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to /kaggle/working/, which is preserved as output
# when you create a version using "Save & Run All"

# scikit-learn, pandas, numpy, matplotlib, and joblib are already installed.
# optuna, autogluon.tabular, wandb, and trackio are NOT preinstalled on Kaggle
# - each is installed right where it's first used, not here, so you only pay
# the install cost for the sections you actually run.

for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

import json, time, os, platform, subprocess
from pathlib import Path
from datetime import datetime

RUNS_DIR = Path("/kaggle/working/runs"); RUNS_DIR.mkdir(parents=True, exist_ok=True)

from sklearn.datasets import load_breast_cancer

data = load_breast_cancer(as_frame=True)
df = data.frame.copy()
df["diagnosis"] = df["target"].map({0: "malignant", 1: "benign"})
df = df.drop(columns=["target"])
print(df.shape)
df.head()

(569, 31)

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

X = df[data.feature_names]
y = df["diagnosis"]

# Train the SAME model three times, with no seed set anywhere
for i in range(3):
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2)
    m = RandomForestClassifier(n_estimators=50).fit(Xtr, ytr)
    print(f"Run {i+1}: accuracy = {m.score(Xte, yte):.4f}")

Run 1: accuracy = 0.9561
Run 2: accuracy = 0.9386
Run 3: accuracy = 0.9386

print(f"Run {i+1}: accuracy = {m.score(Xte, yte):.4f}")

import random

def set_seed(seed=42):
    """Set the seed for every library that might use randomness."""
    random.seed(seed)                    # Python's own random
    np.random.seed(seed)                 # NumPy
    os.environ["PYTHONHASHSEED"] = str(seed)
    try:
        import torch
        torch.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    except Exception:
        # Not installed, or installed but broken (e.g. a numpy/torch binary
        # ABI mismatch on this particular environment) - this lab never
        # actually needs torch, so skip it either way rather than crash.
        pass
    print(f"All seeds set to {seed}")

set_seed(42)

All seeds set to 42

try:
    import torch
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
except Exception:
    # No PyTorch installed - that's fine, skip it
    pass

# The try/except-import pattern in isolation, with a module that
# definitely doesn't exist, so you can see it fail safely.
try:
    import this_library_does_not_exist  # noqa
    print("imported it")
except Exception as e:
    print(f"Skipped cleanly: {type(e).__name__}: {e}")

print("Execution continues normally after the except block.")

# TODO

# 1. Rerun the Section 3 experiment with set_seed(42) + random_state=42
#    everywhere, three times, and confirm the accuracies are identical.

# 2. Rerun with seeds 0, 1, 2, 3, 4. Print the five accuracies and their
#    mean and standard deviation.

# TODO (Extension) - which source of randomness matters most?

# 1. Split only: fix the model's random_state=42, vary the split 15 times.

# 2. Model only: fix the split's random_state=42, vary the model 15 times.

# 3. Both unseeded, 15 times.

# Print a table: source | mean | std

# TODO (Extension) - the KFold shuffle trap: KFold(shuffle=True) without a
# random_state reshuffles differently every call, even inside the same script.

# 1. KFold(shuffle=True), no random_state, 5 repeats - print the scores.

# 2. Same but with random_state=42 - confirm the scores are now identical.

import sklearn
print("Python :", platform.python_version())
print("OS     :", platform.system(), platform.release())
print("NumPy  :", np.__version__)
print("Pandas :", pd.__version__)
print("Sklearn:", sklearn.__version__)

Python : 3.12.13
OS     : Linux 6.12.90+
NumPy  : 2.0.2
Pandas : 2.3.3
Sklearn: 1.6.1

# TODO

# 1. Print a requirements.txt (package==version) and an environment.yml
#    using the versions above.

# 2. Write capture_environment() -> dict with python/os/library versions,
#    git commit hash (or "not a git repo"), and a timestamp.
#    Tip: subprocess.run(["git","rev-parse","HEAD"], capture_output=True)

subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True)

import subprocess

# Demo with a command that always exists, so it runs even outside a git repo
result = subprocess.run(["echo", "hello from a subprocess"],
                         capture_output=True, text=True)

print("return code:", result.returncode)   # 0 means success
print("stdout:", result.stdout.strip())

# What capture_environment() should do with git specifically:
def git_commit_or_message():
    try:
        result = subprocess.run(["git", "rev-parse", "HEAD"],
                                 capture_output=True, text=True)
        if result.returncode == 0:
            return result.stdout.strip()
        return "not a git repo"
    except FileNotFoundError:
        return "git not installed"

print(git_commit_or_message())

def log_run(name, hyperparams, metrics, notes=""):
    """Save one experiment as a JSON file in runs/."""
    run = {
        "name": name,
        "time": datetime.now().isoformat(timespec="seconds"),
        "hyperparams": hyperparams,
        "metrics": metrics,
        "environment": {"python": platform.python_version(),
                        "numpy": np.__version__},
        "notes": notes,
    }
    path = RUNS_DIR / f"{name}_{run['time'].replace(':','-')}.json"
    path.write_text(json.dumps(run, indent=2))
    return path

log_run("baseline_rf",
        {"n_estimators": 50, "max_depth": None, "seed": 42},
        {"val_accuracy": 0.94, "train_time_s": 0.4},
        notes="First try, no tuning")

PosixPath('/kaggle/working/runs/baseline_rf_2026-08-24T07-14-00.json')

run = {
    "name": name,
    "time": datetime.now().isoformat(timespec="seconds"),
    "hyperparams": hyperparams,
    ...
}

from datetime import datetime

# isoformat() timespec options, side by side
now = datetime.now()
print(now.isoformat())                       # full precision, microseconds included
print(now.isoformat(timespec="seconds"))      # trimmed to seconds - what log_run() uses
print(now.isoformat(timespec="minutes"))      # trimmed further

# Building the same nested-dict shape log_run() uses, standalone:
example_run = {
    "name": "demo-run",
    "time": now.isoformat(timespec="seconds"),
    "hyperparams": {"n_estimators": 100, "max_depth": 8},
    "metrics": {"val_acc": 0.94},
}
import json
print(json.dumps(example_run, indent=2))

# TODO

# 1. Train a RandomForestClassifier (n_estimators=100, max_depth=8,
#    min_samples_split=2, seed=42) and log it with log_run(), including
#    train accuracy, validation accuracy, training time, and
#    capture_environment().

# TODO

# 2. Train a SGDClassifier for 20 rounds with partial_fit, logging
#    train/val accuracy each round. Plot both curves.

# TODO

# 3. Write load_all_runs() -> DataFrame of every run in runs/, sorted by
#    val_accuracy (highest first).

# TODO

# 4. Save one run's model, config, and metrics into its own folder instead
#    of one flat JSON file, e.g. runs/2026-08-09_rf-baseline_seed42/.

!pip install -q trackio

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.0/3.0 MB 44.1 MB/s eta 0:00:00a 0:00:01
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.7/61.7 kB 2.8 MB/s eta 0:00:00
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
gradio 5.50.0 requires gradio-client==1.14.0, but you have gradio-client 2.6.0 which is incompatible.

import trackio

trackio.init(project="stt-ai-breast-cancer", name="rf-baseline",
             config={"n_estimators": 100, "max_depth": 8, "seed": 42})

model = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42).fit(Xtr, ytr)

trackio.log({"train_acc": model.score(Xtr, ytr), "val_acc": model.score(Xte, yte)})
trackio.finish()
print("Run logged.")

* Trackio project initialized: stt-ai-breast-cancer
* Trackio metrics logged to: /root/.cache/huggingface/trackio
* psutil detected, enabling automatic CPU/system metrics logging
* Created new run: rf-baseline

* Run finished. Uploading logs to Trackio (please wait...)
Run logged.

# TODO

# 1. ONE TrackIO run (trackio.init once): train a GradientBoostingClassifier,
#    sweeping n_estimators from 10 to 100 in steps of 10. At each step, call
#    trackio.log({...}, step=n_estimators) with train_accuracy and
#    test_accuracy. trackio.finish() once at the end. This is the difference
#    between Section 6's "log the final number" and "log a curve".

# TODO

# 2. Sweep max_depth 1-20 for a DecisionTreeClassifier, log the overfit gap,
#    alert if gap > 0.05, then plot train vs test accuracy vs depth.

# TODO

# 3. Log 5 classifiers (LogisticRegression, DecisionTreeClassifier,
#    RandomForestClassifier, GradientBoostingClassifier, and an SVM inside a
#    Pipeline with StandardScaler) as 5 SEPARATE TrackIO runs. For each, log
#    cv_accuracy (5-fold), test_accuracy, and fit_time_s. Print a leaderboard
#    DataFrame sorted by cv_accuracy - build it from values computed locally
#    in Python, not by reading them back from TrackIO (its API is for
#    browsing/managing runs, not a metrics database you query from code).

trackio.show(project="stt-ai-breast-cancer", open_browser=False, block_thread=False)
print("Dashboard starting - open the printed URL above in your browser.")

m = RandomForestClassifier(n_estimators=10, max_depth=4, random_state=0).fit(X, y)

# Hyperparameters: what YOU chose
print("Hyperparameters:", {k: m.get_params()[k]
                           for k in ["n_estimators", "max_depth", "criterion"]})

# Parameters: what TRAINING learned - here, the tree splits
total_nodes = sum(t.tree_.node_count for t in m.estimators_)
print("Learned parameters (total tree nodes):", total_nodes)

Hyperparameters: {'n_estimators': 10, 'max_depth': 4, 'criterion': 'gini'}
Learned parameters (total tree nodes): 228

# TODO

# 1. Sort each of these into "parameter" or "hyperparameter": learning rate,
#    weights of a neural network layer, number of epochs, the split value at
#    a decision tree node, batch size, dropout rate, the bias term, k in
#    k-nearest-neighbours, the centroids found by k-means.
parameters = ["..."]
hyperparameters = ["..."]

# 2. Train a DecisionTreeClassifier at max_depth = 1, 3, 5, 10, and None.
#    Plot train and validation accuracy against depth, and mark where
#    validation accuracy peaks and where overfitting begins.

{k: m.get_params()[k] for k in ["n_estimators", "max_depth", "criterion"]}

# Dict comprehension, broken down step by step
params_i_want = ["n_estimators", "max_depth", "criterion"]
all_params = {"n_estimators": 10, "max_depth": 4, "criterion": "gini", "n_jobs": None}

# The comprehension:
picked = {k: all_params[k] for k in params_i_want}
print(picked)

# ...is shorthand for this loop:
picked_the_long_way = {}
for k in params_i_want:
    picked_the_long_way[k] = all_params[k]
print(picked_the_long_way)

# Generator expression vs list comprehension - same values, sum() doesn't care
node_counts_list = [3, 7, 5]          # pretend these came from t.tree_.node_count
total_a = sum(n for n in node_counts_list)   # generator expression
total_b = sum([n for n in node_counts_list]) # list comprehension - also works, just builds a list first
print(total_a, total_b)

# TODO

# 1. Sort each of these into "parameter" or "hyperparameter": learning rate,
#    weights of a neural network layer, number of epochs, the split value at
#    a decision tree node, batch size, dropout rate, the bias term, k in
#    k-nearest-neighbours, the centroids found by k-means.
parameters = ["..."]
hyperparameters = ["..."]

# 2. Train a DecisionTreeClassifier at max_depth = 1, 3, 5, 10, and None.
#    Plot train and validation accuracy against depth, and mark where
#    validation accuracy peaks and where overfitting begins.

from sklearn.model_selection import GridSearchCV, KFold

grid = {
    "n_estimators": [50, 100],
    "max_depth": [4, 8, None],
}
gs = GridSearchCV(RandomForestClassifier(random_state=42),
                  grid, cv=3, scoring="accuracy", n_jobs=-1)
gs.fit(Xtr, ytr)
print("Best settings:", gs.best_params_)
print("Best CV score:", round(gs.best_score_, 4))
print("Total fits:", len(gs.cv_results_["params"]) * 3)

Best settings: {'max_depth': 4, 'n_estimators': 100}
Best CV score: 0.9472
Total fits: 18

# TODO

# 1. Combinations, total fits, total time for 5x4x3x3 values, 5-fold CV.

# 2. Real grid search over n_estimators/max_depth: top 5 settings + heatmap.

# 3. Recompute the combination count for 8 hyperparameters, 3 values each.

# TODO (Extension) - build MyGridSearchCV from scratch. GridSearchCV feels
# like magic until you build it yourself - fill in the three methods below,
# then test on breast cancer with a small DecisionTreeClassifier grid and
# compare against sklearn's GridSearchCV on the same grid.
import itertools

class MyGridSearchCV:
    def __init__(self, estimator, param_grid, cv=5, scoring="accuracy"):
        self.estimator = estimator
        self.param_grid = param_grid
        self.cv = cv
        self.scoring = scoring
        self.best_params_ = None
        self.best_score_ = -np.inf
        self.cv_results_ = []

    def _get_all_combinations(self):
        """Return a list of dicts, one per combination of self.param_grid."""
        # YOUR CODE HERE
        pass

    def _score_single_combination(self, params, X, y):
        """Run k-fold CV for one param combination using KFold directly.
        Return the mean accuracy across folds."""
        # YOUR CODE HERE
        pass

    def fit(self, X, y):
        """Run the grid search. Populate best_params_/best_score_/cv_results_."""
        # YOUR CODE HERE
        pass

    def get_results_df(self):
        return pd.DataFrame(self.cv_results_).sort_values("mean_test_score", ascending=False)


# Test on breast cancer + compare with sklearn's GridSearchCV

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint

dist = {
    "n_estimators": randint(20, 300),
    "max_depth": randint(2, 20),
    "min_samples_split": randint(2, 20),
}
rs = RandomizedSearchCV(RandomForestClassifier(random_state=42),
                        dist, n_iter=20, cv=3, random_state=42, n_jobs=-1)
rs.fit(Xtr, ytr)
print("Best settings:", rs.best_params_)
print("Best CV score:", round(rs.best_score_, 4))

Best settings: {'max_depth': 10, 'min_samples_split': 3, 'n_estimators': 111}
Best CV score: 0.9494

from scipy.stats import randint
dist = {
    "n_estimators": randint(20, 300),
    ...
}

from scipy.stats import randint, loguniform

dist = randint(20, 300)     # a distribution object - nothing sampled yet
print(type(dist))
print(dist.rvs())           # sample one value
print(dist.rvs(random_state=42))   # sample one value, reproducibly
print(dist.rvs(size=5, random_state=42))  # sample 5 at once

# Telling a list apart from a distribution object, the way MyRandomSearchCV needs to:
def sample_one(value, rng):
    if hasattr(value, "rvs"):          # it's a scipy distribution
        return value.rvs(random_state=rng)
    else:                              # assume it's a plain list -> sample uniformly
        return rng.choice(value)

import numpy as np
rng = np.random.default_rng(42)
print(sample_one(randint(2, 20), rng))
print(sample_one([2, 5, 10, 20], rng))

# TODO

# 1. Grid search vs random search, both exactly 20 fits. Compare best score
#    and time in a small table.

# 2. Draw 2000 samples from np.random.uniform(0.0001, 0.1) and 2000 from
#    scipy.stats.loguniform(0.0001, 0.1). Print what share of each falls
#    below 0.001 - this is why log-uniform is the better choice for
#    something like a learning rate.

# TODO (Extension) - build MyRandomSearchCV from scratch. Same interface as
# MyGridSearchCV, plus n_iter and random_state. Each param_distributions
# value is either a list (sample uniformly) or a scipy distribution (anything
# with .rvs(random_state=...), like randint(2, 20)). Fill in the two methods
# below, then test on breast cancer and compare against sklearn's
# RandomizedSearchCV on the same distributions.
class MyRandomSearchCV:
    def __init__(self, estimator, param_distributions, n_iter=20, cv=5,
                 scoring="accuracy", random_state=None):
        self.estimator = estimator
        self.param_distributions = param_distributions
        self.n_iter = n_iter
        self.cv = cv
        self.scoring = scoring
        self.random_state = random_state
        self.best_params_ = None
        self.best_score_ = -np.inf
        self.cv_results_ = []

    def _sample_params(self, rng):
        """One value per hyperparameter: rng.choice() for a list,
        .rvs(random_state=rng) for a scipy distribution."""
        # YOUR CODE HERE
        pass

    def fit(self, X, y):
        """Run n_iter random trials with a manual KFold loop
        (reuse the same CV logic as MyGridSearchCV)."""
        # YOUR CODE HERE
        pass

    def get_results_df(self):
        return pd.DataFrame(self.cv_results_).sort_values("mean_test_score", ascending=False)


# Test on breast cancer + compare with sklearn's RandomizedSearchCV

# TODO (Extension) - coverage plot. Grid and random search don't just score
# differently, they land in different places in parameter space. Scatter the
# 20 grid points and the 20 random-search points from the first TODO above,
# colour-coded, on the same (n_estimators, max_depth) axes.

!pip install -q optuna

import optuna
from sklearn.model_selection import cross_val_score

optuna.logging.set_verbosity(optuna.logging.WARNING)

def objective(trial):
    params = {
        "n_estimators":      trial.suggest_int("n_estimators", 20, 300),
        "max_depth":         trial.suggest_int("max_depth", 2, 20),
        "min_samples_split": trial.suggest_int("min_samples_split", 2, 20),
    }
    model = RandomForestClassifier(**params, random_state=42)
    return cross_val_score(model, Xtr, ytr, cv=3).mean()

study = optuna.create_study(direction="maximize",
                            sampler=optuna.samplers.TPESampler(seed=42))
study.optimize(objective, n_trials=20)
print("Best settings:", study.best_params)
print("Best score:", round(study.best_value, 4))

Best settings: {'n_estimators': 59, 'max_depth': 7, 'min_samples_split': 8}
Best score: 0.9472

def objective(trial):
    params = {
        "n_estimators": trial.suggest_int("n_estimators", 20, 300),
        ...
    }
    model = RandomForestClassifier(**params, random_state=42)
    return cross_val_score(model, Xtr, ytr, cv=3).mean()

study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=20)

# **kwargs unpacking, isolated from Optuna entirely
def describe_forest(n_estimators, max_depth):
    print(f"A forest with {n_estimators} trees, max depth {max_depth}")

settings = {"n_estimators": 100, "max_depth": 8}
describe_forest(**settings)          # same as describe_forest(n_estimators=100, max_depth=8)

# A minimal objective(trial) that doesn't need real data or sklearn,
# to see the suggest_* / optimize shape in isolation:
import optuna
optuna.logging.set_verbosity(optuna.logging.WARNING)

def toy_objective(trial):
    x = trial.suggest_float("x", -10, 10)
    return -(x - 3) ** 2   # maximised when x == 3

toy_study = optuna.create_study(direction="maximize")
toy_study.optimize(toy_objective, n_trials=30)
print("Best x found:", toy_study.best_params)

# TODO

# 1. Grid vs random vs Optuna, 20 trials each, one comparison table.

# 2. Plot best-score-so-far for random search and Optuna on the same graph.

# TODO (Extension) - parameter importance. Optuna can compute how much each
# hyperparameter contributed to the score variance across your `study` from
# above. Print optuna.importance.get_param_importances(study) and plot a
# horizontal bar chart (param name on y-axis, importance on x-axis).

# TODO (Extension) - is Optuna itself reproducible? Run two fresh studies
# with the exact same objective, n_trials=10, and TPESampler(seed=42).
# Extract [t.value for t in study.trials] from each and assert they are
# equal. What does this prove - and where would it break?

# TODO (Extension) - a regression task. Everything so far has been
# classification; switch to regression. Tune GradientBoostingRegressor on the
# California Housing dataset (built into scikit-learn, downloaded on first
# use - needs Internet On). Minimise RMSE, 20 trials. n_estimators 50-300,
# learning_rate log-uniform 1e-3 to 1.0, max_depth 2-8.

# TODO (Extension) - let Optuna pick the model family. Use
# trial.suggest_categorical to let Optuna choose both the model family AND
# its hyperparameters in a single study - a tiny hand-rolled AutoML. Include
# DecisionTree, RandomForest, and KNN. Run 30 trials on the breast cancer
# data, then count how many trials picked each family.
def objective_automl(trial):
    classifier = trial.suggest_categorical("classifier", ["DT", "RF", "KNN"])
    # if classifier == "DT": suggest DT params, build DecisionTreeClassifier
    # elif classifier == "RF": suggest RF params, build RandomForestClassifier
    # elif classifier == "KNN": suggest KNN params, build KNeighborsClassifier
    # return 5-fold CV accuracy on Xtr, ytr
    pass

# Run 30 trials, print best, then count how often each family was chosen.

from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

# --- WRONG: scale before CV (leaky) ---
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)  # sees ALL data, including future validation folds!

leaky_score = cross_val_score(
    SVC(C=1, kernel="rbf", random_state=42), X_scaled, y, cv=5
).mean()

# --- RIGHT: scale inside a Pipeline ---
pipe = Pipeline([
    ("scaler", StandardScaler()),
    ("svm", SVC(C=1, kernel="rbf", random_state=42)),
])
correct_score = cross_val_score(pipe, X, y, cv=5).mean()

print(f"Leaky  (wrong)    CV accuracy: {leaky_score:.4f}")
print(f"Correct (Pipeline) CV accuracy: {correct_score:.4f}")
print(f"Difference: {leaky_score - correct_score:+.4f}")
if abs(leaky_score - correct_score) < 0.005:
    print("(Small or zero gap here - breast cancer has 569 fairly stable rows, so a "
          "per-fold scaler and a whole-dataset scaler end up with nearly identical "
          "mean/std anyway. The leak is still real; it just doesn't move the needle "
          "much on this particular dataset. The Pipeline fix costs nothing, so use it "
          "regardless of whether the gap happens to be visible - on a smaller dataset, "
          "or with a leakier step like selecting features by correlation with the "
          "label, the same mistake can inflate scores by many points.)")

Leaky  (wrong)    CV accuracy: 0.9736
Correct (Pipeline) CV accuracy: 0.9736
Difference: +0.0000
(Small or zero gap here - breast cancer has 569 fairly stable rows, so a per-fold scaler and a whole-dataset scaler end up with nearly identical mean/std anyway. The leak is still real; it just doesn't move the needle much on this particular dataset. The Pipeline fix costs nothing, so use it regardless of whether the gap happens to be visible - on a smaller dataset, or with a leakier step like selecting features by correlation with the label, the same mistake can inflate scores by many points.)

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC

pipe = Pipeline([
    ("scaler", StandardScaler()),
    ("svm", SVC()),
])

# get_params() shows exactly the step__param names GridSearchCV expects:
for name in pipe.get_params():
    if "__" in name:
        print(name)

# TODO

# 1. One sentence: why does fitting StandardScaler on the whole dataset
#    before cross_val_score leak information from the validation folds?
answer = "..."

# 2. Build Pipeline(StandardScaler -> SVC) and tune it with GridSearchCV on
#    the breast cancer data. svm__C: [0.01, 0.1, 1, 10, 100],
#    svm__kernel: ["linear", "rbf"] - note the svm__ prefix, needed because
#    "svm" is the pipeline step's name.

!pip install -q autogluon.tabular

   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 514.8/514.8 kB 13.7 MB/s eta 0:00:0000:01
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 248.4/248.4 kB 11.5 MB/s eta 0:00:00
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 112.3/112.3 kB 4.8 MB/s eta 0:00:00
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 90.7/90.7 kB 3.3 MB/s eta 0:00:00

from autogluon.tabular import TabularPredictor

train_df = pd.concat([Xtr, ytr], axis=1)
test_df  = pd.concat([Xte, yte], axis=1)

predictor = TabularPredictor(label="diagnosis", path="ag_models").fit(
    train_df,
    time_limit=60,                # stop after 1 minute - this is a small dataset
    presets="medium_quality",     # use "best_quality" if you have more time
)
print(predictor.evaluate(test_df))

Verbosity: 2 (Standard Logging)
=================== System Info ===================
AutoGluon Version:  1.6.1
Python Version:     3.12.13
Operating System:   Linux
Platform Machine:   x86_64
Platform Version:   #1 SMP Sat May 30 15:40:53 UTC 2026
CPU Count:          4
Pytorch Version:    2.10.0+cpu
CUDA Version:       CUDA is not available
Memory Avail:       29.39 GB / 31.35 GB (93.8%)
Disk Space Avail:   19.50 GB / 19.52 GB (99.9%)
===================================================
Presets specified: ['medium_quality']
Using hyperparameters preset: hyperparameters='default'
Beginning AutoGluon training ... Time limit = 60s
AutoGluon will save models to "/kaggle/working/ag_models"
Train Data Rows:    455
Train Data Columns: 30
Label Column:       diagnosis
AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).
	2 unique label values:  ['benign', 'malignant']
	If 'binary' is not the correct problem_type, please manually specify the problem_type parameter during Predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression', 'quantile'])
Problem Type:       binary
Preprocessing data...
Selected class <--> label mapping:  class 1 = malignant, class 0 = benign
	Note: For your binary classification, AutoGluon arbitrarily selected which label-value represents positive (malignant) vs negative (benign) class.
	To explicitly set the positive_class, either rename classes to 1 and 0, or specify positive_class in Predictor init.
Using Feature Generators to preprocess the data ...
Fitting AutoMLPipelineFeatureGenerator...
	Available Memory:                    30094.77 MB
	Train Data (Original)  Memory Usage: 0.10 MB (0.0% of available memory)
	Inferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.
	Stage 1 Generators:
		Fitting AsTypeFeatureGenerator...
	Stage 2 Generators:
		Fitting FillNaFeatureGenerator...
	Stage 3 Generators:
		Fitting IdentityFeatureGenerator...
	Stage 4 Generators:
		Fitting DropUniqueFeatureGenerator...
	Stage 5 Generators:
		Fitting DropDuplicatesFeatureGenerator...
	Types of features in original data (raw dtype, special dtypes):
		('float', []) : 30 | ['mean radius', 'mean texture', 'mean perimeter', 'mean area', 'mean smoothness', ...]
	Types of features in processed data (raw dtype, special dtypes):
		('float', []) : 30 | ['mean radius', 'mean texture', 'mean perimeter', 'mean area', 'mean smoothness', ...]
	0.1s = Fit runtime
	30 features in original data used to generate 30 features in processed data.
	Train Data (Processed) Memory Usage: 0.10 MB (0.0% of available memory)
Data preprocessing and feature engineering runtime = 0.09s ...
AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'
	To change this, specify the eval_metric parameter of Predictor()
Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 364, Val Rows: 91
User-specified model hyperparameters to be fit:
{
	'NN_TORCH': [{}],
	'GBM': [{'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}}, {}, {'learning_rate': 0.03, 'num_leaves': 128, 'feature_fraction': 0.9, 'min_data_in_leaf': 3, 'ag_args': {'name_suffix': 'Large', 'priority': 0, 'hyperparameter_tune_kwargs': None}}],
	'CAT': [{}],
	'XGB': [{}],
	'FASTAI': [{}],
	'RF': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],
	'XT': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],
}
Fitting 11 L1 models, fit_strategy="sequential" ...
Fitting model: LightGBMXT ... Training model for up to 59.91s of the 59.90s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.1/29.4 GB
	0.978	 = Validation score   (accuracy)
	5.07s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: LightGBM ... Training model for up to 54.82s of the 54.82s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.1/29.3 GB
	0.967	 = Validation score   (accuracy)
	0.51s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: RandomForestGini ... Training model for up to 54.29s of the 54.29s of remaining time.
	Fitting with cpus=4, gpus=0
	0.956	 = Validation score   (accuracy)
	1.03s	 = Training   runtime
	0.11s	 = Validation runtime
Fitting model: RandomForestEntr ... Training model for up to 53.14s of the 53.14s of remaining time.
	Fitting with cpus=4, gpus=0
	0.956	 = Validation score   (accuracy)
	1.02s	 = Training   runtime
	0.11s	 = Validation runtime
Fitting model: CatBoost ... Training model for up to 51.99s of the 51.99s of remaining time.
	Fitting with cpus=2, gpus=0
	0.978	 = Validation score   (accuracy)
	3.61s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: ExtraTreesGini ... Training model for up to 48.37s of the 48.37s of remaining time.
	Fitting with cpus=4, gpus=0
	0.967	 = Validation score   (accuracy)
	1.05s	 = Training   runtime
	0.11s	 = Validation runtime
Fitting model: ExtraTreesEntr ... Training model for up to 47.19s of the 47.19s of remaining time.
	Fitting with cpus=4, gpus=0
	0.967	 = Validation score   (accuracy)
	1.04s	 = Training   runtime
	0.11s	 = Validation runtime
Fitting model: NeuralNetFastAI ... Training model for up to 46.02s of the 46.01s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.0/29.3 GB
	0.978	 = Validation score   (accuracy)
	3.4s	 = Training   runtime
	0.02s	 = Validation runtime
Fitting model: XGBoost ... Training model for up to 42.57s of the 42.56s of remaining time.
	Fitting with cpus=2, gpus=0
	0.967	 = Validation score   (accuracy)
	0.31s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: NeuralNetTorch ... Training model for up to 42.23s of the 42.23s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.0/29.3 GB
	0.978	 = Validation score   (accuracy)
	6.47s	 = Training   runtime
	0.02s	 = Validation runtime
Fitting model: LightGBMLarge ... Training model for up to 35.73s of the 35.72s of remaining time.
	Fitting with cpus=2, gpus=0, mem=0.1/29.2 GB
	0.956	 = Validation score   (accuracy)
	1.21s	 = Training   runtime
	0.0s	 = Validation runtime
Fitting model: WeightedEnsemble_L2 ... Training model for up to 59.91s of the 34.50s of remaining time.
	Fitting 1 model on all data | Fitting with cpus=4, gpus=0, mem=0.0/29.2 GB
	Ensemble Weights: {'LightGBMXT': 1.0}
	0.978	 = Validation score   (accuracy)
	0.11s	 = Training   runtime
	0.0s	 = Validation runtime
AutoGluon training complete, total runtime = 25.95s ... Best model: WeightedEnsemble_L2 | Estimated inference throughput: 22332.3 rows/s (91 batch size)
Disabling decision threshold calibration for metric `accuracy` due to having fewer than 10000 rows of validation data for calibration, to avoid overfitting (91 rows).
	`accuracy` is generally not improved through threshold calibration. Force calibration via specifying `calibrate_decision_threshold=True`.
TabularPredictor saved. To load, use: predictor = TabularPredictor.load("/kaggle/working/ag_models")

{'accuracy': 0.9385964912280702, 'balanced_accuracy': np.float64(0.9299999999999999), 'mcc': np.float64(0.8804608268433263), 'roc_auc': np.float64(0.9846874999999999), 'f1': 0.9247311827956989, 'precision': 1.0, 'recall': 0.86}

train_df = pd.concat([Xtr, ytr], axis=1)

import pandas as pd

# axis=1 (side by side) vs axis=0 (stacked), on tiny toy frames
features = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
labels = pd.Series(["yes", "no"], name="target")

side_by_side = pd.concat([features, labels], axis=1)   # what AutoGluon wants
print(side_by_side)

stacked = pd.concat([features, features], axis=0)      # NOT what you want here
print(stacked)

# TODO

# 1. Leaderboard: models trained, winner, is it a WeightedEnsemble blend?

# 2. Feature importance ranking - does the top feature match intuition?

# 3. Compare Optuna (Section 11) vs AutoGluon: score, wall time, human time.

!pip install -q wandb

# Run once per session
from kaggle_secrets import UserSecretsClient

# Initialize the secrets client
user_secrets = UserSecretsClient()

# Retrieve your secret value using its label
secret_value = user_secrets.get_secret("WANDB_API_KEY")

import os
os.environ["WANDB_API_KEY"] = secret_value

import wandb
wandb.login()
print("Logged in to W&B.")

wandb.init(project="stt-ai-breast-cancer-hpo", name="rf-baseline",
           config={"n_estimators": 100, "max_depth": 8, "seed": 42})

cfg = wandb.config
model = RandomForestClassifier(n_estimators=cfg.n_estimators,
                               max_depth=cfg.max_depth,
                               random_state=cfg.seed).fit(Xtr, ytr)

wandb.log({"train_acc": model.score(Xtr, ytr),
           "val_acc":   model.score(Xte, yte)})
wandb.finish()

# All three parameters use discrete "values" lists, not "min"/"max" ranges.
# This matters specifically for method="grid": grid search needs a finite set
# of values for every parameter. Give it a "min"/"max" range instead and it
# will silently enumerate EVERY integer in that range - for n_estimators
# (20-300) x max_depth (2-20) x 4 values that's ~21,000 grid points, which
# looks like an infinite loop (it's not infinite, just enormous and slow).
# Using "values" everywhere also means grid/random/bayes all search the exact
# same space, making the count=8 comparison in the next TODO a fair one.
sweep_config = {
    "method": "bayes",                                   # or "grid" / "random"
    "metric": {"name": "val_acc", "goal": "maximize"},
    "parameters": {
        "n_estimators":      {"values": [20, 60, 100, 150, 200, 300]},
        "max_depth":         {"values": [2, 4, 8, 12, 16, 20]},
        "min_samples_split": {"values": [2, 5, 10, 20]},
    },
}

def train_one():
    wandb.init()
    cfg = wandb.config
    model = RandomForestClassifier(n_estimators=cfg.n_estimators,
                                   max_depth=cfg.max_depth,
                                   min_samples_split=cfg.min_samples_split,
                                   random_state=42).fit(Xtr, ytr)
    wandb.log({"val_acc": model.score(Xte, yte)})

# sweep_id = wandb.sweep(sweep_config, project="stt-ai-breast-cancer-hpo")
# wandb.agent(sweep_id, function=train_one, count=8)

# TODO

# 1. Run the sweep 3 times (method = grid, random, bayes), 8 runs each.
#    Print the best val_acc from each.

# 2. (On wandb.ai) note which hyperparameter matters most in the
#    parallel-coordinates chart - write your observation as a comment.

# ==========================================================
# 🏆 Lab Exercise: One Reproducible Training Script
# ==========================================================
#
# Objective
# ---------
# Combine everything from Sections 1-6 into one script that a classmate can
# run to get your exact validation accuracy - the real test of reproducibility.

# ✅ Task 1: Write train.py that reads settings from config.json.
#
# ✅ Task 2: Set every seed inside the script.
#
# ✅ Task 3: Load the data and print its file hash.
#
# ✅ Task 4: Train the model and print the validation accuracy.
#
# ✅ Task 5: Save the model, metrics, and environment info into one run folder.
#
# ✅ Task 6: Swap train.py, config.json, and requirements.txt with a
#            classmate. Run theirs. Do you get their exact number? If not,
#            find out why and write down the cause.

# TODO

# On the Wine dataset, build a leaderboard with:
# 1. DummyClassifier baseline (most-frequent-class strategy) - what does a
#    classifier that isn't even trying score?
# 2. LogisticRegression (default)
# 3. RandomForest tuned with Optuna (30 trials)
# 4. DIY AutoML: >=4 model families, each with its own small GridSearchCV
# Print one leaderboard sorted by CV accuracy.

# TODO

# Create a reusable function compare_search_methods(X, y, n_trials=20) that
# runs grid search, random search, and Bayesian search (Optuna) on any
# (X, y) with the same fit budget, and returns one DataFrame comparing best
# score and time taken for all three - so you never have to copy-paste the
# Section 9-11 code again for a new dataset. Test it on the Wine data
# from Part A.

# TODO - write your answers as a dict of short strings, no need to run code
reflection_answers = {
    1: "...",
    2: "...",
    3: "...",
    4: "...",
    5: "...",
}

	mean radius	mean texture	mean perimeter	mean area	mean smoothness	mean compactness	mean concavity	mean concave points	mean symmetry	mean fractal dimension	...	worst texture	worst perimeter	worst area	worst smoothness	worst compactness	worst concavity	worst concave points	worst symmetry	worst fractal dimension	diagnosis
0	17.99	10.38	122.80	1001.0	0.11840	0.27760	0.3001	0.14710	0.2419	0.07871	...	17.33	184.60	2019.0	0.1622	0.6656	0.7119	0.2654	0.4601	0.11890	malignant
1	20.57	17.77	132.90	1326.0	0.08474	0.07864	0.0869	0.07017	0.1812	0.05667	...	23.41	158.80	1956.0	0.1238	0.1866	0.2416	0.1860	0.2750	0.08902	malignant
2	19.69	21.25	130.00	1203.0	0.10960	0.15990	0.1974	0.12790	0.2069	0.05999	...	25.53	152.50	1709.0	0.1444	0.4245	0.4504	0.2430	0.3613	0.08758	malignant
3	11.42	20.38	77.58	386.1	0.14250	0.28390	0.2414	0.10520	0.2597	0.09744	...	26.50	98.87	567.7	0.2098	0.8663	0.6869	0.2575	0.6638	0.17300	malignant
4	20.29	14.34	135.10	1297.0	0.10030	0.13280	0.1980	0.10430	0.1809	0.05883	...	16.67	152.20	1575.0	0.1374	0.2050	0.4000	0.1625	0.2364	0.07678	malignant

Word	Meaning
Reproducibility	Same code + same data → same result. Anyone can rerun your work.
Replicability	New code or new data → same conclusion. Someone else confirms your finding.

	Parameters	Hyperparameters
Who sets them	the training process	you, before training
Examples	weights, biases, split points in a tree	learning rate, `max_depth`, batch size, number of trees
When fixed	after training	before training

1. Import Libraries¶

Why these libraries?¶

Learning Outcomes¶

2. Load Dataset¶

Why a built-in dataset for this lab?¶

Learning Outcomes¶

3. Why Reproducibility Matters¶

Learning Outcomes¶

4. Random Seeds¶

A Function That Seeds Everything¶

Learning Outcomes¶

5. Environment Management¶

Learning Outcomes¶

6. Recording Experiments¶

Learning Outcomes¶

7. Experiment Tracking with TrackIO¶

Learning Outcomes¶

Launch the dashboard¶

8. Parameters vs Hyperparameters¶

Learning Outcomes¶

9. Grid Search¶

Learning Outcomes¶

10. Random Search¶

Learning Outcomes¶

11. Bayesian Optimization¶

Learning Outcomes¶

12. Data Leakage & Pipelines¶

Learning Outcomes¶

13. AutoML and AutoGluon¶

Learning Outcomes¶

14. Weights & Biases Sweeps¶

One-time setup: create your account and connect it to Kaggle¶

Learning Outcomes¶

15. Lab Exercise¶

16. Final Challenge¶

Part A: build a leaderboard from the floor up¶

Part B: a reusable comparison function¶

Reflection (write your answers as comments, no code needed)¶