Self-contained ML take-home with live review

"""Practice data: will a customer cancel next month? One row per customer.

load() returns (train, test); each is a list of dicts with the features
below and "label" (1 = cancelled). Treat it like data you were handed:
read it before you trust it.
"""

import math
import random

FEATURES = ["tenure_months", "monthly_spend", "support_tickets", "logins_last_30d", "discount_pct", "days_since_refund"]


def _customer(rng):
    tenure = rng.randint(1, 72)
    spend = round(rng.lognormvariate(3.5, 0.5), 2)
    tickets = min(int(rng.expovariate(0.7)), 12)
    logins = max(0, int(rng.gauss(18, 8)))
    discount = rng.choice([0, 0, 0, 10, 20])
    score = -0.4 - 0.035 * tenure + 0.45 * tickets - 0.08 * logins + 0.004 * spend + 0.03 * discount
    label = int(rng.random() < 1 / (1 + math.exp(-score)))
    if rng.random() < 0.15:
        logins = -1  # the export writes -1 when the login service had no record
    refund = rng.randint(0, 10) if label and rng.random() < 0.9 else rng.choice([999] * 9 + [rng.randint(0, 300)])
    return {
        "tenure_months": tenure, "monthly_spend": spend, "support_tickets": tickets,
        "logins_last_30d": logins, "discount_pct": discount, "days_since_refund": refund, "label": label,
    }  # fmt: skip


def load(seed=0, n_train=3000, n_test=1000):
    rng = random.Random(seed)
    train = [_customer(rng) for _ in range(n_train)]
    test = [_customer(rng) for _ in range(n_test)]
    for i in rng.sample(range(n_test), n_test // 10):
        test[i] = dict(rng.choice(train))  # the export overlapped two date ranges
    return train, test

"""Checks to run on a dataset before modeling it, and an honest error bar.
Fill these in first; they are what you will show in the review."""


def auc(scores, labels):
    """Area under the ROC curve: the chance a random positive outscores a
    random negative (ties count half)."""
    raise NotImplementedError


def overlap(train, test, features):
    """Share of test rows that also appear in train (same features)."""
    raise NotImplementedError


def single_feature_auc(rows, feature):
    """How well one raw feature alone separates the classes, in either
    direction: 0.5 = not at all, 1.0 = perfectly."""
    raise NotImplementedError


def bootstrap_ci(scores, labels, metric=None, n=1000, seed=0, level=0.95):
    """Percentile bootstrap interval (lo, hi) for a metric (default: auc)
    computed over resampled test rows."""
    raise NotImplementedError