Capstone Session 8

from pathlib import Path
import importlib
from importlib import metadata as importlib_metadata
import json
import subprocess
import sys
from urllib.parse import quote

IS_COLAB = 'google.colab' in sys.modules
GITHUB_REPO_OWNER = 'FrancisBurnet'
GITHUB_REPO_NAME = 'francisburnet'
GITHUB_REPO_BRANCH = 'main'
CAPSTONE_ROOT = Path('Incremental Capstones/Machine Learning Using Python/Capstone Session 8')
MOVIES_FILENAME = 'movies.csv'
RATINGS_FILENAME = 'ratings.csv'


def build_raw_github_url(relative_path: Path) -> str:
    encoded_path = quote(relative_path.as_posix(), safe='/')
    return (
        f"https://raw.githubusercontent.com/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/"
        f"{GITHUB_REPO_BRANCH}/{encoded_path}"
    )


def resolve_capstone_dir() -> Path | None:
    current = Path.cwd().resolve()
    capstone_parts = CAPSTONE_ROOT.parts
    for candidate in [current, *current.parents]:
        if len(candidate.parts) >= len(capstone_parts) and candidate.parts[-len(capstone_parts):] == capstone_parts:
            return candidate
        nested_candidate = candidate / CAPSTONE_ROOT
        if nested_candidate.exists():
            return nested_candidate
    return None


CAPSTONE_DIR = resolve_capstone_dir()
MOVIES_URL = build_raw_github_url(CAPSTONE_ROOT / MOVIES_FILENAME)
RATINGS_URL = build_raw_github_url(CAPSTONE_ROOT / RATINGS_FILENAME)

if CAPSTONE_DIR is not None:
    OUTPUT_ROOT = CAPSTONE_DIR
    OUTPUT_MODE = 'permanent capstone outputs'
    OUTPUT_DISPLAY = (CAPSTONE_ROOT / 'outputs').as_posix()
else:
    runtime_root = Path('/content/capstone-session-8-runtime') if IS_COLAB else Path.cwd().resolve() / 'capstone-session-8-runtime'
    OUTPUT_ROOT = runtime_root
    OUTPUT_MODE = 'runtime scratch outputs; export final artifacts back into the capstone outputs folder'
    OUTPUT_DISPLAY = 'capstone-session-8-runtime/outputs'

OUTPUTS_DIR = (OUTPUT_ROOT / 'outputs').resolve()
PLOTS_DIR = OUTPUTS_DIR / 'plots'
OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
PLOTS_DIR.mkdir(parents=True, exist_ok=True)


def installed_version(package_name: str) -> str | None:
    try:
        return importlib_metadata.version(package_name)
    except importlib_metadata.PackageNotFoundError:
        return None


def surprise_import_ready() -> bool:
    try:
        importlib.import_module('surprise')
        return True
    except Exception:
        return False


numpy_version = installed_version('numpy')
needs_numpy_pin = numpy_version is None or int(numpy_version.split('.')[0]) >= 2
needs_surprise_setup = needs_numpy_pin or not surprise_import_ready()

if needs_surprise_setup:
    try:
        if IS_COLAB:
            subprocess.run(['apt-get', 'update', '-qq'], check=True)
            subprocess.run(['apt-get', 'install', '-y', 'build-essential'], check=True)
        subprocess.run([sys.executable, '-m', 'pip', 'install', '--force-reinstall', 'numpy<2'], check=True)
        subprocess.run([sys.executable, '-m', 'pip', 'install', '--force-reinstall', '--no-deps', 'scikit-surprise'], check=True)
        importlib.invalidate_caches()
    except subprocess.CalledProcessError as exc:
        if not IS_COLAB:
            raise RuntimeError(
                'Session 8 requires Microsoft Visual C++ Build Tools and a NumPy 1.x runtime for scikit-surprise. '
                'Install the Visual Studio C++ workload, then rerun this cell.'
            ) from exc
        raise

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from IPython.display import display
from surprise import Dataset, KNNBasic, NMF as SurpriseNMF, Reader, SVD
from surprise.model_selection import KFold as SurpriseKFold, cross_validate

sns.set_theme(style='whitegrid')
pd.set_option('display.max_columns', 100)

print('Runtime:', 'Google Colab' if IS_COLAB else 'Notebook runtime')
print('Capstone artifact path:', CAPSTONE_ROOT.as_posix())
print('Movies source:', MOVIES_URL)
print('Ratings source:', RATINGS_URL)
print('Output mode:', OUTPUT_MODE)
print('Output target:', OUTPUT_DISPLAY)
print('NumPy version:', np.__version__)
print('scikit-surprise import ready')

movies = pd.read_csv(MOVIES_URL)
ratings = pd.read_csv(RATINGS_URL)
merged = ratings.merge(movies[['movieId', 'title']], on='movieId', how='left')
user_item = merged.pivot_table(index='userId', columns='title', values='rating')
display(merged.head())
print('Movies source used:', MOVIES_URL)
print('Ratings source used:', RATINGS_URL)
print('Movies shape:', movies.shape)
print('Ratings shape:', ratings.shape)
print('Merged shape:', merged.shape)
print('User-item shape:', user_item.shape)

user_filled = user_item.apply(lambda row: row.fillna(row.mean()), axis=1)
user_corr = user_filled.T.corr()
user_1_corr = user_corr.loc[1].drop(index=1).dropna().sort_values(ascending=False)
top_50_users = user_1_corr.head(50)
movie_32_title = movies.loc[movies['movieId'] == 32, 'title'].iloc[0]
movie_32_ratings = merged.loc[merged['movieId'] == 32, ['userId', 'rating']].set_index('userId')
eligible = top_50_users[top_50_users.index.isin(movie_32_ratings.index)]
if eligible.empty:
    predicted_user_1_rating = float(merged.loc[merged['movieId'] == 32, 'rating'].mean())
else:
    weighted_ratings = movie_32_ratings.loc[eligible.index, 'rating']
    denominator = float(np.abs(eligible).sum())
    predicted_user_1_rating = float(np.dot(eligible.values, weighted_ratings.values) / denominator) if denominator else float(weighted_ratings.mean())

top_50_df = top_50_users.reset_index()
top_50_df.columns = ['userId', 'correlation']
top_50_df.to_csv(OUTPUTS_DIR / 'session_8_top_50_user_correlations.csv', index=False)
display(top_50_df.head(10))
{'movieId_32_title': movie_32_title, 'predicted_user_1_rating_for_movie_32': round(predicted_user_1_rating, 4)}

item_filled = user_item.apply(lambda column: column.fillna(column.mean()), axis=0)
movie_corr = item_filled.corr()
jurassic_title = 'Jurassic Park (1993)'
jurassic_similar = movie_corr[jurassic_title].drop(index=jurassic_title).dropna().sort_values(ascending=False).head(10)
similar_movies_df = jurassic_similar.reset_index()
similar_movies_df.columns = ['title', 'correlation']
similar_movies_df.to_csv(OUTPUTS_DIR / 'session_8_similar_movies.csv', index=False)
display(similar_movies_df)

reader = Reader(rating_scale=(float(ratings['rating'].min()), float(ratings['rating'].max())))
surprise_data = Dataset.load_from_df(ratings[['userId', 'movieId', 'rating']], reader)
surprise_cv = SurpriseKFold(n_splits=5, random_state=42, shuffle=True)

model_specs = [
    (
        'KNNBasic',
        KNNBasic(k=20, sim_options={'name': 'msd', 'user_based': True}),
        {'k': 20, 'sim_options': {'name': 'msd', 'user_based': True}},
    ),
    (
        'SVD',
        SVD(random_state=42),
        {'random_state': 42},
    ),
    (
        'NMF',
        SurpriseNMF(random_state=42),
        {'random_state': 42},
    ),
]

fold_records = []
model_summaries = []
for model_name, algorithm, parameters in model_specs:
    cv_result = cross_validate(
        algorithm,
        surprise_data,
        measures=['RMSE'],
        cv=surprise_cv,
        verbose=False,
        n_jobs=1,
    )
    rmse_scores = [float(score) for score in cv_result['test_rmse']]
    for fold_index, rmse_score in enumerate(rmse_scores, start=1):
        fold_records.append(
            {
                'fold': fold_index,
                'model': model_name,
                'rmse': rmse_score,
                'parameters': json.dumps(parameters, sort_keys=True),
            }
        )
    model_summaries.append(
        {
            'model': model_name,
            'parameters': parameters,
            'rmse': float(np.mean(rmse_scores)),
            'best_score': float(np.min(rmse_scores)),
        }
    )

fold_results = pd.DataFrame(fold_records)
display(fold_results.head(9))
summary_results = pd.DataFrame(model_summaries).sort_values('rmse').reset_index(drop=True)
display(summary_results)
best_model = summary_results.iloc[0].to_dict()
best_model

fig, ax = plt.subplots(figsize=(10, 5))
bar_colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
bars = ax.bar(summary_results['model'], summary_results['rmse'], color=bar_colors)
ax.set_title('Session 8 Model-Based RMSE Comparison')
ax.set_ylabel('Average 5-Fold RMSE')
ax.set_xlabel('Model')
ax.bar_label(bars, fmt='%.3f', padding=3)
ax.set_ylim(0, summary_results['rmse'].max() + 0.08)
fig.tight_layout()
fig.savefig(PLOTS_DIR / 'model_based_rmse.png', dpi=150)
plt.show()
plt.close(fig)

fold_results.to_csv(OUTPUTS_DIR / 'session_8_model_cv_results.csv', index=False)
summary = {
    'movie_id_32_title': movie_32_title,
    'predicted_user_1_rating_for_movie_32': round(predicted_user_1_rating, 4),
    'top_50_user_correlations_saved': 'session_8_top_50_user_correlations.csv',
    'similar_movies_for_jurassic_park': similar_movies_df.to_dict(orient='records'),
    'model_cv_results': summary_results.to_dict(orient='records'),
    'best_model': best_model,
    'environment_note': 'Model-based recommendation is executed with scikit-surprise using KNNBasic, SVD, and NMF.',
}
with open(OUTPUTS_DIR / 'session_8_summary.json', 'w', encoding='utf-8') as handle:
    json.dump(summary, handle, indent=2)
summary