Capstone 1 Action Notebook

from pathlib import Path
from urllib.request import urlretrieve
import os

SITE_BASE = os.environ.get('FRANCISBURNET_SITE_BASE', 'https://francisburnet.com')
BASE_DIR = Path('/content/francisburnet_capstone_1')
INPUT_DIR = BASE_DIR / 'inputs'
OUTPUT_DIR = BASE_DIR / 'outputs'
INPUT_DIR.mkdir(parents=True, exist_ok=True)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

PDF_URL = SITE_BASE + '/artifact.php?path=Incremental+Capstones%2FApplied+Data+Science+with+Python%2FCapstone+1%2FCapstone_Session_1.pdf&download=1'
DATASET_URL = SITE_BASE + '/artifact.php?path=Incremental+Capstones%2FApplied+Data+Science+with+Python%2FCapstone+1%2FNSMES1988.csv&download=1'
NOTEBOOK_URL = SITE_BASE + '/artifact.php?path=Incremental+Capstones%2FApplied+Data+Science+with+Python%2FCapstone+1%2Fcapstone_1.ipynb&download=1'

pdf_path, _ = urlretrieve(PDF_URL, INPUT_DIR / 'Capstone_Session_1.pdf')
dataset_path, _ = urlretrieve(DATASET_URL, INPUT_DIR / 'NSMES1988.csv')
source_notebook_path, _ = urlretrieve(NOTEBOOK_URL, INPUT_DIR / 'capstone_1.ipynb')

print('SITE_BASE =', SITE_BASE)
print('PDF path =', pdf_path)
print('Dataset path =', dataset_path)
print('Source notebook path =', source_notebook_path)
print('Output directory =', OUTPUT_DIR)

import io
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display

print('Loaded libraries: io, json, numpy, pandas, matplotlib, display')

df = pd.read_csv(dataset_path)

print('Loaded dataset from:', dataset_path)
print('Shape:', df.shape)
display(df.head())

info_buffer = io.StringIO()
df.info(buf=info_buffer)

print('Rows:', df.shape[0])
print('Columns:', df.shape[1])
print('Column names:', df.columns.tolist())
print('\nData types and null counts:')
print(info_buffer.getvalue())

display(df.describe(include='all').transpose())

missing_summary = pd.DataFrame({
    'missing_count': df.isna().sum(),
    'missing_pct': (df.isna().mean() * 100).round(2),
}).sort_values(['missing_count', 'missing_pct'], ascending=False)

print('Columns with missing values:', int((missing_summary['missing_count'] > 0).sum()))
display(missing_summary)

age_summary = df['age'].describe()
income_summary = df['income'].describe()

print('age dtype:', df['age'].dtype)
print(age_summary)
print('\nincome dtype:', df['income'].dtype)
print(income_summary)
print('\nInterpretation note: the `age` values appear to be stored as age divided by 10, so 6.9 corresponds to approximately 69 years.')

json_path = OUTPUT_DIR / 'NSMES1988.json'
df.to_json(json_path, orient='records')

print('Saved JSON artifact to:', json_path)
with open(json_path, 'r', encoding='utf-8') as handle:
    json_preview = handle.read(500)
print('\nJSON preview:\n', json_preview)

memory_bytes = df.memory_usage(deep=True).sum()
recommended_category_columns = [
    column for column in ['health', 'adl', 'region', 'gender', 'married', 'employed', 'insurance', 'medicaid']
    if column in df.columns
]

print('Total memory (bytes):', memory_bytes)
print('Total memory (MB):', round(memory_bytes / (1024 ** 2), 3))
print('Recommended category columns:', recommended_category_columns)

if recommended_category_columns:
    category_memory = df[recommended_category_columns].astype('category').memory_usage(deep=True).sum()
    original_memory = df[recommended_category_columns].memory_usage(deep=True).sum()
    print('Potential memory reduction on recommended columns (bytes):', original_memory - category_memory)

df_clean = df.copy()
cleanup_notes = []

if 'Unnamed: 0' in df_clean.columns:
    df_clean = df_clean.drop(columns=['Unnamed: 0'])
    cleanup_notes.append("Dropped the index-like 'Unnamed: 0' column before detailed analysis.")
elif '' in df_clean.columns:
    df_clean = df_clean.drop(columns=[''])
    cleanup_notes.append("Dropped the unlabeled index-like first column before detailed analysis.")
else:
    cleanup_notes.append('No index-like placeholder column was found.')

for note in cleanup_notes:
    print(note)
print('Clean dataframe shape:', df_clean.shape)

cleaned_csv_path = OUTPUT_DIR / 'NSMES1988new.csv'
df_clean.to_csv(cleaned_csv_path, index=False)

print('Saved cleaned CSV artifact to:', cleaned_csv_path)
print('Cleaned dataframe shape:', df_clean.shape)

fig, axes = plt.subplots(1, 3, figsize=(18, 5))

if 'health' in df.columns:
    df['health'].value_counts(dropna=False).plot(kind='bar', ax=axes[0], color='#328cc1', title='Health Category Counts')
    axes[0].set_xlabel('health')
    axes[0].set_ylabel('count')
else:
    axes[0].text(0.5, 0.5, 'health column not available', ha='center', va='center')
    axes[0].set_axis_off()

income_min = None
income_max = None
income_median = None
income_p95 = None
income_p99 = None

if 'income' in df.columns:
    income = df['income'].dropna()
    income_min = float(income.min())
    income_max = float(income.max())
    income_median = float(income.median())
    income_p95 = float(income.quantile(0.95))
    income_p99 = float(income.quantile(0.99))
    focused_income = income[income <= income_p99]

    axes[1].hist(focused_income, bins=30, color='#d9b310', edgecolor='white')
    axes[1].axvline(income_median, color='#0b3c5d', linestyle='--', linewidth=2, label=f'median = {income_median:.2f}')
    axes[1].axvline(income_p95, color='#b45309', linestyle=':', linewidth=2, label=f'95th pct = {income_p95:.2f}')
    axes[1].set_title('Income Distribution (up to 99th percentile)')
    axes[1].set_xlabel('income')
    axes[1].set_ylabel('count')
    axes[1].legend(frameon=False)
    axes[1].text(
        0.98,
        0.95,
        f'99th pct = {income_p99:.2f}\nmax = {income_max:.2f}\n1% of rows exceed the chart range',
        transform=axes[1].transAxes,
        ha='right',
        va='top',
        fontsize=9,
        bbox={'facecolor': 'white', 'edgecolor': '#cbd5e1', 'boxstyle': 'round,pad=0.3'}
    )

    axes[2].boxplot(
        income,
        vert=False,
        patch_artist=True,
        boxprops={'facecolor': '#fde68a', 'edgecolor': '#b45309'},
        medianprops={'color': '#0b3c5d', 'linewidth': 2},
        whiskerprops={'color': '#b45309'},
        capprops={'color': '#b45309'},
        flierprops={'marker': 'o', 'markerfacecolor': '#dc2626', 'markeredgecolor': '#dc2626', 'markersize': 3, 'alpha': 0.4}
    )
    axes[2].set_title('Income Spread and Outliers')
    axes[2].set_xlabel('income')
    axes[2].set_yticks([])
else:
    axes[1].text(0.5, 0.5, 'income column not available', ha='center', va='center')
    axes[1].set_axis_off()
    axes[2].text(0.5, 0.5, 'income column not available', ha='center', va='center')
    axes[2].set_axis_off()

plt.tight_layout()
visual_report_path = OUTPUT_DIR / 'capstone_1_visual_observations.png'
fig.savefig(visual_report_path, bbox_inches='tight')
plt.show()

most_common_health = df['health'].mode().iat[0] if 'health' in df.columns else 'not available'
mean_visits = float(df['visits'].mean()) if 'visits' in df.columns else None

print('Saved visual observations figure to:', visual_report_path)
print('Visual observations report:')
print('- The most common health category in the dataset is:', most_common_health)
if income_min is not None and income_max is not None:
    print(f'- Income ranges from {income_min:.4f} to {income_max:.4f}.')
if income_median is not None and income_p99 is not None:
    print(f'- The income chart is focused on values up to the 99th percentile ({income_p99:.2f}) so the main distribution is readable, while the boxplot still shows the full outlier range.')
    print(f'- The median income is {income_median:.2f}, and the 95th percentile is {income_p95:.2f}.')
if mean_visits is not None:
    print(f'- The average number of visits is {mean_visits:.2f}.')
print('- The dataset is concentrated in the `average` health category, while income is strongly right-skewed with a small number of extreme high-income observations.')