Capstone Session 12

from pathlib import Path
import json
import os
import sys
from urllib.parse import quote
from urllib.request import urlretrieve

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
tf.keras.utils.set_random_seed(42)

IS_COLAB = 'google.colab' in sys.modules
GITHUB_REPO_OWNER = 'FrancisBurnet'
GITHUB_REPO_NAME = 'francisburnet'
GITHUB_REPO_BRANCH = 'main'
CAPSTONE_ROOT = Path('Incremental Capstones/Deep Learning Specialization/Capstone Session 12')
DATA_FILENAME = 'Dental-Panaromic-Autoencoder.npz'


def build_github_url(relative_path: Path, media: bool = False) -> str:
    encoded_path = quote(relative_path.as_posix(), safe='/')
    if media:
        return (
            f"https://media.githubusercontent.com/media/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/"
            f"{GITHUB_REPO_BRANCH}/{encoded_path}"
        )
    return (
        f"https://raw.githubusercontent.com/{GITHUB_REPO_OWNER}/{GITHUB_REPO_NAME}/"
        f"{GITHUB_REPO_BRANCH}/{encoded_path}"
    )


def resolve_capstone_dir() -> Path | None:
    current = Path.cwd().resolve()
    capstone_parts = CAPSTONE_ROOT.parts
    for candidate in [current, *current.parents]:
        if len(candidate.parts) >= len(capstone_parts) and candidate.parts[-len(capstone_parts):] == capstone_parts:
            return candidate
        nested_candidate = candidate / CAPSTONE_ROOT
        if nested_candidate.exists():
            return nested_candidate
    return None


CAPSTONE_DIR = resolve_capstone_dir()
DATA_URL = build_github_url(CAPSTONE_ROOT / DATA_FILENAME, media=True)

if CAPSTONE_DIR is not None:
    OUTPUT_ROOT = CAPSTONE_DIR
    OUTPUT_MODE = 'permanent capstone outputs'
    OUTPUT_DISPLAY = (CAPSTONE_ROOT / 'outputs').as_posix()
else:
    runtime_root = Path('/content/capstone-session-12-runtime') if IS_COLAB else Path.cwd().resolve() / 'capstone-session-12-runtime'
    OUTPUT_ROOT = runtime_root
    OUTPUT_MODE = 'runtime scratch outputs; export final artifacts back into the capstone outputs folder'
    OUTPUT_DISPLAY = 'capstone-session-12-runtime/outputs'

RUNTIME_DATA_ROOT = Path('/content/capstone-session-12-data') if IS_COLAB else Path.cwd().resolve() / 'capstone-session-12-data'
DATA_CACHE_PATH = RUNTIME_DATA_ROOT / DATA_FILENAME
RUNTIME_DATA_ROOT.mkdir(parents=True, exist_ok=True)
if not DATA_CACHE_PATH.exists():
    urlretrieve(DATA_URL, DATA_CACHE_PATH)

OUTPUTS_DIR = (OUTPUT_ROOT / 'outputs').resolve()
PLOTS_DIR = OUTPUTS_DIR / 'plots'
OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
PLOTS_DIR.mkdir(parents=True, exist_ok=True)

print('Runtime:', 'Google Colab' if IS_COLAB else 'Notebook runtime')
print('Capstone artifact path:', CAPSTONE_ROOT.as_posix())
print('Data source:', DATA_URL)
print('Data runtime file:', 'capstone-session-12-data/Dental-Panaromic-Autoencoder.npz')
print('Output mode:', OUTPUT_MODE)
print('Output target:', OUTPUT_DISPLAY)

data = np.load(DATA_CACHE_PATH)
x_train = data['x_train'].astype('float32') / 255.0 if data['x_train'].max() > 1 else data['x_train'].astype('float32')
x_test = data['x_test'].astype('float32') / 255.0 if data['x_test'].max() > 1 else data['x_test'].astype('float32')
x_train_gray = x_train.mean(axis=-1, keepdims=True)
x_test_gray = x_test.mean(axis=-1, keepdims=True)
noise_factor = 0.2
x_train_noisy = np.clip(x_train_gray + noise_factor * np.random.normal(loc=0.0, scale=1.0, size=x_train_gray.shape), 0.0, 1.0)
x_test_noisy = np.clip(x_test_gray + noise_factor * np.random.normal(loc=0.0, scale=1.0, size=x_test_gray.shape), 0.0, 1.0)
print('Data source used:', DATA_URL)
print({'x_train': x_train.shape, 'x_train_gray': x_train_gray.shape, 'x_test': x_test.shape, 'x_test_gray': x_test_gray.shape})

fig, axes = plt.subplots(2, 5, figsize=(14, 6))
for index in range(5):
    axes[0, index].imshow(x_train_gray[index].squeeze(), cmap='gray')
    axes[0, index].axis('off')
    axes[0, index].set_title(f'Original {index + 1}')
    axes[1, index].imshow(x_train_noisy[index].squeeze(), cmap='gray')
    axes[1, index].axis('off')
    axes[1, index].set_title(f'Noisy {index + 1}')
fig.tight_layout()
fig.savefig(PLOTS_DIR / 'original_vs_noisy_train.png', dpi=150)
plt.show()
plt.close(fig)

class Denoise(tf.keras.Model):
    def __init__(self):
        super().__init__()
        self.encoder = tf.keras.Sequential([
            tf.keras.layers.Input(shape=(256, 256, 1)),
            tf.keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same', strides=2),
            tf.keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same', strides=2),
        ])
        self.decoder = tf.keras.Sequential([
            tf.keras.layers.Conv2DTranspose(32, kernel_size=3, strides=2, activation='relu', padding='same'),
            tf.keras.layers.Conv2DTranspose(64, kernel_size=3, strides=2, activation='relu', padding='same'),
            tf.keras.layers.Conv2D(1, kernel_size=(3, 3), activation='sigmoid', padding='same'),
        ])

    def call(self, inputs):
        encoded = self.encoder(inputs)
        decoded = self.decoder(encoded)
        return decoded

autoencoder = Denoise()
autoencoder.compile(optimizer='adam', loss=tf.keras.losses.MeanSquaredError(), metrics=['mae'])

history = autoencoder.fit(
    x_train_noisy,
    x_train_gray,
    epochs=50,
    batch_size=8,
    shuffle=True,
    validation_data=(x_test_noisy, x_test_gray),
    verbose=0,
)
pd.DataFrame(history.history).tail()

evaluation = autoencoder.evaluate(x_test_noisy, x_test_gray, verbose=0)
encoded_images = autoencoder.encoder(x_test_noisy).numpy()
decoded_images = autoencoder.decoder(encoded_images).numpy()
print({'test_loss': float(evaluation[0]), 'test_mae': float(evaluation[1]), 'encoded_shape': encoded_images.shape, 'decoded_shape': decoded_images.shape})

fig, axes = plt.subplots(2, 10, figsize=(18, 5))
for index in range(10):
    axes[0, index].imshow(x_test_noisy[index].squeeze(), cmap='gray')
    axes[0, index].axis('off')
    axes[0, index].set_title(f'Noisy {index + 1}')
    axes[1, index].imshow(decoded_images[index].squeeze(), cmap='gray')
    axes[1, index].axis('off')
    axes[1, index].set_title(f'Denoised {index + 1}')
fig.tight_layout()
fig.savefig(PLOTS_DIR / 'noisy_vs_denoised_test.png', dpi=150)
plt.show()
plt.close(fig)

fig, axes = plt.subplots(1, 2, figsize=(12, 4))
axes[0].plot(history.history['mae'], label='train')
axes[0].plot(history.history['val_mae'], label='validation')
axes[0].set_title('MAE by Epoch')
axes[0].legend()
axes[1].plot(history.history['loss'], label='train')
axes[1].plot(history.history['val_loss'], label='validation')
axes[1].set_title('Loss by Epoch')
axes[1].legend()
fig.tight_layout()
fig.savefig(PLOTS_DIR / 'training_history.png', dpi=150)
plt.show()
plt.close(fig)

history_df = pd.DataFrame(history.history)
history_df.to_csv(OUTPUTS_DIR / 'session_12_training_history.csv', index=False)
summary = {
    'original_shapes': {'x_train': list(x_train.shape), 'x_test': list(x_test.shape)},
    'grayscale_shapes': {'x_train_gray': list(x_train_gray.shape), 'x_test_gray': list(x_test_gray.shape)},
    'noise_factor': noise_factor,
    'test_loss': float(evaluation[0]),
    'test_mae': float(evaluation[1]),
    'encoded_shape': list(encoded_images.shape),
    'decoded_shape': list(decoded_images.shape),
}
with open(OUTPUTS_DIR / 'session_12_summary.json', 'w', encoding='utf-8') as handle:
    json.dump(summary, handle, indent=2)
summary