# data_pipeline.py
import pandas as pd
from datetime import datetime
import hashlib

def load_credit_data(file_path):
    """
    Load data and create a 'fingerprint' of it
    A fingerprint is a unique code that changes if even one number changes
    """
    print(f"📂 Loading data from: {file_path}")
    data = pd.read_csv(file_path)
    
    # Create unique fingerprint for this exact dataset
    data_string = data.to_string()
    fingerprint = hashlib.sha256(data_string.encode()).hexdigest()
    
    # Save metadata (information about the data)
    metadata = {
        'file_path': file_path,
        'load_date': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        'row_count': len(data),
        'fingerprint': fingerprint[:16],
    }
    
    print(f"✅ Loaded {len(data):,} rows")
    print(f"🔐 Data fingerprint: {metadata['fingerprint']}")
    
    data.attrs['metadata'] = metadata
    return data

def validate_data(data):
    """Run sanity checks - catch problems early!"""
    print("\n🔍 Running data quality checks...")
    problems = []
    
    # Check 1: Too many missing values?
    missing_percent = (data.isnull().sum().sum() / (len(data) * len(data.columns))) * 100
    if missing_percent > 5:
        problems.append(f"⚠️ Too many missing values: {missing_percent:.1f}%")
    else:
        print(f"✅ Missing values: {missing_percent:.1f}% (acceptable)")
    
    # Check 2: Are loan amounts positive?
    if 'loan_amount' in data.columns:
        if data['loan_amount'].min() <= 0:
            problems.append("⚠️ Found negative loan amounts!")
        else:
            print(f"✅ Loan amounts look good")
    
    if problems:
        raise ValueError("Data validation failed! Fix these issues before training.")
    
    print("✅ All checks passed!\n")
    return True

def engineer_features(data):
    """Transform raw data into features the model can learn from"""
    print("🔧 Engineering features...")
    data = data.copy()
    
    # Debt-to-Income Ratio (lower is better)
    data['debt_to_income_ratio'] = data['total_debt'] / (data['annual_income'] + 0.01)
    
    # Credit Utilization (lower is better)
    data['credit_utilization'] = data['credit_card_balance'] / (data['credit_limit'] + 0.01)
    
    # Payment History Score (higher is better)
    data['payment_score'] = (data['on_time_payments'] / (data['total_payments'] + 1)) * 100
    
    print(f"✅ Created 3 new features\n")
    return data

# Usage
credit_data = load_credit_data('loan_applications.csv')
validate_data(credit_data)
credit_data = engineer_features(credit_data)

# model_training.py
import mlflow
import mlflow.sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

def train_credit_risk_model(data, model_name="CreditRiskModel"):
    """Train model with complete tracking"""
    
    print("🚀 Starting model training with MLOps tracking...\n")
    
    # Prepare data
    feature_columns = ['debt_to_income_ratio', 'credit_utilization', 
                      'payment_score', 'annual_income']
    
    X = data[feature_columns]
    y = data['loan_default']  # 1 = defaulted, 0 = paid back
    
    # Split: 80% to learn from, 20% to test on
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    print(f"📊 Training set: {len(X_train):,} applications")
    print(f"📊 Testing set: {len(X_test):,} applications\n")
    
    # Start tracking this experiment
    mlflow.set_experiment("Credit_Risk_Models")
    
    with mlflow.start_run(run_name=model_name):
        # Record info about this training run
        mlflow.log_param("data_version", data.attrs.get('metadata', {}).get('fingerprint'))
        mlflow.log_param("training_samples", len(X_train))
        
        # Model settings
        model_settings = {
            'n_estimators': 200,        # Number of decision trees
            'max_depth': 10,            # How complex each tree can be
            'min_samples_split': 100,   # Minimum data points to split
            'class_weight': 'balanced', # Handle imbalanced data
            'random_state': 42
        }
        
        # Log all settings
        for setting, value in model_settings.items():
            mlflow.log_param(setting, value)
        
        # Train the model
        print("🎯 Training Random Forest model...")
        model = RandomForestClassifier(**model_settings)
        model.fit(X_train, y_train)
        print("✅ Training complete!\n")
        
        # Evaluate performance
        y_pred_proba = model.predict_proba(X_test)[:, 1]
        test_auc = roc_auc_score(y_test, y_pred_proba)
        
        mlflow.log_metric("test_auc", test_auc)
        
        print(f"📊 Test AUC: {test_auc:.3f}")
        print("   (0.5 = random, 0.7 = okay, 0.85+ = good)\n")
        
        # Save feature importance (for explainability)
        feature_importance = pd.DataFrame({
            'feature': feature_columns,
            'importance': model.feature_importances_
        }).sort_values('importance', ascending=False)
        
        print("🔍 Most important features:")
        for idx, row in feature_importance.iterrows():
            print(f"   {row['feature']}: {row['importance']:.3f}")
        
        mlflow.log_dict(feature_importance.to_dict(), "feature_importance.json")
        
        # Save model with version control
        mlflow.sklearn.log_model(model, "model", 
                                registered_model_name="CreditRiskScorer")
        
        print("\n✅ Model saved with version control!")
        return model, test_auc

# Usage
model, auc = train_credit_risk_model(credit_data, "CreditModel_v1")

# model_monitor.py
import numpy as np
from datetime import datetime
from scipy import stats

class SimpleModelMonitor:
    """A beginner-friendly monitoring system"""
    
    def __init__(self, expected_accuracy=0.85):
        self.expected_accuracy = expected_accuracy
        self.alerts = []
        print(f"🔔 Monitoring initialized (expected accuracy: {expected_accuracy*100:.0f}%)")
    
    def check_accuracy(self, y_true, y_pred):
        """Check if model accuracy is still good"""
        current_accuracy = (y_true == y_pred).mean()
        
        print(f"\n📊 Current accuracy: {current_accuracy*100:.1f}%")
        
        accuracy_drop = self.expected_accuracy - current_accuracy
        
        if accuracy_drop > 0.05:  # More than 5% drop
            alert = f"⚠️ ALERT: Accuracy dropped to {current_accuracy*100:.1f}%"
            self.alerts.append(alert)
            print(alert)
            print("   → Action needed: Review recent data and consider retraining")
        else:
            print("✅ Accuracy within acceptable range")
        
        return current_accuracy
    
    def check_data_drift(self, training_data, production_data, feature_name):
        """Check if incoming data looks different from training data"""
        
        print(f"\n🔍 Checking data drift for: {feature_name}")
        
        # Statistical test: Are these two datasets similar?
        statistic, p_value = stats.ks_2samp(training_data, production_data)
        
        # p_value: High (>0.05) = similar ✅, Low (<0.05) = different ⚠️
        
        if p_value < 0.05:
            alert = f"⚠️ Data drift detected in '{feature_name}'"
            self.alerts.append(alert)
            print(alert)
            print(f"   Training mean: {training_data.mean():.2f}")
            print(f"   Production mean: {production_data.mean():.2f}")
        else:
            print(f"✅ No significant drift (p-value: {p_value:.3f})")
        
        return p_value
    
    def check_fairness(self, predictions, groups, attribute_name):
        """Check if model treats different groups fairly"""
        
        print(f"\n⚖️ Checking fairness across: {attribute_name}")
        
        df = pd.DataFrame({'prediction': predictions, 'group': groups})
        approval_rates = df.groupby('group')['prediction'].mean()
        
        print("\n   Approval rates by group:")
        for group, rate in approval_rates.items():
            print(f"   {group}: {rate*100:.1f}%")
        
        disparity = approval_rates.max() - approval_rates.min()
        
        if disparity > 0.20:  # More than 20 percentage points
            alert = f"🚨 CRITICAL: Large disparity detected ({disparity*100:.1f}%)"
            self.alerts.append(alert)
            print(f"\n{alert}")
            print("   → Immediate review required for compliance")
        else:
            print(f"\n✅ Reasonably balanced (disparity: {disparity*100:.1f}%)")
        
        return disparity
    
    def generate_report(self):
        """Create daily report"""
        print("\n" + "="*50)
        print("📊 DAILY MONITORING REPORT")
        print("="*50)
        print(f"Total alerts: {len(self.alerts)}")
        
        if len(self.alerts) == 0:
            print("✅ No issues - model is healthy!")
        else:
            print("⚠️ ALERTS:")
            for alert in self.alerts:
                print(f"   • {alert}")

# Usage
monitor = SimpleModelMonitor(expected_accuracy=0.85)

# Daily checks
monitor.check_accuracy(actual_outcomes, predictions)
monitor.check_data_drift(training_income, production_income, 'annual_income')
monitor.check_fairness(predictions, age_groups, 'age_group')

monitor.generate_report()