"""
Phase 6: Predictive Analytics & Anomaly Detection
Resolution time forecasting, anomaly detection, incident-to-task conversion analysis
"""

import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from pathlib import Path
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, classification_report
import warnings
warnings.filterwarnings('ignore')

# Paths
OUTPUT_DIR = Path("/app/workspace/temp_files")
IMG_DIR = Path("/app/workspace/assets/images")
MODEL_DIR = Path("/app/workspace/models")

print("=" * 80)
print("PHASE 6: PREDICTIVE ANALYTICS & ANOMALY DETECTION")
print("=" * 80)

# Load cleaned data
incidents_df = pd.read_pickle(OUTPUT_DIR / "incidents_cleaned.pkl")
tasks_df = pd.read_pickle(OUTPUT_DIR / "tasks_cleaned.pkl")

# ============================================================================
# 1. RESOLUTION TIME FORECASTING (REGRESSION MODEL)
# ============================================================================
print("\n🔮 RESOLUTION TIME FORECASTING MODEL")
print("-" * 40)

# Prepare data for regression
task_features = tasks_df.copy()

# Encode categorical variables
le_priority = LabelEncoder()
le_category = LabelEncoder()
le_group = LabelEncoder()
le_subcat = LabelEncoder()

task_features['Priority_Encoded'] = le_priority.fit_transform(task_features['Priority'].fillna('Unknown'))
task_features['Category_Encoded'] = le_category.fit_transform(task_features['Category'].fillna('Unknown'))
task_features['Group_Encoded'] = le_group.fit_transform(task_features['Assignment group'].fillna('Unknown'))
task_features['Subcategory_Encoded'] = le_subcat.fit_transform(task_features['Subcategory'].fillna('Unknown'))

# Add temporal features
task_features['DayOfWeek_num'] = task_features['DayOfWeek']
task_features['Month_num'] = task_features['Opened'].dt.month
task_features['Hour_num'] = task_features['HourOfDay']

# Define features and target
feature_cols = ['Priority_Encoded', 'Category_Encoded', 'Group_Encoded', 
                'Subcategory_Encoded', 'DayOfWeek_num', 'Month_num', 'Hour_num']
X = task_features[feature_cols].dropna()
y = task_features.loc[X.index, 'Business_Duration_Hours']

# Remove extreme outliers for training
mask = y < y.quantile(0.99)
X = X[mask]
y = y[mask]

print(f"  Training samples: {len(X):,}")
print(f"  Features: {feature_cols}")

# Train/test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Random Forest Regressor
rf_model = RandomForestRegressor(n_estimators=100, max_depth=10, random_state=42, n_jobs=-1)
rf_model.fit(X_train, y_train)

# Predictions
y_pred_train = rf_model.predict(X_train)
y_pred_test = rf_model.predict(X_test)

print(f"\n  Random Forest Model Performance:")
train_rmse = np.sqrt(mean_squared_error(y_train, y_pred_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred_test))
print(f"  Training MAE: {mean_absolute_error(y_train, y_pred_train):.1f} hours")
print(f"  Training RMSE: {train_rmse:.1f} hours")
print(f"  Training R²: {r2_score(y_train, y_pred_train):.3f}")
print(f"  Test MAE: {mean_absolute_error(y_test, y_pred_test):.1f} hours")
print(f"  Test RMSE: {test_rmse:.1f} hours")
print(f"  Test R²: {r2_score(y_test, y_pred_test):.3f}")

# Feature importance
feature_importance = pd.DataFrame({
    'Feature': feature_cols,
    'Importance': rf_model.feature_importances_
}).sort_values('Importance', ascending=False)

fig_feat_imp = px.bar(
    feature_importance, x='Importance', y='Feature', orientation='h',
    title='Feature Importance for Resolution Time Prediction',
    labels={'Importance': 'Importance Score', 'Feature': 'Feature'},
    color='Importance',
    color_continuous_scale='Blues'
)
fig_feat_imp.update_layout(template='plotly_white', height=350)
fig_feat_imp.write_html(IMG_DIR / "feature_importance_resolution.html")
fig_feat_imp.write_image(IMG_DIR / "feature_importance_resolution.png")
print("✓ Saved: feature_importance_resolution")

# Actual vs Predicted scatter
fig_actual_pred = px.scatter(
    x=y_test, y=y_pred_test,
    title='Actual vs Predicted Resolution Time (Test Set)',
    labels={'x': 'Actual Duration (Hours)', 'y': 'Predicted Duration (Hours)'},
    opacity=0.5
)
fig_actual_pred.add_shape(type='line', x0=0, y0=0, x1=y_test.max(), y1=y_test.max(),
                         line=dict(color='red', dash='dash'))
fig_actual_pred.update_layout(template='plotly_white', height=400)
fig_actual_pred.write_html(IMG_DIR / "actual_vs_predicted_resolution.html")
fig_actual_pred.write_image(IMG_DIR / "actual_vs_predicted_resolution.png")
print("✓ Saved: actual_vs_predicted_resolution")

# Save model
import joblib
joblib.dump(rf_model, MODEL_DIR / "resolution_time_rf_model.pkl")
joblib.dump(le_priority, MODEL_DIR / "le_priority.pkl")
joblib.dump(le_category, MODEL_DIR / "le_category.pkl")
joblib.dump(le_group, MODEL_DIR / "le_group.pkl")
joblib.dump(le_subcat, MODEL_DIR / "le_subcat.pkl")
print("✓ Saved: resolution_time_rf_model.pkl")

# ============================================================================
# 2. ANOMALY DETECTION ON RESOLUTION DURATION
# ============================================================================
print("\n🔍 ANOMALY DETECTION")
print("-" * 40)

# IQR-based outlier detection
Q1 = tasks_df['Business_Duration_Hours'].quantile(0.25)
Q3 = tasks_df['Business_Duration_Hours'].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
extreme_upper = tasks_df['Business_Duration_Hours'].quantile(0.95)

tasks_df['Anomaly_IQR'] = np.where(
    (tasks_df['Business_Duration_Hours'] > upper_bound) | 
    (tasks_df['Business_Duration_Hours'] < 0),
    'Outlier', 'Normal'
)

tasks_df['Anomaly_Extreme'] = np.where(
    tasks_df['Business_Duration_Hours'] > extreme_upper,
    'Extreme', 'Normal'
)

# Anomaly summary
anomaly_counts = tasks_df['Anomaly_IQR'].value_counts()
extreme_counts = tasks_df['Anomaly_Extreme'].value_counts()

print(f"  IQR Analysis:")
print(f"    Normal cases: {anomaly_counts.get('Normal', 0):,} ({anomaly_counts.get('Normal', 0)/len(tasks_df)*100:.1f}%)")
print(f"    IQR Outliers: {anomaly_counts.get('Outlier', 0):,} ({anomaly_counts.get('Outlier', 0)/len(tasks_df)*100:.1f}%)")
print(f"    Extreme (>95th pctl): {extreme_counts.get(' Extreme', 0):,} ({extreme_counts.get(' Extreme', 0)/len(tasks_df)*100:.1f}%)")

# Z-score based detection
mean_dur = tasks_df['Business_Duration_Hours'].mean()
std_dur = tasks_df['Business_Duration_Hours'].std()
z_threshold = 3

tasks_df['Z_Score'] = (tasks_df['Business_Duration_Hours'] - mean_dur) / std_dur
tasks_df['Anomaly_ZScore'] = np.where(
    np.abs(tasks_df['Z_Score']) > z_threshold,
    'Outlier', 'Normal'
)

zscore_outliers = (tasks_df['Anomaly_ZScore'] == 'Outlier').sum()
print(f"\n  Z-Score Analysis (threshold={z_threshold}):")
print(f"    Z-score Outliers: {zscore_outliers:,} ({zscore_outliers/len(tasks_df)*100:.1f}%)")

# Anomaly visualization
fig_anomaly = go.Figure()
fig_anomaly.add_trace(go.Box(
    y=tasks_df['Business_Duration_Hours'],
    name='Duration',
    boxpoints='outliers',
    marker_color='#457B9D'
))
fig_anomaly.add_hline(y=upper_bound, line_dash="dash", line_color="orange", 
                       annotation_text=f"Upper Bound: {upper_bound:.0f}h")
fig_anomaly.add_hline(y=extreme_upper, line_dash="dot", line_color="red",
                       annotation_text=f"95th Percentile: {extreme_upper:.0f}h")
fig_anomaly.update_layout(
    title='Resolution Duration Anomaly Detection',
    template='plotly_white', height=400
)
fig_anomaly.write_html(IMG_DIR / "anomaly_detection_boxplot.html")
fig_anomaly.write_image(IMG_DIR / "anomaly_detection_boxplot.png")
print("✓ Saved: anomaly_detection_boxplot")

# Anomalies by category
anomaly_by_cat = tasks_df[tasks_df['Anomaly_IQR'] == 'Outlier'].groupby('Subcategory').size().sort_values(ascending=False).head(10)

fig_anom_cat = px.bar(
    anomaly_by_cat, x=anomaly_by_cat.values, y=anomaly_by_cat.index, orientation='h',
    title='Top 10 Subcategories with Most Outliers',
    labels={'index': 'Subcategory', 'value': 'Outlier Count'},
    color_discrete_sequence=['#E63946']
)
fig_anom_cat.update_layout(template='plotly_white', height=400)
fig_anom_cat.write_html(IMG_DIR / "anomalies_by_subcategory.html")
fig_anom_cat.write_image(IMG_DIR / "anomalies_by_subcategory.png")
print("✓ Saved: anomalies_by_subcategory")

# ============================================================================
# 3. INCIDENT-TO-TASK CONVERSION PATTERN ANALYSIS
# ============================================================================
print("\n🔗 INCIDENT-TO-TASK CONVERSION ANALYSIS")
print("-" * 40)

# Identify linked tasks
linked_tasks = tasks_df[tasks_df['Incident to request'].notna()].copy()
unlinked_tasks = tasks_df[tasks_df['Incident to request'].isna()].copy()

print(f"\n  Conversion Summary:")
print(f"    Total Tasks: {len(tasks_df):,}")
print(f"    Linked to Incidents: {len(linked_tasks):,} ({len(linked_tasks)/len(tasks_df)*100:.1f}%)")
print(f"    Standalone Tasks: {len(unlinked_tasks):,} ({len(unlinked_tasks)/len(tasks_df)*100:.1f}%)")

# Compare metrics
linked_metrics = linked_tasks.agg({
    'Business_Duration_Hours': 'mean',
    'Resolution_Outcome': lambda x: (x == 'Success').mean() * 100
})
unlinked_metrics = unlinked_tasks.agg({
    'Business_Duration_Hours': 'mean',
    'Resolution_Outcome': lambda x: (x == 'Success').mean() * 100
})

print(f"\n  Linked Tasks:")
print(f"    Mean Duration: {linked_metrics['Business_Duration_Hours']:.1f} hours")
print(f"    Success Rate: {linked_metrics['Resolution_Outcome']:.1f}%")

print(f"\n  Standalone Tasks:")
print(f"    Mean Duration: {unlinked_metrics['Business_Duration_Hours']:.1f} hours")
print(f"    Success Rate: {unlinked_metrics['Resolution_Outcome']:.1f}%")

# Visualization of conversion comparison
fig_conversion = make_subplots(
    rows=1, cols=2,
    subplot_titles=['Mean Duration: Linked vs Standalone', 'Success Rate: Linked vs Standalone']
)

fig_conversion.add_trace(
    go.Bar(x=['Linked', 'Standalone'], 
           y=[linked_metrics['Business_Duration_Hours'], unlinked_metrics['Business_Duration_Hours']],
           marker_color=['#457B9D', '#E63946'], name='Mean Hours'),
    row=1, col=1
)

fig_conversion.add_trace(
    go.Bar(x=['Linked', 'Standalone'],
           y=[linked_metrics['Resolution_Outcome'], unlinked_metrics['Resolution_Outcome']],
           marker_color=['#2A9D8F', '#F4A261'], name='Success %'),
    row=1, col=2
)

fig_conversion.update_layout(template='plotly_white', height=400, showlegend=False)
fig_conversion.write_html(IMG_DIR / "incident_task_conversion_comparison.html")
fig_conversion.write_image(IMG_DIR / "incident_task_conversion_comparison.png")
print("✓ Saved: incident_task_conversion_comparison")

# Category distribution of linked vs unlinked
fig_cat_compare = make_subplots(
    rows=1, cols=2,
    subplot_titles=['Top Categories - Linked Tasks', 'Top Categories - Standalone Tasks']
)

linked_cat = linked_tasks['Subcategory'].value_counts().head(5)
fig_cat_compare.add_trace(
    go.Bar(x=linked_cat.values, y=linked_cat.index, orientation='h',
           marker_color='#457B9D', name='Linked'),
    row=1, col=1
)

unlinked_cat = unlinked_tasks['Subcategory'].value_counts().head(5)
fig_cat_compare.add_trace(
    go.Bar(x=unlinked_cat.values, y=unlinked_cat.index, orientation='h',
           marker_color='#E63946', name='Standalone'),
    row=1, col=2
)

fig_cat_compare.update_layout(template='plotly_white', height=400, showlegend=False)
fig_cat_compare.write_html(IMG_DIR / "category_comparison_linked_vs_standalone.html")
fig_cat_compare.write_image(IMG_DIR / "category_comparison_linked_vs_standalone.png")
print("✓ Saved: category_comparison_linked_vs_standalone")

# ============================================================================
# 4. BACKLOG AGING & OVERDUE PREDICTION
# ============================================================================
print("\n⏳ BACKLOG AGING & OVERDUE PREDICTION")
print("-" * 40)

# Calculate case age (days since opened)
tasks_df['Case_Age_Days'] = (pd.Timestamp.now() - tasks_df['Opened']).dt.total_seconds() / 86400

# Open/pending cases
open_cases = tasks_df[tasks_df['State'].isin(['Open', 'In Progress', 'Pending Vendor', 'Pending Customer'])]
closed_cases = tasks_df[tasks_df['State'].str.contains('Closed', na=False)]

print(f"\n  Case Status:")
print(f"    Total Cases: {len(tasks_df):,}")
print(f"    Closed: {len(closed_cases):,} ({len(closed_cases)/len(tasks_df)*100:.1f}%)")
print(f"    Open/Pending: {len(open_cases):,} ({len(open_cases)/len(tasks_df)*100:.1f}%)")

# Age distribution for open cases
if len(open_cases) > 0:
    print(f"\n  Open Case Age (Days):")
    print(f"    Mean: {open_cases['Case_Age_Days'].mean():.1f}")
    print(f"    Median: {open_cases['Case_Age_Days'].median():.1f}")
    print(f"    Max: {open_cases['Case_Age_Days'].max():.1f}")

    # Cases at risk (older than 30 days)
    at_risk = len(open_cases[open_cases['Case_Age_Days'] > 30])
    print(f"\n  At-Risk Cases (>30 days old): {at_risk:,} ({at_risk/len(open_cases)*100:.1f}%)")

# Build overdue prediction model
tasks_df['Overdue_Risk'] = np.where(
    (tasks_df['State'].isin(['Open', 'In Progress'])) & 
    (tasks_df['Case_Age_Days'] > 7),
    'High Risk', 'Normal'
)

risk_summary = tasks_df['Overdue_Risk'].value_counts()
print(f"\n  Overdue Risk Distribution:")
for risk, count in risk_summary.items():
    print(f"    {risk}: {count:,} ({count/len(tasks_df)*100:.1f}%)")

# Risk by assignment group
risk_by_group = tasks_df[tasks_df['Overdue_Risk'] == 'High Risk'].groupby('Assignment group').size()
risk_by_group = risk_by_group.sort_values(ascending=False).reset_index()
risk_by_group.columns = ['Assignment group', 'High Risk Cases']

fig_risk = px.bar(
    risk_by_group, x='High Risk Cases', y='Assignment group', orientation='h',
    title='High Risk Cases by Assignment Group',
    labels={'Assignment group': 'Assignment Group', 'High Risk Cases': 'High Risk Cases'},
    color_discrete_sequence=['#DC3545']
)
fig_risk.update_layout(template='plotly_white', height=400)
fig_risk.write_html(IMG_DIR / "overdue_risk_by_team.html")
fig_risk.write_image(IMG_DIR / "overdue_risk_by_team.png")
print("✓ Saved: overdue_risk_by_team")

# ============================================================================
# KEY PREDICTIVE INSIGHTS
# ============================================================================
print("\n📊 PREDICTIVE ANALYTICS KEY INSIGHTS")
print("-" * 40)

print("\n🔮 RESOLUTION TIME MODEL:")
print(f"  Primary Predictors: {', '.join(feature_importance.head(3)['Feature'].tolist())}")
print(f"  Model Accuracy (R²): {r2_score(y_test, y_pred_test):.3f}")
print(f"  Mean Prediction Error: {mean_absolute_error(y_test, y_pred_test):.1f} hours")

print("\n🔍 ANOMALY DETECTION:")
print(f"  Outlier Cases: {anomaly_counts.get('Outlier', 0):,} ({anomaly_counts.get('Outlier', 0)/len(tasks_df)*100:.1f}%)")
print(f"  Extreme Cases: {extreme_counts.get(' Extreme', 0):,}")
print(f"  Top Outlier Subcategory: {anomaly_by_cat.index[0] if len(anomaly_by_cat) > 0 else 'N/A'}")

print("\n🔗 INCIDENT-TASK LINKAGE:")
print(f"  Conversion Rate: {len(linked_tasks)/len(tasks_df)*100:.1f}%")
print(f"  Linked vs Standalone Duration Delta: {linked_metrics['Business_Duration_Hours'] - unlinked_metrics['Business_Duration_Hours']:.1f} hours")

print("\n⏳ BACKLOG RISK:")
print(f"  High-Risk Cases: {risk_summary.get('High Risk', 0):,}")
print(f"  At-Risk Teams: {', '.join(risk_by_group.head(3).index.tolist())}")

print("\n✅ PREDICTIVE ANALYTICS COMPLETE")
print(f"✓ {len(list(IMG_DIR.glob('*predict*')))} predictive visualizations saved")
print(f"✓ Models saved to: {MODEL_DIR}")