Chapter 40: EDA: Advanced Visualization and Pattern Discovery

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import fetch_openml
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
import umap
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import warnings
warnings.filterwarnings('ignore')

# Set style
plt.style.use('seaborn-v0_8')
sns.set_palette("husl")

print("Libraries imported successfully!")

# Load the Online Shoppers Purchasing Intention dataset
# Use parser='liac-arff' to avoid potential loading issues
data = fetch_openml(name='online_shoppers_intention', version=1, as_frame=True, parser='liac-arff')
df = data.frame.copy()

# The target column is 'Revenue', which we rename to 'class' for consistency
df.rename(columns={'Revenue': 'class'}, inplace=True)


print(f"Dataset shape: {df.shape}")
print(f"Dataset columns: {list(df.columns)}")
print("\nDataset info:")
df.info()

# Examine the target variable
print(f"\nTarget distribution:")
print(df['class'].value_counts())

# Preprocess the data
def preprocess_data(df):
    """Clean and prepare the data for analysis"""
    # Handle categorical variables
    categorical_cols = ['Month', 'OperatingSystems', 'Browser', 'Region', 
                       'TrafficType', 'VisitorType', 'Weekend']

    # Label encode categorical feature variables
    le_dict = {}
    for col in categorical_cols:
        if col in df.columns:
            le = LabelEncoder()
            df[col] = le.fit_transform(df[col].astype(str))
            le_dict[col] = le

    # Separate features and target
    X = df.drop('class', axis=1)
    
    # FIX: Convert target variable from string ('TRUE'/'FALSE') to boolean (True/False)
    # This is the key fix that resolves the empty plots and incorrect purchase rates.
    y = (df['class'].astype(str).str.upper() == 'TRUE')

    # Scale the features
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    X_scaled_df = pd.DataFrame(X_scaled, columns=X.columns)

    return X_scaled_df, y, scaler, le_dict

X_scaled, y, scaler, le_dict = preprocess_data(df)
print("Data preprocessing completed!")
print(f"Feature matrix shape: {X_scaled.shape}")

def perform_pca_analysis(X_scaled, y, n_components=2):
    """Perform PCA and visualize results"""
    # Fit PCA
    pca = PCA(n_components=10)  # First get more components for analysis
    X_pca_full = pca.fit_transform(X_scaled)

    # Create explained variance plot
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))

    # Explained variance ratio
    ax1.bar(range(1, 11), pca.explained_variance_ratio_[:10])
    ax1.set_xlabel('Principal Component')
    ax1.set_ylabel('Explained Variance Ratio')
    ax1.set_title('PCA: Explained Variance by Component')
    ax1.grid(True, alpha=0.3)

    # Cumulative explained variance
    cumvar = np.cumsum(pca.explained_variance_ratio_[:10])
    ax2.plot(range(1, 11), cumvar, 'bo-')
    ax2.axhline(y=0.8, color='r', linestyle='--', alpha=0.7, label='80% variance')
    ax2.set_xlabel('Number of Components')
    ax2.set_ylabel('Cumulative Explained Variance')
    ax2.set_title('PCA: Cumulative Explained Variance')
    ax2.legend()
    ax2.grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()

    # Get 2D projection for visualization
    pca_2d = PCA(n_components=2)
    X_pca_2d = pca_2d.fit_transform(X_scaled)

    return X_pca_2d, pca_2d

X_pca, pca_model = perform_pca_analysis(X_scaled, y)
print(f"PCA completed. First 2 components explain {pca_model.explained_variance_ratio_.sum():.2%} of variance")

def perform_tsne_analysis(X_scaled, y, perplexity=30, random_state=42):
    """Perform t-SNE dimensionality reduction"""
    print("Performing t-SNE analysis (this may take a few minutes)...")

    # For large datasets, sample to make t-SNE computation feasible
    n_sample = min(5000, len(X_scaled))
    sample_idx = np.random.choice(len(X_scaled), n_sample, replace=False)
    X_sample = X_scaled.iloc[sample_idx]
    y_sample = y.iloc[sample_idx]

    tsne = TSNE(n_components=2, perplexity=perplexity, random_state=random_state, 
                n_jobs=-1, verbose=1)
    X_tsne = tsne.fit_transform(X_sample)

    print("t-SNE analysis completed!")
    return X_tsne, y_sample, sample_idx

X_tsne, y_tsne, tsne_idx = perform_tsne_analysis(X_scaled, y)

def perform_umap_analysis(X_scaled, y, n_neighbors=15, min_dist=0.1, random_state=42):
    """Perform UMAP dimensionality reduction"""
    print("Performing UMAP analysis...")

    # UMAP is more scalable than t-SNE, but we'll still sample for consistency
    n_sample = min(5000, len(X_scaled))
    sample_idx = np.random.choice(len(X_scaled), n_sample, replace=False)
    X_sample = X_scaled.iloc[sample_idx]
    y_sample = y.iloc[sample_idx]

    reducer = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, 
                       random_state=random_state, verbose=True)
    X_umap = reducer.fit_transform(X_sample)

    print("UMAP analysis completed!")
    return X_umap, y_sample, sample_idx, reducer

X_umap, y_umap, umap_idx, umap_model = perform_umap_analysis(X_scaled, y)

def compare_dim_reduction_methods():
    """Create comparative visualization of PCA, t-SNE, and UMAP"""
    # Sample PCA results to match other methods
    pca_sample_idx = tsne_idx  # Use same indices for fair comparison
    X_pca_sample = X_pca[pca_sample_idx]
    y_pca_sample = y.iloc[pca_sample_idx]

    # Create subplot
    fig, axes = plt.subplots(1, 3, figsize=(18, 6))

    # Color map for target classes
    colors = ['#FF6B6B', '#4ECDC4']
    class_names = ['No Purchase', 'Purchase']

    # PCA Plot
    for i, (class_val, color, name) in enumerate(zip([False, True], colors, class_names)):
        mask = y_pca_sample == class_val
        axes[0].scatter(X_pca_sample[mask, 0], X_pca_sample[mask, 1], 
                       c=color, label=name, alpha=0.6, s=30)
    axes[0].set_title('PCA Projection\n(Linear Dimensionality Reduction)', fontsize=12, fontweight='bold')
    axes[0].set_xlabel('First Principal Component')
    axes[0].set_ylabel('Second Principal Component')
    axes[0].legend()
    axes[0].grid(True, alpha=0.3)

    # t-SNE Plot
    for i, (class_val, color, name) in enumerate(zip([False, True], colors, class_names)):
        mask = y_tsne == class_val
        axes[1].scatter(X_tsne[mask, 0], X_tsne[mask, 1], 
                       c=color, label=name, alpha=0.6, s=30)
    axes[1].set_title('t-SNE Projection\n(Non-linear, Local Structure)', fontsize=12, fontweight='bold')
    axes[1].set_xlabel('t-SNE Dimension 1')
    axes[1].set_ylabel('t-SNE Dimension 2')
    axes[1].legend()
    axes[1].grid(True, alpha=0.3)

    # UMAP Plot
    for i, (class_val, color, name) in enumerate(zip([False, True], colors, class_names)):
        mask = y_umap == class_val
        axes[2].scatter(X_umap[mask, 0], X_umap[mask, 1], 
                       c=color, label=name, alpha=0.6, s=30)
    axes[2].set_title('UMAP Projection\n(Non-linear, Global + Local)', fontsize=12, fontweight='bold')
    axes[2].set_xlabel('UMAP Dimension 1')
    axes[2].set_ylabel('UMAP Dimension 2')
    axes[2].legend()
    axes[2].grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()

compare_dim_reduction_methods()

def create_interactive_dashboard():
    """Create an interactive dashboard for exploring customer segments"""

    # Prepare data for dashboard
    sample_idx = umap_idx  # Use UMAP sample for consistency
    df_sample = df.iloc[sample_idx].copy()
    df_sample['class'] = y.iloc[sample_idx] # Use the corrected boolean target

    # Add dimensionality reduction coordinates
    df_sample['PCA_1'] = X_pca[sample_idx, 0]
    df_sample['PCA_2'] = X_pca[sample_idx, 1]
    df_sample['UMAP_1'] = X_umap[:, 0]
    df_sample['UMAP_2'] = X_umap[:, 1]
    df_sample['t-SNE_1'] = X_tsne[:, 0]
    df_sample['t-SNE_2'] = X_tsne[:, 1]

    # Convert boolean target to string for better visualization
    df_sample['Purchase_Intent'] = df_sample['class'].map({True: 'Purchase', False: 'No Purchase'})

    # Create interactive scatter plot with UMAP
    fig = px.scatter(df_sample, x='UMAP_1', y='UMAP_2', 
                    color='Purchase_Intent',
                    hover_data=['Administrative', 'Informational', 'ProductRelated',
                               'PageValues', 'ExitRates', 'BounceRates'],
                    title='Interactive Customer Segmentation Analysis (UMAP Projection)',
                    color_discrete_map={'Purchase': '#4ECDC4', 'No Purchase': '#FF6B6B'})

    fig.update_layout(
        width=900,
        height=600,
        title_font_size=16,
        showlegend=True
    )

    fig.show()

    return df_sample

df_dashboard = create_interactive_dashboard()

def analyze_customer_segments():
    """Analyze different customer segments identified through UMAP"""

    # Use UMAP coordinates to identify clusters (simple approach)
    kmeans = KMeans(n_clusters=4, random_state=42, n_init=10)
    clusters = kmeans.fit_predict(X_umap)

    # Add cluster information to sample dataframe
    df_analysis = df.iloc[umap_idx].copy()
    df_analysis['Cluster'] = clusters
    df_analysis['Purchase'] = y_umap.astype(int) # y_umap is already boolean, convert to 0/1

    # Analyze clusters
    print("=== Customer Segment Analysis ===\n")

    for cluster_id in range(4):
        cluster_data = df_analysis[df_analysis['Cluster'] == cluster_id]
        purchase_rate = cluster_data['Purchase'].mean()

        print(f"--- Cluster {cluster_id + 1} ---")
        print(f"Size: {len(cluster_data)} customers ({len(cluster_data)/len(df_analysis)*100:.1f}%)")
        print(f"Purchase Rate: {purchase_rate:.2%}")
        print(f"Avg Page Values: {cluster_data['PageValues'].mean():.2f}")
        print(f"Avg Exit Rates: {cluster_data['ExitRates'].mean():.3f}")
        print(f"Avg Bounce Rates: {cluster_data['BounceRates'].mean():.3f}")
        print()

    # Visualize clusters
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))

    # Cluster visualization
    scatter = ax1.scatter(X_umap[:, 0], X_umap[:, 1], c=clusters, cmap='viridis', alpha=0.6)
    ax1.set_title('Customer Clusters (UMAP Projection)')
    ax1.set_xlabel('UMAP Dimension 1')
    ax1.set_ylabel('UMAP Dimension 2')
    plt.colorbar(scatter, ax=ax1, label='Cluster ID')

    # Purchase rate by cluster
    cluster_stats = df_analysis.groupby('Cluster')['Purchase'].agg(['mean', 'count']).reset_index()
    bars = ax2.bar(cluster_stats['Cluster'], cluster_stats['mean'], 
                   color=['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4'])
    ax2.set_title('Purchase Rate by Customer Segment')
    ax2.set_xlabel('Customer Segment')
    ax2.set_ylabel('Purchase Rate')
    ax2.set_ylim(0, max(cluster_stats['mean']) * 1.1 if not cluster_stats.empty else 1)


    # Add value labels on bars
    for bar, rate in zip(bars, cluster_stats['mean']):
        ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01,
                f'{rate:.2%}', ha='center', va='bottom', fontweight='bold')

    plt.tight_layout()
    plt.show()

    return df_analysis

segment_analysis = analyze_customer_segments()

def summarize_insights():
    """Summarize key insights from the analysis"""

    print("=== KEY INSIGHTS FROM CUSTOMER SEGMENTATION ANALYSIS ===\n")

    print("1. DIMENSIONALITY REDUCTION COMPARISON:")
    print("   • PCA: Shows broad linear trends but limited cluster separation.")
    print("   • t-SNE: Reveals tight, well-separated local clusters, but may obscure global structure.")
    print("   • UMAP: Provides a good balance, showing local clusters while preserving global relationships.")
    print()

    print("2. CUSTOMER SEGMENTS IDENTIFIED:")
    print("   The analysis identified distinct customer segments with varying purchase rates and behaviors.")
    print("   Detailed segment profiles are printed in the 'Customer Segment Analysis' section above.")
    print()

    print("3. BUSINESS ACTIONABLE INSIGHTS:")
    print("   • High-value segments (high purchase rate, high page values) can be targeted with loyalty programs or premium offers.")
    print("   • Low-engagement segments (high bounce/exit rates, low purchase rate) may need re-engagement campaigns or website usability improvements.")
    print("   • 'PageValues' is a strong indicator of purchase intent. Optimizing pages to increase this metric is crucial.")
    print()

    print("4. RECOMMENDED NEXT STEPS:")
    print("   • Develop targeted marketing campaigns tailored to the characteristics of each identified segment.")
    print("   • Conduct A/B testing on website design and offers for different segments to optimize conversion.")
    print("   • Monitor how segments evolve over time to adapt business strategy accordingly.")
    print("   • Use these segments to build more complex predictive models for churn or lifetime value.")

summarize_insights()

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import fetch_openml
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import IsolationForest
from sklearn.manifold import TSNE
import umap
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.figure_factory as ff
from scipy import stats
import warnings
warnings.filterwarnings('ignore')

# Set plotting style
plt.style.use('seaborn-v0_8')
colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4', '#F7DC6F', '#BB8FCE']
sns.set_palette(colors)

print("Libraries imported successfully!")

# Load the KDD Cup 1999 dataset
# Note: This is a large dataset, so we'll work with a sample
print("Loading network intrusion detection dataset...")

# Load KDD dataset
data = fetch_openml(name='kddcup99', version=1, as_frame=True)
df_full = data.frame.copy()

print(f"Full dataset shape: {df_full.shape}")
print(f"Dataset columns: {list(df_full.columns)}")

# Sample the dataset for manageable analysis
np.random.seed(42)
sample_size = 10000
df = df_full.sample(n=sample_size, random_state=42).reset_index(drop=True)

print(f"Sampled dataset shape: {df.shape}")
print("\nClass distribution:")
print(df['label'].value_counts())

def preprocess_network_data(df):
    """Preprocess network data for anomaly detection analysis"""

    # Create binary target: normal vs attack
    df['is_attack'] = (df['label'] != 'normal').astype(int)

    # Identify categorical and numerical columns
    categorical_cols = ['protocol_type', 'service', 'flag']
    numerical_cols = [col for col in df.columns if col not in categorical_cols + ['label', 'is_attack']]

    print(f"Categorical columns: {categorical_cols}")
    print(f"Numerical columns: {len(numerical_cols)} features")

    # Encode categorical variables
    df_processed = df.copy()
    le_dict = {}

    for col in categorical_cols:
        le = LabelEncoder()
        df_processed[col + '_encoded'] = le.fit_transform(df_processed[col])
        le_dict[col] = le

    # Select features for analysis (exclude original categorical and target)
    feature_cols = [col for col in df_processed.columns 
                   if col not in categorical_cols + ['label']]
    feature_cols.remove('is_attack')

    X = df_processed[feature_cols]
    y = df_processed['is_attack']

    # Scale numerical features
    scaler = StandardScaler()
    X_scaled = pd.DataFrame(
        scaler.fit_transform(X),
        columns=X.columns,
        index=X.index
    )

    # Keep original data for interpretation
    df_analysis = df_processed.copy()

    return X_scaled, y, df_analysis, scaler, le_dict, feature_cols

X_scaled, y, df_analysis, scaler, le_dict, feature_cols = preprocess_network_data(df)

print(f"\nPreprocessed feature matrix shape: {X_scaled.shape}")
print(f"Attack rate: {y.mean():.2%}")

def perform_anomaly_detection(X_scaled, contamination=0.1):
    """Perform anomaly detection using Isolation Forest"""

    print("Training Isolation Forest for anomaly detection...")

    # Train Isolation Forest
    iso_forest = IsolationForest(
        contamination=contamination,
        random_state=42,
        n_jobs=-1
    )

    # Fit and predict
    anomaly_scores = iso_forest.fit_predict(X_scaled)
    anomaly_scores_proba = iso_forest.score_samples(X_scaled)

    # Convert to binary (1 = normal, -1 = anomaly)
    is_anomaly = (anomaly_scores == -1).astype(int)

    print(f"Detected {is_anomaly.sum()} anomalies ({is_anomaly.mean():.2%} of data)")

    return iso_forest, anomaly_scores, anomaly_scores_proba, is_anomaly

iso_forest, anomaly_scores, anomaly_scores_proba, is_anomaly = perform_anomaly_detection(X_scaled)

# Add anomaly information to analysis dataframe
df_analysis['anomaly_score'] = anomaly_scores_proba
df_analysis['is_anomaly_detected'] = is_anomaly
df_analysis['is_attack'] = y

def create_network_overview_dashboard():
    """Create comprehensive network traffic overview"""

    # Create subplots
    fig = make_subplots(
        rows=2, cols=2,
        subplot_titles=['Protocol Distribution', 'Service Distribution', 
                       'Attack Types', 'Anomaly Score Distribution'],
        specs=[[{'type': 'bar'}, {'type': 'bar'}],
               [{'type': 'bar'}, {'type': 'histogram'}]]
    )

    # Protocol distribution
    protocol_counts = df_analysis['protocol_type'].value_counts()
    fig.add_trace(
        go.Bar(x=protocol_counts.index, y=protocol_counts.values, 
               name='Protocol', marker_color='#4ECDC4'),
        row=1, col=1
    )

    # Service distribution (top 10)
    service_counts = df_analysis['service'].value_counts().head(10)
    fig.add_trace(
        go.Bar(x=service_counts.index, y=service_counts.values,
               name='Service', marker_color='#FF6B6B'),
        row=1, col=2
    )

    # Attack types
    attack_counts = df_analysis[df_analysis['is_attack'] == 1]['label'].value_counts().head(10)
    fig.add_trace(
        go.Bar(x=attack_counts.index, y=attack_counts.values,
               name='Attacks', marker_color='#45B7D1'),
        row=2, col=1
    )

    # Anomaly score distribution
    fig.add_trace(
        go.Histogram(x=df_analysis['anomaly_score'], nbinsx=50,
                    name='Anomaly Scores', marker_color='#96CEB4'),
        row=2, col=2
    )

    fig.update_layout(
        title_text="Network Traffic Analysis Dashboard",
        title_font_size=16,
        showlegend=False,
        height=800,
        width=1000
    )

    # Update x-axis labels for readability
    fig.update_xaxes(tickangle=45, row=1, col=2)
    fig.update_xaxes(tickangle=45, row=2, col=1)

    fig.show()

    return fig

dashboard_fig = create_network_overview_dashboard()

def create_parallel_coordinates_plot():
    """Create parallel coordinates plot for network analysis"""

    # Select key network features for parallel coordinates
    key_features = ['duration', 'src_bytes', 'dst_bytes', 'count', 
                   'srv_count', 'same_srv_rate', 'diff_srv_rate', 'dst_host_count']

    # Sample data to avoid overcrowding
    n_sample = 1000
    sample_idx = np.random.choice(len(df_analysis), n_sample, replace=False)
    df_sample = df_analysis.iloc[sample_idx].copy()

    # Normalize the selected features for better visualization
    df_parallel = df_sample[key_features].copy()
    for col in key_features:
        df_parallel[col] = (df_parallel[col] - df_parallel[col].min()) / (
            df_parallel[col].max() - df_parallel[col].min())

    # Add categorical information
    df_parallel['attack_type'] = df_sample['class']
    df_parallel['is_attack'] = df_sample['is_attack']
    df_parallel['is_anomaly'] = df_sample['is_anomaly_detected']

    # Create parallel coordinates plot
    fig = go.Figure(data=
        go.Parcoords(
            line=dict(color=df_parallel['is_attack'],
                     colorscale=[[0, '#4ECDC4'], [1, '#FF6B6B']],
                     showscale=True,
                     colorbar=dict(title="Attack Status")),
            dimensions=[
                dict(range=[0,1], label=col.replace('_', ' ').title(), values=df_parallel[col])
                for col in key_features
            ]
        )
    )

    fig.update_layout(
        title="Network Traffic Parallel Coordinates Analysis<br><sub>Blue: Normal Traffic, Red: Attack Traffic</sub>",
        title_font_size=16,
        width=1200,
        height=600
    )

    fig.show()

    # Create anomaly-focused version
    fig_anomaly = go.Figure(data=
        go.Parcoords(
            line=dict(color=df_parallel['is_anomaly'],
                     colorscale=[[0, '#45B7D1'], [1, '#F7DC6F']],
                     showscale=True,
                     colorbar=dict(title="Anomaly Detection")),
            dimensions=[
                dict(range=[0,1], label=col.replace('_', ' ').title(), values=df_parallel[col])
                for col in key_features
            ]
        )
    )

    fig_anomaly.update_layout(
        title="Network Traffic Anomaly Detection Analysis<br><sub>Blue: Normal, Yellow: Detected Anomaly</sub>",
        title_font_size=16,
        width=1200,
        height=600
    )

    fig_anomaly.show()

    return df_parallel

df_parallel = create_parallel_coordinates_plot()

def visualize_anomalies_with_umap():
    """Use UMAP to visualize network traffic anomalies"""

    print("Performing UMAP dimensionality reduction for anomaly visualization...")

    # Sample data for UMAP (computational efficiency)
    n_sample = 3000
    sample_idx = np.random.choice(len(X_scaled), n_sample, replace=False)
    X_sample = X_scaled.iloc[sample_idx]
    y_sample = y.iloc[sample_idx]
    anomaly_sample = is_anomaly[sample_idx]
    df_sample = df_analysis.iloc[sample_idx]

    # Perform UMAP
    reducer = umap.UMAP(n_neighbors=15, min_dist=0.1, random_state=42)
    X_umap = reducer.fit_transform(X_sample)

    # Create visualization
    fig, axes = plt.subplots(1, 2, figsize=(16, 7))

    # Plot 1: Actual attacks vs normal traffic
    for attack_status, color, label in zip([0, 1], ['#4ECDC4', '#FF6B6B'], ['Normal', 'Attack']):
        mask = y_sample == attack_status
        axes[0].scatter(X_umap[mask, 0], X_umap[mask, 1], 
                       c=color, label=label, alpha=0.6, s=30)

    axes[0].set_title('UMAP Projection: Actual Network Attacks', fontsize=14, fontweight='bold')
    axes[0].set_xlabel('UMAP Dimension 1')
    axes[0].set_ylabel('UMAP Dimension 2')
    axes[0].legend()
    axes[0].grid(True, alpha=0.3)

    # Plot 2: Detected anomalies
    for anomaly_status, color, label in zip([0, 1], ['#45B7D1', '#F7DC6F'], ['Normal', 'Detected Anomaly']):
        mask = anomaly_sample == anomaly_status
        axes[1].scatter(X_umap[mask, 0], X_umap[mask, 1], 
                       c=color, label=label, alpha=0.6, s=30)

    axes[1].set_title('UMAP Projection: Anomaly Detection Results', fontsize=14, fontweight='bold')
    axes[1].set_xlabel('UMAP Dimension 1')
    axes[1].set_ylabel('UMAP Dimension 2')
    axes[1].legend()
    axes[1].grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()

    # Interactive version with detailed information
    df_umap = df_sample.copy()
    df_umap['UMAP_1'] = X_umap[:, 0]
    df_umap['UMAP_2'] = X_umap[:, 1]
    df_umap['Attack_Status'] = df_umap['is_attack'].map({0: 'Normal', 1: 'Attack'})
    df_umap['Anomaly_Status'] = anomaly_sample
    df_umap['Anomaly_Status_Label'] = df_umap['Anomaly_Status'].map({0: 'Normal', 1: 'Detected Anomaly'})

    # Create interactive plot
    fig_interactive = px.scatter(
        df_umap, x='UMAP_1', y='UMAP_2', 
        color='Attack_Status',
        symbol='Anomaly_Status_Label',
        hover_data=['protocol_type', 'service', 'duration', 'src_bytes', 'dst_bytes', 'label'],
        title='Interactive Network Traffic Analysis (UMAP Projection)',
        color_discrete_map={'Normal': '#4ECDC4', 'Attack': '#FF6B6B'}
    )

    fig_interactive.update_layout(
        width=900,
        height=600,
        title_font_size=16
    )

    fig_interactive.show()

    return X_umap, df_umap

X_umap, df_umap = visualize_anomalies_with_umap()

def create_security_analysis_dashboard():
    """Create comprehensive security analysis dashboard with multiple linked views"""

    # Create subplots for linked analysis
    fig = make_subplots(
        rows=2, cols=2,
        subplot_titles=['Network Traffic Clusters (UMAP)', 'Attack Type Distribution', 
                       'Temporal Patterns', 'Feature Importance'],
        specs=[[{'type': 'scatter'}, {'type': 'bar'}],
               [{'type': 'scatter'}, {'type': 'bar'}]]
    )

    # 1. UMAP scatter plot
    normal_mask = df_umap['is_attack'] == 0
    attack_mask = df_umap['is_attack'] == 1

    fig.add_trace(
        go.Scatter(x=df_umap[normal_mask]['UMAP_1'], y=df_umap[normal_mask]['UMAP_2'],
                  mode='markers', name='Normal Traffic', 
                  marker=dict(color='#4ECDC4', size=4, opacity=0.6)),
        row=1, col=1
    )

    fig.add_trace(
        go.Scatter(x=df_umap[attack_mask]['UMAP_1'], y=df_umap[attack_mask]['UMAP_2'],
                  mode='markers', name='Attack Traffic',
                  marker=dict(color='#FF6B6B', size=4, opacity=0.8)),
        row=1, col=1
    )

    # 2. Attack type distribution
    attack_types = df_umap[df_umap['is_attack'] == 1]['label'].value_counts().head(8)
    fig.add_trace(
        go.Bar(x=attack_types.values, y=attack_types.index, 
               orientation='h', name='Attack Types',
               marker_color='#45B7D1'),
        row=1, col=2
    )

    # 3. Temporal analysis (simulate time-based patterns)
    # Create synthetic time data for demonstration
    np.random.seed(42)
    df_umap['hour'] = np.random.randint(0, 24, len(df_umap))
    hourly_attacks = df_umap[df_umap['is_attack'] == 1].groupby('hour').size().reindex(range(24), fill_value=0)

    fig.add_trace(
        go.Scatter(x=list(range(24)), y=hourly_attacks.values,
                  mode='lines+markers', name='Attacks by Hour',
                  line=dict(color='#96CEB4', width=3)),
        row=2, col=1
    )

    # 4. Feature importance (based on anomaly detection)
    # Calculate feature statistics for anomalies vs normal
    feature_importance = []
    key_features = ['duration', 'src_bytes', 'dst_bytes', 'count', 'srv_count']

    for feature in key_features:
        if feature in df_umap.columns:
            normal_mean = df_umap[df_umap['is_attack'] == 0][feature].mean()
            attack_mean = df_umap[df_umap['is_attack'] == 1][feature].mean()
            importance = abs(attack_mean - normal_mean) / (normal_mean + 1e-6)
            feature_importance.append((feature, importance))

    feature_importance.sort(key=lambda x: x[1], reverse=True)
    features, importances = zip(*feature_importance)

    fig.add_trace(
        go.Bar(x=importances, y=features, orientation='h',
               name='Feature Discrimination', marker_color='#BB8FCE'),
        row=2, col=2
    )

    fig.update_layout(
        title_text="Network Security Analysis Dashboard",
        title_font_size=16,
        showlegend=True,
        height=800,
        width=1200
    )

    fig.show()

    return fig

security_dashboard = create_security_analysis_dashboard()

def evaluate_anomaly_detection_performance():
    """Evaluate the performance of anomaly detection against known attacks"""

    from sklearn.metrics import classification_report, confusion_matrix
    import seaborn as sns

    # Calculate performance metrics
    print("=== ANOMALY DETECTION PERFORMANCE EVALUATION ===\n")

    # Classification report
    print("Classification Report:")
    print(classification_report(y, is_anomaly, target_names=['Normal', 'Attack']))

    # Confusion matrix
    cm = confusion_matrix(y, is_anomaly)

    # Visualization
    fig, axes = plt.subplots(1, 2, figsize=(15, 6))

    # Confusion Matrix
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
                xticklabels=['Predicted Normal', 'Predicted Attack'],
                yticklabels=['Actual Normal', 'Actual Attack'],
                ax=axes[0])
    axes[0].set_title('Confusion Matrix\n(Anomaly Detection vs Actual Attacks)', fontweight='bold')

    # Performance by attack type
    attack_data = df_analysis[df_analysis['is_attack'] == 1].copy()
    attack_performance = attack_data.groupby('label').agg({
        'is_anomaly_detected': ['sum', 'count']
    }).round(3)

    attack_performance.columns = ['Detected', 'Total']
    attack_performance['Detection_Rate'] = (attack_performance['Detected'] / 
                                           attack_performance['Total']).round(3)
    attack_performance = attack_performance.sort_values('Detection_Rate', ascending=True).tail(10)

    # Plot detection rates
    bars = axes[1].barh(range(len(attack_performance)), attack_performance['Detection_Rate'], 
                       color=colors[:len(attack_performance)])
    axes[1].set_yticks(range(len(attack_performance)))
    axes[1].set_yticklabels(attack_performance.index, fontsize=10)
    axes[1].set_xlabel('Detection Rate')
    axes[1].set_title('Attack Detection Rate by Attack Type', fontweight='bold')
    axes[1].set_xlim(0, 1)

    # Add value labels
    for i, (bar, rate) in enumerate(zip(bars, attack_performance['Detection_Rate'])):
        axes[1].text(bar.get_width() + 0.02, bar.get_y() + bar.get_height()/2,
                    f'{rate:.2f}', va='center', ha='left', fontweight='bold')

    plt.tight_layout()
    plt.show()

    return attack_performance

performance_results = evaluate_anomaly_detection_performance()

def simulate_security_investigation():
    """Simulate a security analyst's investigation workflow"""

    print("=== SIMULATED SECURITY INVESTIGATION WORKFLOW ===\n")

    # Step 1: Identify high-priority anomalies
    high_priority = df_analysis[
        (df_analysis['is_anomaly_detected'] == 1) & 
        (df_analysis['anomaly_score'] < -0.3)  # Very low anomaly scores
    ].copy()

    print(f"Step 1: Identified {len(high_priority)} high-priority anomalies")
    print(f"Percentage of total traffic: {len(high_priority)/len(df_analysis)*100:.2f}%")

    # Step 2: Analyze attack types in high-priority anomalies
    print(f"\nStep 2: Attack type analysis of high-priority anomalies:")
    attack_types_priority = high_priority['label'].value_counts()
    for attack_type, count in attack_types_priority.head().items():
        percentage = count / len(high_priority) * 100
        print(f"  • {attack_type}: {count} instances ({percentage:.1f}%)")

    # Step 3: Network behavior analysis
    print(f"\nStep 3: Network behavior patterns:")

    # Protocol analysis
    protocol_analysis = high_priority['protocol_type'].value_counts()
    print(f"  • Most common protocols in anomalies: {dict(protocol_analysis.head(3))}")

    # Service analysis  
    service_analysis = high_priority['service'].value_counts()
    print(f"  • Most targeted services: {dict(service_analysis.head(3))}")

    # Statistical analysis
    normal_traffic = df_analysis[df_analysis['is_attack'] == 0]
    print(f"\nStep 4: Statistical comparison (anomalies vs normal):")

    key_metrics = ['duration', 'src_bytes', 'dst_bytes', 'count']
    for metric in key_metrics:
        if metric in high_priority.columns:
            normal_mean = normal_traffic[metric].mean()
            anomaly_mean = high_priority[metric].mean()
            ratio = anomaly_mean / (normal_mean + 1e-6)
            print(f"  • {metric}: {ratio:.2f}x difference (anomaly/normal)")

    # Step 5: Visualization for investigation
    print(f"\nStep 5: Creating investigation visualization...")

    fig, axes = plt.subplots(2, 2, figsize=(16, 12))

    # Top-left: Connection duration distribution
    axes[0,0].hist(normal_traffic['duration'].clip(0, 1000), bins=50, alpha=0.7, 
                   label='Normal', color='#4ECDC4', density=True)
    axes[0,0].hist(high_priority['duration'].clip(0, 1000), bins=50, alpha=0.7,
                   label='High Priority Anomalies', color='#FF6B6B', density=True)
    axes[0,0].set_xlabel('Connection Duration (seconds)')
    axes[0,0].set_ylabel('Density')
    axes[0,0].set_title('Connection Duration Distribution')
    axes[0,0].legend()
    axes[0,0].grid(True, alpha=0.3)

    # Top-right: Bytes transferred analysis
    normal_bytes = np.log10(normal_traffic['src_bytes'].clip(1, None))
    anomaly_bytes = np.log10(high_priority['src_bytes'].clip(1, None))

    axes[0,1].hist(normal_bytes, bins=50, alpha=0.7, label='Normal', 
                   color='#4ECDC4', density=True)
    axes[0,1].hist(anomaly_bytes, bins=50, alpha=0.7, 
                   label='High Priority Anomalies', color='#FF6B6B', density=True)
    axes[0,1].set_xlabel('Log10(Source Bytes)')
    axes[0,1].set_ylabel('Density')
    axes[0,1].set_title('Data Transfer Volume Distribution')
    axes[0,1].legend()
    axes[0,1].grid(True, alpha=0.3)

    # Bottom-left: Protocol distribution
    protocol_normal = normal_traffic['protocol_type'].value_counts()
    protocol_anomaly = high_priority['protocol_type'].value_counts()

    protocols = list(set(protocol_normal.index) | set(protocol_anomaly.index))
    normal_counts = [protocol_normal.get(p, 0) for p in protocols]
    anomaly_counts = [protocol_anomaly.get(p, 0) for p in protocols]

    x = np.arange(len(protocols))
    width = 0.35

    axes[1,0].bar(x - width/2, normal_counts, width, label='Normal', color='#4ECDC4')
    axes[1,0].bar(x + width/2, anomaly_counts, width, label='Anomalies', color='#FF6B6B')
    axes[1,0].set_xlabel('Protocol Type')
    axes[1,0].set_ylabel('Count')
    axes[1,0].set_title('Protocol Distribution Comparison')
    axes[1,0].set_xticks(x)
    axes[1,0].set_xticklabels(protocols)
    axes[1,0].legend()
    axes[1,0].grid(True, alpha=0.3)

    # Bottom-right: Time-based analysis (simulated)
    np.random.seed(42)
    normal_traffic_sample = normal_traffic.sample(min(1000, len(normal_traffic)))
    normal_traffic_sample['hour'] = np.random.randint(0, 24, len(normal_traffic_sample))
    high_priority['hour'] = np.random.randint(0, 24, len(high_priority))

    normal_hourly = normal_traffic_sample.groupby('hour').size()
    anomaly_hourly = high_priority.groupby('hour').size().reindex(range(24), fill_value=0)

    axes[1,1].plot(range(24), normal_hourly, marker='o', label='Normal Traffic', 
                   color='#4ECDC4', linewidth=2)
    axes[1,1].plot(range(24), anomaly_hourly, marker='s', label='Anomalies',
                   color='#FF6B6B', linewidth=2)
    axes[1,1].set_xlabel('Hour of Day')
    axes[1,1].set_ylabel('Traffic Count')
    axes[1,1].set_title('Temporal Distribution Pattern')
    axes[1,1].legend()
    axes[1,1].grid(True, alpha=0.3)
    axes[1,1].set_xticks(range(0, 24, 4))

    plt.suptitle('Security Investigation Dashboard: High-Priority Anomalies Analysis', 
                 fontsize=16, fontweight='bold')
    plt.tight_layout()
    plt.show()

    return high_priority

investigation_results = simulate_security_investigation()