Ethics & Best Practices

# Technique 1: Balance representation
from sklearn.utils import resample

def balance_dataset(df, protected_attr, target_col):
    """Balance dataset so all groups have equal representation."""

    # Find minimum group size
    min_size = df.groupby(protected_attr).size().min()

    # Downsample all groups to minimum size
    balanced_dfs = []
    for group in df[protected_attr].unique():
        group_df = df[df[protected_attr] == group]
        balanced_group = resample(
            group_df,
            n_samples=min_size,
            random_state=42
        )
        balanced_dfs.append(balanced_group)

    return pd.concat(balanced_dfs, ignore_index=True)


# Technique 2: Remove biased correlations
def remove_proxy_features(df, protected_attrs):
    """
    Remove features that are proxies for protected attributes.

    Example: ZIP code often correlates with race, income
    """
    # Calculate correlation with protected attributes
    proxy_features = []

    for col in df.columns:
        if col in protected_attrs:
            continue

        # Check if categorical or numeric
        if df[col].dtype in ['object', 'category']:
            # For categorical, check if distribution varies by protected attr
            for protected in protected_attrs:
                # Chi-square test or similar
                pass
        else:
            # For numeric, check correlation
            for protected in protected_attrs:
                if df[protected].dtype not in ['object', 'category']:
                    corr = df[col].corr(df[protected])
                    if abs(corr) > 0.7:  # High correlation threshold
                        proxy_features.append(col)

    print(f"Identified proxy features: {proxy_features}")
    return df.drop(columns=proxy_features)


# Technique 3: Reweighting samples
from sklearn.utils.class_weight import compute_sample_weight

def compute_fair_weights(y, protected_attr):
    """
    Compute sample weights to balance groups.

    Gives higher weight to underrepresented groups.
    """
    # Combine target and protected attribute
    combined = y.astype(str) + "_" + protected_attr.astype(str)

    weights = compute_sample_weight('balanced', combined)

    return weights

# Use in training:
# model.fit(X, y, sample_weight=compute_fair_weights(y, df['gender']))

from sklearn.base import BaseEstimator, ClassifierMixin
import numpy as np

class FairClassifier(BaseEstimator, ClassifierMixin):
    """
    Classifier with fairness constraints.

    Adds fairness penalty to loss function.
    """

    def __init__(self, base_model, fairness_penalty=0.1):
        self.base_model = base_model
        self.fairness_penalty = fairness_penalty

    def fit(self, X, y, protected_attribute):
        """
        Train with fairness constraint.

        Minimizes: prediction_error + fairness_penalty * fairness_violation
        """
        # Custom training loop that penalizes disparate impact
        # (Simplified - real implementation would use gradient-based optimization)

        self.base_model.fit(X, y)

        # Iteratively adjust to reduce bias
        for iteration in range(10):
            y_pred = self.base_model.predict(X)

            # Calculate bias
            bias_score = self._calculate_bias(y_pred, protected_attribute)

            if bias_score < 0.1:  # Acceptable threshold
                break

            # Adjust model (simplified)
            # Real implementation: adjust loss function or decision threshold
            pass

        return self

    def _calculate_bias(self, predictions, protected_attr):
        """Calculate bias metric (e.g., demographic parity)."""
        rates = []
        for group in np.unique(protected_attr):
            mask = protected_attr == group
            rate = predictions[mask].mean()
            rates.append(rate)

        return max(rates) - min(rates)

    def predict(self, X):
        return self.base_model.predict(X)


# Using fairness-aware libraries
from fairlearn.reductions import ExponentiatedGradient, DemographicParity
from sklearn.linear_model import LogisticRegression

# Train with fairness constraint
base_model = LogisticRegression()
fair_model = ExponentiatedGradient(
    estimator=base_model,
    constraints=DemographicParity()
)

fair_model.fit(X_train, y_train, sensitive_features=protected_attr_train)

def calibrate_by_group(model, X, protected_attr, target_fpr=0.1):
    """
    Adjust decision thresholds per group to equalize false positive rates.

    Args:
        model: Trained model with predict_proba
        X: Features
        protected_attr: Protected attribute values
        target_fpr: Desired false positive rate

    Returns:
        Adjusted predictions
    """
    # Get prediction probabilities
    probabilities = model.predict_proba(X)[:, 1]

    adjusted_predictions = np.zeros(len(X))

    # Calculate optimal threshold per group
    for group in np.unique(protected_attr):
        group_mask = protected_attr == group
        group_probs = probabilities[group_mask]

        # Find threshold that achieves target FPR for this group
        # (Simplified - real implementation would use validation set)
        threshold = np.percentile(group_probs, (1 - target_fpr) * 100)

        adjusted_predictions[group_mask] = (group_probs >= threshold).astype(int)

    return adjusted_predictions


# Alternative: Equalized odds post-processing
from fairlearn.postprocessing import ThresholdOptimizer

threshold_optimizer = ThresholdOptimizer(
    estimator=trained_model,
    constraints="equalized_odds"
)

threshold_optimizer.fit(X_val, y_val, sensitive_features=protected_attr_val)
fair_predictions = threshold_optimizer.predict(X_test, sensitive_features=protected_attr_test)

# Bad: Collecting everything
user_data = {
    'name': 'John Doe',
    'ssn': '123-45-6789',
    'address': '123 Main St',
    'phone': '555-1234',
    'email': 'john@example.com',
    'browsing_history': [...],
    'purchase_history': [...],
    'location_history': [...]
}

# Good: Collect only what you need for the task
# For a recommendation system:
user_data = {
    'user_id': 'uuid-123',  # Pseudonymous identifier
    'age_group': '25-34',   # Aggregated, not exact age
    'category_preferences': ['electronics', 'books'],  # Not full history
    'region': 'Northeast'   # Not exact location
}

# Automatically delete old data
def cleanup_old_data(retention_days=90):
    """Delete data older than retention period."""
    cutoff_date = datetime.now() - timedelta(days=retention_days)

    # Delete old records
    db.execute(
        "DELETE FROM user_interactions WHERE created_at < ?",
        (cutoff_date,)
    )

import hashlib
import pandas as pd

def pseudonymize_dataframe(df, id_columns, sensitive_columns):
    """
    Pseudonymize a dataframe for analysis.

    Args:
        df: Original dataframe
        id_columns: Columns with direct identifiers (names, emails, SSN)
        sensitive_columns: Columns to aggregate or remove

    Returns:
        Pseudonymized dataframe safe for analysis
    """
    df_anon = df.copy()

    # Hash direct identifiers
    for col in id_columns:
        df_anon[col] = df_anon[col].apply(
            lambda x: hashlib.sha256(str(x).encode()).hexdigest()[:16]
        )

    # Aggregate sensitive numeric data
    for col in sensitive_columns:
        if df_anon[col].dtype in [np.int64, np.float64]:
            # Bin into ranges
            df_anon[col] = pd.cut(df_anon[col], bins=5, labels=['very_low', 'low', 'medium', 'high', 'very_high'])

    return df_anon


# Example: Anonymize medical data
medical_data = pd.DataFrame({
    'patient_id': ['P001', 'P002', 'P003'],
    'name': ['Alice', 'Bob', 'Charlie'],
    'age': [34, 45, 29],
    'salary': [75000, 120000, 45000],
    'diagnosis': ['diabetes', 'hypertension', 'healthy']
})

anonymized = pseudonymize_dataframe(
    medical_data,
    id_columns=['patient_id', 'name'],
    sensitive_columns=['age', 'salary']
)

print(anonymized)
# patient_id: hashed
# name: hashed
# age: binned into ranges
# salary: binned into ranges

# Differential privacy: Add noise to protect individuals

def add_laplace_noise(value, sensitivity, epsilon):
    """
    Add Laplace noise for differential privacy.

    Args:
        value: True value
        sensitivity: Maximum change one individual can cause
        epsilon: Privacy parameter (smaller = more privacy, less accuracy)

    Returns:
        Noisy value
    """
    scale = sensitivity / epsilon
    noise = np.random.laplace(0, scale)
    return value + noise


def private_mean(data, epsilon=1.0):
    """
    Calculate mean with differential privacy guarantee.

    Args:
        data: Array of values
        epsilon: Privacy budget

    Returns:
        Differentially private mean
    """
    true_mean = np.mean(data)

    # Sensitivity: how much one person can change the mean
    sensitivity = (np.max(data) - np.min(data)) / len(data)

    private_mean = add_laplace_noise(true_mean, sensitivity, epsilon)

    return private_mean


# Example: Private age statistics
ages = np.array([25, 30, 35, 40, 45, 50, 55, 60])

true_mean = np.mean(ages)
private_mean_age = private_mean(ages, epsilon=1.0)

print(f"True mean age: {true_mean:.1f}")
print(f"Private mean age: {private_mean_age:.1f}")  # Slightly different due to noise


# Using differential privacy libraries
from diffprivlib.models import LogisticRegression as PrivateLogisticRegression

# Train with privacy guarantee
private_model = PrivateLogisticRegression(epsilon=1.0)
private_model.fit(X_train, y_train)

from cryptography.fernet import Fernet
import json

class SecureDataStore:
    """Store sensitive data with encryption."""

    def __init__(self, encryption_key=None):
        if encryption_key is None:
            encryption_key = Fernet.generate_key()
        self.cipher = Fernet(encryption_key)

    def encrypt_data(self, data):
        """Encrypt data before storage."""
        json_data = json.dumps(data)
        encrypted = self.cipher.encrypt(json_data.encode())
        return encrypted

    def decrypt_data(self, encrypted_data):
        """Decrypt data when needed."""
        decrypted = self.cipher.decrypt(encrypted_data)
        return json.loads(decrypted.decode())

    def store_user_data(self, user_id, data):
        """Store user data encrypted."""
        encrypted = self.encrypt_data(data)
        # Store in database
        db.execute(
            "INSERT INTO secure_data (user_id, encrypted_data) VALUES (?, ?)",
            (user_id, encrypted)
        )

        # Audit log
        self._log_access(user_id, action="store")

    def _log_access(self, user_id, action):
        """Log all data access for audit trail."""
        db.execute(
            "INSERT INTO audit_log (user_id, action, timestamp) VALUES (?, ?, ?)",
            (user_id, action, datetime.now())
        )


# Usage
store = SecureDataStore()

# Store sensitive data encrypted
user_data = {
    'medical_history': [...],
    'financial_info': {...}
}

store.store_user_data('user_123', user_data)

# Defense: Input validation and adversarial training

def detect_adversarial_input(input_data, model, threshold=0.5):
    """
    Detect potential adversarial inputs.

    Uses prediction confidence and consistency checks.
    """
    # Get prediction confidence
    proba = model.predict_proba([input_data])[0]
    max_confidence = np.max(proba)

    # Low confidence might indicate adversarial input
    if max_confidence < threshold:
        return True, "Low confidence prediction"

    # Check consistency with similar inputs
    # (Add small random noise and see if prediction changes drastically)
    predictions = []
    for _ in range(10):
        noisy_input = input_data + np.random.normal(0, 0.01, input_data.shape)
        pred = model.predict([noisy_input])[0]
        predictions.append(pred)

    # If predictions vary wildly, might be adversarial
    if len(set(predictions)) > 5:
        return True, "Unstable predictions"

    return False, "Input appears legitimate"


# Defense: Adversarial training
def adversarial_training(model, X_train, y_train, epsilon=0.1):
    """
    Train model on both clean and adversarial examples.

    Makes model more robust to attacks.
    """
    X_adversarial = []
    y_adversarial = []

    for x, y in zip(X_train, y_train):
        # Generate adversarial example
        # (Simplified - real implementation uses gradient-based methods)
        adv_x = x + np.random.uniform(-epsilon, epsilon, x.shape)
        X_adversarial.append(adv_x)
        y_adversarial.append(y)

    # Combine original and adversarial data
    X_combined = np.vstack([X_train, X_adversarial])
    y_combined = np.concatenate([y_train, y_adversarial])

    # Train on both
    model.fit(X_combined, y_combined)

    return model