Feature Engineering & Deployment

Feature Engineering and Preprocessing Guide

Essential techniques for preparing data and maintaining machine learning models

Introduction

Feature engineering and preprocessing are critical steps in the machine learning pipeline that can significantly impact model performance. This guide covers essential techniques for handling missing data, encoding categorical variables, scaling features, feature selection, and model deployment.

Proper data preparation not only improves model accuracy but also ensures stability, reproducibility, and efficient operation in production environments.

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Check for missing values
missing_values = df.isnull().sum()
missing_percent = (missing_values / len(df)) * 100

# Display results
missing_info = pd.DataFrame({
    'Missing Values': missing_values,
    'Percent Missing': missing_percent
})
print(missing_info[missing_info['Missing Values'] > 0])

# Visualize missing values
plt.figure(figsize=(10, 6))
sns.heatmap(df.isnull(), cbar=False, yticklabels=False, cmap='viridis')
plt.title('Missing Value Heatmap')
plt.show()

# Drop rows with any missing values
df_dropped = df.dropna()

# Drop rows only if all columns are missing
df_dropped = df.dropna(how='all')

# Drop columns with more than 50% missing values
threshold = len(df) * 0.5
df_dropped = df.dropna(axis=1, thresh=threshold)

Imputation

Filling missing values with estimated values:

# Simple imputation
df_filled = df.copy()
df_filled['numeric_column'].fillna(df_filled['numeric_column'].mean(), inplace=True)  # Mean
df_filled['numeric_column'].fillna(df_filled['numeric_column'].median(), inplace=True)  # Median
df_filled['categorical_column'].fillna(df_filled['categorical_column'].mode()[0], inplace=True)  # Mode

# Using scikit-learn's SimpleImputer
from sklearn.impute import SimpleImputer

# For numeric columns
numeric_imputer = SimpleImputer(strategy='mean')
df_numeric = pd.DataFrame(numeric_imputer.fit_transform(df[['age', 'income']]),
                          columns=['age', 'income'])

# For categorical columns
cat_imputer = SimpleImputer(strategy='most_frequent')
df_cat = pd.DataFrame(cat_imputer.fit_transform(df[['education', 'occupation']]),
                      columns=['education', 'occupation'])

Advanced Imputation

Using more sophisticated methods to estimate missing values:

# KNN Imputation
from sklearn.impute import KNNImputer
imputer = KNNImputer(n_neighbors=5)
df_knn_imputed = pd.DataFrame(
    imputer.fit_transform(df),
    columns=df.columns
)

# Iterative Imputation (MICE)
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
imputer = IterativeImputer(max_iter=10, random_state=42)
df_mice_imputed = pd.DataFrame(
    imputer.fit_transform(df),
    columns=df.columns
)

Adding Missing Indicators

Creating binary flags to indicate where values were missing:

# Create missing indicators
for col in ['age', 'income', 'education']:
    df[f'{col}_was_missing'] = df[col].isnull().astype(int)

# Using scikit-learn's MissingIndicator
from sklearn.impute import MissingIndicator
indicator = MissingIndicator()
missing_indicators = indicator.fit_transform(df)
indicator_df = pd.DataFrame(missing_indicators, columns=[f'{col}_missing' for col in df.columns])

Categorical Variable Encoding

Machine learning algorithms generally require numerical input, so categorical variables need to be converted to numbers.

Label Encoding

Assigning a unique integer to each category (suitable for ordinal data):

from sklearn.preprocessing import LabelEncoder

# Create encoder
le = LabelEncoder()

# Example with a single column
df['encoded_education'] = le.fit_transform(df['education'])

# Get mapping
mapping = dict(zip(le.classes_, le.transform(le.classes_)))
print("Mapping:", mapping)

# Example output (mapping):
# {'high school': 0, 'bachelor': 1, 'master': 2, 'phd': 3}

One-Hot Encoding

Creating binary columns for each category (suitable for nominal data):

# Using pandas
df_one_hot = pd.get_dummies(df, columns=['education', 'occupation'], drop_first=True)

# Using scikit-learn's OneHotEncoder
from sklearn.preprocessing import OneHotEncoder

# Create encoder (sparse matrix output by default)
encoder = OneHotEncoder(sparse=False, drop='first')  # drop='first' eliminates multicollinearity
encoded_data = encoder.fit_transform(df[['education', 'occupation']])

# Convert to DataFrame with column names
encoded_df = pd.DataFrame(
    encoded_data,
    columns=encoder.get_feature_names_out(['education', 'occupation'])
)

# Example output columns:
# ['education_bachelor', 'education_master', 'education_phd', 'occupation_engineer', ...]

Binary Encoding

More memory-efficient for high-cardinality features:

# Install category_encoders if needed: pip install category_encoders
import category_encoders as ce

# Binary encoding
encoder = ce.BinaryEncoder(cols=['zipcode', 'product_id'])
df_binary_encoded = encoder.fit_transform(df)

Target Encoding

Using target statistics to encode categories:

# Simple target mean encoding
mean_target = df.groupby('category')['target'].mean()
df['category_encoded'] = df['category'].map(mean_target)

# Using category_encoders
encoder = ce.TargetEncoder(cols=['category'])
df_target_encoded = encoder.fit_transform(df, df['target'])

Feature Scaling

Standardizing or normalizing features to the same scale to improve model performance.

StandardScaler

Standardizes features by removing the mean and scaling to unit variance (z-score normalization):

from sklearn.preprocessing import StandardScaler

# Create scaler
scaler = StandardScaler()

# Fit and transform
X_scaled = scaler.fit_transform(X)

# For DataFrames, preserving column names
X_scaled_df = pd.DataFrame(
    scaler.fit_transform(df[['age', 'income', 'experience']]),
    columns=['age', 'income', 'experience'],
    index=df.index
)

The formula applied is: z = (x - μ) / σ

This transformation results in features with a mean of 0 and a standard deviation of 1.

MinMaxScaler

Scales features to a specified range (default [0, 1]):

from sklearn.preprocessing import MinMaxScaler

# Create scaler with default range [0, 1]
scaler = MinMaxScaler()

# Or specify a custom range, e.g., [-1, 1]
custom_scaler = MinMaxScaler(feature_range=(-1, 1))

# Fit and transform
X_minmax = scaler.fit_transform(X)

# For DataFrames
X_minmax_df = pd.DataFrame(
    scaler.fit_transform(df[['age', 'income', 'experience']]),
    columns=['age', 'income', 'experience'],
    index=df.index
)

The formula applied is: X_scaled = (X - X_min) / (X_max - X_min)

RobustScaler

Scales features using statistics that are robust to outliers:

from sklearn.preprocessing import RobustScaler

# Create scaler
scaler = RobustScaler()

# Fit and transform
X_robust = scaler.fit_transform(X)

This scaler uses the median and interquartile range instead of the mean and variance, making it robust to outliers.

When to Use Each Scaler

Feature Selection Techniques

Selecting the most relevant features to improve model performance and reduce overfitting.

Filter Methods

Using statistical tests to select features without involving the model:

# Variance Threshold (removing constant features)
from sklearn.feature_selection import VarianceThreshold
selector = VarianceThreshold(threshold=0.1)  # features with variance <= 0.1 will be removed
X_selected = selector.fit_transform(X)

# Correlation with target (for regression)
import pandas as pd
import numpy as np

# Calculate correlations
correlations = df.corr()['target'].sort_values(ascending=False)
print(correlations)

# Select features with correlation above threshold
correlation_threshold = 0.2
selected_features = correlations[abs(correlations) > correlation_threshold].index.tolist()
selected_features.remove('target')  # Remove target itself
X_selected = df[selected_features]

# Chi-squared test (for classification with categorical features)
from sklearn.feature_selection import chi2, SelectKBest
selector = SelectKBest(chi2, k=10)  # select top 10 features
X_selected = selector.fit_transform(X, y)
selected_feature_indices = selector.get_support(indices=True)
selected_features = [X.columns[i] for i in selected_feature_indices]

Wrapper Methods

Evaluating subsets of features using the model itself:

# Recursive Feature Elimination (RFE)
from sklearn.feature_selection import RFE
from sklearn.ensemble import RandomForestClassifier

# Create the RFE object
estimator = RandomForestClassifier(n_estimators=100, random_state=42)
selector = RFE(estimator, n_features_to_select=10, step=1)
selector = selector.fit(X, y)

# Get selected features
selected_features = X.columns[selector.support_]
print("Selected features:", selected_features)
print("Feature ranking:", selector.ranking_)

# Forward Feature Selection
from mlxtend.feature_selection import SequentialFeatureSelector
from sklearn.linear_model import LogisticRegression

sfs = SequentialFeatureSelector(
    LogisticRegression(),
    k_features=10,
    forward=True,
    scoring='accuracy',
    cv=5
)
sfs.fit(X, y)
selected_features = list(X.columns[list(sfs.k_feature_idx_)])

Embedded Methods

Using models that perform feature selection as part of their algorithm:

# LASSO Regression
from sklearn.linear_model import Lasso
from sklearn.preprocessing import StandardScaler

# Scale features first
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Fit LASSO model
lasso = Lasso(alpha=0.1)
lasso.fit(X_scaled, y)

# Get feature importance
feature_importance = pd.DataFrame({
    'Feature': X.columns,
    'Coefficient': lasso.coef_
})
feature_importance = feature_importance.sort_values(by='Coefficient', key=abs, ascending=False)
print(feature_importance)

# Random Forest Feature Importance
from sklearn.ensemble import RandomForestClassifier
import matplotlib.pyplot as plt

# Train model
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X, y)

# Get feature importances
feature_importance = pd.DataFrame({
    'Feature': X.columns,
    'Importance': rf.feature_importances_
})
feature_importance = feature_importance.sort_values('Importance', ascending=False)

# Plot
plt.figure(figsize=(10, 6))
plt.barh(feature_importance['Feature'], feature_importance['Importance'])
plt.xlabel('Importance')
plt.title('Feature Importance')
plt.tight_layout()
plt.show()

Dimensionality Reduction

Techniques that transform the feature space:

# Principal Component Analysis (PCA)
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# Create PCA object
pca = PCA(n_components=0.95)  # Keep components that explain 95% of variance

# Fit and transform
X_pca = pca.fit_transform(X_scaled)  # Use scaled data

# Plot explained variance
plt.figure(figsize=(10, 6))
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Explained Variance')
plt.grid(True)
plt.show()

# Check shape after PCA
print(f"Original shape: {X.shape}, PCA shape: {X_pca.shape}")

# t-SNE for visualization
from sklearn.manifold import TSNE

# Create t-SNE object
tsne = TSNE(n_components=2, random_state=42)
X_tsne = tsne.fit_transform(X_scaled)

# Plot
plt.figure(figsize=(10, 8))
plt.scatter(X_tsne[:, 0], X_tsne[:, 1], c=y, cmap='viridis')
plt.colorbar()
plt.title('t-SNE Visualization')
plt.show()