DATA
PREPROCESSING CHEATSHEET.

Check for Missing

df.isnull().sum()
df.isnull().sum().sum()
df.info()  # shows non‑null counts

Remove Missing

df.dropna()  # drop rows with any NaN
df.dropna(thresh=3)  # rows with ≥3 non‑null
df.dropna(subset=['col1'])  # specific columns

Imputation

# Simple
df.fillna(0)
df.fillna('missing')
df['col'].fillna(df['col'].mean())
df['col'].fillna(df['col'].median())
df['col'].fillna(df['col'].mode()[0])

# Forward/Backward
df.fillna(method='ffill')
df.fillna(method='bfill')

Advanced Imputation

from sklearn.impute import SimpleImputer, KNNImputer

# SimpleImputer
imputer = SimpleImputer(strategy='mean')
df_imputed = imputer.fit_transform(df)

# KNN Imputer
imputer = KNNImputer(n_neighbors=5)
df_imputed = imputer.fit_transform(df)

# Iterative Imputer
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
imputer = IterativeImputer()
df_imputed = imputer.fit_transform(df)

Detection

# Z‑score
from scipy import stats
z_scores = np.abs(stats.zscore(df))
outliers = (z_scores > 3).all(axis=1)

# IQR
Q1 = df.quantile(0.25)
Q3 = df.quantile(0.75)
IQR = Q3 - Q1
outliers = ((df < (Q1 - 1.5 * IQR)) | (df > (Q3 + 1.5 * IQR))).any(axis=1)

# Isolation Forest
from sklearn.ensemble import IsolationForest
iso = IsolationForest(contamination=0.1)
outliers = iso.fit_predict(df) == -1

Treatment

# Remove
df_clean = df[~outliers]

# Cap/Winsorize
def cap_outliers(df, col, lower, upper):
    df[col] = df[col].clip(lower=lower, upper=upper)

# Transform
df['col_log'] = np.log(df['col'])
df['col_sqrt'] = np.sqrt(df['col'])
df['col_inv'] = 1 / df['col']

Standardization (Z‑Score)

• Mean = 0, Std = 1
• Formula: (x - μ) / σ
• Use when features have different units
• Works for most algorithms (SVM, Logistic Regression, Neural Nets)

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
df_scaled = scaler.fit_transform(df)

Min‑Max Scaling (Normalization)

• Range: [0, 1]
• Formula: (x - min) / (max - min)
• Use when data has bounded range
• Sensitive to outliers
• Good for neural networks, K‑NN

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
df_scaled = scaler.fit_transform(df)

// Custom range
scaler = MinMaxScaler(feature_range=(-1, 1))

Robust Scaling

• Uses median and IQR
• Formula: (x - median) / IQR
• Robust to outliers
• Use when outliers are present

from sklearn.preprocessing import RobustScaler
scaler = RobustScaler()
df_scaled = scaler.fit_transform(df)

Normalizer (L2/L1 Norm)

• Scales rows (samples) to unit norm
• L2: Euclidean norm = 1
• L1: Manhattan norm = 1
• Use for text data, embeddings, distance‑based

from sklearn.preprocessing import Normalizer
scaler = Normalizer(norm='l2')  // 'l1', 'l2', 'max'
df_scaled = scaler.fit_transform(df)

Label Encoding

• Maps categories to integers (0, 1, 2, ...)
• Use for ordinal data (order matters)
• May imply ordinal relationship

from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
df['col_encoded'] = le.fit_transform(df['col'])

One‑Hot Encoding

• Creates binary columns for each category
• Use for nominal data (no order)
• Avoids ordinal assumption
• Can create many columns (curse of dimensionality)

from sklearn.preprocessing import OneHotEncoder
encoder = OneHotEncoder(sparse_output=False)
encoded = encoder.fit_transform(df[['col']])

// With pandas
df_encoded = pd.get_dummies(df, columns=['col1', 'col2'])

Ordinal Encoding

• Maps categories to ordered integers
• Define mapping explicitly
• Use when order is known

from sklearn.preprocessing import OrdinalEncoder
encoder = OrdinalEncoder(categories=[['low', 'medium', 'high']])
df_encoded = encoder.fit_transform(df[['col']])

// Custom mapping
mapping = {'low': 0, 'medium': 1, 'high': 2}
df['col_encoded'] = df['col'].map(mapping)

Target Encoding

• Replace categories with target mean
• Use for high cardinality
• Risk of overfitting (use smoothing)

// Mean encoding
df['col_target'] = df.groupby('col')['target'].transform('mean')

// With smoothing
global_mean = df['target'].mean()
k = 5
df['col_encoded'] = (df.groupby('col')['target'].transform('sum') + global_mean * k) /
                    (df.groupby('col')['target'].transform('count') + k)

Frequency Encoding

• Replace with category frequency
• Use for high cardinality
• Simple and effective

df['col_freq'] = df['col'].map(df['col'].value_counts() / len(df))

Binary Encoding

• Convert to binary, then one‑hot
• Less columns than one‑hot
• Use when many categories

!pip install category_encoders
import category_encoders as ce
encoder = ce.BinaryEncoder()
df_encoded = encoder.fit_transform(df['col'])

Polynomial Features

• Adds polynomial combinations
• Captures non‑linear relationships
• Risk of overfitting

from sklearn.preprocessing import PolynomialFeatures
poly = PolynomialFeatures(degree=2, include_bias=False)
df_poly = poly.fit_transform(df)

Interaction Terms

• Product of two features
• Captures interactions
• Use with caution (increases dimensionality)

df['interact'] = df['col1'] * df['col2']
df['interact'] = df['col1'] / df['col2']

Binning

• Convert continuous to categorical
• Use equal‑width or equal‑frequency
• Can lose information

df['binned'] = pd.cut(df['col'], bins=5)
df['binned'] = pd.qcut(df['col'], q=5, labels=False)

Log / Transform

• Log transform for skewed data
• Square root, Box‑Cox, Yeo‑Johnson

df['log'] = np.log1p(df['col'])  // log(1+x)
df['sqrt'] = np.sqrt(df['col'])
df['boxcox'], _ = stats.boxcox(df['col'] + 1)

Date/Time Features

• Extract year, month, day, hour, etc.
• Day of week, quarter, week number
• Cyclical encoding (sin/cos)

df['year'] = df['date'].dt.year
df['month'] = df['date'].dt.month
df['day'] = df['date'].dt.day
df['weekday'] = df['date'].dt.dayofweek
df['hour'] = df['date'].dt.hour

// Cyclical
df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12)
df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)

Text Features

• Length, word count
• TF‑IDF, Bag‑of‑Words
• Sentiment, entities, keywords

df['len'] = df['text'].str.len()
df['word_count'] = df['text'].str.split().str.len()

// TF‑IDF
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(max_features=100)
X_tfidf = tfidf.fit_transform(df['text'])