DEEP LEARNING
CNN & RNN CHEATSHEET.

Convolutional Neural Networks (CNN)

Convolution Formulas

Output Size = (Input Size - Kernel Size + 2 * Padding) / Stride + 1

// Example: 32x32 input, 3x3 kernel, stride 1, padding 0
// Output = (32 - 3 + 0) / 1 + 1 = 30

// With padding (same size)
Padding = (Kernel Size - 1) / 2
// 3x3 kernel → padding = 1 → same size

Common CNN Architectures

CNN Building Blocks in Code

// PyTorch
import torch.nn as nn

class SimpleCNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(64 * 8 * 8, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.pool(nn.functional.relu(self.conv1(x)))
        x = self.pool(nn.functional.relu(self.conv2(x)))
        x = x.view(-1, 64 * 8 * 8)  // flatten
        x = nn.functional.relu(self.fc1(x))
        x = self.fc2(x)
        return x

// Keras
from tensorflow.keras import layers, models

model = models.Sequential([
    layers.Conv2D(32, (3,3), activation='relu', padding='same', input_shape=(32,32,3)),
    layers.MaxPooling2D((2,2)),
    layers.Conv2D(64, (3,3), activation='relu', padding='same'),
    layers.MaxPooling2D((2,2)),
    layers.Flatten(),
    layers.Dense(128, activation='relu'),
    layers.Dense(10, activation='softmax')
])

Residual Block (ResNet)

// PyTorch
class ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)
        self.shortcut = nn.Sequential()
        if stride != 1 or in_channels != out_channels:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(out_channels)
            )

    def forward(self, x):
        out = nn.functional.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = nn.functional.relu(out)
        return out

Recurrent Neural Networks (RNN)

RNN Cell Equation

h_t = σ(W_h * h_{t-1} + W_x * x_t + b)
y_t = σ(W_y * h_t + b_y)

RNN Variants

LSTM Equations

f_t = σ(W_f · [h_{t-1}, x_t] + b_f)   // forget gate
i_t = σ(W_i · [h_{t-1}, x_t] + b_i)   // input gate
C̃_t = tanh(W_C · [h_{t-1}, x_t] + b_C) // candidate cell
C_t = f_t * C_{t-1} + i_t * C̃_t       // cell state
o_t = σ(W_o · [h_{t-1}, x_t] + b_o)   // output gate
h_t = o_t * tanh(C_t)                // hidden state

GRU Equations

z_t = σ(W_z · [h_{t-1}, x_t] + b_z)   // update gate
r_t = σ(W_r · [h_{t-1}, x_t] + b_r)   // reset gate
h̃_t = tanh(W_h · [r_t * h_{t-1}, x_t] + b_h)
h_t = (1 - z_t) * h_{t-1} + z_t * h̃_t

RNN in Code

// PyTorch
import torch.nn as nn

// Vanilla RNN
rnn = nn.RNN(input_size=10, hidden_size=20, num_layers=2, batch_first=True)

// LSTM
lstm = nn.LSTM(input_size=10, hidden_size=20, num_layers=2, batch_first=True, bidirectional=True)

// GRU
gru = nn.GRU(input_size=10, hidden_size=20, num_layers=2, batch_first=True)

// Custom LSTM Classifier
class LSTMClassifier(nn.Module):
    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers, num_classes):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers, batch_first=True)
        self.fc = nn.Linear(hidden_dim, num_classes)

    def forward(self, x):
        x = self.embedding(x)
        out, (hidden, cell) = self.lstm(x)
        out = out[:, -1, :]  // last hidden state
        out = self.fc(out)
        return out

// Keras
from tensorflow.keras import layers, models

model = models.Sequential([
    layers.Embedding(vocab_size, embed_dim),
    layers.LSTM(hidden_dim, return_sequences=True),
    layers.LSTM(hidden_dim),
    layers.Dense(num_classes, activation='softmax')
])

Text Input - Packing & Padding

// PyTorch packing
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

// Pack sequence (variable lengths)
packed = pack_padded_sequence(embedded, lengths, batch_first=True, enforce_sorted=False)
output, _ = lstm(packed)
output, _ = pad_packed_sequence(output, batch_first=True)  // pad back

Activation Functions

// PyTorch
import torch.nn.functional as F
x = F.relu(x)
x = F.sigmoid(x)
x = F.softmax(x, dim=1)

// Keras
layers.Dense(128, activation='relu')
layers.Dense(10, activation='softmax')

Loss Functions

Loss in Code

// PyTorch
criterion = nn.CrossEntropyLoss()
criterion = nn.BCEWithLogitsLoss()
criterion = nn.MSELoss()

// Keras
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.compile(loss='binary_crossentropy', optimizer='adam')

Optimizers

Optimizer in Code

// PyTorch
optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)
optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)

// Keras
model.compile(optimizer='adam', loss='categorical_crossentropy')
model.compile(optimizer='sgd', loss='categorical_crossentropy')

Regularization in Deep Learning

• Dropout – randomly drop neurons during training
• Batch Normalization – normalise layer inputs
• Weight Decay – L2 regularisation
• Data Augmentation – generate additional training samples
• Early Stopping – stop when validation stops improving
• Label Smoothing – reduce overconfidence

Regularization in Code

// PyTorch
// Dropout
self.dropout = nn.Dropout(0.5)

// Batch Normalization
self.bn1 = nn.BatchNorm2d(32)

// Weight Decay (in optimizer)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-4)

// Keras
layers.Dropout(0.5)
layers.BatchNormalization()
model.add(layers.Dropout(0.5))

// Weight Decay (in optimizers)
model.compile(optimizer=tf.keras.optimizers.Adam(weight_decay=1e-4), loss='categorical_crossentropy')

Transfer Learning

• Feature Extraction – freeze base model, train classifier
• Fine‑tuning – unfreeze some layers and train
• Common Pre‑trained Models – ResNet, VGG, EfficientNet (vision), BERT, GPT (text)
• Use torchvision.models or tensorflow.keras.applications

Transfer Learning in Code

// PyTorch (feature extraction)
import torchvision.models as models

resnet = models.resnet18(pretrained=True)
for param in resnet.parameters():
    param.requires_grad = False
resnet.fc = nn.Linear(512, num_classes)

// Keras (feature extraction)
from tensorflow.keras.applications import ResNet50
base_model = ResNet50(weights='imagenet', include_top=False)
base_model.trainable = False
model = models.Sequential([
    base_model,
    layers.GlobalAveragePooling2D(),
    layers.Dense(num_classes, activation='softmax')
])

Training Tips

• Learning Rate Scheduling – reduce lr over time
• Gradient Clipping – prevent exploding gradients (RNNs)
• Mixed Precision Training – faster, less memory
• Gradient Accumulation – simulate larger batch sizes
• Kaiming / Xavier Initialization – proper weight initialisation
• Monitoring – track train/val loss and metrics
• Data Augmentation – flip, rotate, crop, colour jitter
• Normalize Input – mean=0, std=1 for each channel

Learning Rate Schedulers

// PyTorch
from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR

scheduler = StepLR(optimizer, step_size=10, gamma=0.1)
scheduler = ReduceLROnPlateau(optimizer, mode='min', patience=5, factor=0.5)
scheduler = CosineAnnealingLR(optimizer, T_max=50)

// Keras
from tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler

callback = ReduceLROnPlateau(monitor='val_loss', patience=5, factor=0.5)

Common Data Augmentations