A convolutional network

Examples

The architecture

import numpy as np
import torch
import torch.nn as nn
import matplotlib.pyplot as plt

torch.manual_seed(9)

cnn = nn.Sequential(
    nn.Conv2d(1, 16, kernel_size=3, padding=1), nn.ReLU(),
    nn.MaxPool2d(2),
    nn.Conv2d(16, 32, kernel_size=3, padding=1), nn.ReLU(),
    nn.MaxPool2d(2),
    nn.Flatten(),
    nn.Linear(32 * 7 * 7, 64), nn.ReLU(),
    nn.Linear(64, 10),
)

print("parameters:", sum(p.numel() for p in cnn.parameters()))

1

One input channel because the images are grey. 16 filters of size 3 by 3, with padding 1 so the size stays 28 by 28.

2

Halves the width and the height, so 28 becomes 14.

3

Turns the volume \(32 \times 7 \times 7\) into a vector of 1568 numbers.

parameters: 105866

Let us check the shapes step by step.

x = torch.zeros(1, 1, 28, 28)
for layer in cnn:
    x = layer(x)
    print(f"{layer.__class__.__name__:12s} {tuple(x.shape)}")

Conv2d       (1, 16, 28, 28)
ReLU         (1, 16, 28, 28)
MaxPool2d    (1, 16, 14, 14)
Conv2d       (1, 32, 14, 14)
ReLU         (1, 32, 14, 14)
MaxPool2d    (1, 32, 7, 7)
Flatten      (1, 1568)
Linear       (1, 64)
ReLU         (1, 64)
Linear       (1, 10)

Comparing the parameter count

dense = nn.Sequential(
    nn.Flatten(), nn.Linear(784, 128), nn.ReLU(), nn.Linear(128, 10))

print("convolutional:", sum(p.numel() for p in cnn.parameters()))
print("fully connected:", sum(p.numel() for p in dense.parameters()))

convolutional: 105866
fully connected: 101770

The convolutional network has more parameters here, because the flattened layer is large. On bigger images the picture reverses sharply, because the convolutional part does not grow with the image size while the dense part does.

Training on MNIST

from sklearn.datasets import fetch_openml

mnist = fetch_openml("mnist_784", version=1, as_frame=True, parser="auto")
X = mnist.data.values.astype(np.float32).reshape(-1, 1, 28, 28) / 255
y = mnist.target.values.astype(np.int64)

Xtr, ytr = torch.tensor(X[:60000]), torch.tensor(y[:60000])
Xte, yte = torch.tensor(X[60000:]), torch.tensor(y[60000:])

loss_fn = nn.CrossEntropyLoss()
opt = torch.optim.Adam(cnn.parameters(), lr=1e-3)
loader = torch.utils.data.DataLoader(
    torch.utils.data.TensorDataset(Xtr, ytr), batch_size=128, shuffle=True)

for epoch in range(5):
    cnn.train()
    for xb, yb in loader:
        opt.zero_grad()
        loss_fn(cnn(xb), yb).backward()
        opt.step()

cnn.eval()
with torch.no_grad():
    pred = cnn(Xte).argmax(dim=1)
print("test accuracy:", float((pred == yte).float().mean()))

A network of this size reaches about 99 percent. The fully connected network from week 7 reached about 98 percent, and multinomial logistic regression about 92 percent.

A smaller run

The download takes a while, so here is the same code on the small digits data set, where the images are 8 by 8.

from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split

digits = load_digits()
X = digits.images.astype(np.float32).reshape(-1, 1, 8, 8) / 16.0
y = digits.target.astype(np.int64)

Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=0)
Xtr, Xte = torch.tensor(Xtr), torch.tensor(Xte)
ytr, yte = torch.tensor(ytr), torch.tensor(yte)

small = nn.Sequential(
    nn.Conv2d(1, 8, kernel_size=3, padding=1), nn.ReLU(),
    nn.MaxPool2d(2),
    nn.Flatten(),
    nn.Linear(8 * 4 * 4, 32), nn.ReLU(),
    nn.Linear(32, 10),
)

loss_fn = nn.CrossEntropyLoss()
opt = torch.optim.Adam(small.parameters(), lr=5e-3)
loader = torch.utils.data.DataLoader(
    torch.utils.data.TensorDataset(Xtr, ytr), batch_size=64, shuffle=True)

for epoch in range(25):
    small.train()
    for xb, yb in loader:
        opt.zero_grad()
        loss_fn(small(xb), yb).backward()
        opt.step()

small.eval()
with torch.no_grad():
    pred = small(Xte).argmax(dim=1)
print("test accuracy:", round(float((pred == yte).float().mean()), 4))

test accuracy: 0.9759

Looking at the filters

w = small[0].weight.detach().numpy()

fig, axes = plt.subplots(1, 8, figsize=(6.6, 1.0))
for ax, f in zip(axes, w):
    ax.imshow(f[0], cmap="gray")
    ax.set(xticks=[], yticks=[])
plt.show()

Figure 45.1: The eight learned filters of the first layer. Each one responds to a different orientation or contrast.

Nobody told the network what an edge is. These filters came out of the gradient.