Data Science and Machine Learning Lab - Lab #6: PyTorch Introduction¶
In [1]:
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
1. Warm-up: PyTorch Tensors¶
In [2]:
# === 1.1 Creating Tensors ===
# Create a tensor
a = torch.tensor([1.0, 2.0, 3.0])
# Perform basic tensor operations
b = torch.tensor([4.0, 5.0, 6.0])
c = a + b # Element-wise addition
print(f"Tensor c: {c}")
Tensor c: tensor([5., 7., 9.])
In [3]:
# === 1.2 Inspecting Tensor Attributes ===
print(f"Shape of tensor c: {c.shape}")
print(f"Number of dimensions of tensor c: {c.ndim}")
print(f"Data type of tensor c: {c.dtype}")
Shape of tensor c: torch.Size([3]) Number of dimensions of tensor c: 1 Data type of tensor c: torch.float32
In [4]:
# === 1.3 Performance comparison of data types ===
from timeit import timeit
mat_size = 1000
M1_64 = torch.randn(mat_size, mat_size, dtype=torch.float64)
M2_64 = torch.randn(mat_size, mat_size, dtype=torch.float64)
M1_32 = torch.randn(mat_size, mat_size)
M2_32 = torch.randn(mat_size, mat_size)
t64 = timeit(lambda: M1_64 @ M2_64, number=100)
t32 = timeit(lambda: M1_32 @ M2_32, number=100)
print(f"Time for matrix multiplication (float64): {t64:.4f}s")
print(f"Time for matrix multiplication (float32): {t32:.4f}s")
print(f"Speedup: {t64 / t32:.2f}x")
Time for matrix multiplication (float64): 0.2299s Time for matrix multiplication (float32): 0.1252s Speedup: 1.84x
2. Datasets and Dataloaders¶
In [5]:
# == 2.1 Dataset generation ==
n_pts = 2048
X = torch.randn(n_pts, 1)
y = 5 * X + 3 + 0.1 * torch.randn(n_pts, 1)
In [6]:
# == 2.2 Creating a Dataset object ==
from torch.utils.data import TensorDataset
dataset = TensorDataset(X, y)
x, y = dataset[0]
print(f"First sample - x: {x}, y: {y}")
print(f"Total number of samples in dataset: {len(dataset)}")
First sample - x: tensor([0.5658]), y: tensor([5.9369]) Total number of samples in dataset: 2048
In [7]:
# == 2.3 Building a DataLoader and inspect batches ==
trainloader = DataLoader(dataset, batch_size=256, shuffle=True)
for i, (inputs, targets) in enumerate(trainloader):
print(f"Batch {i} - inputs: {inputs.reshape(-1)[:3]}, targets: {targets.reshape(-1)[:3]}")
print(f"Batch {i} - inputs shape: {inputs.shape}, targets shape: {targets.shape}")
if i == 2:
break
Batch 0 - inputs: tensor([ 1.0953, 0.0485, -1.1430]), targets: tensor([ 8.5295, 3.2383, -2.6865]) Batch 0 - inputs shape: torch.Size([256, 1]), targets shape: torch.Size([256, 1]) Batch 1 - inputs: tensor([-1.8259, 0.8842, 0.4873]), targets: tensor([-6.2385, 7.4543, 5.5134]) Batch 1 - inputs shape: torch.Size([256, 1]), targets shape: torch.Size([256, 1]) Batch 2 - inputs: tensor([0.2223, 2.1809, 1.2415]), targets: tensor([ 4.1065, 13.7996, 9.2403]) Batch 2 - inputs shape: torch.Size([256, 1]), targets shape: torch.Size([256, 1])
In [8]:
trainloader = DataLoader(dataset, batch_size=128, shuffle=True)
for i, (inputs, targets) in enumerate(trainloader):
print(f"Batch {i} - inputs: {inputs.reshape(-1)[:3]}, targets: {targets.reshape(-1)[:3]}")
print(f"Batch {i} - inputs shape: {inputs.shape}, targets shape: {targets.shape}")
if i == 2:
break
Batch 0 - inputs: tensor([0.9843, 0.3815, 0.1245]), targets: tensor([7.9351, 5.0816, 3.4754]) Batch 0 - inputs shape: torch.Size([128, 1]), targets shape: torch.Size([128, 1]) Batch 1 - inputs: tensor([ 0.4099, -0.4162, 0.1998]), targets: tensor([5.2598, 0.6866, 3.9683]) Batch 1 - inputs shape: torch.Size([128, 1]), targets shape: torch.Size([128, 1]) Batch 2 - inputs: tensor([-1.4482, 1.5792, 1.1330]), targets: tensor([-4.2729, 10.8469, 8.8186]) Batch 2 - inputs shape: torch.Size([128, 1]), targets shape: torch.Size([128, 1])
3. Building and understanding a simple linear model¶
In [9]:
# == 3.1 Model definition: SimpleLinearModel ==
from torch import nn
class SimpleLinearModel(nn.Module):
def __init__(self, input_size, output_size):
super(SimpleLinearModel, self).__init__()
self.linear = nn.Linear(input_size, output_size)
def forward(self, x):
return self.linear(x)
In [10]:
# Initialize and analyze your model
model = SimpleLinearModel(input_size=1, output_size=1)
y = model(dataset[0][0])
y
Out[10]:
tensor([-0.4439], grad_fn=<ViewBackward0>)
In [11]:
model_weights = model.linear.weight
model_bias = model.linear.bias
print("Weight", model_weights)
print("Bias", model_bias)
Weight Parameter containing: tensor([[-0.4860]], requires_grad=True) Bias Parameter containing: tensor([-0.1689], requires_grad=True)
In [12]:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")
model = model.to(device)
The device is set to: cuda
In [13]:
# == 3.2 Criterion and Optimizer ==
criterion = nn.MSELoss()
optimizer = optim.SGD(model.parameters(), lr=0.01)
In [14]:
# == 3.3 Training Loop ==
num_epochs = 50
losses = []
weights = []
biases = []
model.train()
for epoch in range(num_epochs):
running_loss = 0.0
for inputs, labels in trainloader:
inputs, labels = inputs.to(device), labels.to(device)
# Zero the parameter gradients
optimizer.zero_grad()
# Forward pass
outputs = model(inputs)
loss = criterion(outputs, labels)
# Backward pass and optimization
loss.backward()
optimizer.step()
# Save the loss, weight, and bias
losses.append(loss.item())
weights.append(model.linear.weight.item())
biases.append(model.linear.bias.item())
running_loss += loss.item()
print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {running_loss/len(trainloader):.4f}")
Epoch [1/50], Loss: 30.2832 Epoch [2/50], Loss: 15.8115 Epoch [3/50], Loss: 8.2607 Epoch [4/50], Loss: 4.3172 Epoch [5/50], Loss: 2.2591 Epoch [6/50], Loss: 1.1845 Epoch [7/50], Loss: 0.6230 Epoch [8/50], Loss: 0.3300 Epoch [9/50], Loss: 0.1770 Epoch [10/50], Loss: 0.0971 Epoch [11/50], Loss: 0.0554 Epoch [12/50], Loss: 0.0337 Epoch [13/50], Loss: 0.0223 Epoch [14/50], Loss: 0.0164 Epoch [15/50], Loss: 0.0133 Epoch [16/50], Loss: 0.0117 Epoch [17/50], Loss: 0.0109 Epoch [18/50], Loss: 0.0104 Epoch [19/50], Loss: 0.0102 Epoch [20/50], Loss: 0.0101 Epoch [21/50], Loss: 0.0100 Epoch [22/50], Loss: 0.0100 Epoch [23/50], Loss: 0.0099 Epoch [24/50], Loss: 0.0099 Epoch [25/50], Loss: 0.0099 Epoch [26/50], Loss: 0.0099 Epoch [27/50], Loss: 0.0099 Epoch [28/50], Loss: 0.0099 Epoch [29/50], Loss: 0.0099 Epoch [30/50], Loss: 0.0099 Epoch [31/50], Loss: 0.0099 Epoch [32/50], Loss: 0.0099 Epoch [33/50], Loss: 0.0099 Epoch [34/50], Loss: 0.0099 Epoch [35/50], Loss: 0.0099 Epoch [36/50], Loss: 0.0099 Epoch [37/50], Loss: 0.0099 Epoch [38/50], Loss: 0.0099 Epoch [39/50], Loss: 0.0099 Epoch [40/50], Loss: 0.0099 Epoch [41/50], Loss: 0.0099 Epoch [42/50], Loss: 0.0099 Epoch [43/50], Loss: 0.0099 Epoch [44/50], Loss: 0.0099 Epoch [45/50], Loss: 0.0099 Epoch [46/50], Loss: 0.0099 Epoch [47/50], Loss: 0.0099 Epoch [48/50], Loss: 0.0099 Epoch [49/50], Loss: 0.0099 Epoch [50/50], Loss: 0.0099
In [15]:
fig, ax = plt.subplots(1, 3, figsize=(15, 3))
ax[0].plot(losses)
ax[0].set_title("Loss")
ax[0].set_xlabel("Steps")
ax[0].set_ylabel("Loss")
ax[0].grid()
ax[1].plot(weights)
ax[1].set_title("Weight")
ax[1].set_xlabel("Steps")
ax[1].set_ylabel("Weight")
ax[1].grid()
ax[2].plot(biases)
ax[2].set_title("Bias")
ax[2].set_xlabel("Steps")
ax[2].set_ylabel("Bias")
ax[2].grid()
In [16]:
# Inspecting the learned weights and biases
weights = model.linear.weight.data
biases = model.linear.bias.data
print(f'Learned weights: {weights}')
print(f'Learned biases: {biases}')
Learned weights: tensor([[4.9986]], device='cuda:0') Learned biases: tensor([2.9998], device='cuda:0')
4. MNIST¶
In [17]:
dataset_dir = "~/data"
In [18]:
from torchvision import datasets,transforms
train_dataset=datasets.MNIST(root="data",train=True,download=True)
test_dataset=datasets.MNIST(root="data",train=False,download=True)
print(f"Number of training samples: {len(train_dataset)}")
print(f"Number of test samples: {len(test_dataset)}")
Number of training samples: 60000 Number of test samples: 10000
In [19]:
image, label = train_dataset[0]
print(f"Image shape: {image.size}, Label: {label}")
image
Image shape: (28, 28), Label: 5
Out[19]:
In [20]:
fig, ax = plt.subplots(3, 3, figsize=(8, 8))
plt.subplots_adjust(hspace=0.25, wspace=0.1)
for i in range(3):
for j in range(3):
idx = i * 3 + j
img, label = train_dataset[idx]
ax[i, j].imshow(img, cmap='gray')
ax[i, j].set_title(label)
ax[i, j].axis('off')
In [21]:
# == 4.1 Preprocessing and Transforms ==
# Conversion to tensor
transform = transforms.ToTensor()
train_dataset.transform = transform
test_dataset.transform = transform
image, label = train_dataset[0]
print(f"Image shape after transform: {image.shape}, Label: {label}")
print("max and min pixel values:", image.max(), image.min())
Image shape after transform: torch.Size([1, 28, 28]), Label: 5 max and min pixel values: tensor(1.) tensor(0.)
In [22]:
# Evaluate dataset mean and std
mean = train_dataset.data.float().mean() / 255.0
std = train_dataset.data.float().std() / 255.0
print(f"Dataset mean: {mean}, Dataset std: {std}")
Dataset mean: 0.13066047430038452, Dataset std: 0.30810779333114624
In [23]:
# Normalization
normalize_transform = transforms.Normalize(mean, std)
train_dataset.transform = transforms.Compose([
transforms.ToTensor(),
normalize_transform
])
test_dataset.transform = transforms.Compose([
transforms.ToTensor(),
normalize_transform
])
image, label = train_dataset[0]
print(f"Image shape after normalization: {image.shape}, Label: {label}")
print("max and min pixel values after normalization:", image.max(), image.min())
Image shape after normalization: torch.Size([1, 28, 28]), Label: 5 max and min pixel values after normalization: tensor(2.8215) tensor(-0.4241)
In [24]:
# Data augmentation
transform = transforms.Compose([
transforms.RandomRotation(45),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
train_dataset.transform = transform
test_dataset.transform = transform
In [25]:
fig, ax = plt.subplots(3, 3, figsize=(8, 8))
plt.subplots_adjust(hspace=0.25, wspace=0.1)
for i in range(3):
for j in range(3):
idx = i * 3 + j
img, label = train_dataset[idx]
ax[i, j].imshow(img.squeeze(), cmap='gray')
ax[i, j].set_title(label)
ax[i, j].axis('off')
In [26]:
# (*) Experimenting with transformations
train_dataset.transform = transforms.ToTensor()
transformations = {
'Original': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'RandomAffine (Light)': transforms.Compose([
transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'RandomAffine (Heavy)': transforms.Compose([
transforms.RandomAffine(degrees=45, translate=(0.2, 0.2), scale=(0.7, 1.3)),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'RandomCrop': transforms.Compose([
transforms.Resize(32),
transforms.RandomCrop(28, padding=4),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'ColorJitter': transforms.Compose([
transforms.ColorJitter(brightness=0.3, contrast=0.3),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'Combined Light': transforms.Compose([
transforms.RandomRotation(10),
transforms.RandomAffine(degrees=0, translate=(0.05, 0.05)),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'Combined Heavy': transforms.Compose([
transforms.RandomRotation(30),
transforms.RandomAffine(degrees=0, translate=(0.15, 0.15), scale=(0.8, 1.2)),
transforms.ColorJitter(brightness=0.2, contrast=0.2),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
}
def visualize_transformations(dataset, transformations_dict, sample_idx=0):
original_img, label = dataset[sample_idx]
n_transforms = len(transformations_dict)
cols = 4
rows = (n_transforms + cols - 1) // cols
fig, axes = plt.subplots(rows, cols, figsize=(15, 4*rows))
axes = axes.flatten() if rows > 1 else [axes] if cols == 1 else axes
axes[0].imshow(original_img.squeeze().numpy(), cmap='gray')
axes[0].set_title(f'Original\nLabel: {label}')
axes[0].axis('off')
for idx, (name, transform) in enumerate(transformations_dict.items()):
dataset.transform = transform
transformed_img, _ = dataset[sample_idx]
if transformed_img.dim() == 3:
img_np = transformed_img.squeeze().numpy()
else:
img_np = transformed_img.numpy()
axes[idx+1].imshow(img_np, cmap='gray')
axes[idx+1].set_title(f'{name}\nLabel: {label}')
axes[idx+1].axis('off')
plt.tight_layout()
plt.show()
print("Visualizing transformations on digit '5':")
for i, (img, label) in enumerate(train_dataset):
if label == 5:
visualize_transformations(train_dataset, transformations, i)
break
print("\nVisualizing transformations on digit '3':")
for i, (img, label) in enumerate(train_dataset):
if label == 3:
visualize_transformations(train_dataset, transformations, i)
break
Visualizing transformations on digit '5':
Visualizing transformations on digit '3':
5. A more complex neural network¶
In [27]:
# == 5.1 Model definition: SimpleNN ==
# Define a more complex neural network model
class SimpleNN(nn.Module):
def __init__(self):
super(SimpleNN, self).__init__()
# number of input features: 28*28 (image size)
self.fc1 = nn.Linear(28 * 28, 512) # Input layer to first hidden layer
self.fc2 = nn.Linear(512, 256) # First hidden layer to second hidden layer
self.fc3 = nn.Linear(256, 10) # Second hidden layer to output layer
def forward(self, x):
# Flatten the input
x = x.view(-1, 28 * 28)
# Apply layers with ReLU activation
x = torch.relu(self.fc1(x))
x = torch.relu(self.fc2(x))
x = self.fc3(x)
return x
model = SimpleNN()
model = model.to(device)
In [28]:
# == 5.2 Training setup ==
# Prepare the data
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
train_dataset.transform = transform
test_dataset.transform = transform
batch_size = 1024
trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
testloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
In [29]:
# Define the loss function and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
In [30]:
# Create a validation function
def val_model(model, testloader):
model.eval()
correct = 0
total = 0
with torch.no_grad():
for inputs, labels in testloader:
inputs, labels = inputs.to(device), labels.to(device)
outputs = model(inputs)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print(f"Accuracy of the network on the test images: {100 * correct / total:.2f}%")
In [31]:
val_model(model, testloader)
Accuracy of the network on the test images: 7.37%
In [32]:
# == 5.3 Training loop ==
num_epochs = 5
losses = []
model.train()
for epoch in range(num_epochs):
running_loss = 0.0
for inputs, labels in trainloader:
inputs, labels = inputs.to(device), labels.to(device)
# Zero the parameter gradients
optimizer.zero_grad()
# Forward pass
outputs = model(inputs)
loss = criterion(outputs, labels)
# Backward pass and optimization
loss.backward()
optimizer.step()
losses.append(loss.item())
running_loss += loss.item()
print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {running_loss/len(trainloader):.4f}")
Epoch [1/5], Loss: 1.3308 Epoch [2/5], Loss: 0.3773 Epoch [3/5], Loss: 0.3017 Epoch [4/5], Loss: 0.2631 Epoch [5/5], Loss: 0.2329
In [33]:
val_model(model, testloader)
Accuracy of the network on the test images: 93.95%