I followed this tutorial for object detection: https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html
and their GitHub repository that contains the following train_one_epoch and evaluate functions:
https://github.com/pytorch/vision/blob/main/references/detection/engine.py
However, I want to calculate losses during validation. I implemented this for the evaluation loss, where essentially to obtain losses, model.train() needs to be on:
@torch.no_grad()
def evaluate_loss(model, data_loader, device):
val_loss = 0
model.train()
for images, targets in data_loader:
images = list(image.to(device) for image in images)
targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
loss_dict = model(images, targets)
losses = sum(loss for loss in loss_dict.values())
# reduce losses over all GPUs for logging purposes
loss_dict_reduced = utils.reduce_dict(loss_dict)
losses_reduced = sum(loss for loss in loss_dict_reduced.values())
val_loss += losses_reduced
validation_loss = val_loss/ len(data_loader)
return validation_loss
I then place it after the learning rate scheduler step in my for loop:
for epoch in range(args.num_epochs):
# train for one epoch, printing every 10 iterations
train_one_epoch(model, optimizer, train_data_loader, device, epoch, print_freq=10)
# update the learning rate
lr_scheduler.step()
validation_loss = evaluate_loss(model, valid_data_loader, device=device)
# evaluate on the test dataset
evaluate(model, valid_data_loader, device=device)
Does this look correct or can it interfere with training or produce inaccurate validation losses?
If ok, by using this, is there is a simple way in applying early stopping for validation loss?
I'm considering just adding something like this after the evaluate model function shown above:
torch.save({
'epoch': epoch,
'model_state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'validation loss': valid_loss,
}, PATH)
where I also aim to save the model at every epoch for checkpointing purposes. However I need to determine the validation "loss" for saving the "best" model.
So it turns out no stages of the pytorch fasterrcnn return losses when model.eval() is set. However, you can just manually use the forward code to generate the losses in evaluation mode:
from typing import Tuple, List, Dict, Optional
import torch
from torch import Tensor
from collections import OrderedDict
from torchvision.models.detection.roi_heads import fastrcnn_loss
from torchvision.models.detection.rpn import concat_box_prediction_layers
def eval_forward(model, images, targets):
# type: (List[Tensor], Optional[List[Dict[str, Tensor]]]) -> Tuple[Dict[str, Tensor], List[Dict[str, Tensor]]]
"""
Args:
images (list[Tensor]): images to be processed
targets (list[Dict[str, Tensor]]): ground-truth boxes present in the image (optional)
Returns:
result (list[BoxList] or dict[Tensor]): the output from the model.
It returns list[BoxList] contains additional fields
like `scores`, `labels` and `mask` (for Mask R-CNN models).
"""
model.eval()
original_image_sizes: List[Tuple[int, int]] = []
for img in images:
val = img.shape[-2:]
assert len(val) == 2
original_image_sizes.append((val[0], val[1]))
images, targets = model.transform(images, targets)
# Check for degenerate boxes
# TODO: Move this to a function
if targets is not None:
for target_idx, target in enumerate(targets):
boxes = target["boxes"]
degenerate_boxes = boxes[:, 2:] <= boxes[:, :2]
if degenerate_boxes.any():
# print the first degenerate box
bb_idx = torch.where(degenerate_boxes.any(dim=1))[0][0]
degen_bb: List[float] = boxes[bb_idx].tolist()
raise ValueError(
"All bounding boxes should have positive height and width."
f" Found invalid box {degen_bb} for target at index {target_idx}."
)
features = model.backbone(images.tensors)
if isinstance(features, torch.Tensor):
features = OrderedDict([("0", features)])
model.rpn.training=True
#model.roi_heads.training=True
#####proposals, proposal_losses = model.rpn(images, features, targets)
features_rpn = list(features.values())
objectness, pred_bbox_deltas = model.rpn.head(features_rpn)
anchors = model.rpn.anchor_generator(images, features_rpn)
num_images = len(anchors)
num_anchors_per_level_shape_tensors = [o[0].shape for o in objectness]
num_anchors_per_level = [s[0] * s[1] * s[2] for s in num_anchors_per_level_shape_tensors]
objectness, pred_bbox_deltas = concat_box_prediction_layers(objectness, pred_bbox_deltas)
# apply pred_bbox_deltas to anchors to obtain the decoded proposals
# note that we detach the deltas because Faster R-CNN do not backprop through
# the proposals
proposals = model.rpn.box_coder.decode(pred_bbox_deltas.detach(), anchors)
proposals = proposals.view(num_images, -1, 4)
proposals, scores = model.rpn.filter_proposals(proposals, objectness, images.image_sizes, num_anchors_per_level)
proposal_losses = {}
assert targets is not None
labels, matched_gt_boxes = model.rpn.assign_targets_to_anchors(anchors, targets)
regression_targets = model.rpn.box_coder.encode(matched_gt_boxes, anchors)
loss_objectness, loss_rpn_box_reg = model.rpn.compute_loss(
objectness, pred_bbox_deltas, labels, regression_targets
)
proposal_losses = {
"loss_objectness": loss_objectness,
"loss_rpn_box_reg": loss_rpn_box_reg,
}
#####detections, detector_losses = model.roi_heads(features, proposals, images.image_sizes, targets)
image_shapes = images.image_sizes
proposals, matched_idxs, labels, regression_targets = model.roi_heads.select_training_samples(proposals, targets)
box_features = model.roi_heads.box_roi_pool(features, proposals, image_shapes)
box_features = model.roi_heads.box_head(box_features)
class_logits, box_regression = model.roi_heads.box_predictor(box_features)
result: List[Dict[str, torch.Tensor]] = []
detector_losses = {}
loss_classifier, loss_box_reg = fastrcnn_loss(class_logits, box_regression, labels, regression_targets)
detector_losses = {"loss_classifier": loss_classifier, "loss_box_reg": loss_box_reg}
boxes, scores, labels = model.roi_heads.postprocess_detections(class_logits, box_regression, proposals, image_shapes)
num_images = len(boxes)
for i in range(num_images):
result.append(
{
"boxes": boxes[i],
"labels": labels[i],
"scores": scores[i],
}
)
detections = result
detections = model.transform.postprocess(detections, images.image_sizes, original_image_sizes) # type: ignore[operator]
model.rpn.training=False
model.roi_heads.training=False
losses = {}
losses.update(detector_losses)
losses.update(proposal_losses)
return losses, detections
Testing this code gives me:
import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
# load a model pre-trained on COCO
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)
# replace the classifier with a new one, that has
# num_classes which is user-defined
num_classes = 2 # 1 class (person) + background
# get number of input features for the classifier
in_features = model.roi_heads.box_predictor.cls_score.in_features
# replace the pre-trained head with a new one
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
losses, detections = eval_forward(model,torch.randn([1,3,300,300]),[{'boxes':torch.tensor([[100,100,200,200]]),'labels':torch.tensor([0])}])
{'loss_classifier': tensor(0.6594, grad_fn=<NllLossBackward0>),
'loss_box_reg': tensor(0., grad_fn=<DivBackward0>),
'loss_objectness': tensor(0.5108, grad_fn=<BinaryCrossEntropyWithLogitsBackward0>),
'loss_rpn_box_reg': tensor(0.0160, grad_fn=<DivBackward0>)}