# Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch import torch.nn as nn from mmcv.cnn.bricks import Swish, build_norm_layer from mmengine.model import bias_init_with_prob from torch import Tensor from mmdet.models.dense_heads.anchor_head import AnchorHead from mmdet.models.utils import images_to_levels, multi_apply from mmdet.registry import MODELS from mmdet.structures.bbox import cat_boxes, get_box_tensor from mmdet.utils import (InstanceList, OptConfigType, OptInstanceList, OptMultiConfig, reduce_mean) from .utils import DepthWiseConvBlock @MODELS.register_module() class EfficientDetSepBNHead(AnchorHead): """EfficientDetHead with separate BN. num_classes (int): Number of categories num_ins (int): Number of the input feature map. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels. stacked_convs (int): Number of repetitions of conv norm_cfg (dict): Config dict for normalization layer. anchor_generator (dict): Config dict for anchor generator bbox_coder (dict): Config of bounding box coder. loss_cls (dict): Config of classification loss. loss_bbox (dict): Config of localization loss. train_cfg (dict): Training config of anchor head. test_cfg (dict): Testing config of anchor head. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, num_classes: int, num_ins: int, in_channels: int, feat_channels: int, stacked_convs: int = 3, norm_cfg: OptConfigType = dict( type='BN', momentum=1e-2, eps=1e-3), init_cfg: OptMultiConfig = None, **kwargs) -> None: self.num_ins = num_ins self.stacked_convs = stacked_convs self.norm_cfg = norm_cfg super().__init__( num_classes=num_classes, in_channels=in_channels, feat_channels=feat_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.reg_conv_list = nn.ModuleList() self.cls_conv_list = nn.ModuleList() for i in range(self.stacked_convs): channels = self.in_channels if i == 0 else self.feat_channels self.reg_conv_list.append( DepthWiseConvBlock( channels, self.feat_channels, apply_norm=False)) self.cls_conv_list.append( DepthWiseConvBlock( channels, self.feat_channels, apply_norm=False)) self.reg_bn_list = nn.ModuleList([ nn.ModuleList([ build_norm_layer( self.norm_cfg, num_features=self.feat_channels)[1] for j in range(self.num_ins) ]) for i in range(self.stacked_convs) ]) self.cls_bn_list = nn.ModuleList([ nn.ModuleList([ build_norm_layer( self.norm_cfg, num_features=self.feat_channels)[1] for j in range(self.num_ins) ]) for i in range(self.stacked_convs) ]) self.cls_header = DepthWiseConvBlock( self.in_channels, self.num_base_priors * self.cls_out_channels, apply_norm=False) self.reg_header = DepthWiseConvBlock( self.in_channels, self.num_base_priors * 4, apply_norm=False) self.swish = Swish() def init_weights(self) -> None: """Initialize weights of the head.""" for m in self.reg_conv_list: nn.init.constant_(m.pointwise_conv.bias, 0.0) for m in self.cls_conv_list: nn.init.constant_(m.pointwise_conv.bias, 0.0) bias_cls = bias_init_with_prob(0.01) nn.init.constant_(self.cls_header.pointwise_conv.bias, bias_cls) nn.init.constant_(self.reg_header.pointwise_conv.bias, 0.0) def forward_single_bbox(self, feat: Tensor, level_id: int, i: int) -> Tensor: conv_op = self.reg_conv_list[i] bn = self.reg_bn_list[i][level_id] feat = conv_op(feat) feat = bn(feat) feat = self.swish(feat) return feat def forward_single_cls(self, feat: Tensor, level_id: int, i: int) -> Tensor: conv_op = self.cls_conv_list[i] bn = self.cls_bn_list[i][level_id] feat = conv_op(feat) feat = bn(feat) feat = self.swish(feat) return feat def forward(self, feats: Tuple[Tensor]) -> tuple: cls_scores = [] bbox_preds = [] for level_id in range(self.num_ins): feat = feats[level_id] for i in range(self.stacked_convs): feat = self.forward_single_bbox(feat, level_id, i) bbox_pred = self.reg_header(feat) bbox_preds.append(bbox_pred) for level_id in range(self.num_ins): feat = feats[level_id] for i in range(self.stacked_convs): feat = self.forward_single_cls(feat, level_id, i) cls_score = self.cls_header(feat) cls_scores.append(cls_score) return cls_scores, bbox_preds def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor concat_anchor_list = [] for i in range(len(anchor_list)): concat_anchor_list.append(cat_boxes(anchor_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() avg_factor = max(avg_factor, 1.0) losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, all_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) def loss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, anchors: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, avg_factor: int) -> tuple: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W). bbox_pred (Tensor): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (N, num_total_anchors, 4). bbox_weights (Tensor): BBox regression loss weights of each anchor with shape (N, num_total_anchors, 4). avg_factor (int): Average factor that is used to average the loss. Returns: tuple: loss components. """ # classification loss labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor) # regression loss target_dim = bbox_targets.size(-1) bbox_targets = bbox_targets.reshape(-1, target_dim) bbox_weights = bbox_weights.reshape(-1, target_dim) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, self.bbox_coder.encode_size) if self.reg_decoded_bbox: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, it # decodes the already encoded coordinates to absolute format. anchors = anchors.reshape(-1, anchors.size(-1)) bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) bbox_pred = get_box_tensor(bbox_pred) loss_bbox = self.loss_bbox( bbox_pred, bbox_targets, bbox_weights, avg_factor=avg_factor * 4) return loss_cls, loss_bbox