This looks backwards possibly, in this file

    if tf_utils.is_ragged_tensor(boxes):
        bounding_boxes["boxes"] = bounding_boxes["boxes"].to_tensor(
            default_value=boxes_default_value,
            shape=_classes_shape(
                is_batched, bounding_boxes["boxes"].shape, max_boxes
            ),
        )
        bounding_boxes["labels"] = bounding_boxes["labels"].to_tensor(
            default_value=labels_default_value,
            shape=_box_shape(
                is_batched, bounding_boxes["labels"].shape, max_boxes
            ),
        )
        return bounding_boxes

Boxes are getting the classes_shape and labels are getting the box_shape?