Notebooks

Analyse annotation quality

iPython Project Created 4 days ago Free
Ready to use notebook to perform: labeling consensus, training data verification and analysis of neural network predictions
Free Signup

Analyse annotation quality

Ready to use notebook to perform: labeling consensus, training data verification and analysis of neural network predictions.

Input:

  • Existing Project
  • List of classes to compare

Output:

  • New Project in Supervisely with visualized difference

Configuration

Edit the following settings for your own case

In [1]:
%matplotlib inline
In [2]:
import supervisely_lib as sly
import os
import collections
from tqdm import tqdm
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(color_codes=True)

ClassComparison = collections.namedtuple('ClassComparison', 'class1 class2 diff_suffix diff_color')
In [3]:
team_name = "jupyter_tutorials"
workspace_name = "cookbook"
project_name = "tutorial_project_inf"

dst_project_name = "tutorial_project_analysis"

classes_to_compare = [ClassComparison(class1="car", class2="car_dl", diff_suffix="car", diff_color=[255, 0, 0]),
                      ClassComparison(class1="person", class2="person_dl", diff_suffix="person", diff_color=[255, 51, 255])]

# Obtain server address and your api_token from environment variables
# Edit those values if you run this notebook on your own PC
address = os.environ['SERVER_ADDRESS']
token = os.environ['API_TOKEN']
In [5]:
# Initialize API object
api = sly.Api(address, token)

Verify input values

Test that context (team / workspace / project) exists

In [6]:
# get IDs of team, workspace and project by names

team = api.team.get_info_by_name(team_name)
if team is None:
    raise RuntimeError("Team {!r} not found".format(team_name))

workspace = api.workspace.get_info_by_name(team.id, workspace_name)
if workspace is None:
    raise RuntimeError("Workspace {!r} not found".format(workspace_name))
    
project = api.project.get_info_by_name(workspace.id, project_name)
if project is None:
    raise RuntimeError("Project {!r} not found".format(project_name))
    
print("Team: id={}, name={}".format(team.id, team.name))
print("Workspace: id={}, name={}".format(workspace.id, workspace.name))
print("Project: id={}, name={}".format(project.id, project.name))
Out [6]:
Team: id=30, name=jupyter_tutorials
Workspace: id=76, name=cookbook
Project: id=909, name=tutorial_project_inf

Get Source Project Meta

In [7]:
project = api.project.get_info_by_name(workspace.id, project_name)
meta_json = api.project.get_meta(project.id)
meta = sly.ProjectMeta.from_json(meta_json)
print("Source ProjectMeta: \n", meta)
Out [7]:
Source ProjectMeta: 
 ProjectMeta:
Object Classes
+-----------+-----------+----------------+
|    Name   |   Shape   |     Color      |
+-----------+-----------+----------------+
| person_dl |   Bitmap  |  [0, 128, 90]  |
|   dog_dl  |   Bitmap  |  [0, 81, 128]  |
|   car_dl  |   Bitmap  |  [9, 0, 255]   |
|    bike   | Rectangle | [246, 255, 0]  |
|    car    |  Polygon  | [190, 85, 206] |
|    dog    |  Polygon  |  [253, 0, 0]   |
|   person  |   Bitmap  |  [0, 255, 18]  |
+-----------+-----------+----------------+
Image Tags
+-------------+--------------+-----------------------+
|     Name    |  Value type  |    Possible values    |
+-------------+--------------+-----------------------+
| cars_number |  any_number  |          None         |
|     like    |     none     |          None         |
|   situated  | oneof_string | ['inside', 'outside'] |
+-------------+--------------+-----------------------+
Object Tags
+---------------+--------------+-----------------------+
|      Name     |  Value type  |    Possible values    |
+---------------+--------------+-----------------------+
|   car_color   |  any_string  |          None         |
| person_gender | oneof_string |   ['male', 'female']  |
|  vehicle_age  | oneof_string | ['modern', 'vintage'] |
+---------------+--------------+-----------------------+

Construct Destination ProjectMeta

In [8]:
# prepare new tags for metrics and object classes for difference visualization
new_meta_items = {}
for cc in classes_to_compare:
    new_meta_items[cc.class1] = [sly.TagMeta('iou_{}'.format(cc.diff_suffix), sly.TagValueType.ANY_NUMBER),
                                 sly.ObjClass("diff_{}".format(cc.diff_suffix), sly.Bitmap, cc.diff_color) ]
In [9]:
def process_meta(input_meta):
    output_meta = input_meta.clone()
    for items in new_meta_items.values():
        output_meta = output_meta.add_img_tag_meta(items[0])
        output_meta = output_meta.add_obj_class(items[1])    
    return output_meta
In [10]:
dst_meta = process_meta(meta)
print("Destination ProjectMeta:\n", dst_meta)
Out [10]:
Destination ProjectMeta:
 ProjectMeta:
Object Classes
+-------------+-----------+----------------+
|     Name    |   Shape   |     Color      |
+-------------+-----------+----------------+
|  person_dl  |   Bitmap  |  [0, 128, 90]  |
|    dog_dl   |   Bitmap  |  [0, 81, 128]  |
|    car_dl   |   Bitmap  |  [9, 0, 255]   |
|     bike    | Rectangle | [246, 255, 0]  |
|     car     |  Polygon  | [190, 85, 206] |
|     dog     |  Polygon  |  [253, 0, 0]   |
|    person   |   Bitmap  |  [0, 255, 18]  |
|   diff_car  |   Bitmap  |  [255, 0, 0]   |
| diff_person |   Bitmap  | [255, 51, 255] |
+-------------+-----------+----------------+
Image Tags
+-------------+--------------+-----------------------+
|     Name    |  Value type  |    Possible values    |
+-------------+--------------+-----------------------+
| cars_number |  any_number  |          None         |
|     like    |     none     |          None         |
|   situated  | oneof_string | ['inside', 'outside'] |
|   iou_car   |  any_number  |          None         |
|  iou_person |  any_number  |          None         |
+-------------+--------------+-----------------------+
Object Tags
+---------------+--------------+-----------------------+
|      Name     |  Value type  |    Possible values    |
+---------------+--------------+-----------------------+
|   car_color   |  any_string  |          None         |
| person_gender | oneof_string |   ['male', 'female']  |
|  vehicle_age  | oneof_string | ['modern', 'vintage'] |
+---------------+--------------+-----------------------+

Create Destination project

In [11]:
# check if destination project already exists. If yes - generate new free name
if api.project.exists(workspace.id, dst_project_name):
    dst_project_name = api.project.get_free_name(workspace.id, dst_project_name)
print("Destination project name: ", dst_project_name)
Out [11]:
Destination project name:  tutorial_project_analysis
In [12]:
dst_project = api.project.create(workspace.id, dst_project_name)
api.project.update_meta(dst_project.id, dst_meta.to_json())
print("Destination project has been created: id={}, name={!r}".format(dst_project.id, dst_project.name))
Out [12]:
Destination project has been created: id=924, name='tutorial_project_analysis'

Iterate over all images and calculate annotation quality

In [13]:
def _render_labels_for_class_name(labels, class_name, canvas):
    for label in labels:
        if label.obj_class.name == class_name:
            label.geometry.draw(canvas, True)

def safe_ratio(num, denom):
    return (num / denom) if denom != 0 else 0
In [14]:
def process(original_ann, class_comparison):
    ann = original_ann.clone()
    
    # rasterize all objects of class1
    mask_class1 = np.full(ann.img_size, False)
    _render_labels_for_class_name(ann.labels, class_comparison.class1, mask_class1)
    
    # rasterize all objects of class2
    mask_class2 = np.full(ann.img_size, False)
    _render_labels_for_class_name(ann.labels, class_comparison.class2, mask_class2)
    
    # construct intersection and union bitmaps, calculate IoU metric
    intersection = mask_class1 & mask_class2
    union = mask_class1 | mask_class2
    iou = safe_ratio(intersection.sum(), union.sum())
    
    # create and add tag to annotation
    diff_tagmeta = new_meta_items[class_comparison.class1][0]
    diff_tag = sly.Tag(meta=diff_tagmeta, value=iou)
    ann = ann.add_tag(diff_tag)
    
    if iou != 0:
        # create and add difference object to annotation
        mask_difference = union ^ intersection
        diff_geometry = sly.Bitmap(data=mask_difference)
        diff_object_class = new_meta_items[class_comparison.class1][1]
        diff_label = sly.Label(diff_geometry, diff_object_class)
        ann = ann.add_label(diff_label)
    else:
        diff_label = None
    
    return ann, iou, diff_label # iou, diff_label - are used for additional visualization
In [15]:
buffer_iou = {}
buffer_id = {}
for cc in classes_to_compare:
    buffer_iou[cc.class1] = []
    buffer_id[cc.class1] = []

for dataset in api.dataset.get_list(project.id):
    print('Dataset: {}'.format(dataset.name))
    dst_dataset = api.dataset.create(dst_project.id, dataset.name)

    for image in tqdm(api.image.get_list(dataset.id)):
        #img = api.image.download_np(image.id)
        ann_json = api.annotation.download(image.id).annotation
        ann = sly.Annotation.from_json(ann_json, meta)
        
        dst_image = api.image.add(dst_dataset.id, image.name, image.hash)
        
        new_ann = ann.clone()
        for cc in classes_to_compare:
            new_ann, iou, diff_label = process(new_ann, cc)
            buffer_iou[cc.class1].append(iou)
            buffer_id[cc.class1].append(dst_image.id)
            
        api.annotation.upload(dst_image.id, new_ann.to_json())
Out [15]:
  0%|          | 0/3 [00:00<?, ?it/s]
Dataset: dataset_01
100%|██████████| 3/3 [00:00<00:00,  3.95it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
Dataset: dataset_02
100%|██████████| 2/2 [00:00<00:00,  4.75it/s]

Visualize IoU distribution

In [51]:
for cc in classes_to_compare:
    fig = plt.figure()
    plt.title('{} <-> {}'.format(cc.class1, cc.class2))
    plt.xlabel("IoU value")
    plt.ylabel("Images number")
    sns.distplot(buffer_iou[cc.class1], bins=10, kde=False)
Out [51]:
<Figure size 432x288 with 1 Axes>
<Figure size 432x288 with 1 Axes>

Helper function to draw only difference

In [94]:
plt.rcParams["axes.grid"] = False
In [95]:
def _draw_diff(image_id, class_compare):
    diff_object_class = new_meta_items[class_compare.class1][1]
    
    img = api.image.download_np(image_id)
    
    ann_json = api.annotation.download(image_id).annotation
    ann = sly.Annotation.from_json(ann_json, dst_meta)
    
    for label in ann.labels:
        if label.obj_class.name == diff_object_class.name:
            label_to_draw = label
            break
    
    label.draw(img)
    return img
In [100]:
def draw_diff_for_all_pairs(functor_min_index):
    for cc in classes_to_compare:
        arr = np.array(buffer_iou[cc.class1])
        index = functor_min_index(arr)

        iou = buffer_iou[cc.class1][index]
        image_id = buffer_id[cc.class1][index]

        img = _draw_diff(image_id, cc)
        fig = plt.figure(figsize=(15, 15))
        plt.title('{} <-> {} : {}'.format(cc.class1, cc.class2, iou))
        plt.imshow(img)

Draw difference with minimum IoU for all class pairs

In [98]:
draw_diff_for_all_pairs(np.argmin)
Out [98]: