add WSGCNI

Data/IRN4GCN/AFF_MATRIX/README.md

@@ -0,0 +1 @@
+# Directory for affinity matrix

Data/VOC12/VOC2012/README.md

@@ -0,0 +1,16 @@
+### Please put the Pascal VOC2012 dataset here
+Note that the SegmentationClassAug is the ground truth from the Pascal VOC2012 augmentation dataset
+
+```
+./
+├── Annotations
+├── ImageSets
+│   ├── Action
+│   ├── Layout
+│   ├── Main
+│   └── Segmentation
+├── JPEGImages
+├── SegmentationClass
+├── SegmentationClassAug
+└── SegmentationObject
+```

Data/VOC12/easy.txt

@@ -0,0 +1,20 @@
+/JPEGImages/2007_000032.jpg /SegmentationClassAug/2007_000032.png
+/JPEGImages/2007_000039.jpg /SegmentationClassAug/2007_000039.png
+/JPEGImages/2007_000063.jpg /SegmentationClassAug/2007_000063.png
+/JPEGImages/2007_000068.jpg /SegmentationClassAug/2007_000068.png
+/JPEGImages/2007_000121.jpg /SegmentationClassAug/2007_000121.png
+/JPEGImages/2007_000170.jpg /SegmentationClassAug/2007_000170.png
+/JPEGImages/2007_000241.jpg /SegmentationClassAug/2007_000241.png
+/JPEGImages/2007_000243.jpg /SegmentationClassAug/2007_000243.png
+/JPEGImages/2007_000250.jpg /SegmentationClassAug/2007_000250.png
+/JPEGImages/2007_000256.jpg /SegmentationClassAug/2007_000256.png
+/JPEGImages/2007_000333.jpg /SegmentationClassAug/2007_000333.png
+/JPEGImages/2007_000363.jpg /SegmentationClassAug/2007_000363.png
+/JPEGImages/2007_000364.jpg /SegmentationClassAug/2007_000364.png
+/JPEGImages/2007_000392.jpg /SegmentationClassAug/2007_000392.png
+/JPEGImages/2007_000480.jpg /SegmentationClassAug/2007_000480.png
+/JPEGImages/2007_000504.jpg /SegmentationClassAug/2007_000504.png
+/JPEGImages/2007_000515.jpg /SegmentationClassAug/2007_000515.png
+/JPEGImages/2007_000528.jpg /SegmentationClassAug/2007_000528.png
+/JPEGImages/2007_000549.jpg /SegmentationClassAug/2007_000549.png
+/JPEGImages/2007_000584.jpg /SegmentationClassAug/2007_000584.png

GCN/CRF.py

@@ -0,0 +1,139 @@
+import numpy as np
+import torch
+import scipy.misc
+import os
+import datetime
+import time
+import pydensecrf.densecrf as dcrf
+import tqdm
+import misc_old
+
+from cv2 import imread
+from torch import nn
+from multiprocessing import Pool
+from functools import partial
+from pydensecrf.utils import unary_from_softmax
+from utils import colors_map, load_img_name_list, show_timing
+from utils import evaluate_dataset_IoU
+from utils import get_least_modify_file
+from config import opt as args
+from PIL import Image
+
+
+def crf_inference(img, probs, CRF_parameter, scale_factor=1, labels=21):
+    h, w = img.shape[:2]
+    n_labels = labels
+
+    d = dcrf.DenseCRF2D(w, h, n_labels)
+    pred_softmax = torch.nn.Softmax(dim=0)
+    probs = pred_softmax(torch.tensor(probs)).numpy()
+    unary = unary_from_softmax(probs)
+    unary = np.ascontiguousarray(unary)
+
+    d.setUnaryEnergy(unary)
+    d.addPairwiseGaussian(sxy=CRF_parameter["pos_xy_std"] / scale_factor,
+                          compat=CRF_parameter["pos_w"])
+    d.addPairwiseBilateral(sxy=CRF_parameter["bi_xy_std"] / scale_factor,
+                           srgb=CRF_parameter["bi_rgb_std"],
+                           rgbim=np.copy(img),
+                           compat=CRF_parameter["bi_w"])
+    Q = d.inference(CRF_parameter["iter_max"])
+    return np.array(Q).reshape((n_labels, h, w))
+
+
+def crf(img_name, CRF_parameter, save_path_label, save_path_logit, img=None,
+        probs=None, prediction_root=None, scale_factor=1, labels=21):
+
+    if img is None:
+        img = imread(os.path.join(args.path4Image, img_name + '.jpg'))
+    H, W = img.shape[:2]
+    # load predict_dict 
+    if prediction_root is None:
+        prediction_root = os.path.join("predict_result_matrix_visual_new",
+                                       "250")
+    prect_dict = np.load(os.path.join(prediction_root, img_name + '.npy'),
+                         allow_pickle=True).item()
+
+    def crf_inf(predicted_dict, name=None):
+        v = np.array(list(predicted_dict.values()))
+        img_path = os.path.join(args.path4Image, name + '.jpg')
+        orig_img = np.asarray(Image.open(img_path))
+        crf_score = crf_inference(orig_img, v, labels=v.shape[0],
+                                  CRF_parameter=CRF_parameter)
+        h, w = orig_img.shape[:2]
+        crf_dict = dict()
+        crf_score_np = np.zeros(shape=(args.num_class, h, w))
+        for i, key in enumerate(predicted_dict.keys()):
+            crf_score_np[key] = crf_score[i]
+            crf_dict[key] = crf_score[i]
+        return crf_score_np, crf_dict
+
+    crf_resut, crf_dict = crf_inf(predicted_dict=prect_dict, name=img_name)
+
+    # save crf logit
+    if not os.path.exists(save_path_logit):
+        os.makedirs(save_path_logit)
+    np.save(os.path.join(save_path_logit, img_name + '.npy'), crf_dict)
+
+    # save crf label
+    if not os.path.exists(save_path_label):
+        os.makedirs(save_path_label)
+    misc_old.toimage(crf_resut.argmax(axis=0), cmin=0, cmax=255, 
+                     pal=colors_map, mode="P").save(
+                     os.path.join(save_path_label, img_name + '.png'))
+
+
+def apply(**kwargs):
+    parameter_dict = dict()
+    t_start = time.time()
+    time_now = datetime.datetime.today()
+    time_now = "{}_{}_{}_{}h{}m".format(time_now.year, time_now.month,
+                                        time_now.day, time_now.hour,
+                                        time_now.minute)
+    descript = ""
+    parameter_dict["num_cpu"] = os.cpu_count()//2
+    parameter_dict["CRF_parameter"] = args.CRF
+    parameter_dict["path4saveCRF_label"] = args.path4Complete_label_label
+    parameter_dict["path4saveCRF_logit"] = args.path4Complete_label_logit
+    if "pred_root" not in kwargs.keys():
+        parameter_dict["pred_root"] = args.path4GCN_logit
+    else:
+        parameter_dict["pred_root"] = kwargs["pred_root"]
+
+    parameter_dict["f_list"] = args.path4train_images
+
+    evaluate_folder = parameter_dict["path4saveCRF_label"]
+    img_list = load_img_name_list(parameter_dict["f_list"])
+    # === load parameter
+    for k, v in kwargs.items():
+        if k in parameter_dict.keys():
+            if "CRF_parameter" == k:
+                parameter_dict[k] = eval(v)
+            else:
+                parameter_dict[k] = v
+            print("{}: {}".format(k, parameter_dict[k]))
+
+    print("path4saveCRF_label: ", parameter_dict["path4saveCRF_label"])
+    print("pred_root: ", parameter_dict["pred_root"])
+
+    p = Pool(parameter_dict["num_cpu"])
+    crfP = partial(crf,
+                   prediction_root=parameter_dict["pred_root"],
+                   save_path_label=parameter_dict["path4saveCRF_label"],
+                   save_path_logit=parameter_dict["path4saveCRF_logit"],
+                   CRF_parameter=parameter_dict["CRF_parameter"])
+    # run crf by multiprocessing
+    for _ in tqdm.tqdm(p.imap_unordered(crfP, img_list), total=len(img_list)):
+        pass
+    p.close()
+    p.join()
+    evaluate_dataset_IoU(file_list=parameter_dict["f_list"],
+                         predicted_folder=evaluate_folder,
+                         descript=descript,
+                         path4GT=args.path4VOC_class_aug)
+    show_timing(time_start=t_start, time_end=time.time())
+
+
+if __name__ == "__main__":
+    apply()
+

GCN/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import print_function
+from __future__ import division

GCN/config.py

@@ -0,0 +1,94 @@
+import os
+import torch
+import warnings
+import argparse
+
+class DefaultConfig(object):
+
+    debug = False
+    seed = 42
+    cuda = torch.cuda.is_available()
+    use_TB = True
+    process_id = 1
+
+    # === parameters for GCN ===
+    lr = 0.01
+    weight_decay = 5e-4
+    num_class = 21
+    max_epoch = 250
+    num_hid_unit = 16
+    drop_rate = .3
+    use_lap = True
+    use_ent = True
+
+    # === parameter for preprocessing ===
+    confident_ratio = 0.3
+
+    # === VOC dataset ===
+    path4Data = os.path.join("..", "Data")
+    path4VOC_root = os.path.join(path4Data, "VOC12", "VOC2012")
+    path4Image = os.path.join(path4VOC_root, "JPEGImages")
+    # path4Class
+    path4VOC_class_aug = os.path.join(path4VOC_root, "SegmentationClassAug")
+    # image list
+    path4train_images = os.path.join(path4Data, "VOC12", "train.txt")
+    path4train_aug_images = os.path.join(path4Data, "VOC12", "train_aug2.txt")
+    path4val_images = os.path.join(path4Data, "VOC12", "val3.txt")
+    path4trainval_images = os.path.join(path4Data, "trainval4.txt")
+    eval_dataset = True
+
+    # === IRN4GCN ===
+    path4IRN4GCN = os.path.join(path4Data, "IRN4GCN")
+    # path4boundaryMap 
+    path4CAM = os.path.join("../IRN/result/cam")
+    # path4boundaryMap_logit 
+    path4CAMLogit = os.path.join(path4IRN4GCN, "CAMLogit")
+    path4AffGraph = os.path.join(path4IRN4GCN, "AFF_MATRIX")
+    path4node_feat = os.path.join(path4IRN4GCN, "AFF_FEATURE")
+    partial_label_label = os.path.join(path4IRN4GCN, "PARTIAL_PSEUDO_LABEL")
+    partial_label_logit = os.path.join(path4IRN4GCN, "PARTIAL_PSEUDO_LABEL_LOGIT")
+    path4partial_label_label = os.path.join(partial_label_label+"_DN")
+    path4partial_label_logit = os.path.join(partial_label_logit+"_DN")
+
+    output_rate = 4
+
+    # === parameter for postprocessing ===
+    path4GCN4DeepLab = os.path.join(path4Data, "GCN4DeepLab")
+    path4GCN_logit = os.path.join(path4GCN4DeepLab, "Logit")
+    path4GCN_label = os.path.join(path4GCN4DeepLab, "Label")
+    path4Complete_label_label = os.path.join(path4GCN4DeepLab, "CRF_Label")
+    path4Complete_label_logit = os.path.join(path4GCN4DeepLab, "CRF_Logit")
+    save_prediction_np = True
+    save_mask = True
+
+    # === CRF ===
+    CRF = dict()
+
+    CRF["iter_max"] = 10
+    CRF["pos_w"] = 3
+    CRF["pos_xy_std"] = 3
+    CRF["bi_w"] = 3
+    CRF["bi_xy_std"] = 50
+    CRF["bi_rgb_std"] = 5
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--train_list", default=path4train_images, type=str)
+    args = parser.parse_args()
+    path4train_images = args.train_list
+
+
+    def parse(self, **kwargs):
+        """
+        update config
+        """
+        for k, v in kwargs.items():
+            if not hasattr(self, k):
+                continue
+                #warnings.warn("Warning: opt does not have attribute:  {}".format(k))
+            else:
+                setattr(self, k, v)
+
+        opt.device = torch.device('cuda') if opt.cuda else torch.device('cpu')
+
+
+opt = DefaultConfig()

GCN/dataset.py

@@ -0,0 +1,120 @@
+import torch
+import torch.nn.functional as F
+import os
+import scipy.sparse as sp
+import numpy as np
+import time
+
+from PIL import Image
+from torchvision import transforms as T
+from torch.utils import data
+from cv2 import imread
+from config import opt as args
+from utils import load_img_name_list
+
+
+def normalize_t(mx):
+    """Row-normalize sparse matrix in tensor"""
+    rowsum = torch.sum(mx, dim=1)
+    r_inv = torch.pow(rowsum, -1).flatten()
+    r_inv[torch.isinf(r_inv)] = 0.
+    r_mat_inv = torch.diagflat(r_inv)
+    mx = torch.mm(r_mat_inv, mx)
+    return mx
+
+
+def preprocess_adj(aff_mat):
+    adjT = torch.t(aff_mat)
+    adj = torch.stack([aff_mat, adjT])
+    adj, _ = adj.max(dim=0)
+    return normalize_t(adj + torch.eye(adj.shape[0]))
+
+
+class graph_voc(data.Dataset):
+    def __init__(self, root=args.path4Image, graph_type="AFF", start_idx=0,
+                 end_idx=None, device=None):
+        self.label_list = load_img_name_list(args.path4train_images)
+        self.seg_label_dict = dict()
+        # AFF
+        self.graph_type = graph_type  
+        self.train_file = load_img_name_list(args.path4train_images)
+        self.start_idx = start_idx
+        self.end_idx = len(self.label_list) if end_idx is None else end_idx
+        self.device = device
+        print("self.device: ", self.device)
+        self.ignore_list = []
+
+    def load_data(self, graph_type='AFF', path=None, img_name=None,
+                  path4Data=None, load_adjacency_mat=True):
+        """
+        return adj, features, labels, idx_train, idx_test, rgbxy, img_name
+        adj: sparse matrix
+        """
+        t_start = time.time()
+        graph = np.load(os.path.join(args.path4AffGraph, img_name + ".npy"))
+        adj = preprocess_adj(torch.FloatTensor(graph))
+        labels = Image.open(
+            os.path.join(args.path4partial_label_label, img_name + '.png'))
+        labels = np.asarray(labels)
+        labels = np.reshape(labels, (-1)).astype(np.int16)
+
+        # np.int8 turns 255 to -1
+        labels = np.where(labels == -1, 255,
+                          labels)
+        # split foreground and background label
+        label_fg = labels.copy()
+        label_fg[label_fg == 0] = 255
+
+        label_bg = labels.copy()
+        label_bg[label_bg != 0] = 255
+
+        # to tensor
+        labels = torch.LongTensor(labels)
+        label_fg_t = torch.LongTensor(label_fg)
+        label_bg_t = torch.LongTensor(label_bg)
+
+        img = imread(os.path.join(args.path4Image, img_name + ".jpg"))
+        H_origin, W_origin, C = img.shape
+        H = int(np.ceil(H_origin / 4))
+        W = int(np.ceil(W_origin / 4))
+
+        f_aff = np.load(os.path.join(args.path4node_feat, img_name + ".npy"))
+        f_aff = np.squeeze(f_aff)
+        f_aff = np.reshape(f_aff, (np.shape(f_aff)[0], H * W))
+        allx = np.transpose(f_aff, [1, 0])
+        feat = torch.FloatTensor(np.array(allx))
+        # get rgb    
+        img_dn = Image.fromarray(img).resize((W, H), Image.LANCZOS)
+        img_dn = np.asarray(img_dn)
+        rgbxy = np.zeros(shape=(H, W, 5))
+        rgbxy[:, :, :3] = img_dn / 255.
+
+        # get xy
+        for i in range(H):
+            for j in range(W):
+                rgbxy[i, j, 3] = float(i) # / H
+                rgbxy[i, j, 4] = float(j) # / W
+
+        rgbxy_t = torch.FloatTensor(rgbxy)
+        return {"adj_t": adj, "features_t": feat, "labels_t": labels,
+            "rgbxy_t": rgbxy_t, "img_name": img_name, 
+            "label_fg_t": label_fg_t, "label_bg_t": label_bg_t}
+
+    def __getitem__(self, index):
+        """
+        return adj, feat, labels, idx_train_t, rgbxy, img_name, label_fg_t, label_bg_t
+        """
+
+        img_name = self.train_file[index]
+        if self.start_idx <= index < self.end_idx:
+            if img_name in self.ignore_list:
+                print("[{}] ignore: {}".format(index, img_name))
+                return None
+            return self.load_data(graph_type=self.graph_type, 
+                                  path=args.path4AffGraph, img_name=img_name,
+                                  path4Data=args.path4Data)
+        else:
+            return None
+
+    def __len__(self):
+        return len(self.train_file)