Add files via upload

Author: nmaffe

CFG.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+class CFG:
+    target_cols = ['camp', 'corylus', 'dust', 'grim', 'qrob', 'qsub', 'cont']
+    target_size = len(target_cols)
+    prob_cols = ['p_' + i for i in target_cols]
+    cols_mva = ['Area (ABD)', 'Area (Filled)', 'Aspect Ratio', 'Biovolume (Cylinder)',
+                'Biovolume (P. Spheroid)', 'Circle Fit',
+                'Circularity', 'Circularity (Hu)', 'Compactness', 'Convex Perimeter',
+                'Convexity', 'Diameter (ABD)', 'Diameter (ESD)', 'Edge Gradient',
+                'Elongation', 'Feret Angle Max', 'Feret Angle Min', 'Fiber Curl',
+                'Fiber Straightness', 'Geodesic Aspect Ratio', 'Geodesic Length',
+                'Geodesic Thickness', 'Intensity', 'Length', 'Particles Per Chain',
+                'Perimeter', 'Roughness', 'Sigma Intensity', 'Sum Intensity',
+                'Symmetry', 'Transparency', 'Volume (ABD)', 'Volume (ESD)', 'Width']
+
+    size = 128
+    n_fold = 1
+    num_workers = 8
+    batch_size = 512
+    model_name = 'resnet18'
+    if_pretrained = True
+
+    lr = 1e-4
+    epochs = 40
+
+    run_umap_test = True
+
+    save_model = True
+    load_model = False
+    model_name_saved = 'ICELEARNING_net'
+    OUTPUT_DIR = 'saved_model/'
+    save_conf_matrix = 'confusion_matrix_test_dataset.pdf'
+
+    save_inference_csv_files = True

data.py

@@ -0,0 +1,57 @@
+from CFG import CFG
+import torch
+from torch.utils.data import Dataset
+import albumentations as A
+import cv2
+from albumentations.pytorch import ToTensorV2
+
+# ====================================================
+# Dataset
+# ====================================================
+class ParticleDataset(Dataset):
+
+    def __init__(self, df, transform=None):
+
+        self.df = df
+        self.imgpaths = df['imgpaths'].to_numpy()
+        self.labels = df[CFG.target_cols].to_numpy()
+        self.transform = transform
+        self.X_features = df[CFG.cols_mva].to_numpy()
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, idx):
+
+        imgpath = self.imgpaths[idx]
+
+        image = cv2.imread(imgpath, cv2.IMREAD_GRAYSCALE)
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+        if self.transform:
+            augmented = self.transform(image=image)
+            image = augmented['image']
+
+        label = torch.tensor(self.labels[idx]).float()
+        xfeatures = torch.from_numpy(self.X_features[idx]).float()
+
+        # print(type(image), type(label), type(xfeatures))
+        return image, label, imgpath, xfeatures
+
+# ====================================================
+# Transformations
+# ====================================================
+def get_transforms(*, data):
+    if data == 'train':
+        return A.Compose([
+            A.Flip(p=0.5),
+            A.Resize(CFG.size, CFG.size),
+            A.Normalize(mean=[94., 94., 94.], std=[12., 12., 12.], max_pixel_value=1.0),
+            ToTensorV2()
+        ])
+    elif data == 'valid':
+        return A.Compose([
+            A.Resize(CFG.size, CFG.size),
+            A.Normalize(mean=[94., 94., 94.], std=[12., 12., 12.], max_pixel_value=1.0),
+            ToTensorV2()
+        ])

functs.py

@@ -0,0 +1,92 @@
+import torch
+
+# ====================================================
+# Helper functions
+# ====================================================
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+        self.history = []
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+        self.history.append(self.sum / self.count)
+
+    def update_simplesum(self, val, n=1):
+        self.val = val
+        self.sum += val
+        self.count += n
+        self.avg = self.sum / self.count
+        self.history.append(self.sum / self.count)
+
+
+def train_fn(train_loader, model, criterion, optimizer, scheduler, device):
+    losses = AverageMeter()
+    accuracies = AverageMeter()
+
+    model.train()
+
+    for step, (images, labels, paths, xfeatures) in enumerate(train_loader):
+        images = images.to(device)
+        labels = labels.to(device)
+        xfeatures = xfeatures.to(device)
+
+        # with torch.set_grad_enabled(True):
+        y_preds = model(images, xfeatures)
+        loss = criterion(y_preds, labels)
+        preds = (y_preds == y_preds.max(dim=1, keepdim=True)[0]).to(dtype=torch.int32)
+
+        # statistics
+        losses.update(loss.item(), images.size(0))
+        how_many_correct = torch.sum(torch.all(torch.eq(preds, labels), dim=1))
+        accuracies.update_simplesum(how_many_correct.item(), images.size(0))
+
+        # compute gradient and do SGD step
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+    scheduler.step()
+
+    print(f'Train Loss: {losses.avg:.4f} Acc: {accuracies.avg:.4f}')
+
+    return losses.history, accuracies.history
+
+
+def valid_fn(valid_loader, model, criterion, device):
+    losses = AverageMeter()
+    accuracies = AverageMeter()
+
+    model.eval()
+
+    for step, (images, labels, paths, xfeatures) in enumerate(valid_loader):
+        images = images.to(device)
+        labels = labels.to(device)
+        xfeatures = xfeatures.to(device)
+
+        # compute loss
+        with torch.no_grad():
+            y_preds = model(images, xfeatures)
+            loss = criterion(y_preds, labels)  
+            preds = (y_preds == y_preds.max(dim=1, keepdim=True)[0]).to(dtype=torch.int32)
+
+        # statistics
+        losses.update(loss.item(), images.size(0))
+        how_many_correct = torch.sum(torch.all(torch.eq(preds, labels), dim=1))
+        accuracies.update_simplesum(how_many_correct.item(), images.size(0))
+
+    print(f'Val Loss: {losses.avg:.4f} Acc: {accuracies.avg:.4f}')
+
+    return losses.history, accuracies.history
+

model.py

@@ -0,0 +1,86 @@
+from CFG import CFG
+import torch
+import torch.nn as nn
+import torchvision.models as models
+
+# ====================================================
+# MODEL
+# ====================================================
+class CustomModel(nn.Module):
+    def __init__(self, model_name=CFG.model_name, pretrained=CFG.if_pretrained):
+
+        super().__init__()
+
+        if model_name == 'resnet18':
+            self.base = models.resnet18(weights='ResNet18_Weights.DEFAULT')
+        if model_name == 'resnet34':
+            self.base = models.resnet34(weights='ResNet34_Weights.DEFAULT')
+        if model_name == 'resnet152':
+            self.base = models.resnet152(weights='ResNet152_Weights.DEFAULT')
+        if model_name == 'resnet101':
+            self.base = timm.create_model(model_name)
+        elif model_name == 'resnext50_32x4d':
+            self.base = timm.create_model(model_name)
+
+
+        n_features = self.base.fc.in_features  # 512
+
+        self.base.fc = nn.Linear(n_features, 64)
+
+        self.bn1 = nn.BatchNorm1d(64)
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(p=0.2)
+
+        self.meta_net = nn.Sequential(nn.Linear(34, 128),
+                                      nn.BatchNorm1d(128),
+                                      nn.ReLU(),
+                                      nn.Dropout(p=0.5),
+                                      nn.Linear(128, 64),
+                                      nn.BatchNorm1d(64),
+                                      nn.ReLU(),
+                                      nn.Dropout(p=0.5),
+                                      nn.Linear(64, 32)
+                                      )
+
+        self.fc3 = nn.Linear(96, 40)
+        self.bn3 = nn.BatchNorm1d(40)
+
+        self.layer_out = nn.Linear(40, CFG.target_size)
+
+    def forward(self, imgs, metas):
+        cnn1 = self.base(imgs)
+        x = self.bn1(cnn1)
+        x = self.relu(x)
+        x = self.dropout(x)
+
+        meta_ = self.meta_net(metas)
+
+        x = torch.cat((x, meta_), 1)
+
+        x = self.fc3(x)
+        x = self.bn3(x)
+        x = self.relu(x)
+        x = self.dropout(x)
+
+        x = self.layer_out(x)
+        return x
+
+# ====================================================
+# Hooks
+# ====================================================
+class Hook():
+    def __init__(self, name, module, backward=False):
+
+        self.name = name
+
+        if backward == False:
+            self.hook = module.register_forward_hook(self.hook_fn)
+        else:
+            self.hook = module.register_backward_hook(self.hook_fn)
+
+    def hook_fn(self, module, input, output):
+        self.input = input
+        self.output = output
+
+    def close(self):
+        self.hook.remove()