Upgrade to DLSIA Suite

.gitignore

@@ -0,0 +1,3 @@
+*pycache*
+data/
+.DS_Store

Makefile

@@ -1,6 +1,6 @@
 TAG 			:= latest	
-USER 			:= mlexchange
-PROJECT			:= msdnetwork-notebook
+USER 			:= mlexchange1
+PROJECT			:= dlsia
 
 IMG_WEB_SVC    		:= ${USER}/${PROJECT}:${TAG}
 IMG_WEB_SVC_JYP    	:= ${USER}/${PROJECT_JYP}:${TAG}
@@ -22,12 +22,20 @@ build_docker:
 run_docker:
 	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/local -p 8888:8888 ${IMG_WEB_SVC}
 
-train_example:
-	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/train.py data/mask data/images/train data/output '{"num_epochs": 200, "optimizer": "Adam", "criterion": "CrossEntropyLoss", "learning_rate": 0.01, "num_layers": 10, "max_dilation": 10}'
+train_msdnet_maxdil:
+	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/train.py data/mask data/images/train data/output '{"model": "MSDNet", "num_epochs": 10, "optimizer": "Adam", "criterion": "CrossEntropyLoss", "learning_rate": 0.01, "num_layers": 10, "custom_dilation": false, "max_dilation": 5}'
 
-test_example:
-	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/segment.py data/images/test/segment_series.tif data/output/state_dict_net.pt data/output '{"show_progress": 20}'
+train_msdnet_customdil:
+	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/train.py data/mask data/images/train data/output '{"model": "MSDNet", "num_epochs": 10, "optimizer": "Adam", "criterion": "CrossEntropyLoss", "learning_rate": 0.01, "num_layers": 10, "custom_dilation": true, "dilation_array": [1,2,5]}'
 
+train_tunet:
+	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/train.py data/mask data/images/train data/output '{"model": "TUNet", "num_epochs": 10, "optimizer": "Adam", "criterion": "CrossEntropyLoss", "learning_rate": 0.01, "depth": 4, "base_channels": 16, "growth_rate": 2, "hidden_rate": 1}'
+
+train_tunet3plus:
+	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/train.py data/mask data/images/train data/output '{"model": "TUNet3+", "num_epochs": 10, "optimizer": "Adam", "criterion": "CrossEntropyLoss", "learning_rate": 0.01, "depth": 4, "base_channels": 16, "carryover_channels": 5, "growth_rate": 2, "hidden_rate": 1}'
+
+test_segment:
+	docker run -u ${ID_USER $USER}:${ID_GROUP $USER} --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 --memory-swap -1 -it -v ${PWD}:/app/work/ ${IMG_WEB_SVC} python3 src/segment.py data/images/test/segment_series.tif data/output/state_dict_net.pt data/output '{"show_progress": 1}'
 
 clean: 
 	find -name "*~" -delete

docker/Dockerfile

@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     pip install --upgrade pip  
 
 RUN pip install -r requirements.txt
-RUN pip install git+https://bitbucket.org/berkeleylab/pymsdtorch.git
+RUN pip install git+https://github.com/phzwart/dlsia.git
 
 WORKDIR /app/work/
 COPY src src

docker/requirements.txt

@@ -1,10 +1,10 @@
-einops==0.3.2
-imageio==2.9.0
-matplotlib==3.4.3
-numpy==1.21.2
+#einops==0.3.2
+#imageio==2.9.0
+#matplotlib==3.4.3
+#numpy==1.21.2
 patchify==0.2.3
 pydantic==1.8.2
-scikit-image==0.18.3
-torch==1.9.0
-torchvision==0.10.0
+#scikit-image==0.18.3
+#torch==1.9.0
+#torchvision==0.10.0
 tifffile==2022.2.9

src/seg_helpers/model.py → src/model.py

@@ -1,6 +1,11 @@
 from enum import Enum
 from pydantic import BaseModel, Field
-from typing import Optional
+from typing import Optional, List
+
+class NNModel(str, Enum):
+    msdnet = 'MSDNet'
+    tunet = 'TUNet'
+    tunet3plus = 'TUNet3+'
 
 
 class Optimizer(str, Enum):
@@ -18,16 +23,15 @@ class Optimizer(str, Enum):
 
 
 class Criterion(str, Enum):
-    l1loss = "L1Loss" #*
-    mseloss = "MSELoss" #*
+    l1loss = "L1Loss" 
+    mseloss = "MSELoss" 
     crossentropyloss = "CrossEntropyLoss"
     ctcloss = "CTCLoss"
-#    nllloss = "NLLLoss"
     poissonnllloss = "PoissonNLLLoss"
     gaussiannllloss = "GaussianNLLLoss"
     kldivloss = "KLDivLoss"
-    bceloss = "BCELoss" #*
-    bcewithlogitsloss = "BCEWithLogitsLoss" #*
+    bceloss = "BCELoss"
+    bcewithlogitsloss = "BCEWithLogitsLoss"
     marginrankingloss = "MarginRankingLoss"
     hingeembeddingloss = "HingeEnbeddingLoss"
     multilabelmarginloss = "MultiLabelMarginLoss"
@@ -49,12 +53,20 @@ class LoadParameters(BaseModel):
 
 
 class TrainingParameters(BaseModel):
+    model: NNModel
     num_epochs: int = Field(description="number of epochs")
     optimizer: Optimizer
     criterion: Criterion
     learning_rate: float = Field(description='learning rate')
-    num_layers: int = Field(description="number of layers")
-    max_dilation: int = Field(description="maximum dilation")
+    num_layers: Optional[int] = Field(description="number of layers for MSDNet")
+    custom_dilation: Optional[bool] = Field(description="whether to customize dilation for MSDNet")
+    max_dilation: Optional[int] = Field(description="maximum dilation for MSDNet")
+    dilation_array: Optional[List[int]] = Field(description="customized dilation array for MSDNet")
+    depth: Optional[int] = Field(description='the depth of the UNet')
+    base_channels: Optional[int] = Field(description='the number of initial channels for UNet')
+    growth_rate: Optional[int] = Field(description='multiplicative growth factor of number of channels per layer of depth for UNet')
+    hidden_rate: Optional[int] = Field(description='multiplicative growth factor of channels within each layer for UNet')
+    carryover_channels: Optional[int] = Field(description='the number of channels in each skip connection for UNet3+')
     load: Optional[LoadParameters]
 
 

src/segment.py

@@ -6,8 +6,8 @@
 import torch
 from torch.utils.data import Dataset, DataLoader
 
-from pyMSDtorch.core import helpers
-from seg_helpers.model import TestingParameters, TestingResults
+from dlsia.core import helpers
+from model import TestingParameters, TestingResults
 
 
 class PredictDataset(Dataset):