From 53260c2c0b4c6b0870c788ba4ad5bb61dc890f58 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Fri, 10 Feb 2023 15:12:52 -0500 Subject: [PATCH 1/6] Updated with latest GT file name --- preprocessing/qc_preprocess.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/preprocessing/qc_preprocess.py b/preprocessing/qc_preprocess.py index 3423292..b48629c 100644 --- a/preprocessing/qc_preprocess.py +++ b/preprocessing/qc_preprocess.py @@ -60,10 +60,8 @@ resolutions.append(resolution) # Read original and cropped subject ground-truths (GT) - gt1_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manual.nii.gz' % subject) - gt1_crop_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manual_crop.nii.gz' % subject) - gt2_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manual2.nii.gz' % subject) - gt2_crop_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manual2_crop.nii.gz' % subject) + gt1_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manualNeuroPoly.nii.gz' % subject) + gt1_crop_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manualNeuroPoly_crop.nii.gz' % subject) gt1 = nib.load(gt1_fpath) gt1_crop = nib.load(gt1_crop_fpath) From b24e1b4af51c596f32be91e2c4ff229b5d767dff Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Fri, 10 Feb 2023 15:13:11 -0500 Subject: [PATCH 2/6] Updated config file --- config/config_seg_lesion.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config_seg_lesion.json b/config/config_seg_lesion.json index 4aee2c5..7248da6 100644 --- a/config/config_seg_lesion.json +++ b/config/config_seg_lesion.json @@ -1,19 +1,19 @@ { "command": "train", "gpu_ids": [4], - "path_output": "/home/GRAMES.POLYMTL.CA/p101317/data_nvme_p101317/model_seg_lesion_mp2rage_20230102_144854", + "path_output": "/home/GRAMES.POLYMTL.CA/p101317/data_nvme_p101317/model_seg_lesion_mp2rage_", "model_name": "model_seg_lesion_mp2rage", "debugging": true, "log_file": "train.log", "wandb": { "wandb_api_key": "9095e2bc9e4ab445d478c9c8a81759ae908be8c6", "project_name": "basel-mp2rage-lesion", - "group_name": "r20230127", - "run_name": "seed11", + "group_name": "r20230210", + "run_name": "seed01", "log_grads_every": 100 }, "loader_parameters": { - "path_data": ["/home/GRAMES.POLYMTL.CA/p101317/data_nvme_p101317/data_seg_mp2rage_20230124_132415/data_processed_lesionseg"], + "path_data": ["/home/GRAMES.POLYMTL.CA/p101317/data_nvme_p101317/data_seg_mp2rage_20230210_144231/data_processed_lesionseg"], "subject_selection": {"n": [], "metadata": [], "value": []}, "target_suffix": ["_lesion-manualNeuroPoly"], "extensions": [".nii.gz"], @@ -37,7 +37,7 @@ }, "split_dataset": { "fname_split": null, - "random_seed": 11, + "random_seed": 1, "split_method" : "participant_id", "data_testing": {"data_type": null, "data_value":[]}, "balance": null, From 4b582ff7047e04eeba9731c98a36e9a6762ec806 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Fri, 10 Feb 2023 15:13:18 -0500 Subject: [PATCH 3/6] Removed need for mask dilation Fixes #62 --- preprocessing/preprocess_data.sh | 69 +++++--------------------------- 1 file changed, 9 insertions(+), 60 deletions(-) diff --git a/preprocessing/preprocess_data.sh b/preprocessing/preprocess_data.sh index 7b405e1..ca34db9 100644 --- a/preprocessing/preprocess_data.sh +++ b/preprocessing/preprocess_data.sh @@ -107,6 +107,7 @@ cd ${SUBJECT}/anat # Define variables file="${SUBJECT}_UNIT1" +file_gt1="${SUBJECT}_UNIT1_lesion-manualNeuroPoly" # Make sure the image metadata is a valid JSON object if [[ ! -s ${file}.json ]]; then @@ -119,65 +120,22 @@ fi segment_if_does_not_exist ${file} t1 svm file_seg="${FILESEG}" -# Dilate spinal cord mask -sct_maths -i ${file_seg}.nii.gz -dilate 2 -shape ball -o ${file_seg}_dilate.nii.gz -sct_maths -i ${file_seg}_dilate.nii.gz -dilate 32 -dim 1 -shape disk -o ${file_seg}_dilate.nii.gz - -# Use dilated mask to crop the original image and manual MS segmentations -sct_crop_image -i ${file}.nii.gz -m ${file_seg}_dilate.nii.gz -o ${file}_crop.nii.gz +# Use mask to crop the original image +sct_crop_image -i ${file}.nii.gz -m ${file_seg}.nii.gz -dilate 3 -o ${file}_crop.nii.gz -# Go to subject folder for segmentation GTs -cd $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat +# Crop the manual MS lesion segmentation +sct_crop_image -i $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.nii.gz -m ${file_seg}.nii.gz -dilate 3 -o $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}_crop.nii.gz -# Define variables -file_gt1="${SUBJECT}_UNIT1_lesion-manualNeuroPoly" -# Until we figure out a way to inclure more ground truth, we only use manualHaris segmentation. -# In the future, manualNeuroPoly should be use if it exists. -#file_gt2="${SUBJECT}_UNIT1_lesion-manual2" -#file_gtc="${SUBJECT}_UNIT1_lesion-manual-majvote" -#file_soft="${SUBJECT}_UNIT1_lesion-manual-soft" -# 'c' stands for the consensus GT - -# Redefine variable for final SC segmentation mask as path changed -file_seg_dil=${PATH_DATA_PROCESSED}/${SUBJECT}/anat/${file_seg}_dilate - -# Make sure the first rater metadata is a valid JSON object -if [[ ! -s ${file_gt1}.json ]]; then - echo "{}" >> ${file_gt1}.json +# Make sure a JSON file is present, if not create an empty one +if [[ ! -s $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.json ]]; then + echo "{}" >> $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.json fi -# Aggregate multiple raters if second rater is present -#if [[ -f ${file_gt2}.nii.gz ]]; then - # Make sure the second rater metadata is a valid JSON object -# if [[ ! -s ${file_gt2}.json ]]; then -# echo "{}" >> ${file_gt2}.json -# fi - # Create consensus ground truth by majority vote -# sct_maths -i ${file_gt1}.nii.gz -add ${file_gt2}.nii.gz -o lesion_sum.nii.gz -# sct_maths -i lesion_sum.nii.gz -sub 1 -o lesion_sum_minusone.nii.gz - # binarize: everything that is 0.5 and below 0.5 becomes 0. -# sct_maths -i lesion_sum_minusone.nii.gz -thr 0.5 -o ${file_gtc}.nii.gz - - # Create soft ground truth by averaging all raters -# sct_maths -i lesion_sum.nii.gz -div 2 -o ${file_soft}.nii.gz - - # Crop the manual segs -# sct_crop_image -i ${file_gt2}.nii.gz -m ${file_seg_dil}.nii.gz -o ${file_gt2}_crop.nii.gz -# sct_crop_image -i ${file_gtc}.nii.gz -m ${file_seg_dil}.nii.gz -o ${file_gtc}_crop.nii.gz -# sct_crop_image -i ${file_soft}.nii.gz -m ${file_seg_dil}.nii.gz -o ${file_soft}_crop.nii.gz -#fi - -# Crop the manual seg -sct_crop_image -i ${file_gt1}.nii.gz -m ${file_seg_dil}.nii.gz -o ${file_gt1}_crop.nii.gz - -# Go back to the root output path -cd $PATH_OUTPUT - # Create clean data processed folders for two tasks: spinal cord (SC) segmentation and lesion segmentation PATH_DATA_PROCESSED_SCSEG="${PATH_DATA_PROCESSED}_scseg" PATH_DATA_PROCESSED_LESIONSEG="${PATH_DATA_PROCESSED}_lesionseg" -# Copy over required BIDs files to both folders +# Copy over required BIDS files to both folders mkdir -p $PATH_DATA_PROCESSED_SCSEG $PATH_DATA_PROCESSED_SCSEG/${SUBJECT} $PATH_DATA_PROCESSED_SCSEG/${SUBJECT}/anat mkdir -p $PATH_DATA_PROCESSED_LESIONSEG $PATH_DATA_PROCESSED_LESIONSEG/${SUBJECT} $PATH_DATA_PROCESSED_LESIONSEG/${SUBJECT}/anat rsync -avzh $PATH_DATA_PROCESSED/dataset_description.json $PATH_DATA_PROCESSED_SCSEG/ @@ -208,15 +166,6 @@ rsync -avzh $PATH_DATA_PROCESSED/${SUBJECT}/anat/${file}.json $PATH_DATA_PROCESS mkdir -p $PATH_DATA_PROCESSED_LESIONSEG/derivatives $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT} $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/ rsync -avzh $PATH_DATA_PROCESSED/derivatives/labels/${SUBJECT}/anat/${file_gt1}_crop.nii.gz $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/${file_gt1}.nii.gz rsync -avzh $PATH_DATA_PROCESSED/derivatives/labels/${SUBJECT}/anat/${file_gt1}.json $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/${file_gt1}.json -# If second rater is present, copy the other files -#if [[ -f ${PATH_DATA_PROCESSED}/derivatives/labels/${SUBJECT}/anat/${file_gt2}.nii.gz ]]; then - # Copy the second rater GT and aggregated GTs if second rater is present -# rsync -avzh $PATH_DATA_PROCESSED/derivatives/labels/${SUBJECT}/anat/${file_gt2}_crop.nii.gz $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/${file_gt2}.nii.gz -# rsync -avzh $PATH_DATA_PROCESSED/derivatives/labels/${SUBJECT}/anat/${file_gt2}.json $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/${file_gt2}.json -# rsync -avzh $PATH_DATA_PROCESSED/derivatives/labels/${SUBJECT}/anat/${file_gtc}_crop.nii.gz $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/${file_gtc}.nii.gz -# rsync -avzh $PATH_DATA_PROCESSED/derivatives/labels/${SUBJECT}/anat/${file_soft}_crop.nii.gz $PATH_DATA_PROCESSED_LESIONSEG/derivatives/labels/${SUBJECT}/anat/${file_soft}.nii.gz -#fi - # Display useful info for the log end=`date +%s` From c96be05db23ea06b00892350aaf6627cb24ee7dc Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Fri, 10 Feb 2023 15:17:56 -0500 Subject: [PATCH 4/6] Fixes bug if ground truth file not present --- preprocessing/qc_preprocess.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/preprocessing/qc_preprocess.py b/preprocessing/qc_preprocess.py index b48629c..c05b2f1 100644 --- a/preprocessing/qc_preprocess.py +++ b/preprocessing/qc_preprocess.py @@ -34,6 +34,7 @@ # Log problematic subjects for QC failed_crop_subjects, shape_mismatch_subjects, left_out_lesion_subjects = [], [], [] +no_groundtruth = [] # Perform QC on each subject for subject in tqdm(subjects, desc='Iterating over Subjects'): @@ -61,27 +62,27 @@ # Read original and cropped subject ground-truths (GT) gt1_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manualNeuroPoly.nii.gz' % subject) + if not os.path.exists(gt1_fpath): + no_groundtruth.append(subject) + continue gt1_crop_fpath = os.path.join(subject_labels_path, '%s_UNIT1_lesion-manualNeuroPoly_crop.nii.gz' % subject) gt1 = nib.load(gt1_fpath) gt1_crop = nib.load(gt1_crop_fpath) - gt2 = nib.load(gt2_fpath) - gt2_crop = nib.load(gt2_crop_fpath) # Basic shape checks - if not img_crop.shape == gt1_crop.shape == gt2_crop.shape: + if not img_crop.shape == gt1_crop.shape: shape_mismatch_subjects.append(subject) continue # Check if the dilated SC mask leaves out any lesions from GTs (from each rater) - if not (np.allclose(np.sum(gt1.get_fdata()), np.sum(gt1_crop.get_fdata())) and - np.allclose(np.sum(gt2.get_fdata()), np.sum(gt2_crop.get_fdata()))): + if not (np.allclose(np.sum(gt1.get_fdata()), np.sum(gt1_crop.get_fdata()))): left_out_lesion_subjects.append(subject) print('RESOLUTIONS: ', Counter(resolutions)) print('SIZES: ', Counter(sizes)) print('CROP SIZES: ', Counter(crop_sizes)) - +print('List of missing ground truth: ', no_groundtruth) print('Could not find cropped image for the following subjects: ', failed_crop_subjects) print('Found shape mismatch in images and GTs for the following subjects: ', shape_mismatch_subjects) print('ALERT: Lesion(s) from raters cropped during preprocessing for the following subjects: ', left_out_lesion_subjects) From 07205bdfa442b75c81ddf8deb2aa74d8dfd209e5 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Fri, 10 Feb 2023 16:51:08 -0500 Subject: [PATCH 5/6] Increased dilation See: https://github.com/ivadomed/model_seg_ms_mp2rage/issues/67 --- preprocessing/preprocess_data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocessing/preprocess_data.sh b/preprocessing/preprocess_data.sh index ca34db9..38835f5 100644 --- a/preprocessing/preprocess_data.sh +++ b/preprocessing/preprocess_data.sh @@ -121,7 +121,7 @@ segment_if_does_not_exist ${file} t1 svm file_seg="${FILESEG}" # Use mask to crop the original image -sct_crop_image -i ${file}.nii.gz -m ${file_seg}.nii.gz -dilate 3 -o ${file}_crop.nii.gz +sct_crop_image -i ${file}.nii.gz -m ${file_seg}.nii.gz -dilate 32x2x32 -o ${file}_crop.nii.gz # Crop the manual MS lesion segmentation sct_crop_image -i $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.nii.gz -m ${file_seg}.nii.gz -dilate 3 -o $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}_crop.nii.gz From 515d35aaa3043cf888a9680ec873fe20cd177c35 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Fri, 10 Feb 2023 17:16:04 -0500 Subject: [PATCH 6/6] Fixed dilation value --- preprocessing/preprocess_data.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/preprocessing/preprocess_data.sh b/preprocessing/preprocess_data.sh index 38835f5..0491019 100644 --- a/preprocessing/preprocess_data.sh +++ b/preprocessing/preprocess_data.sh @@ -120,11 +120,12 @@ fi segment_if_does_not_exist ${file} t1 svm file_seg="${FILESEG}" -# Use mask to crop the original image -sct_crop_image -i ${file}.nii.gz -m ${file_seg}.nii.gz -dilate 32x2x32 -o ${file}_crop.nii.gz +# Use mask to crop the original image +DILATE="32x3x32" +sct_crop_image -i ${file}.nii.gz -m ${file_seg}.nii.gz -dilate ${DILATE} -o ${file}_crop.nii.gz # Crop the manual MS lesion segmentation -sct_crop_image -i $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.nii.gz -m ${file_seg}.nii.gz -dilate 3 -o $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}_crop.nii.gz +sct_crop_image -i $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.nii.gz -m ${file_seg}.nii.gz -dilate ${DILATE} -o $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}_crop.nii.gz # Make sure a JSON file is present, if not create an empty one if [[ ! -s $PATH_DATA_PROCESSED/derivatives/labels/$SUBJECT/anat/${file_gt1}.json ]]; then