Merge pull request #749 from I2PC/release-3.23.11

Release 3.23.11 (Nereus)
I2PC · Nov 28, 2023 · b9eb740 · b9eb740
2 parents c455334 + 33dabd0
commit b9eb740
Show file tree

Hide file tree

Showing 63 changed files with 994 additions and 1,455 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,20 +1,90 @@
-name: Build
+# Name of the GitHub Action
+name: Build scipion-em-xmipp on Pull Request
+
+# Specify when the Action should be triggered: when a pull request is opened against the 'devel' or 'master' branch
 on:
-  push:
-    branches:
-      - master
   pull_request:
-    types: [opened, synchronize, reopened]
+    branches: [devel, master]
+
+# Define the job that should be run
 jobs:
-  sonarcloud:
-    name: SonarCloud
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-      - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+  build:
+    # Specify the machine to run the job on
+    runs-on: ubuntu-22.04
+
+    # Define the steps to be taken in the job
+    steps:    
+    # Installing dependencies
+    - name: Install dependencies 
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libfftw3-dev libopenmpi-dev openmpi-bin libhdf5-dev python3-numpy python3-dev libtiff5-dev unzip libsqlite3-dev default-jdk git cmake libopencv-dev libopenmpi-dev make cmake
+    
+    # Installing CUDA
+    - name: Install CUDA
+      uses: Jimver/cuda-toolkit@v0.2.11
+      id: cuda-toolkit
+      with:
+        cuda: '11.8.0'
+        method: network
+        sub-packages: '["nvcc", "toolkit"]'
+
+    # Installing Miniconda
+    - name: Install Miniconda
+      working-directory: ${{ github.workspace }}/../
+      run: |
+        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+        bash Miniconda3-latest-Linux-x86_64.sh -b -p ${{ github.workspace }}/../miniconda/
+        source ./miniconda/etc/profile.d/conda.sh
+        conda update -n base -c defaults conda -y
+        
+    # Installing Scipion
+    - name: Install Scipion
+      working-directory: ${{ github.workspace }}/../
+      run: |
+        eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)"
+        pip3 install --user scipion-installer
+        python3 -m scipioninstaller -conda -noXmipp -noAsk scipion
+    
+    # Installing Xmipp's Python dependencies in scipion conda enviroment
+    - name: Install scons in scipion enviroment
+      run: |
+        eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)"
+        conda activate scipion3
+        pip install scons
+        
+    # Cloning Xmipp 
+    - name: Cloning Xmipp
+      working-directory: ${{ github.workspace }}/../
+      run: git clone https://github.com/I2PC/xmipp.git
+
+    # Checkout Xmipp to Pull Request branch if exists, by default stays in devel
+    - name: Conditionally checkout Xmipp to ${{ github.head_ref }}
+      working-directory: ${{ github.workspace }}/../xmipp
+      env:
+        BRANCH_NAME: ${{ github.head_ref }}
+      run: |
+        if [ $(git ls-remote --heads https://github.com/I2PC/xmipp.git $BRANCH_NAME | wc -l) -eq 1 ]; then
+          git checkout $BRANCH_NAME
+        fi
+      
+    # Installing Xmipp
+    - name: Compile Xmipp and show log
+      working-directory: ${{ github.workspace }}/../xmipp
+      env:
+        BUILD_TESTS: True
+      run: |
+        ../scipion/scipion3 run ./xmipp || (cat compileLOG.txt && false)
+        cat compileLOG.txt
+    #TODO: Remove last "cat compileLOG.txt" once there is a new Scipion release 
+
+    # Check out the repository in the pull request
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.head_ref }}
+
+    # Install plugin from the pull request using the Scipion3 installp command
+    - name: Install plugin from pull request
+      working-directory: ${{ github.workspace }}
+      run: ../scipion/scipion3 installp -p . --devel --noBin
diff --git a/.github/workflows/buildPlugin.yml b/.github/workflows/buildPlugin.yml
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,30 @@
+## Release 3.23.11 - Nereus
+   - New protocols
+      - Volume local adjustment
+   - Protocols updated
+      - convert_pdb: Allowed conversion natively from CIFs
+      - particle_pick_automatic: The model can now be given by a directory
+      - volume_local_adjust: Save occupancy volume
+      - extract_particles: Added two different cases for downsampling, by dimensions and by sampling rate
+   - Protocols fixed
+      - movie_resize: Fixed movie resize output size
+      - movie_gain: Fix update output step by using a generic one from scipion
+      - tilt_analysis: Fixes in the generated tilt images and in updating correctly the output sets
+      - ctf_consensus: Fix the dependencies of the step
+      - preprocess_micrographs: Fixed output size in preprocess micrographs
+      - deep_center_assignment: Fixed deep center calls
+      - extract_particles_movies: Get coords correctly
+      - particle_pick_consensus: Fix MicsPointer
+      - trigger_data: fix updateOutput and close correclty the output set
+   - Protocols deprecated (For more details visit [this](https://github.com/I2PC/xmipp/wiki/Deprecating-programs-and-protocols))
+      - classification_gpuCorr
+      - classification_gpuCorr_full
+      - classification_gpuCorr_semi
+   - More scipio-em-xmipp
+      - Updated Nvidia driver required version
+
 
-## Release 3.23.07 - 
+## Release 3.23.07 - Morpheus 
    - New protocols
       - Movie Dose analysis
       - deep_center

diff --git a/sonar-project.properties b/sonar-project.properties
@@ -1,13 +1,5 @@
-sonar.projectKey=ScipionEmXmipp
-sonar.organization=i2pc
-
-# This is the name and version displayed in the SonarCloud UI.
-sonar.projectName=Scipion-em-Xmipp
-sonar.projectVersion=3.0
+sonar.python.version=3
 
 # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows.
 sonar.sources=.
 sonar.exlusions=**/legacy/**
-
-# Encoding of the source code. Default is default system encoding
-#sonar.sourceEncoding=UTF-8
diff --git a/xmipp3/__init__.py b/xmipp3/__init__.py
@@ -37,12 +37,15 @@
 from .base import *
 from .constants import XMIPP_HOME, XMIPP_URL, XMIPP_DLTK_NAME
 
-_logo = "xmipp_logo.png"
+type_of_version = 'release' #'devel'
+_logo = "xmipp_logo" + ("" if type_of_version == 'release' else '_devel') + '.png'
+
 _references = ['delaRosaTrevin2013', 'Sorzano2013', 'Strelak2021']
-_currentBinVersion = '3.23.07.0'
+_currentBinVersion = '3.23.11.0'
 __version__ = _currentBinVersion[2:] + ".0"  # Set this to ".0" on each xmipp binary release, otherwise increase it --> ".1", ".2", ...
 
-
+# Requirement version variables
+NVIDIA_DRIVERS_MINIMUM_VERSION = 450
 
 class Plugin(pwem.Plugin):
     _homeVar = XMIPP_HOME
@@ -209,12 +212,12 @@ def installDeepLearningToolkit(plugin, env):
                                                env=plugin.getEnviron(),
                                                stdout=subprocess.PIPE
                                                ).stdout.read().decode('utf-8').split(".")[0]
-            if int(nvidiaDriverVer) < 390:
+            if int(nvidiaDriverVer) < NVIDIA_DRIVERS_MINIMUM_VERSION:
                 preMsgs.append("Incompatible driver %s" % nvidiaDriverVer)
-                cudaMsgs.append("Your NVIDIA drivers are too old (<390). "
+                cudaMsgs.append(f"Your NVIDIA drivers are too old (<{NVIDIA_DRIVERS_MINIMUM_VERSION}). "
                                 "Tensorflow was installed without GPU support. "
                                 "Just CPU computations enabled (slow computations)."
-                                "To enable CUDA (drivers>390 needed), "
+                                f"To enable CUDA (drivers>{NVIDIA_DRIVERS_MINIMUM_VERSION} needed), "
                                 "set CUDA=True in 'scipion.conf' file")
                 nvidiaDriverVer = None
         except (ValueError, TypeError):
@@ -239,7 +242,7 @@ def installDeepLearningToolkit(plugin, env):
     cmdsInstall = list(CondaEnvManager.yieldInstallAllCmds(useGpu=useGpu))
 
     now = datetime.now()
-    installDLvars = {'modelsUrl': "http://scipion.cnb.csic.es/downloads/scipion/software/em",
+    installDLvars = {'modelsUrl': "https://scipion.cnb.csic.es/downloads/scipion/software/em",
                      'syncBin': plugin.getHome('bin/xmipp_sync_data'),
                      'modelsDir': plugin.getHome('models'),
                      'modelsPrefix': "models_UPDATED_on",

diff --git a/xmipp3/bibtex.py b/xmipp3/bibtex.py
@@ -620,5 +620,43 @@
 }
 
 
+@article{Fernandez-Gimenez2023a,
+title = {A new algorithm for particle weighted subtraction to eliminate signals from unwanted components in Single Particle Analysis},
+journal = {Journal of Structural Biology},
+volume = {215},
+number = {4},
+year = {2023},
+issn = {1047-8477},
+doi = {https://doi.org/10.1016/j.jsb.2023.108024},
+url = {https://www.sciencedirect.com/science/article/pii/S1047847723000874},
+author = {E. Fernández-Giménez, M. Martínez, R. Marabini, D. Strelak, R. Sánchez-García, J.M. Carazo and C.O.S. Sorzano},
+}
+
+@article{Fernandez-Gimenez2023b,
+title = {Local defocus estimation in Single Particle Analysis in Cryo-Electron Microscopy},
+journal = {Journal of Structural Biology},
+volume = {215},
+number = {4},
+year = {2023},
+issn = {1047-8477},
+doi = {https://doi.org/10.1016/j.jsb.2023.108030},
+url = {https://www.sciencedirect.com/science/article/pii/S104784772300093X},
+author = {E. Fernández-Giménez, J.M. Carazo and C.O.S. Sorzano},
+
+
+@Article{Strelak2023performance,
+AUTHOR = {Střelák, David and Marchán, Daniel and Carazo, José María and S. Sorzano, Carlos O.},
+TITLE = {Performance and Quality Comparison of Movie Alignment Software for Cryogenic Electron Microscopy},
+JOURNAL = {Micromachines},
+VOLUME = {14},
+YEAR = {2023},
+NUMBER = {10},
+ARTICLE-NUMBER = {1835},
+URL = {https://www.mdpi.com/2072-666X/14/10/1835},
+ISSN = {2072-666X},
+DOI = {https://doi.org/10.3390/mi14101835}
+
+}
+
 """
 
diff --git a/xmipp3/convert/io_coordinates.py b/xmipp3/convert/io_coordinates.py
@@ -17,9 +17,10 @@ def readSetOfCoordsFromPosFnames(posDir, setOfInputCoords, sqliteOutName,
     """
 
     inputMics = setOfInputCoords.getMicrographs()
+    inputMicsPointer = setOfInputCoords.getMicrographs(asPointer=True)
     cleanPath(sqliteOutName)
     setOfOutputCoordinates= SetOfCoordinates(filename= sqliteOutName)
-    setOfOutputCoordinates.setMicrographs(inputMics)
+    setOfOutputCoordinates.setMicrographs(inputMicsPointer)
     setOfOutputCoordinates.setBoxSize(setOfInputCoords.getBoxSize())
     readSetOfCoordinates(posDir, micSet=inputMics,
                            coordSet=setOfOutputCoordinates,

diff --git a/...tocols/protocol_classification_gpuCorr.py → ...tocols/protocol_classification_gpuCorr.py b/...tocols/protocol_classification_gpuCorr.py → ...tocols/protocol_classification_gpuCorr.py
diff --git a/...s/protocol_classification_gpuCorr_full.py → ...s/protocol_classification_gpuCorr_full.py b/...s/protocol_classification_gpuCorr_full.py → ...s/protocol_classification_gpuCorr_full.py
diff --git a/...s/protocol_classification_gpuCorr_semi.py → ...s/protocol_classification_gpuCorr_semi.py b/...s/protocol_classification_gpuCorr_semi.py → ...s/protocol_classification_gpuCorr_semi.py
diff --git a/...ests/test_protocols_gpuCorr_classifier.py → ...ests/test_protocols_gpuCorr_classifier.py b/...ests/test_protocols_gpuCorr_classifier.py → ...ests/test_protocols_gpuCorr_classifier.py
diff --git a/...s/test_protocols_gpuCorr_fullStreaming.py → ...s/test_protocols_gpuCorr_fullStreaming.py b/...s/test_protocols_gpuCorr_fullStreaming.py → ...s/test_protocols_gpuCorr_fullStreaming.py
diff --git a/...s/test_protocols_gpuCorr_semiStreaming.py → ...s/test_protocols_gpuCorr_semiStreaming.py b/...s/test_protocols_gpuCorr_semiStreaming.py → ...s/test_protocols_gpuCorr_semiStreaming.py
diff --git a/xmipp3/protocols.conf b/xmipp3/protocols.conf
@@ -64,9 +64,6 @@ Protocols SPA = [
 		{"tag": "protocol_group", "text": "Classify", "openItem": "False", "children": [
 			{"tag": "protocol", "value": "XmippProtCL2D",   "text": "default"},			
 			{"tag": "protocol", "value": "XmippProtCoreAnalysis",   "text": "default"},
-			{"tag": "protocol", "value": "XmippProtGpuCrrCL2D",   "text": "default"},
-			{"tag": "protocol", "value": "XmippProtStrGpuCrrCL2D",   "text": "default"},
-			{"tag": "protocol", "value": "XmippProtStrGpuCrrSimple",   "text": "default"},
 			{"tag": "section", "text": "more", "openItem": "False", "children": [
 				{"tag": "protocol", "value": "XmippProtML2D", "text": "default"},
 				{"tag": "protocol", "value": "XmippProtKerdensom", "text": "default"}
@@ -157,6 +154,7 @@ Protocols SPA = [
 		    {"tag": "protocol", "value": "XmippProtVolSubtraction", "text": "default"},
 		    {"tag": "protocol", "value": "XmippProtVolAdjust", "text": "default"},
             {"tag": "protocol", "value": "XmippProtVolConsensus", "text": "default"},
+            {"tag": "protocol", "value": "XmippProtLocalVolAdj", "text": "default"},
             {"tag": "protocol", "value": "XmippProtShiftVolume", "text": "default"},
             {"tag": "protocol", "value": "XmippProtSubtractProjection", "text": "default"}
 		]}

diff --git a/xmipp3/protocols/__init__.py b/xmipp3/protocols/__init__.py
@@ -40,9 +40,6 @@
 from .protocol_cl2d_align import XmippProtCL2DAlign
 from .protocol_cl2d import XmippProtCL2D
 #from .protocol_classify_kmeans2d import XmippProtKmeansClassif2D
-from .protocol_classification_gpuCorr import XmippProtGpuCrrCL2D
-from .protocol_classification_gpuCorr_semi import XmippProtStrGpuCrrSimple
-from .protocol_classification_gpuCorr_full import XmippProtStrGpuCrrCL2D
 from .protocol_ctf_defocus_group import XmippProtCTFDefocusGroup
 from .protocol_compare_reprojections import XmippProtCompareReprojections
 from .protocol_compare_angles import XmippProtCompareAngles
@@ -136,9 +133,7 @@
 from .protocol_apply_zernike3d import XmippApplyZernike3D
 from .protocol_volume_adjust_sub import XmippProtVolAdjust, XmippProtVolSubtraction
 from .protocol_volume_consensus import XmippProtVolConsensus
+from .protocol_volume_local_adjust import XmippProtLocalVolAdj
 from .protocol_classes_2d_mapping import XmippProtCL2DMap
 from .protocol_deep_hand import XmippProtDeepHand
-from .protocol_deep_center import XmippProtDeepCenter
-from .protocol_deep_center_predict import XmippProtDeepCenterPredict
-from .protocol_deep_global_assignment import XmippProtDeepGlobalAssignment
-from .protocol_deep_global_assignment_predict import XmippProtDeepGlobalAssignmentPredict
+from .protocol_deep_center_assignment import XmippProtDeepCenter, XmippProtDeepGlobalAssignment
diff --git a/xmipp3/protocols/protocol_align_volume.py b/xmipp3/protocols/protocol_align_volume.py
@@ -52,7 +52,7 @@ class XmippProtAlignVolume(ProtAlignVolume):
      """
     _label = 'align volume'
     nVols = 0
-    _devStatus = UPDATED
+    _devStatus = PROD
 
 
     def __init__(self, **args):