diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3c313d3f..991b4c184 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,20 +1,90 @@ -name: Build +# Name of the GitHub Action +name: Build scipion-em-xmipp on Pull Request + +# Specify when the Action should be triggered: when a pull request is opened against the 'devel' or 'master' branch on: - push: - branches: - - master pull_request: - types: [opened, synchronize, reopened] + branches: [devel, master] + +# Define the job that should be run jobs: - sonarcloud: - name: SonarCloud - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + build: + # Specify the machine to run the job on + runs-on: ubuntu-22.04 + + # Define the steps to be taken in the job + steps: + # Installing dependencies + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libfftw3-dev libopenmpi-dev openmpi-bin libhdf5-dev python3-numpy python3-dev libtiff5-dev unzip libsqlite3-dev default-jdk git cmake libopencv-dev libopenmpi-dev make cmake + + # Installing CUDA + - name: Install CUDA + uses: Jimver/cuda-toolkit@v0.2.11 + id: cuda-toolkit + with: + cuda: '11.8.0' + method: network + sub-packages: '["nvcc", "toolkit"]' + + # Installing Miniconda + - name: Install Miniconda + working-directory: ${{ github.workspace }}/../ + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh -b -p ${{ github.workspace }}/../miniconda/ + source ./miniconda/etc/profile.d/conda.sh + conda update -n base -c defaults conda -y + + # Installing Scipion + - name: Install Scipion + working-directory: ${{ github.workspace }}/../ + run: | + eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)" + pip3 install --user scipion-installer + python3 -m scipioninstaller -conda -noXmipp -noAsk scipion + + # Installing Xmipp's Python dependencies in scipion conda enviroment + - name: Install scons in scipion enviroment + run: | + eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)" + conda activate scipion3 + pip install scons + + # Cloning Xmipp + - name: Cloning Xmipp + working-directory: ${{ github.workspace }}/../ + run: git clone https://github.com/I2PC/xmipp.git + + # Checkout Xmipp to Pull Request branch if exists, by default stays in devel + - name: Conditionally checkout Xmipp to ${{ github.head_ref }} + working-directory: ${{ github.workspace }}/../xmipp + env: + BRANCH_NAME: ${{ github.head_ref }} + run: | + if [ $(git ls-remote --heads https://github.com/I2PC/xmipp.git $BRANCH_NAME | wc -l) -eq 1 ]; then + git checkout $BRANCH_NAME + fi + + # Installing Xmipp + - name: Compile Xmipp and show log + working-directory: ${{ github.workspace }}/../xmipp + env: + BUILD_TESTS: True + run: | + ../scipion/scipion3 run ./xmipp || (cat compileLOG.txt && false) + cat compileLOG.txt + #TODO: Remove last "cat compileLOG.txt" once there is a new Scipion release + + # Check out the repository in the pull request + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + # Install plugin from the pull request using the Scipion3 installp command + - name: Install plugin from pull request + working-directory: ${{ github.workspace }} + run: ../scipion/scipion3 installp -p . --devel --noBin diff --git a/.github/workflows/buildPlugin.yml b/.github/workflows/buildPlugin.yml deleted file mode 100644 index 9f49edd05..000000000 --- a/.github/workflows/buildPlugin.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Name of the GitHub Action -name: Build scipion-em-xmipp on Pull Request - -# Specify when the Action should be triggered: when a pull request is opened against the 'devel' or 'master' branch -on: - pull_request: - branches: [devel, master] - -# Define the job that should be run -jobs: - build: - # Specify the machine to run the job on - runs-on: ubuntu-22.04 - - # Define the steps to be taken in the job - steps: - # Installing dependencies - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libfftw3-dev libopenmpi-dev openmpi-bin libhdf5-dev python3-numpy python3-dev libtiff5-dev unzip libsqlite3-dev default-jdk git cmake libopencv-dev libopenmpi-dev make cmake - - # Installing CUDA - - name: Install CUDA - uses: Jimver/cuda-toolkit@v0.2.10 - id: cuda-toolkit - with: - cuda: '11.7.0' - linux-local-args: '[ "--toolkit" ]' - - # Installing Miniconda - - name: Install Miniconda - working-directory: ${{ github.workspace }}/../ - run: | - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh -b -p ${{ github.workspace }}/../miniconda/ - source ./miniconda/etc/profile.d/conda.sh - conda update -n base -c defaults conda -y - - # Installing Scipion - - name: Install Scipion with dependencies - working-directory: ${{ github.workspace }}/../ - run: | - eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)" - pip3 install --user scipion-installer - python3 -m scipioninstaller -conda -noXmipp -noAsk scipion - - # Installing Xmipp's Python dependencies in scipion conda enviroment - - name: Install scons in scipion enviroment - run: | - eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)" - conda activate scipion3 - pip install scons - - # Cloning Xmipp - - name: Cloning Xmipp - working-directory: ${{ github.workspace }}/../ - run: git clone https://github.com/I2PC/xmipp.git - - # Checkout Xmipp to Pull Request branch if exists, by default stays in devel - - name: Conditionally checkout Xmipp to ${{ github.head_ref }} - working-directory: ${{ github.workspace }}/../xmipp - env: - BRANCH_NAME: ${{ github.head_ref }} - run: | - if [ $(git ls-remote --heads https://github.com/I2PC/xmipp.git $BRANCH_NAME | wc -l) -eq 1 ]; then - git checkout $BRANCH_NAME - fi - - # Installing Xmipp - - name: Compile Xmipp and show log - working-directory: ${{ github.workspace }}/../xmipp - env: - BUILD_TESTS: True - run: | - ../scipion/scipion3 run ./xmipp - cat compileLOG.txt - - # Check out the repository in the pull request - - name: Checkout repository - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - # Install plugin from the pull request using the Scipion3 installp command - - name: Install plugin from pull request - working-directory: ${{ github.workspace }} - run: ../scipion/scipion3 installp -p . --devel --noBin diff --git a/CHANGES.md b/CHANGES.md index d46d88e90..85cb5a49c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,30 @@ +## Release 3.23.11 - Nereus + - New protocols + - Volume local adjustment + - Protocols updated + - convert_pdb: Allowed conversion natively from CIFs + - particle_pick_automatic: The model can now be given by a directory + - volume_local_adjust: Save occupancy volume + - extract_particles: Added two different cases for downsampling, by dimensions and by sampling rate + - Protocols fixed + - movie_resize: Fixed movie resize output size + - movie_gain: Fix update output step by using a generic one from scipion + - tilt_analysis: Fixes in the generated tilt images and in updating correctly the output sets + - ctf_consensus: Fix the dependencies of the step + - preprocess_micrographs: Fixed output size in preprocess micrographs + - deep_center_assignment: Fixed deep center calls + - extract_particles_movies: Get coords correctly + - particle_pick_consensus: Fix MicsPointer + - trigger_data: fix updateOutput and close correclty the output set + - Protocols deprecated (For more details visit [this](https://github.com/I2PC/xmipp/wiki/Deprecating-programs-and-protocols)) + - classification_gpuCorr + - classification_gpuCorr_full + - classification_gpuCorr_semi + - More scipio-em-xmipp + - Updated Nvidia driver required version + -## Release 3.23.07 - +## Release 3.23.07 - Morpheus - New protocols - Movie Dose analysis - deep_center diff --git a/sonar-project.properties b/sonar-project.properties index 45b7ed30c..a242b15e2 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,13 +1,5 @@ -sonar.projectKey=ScipionEmXmipp -sonar.organization=i2pc - -# This is the name and version displayed in the SonarCloud UI. -sonar.projectName=Scipion-em-Xmipp -sonar.projectVersion=3.0 +sonar.python.version=3 # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. sonar.sources=. sonar.exlusions=**/legacy/** - -# Encoding of the source code. Default is default system encoding -#sonar.sourceEncoding=UTF-8 diff --git a/xmipp3/__init__.py b/xmipp3/__init__.py index a9e4f57ff..ec8456111 100644 --- a/xmipp3/__init__.py +++ b/xmipp3/__init__.py @@ -37,12 +37,15 @@ from .base import * from .constants import XMIPP_HOME, XMIPP_URL, XMIPP_DLTK_NAME -_logo = "xmipp_logo.png" +type_of_version = 'release' #'devel' +_logo = "xmipp_logo" + ("" if type_of_version == 'release' else '_devel') + '.png' + _references = ['delaRosaTrevin2013', 'Sorzano2013', 'Strelak2021'] -_currentBinVersion = '3.23.07.0' +_currentBinVersion = '3.23.11.0' __version__ = _currentBinVersion[2:] + ".0" # Set this to ".0" on each xmipp binary release, otherwise increase it --> ".1", ".2", ... - +# Requirement version variables +NVIDIA_DRIVERS_MINIMUM_VERSION = 450 class Plugin(pwem.Plugin): _homeVar = XMIPP_HOME @@ -209,12 +212,12 @@ def installDeepLearningToolkit(plugin, env): env=plugin.getEnviron(), stdout=subprocess.PIPE ).stdout.read().decode('utf-8').split(".")[0] - if int(nvidiaDriverVer) < 390: + if int(nvidiaDriverVer) < NVIDIA_DRIVERS_MINIMUM_VERSION: preMsgs.append("Incompatible driver %s" % nvidiaDriverVer) - cudaMsgs.append("Your NVIDIA drivers are too old (<390). " + cudaMsgs.append(f"Your NVIDIA drivers are too old (<{NVIDIA_DRIVERS_MINIMUM_VERSION}). " "Tensorflow was installed without GPU support. " "Just CPU computations enabled (slow computations)." - "To enable CUDA (drivers>390 needed), " + f"To enable CUDA (drivers>{NVIDIA_DRIVERS_MINIMUM_VERSION} needed), " "set CUDA=True in 'scipion.conf' file") nvidiaDriverVer = None except (ValueError, TypeError): @@ -239,7 +242,7 @@ def installDeepLearningToolkit(plugin, env): cmdsInstall = list(CondaEnvManager.yieldInstallAllCmds(useGpu=useGpu)) now = datetime.now() - installDLvars = {'modelsUrl': "http://scipion.cnb.csic.es/downloads/scipion/software/em", + installDLvars = {'modelsUrl': "https://scipion.cnb.csic.es/downloads/scipion/software/em", 'syncBin': plugin.getHome('bin/xmipp_sync_data'), 'modelsDir': plugin.getHome('models'), 'modelsPrefix': "models_UPDATED_on", diff --git a/xmipp3/bibtex.py b/xmipp3/bibtex.py index c1bca0e78..231f831c5 100644 --- a/xmipp3/bibtex.py +++ b/xmipp3/bibtex.py @@ -620,5 +620,43 @@ } +@article{Fernandez-Gimenez2023a, +title = {A new algorithm for particle weighted subtraction to eliminate signals from unwanted components in Single Particle Analysis}, +journal = {Journal of Structural Biology}, +volume = {215}, +number = {4}, +year = {2023}, +issn = {1047-8477}, +doi = {https://doi.org/10.1016/j.jsb.2023.108024}, +url = {https://www.sciencedirect.com/science/article/pii/S1047847723000874}, +author = {E. Fernández-Giménez, M. Martínez, R. Marabini, D. Strelak, R. Sánchez-García, J.M. Carazo and C.O.S. Sorzano}, +} + +@article{Fernandez-Gimenez2023b, +title = {Local defocus estimation in Single Particle Analysis in Cryo-Electron Microscopy}, +journal = {Journal of Structural Biology}, +volume = {215}, +number = {4}, +year = {2023}, +issn = {1047-8477}, +doi = {https://doi.org/10.1016/j.jsb.2023.108030}, +url = {https://www.sciencedirect.com/science/article/pii/S104784772300093X}, +author = {E. Fernández-Giménez, J.M. Carazo and C.O.S. Sorzano}, + + +@Article{Strelak2023performance, +AUTHOR = {Střelák, David and Marchán, Daniel and Carazo, José María and S. Sorzano, Carlos O.}, +TITLE = {Performance and Quality Comparison of Movie Alignment Software for Cryogenic Electron Microscopy}, +JOURNAL = {Micromachines}, +VOLUME = {14}, +YEAR = {2023}, +NUMBER = {10}, +ARTICLE-NUMBER = {1835}, +URL = {https://www.mdpi.com/2072-666X/14/10/1835}, +ISSN = {2072-666X}, +DOI = {https://doi.org/10.3390/mi14101835} + +} + """ diff --git a/xmipp3/convert/io_coordinates.py b/xmipp3/convert/io_coordinates.py index 1eb323292..b91c6ccd3 100644 --- a/xmipp3/convert/io_coordinates.py +++ b/xmipp3/convert/io_coordinates.py @@ -17,9 +17,10 @@ def readSetOfCoordsFromPosFnames(posDir, setOfInputCoords, sqliteOutName, """ inputMics = setOfInputCoords.getMicrographs() + inputMicsPointer = setOfInputCoords.getMicrographs(asPointer=True) cleanPath(sqliteOutName) setOfOutputCoordinates= SetOfCoordinates(filename= sqliteOutName) - setOfOutputCoordinates.setMicrographs(inputMics) + setOfOutputCoordinates.setMicrographs(inputMicsPointer) setOfOutputCoordinates.setBoxSize(setOfInputCoords.getBoxSize()) readSetOfCoordinates(posDir, micSet=inputMics, coordSet=setOfOutputCoordinates, diff --git a/xmipp3/protocols/protocol_classification_gpuCorr.py b/xmipp3/legacy/protocols/protocol_classification_gpuCorr.py similarity index 100% rename from xmipp3/protocols/protocol_classification_gpuCorr.py rename to xmipp3/legacy/protocols/protocol_classification_gpuCorr.py diff --git a/xmipp3/protocols/protocol_classification_gpuCorr_full.py b/xmipp3/legacy/protocols/protocol_classification_gpuCorr_full.py similarity index 100% rename from xmipp3/protocols/protocol_classification_gpuCorr_full.py rename to xmipp3/legacy/protocols/protocol_classification_gpuCorr_full.py diff --git a/xmipp3/protocols/protocol_classification_gpuCorr_semi.py b/xmipp3/legacy/protocols/protocol_classification_gpuCorr_semi.py similarity index 100% rename from xmipp3/protocols/protocol_classification_gpuCorr_semi.py rename to xmipp3/legacy/protocols/protocol_classification_gpuCorr_semi.py diff --git a/xmipp3/tests/test_protocols_gpuCorr_classifier.py b/xmipp3/legacy/tests/test_protocols_gpuCorr_classifier.py similarity index 100% rename from xmipp3/tests/test_protocols_gpuCorr_classifier.py rename to xmipp3/legacy/tests/test_protocols_gpuCorr_classifier.py diff --git a/xmipp3/tests/test_protocols_gpuCorr_fullStreaming.py b/xmipp3/legacy/tests/test_protocols_gpuCorr_fullStreaming.py similarity index 100% rename from xmipp3/tests/test_protocols_gpuCorr_fullStreaming.py rename to xmipp3/legacy/tests/test_protocols_gpuCorr_fullStreaming.py diff --git a/xmipp3/tests/test_protocols_gpuCorr_semiStreaming.py b/xmipp3/legacy/tests/test_protocols_gpuCorr_semiStreaming.py similarity index 100% rename from xmipp3/tests/test_protocols_gpuCorr_semiStreaming.py rename to xmipp3/legacy/tests/test_protocols_gpuCorr_semiStreaming.py diff --git a/xmipp3/protocols.conf b/xmipp3/protocols.conf index 519d3db6f..1fd80b05b 100644 --- a/xmipp3/protocols.conf +++ b/xmipp3/protocols.conf @@ -64,9 +64,6 @@ Protocols SPA = [ {"tag": "protocol_group", "text": "Classify", "openItem": "False", "children": [ {"tag": "protocol", "value": "XmippProtCL2D", "text": "default"}, {"tag": "protocol", "value": "XmippProtCoreAnalysis", "text": "default"}, - {"tag": "protocol", "value": "XmippProtGpuCrrCL2D", "text": "default"}, - {"tag": "protocol", "value": "XmippProtStrGpuCrrCL2D", "text": "default"}, - {"tag": "protocol", "value": "XmippProtStrGpuCrrSimple", "text": "default"}, {"tag": "section", "text": "more", "openItem": "False", "children": [ {"tag": "protocol", "value": "XmippProtML2D", "text": "default"}, {"tag": "protocol", "value": "XmippProtKerdensom", "text": "default"} @@ -157,6 +154,7 @@ Protocols SPA = [ {"tag": "protocol", "value": "XmippProtVolSubtraction", "text": "default"}, {"tag": "protocol", "value": "XmippProtVolAdjust", "text": "default"}, {"tag": "protocol", "value": "XmippProtVolConsensus", "text": "default"}, + {"tag": "protocol", "value": "XmippProtLocalVolAdj", "text": "default"}, {"tag": "protocol", "value": "XmippProtShiftVolume", "text": "default"}, {"tag": "protocol", "value": "XmippProtSubtractProjection", "text": "default"} ]} diff --git a/xmipp3/protocols/__init__.py b/xmipp3/protocols/__init__.py index 38d254ec8..9c8244e0b 100644 --- a/xmipp3/protocols/__init__.py +++ b/xmipp3/protocols/__init__.py @@ -40,9 +40,6 @@ from .protocol_cl2d_align import XmippProtCL2DAlign from .protocol_cl2d import XmippProtCL2D #from .protocol_classify_kmeans2d import XmippProtKmeansClassif2D -from .protocol_classification_gpuCorr import XmippProtGpuCrrCL2D -from .protocol_classification_gpuCorr_semi import XmippProtStrGpuCrrSimple -from .protocol_classification_gpuCorr_full import XmippProtStrGpuCrrCL2D from .protocol_ctf_defocus_group import XmippProtCTFDefocusGroup from .protocol_compare_reprojections import XmippProtCompareReprojections from .protocol_compare_angles import XmippProtCompareAngles @@ -136,9 +133,7 @@ from .protocol_apply_zernike3d import XmippApplyZernike3D from .protocol_volume_adjust_sub import XmippProtVolAdjust, XmippProtVolSubtraction from .protocol_volume_consensus import XmippProtVolConsensus +from .protocol_volume_local_adjust import XmippProtLocalVolAdj from .protocol_classes_2d_mapping import XmippProtCL2DMap from .protocol_deep_hand import XmippProtDeepHand -from .protocol_deep_center import XmippProtDeepCenter -from .protocol_deep_center_predict import XmippProtDeepCenterPredict -from .protocol_deep_global_assignment import XmippProtDeepGlobalAssignment -from .protocol_deep_global_assignment_predict import XmippProtDeepGlobalAssignmentPredict +from .protocol_deep_center_assignment import XmippProtDeepCenter, XmippProtDeepGlobalAssignment diff --git a/xmipp3/protocols/protocol_align_volume.py b/xmipp3/protocols/protocol_align_volume.py index 6c03ebbb5..3029cef51 100644 --- a/xmipp3/protocols/protocol_align_volume.py +++ b/xmipp3/protocols/protocol_align_volume.py @@ -52,7 +52,7 @@ class XmippProtAlignVolume(ProtAlignVolume): """ _label = 'align volume' nVols = 0 - _devStatus = UPDATED + _devStatus = PROD def __init__(self, **args): diff --git a/xmipp3/protocols/protocol_analyze_local_ctf.py b/xmipp3/protocols/protocol_analyze_local_ctf.py index 928545bf6..5d52796d2 100644 --- a/xmipp3/protocols/protocol_analyze_local_ctf.py +++ b/xmipp3/protocols/protocol_analyze_local_ctf.py @@ -34,6 +34,8 @@ import pwem.emlib.metadata as md from xmipp3.convert import readSetOfMicrographs, writeSetOfMicrographs +CITE ='Fernandez-Gimenez2023b' + class XmippProtAnalyzeLocalCTF(ProtAnalysis3D): """Assigns to each micrograph a coefficient (R2) which evaluates the result of the diff --git a/xmipp3/protocols/protocol_consensus_classes.py b/xmipp3/protocols/protocol_consensus_classes.py index 7434c7d6a..d52195dc2 100644 --- a/xmipp3/protocols/protocol_consensus_classes.py +++ b/xmipp3/protocols/protocol_consensus_classes.py @@ -164,18 +164,6 @@ def convert_index(i: int) -> int: self._writeElbows(elbows) # --------------------------- INFO functions ------------------------------- - """ - def _validate(self): - errors = [] - - # Ensure that all classifications are made of the same images - items = self._getInputImages(0) - for i in range(1, len(self.inputClassifications)): - if self._getInputImages(i) != items: - errors.append(f'Classification {i} has been done with different images') - - return errors - """ # --------------------------- UTILS functions ------------------------------ def _getInputClassificationCount(self) -> int: @@ -479,7 +467,6 @@ def cost_fun(i: int) -> float: mu2 = np.mean(d2) sigma1 = np.std(d1, ddof=1) sigma2 = np.std(d2, ddof=1) - #sigma = (len(d1)*sigma1 + len(d2)*sigma2) / (len(d1) + len(d2)) # Weighted average # Compute the log likelihood logLikelihoods1 = scipy.stats.norm.logpdf(d1, mu1, sigma1) diff --git a/xmipp3/protocols/protocol_consensus_local_ctf.py b/xmipp3/protocols/protocol_consensus_local_ctf.py index 0d9326e2e..6e3eadd18 100644 --- a/xmipp3/protocols/protocol_consensus_local_ctf.py +++ b/xmipp3/protocols/protocol_consensus_local_ctf.py @@ -40,11 +40,12 @@ from pyworkflow import BETA, UPDATED, NEW, PROD OUTPUTNAME = "outputParticles" +CITE ='Fernandez-Gimenez2023b' class XmippProtConsensusLocalCTF(ProtAnalysis3D): """This protocol compares the estimations of local defocus computed by different protocols for a set of particles""" _label = 'consensus local defocus' - _devStatus = UPDATED + _devStatus = PROD _lastUpdateVersion = VERSION_2_0 _possibleOutputs = {OUTPUTNAME:SetOfParticles} diff --git a/xmipp3/protocols/protocol_convert_pdb.py b/xmipp3/protocols/protocol_convert_pdb.py index c9f271828..0697299ce 100644 --- a/xmipp3/protocols/protocol_convert_pdb.py +++ b/xmipp3/protocols/protocol_convert_pdb.py @@ -28,6 +28,9 @@ # * # ************************************************************************** +# General imports +import os + # Scipion em imports from pwem.convert.headers import setMRCSamplingRate from pwem.emlib.image import ImageHandler @@ -37,7 +40,7 @@ import pyworkflow.protocol.params as params import pyworkflow.protocol.constants as const from pyworkflow.utils import replaceBaseExt, removeExt, getExt, createLink, replaceExt, removeBaseExt -from pyworkflow import BETA, UPDATED, NEW, PROD +from pyworkflow import UPDATED class XmippProtConvertPdb(ProtInitialVolume): """ Convert atomic structure(s) into volume(s) """ @@ -70,7 +73,7 @@ def _defineParams(self, form): coordsCondition = 'setSize and not vol' # Defining parallel arguments - form.addParallelSection(threads=4) + form.addParallelSection(threads=4, mpi=1) # Generating form form.addSection(label='Input') @@ -102,12 +105,16 @@ def _defineParams(self, form): form.addParam('centerPdb', params.BooleanParam, default=True, expertLevel=const.LEVEL_ADVANCED, label="Center PDB", - help='Center PDB with the center of mass') + help='Center PDB with the center of mass.') form.addParam('outPdb', params.BooleanParam, default=False, expertLevel=const.LEVEL_ADVANCED, label="Store centered PDB", help='Set to \'Yes\' if you want to save centered PDB. ' - 'It will be stored in the output directory of this protocol') + 'It will be stored in the output directory of this protocol.') + form.addParam('convertCif', params.BooleanParam, default=True, + expertLevel=const.LEVEL_ADVANCED, + label="Convert CIF to PDB", + help='If set to true and input atom struct file is a CIF, it will get converted to PDB.') form.addParam('clean', params.BooleanParam, default=True, expertLevel=const.LEVEL_ADVANCED, label="Clean tmp files", @@ -138,8 +145,8 @@ def _insertAllSteps(self): # --------------------------- STEPS functions -------------------------------------------- def processConversion(self, pdb, samplingR, isSet): """ This step runs the pdb conversion. """ - # Converting all input atomic structures to .pdb - pdb = self._convertToPdb(pdb) + # Conditionally converting all input atomic structures to .pdb + pdb = self._convertAtomStruct(pdb) # Generating output file for each input outFile = removeExt(pdb) @@ -260,13 +267,14 @@ def _getPdbFileNames(self): # If it is a SetOfAtom Structs, get all of the elements iterating the set return [i.getFileName() for i in pbdObj] - def _convertToPdb(self, pdbRaw): - """ This function receives an atomic struct file, and converts it to .pdb if it is in .cif format. """ - # Get output path for pdb file - convertedPdb = self._getExtraPath(replaceBaseExt(pdbRaw, 'pdb')).replace(" ", "_") + def _convertAtomStruct(self, pdbRaw): + """ This function receives an atomic struct file, and conditionally converts it to .pdb if it is in .cif format. """ + # Get output path for atomic struct file + baseExt = replaceBaseExt(pdbRaw, 'pdb') if self.convertCif.get() else os.path.basename(pdbRaw) + convertedPdb = self._getExtraPath(baseExt).replace(" ", "_") - # If file extension is .cif, convert to .pdb, or else we link it as is - if getExt(pdbRaw) == ".cif": + # If file extension is .cif and conversion is required, convert to .pdb, or else we link it as is + if getExt(pdbRaw) == ".cif" and self.convertCif.get(): cifToPdb(pdbRaw, convertedPdb) else: createLink(pdbRaw, convertedPdb) diff --git a/xmipp3/protocols/protocol_create_gallery.py b/xmipp3/protocols/protocol_create_gallery.py index 781fb87d6..2f4b4b9d5 100644 --- a/xmipp3/protocols/protocol_create_gallery.py +++ b/xmipp3/protocols/protocol_create_gallery.py @@ -51,7 +51,7 @@ def _defineParams(self, form): form.addParam('symmetryGroup', StringParam, default="c1", label='Symmetry group', help='See' - 'http://xmipp.cnb.uam.es/twiki/bin/view/Xmipp/Symmetry ' + 'https://github.com/I2PC/xmipp-portal/wiki/Symmetry ' 'for a description of the symmetry groups format. ' 'If no symmetry is present, give c1') diff --git a/xmipp3/protocols/protocol_ctf_consensus.py b/xmipp3/protocols/protocol_ctf_consensus.py index eb153012a..f9f327233 100644 --- a/xmipp3/protocols/protocol_ctf_consensus.py +++ b/xmipp3/protocols/protocol_ctf_consensus.py @@ -59,7 +59,7 @@ class XmippProtCTFConsensus(ProtCTFMicrographs): the agreement with a secondary CTF for the same set of micrographs. """ _label = 'ctf consensus' - _devStatus = UPDATED + _devStatus = PROD _lastUpdateVersion = VERSION_3_0 def __init__(self, **args): @@ -200,12 +200,11 @@ def _insertAllSteps(self): self.ctfFn2 = self.inputCTF2.get().getFileName() self.allCtf2 = {} - ctfSteps = self._checkNewInput() self._insertFunctionStep('createOutputStep', - prerequisites=ctfSteps, wait=True) + prerequisites=[], wait=True) def createOutputStep(self): - pass + self._closeOutputSet() def initializeParams(self): self.finished = False diff --git a/xmipp3/protocols/protocol_deep_center.py b/xmipp3/protocols/protocol_deep_center.py deleted file mode 100644 index 377075011..000000000 --- a/xmipp3/protocols/protocol_deep_center.py +++ /dev/null @@ -1,174 +0,0 @@ -# ************************************************************************** -# * -# * Authors: COS Sorzano and Adrian Sansinena -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow import VERSION_3_0 -from pyworkflow.protocol import STEPS_PARALLEL -from pyworkflow.protocol.params import (PointerParam, StringParam, FloatParam, EnumParam, - IntParam, BooleanParam, GPU_LIST) -from pyworkflow.protocol.constants import LEVEL_ADVANCED -from pyworkflow.utils.path import moveFile, cleanPattern -from pwem.protocols import ProtAlign2D -from pwem.emlib.metadata import iterRows, getFirstRow -import pwem.emlib.metadata as md -from xmipp3.convert import createItemMatrix, setXmippAttributes, readSetOfParticles, writeSetOfParticles -from pwem import ALIGN_PROJ -from pwem import emlib -from pwem.emlib.image import ImageHandler -import os -import sys -import numpy as np -import math -from shutil import copy -from os import remove -from os.path import exists, join -import xmipp3 -from pyworkflow import BETA, UPDATED, NEW, PROD - -class XmippProtDeepCenter(ProtAlign2D, xmipp3.XmippProtocol): - """Learns a model to center particles using deep learning. Particles must be previously centered with respect - to a volume, and they must have 3D alignment information. """ - _label = 'deep center' - _lastUpdateVersion = VERSION_3_0 - _devStatus = BETA - _conda_env = 'xmipp_DLTK_v1.0' - _cond_modelPretrainTrue = 'modelPretrain==True' - _cond_modelPretrainFalse = 'modelPretrain==False' - - - def __init__(self, **args): - ProtAlign2D.__init__(self, **args) - # --------------------------- DEFINE param functions -------------------------------------------- - def _defineParams(self, form): - form.addHidden(GPU_LIST, StringParam, default='0', - expertLevel=LEVEL_ADVANCED, - label="Choose GPU IDs", - help="GPU may have several cores. Set it to zero" - " if you do not know what we are talking about." - " First core index is 0, second 1 and so on.") - form.addSection(label='Input') - form.addParam('inputTrainSet', PointerParam, label="Input training set", - pointerClass='SetOfParticles', - pointerCondition='hasAlignment2D or hasAlignmentProj', - help='The set of particles previously aligned to be used as training set') - - form.addParam('Xdim', IntParam, label="Size of the images for training", default=128) - - form.addParam('modelPretrain', BooleanParam, default=False, - label='Choose if you want to use a pretrained model', - help='Set "yes" if you want to use a previously trained model. ' - 'If you choose "no" new models will be trained.') - - form.addParam('pretrainedModels', PointerParam, - pointerClass='XmippProtDeepCenter', - condition=self._cond_modelPretrainTrue, - label='Pretrained model', - help='Select the pretrained model. ') - - form.addSection(label='Training parameters') - form.addParam('numCenModels', IntParam, - label="Number of models for particle centering", default=5, - help="Choose number of models you want to train. More than 1 is recommended only if next step " - "is inference.") - - form.addParam('numEpochs_cen', IntParam, - label="Number of epochs", - default=25, expertLevel=LEVEL_ADVANCED, - help="Number of epochs for training.") - - form.addParam('batchSize', IntParam, - label="Batch size for training", - default=32, expertLevel=LEVEL_ADVANCED, - help="Batch size for training.") - - form.addParam('learningRate', FloatParam, - label="Learning rate", - default=0.001, expertLevel=LEVEL_ADVANCED, - help="Learning rate for training.") - - form.addParam('sigma', FloatParam, - label="Image shifting", - default=10, expertLevel=LEVEL_ADVANCED, - help="A measure of the number of pixels that particles can be shifted in each direction from the center.") - - form.addParam('patience', IntParam, - label="Patience", - default=5, expertLevel=LEVEL_ADVANCED, - help="Training will be stopped if the number of epochs without improvement is greater than " - "patience.") - - form.addParallelSection(threads=1, mpi=1) - - # --------------------------- INSERT steps functions -------------------------------------------- - def _insertAllSteps(self): - self.trainImgsFn = self._getExtraPath('train_input_imgs.xmd') - if self.useQueueForSteps() or self.useQueue(): - myStr = os.environ["CUDA_VISIBLE_DEVICES"] - else: - myStr = self.gpuList.get() - os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() - numGPU = myStr.split(',') - - self._insertFunctionStep("convertStep") - self._insertFunctionStep("train", numGPU[0]) - - # --------------------------- STEPS functions --------------------------------------------------- - def convertStep(self): - writeSetOfParticles(self.inputTrainSet.get(), self.trainImgsFn) - self.runJob("xmipp_image_resize", - "-i %s -o %s --save_metadata_stack %s --fourier %d" % - (self.trainImgsFn, - self._getExtraPath('trainingResized.stk'), - self._getExtraPath('trainingResized.xmd'), - self.Xdim), numberOfMpi=self.numberOfThreads.get()*self.numberOfMpi.get()) - - def train(self, gpuId): - - sig = self.sigma.get() - - self.pretrained = 'no' - self.pathToModel = 'none' - if self.modelPretrain: - self.model = self.pretrainedModels.get() - self.pretrained = 'yes' - self.pathToModel = self.model._getExtraPath("modelCenter0.h5") - - args = "%s %s %f %d %d %s %d %f %d %s %s" % ( - self._getExtraPath("trainingResized.xmd"), self._getExtraPath("modelCenter"), sig, - self.numEpochs_cen, self.batchSize.get(), gpuId, self.numCenModels.get(), self.learningRate.get(), - self.patience.get(), self.pretrained, self.pathToModel) - print(args) - self.runJob("xmipp_deep_center", args, numberOfMpi=1, env=self.getCondaEnv()) - - remove(self._getExtraPath("trainingResized.xmd")) - remove(self._getExtraPath("trainingResized.stk")) - - # --------------------------- INFO functions -------------------------------- - def _methods(self): - methods = [] - if hasattr(self, 'outputParticles'): - methods.append("We learned a model to center particles from %i input images (%s)." \ - % (self.inputSet.get().getSize(), self.getObjectTag('inputSet'))) - return methods diff --git a/xmipp3/protocols/protocol_deep_center_assignment.py b/xmipp3/protocols/protocol_deep_center_assignment.py new file mode 100644 index 000000000..2cfe2d4bd --- /dev/null +++ b/xmipp3/protocols/protocol_deep_center_assignment.py @@ -0,0 +1,261 @@ +# ************************************************************************** +# * +# * Authors: COS Sorzano, Erney Ramirez and Adrian Sansinena +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 2 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ************************************************************************** + +from pyworkflow import VERSION_3_0 +from pyworkflow.protocol.params import (PointerParam, StringParam, FloatParam, + IntParam, BooleanParam, GPU_LIST) +from pyworkflow.protocol.constants import LEVEL_ADVANCED +from pyworkflow.utils import Message +from pwem.protocols import ProtAlign2D +from pwem.objects import String +from xmipp3.convert import readSetOfParticles, writeSetOfParticles +import os +from os import remove +import xmipp3 +from xmipp_base import XmippScript +from pyworkflow import BETA, UPDATED, NEW, PROD + +class XmippProtDeepCenterAssignmentPredictBase(ProtAlign2D, xmipp3.XmippProtocol): + """Super protocol of Deep Center and Deep Global Assignment.""" + _lastUpdateVersion = VERSION_3_0 + _conda_env = 'xmipp_DLTK_v1.0' + _trainingResizedFileName = 'trainingResized' + + def __init__(self, **args): + ProtAlign2D.__init__(self, **args) + + # --------------------------- DEFINE param functions -------------------------------------------- + def _defineParams(self, form): + form.addParallelSection(threads=1, mpi=1) + + form.addHidden(GPU_LIST, StringParam, default='0', + expertLevel=LEVEL_ADVANCED, + label="Choose GPU IDs", + help="GPU may have several cores. Set it to zero" + " if you do not know what we are talking about." + " First core index is 0, second 1 and so on.") + + form.addSection(label=Message.LABEL_INPUT) + + form.addParam('inputImageSet', PointerParam, label="Input Image set", + pointerClass='SetOfParticles', + help='The set of particles to predict') + + form.addParam('trainModels', BooleanParam, label="Train models", + pointerClass='SetOfParticles', default=False, + help='Choose if you want to train a model using a centered set of particles') + + trainingGroup = form.addGroup('Training parameters', condition='trainModels==True') + + trainingGroup.addParam('numModels', IntParam, + label="Number of models", default=5, + help="The maximum number of model available in xmipp is 5.") + + trainingGroup.addParam('inputTrainSet', PointerParam, label="Input train set", + pointerClass='SetOfParticles', allowsNull=True, + pointerCondition='hasAlignment2D or hasAlignmentProj', + help='The set of particles to train the models') + + trainingGroup.addParam('numEpochs', IntParam, + label="Number of epochs", + default=25, + expertLevel=LEVEL_ADVANCED, + help="Number of epochs for training.") + + trainingGroup.addParam('batchSize', IntParam, + label="Batch size for training", + default=32, + expertLevel=LEVEL_ADVANCED, + help="Batch size for training.") + + trainingGroup.addParam('learningRate', FloatParam, + label="Learning rate", + default=0.001, + expertLevel=LEVEL_ADVANCED, + help="Learning rate for training.") + + trainingGroup.addParam('sigma', FloatParam, + label="Image shifting", + default=5, + expertLevel=LEVEL_ADVANCED, + help="A measure of the number of pixels that particles can be shifted in each direction from the center.") + + trainingGroup.addParam('patience', IntParam, + label="Patience", + default=5, + expertLevel=LEVEL_ADVANCED, + help="Training will be stopped if the number of epochs without improvement is greater than " + "patience.") + + form.addSection(label='Consensus') + + form.addParam('tolerance', IntParam, + label="Tolerance in pixels", default=3, + help="Max difference between predictions and their mean value.") + + form.addParam('maxModels', IntParam, + label="Maximum number of models dropped per particle", default=0, + help="If more models are dropped, the particle is discarded.") + + # --------------------------- INSERT steps functions -------------------------------------------- + def _insertAllSteps(self): + self.predictImgsFn = self._getExtraPath('predict_input_imgs.xmd') + if self.useQueueForSteps() or self.useQueue(): + myStr = os.environ["CUDA_VISIBLE_DEVICES"] + else: + myStr = self.gpuList.get() + os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() + numGPU = myStr.split(',') + self._insertFunctionStep("convertStep", self.inputImageSet.get()) + self._insertFunctionStep("predict", numGPU[0], self.predictImgsFn) + self._insertFunctionStep('createOutputStep') + + def insertTrainSteps(self): + self.trainImgsFn = self._getExtraPath('train_input_imgs.xmd') + if self.useQueueForSteps() or self.useQueue(): + myStr = os.environ["CUDA_VISIBLE_DEVICES"] + else: + myStr = self.gpuList.get() + os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() + numGPU = myStr.split(',') + + self._insertFunctionStep("convertTrainStep") + self._insertFunctionStep("train", numGPU[0]) + + # --------------------------- STEPS functions --------------------------------------------------- + def convertStep(self, inputSet): + self.Xdim = 128 + writeSetOfParticles(inputSet, self.predictImgsFn) + self.runJob("xmipp_image_resize", + "-i %s -o %s --save_metadata_stack %s --fourier %d" % + (self.predictImgsFn, + self._getExtraPath(f'{self._trainingResizedFileName}.stk'), + self._getExtraPath(f'{self._trainingResizedFileName}.xmd'), + self.Xdim), numberOfMpi=self.numberOfThreads.get() * self.numberOfMpi.get()) + + def convertTrainStep(self): + self.convertStep(self.inputTrainSet.get()) + + def train(self, gpuId, mode="", orderSymmetry=None): + args = "%s %s %f %d %d %s %d %f %d %d" % ( + self._getExtraPath(f"{self._trainingResizedFileName}.xmd"), self._getExtraPath("model"), self.sigma.get(), + self.numEpochs, self.batchSize.get(), gpuId, self.numModels.get(), self.learningRate.get(), + self.patience.get(), 0) + if orderSymmetry: + args += " " + str(orderSymmetry) + self.runJob(f"xmipp_deep_{mode}", args, numberOfMpi=1, env=self.getCondaEnv()) + + remove(self._getExtraPath(f"{self._trainingResizedFileName}.xmd")) + remove(self._getExtraPath(f"{self._trainingResizedFileName}.stk")) + + def predict(self, predictImgsFn, gpuId, mode="", inputModel="", trainedModel=True, orderSymmetry=None): + if mode != "center" or trainedModel: + args = "%s %s %s %s %s %d %d %d" % ( + self._getExtraPath(f"{self._trainingResizedFileName}.xmd"), gpuId, self._getPath(), predictImgsFn, + inputModel, self.numModels.get(), self.tolerance.get(), + self.maxModels.get()) + if orderSymmetry: + args += " " + str(orderSymmetry) + else: + args = "%s %s %s %s %s %d %d %d" % ( + self._getExtraPath(f"{self._trainingResizedFileName}.xmd"), gpuId, self._getPath(), predictImgsFn, + os.path.join(XmippScript.getModel("deep_center"), 'modelCenter'), self.numModels.get(), self.tolerance.get(), + self.maxModels.get()) + self.runJob(f"xmipp_deep_{mode}_predict", args, numberOfMpi=1, env=self.getCondaEnv()) + remove(self._getExtraPath(f"{self._trainingResizedFileName}.xmd")) + remove(self._getExtraPath(f"{self._trainingResizedFileName}.stk")) + + def createOutputStep(self): + imgFname = self._getPath('predict_results.xmd') + outputSet = self._createSetOfParticles() + readSetOfParticles(imgFname, outputSet) + outputSet.copyInfo(self.inputImageSet.get()) + outputSet.setAlignmentProj() + self._defineOutputs(outputParticles=outputSet) + self._store(outputSet) + self._defineSourceRelation(self.inputImageSet.get(), outputSet) + + # --------------------------- INFO functions -------------------------------- + def _methods(self): + methods = [] + if hasattr(self, 'outputParticles'): + methods.append("We learned a model to center particles from %i input images (%s)." \ + % (self.inputSet.get().getSize(), self.getObjectTag('inputSet'))) + return methods + +class XmippProtDeepCenter(XmippProtDeepCenterAssignmentPredictBase): + """Predict the center particles using deep learning.""" + _label = 'deep center' + _devStatus = BETA + # --------------------------- INSERT steps functions -------------------------------------------- + def _insertAllSteps(self): + if self.trainModels.get(): + self.insertTrainSteps() + super()._insertAllSteps() + + # --------------------------- STEPS functions --------------------------------------------------- + def train(self, gpuId, mode="", orderSymmetry=None): + super().train(gpuId, mode="center") + + def predict(self, predictImgsFn, gpuId, mode="", inputModel="", trainedModel=True, orderSymmetry=None): + super().predict(gpuId, predictImgsFn, mode="center", + inputModel=self._getExtraPath("model"), + trainedModel=self.trainModels.get()) + +class XmippProtDeepGlobalAssignment(XmippProtDeepCenterAssignmentPredictBase): + """Predict Euler Angles using deep learning.""" + _label = 'deep global assignment' + _devStatus = BETA + + # --------------------------- DEFINE param functions -------------------------------------------- + def _defineParams(self, form): + # Calling parent form + super()._defineParams(form) + + # Adding extra conditions + trainModels = form.getParam('trainModels') + trainModels.condition = String('False') + + # Always show training parameters group in this protocol + trainingGroup = form.getParam('Training_parameters') + trainingGroup.condition = String('True') + + section = form.getSection(label=Message.LABEL_INPUT) + section.addParam('orderSymmetry', IntParam, + label="Order of symmetry", default=1, + help="Order of the group of the molecule.") + + # --------------------------- INSERT steps functions -------------------------------------------- + def _insertAllSteps(self): + self.insertTrainSteps() + super()._insertAllSteps() + + # --------------------------- STEPS functions --------------------------------------------------- + def train(self, gpuId, mode="", orderSymmetry=None): + super().train(gpuId, mode="global_assignment", orderSymmetry=self.orderSymmetry.get()) + + def predict(self, predictImgsFn, gpuId, mode="", inputModel="", trainedModel=True, orderSymmetry=None): + super().predict(gpuId, predictImgsFn, mode="global_assignment", inputModel=self._getExtraPath("model"), orderSymmetry=self.orderSymmetry.get()) diff --git a/xmipp3/protocols/protocol_deep_center_predict.py b/xmipp3/protocols/protocol_deep_center_predict.py deleted file mode 100644 index 23b3b4a02..000000000 --- a/xmipp3/protocols/protocol_deep_center_predict.py +++ /dev/null @@ -1,158 +0,0 @@ -# ************************************************************************** -# * -# * Authors: COS Sorzano, Erney Ramirez and Adrian Sansinena -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow import VERSION_3_0 -from pyworkflow.protocol import STEPS_PARALLEL -from pyworkflow.protocol.params import (PointerParam, StringParam, FloatParam, - IntParam, BooleanParam, GPU_LIST, EnumParam) -from pyworkflow.protocol.constants import LEVEL_ADVANCED -from pyworkflow.utils.path import moveFile, cleanPattern -from pwem.protocols import ProtAlign2D -from pwem.emlib.metadata import iterRows, getFirstRow -import pwem.emlib.metadata as md -from xmipp3.convert import createItemMatrix, setXmippAttributes, readSetOfParticles, writeSetOfParticles -from pwem import ALIGN_PROJ -from pwem import emlib -from pwem.emlib.image import ImageHandler -import os -import sys -import numpy as np -import math -from shutil import copy -from os import remove -from os.path import exists, join -import xmipp3 -from pyworkflow import BETA, UPDATED, NEW, PROD - -class XmippProtDeepCenterPredict(ProtAlign2D, xmipp3.XmippProtocol): - """Predict the center particles using deep learning.""" - _label = 'deep center predict' - _lastUpdateVersion = VERSION_3_0 - _devStatus = BETA - - _conda_env = 'xmipp_DLTK_v1.0' - _cond_predictAnglesTrue = 'predictAngles==True' - _cond_predictAnglesFalse = 'predictAngles==False' - _cond_trainedModelTrue = 'trainedModel==True' - _cond_trainedModelFalse = 'trainedModel==False' - _cond_needModel = '(trainedModel==True) or (predictAngles==True)' - - def __init__(self, **args): - ProtAlign2D.__init__(self, **args) - - # --------------------------- DEFINE param functions -------------------------------------------- - def _defineParams(self, form): - - form.addHidden(GPU_LIST, StringParam, default='0', - expertLevel=LEVEL_ADVANCED, - label="Choose GPU IDs", - help="GPU may have several cores. Set it to zero" - " if you do not know what we are talking about." - " First core index is 0, second 1 and so on.") - - form.addSection(label='Input') - - form.addParam('inputImageSet', PointerParam, label="Input Image set", - pointerClass='SetOfParticles', - help='The set of particles to predict shift') - - form.addParam('Xdim', IntParam, label="Size of the images for training", default=128) - - form.addParam('inputModel', PointerParam, label="Model trained", - pointerClass='XmippProtDeepCenter', - help='The model to predict angles') - - form.addSection(label='Consensus') - - form.addParam('numModels', IntParam, - label="Number of models trained", default=5) - - form.addParam('tolerance', IntParam, - label="Tolerance in pixels", default=3, - help="Max difference between predictions and their mean value.") - - form.addParam('maxModels', IntParam, - label="Maximum number of models dropped per particle", default=0, - help="If more models are dropped, the particle is discarded.") - - form.addParallelSection(threads=1, mpi=1) - - # --------------------------- INSERT steps functions -------------------------------------------- - def _insertAllSteps(self): - self.predictImgsFn = self._getExtraPath('predict_input_imgs.xmd') - - if self.useQueueForSteps() or self.useQueue(): - myStr = os.environ["CUDA_VISIBLE_DEVICES"] - else: - myStr = self.gpuList.get() - os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() - numGPU = myStr.split(',') - - self._insertFunctionStep("convertStep") - self._insertFunctionStep("predict", numGPU[0]) - self._insertFunctionStep('createOutputStep') - - # --------------------------- STEPS functions --------------------------------------------------- - def convertStep(self): - writeSetOfParticles(self.inputImageSet.get(), self.predictImgsFn) - self.runJob("xmipp_image_resize", - "-i %s -o %s --save_metadata_stack %s --fourier %d" % - (self.predictImgsFn, - self._getExtraPath('trainingResized.stk'), - self._getExtraPath('trainingResized.xmd'), - self.Xdim), numberOfMpi=self.numberOfThreads.get()*self.numberOfMpi.get()) - - def predict(self, gpuId): - self.modelAng = self.inputModel.get() - args = "%s %s %s %s %s %d %d %d" % ( - self._getExtraPath("trainingResized.xmd"), gpuId, self._getPath(), self.predictImgsFn, - self.modelAng._getExtraPath("modelCenter"), self.numModels.get(), self.tolerance.get(), - self.maxModels.get()) - - self.runJob("xmipp_deep_center_predict", args, numberOfMpi=1, env=self.getCondaEnv()) - - remove(self._getExtraPath("trainingResized.xmd")) - remove(self._getExtraPath("trainingResized.stk")) - - def createOutputStep(self): - imgFname = self._getPath('predict_results.xmd') - outputSet = self._createSetOfParticles() - - readSetOfParticles(imgFname, outputSet) - outputSet.copyInfo(self.inputImageSet.get()) - outputSet.setAlignmentProj() - - self._defineOutputs(outputParticles=outputSet) - self._store(outputSet) - self._defineSourceRelation(self.inputImageSet.get(), outputSet) - - # --------------------------- INFO functions -------------------------------- - def _methods(self): - methods = [] - if hasattr(self, 'outputParticles'): - methods.append("We learned a model to center particles from %i input images (%s)." \ - % (self.inputSet.get().getSize(), self.getObjectTag('inputSet'))) - return methods diff --git a/xmipp3/protocols/protocol_deep_global_assignment.py b/xmipp3/protocols/protocol_deep_global_assignment.py deleted file mode 100644 index eeda8ea02..000000000 --- a/xmipp3/protocols/protocol_deep_global_assignment.py +++ /dev/null @@ -1,176 +0,0 @@ -# ************************************************************************** -# * -# * Authors: COS Sorzano and Adrian Sansinena -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow import VERSION_3_0 -from pyworkflow.protocol import STEPS_PARALLEL -from pyworkflow.protocol.params import (PointerParam, StringParam, FloatParam, EnumParam, - IntParam, BooleanParam, GPU_LIST) -from pyworkflow.protocol.constants import LEVEL_ADVANCED -from pyworkflow.utils.path import moveFile, cleanPattern -from pwem.protocols import ProtAlign2D -from pwem.emlib.metadata import iterRows, getFirstRow -import pwem.emlib.metadata as md -from xmipp3.convert import createItemMatrix, setXmippAttributes, readSetOfParticles, writeSetOfParticles -from pwem import ALIGN_PROJ -from pwem import emlib -from pwem.emlib.image import ImageHandler -import os -import sys -import numpy as np -import math -from shutil import copy -from os import remove -from os.path import exists, join -import xmipp3 -from pyworkflow import BETA, UPDATED, NEW, PROD - - -class XmippProtDeepGlobalAssignment(ProtAlign2D, xmipp3.XmippProtocol): - """Learns a model to assign angles to particles using deep learning. Particles must be previously centered with respect - to a volume, and they must have 3D alignment information. """ - _label = 'deep global assignment' - _lastUpdateVersion = VERSION_3_0 - _devStatus = BETA - - _conda_env = 'xmipp_DLTK_v1.0' - _cond_modelPretrainTrue = 'modelPretrain==True' - _cond_modelPretrainFalse = 'modelPretrain==False' - - def __init__(self, **args): - ProtAlign2D.__init__(self, **args) - - # --------------------------- DEFINE param functions -------------------------------------------- - def _defineParams(self, form): - form.addHidden(GPU_LIST, StringParam, default='0', - expertLevel=LEVEL_ADVANCED, - label="Choose GPU IDs", - help="GPU may have several cores. Set it to zero" - " if you do not know what we are talking about." - " First core index is 0, second 1 and so on.") - form.addSection(label='Input') - form.addParam('inputTrainSet', PointerParam, label="Input training set", - pointerClass='SetOfParticles', - pointerCondition='hasAlignment2D or hasAlignmentProj', - help='The set of particles previously aligned to be used as training set') - - form.addParam('Xdim', IntParam, label="Size of the images for training", default=128) - - form.addParam('modelPretrain', BooleanParam, default=False, - label='Choose if you want to use a pretrained model', - help='Set "yes" if you want to use a previously trained model. ' - 'If you choose "no" new models will be trained.') - - form.addParam('pretrainedModels', PointerParam, - pointerClass='XmippProtDeepGlobalAssignment', - condition=self._cond_modelPretrainTrue, - label='Pretrained model', - help='Select the pretrained model. ') - - form.addSection(label='Training parameters') - form.addParam('numAngModels', IntParam, - label="Number of models for angular assignment", default=5, - help="Choose number of models you want to train. More than 1 is recommended only if next step " - "is inference.") - - form.addParam('numEpochs_ang', IntParam, - label="Number of epochs", - default=25, expertLevel=LEVEL_ADVANCED, - help="Number of epochs for training.") - - form.addParam('batchSize', IntParam, - label="Batch size for training", - default=32, expertLevel=LEVEL_ADVANCED, - help="Batch size for training.") - - form.addParam('learningRate', FloatParam, - label="Learning rate", - default=0.001, expertLevel=LEVEL_ADVANCED, - help="Learning rate for training.") - - form.addParam('sigma', FloatParam, - label="Image shifting", - default=2, expertLevel=LEVEL_ADVANCED, - help="Maximum number of pixels that particles can be shifted in each direction from the center.") - - form.addParam('patience', IntParam, - label="Patience", - default=20, expertLevel=LEVEL_ADVANCED, - help="Training will be stopped if the number of epochs without improvement is greater than " - "patience.") - - form.addParallelSection(threads=1, mpi=1) - - # --------------------------- INSERT steps functions -------------------------------------------- - def _insertAllSteps(self): - self.trainImgsFn = self._getExtraPath('train_input_imgs.xmd') - if self.useQueueForSteps() or self.useQueue(): - myStr = os.environ["CUDA_VISIBLE_DEVICES"] - else: - myStr = self.gpuList.get() - os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() - numGPU = myStr.split(',') - - self._insertFunctionStep("convertStep") - self._insertFunctionStep("train", numGPU[0]) - - # --------------------------- STEPS functions --------------------------------------------------- - def convertStep(self): - writeSetOfParticles(self.inputTrainSet.get(), self.trainImgsFn) - self.runJob("xmipp_image_resize", - "-i %s -o %s --save_metadata_stack %s --fourier %d" % - (self.trainImgsFn, - self._getExtraPath('trainingResized.stk'), - self._getExtraPath('trainingResized.xmd'), - self.Xdim), numberOfMpi=self.numberOfThreads.get() * self.numberOfMpi.get()) - - def train(self, gpuId): - - sig = self.sigma.get() - - self.pretrained = 'no' - self.pathToModel = 'none' - if self.modelPretrain: - self.model = self.pretrainedModels.get() - self.pretrained = 'yes' - self.pathToModel = self.model._getExtraPath("modelAngular0.h5") - - args = "%s %s %f %d %d %s %d %f %d %s %s" % ( - self._getExtraPath("trainingResized.xmd"), self._getExtraPath("modelAngular"), sig, - self.numEpochs_ang, self.batchSize.get(), gpuId, self.numAngModels.get(), self.learningRate.get(), - self.patience.get(), self.pretrained, self.pathToModel) - print(args) - self.runJob("xmipp_deep_global_assignment", args, numberOfMpi=1, env=self.getCondaEnv()) - - remove(self._getExtraPath("trainingResized.xmd")) - remove(self._getExtraPath("trainingResized.stk")) - - # --------------------------- INFO functions -------------------------------- - def _methods(self): - methods = [] - if hasattr(self, 'outputParticles'): - methods.append("We learned a model to center particles from %i input images (%s)." \ - % (self.inputSet.get().getSize(), self.getObjectTag('inputSet'))) - return methods diff --git a/xmipp3/protocols/protocol_deep_global_assignment_predict.py b/xmipp3/protocols/protocol_deep_global_assignment_predict.py deleted file mode 100644 index d1c163800..000000000 --- a/xmipp3/protocols/protocol_deep_global_assignment_predict.py +++ /dev/null @@ -1,152 +0,0 @@ -# ************************************************************************** -# * -# * Authors: COS Sorzano, Erney Ramirez and Adrian Sansinena -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow import VERSION_3_0 -from pyworkflow.protocol import STEPS_PARALLEL -from pyworkflow.protocol.params import (PointerParam, StringParam, FloatParam, - IntParam, BooleanParam, GPU_LIST, EnumParam) -from pyworkflow.protocol.constants import LEVEL_ADVANCED -from pyworkflow.utils.path import moveFile, cleanPattern -from pwem.protocols import ProtAlign2D -from pwem.emlib.metadata import iterRows, getFirstRow -import pwem.emlib.metadata as md -from xmipp3.convert import createItemMatrix, setXmippAttributes, readSetOfParticles, writeSetOfParticles -from pwem import ALIGN_PROJ -from pwem import emlib -from pwem.emlib.image import ImageHandler -import os -import sys -import numpy as np -import math -from shutil import copy -from os import remove -from os.path import exists, join -import xmipp3 -from pyworkflow import BETA, UPDATED, NEW, PROD - - -class XmippProtDeepGlobalAssignmentPredict(ProtAlign2D, xmipp3.XmippProtocol): - """Predict the center particles using deep learning.""" - _label = 'deep global assignment predict' - _lastUpdateVersion = VERSION_3_0 - _devStatus = BETA - - _conda_env = 'xmipp_DLTK_v1.0' - - def __init__(self, **args): - ProtAlign2D.__init__(self, **args) - - # --------------------------- DEFINE param functions -------------------------------------------- - def _defineParams(self, form): - - form.addHidden(GPU_LIST, StringParam, default='0', - expertLevel=LEVEL_ADVANCED, - label="Choose GPU IDs", - help="GPU may have several cores. Set it to zero" - " if you do not know what we are talking about." - " First core index is 0, second 1 and so on.") - form.addSection(label='Input') - - form.addParam('inputImageSet', PointerParam, label="Input Image set", - pointerClass='SetOfParticles', - help='The set of particles to predict shift') - - form.addParam('Xdim', IntParam, label="Size of the images for training", default=128) - - form.addParam('inputModel', PointerParam, label="Model trained", - pointerClass='XmippProtDeepGlobalAssignment', - help='The model to predict angles') - - form.addSection(label='Consensus') - - form.addParam('numAngModels', IntParam, - label="Number of models trained", default=5) - - form.addParam('tolerance', IntParam, - label="Angular tolerance (º)", default=15, - help="Max angular difference between predictions and their mean value.") - - form.addParam('maxModels', IntParam, - label="Maximum number of models dropped per particle", default=0, - help="If more models are dropped, the particle is discarded.") - - form.addParallelSection(threads=1, mpi=1) - - # --------------------------- INSERT steps functions -------------------------------------------- - def _insertAllSteps(self): - self.predictImgsFn = self._getExtraPath('predict_input_imgs.xmd') - - if self.useQueueForSteps() or self.useQueue(): - myStr = os.environ["CUDA_VISIBLE_DEVICES"] - else: - myStr = self.gpuList.get() - os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() - numGPU = myStr.split(',') - - self._insertFunctionStep("convertStep") - self._insertFunctionStep("predict", numGPU[0]) - self._insertFunctionStep('createOutputStep') - - # --------------------------- STEPS functions --------------------------------------------------- - def convertStep(self): - writeSetOfParticles(self.inputImageSet.get(), self.predictImgsFn) - self.runJob("xmipp_image_resize", - "-i %s -o %s --save_metadata_stack %s --fourier %d" % - (self.predictImgsFn, - self._getExtraPath('trainingResized.stk'), - self._getExtraPath('trainingResized.xmd'), - self.Xdim), numberOfMpi=self.numberOfThreads.get() * self.numberOfMpi.get()) - - def predict(self, gpuId): - self.modelAng = self.inputModel.get() - args = "%s %s %s %s %s %d %d %d" % ( - self._getExtraPath("trainingResized.xmd"), gpuId, self._getPath(), self.predictImgsFn, - self.modelAng._getExtraPath("modelAngular"), self.numAngModels.get(), self.tolerance.get(), self.maxModels.get()) - - self.runJob("xmipp_deep_global_assignment_predict", args, numberOfMpi=1, env=self.getCondaEnv()) - - remove(self._getExtraPath("trainingResized.xmd")) - remove(self._getExtraPath("trainingResized.stk")) - - def createOutputStep(self): - imgFname = self._getPath('predict_results.xmd') - outputSet = self._createSetOfParticles() - - readSetOfParticles(imgFname, outputSet) - outputSet.copyInfo(self.inputImageSet.get()) - outputSet.setAlignmentProj() - - self._defineOutputs(outputParticles=outputSet) - self._store(outputSet) - self._defineSourceRelation(self.inputImageSet.get(), outputSet) - - # --------------------------- INFO functions -------------------------------- - def _methods(self): - methods = [] - if hasattr(self, 'outputParticles'): - methods.append("We learned a model to center particles from %i input images (%s)." \ - % (self.inputSet.get().getSize(), self.getObjectTag('inputSet'))) - return methods diff --git a/xmipp3/protocols/protocol_deep_micrograph_screen.py b/xmipp3/protocols/protocol_deep_micrograph_screen.py index 5e4c00838..7293c6d46 100644 --- a/xmipp3/protocols/protocol_deep_micrograph_screen.py +++ b/xmipp3/protocols/protocol_deep_micrograph_screen.py @@ -426,8 +426,7 @@ def getInputMicrographsPointer(self): """ Return the micrographs pointer associated to the SetOfCoordinates or Other micrographs. """ if not self._micsOther(): - inMicsPointer = Pointer(self.getMapper().getParent(self.inputCoordinates.get().getMicrographs()), - extended='outputMicrographs') + inMicsPointer = self.getCoords().getMicrographs(asPointer=True) return inMicsPointer else: return self.inputMicrographs diff --git a/xmipp3/protocols/protocol_extract_particles.py b/xmipp3/protocols/protocol_extract_particles.py index 4c9a8853d..8fb8254c3 100644 --- a/xmipp3/protocols/protocol_extract_particles.py +++ b/xmipp3/protocols/protocol_extract_particles.py @@ -43,11 +43,17 @@ from xmipp3.convert import (micrographToCTFParam, writeMicCoordinates, xmippToLocation, setXmippAttributes) from xmipp3.constants import OTHER +from pyworkflow import BETA, UPDATED, NEW, PROD +FACTOR_BOXSIZE = 1.5 class XmippProtExtractParticles(ProtExtractParticles, XmippProtocol): """Protocol to extract particles from a set of coordinates""" _label = 'extract particles' + _devStatus = UPDATED + RESIZE_FACTOR = 0 + RESIZE_DIMENSIONS = 1 + RESIZE_SAMPLINGRATE = 2 def __init__(self, **kwargs): ProtExtractParticles.__init__(self, **kwargs) @@ -55,26 +61,53 @@ def __init__(self, **kwargs): #--------------------------- DEFINE param functions ------------------------ def _definePreprocessParams(self, form): + + form.addParam('doResize', params.BooleanParam, default=False, + label='Rescale particles?', + help='If you set to *Yes*, you should provide a resize option.') + + form.addParam('resizeOption', params.EnumParam, + choices=['Factor', 'Dimensions', 'Sampling Rate'], + condition='doResize', + default=self.RESIZE_FACTOR, + label="Resize option", display=params.EnumParam.DISPLAY_COMBO, + help='Select an option to resize the images: \n ' + '_Factor_: Set a resize factor to resize. \n ' + '_Dimensions_: Set the output dimensions in pixels. \n' + '_Sampling Rate_: Set the desire sampling rate to resize.') + # downFactor should always be 1.0 or greater - geOne = params.GE(1.0, - error='Value should be greater or equal than 1.0') + geOne = params.GE(1.0, error='Value should be greater or equal than 1.0') form.addParam('downFactor', params.FloatParam, default=1.0, validators=[geOne], label='Downsampling factor', + condition='doResize and resizeOption==%d' % self.RESIZE_FACTOR, help='Select a value greater than 1.0 to reduce the size ' 'of micrographs before extracting the particles. ' 'If 1.0 is used, no downsample is applied. ' 'Non-integer downsample factors are possible. ') + form.addParam('resizeDim', params.IntParam, default=0, + condition='doResize and resizeOption==%d' % self.RESIZE_DIMENSIONS, + allowsPointers=True, + label='Re-scaled size (px)', + help='Size in pixels of the particle images .') + + form.addParam('resizeSamplingRate', params.FloatParam, default=1.0, + condition='doResize and resizeOption==%d' % self.RESIZE_SAMPLINGRATE, + allowsPointers=True, + label='Resize sampling rate (Å/px)', + help='Set the new output sampling rate.') + form.addParam('boxSize', params.IntParam, - label='Particle box size (px)', allowsPointers=True, + label='Particle box size (px)', allowsPointers=True, default=-1, # validators=[params.Positive], - help='This is size of the boxed particles (in pixels). ' + help='This is the size of the boxed particles (in pixels). ' 'Note that if you use downsample option, the ' - 'particles are boxed out after downsampling. ' - 'Use the wizard to check boxSize changes after ' - 'downsampling or using a different pixel size. ') + 'particles are boxed in a downsampled boxsize conserving the same relation. ' + 'Use the wizard to select a boxSize automatically. ' + 'This is calculated by multiplying 1.5 * box size used for picking.') form.addParam('doBorders', params.BooleanParam, default=False, label='Fill pixels outside borders', @@ -174,8 +207,8 @@ def _extractMicrograph(self, mic, doInvert, normalizeArgs, doBorders): baseMicName = pwutils.removeBaseExt(fnLast) outputRoot = str(self._getExtraPath(baseMicName)) fnPosFile = self._getMicPos(mic) - boxSize = self.boxSize.get() - downFactor = self.downFactor.get() + boxSize = self._getExtractBoxSize() + downFactor = self._getDownFactor() patchSize = self.patchSize.get() if self.patchSize.get() > 0 \ else int(boxSize*1.5*downFactor) @@ -267,7 +300,7 @@ def _getNormalizeArgs(self): if normType != "OldXmipp": bgRadius = self.backRadius.get() if bgRadius <= 0: - bgRadius = int(self.boxSize.get() / 2) + bgRadius = int(self._getExtractBoxSize() / 2) args += " --background circle %d" % bgRadius return args @@ -275,6 +308,11 @@ def _getNormalizeArgs(self): #--------------------------- INFO functions -------------------------------- def _validate(self): errors = [] + if self.doResize: + downFactor = self._getDownFactor() + if downFactor < 1: + errors.append('The new particles size should be smaller than the current one') + if self.boxSize.get() == -1: self.boxSize.set(self.getBoxSize()) @@ -348,7 +386,7 @@ def _methods(self): methodsMsgs.append("Inverted contrast on images.") if self._doDownsample(): methodsMsgs.append("Particles downsampled by a factor of %0.2f." - % self.downFactor) + % self._getDownFactor()) if self.doNormalize: methodsMsgs.append("Normalization: %s." % self.getEnumText('normType')) @@ -368,7 +406,7 @@ def _useCTF(self): return self.ctfRelations.hasValue() def _doDownsample(self): - return self.downFactor > 1.0 + return self._getDownFactor() > 1.0 def notOne(self, value): return abs(value - 1) > 0.0001 @@ -379,10 +417,35 @@ def _getNewSampling(self): if self._doDownsample(): # Set new sampling, it should be the input sampling of the used # micrographs multiplied by the downFactor - newSampling *= self.downFactor.get() + newSampling *= self._getDownFactor() return newSampling + def _getDownFactor(self): + downFactor = 1 + if self.resizeOption == self.RESIZE_FACTOR: + downFactor = self.downFactor.get() + elif self.resizeOption == self.RESIZE_SAMPLINGRATE: + downFactor = self.resizeSamplingRate.get()/self.getCoordSampling() + elif self.resizeOption == self.RESIZE_DIMENSIONS: + downFactor = self.boxSize.get()/self.resizeDim.get() + + return float(downFactor) + + def _getExtractBoxSize(self): + if self.boxSize.get() == -1: + boxSize = int(self.getBoxSize()) + else: + boxSize = int(self.boxSize.get()) + + downFactor = self._getDownFactor() + if downFactor > 1: + newBoxSize = self.getEven(boxSize/downFactor) + else: + newBoxSize = boxSize + + return int(newBoxSize) + def _setupBasicProperties(self): # Set sampling rate (before and after doDownsample) and inputMics # according to micsSource type @@ -451,14 +514,14 @@ def getBoxScale(self): samplingPicking = self.getCoordSampling() samplingExtract = self.getMicSampling() f = float(samplingPicking) / samplingExtract - return f / self.downFactor.get() if self._doDownsample() else f + return f / self._getDownFactor() if self._doDownsample() else f def getEven(self, boxSize): return Integer(int(int(boxSize)/2+0.75)*2) def getBoxSize(self): # This function is needed by the wizard and for auto-boxSize selection - return self.getEven(self.getCoords().getBoxSize() * self.getBoxScale()) + return self.getEven(self.getCoords().getBoxSize()*FACTOR_BOXSIZE) def _getOutputImgMd(self): return self._getPath('images.xmd') diff --git a/xmipp3/protocols/protocol_extract_particles_movies.py b/xmipp3/protocols/protocol_extract_particles_movies.py index d68d59414..29881587a 100644 --- a/xmipp3/protocols/protocol_extract_particles_movies.py +++ b/xmipp3/protocols/protocol_extract_particles_movies.py @@ -161,8 +161,6 @@ def _defineParams(self, form): def _insertAllSteps(self): self._createFilenameTemplates() - - makePath(self._getExtraPath('DONE')) # Build the list of all processMovieStep ids by # inserting each of the steps for each movie self.insertedDict = {} @@ -501,10 +499,9 @@ def getCoordSampling(self): def getCoords(self): # to support multiple access to db coordSet = self.inputCoordinates.get() - coordSetCopy = SetOfCoordinates() - coordSetCopy.copy(coordSet) + coordSet.loadAllProperties() coordSet.close() - return coordSetCopy + return coordSet def _stepsCheck(self): # Streaming is not implemented yet for this protocol. @@ -512,10 +509,9 @@ def _stepsCheck(self): def _hasCoordinates(self, movie): coordSet = self.getCoords() - len = 0 for coord in coordSet.iterCoordinates(movie.getObjId()): - len += 1 + len += 1 break if len > 0: return True diff --git a/xmipp3/protocols/protocol_flexalign.py b/xmipp3/protocols/protocol_flexalign.py index 253833c55..939c1307f 100644 --- a/xmipp3/protocols/protocol_flexalign.py +++ b/xmipp3/protocols/protocol_flexalign.py @@ -188,7 +188,7 @@ def getGPUArgs(self): def tryProcessMovie(self, movie): movieFolder = self._getOutputMovieFolder(movie) - x, y, n = movie.getDim() + _, _, n = movie.getDim() a0, aN = self._getFrameRange(n, 'align') s0, sN = self._getFrameRange(n, 'sum') @@ -422,4 +422,4 @@ def _validate(self): return errors def _citations(self): - return ['strelak2020flexalign'] + return ['strelak2020flexalign', 'Strelak2023performance'] diff --git a/xmipp3/protocols/protocol_local_ctf.py b/xmipp3/protocols/protocol_local_ctf.py index 49298eb6e..9ac6c6377 100644 --- a/xmipp3/protocols/protocol_local_ctf.py +++ b/xmipp3/protocols/protocol_local_ctf.py @@ -35,6 +35,8 @@ from xmipp3.convert import readSetOfParticles, writeSetOfParticles +CITE ='Fernandez-Gimenez2023b' + class XmippProtLocalCTF(ProtAnalysis3D): """Compares a set of particles with the corresponding projections of a reference volume. diff --git a/xmipp3/protocols/protocol_movie_dose_analysis.py b/xmipp3/protocols/protocol_movie_dose_analysis.py index 3a1b3b1ff..200190c37 100644 --- a/xmipp3/protocols/protocol_movie_dose_analysis.py +++ b/xmipp3/protocols/protocol_movie_dose_analysis.py @@ -47,7 +47,7 @@ class XmippProtMovieDoseAnalysis(ProtProcessMovies): """ Protocol for the dose analysis """ # FIXME: WITH .mrcs IT DOES NOT FILL THE LABELS - _devStatus = NEW + _devStatus = PROD _label = 'movie dose analysis' _lastUpdateVersion = VERSION_3_0 diff --git a/xmipp3/protocols/protocol_movie_gain.py b/xmipp3/protocols/protocol_movie_gain.py index 409fa8971..3eea48642 100644 --- a/xmipp3/protocols/protocol_movie_gain.py +++ b/xmipp3/protocols/protocol_movie_gain.py @@ -32,7 +32,7 @@ from pyworkflow import VERSION_1_1 import pyworkflow.utils as pwutils from pyworkflow.object import Set -from pyworkflow.protocol import STEPS_PARALLEL +from pyworkflow.protocol import STEPS_PARALLEL, Protocol from pyworkflow.protocol.params import (PointerParam, IntParam, BooleanParam, LEVEL_ADVANCED) from pyworkflow.utils.properties import Message @@ -51,7 +51,7 @@ OUTPUT_RESIDUAL_GAINS = 'residualGains' OUTPUT_MOVIES = 'outputMovies' -class XmippProtMovieGain(ProtProcessMovies): +class XmippProtMovieGain(ProtProcessMovies, Protocol): """ Estimate the gain image of a camera, directly analyzing one of its movies. It can correct the orientation of an external gain image (by comparing it with the estimated). Finally, it estimates the residual gain (the gain of the movie after correcting with a gain). @@ -351,24 +351,6 @@ def updateGainsOutput(self, movie, imgSet, imageFile): imgSet.append(imgOut) return imgSet - def _updateOutputSet(self, outputName, outputSet, state=Set.STREAM_OPEN): - outputSet.setStreamState(state) - - if self.hasAttribute(outputName): - outputSet.write() # Write to commit changes - outputAttr = getattr(self, outputName) - # Copy the properties to the object contained in the protcol - outputAttr.copy(outputSet, copyId=False) - # Persist changes - self._store(outputAttr) - else: - # Here the defineOutputs function will call the write() method - self._defineOutputs(**{outputName: outputSet}) - self._store(outputSet) - - # Close set databaset to avoid locking it - outputSet.close() - def match_orientation(self, exp_gain, est_gain): ''' Calculates the correct orientation of the experimental gain image with respect to the estimated @@ -409,7 +391,6 @@ def match_orientation(self, exp_gain, est_gain): best_cor = minVal best_transf = (angle,imir) best_R = R - # T = np.asarray([[1, 0, np.asscalar(corLoc[1])], [0, 1, np.asscalar(corLoc[0])], [0, 0, 1]]) T = np.asarray([[1, 0, corLoc[1].item()], [0, 1, corLoc[0].item()], [0, 0, 1]]) @@ -418,7 +399,6 @@ def match_orientation(self, exp_gain, est_gain): best_cor = maxVal best_transf = (angle, imir) best_R = R - # T = np.asarray([[1, 0, np.asscalar(corLoc[1])], [0, 1, np.asscalar(corLoc[0])], [0, 0, 1]]) T = np.asarray([[1, 0, corLoc[1].item()], [0, 1, corLoc[0].item()], [0, 0, 1]]) diff --git a/xmipp3/protocols/protocol_particle_pick_automatic.py b/xmipp3/protocols/protocol_particle_pick_automatic.py index adffc6757..851b34a96 100644 --- a/xmipp3/protocols/protocol_particle_pick_automatic.py +++ b/xmipp3/protocols/protocol_particle_pick_automatic.py @@ -26,7 +26,7 @@ # ************************************************************************** from os.path import exists, basename, join -from pyworkflow.protocol.params import STEPS_PARALLEL, PointerParam, EnumParam +from pyworkflow.protocol.params import STEPS_PARALLEL, PointerParam, EnumParam, FileParam from pyworkflow.utils.path import * from pwem.protocols import ProtParticlePickingAuto @@ -34,20 +34,22 @@ from pwem import emlib from xmipp3.base import XmippProtocol from xmipp3.convert import readSetOfCoordinates +from pyworkflow import BETA, UPDATED, NEW, PROD MICS_SAMEASPICKING = 0 MICS_OTHER = 1 +SRC_MANUAL_PICKING = 0 +SRC_DIR = 1 class XmippParticlePickingAutomatic(ProtParticlePickingAuto, XmippProtocol): """Protocol to pick particles automatically in a set of micrographs using previous training """ _label = 'auto-picking (step 2)' - - filesToCopy = ['model_training.txt', 'model_svm.txt', - 'model_pca_model.stk', 'model_rotpca_model.stk', - 'model_particle_avg.xmp', 'config.xmd', 'templates.stk'] + _devStatus = UPDATED + filesToCopy = ['model_svm.txt', 'model_pca_model.stk', 'model_rotpca_model.stk', + 'model_particle_avg.xmp', 'templates.stk'] def __init__(self, **kwargs): ProtParticlePickingAuto.__init__(self, **kwargs) @@ -58,12 +60,26 @@ def _defineParams(self, form): form.addSection(label='Input') + form.addParam('modelSource', EnumParam, label="Model source", + choices=["Manual picking in this project", "External directory"], + default=0, help="The files model_* can be copied from a previous protocol execution within this " + "project or copied from an external directory. This latter option is useful in" + "cases in which the same kind of molecule is processed many times.") + form.addParam('xmippParticlePicking', PointerParam, label="Xmipp particle picking run", pointerClass='XmippProtParticlePicking', + condition="modelSource==%d"%SRC_MANUAL_PICKING, #pointerCondition='isFinished', help='Select the previous xmipp particle picking run.') + form.addParam('xmippParticlePickingDir', FileParam, + label="Xmipp particle picking model directory", + allowsNull=True, + condition="modelSource==%d"%SRC_DIR, + #pointerCondition='isFinished', + help='The directory must contain the files model_*, config.xmd and templates.stk') + form.addParam('micsToPick', EnumParam, choices=['Same as supervised', 'Other'], default=0, label='Micrographs to pick', @@ -98,16 +114,28 @@ def _insertInitialSteps(self): return [copyId] # --------------------------- STEPS functions ------------------------------ + def getSrcDir(self): + if self.modelSource == SRC_MANUAL_PICKING: + return self.xmippParticlePicking.get()._getExtraPath() + else: + return self.xmippParticlePickingDir.get() + def copyInputFilesStep(self): # Copy training model files to current run + srcDir = self.getSrcDir() for f in self.filesToCopy: - copyFile(self.particlePickingRun._getExtraPath(f), - self._getExtraPath(f)) + createLink(os.path.join(srcDir, f), self._getExtraPath(f)) + copyFile(os.path.join(srcDir, "config.xmd"), self._getExtraPath("config.xmd")) + + # Get the box size + mdInfo = emlib.MetaData("properties@"+self._getExtraPath("config.xmd")) + self.boxSize = mdInfo.getValue(emlib.MDL_PICKING_PARTICLE_SIZE,mdInfo.firstObject()) + mdInfo.setValue(emlib.MDL_PICKING_MANUALPARTICLES_SIZE,0,mdInfo.firstObject()) + mdInfo.write("properties@"+self._getExtraPath("config.xmd"),emlib.MD_APPEND) def _pickMicrograph(self, mic, *args): micPath = mic.getFileName() # Get particle picking boxsize from the previous run - boxSize = self.particlePickingRun.outputCoordinates.getBoxSize() modelRoot = self._getExtraPath('model') micName = removeBaseExt(micPath) @@ -131,7 +159,7 @@ def _pickMicrograph(self, mic, *args): if proceed: args = "-i %s " % micPath - args += "--particleSize %d " % boxSize + args += "--particleSize %d " % self.boxSize args += "--model %s " % modelRoot args += "--outputRoot %s " % self._getExtraPath(micName) args += "--mode autoselect --thr %d" % self.numberOfThreads @@ -147,13 +175,13 @@ def readCoordsFromMics(self, workingDir, micList, coordSet): # --------------------------- INFO functions ------------------------------- def _validate(self): validateMsgs = [] - - if not hasattr(self.xmippParticlePicking.get(),"outputCoordinates"): + + if self.modelSource == SRC_MANUAL_PICKING and not hasattr(self.xmippParticlePicking.get(),"outputCoordinates"): validateMsgs.append("You need to generate coordinates for the " "supervised picking") - srcPaths = [self.xmippParticlePicking.get()._getExtraPath(k) - for k in self.filesToCopy] + srcDir = self.getSrcDir() + srcPaths = [os.path.join(srcDir,k) for k in self.filesToCopy] # Check that all needed files exist if missingPaths(*srcPaths): validateMsgs.append('Input picking run has not been trained, ' @@ -161,7 +189,7 @@ def _validate(self): # If other set of micrographs is provided they should have same # sampling rate and acquisition - if self.micsToPick.get() == MICS_OTHER: + if self.micsToPick.get() == MICS_OTHER and self.modelSource != SRC_DIR: inputMics = self.inputMicrographs.get() manualMics = self.xmippParticlePicking.get().inputMicrographs.get() # FIXME: manualMics is always None when scheduled... @@ -181,34 +209,20 @@ def _validate(self): 'acquisition parameters as the ones ' 'already picked.') + if self.modelSource.get()==SRC_DIR and self.micsToPick.get()==MICS_SAMEASPICKING: + validateMsgs.append("You cannot take the model from a directory and indicate that the set of micrograohs " + "is the same as picking. If you take the model from a directory, probably you want " + "to pick from a different set.") return validateMsgs def getSummary(self, coordSet): summary = [] - summary.append("Previous run: %s" % - self.xmippParticlePicking.get().getNameId()) - configfile = join(self._getExtraPath(), 'config.xmd') - existsConfig = exists(configfile) - if existsConfig: - md = emlib.MetaData('properties@' + configfile) - configobj = md.firstObject() - def _get(label): - return md.getValue(label, configobj) - pickingState = _get(emlib.MDL_PICKING_STATE) - particleSize = _get(emlib.MDL_PICKING_PARTICLE_SIZE) - activeMic = _get(emlib.MDL_MICROGRAPH) - isAutopick = pickingState != "Manual" - manualParticlesSize = _get(emlib.MDL_PICKING_MANUALPARTICLES_SIZE) - autoParticlesSize = _get(emlib.MDL_PICKING_AUTOPARTICLES_SIZE) - - summary.append("Manual particles picked: %s" % manualParticlesSize) - summary.append("Particle size:%d" %(particleSize)) - autopick = "Yes" if isAutopick else "No" - summary.append("Autopick: " + autopick) - if isAutopick: - summary.append("Automatic particles picked: %s" - % autoParticlesSize) - summary.append("Last micrograph: " + activeMic) + if self.modelSource == SRC_MANUAL_PICKING: + summary.append("Previous run: %s" % + self.xmippParticlePicking.get().getNameId()) + else: + summary.append("Model from: %s" % + self.xmippParticlePickingDir.get()) return "\n".join(summary) def getMethods(self, output): @@ -221,7 +235,7 @@ def getMethods(self, output): def _citations(self): return ['Abrishami2013'] - + # --------------------------- UTILS functions ------------------------------ def getCoordsDir(self): return self._getExtraPath() diff --git a/xmipp3/protocols/protocol_particle_pick_consensus.py b/xmipp3/protocols/protocol_particle_pick_consensus.py index d913bb308..f6e9df001 100644 --- a/xmipp3/protocols/protocol_particle_pick_consensus.py +++ b/xmipp3/protocols/protocol_particle_pick_consensus.py @@ -236,12 +236,7 @@ def _loadOutputSet(self, SetClass, baseName): outputSet.setStreamState(outputSet.STREAM_OPEN) outputSet.setBoxSize(self.getMainInput().getBoxSize()) - # FIXME: The commented line below should work but it fails in streaming - # when extracting particles. The line below it fixs that error.. - # inMicsPointer = self.getMainInput().getMicrographs() - inMicsPointer = Pointer(self.getMapper().getParent( - self.getMainInput().getMicrographs()), - extended='outputMicrographs') + inMicsPointer = self.getMainInput().getMicrographs(asPointer=True) outputSet.setMicrographs(inMicsPointer) return outputSet diff --git a/xmipp3/protocols/protocol_preprocess/protocol_crop_resize.py b/xmipp3/protocols/protocol_preprocess/protocol_crop_resize.py index 8b6fdcb40..402ee43af 100644 --- a/xmipp3/protocols/protocol_preprocess/protocol_crop_resize.py +++ b/xmipp3/protocols/protocol_preprocess/protocol_crop_resize.py @@ -43,7 +43,7 @@ class XmippResizeHelper: """ Common features to change dimensions of either SetOfParticles, Volume or SetOfVolumes objects. """ - _devStatus = UPDATED + _devStatus = PROD RESIZE_SAMPLINGRATE = 0 RESIZE_DIMENSIONS = 1 diff --git a/xmipp3/protocols/protocol_preprocess/protocol_filter.py b/xmipp3/protocols/protocol_preprocess/protocol_filter.py index dfe493a8a..2371f992c 100644 --- a/xmipp3/protocols/protocol_preprocess/protocol_filter.py +++ b/xmipp3/protocols/protocol_preprocess/protocol_filter.py @@ -69,11 +69,11 @@ class XmippFilterHelper(): #--------------------------- DEFINE param functions -------------------------------------------- @classmethod - def _defineProcessParams(cls, form): + def _defineProcessParams(cls, form, fourierChoices=['low pass', 'high pass', 'band pass', 'ctf']): form.addParam('filterSpace', EnumParam, choices=['fourier', 'real', 'wavelets'], default=FILTER_SPACE_FOURIER, label="Filter space") - form.addParam('filterModeFourier', EnumParam, choices=['low pass', 'high pass', 'band pass', 'ctf'], + form.addParam('filterModeFourier', EnumParam, choices=fourierChoices, default=cls.FM_BAND_PASS, condition='filterSpace == %d' % FILTER_SPACE_FOURIER, label="Filter mode", @@ -119,7 +119,7 @@ def _defineProcessParams(cls, form): #fourier form.addParam('freqInAngstrom', BooleanParam, default=True, - condition='filterSpace == %d' % FILTER_SPACE_FOURIER, + condition='filterSpace == %d and filterModeFourier != %d' % (FILTER_SPACE_FOURIER, cls.FM_CTF), label='Provide resolution in Angstroms?', help='If *Yes*, the resolution values for the filter\n' 'should be provided in Angstroms. If *No*, the\n' @@ -433,7 +433,7 @@ def __init__(self, **kwargs): #--------------------------- DEFINE param functions -------------------------------------------- def _defineProcessParams(self, form): - XmippFilterHelper._defineProcessParams(form) + XmippFilterHelper._defineProcessParams(form, fourierChoices=['low pass', 'high pass', 'band pass']) def _insertProcessStep(self): XmippFilterHelper._insertProcessStep(self) diff --git a/xmipp3/protocols/protocol_preprocess/protocol_movie_resize.py b/xmipp3/protocols/protocol_preprocess/protocol_movie_resize.py index 4dc17b98c..1eb298f2a 100644 --- a/xmipp3/protocols/protocol_preprocess/protocol_movie_resize.py +++ b/xmipp3/protocols/protocol_preprocess/protocol_movie_resize.py @@ -170,6 +170,10 @@ def _checkNewOutput(self): imgOut.setAcquisition(movie.getAcquisition()) imgOut.setSamplingRate(self.newSamplingRate) imgOut.setFramesRange(self.inputMovies.get().getFramesRange()) + + if imageSet.isEmpty(): + imageSet.setDim(imgOut.getDim()) + imageSet.append(imgOut) self._updateOutputSet('outputMovies', imageSet, streamMode) @@ -191,13 +195,11 @@ def _loadOutputSet(self, SetClass, baseName): outputSet.loadAllProperties() outputSet.enableAppend() else: + inputMovies = self.inputMovies.get() outputSet = SetClass(filename=setFile) outputSet.setStreamState(outputSet.STREAM_OPEN) - - inputMovies = self.inputMovies.get() - outputSet.copyInfo(inputMovies) - - outputSet.setSamplingRate(self.newSamplingRate) + outputSet.copyInfo(inputMovies) + outputSet.setSamplingRate(self.newSamplingRate) return outputSet diff --git a/xmipp3/protocols/protocol_preprocess/protocol_preprocess.py b/xmipp3/protocols/protocol_preprocess/protocol_preprocess.py index dabe2e8b4..a0ae55d27 100644 --- a/xmipp3/protocols/protocol_preprocess/protocol_preprocess.py +++ b/xmipp3/protocols/protocol_preprocess/protocol_preprocess.py @@ -38,6 +38,7 @@ from xmipp3.base import isXmippCudaPresent from .protocol_process import XmippProcessParticles,\ XmippProcessVolumes +from pyworkflow import BETA, UPDATED, NEW, PROD class XmippPreprocessHelper: @@ -122,7 +123,7 @@ class XmippProtPreprocessParticles(XmippProcessParticles): """ Preprocess a set of particles. You can remove dust, normalize, apply threshold, etc """ _label = 'preprocess particles' - + _devStatus = PROD # Normalization enum constants NORM_OLD = 0 NORM_NEW = 1 @@ -373,7 +374,7 @@ def _postprocessOutput(self, outputSet): class XmippProtPreprocessVolumes(XmippProcessVolumes): """ Protocol for Xmipp-based preprocess for volumes """ _label = 'preprocess volumes' - + _devStatus = PROD # Aggregation constants AGG_AVERAGE=0 AGG_SUM=1 diff --git a/xmipp3/protocols/protocol_preprocess_micrographs.py b/xmipp3/protocols/protocol_preprocess_micrographs.py index cf54a2fe4..baa4ed185 100644 --- a/xmipp3/protocols/protocol_preprocess_micrographs.py +++ b/xmipp3/protocols/protocol_preprocess_micrographs.py @@ -38,6 +38,8 @@ from pwem.protocols import ProtPreprocessMicrographs from pwem.objects import SetOfMicrographs, Micrograph +OUTPUT_MICROGRAPHS = 'outputMicrographs' + class XmippProtPreprocessMicrographs(ProtPreprocessMicrographs): """Protocol to preprocess a set of micrographs in the project. @@ -260,12 +262,15 @@ def _checkNewOutput(self): # so we exit from the function here return - outSet = self._loadOutputSet(SetOfMicrographs, 'micrographs.sqlite') + outSet = self.getOutputMics() def tryToAppend(outSet, micOut, tries=1): """ When micrograph is very big, sometimes it's not ready to be read Then we will wait for it up to a minute in 6 time-growing tries. """ try: + if outSet.isEmpty(): + outSet.setDim(micOut.getDim()) + outSet.append(micOut) except Exception as ex: micFn = micOut.getFileName() # Runs/..../extra/filename.mrc @@ -289,7 +294,7 @@ def tryToAppend(outSet, micOut, tries=1): micOut.setMicName(mic.getMicName()) tryToAppend(outSet, micOut) - self._updateOutputSet('outputMicrographs', outSet, streamMode) + self._updateOutputSet(OUTPUT_MICROGRAPHS, outSet, streamMode) if len(doneList)==0: #firstTime self._defineTransformRelation(self.inputMicrographs, outSet) @@ -300,40 +305,36 @@ def tryToAppend(outSet, micOut, tries=1): outputStep.setStatus(cons.STATUS_NEW) - def _loadOutputSet(self, SetClass, baseName): - setFile = self._getPath(baseName) - if os.path.exists(setFile): - outputSet = SetClass(filename=setFile) - outputSet.loadAllProperties() - outputSet.enableAppend() - else: - outputSet = SetClass(filename=setFile) + def getOutputMics(self): + + if not hasattr(self, OUTPUT_MICROGRAPHS): + + outputSet = SetOfMicrographs(filename=self.getPath('micrographs.sqlite')) outputSet.setStreamState(outputSet.STREAM_OPEN) + inputs = self.inputMicrographs.get() + outputSet.copyInfo(inputs) + if self.doDownsample: + outputSet.setSamplingRate(self.inputMicrographs.get().getSamplingRate() + * self.downFactor.get()) + + self._defineOutputs(**{OUTPUT_MICROGRAPHS: outputSet}) + self._defineTransformRelation(inputs, outputSet) + self.info("Storing set 1rst time: %s" % outputSet) + # self._store(outputSet) + else: + outputSet = getattr(self, OUTPUT_MICROGRAPHS) - inputs = self.inputMicrographs.get() - outputSet.copyInfo(inputs) - if self.doDownsample: - outputSet.setSamplingRate(self.inputMicrographs.get().getSamplingRate() - * self.downFactor.get()) return outputSet def _updateOutputSet(self, outputName, outputSet, state=Set.STREAM_OPEN): + outputSet.setStreamState(state) - if self.hasAttribute(outputName): - outputSet.write() # Write to commit changes - outputAttr = getattr(self, outputName) - # Copy the properties to the object contained in the protcol - outputAttr.copy(outputSet, copyId=False) - # Persist changes - self._store(outputAttr) - else: - # Here the defineOutputs function will call the write() method - self._defineOutputs(**{outputName: outputSet}) - self._defineTransformRelation(self.inputMicrographs, outputSet) - self._store(outputSet) - # Close set databaset to avoid locking it - outputSet.close() + outputSet.write() # Write to commit changes + self._store(outputSet) + + # Close set dataset to avoid locking it + # outputSet.close() def _insertStepsForMicrograph(self, inputMic, outputMic): diff --git a/xmipp3/protocols/protocol_resolution_deepres.py b/xmipp3/protocols/protocol_resolution_deepres.py index 0607d3ea9..b2064434e 100644 --- a/xmipp3/protocols/protocol_resolution_deepres.py +++ b/xmipp3/protocols/protocol_resolution_deepres.py @@ -45,7 +45,7 @@ def updateEnviron(gpuNum): else: os.environ['CUDA_VISIBLE_DEVICES'] = str(gpuNum) -DEEPRES_METHOD_URL = 'http://github.com/I2PC/scipion/wiki/XmippProtDeepRes' +DEEPRES_METHOD_URL = 'https://github.com/I2PC/scipion/wiki/XmippProtDeepRes' RESIZE_MASK = 'binaryMask.vol' MASK_DILATE = 'Mask_dilate.vol' RESIZE_VOL = 'originalVolume.vol' diff --git a/xmipp3/protocols/protocol_resolution_directional.py b/xmipp3/protocols/protocol_resolution_directional.py index 9bbf6d918..d6a9c6cd4 100644 --- a/xmipp3/protocols/protocol_resolution_directional.py +++ b/xmipp3/protocols/protocol_resolution_directional.py @@ -38,7 +38,7 @@ MDL_MAX, MDL_MIN, MDL_INTSCALE) -MONORES_METHOD_URL = 'http://github.com/I2PC/scipion/wiki/XmippProtMonoDir' +MONORES_METHOD_URL = 'https://github.com/I2PC/scipion/wiki/XmippProtMonoDir' OUTPUT_RADIAL_AVERAGES = 'Radial_averages.xmd' diff --git a/xmipp3/protocols/protocol_resolution_monogenic_signal.py b/xmipp3/protocols/protocol_resolution_monogenic_signal.py index 763652466..e2d4781fe 100644 --- a/xmipp3/protocols/protocol_resolution_monogenic_signal.py +++ b/xmipp3/protocols/protocol_resolution_monogenic_signal.py @@ -37,7 +37,7 @@ from pwem.objects import Volume import pwem.emlib.metadata as md -MONORES_METHOD_URL = 'http://github.com/I2PC/scipion/wiki/XmippProtMonoRes' +MONORES_METHOD_URL = 'https://github.com/I2PC/scipion/wiki/XmippProtMonoRes' OUTPUT_RESOLUTION_FILE = 'monoresResolutionMap.mrc' OUTPUT_RESOLUTION_FILE_CHIMERA = 'monoresResolutionChimera.mrc' OUTPUT_MASK_FILE = 'refinedMask.mrc' diff --git a/xmipp3/protocols/protocol_screen_deepConsensus.py b/xmipp3/protocols/protocol_screen_deepConsensus.py index f7d983453..99e74bbdf 100644 --- a/xmipp3/protocols/protocol_screen_deepConsensus.py +++ b/xmipp3/protocols/protocol_screen_deepConsensus.py @@ -1189,7 +1189,8 @@ def createPreliminarOutput(self, trPass): def getPreCoordinatesOutput(self): print('Creating new preliminarOutputCoordinates set') - self.preliminarOutputCoordinates = self._createSetOfCoordinates(self.coordinatesDict['OR'].getMicrographs()) + self.preliminarOutputCoordinates = \ + self._createSetOfCoordinates(self.coordinatesDict['OR'].getMicrographs(asPointer=True)) self.preliminarOutputCoordinates.copyInfo(self.coordinatesDict['OR']) self.preliminarOutputCoordinates.setBoxSize(self._getBoxSize()) self.preliminarOutputCoordinates.setStreamState(SetOfParticles.STREAM_OPEN) @@ -1203,7 +1204,7 @@ def getPreCoordinatesOutput(self): def getCoordinatesOutput(self): if not hasattr(self, "outputCoordinates"): print('Creating new outputCoordinates set') - self.outputCoordinates = self._createSetOfCoordinates(self.coordinatesDict['OR'].getMicrographs()) + self.outputCoordinates = self._createSetOfCoordinates(self.coordinatesDict['OR'].getMicrographs(asPointer=True)) self.outputCoordinates.copyInfo(self.coordinatesDict['OR']) self.outputCoordinates.setBoxSize(self._getBoxSize()) self.outputCoordinates.setStreamState(SetOfParticles.STREAM_OPEN) @@ -1214,7 +1215,7 @@ def getCoordinatesOutput(self): self.USING_INPUT_COORDS = False else: # Micrographs of the set removed because there might be new ones in streaming - self.outputCoordinates.setMicrographs(self.coordinatesDict['OR'].getMicrographs()) + self.outputCoordinates.setMicrographs(self.coordinatesDict['OR'].getMicrographs(asPointer=True)) return self.outputCoordinates def getPreParticlesOutput(self, partSet): diff --git a/xmipp3/protocols/protocol_subtract_projection.py b/xmipp3/protocols/protocol_subtract_projection.py index ad03a948f..18cc11e6c 100644 --- a/xmipp3/protocols/protocol_subtract_projection.py +++ b/xmipp3/protocols/protocol_subtract_projection.py @@ -35,11 +35,12 @@ from pyworkflow import BETA, UPDATED, NEW, PROD OUTPUT = "output_particles.xmd" +CITE = 'Fernandez-Gimenez2023a' class XmippProtSubtractProjectionBase(EMProtocol): """ Helper class that contains some Protocol utilities methods used by both XmippProtSubtractProjection and XmippProtBoostParticles.""" - _devStatus = UPDATED + _devStatus = PROD # --------------------------- DEFINE param functions -------------------------------------------- @classmethod @@ -241,4 +242,4 @@ def _methods(self): methods.append("Output particles not ready yet.") else: methods.append("Particles boosted according to their equivalent projections from a reference volume.") - return methods \ No newline at end of file + return methods diff --git a/xmipp3/protocols/protocol_tilt_analysis.py b/xmipp3/protocols/protocol_tilt_analysis.py index 2005a3a20..debb95d27 100644 --- a/xmipp3/protocols/protocol_tilt_analysis.py +++ b/xmipp3/protocols/protocol_tilt_analysis.py @@ -32,7 +32,7 @@ from datetime import datetime from collections import OrderedDict from pyworkflow import VERSION_3_0 -from pyworkflow.protocol import STEPS_PARALLEL +from pyworkflow.protocol import STEPS_PARALLEL, Protocol from pyworkflow.protocol.params import (PointerParam, IntParam, BooleanParam, LEVEL_ADVANCED, FloatParam, GE, GT, Range) import pyworkflow.utils as pwutils @@ -47,7 +47,7 @@ from matplotlib import pyplot as plt -class XmippProtTiltAnalysis(ProtMicrographs): +class XmippProtTiltAnalysis(ProtMicrographs, Protocol): """ Estimate the tilt of a micrograph, by analyzing the PSD correlations of different segments of the image. """ _label = 'tilt analysis' @@ -345,7 +345,7 @@ def calculateTiltCorrelationStep(self, mic): window_image = ImageHandler().createImage() rotatedWind_psd = ImageHandler().createImage() output_image = ImageHandler().createImage() - output_array = np.zeros((subWindStep * 2, subWindStep * 2)) + output_array = np.zeros(((subWindStep * 2)-1, (subWindStep * 2)-1)) ih = ImageHandler() for i in range(0, 3, 2): @@ -440,22 +440,6 @@ def _loadOutputSet(self, SetClass, baseName): return outputSet - def _updateOutputSet(self, outputName, outputSet, state=Set.STREAM_OPEN): - outputSet.setStreamState(state) - if self.hasAttribute(outputName): - outputSet.write() # Write to commit changes - outputAttr = getattr(self, outputName) - # Copy the properties to the object contained in the protocol - outputAttr.copy(outputSet, copyId=False) - # Persist changes - self._store(outputAttr) - else: - # Here the defineOutputs function will call the write() method - self._defineOutputs(**{outputName: outputSet}) - self._store(outputSet) - # Close set databaset to avoid locking it - outputSet.close() - # ------------------------- UTILS functions -------------------------------- def _correctFormat(self, micName, micFn, micFolderTmp): if micName.endswith('bz2'): diff --git a/xmipp3/protocols/protocol_trigger_data.py b/xmipp3/protocols/protocol_trigger_data.py index 20928398e..f3af9ed96 100644 --- a/xmipp3/protocols/protocol_trigger_data.py +++ b/xmipp3/protocols/protocol_trigger_data.py @@ -31,14 +31,15 @@ from datetime import datetime import pyworkflow.protocol.constants as cons -from pyworkflow import VERSION_2_0 +from pyworkflow import VERSION_3_0 +from pyworkflow.protocol import Protocol from pwem.protocols import EMProtocol from pyworkflow.object import Set from pyworkflow.protocol.params import BooleanParam, IntParam, PointerParam, GT SIGNAL_FILENAME = "STOP_STREAM.TXT" -class XmippProtTriggerData(EMProtocol): +class XmippProtTriggerData(EMProtocol, Protocol): """ Waits until certain number of images is prepared and then send them to output. @@ -55,7 +56,7 @@ class XmippProtTriggerData(EMProtocol): a certain number of items (completely in streaming). """ _label = 'trigger data' - _lastUpdateVersion = VERSION_2_0 + _lastUpdateVersion = VERSION_3_0 # --------------------------- DEFINE param functions ---------------------- def _defineParams(self, form): @@ -121,7 +122,7 @@ def _stepsCheck(self): self._checkNewOutput() def createOutputStep(self): - pass + self._closeOutputSet() def _checkNewInput(self): imsFile = self.inputImages.get().getFileName() @@ -252,23 +253,6 @@ def _loadOutputSet(self, SetClass, baseName, newImages): outputSet.copyItems(newImages) return outputSet - def _updateOutputSet(self, outputName, outputSet, state=Set.STREAM_OPEN): - outputSet.setStreamState(state) - if self.hasAttribute(outputName): - outputSet.write() # Write to commit changes - outputAttr = getattr(self, outputName) - # Copy the properties to the object contained in the protocol - outputAttr.copy(outputSet, copyId=False) - # Persist changes - self._store(outputAttr) - else: - self._defineOutputs(**{outputName: outputSet}) - self._defineTransformRelation(self.inputImages, outputSet) - self._store(outputSet) - - # Close set database to avoid locking it - outputSet.close() - # --------------------------- INFO functions ------------------------------ def _summary(self): summary = [] diff --git a/xmipp3/protocols/protocol_validate_fscq.py b/xmipp3/protocols/protocol_validate_fscq.py index 504670922..9a6e9bb20 100644 --- a/xmipp3/protocols/protocol_validate_fscq.py +++ b/xmipp3/protocols/protocol_validate_fscq.py @@ -40,7 +40,7 @@ import xmipp3 -VALIDATE_METHOD_URL = 'http://github.com/I2PC/scipion-em-xmipp/wiki/XmippProtValFit' +VALIDATE_METHOD_URL = 'https://github.com/I2PC/scipion-em-xmipp/wiki/XmippProtValFit' OUTPUT_PDBVOL_FILE = 'pdbVol' OUTPUT_PDBMRC_FILE = 'pdb_volume.map' BLOCRES_AVG_FILE = 'blocresAvg' diff --git a/xmipp3/protocols/protocol_volume_consensus.py b/xmipp3/protocols/protocol_volume_consensus.py index deb84001e..291dd6db4 100644 --- a/xmipp3/protocols/protocol_volume_consensus.py +++ b/xmipp3/protocols/protocol_volume_consensus.py @@ -30,6 +30,7 @@ from pwem.objects import Volume, Transform from pwem.protocols import ProtInitialVolume +CITE = 'Fernandez-Gimenez2021' class XmippProtVolConsensus(ProtInitialVolume): """ This protocol performs a fusion of all the input volumes, which should be preprocessed with protocol 'volume diff --git a/xmipp3/protocols/protocol_volume_local_adjust.py b/xmipp3/protocols/protocol_volume_local_adjust.py new file mode 100644 index 000000000..b166f627b --- /dev/null +++ b/xmipp3/protocols/protocol_volume_local_adjust.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# ************************************************************************** +# * +# * Authors: Estrella Fernandez Gimenez (me.fernandez@cnb.csic.es) +# * +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 2 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ************************************************************************** + +from pyworkflow.protocol.params import PointerParam, BooleanParam, IntParam + +from pwem.convert import headers +from pwem.objects import Volume, Transform +from pwem.protocols import EMProtocol + +class XmippProtLocalVolAdj(EMProtocol): + """Protocol to adjust locally volume intensity to a reference volume. Occupancy volume is saved in protocol folder. Based on + https://www.sciencedirect.com/science/article/pii/S1047847723000874?via%3Dihub""" + _label = 'volume local adjustment' + _possibleOutputs = Volume + + # --------------------------- DEFINE param functions -------------------------------------------- + def _defineParams(self, form): + form.addSection(label='Input') + form.addParam('vol1', PointerParam, pointerClass='Volume', label="Reference volume", + help='Specify a volume to be used as reference volume.') + form.addParam('vol2', PointerParam, pointerClass='Volume', label="Input volume", + help='Specify a volume which will be adjusted to the reference volume.') + form.addParam('mask', PointerParam, pointerClass='VolumeMask', label="Mask for reference volume", + help='Specify a mask to define region of interest (which is signal in white (1s) and background in ' + 'black (0s))') + form.addParam('neighborhood', IntParam, label="Neighborhood (A)", default=5, + help='Side length (in Angstroms) of a square which will define the region of adjustment') + form.addParam('subtract', BooleanParam, label="Perform subtraction?", default=False, + help='Perform subtraction of reference volume minus input volume in real space') + # --------------------------- INSERT steps functions -------------------------------------------- + def _insertAllSteps(self): + self._insertFunctionStep('adjustStep') + self._insertFunctionStep('createOutputStep') + + # --------------------------- STEPS functions -------------------------------------------- + def adjustStep(self): + vol1 = self.vol1.get().clone() + fnVol1 = vol1.getFileName() + vol2 = self.vol2.get().getFileName() + if fnVol1.endswith('.mrc'): + fnVol1 += ':mrc' + if vol2.endswith('.mrc'): + vol2 += ':mrc' + program = "xmipp_local_volume_adjust" + args = '--i1 %s --i2 %s -o %s --mask %s --neighborhood %d --sampling %s --save %s' % \ + (fnVol1, vol2, self._getExtraPath("output_volume.mrc"), self.mask.get().getFileName(), + self.neighborhood.get(), vol1.getSamplingRate(), self._getExtraPath()) + if self.subtract.get(): + args += ' --sub' + self.runJob(program, args) + + def createOutputStep(self): + vol1 = self.vol1.get() + volume = Volume() + volume.setSamplingRate(vol1.getSamplingRate()) + if vol1.getFileName().endswith('mrc'): + origin = Transform() + ccp4header = headers.Ccp4Header(vol1.getFileName(), readHeader=True) + shifts = ccp4header.getOrigin() + origin.setShiftsTuple(shifts) + volume.setOrigin(origin) + volume.setFileName(self._getExtraPath("output_volume.mrc")) + filename = volume.getFileName() + if filename.endswith('.mrc') or filename.endswith('.map'): + volume.setFileName(filename + ':mrc') + self._defineOutputs(outputVolume=volume) + + # --------------------------- INFO functions -------------------------------------------- + def _summary(self): + neighborhood = self.neighborhood.get() + vol1 = self.vol1.get() + summary = ["Volume 1: %s\nVolume 2: %s\nInput mask 1: %s\n\nNeighborhood: %d Å (%d px)" % + (vol1.getFileName(), self.vol2.get().getFileName(), self.mask.get().getFileName(), + neighborhood, round(neighborhood/vol1.getSamplingRate()))] + if self.subtract.get(): + summary.append("\nSubtraction performed") + return summary + + def _methods(self): + methods = [] + if not hasattr(self, 'outputVolume'): + methods.append("Output volume not ready yet.") + else: + methods.append("Volume %s adjusted to volume %s" % (self.vol2.get().getFileName(), + self.vol1.get().getFileName())) + return methods + + def _validate(self): + errors = [] + if self.vol1.get().getSamplingRate() != self.vol2.get().getSamplingRate(): + errors.append('Input volumes should have same pixel size') + if self.vol1.get().getSamplingRate() != self.mask.get().getSamplingRate(): + errors.append('\nInput mask and volumes should have same pixel size') + return errors diff --git a/xmipp3/protocols/protocol_volume_local_sharpening.py b/xmipp3/protocols/protocol_volume_local_sharpening.py index 592067ebd..7a95aff09 100644 --- a/xmipp3/protocols/protocol_volume_local_sharpening.py +++ b/xmipp3/protocols/protocol_volume_local_sharpening.py @@ -39,7 +39,7 @@ from ntpath import dirname from os.path import exists -LOCALDEBLUR_METHOD_URL='http://github.com/I2PC/scipion/wiki/XmippProtLocSharp' +LOCALDEBLUR_METHOD_URL='https://github.com/I2PC/scipion/wiki/XmippProtLocSharp' class XmippProtLocSharp(ProtAnalysis3D): """ diff --git a/xmipp3/tests/test_protocol_deep_center.py b/xmipp3/tests/test_protocol_deep_center.py deleted file mode 100644 index ec74959fa..000000000 --- a/xmipp3/tests/test_protocol_deep_center.py +++ /dev/null @@ -1,109 +0,0 @@ -# ************************************************************************** -# * -# * Authors: Adrian Sansinena Rodriguez (adrian.sansinena@cnb.csic.es) -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow.tests import BaseTest, DataSet, setupTestProject - -from pwem.protocols import (ProtImportParticles, ProtSubSet, - exists) - -from pwem import emlib -from xmipp3.protocols import XmippProtDeepCenter - - -class TestDeepCenter(BaseTest): - @classmethod - def runImportParticles(cls): - """ Import Particles. - """ - args = {'importFrom': ProtImportParticles.IMPORT_FROM_SCIPION, - 'sqliteFile': cls.particles, - 'amplitudConstrast': 0.1, - 'sphericalAberration': 2.0, - 'voltage': 200, - 'samplingRate': 0.99, - 'haveDataBeenPhaseFlipped': True - } - - # Id's should be set increasing from 1 if ### is not in the - # pattern - protImport = cls.newProtocol(ProtImportParticles, **args) - protImport.setObjLabel('import particles') - cls.launchProtocol(protImport) - return protImport - - @classmethod - def setUpClass(cls): - setupTestProject(cls) - - # Data - cls.dataset = DataSet.getDataSet('10010') - cls.particles = cls.dataset.getFile('particles') - - cls.protImportParts = cls.runImportParticles() - - def test(self): - subset = self.newProtocol(ProtSubSet, - inputFullSet=self.protImportParts.outputParticles, - chooseAtRandom=True, - nElements=400) - self.launchProtocol(subset) - - deepCenter = self.newProtocol(XmippProtDeepCenter, - inputTrainSet=subset.outputParticles, - Xdim=128, - modelPretrain=False, - numCenModels=5, - numEpochs_cen=1, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepCenter) - - fnModel0 = deepCenter._getExtraPath("modelCenter0.h5") - fnModel2 = deepCenter._getExtraPath("modelCenter2.h5") - - if not exists(fnModel0): - self.assertTrue(False, fnModel0 + " does not exist") - if not exists(fnModel2): - self.assertTrue(False, fnModel2 + " does not exist") - - deepCenter2 = self.newProtocol(XmippProtDeepCenter, - inputTrainSet=subset.outputParticles, - Xdim=128, - modelPretrain=True, - pretrainedModels=deepCenter, - numCenModels=5, - numEpochs_cen=10, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepCenter2) - - fnModel0 = deepCenter2._getExtraPath("modelCenter0.h5") - if not exists(fnModel0): - self.assertTrue(False, fnModel0 + " does not exist") \ No newline at end of file diff --git a/xmipp3/tests/test_protocol_deep_center_assignment.py b/xmipp3/tests/test_protocol_deep_center_assignment.py new file mode 100644 index 000000000..700b1d7b2 --- /dev/null +++ b/xmipp3/tests/test_protocol_deep_center_assignment.py @@ -0,0 +1,132 @@ +# ************************************************************************** +# * +# * Authors: Adrian Sansinena Rodriguez (adrian.sansinena@cnb.csic.es) +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 2 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ************************************************************************** + +################## QUALITY DISCLAIMER ################## +# This tests only proves that this protocol can successfully run without errors! +# It does not guarantee at all that the results will be any good. +# The input train set is the same as the set to predict, which causes artificially great results, +# that will get worse once real use cases are introduced. However, changing that makes no sense +# either, since the quality of the results cannot be checked, due to the non-deterministic nature +# of Neural Networks, specially regarding their training. + +# Scipion em imports +from pwem.protocols import ProtImportParticles, ProtSubSet, exists +from pyworkflow.tests import BaseTest, DataSet, setupTestProject + +# Plugin imports +from ..protocols import XmippProtDeepGlobalAssignment, XmippProtDeepCenter + +# ---------------------------- COMMON FUNCTIONS & VARIABLES ---------------------------- +genericError = "There was a problem while running this protocol." + +def runImportParticles(cls): + """ Import Particles. """ + args = {'importFrom': ProtImportParticles.IMPORT_FROM_SCIPION, + 'sqliteFile': cls.particles, + 'amplitudConstrast': 0.1, + 'sphericalAberration': 2.0, + 'voltage': 200, + 'samplingRate': 0.99, + 'haveDataBeenPhaseFlipped': True + } + + protImport = cls.newProtocol(ProtImportParticles, **args) + protImport.setObjLabel('import particles') + cls.launchProtocol(protImport) + return protImport + +def setUpClassBase(cls): + setupTestProject(cls) + + # Data + cls.dataset = DataSet.getDataSet('10010') + cls.particles = cls.dataset.getFile('particles') + cls.protImportParts = runImportParticles(cls) + +class TestDeepGlobalAssignment(BaseTest): + @classmethod + def setUpClass(cls): + setUpClassBase(cls) + + def test(self): + # Creating subset of particles (same for train and test) + subset = self.newProtocol(ProtSubSet, + inputFullSet=self.protImportParts.outputParticles, + chooseAtRandom=True, + nElements=400) + self.launchProtocol(subset) + + deepGA = self.newProtocol(XmippProtDeepGlobalAssignment, + inputImageSet=subset.outputParticles, + inputTrainSet=subset.outputParticles, + Xdim=128, + numModels=5, + numEpochs=1, + batchSize=32, + learningRate=0.001, + sigma=8, + patience=5) + self.launchProtocol(deepGA) + self.assertIsNotNone(deepGA.outputParticles, genericError) + +class TestDeepCenter(BaseTest): + @classmethod + def setUpClass(cls): + setUpClassBase(cls) + + def test(self): + subset = self.newProtocol(ProtSubSet, + inputFullSet=self.protImportParts.outputParticles, + chooseAtRandom=True, + nElements=400) + self.launchProtocol(subset) + + deepCenter = self.newProtocol(XmippProtDeepCenter, + inputImageSet=subset.outputParticles, + trainModels=True, + inputTrainSet=subset.outputParticles, + numModels=5, + numEpochs=1, + batchSize=32, + learningRate=0.001, + sigma=8, + patience=5) + self.launchProtocol(deepCenter) + + fnModel0 = deepCenter._getExtraPath("model0.h5") + fnModel2 = deepCenter._getExtraPath("model2.h5") + + self.assertTrue(exists(fnModel0), fnModel0 + " does not exist") + self.assertTrue(exists(fnModel2), fnModel2 + " does not exist") + + self.assertIsNotNone(deepCenter.outputParticles, genericError) + + deepCenter2 = self.newProtocol(XmippProtDeepCenter, + inputImageSet=subset.outputParticles, + trainModels=False) + self.launchProtocol(deepCenter2) + + self.assertIsNotNone(deepCenter2.outputParticles, genericError) diff --git a/xmipp3/tests/test_protocol_deep_center_predict.py b/xmipp3/tests/test_protocol_deep_center_predict.py deleted file mode 100644 index a08b2bed2..000000000 --- a/xmipp3/tests/test_protocol_deep_center_predict.py +++ /dev/null @@ -1,95 +0,0 @@ -# ************************************************************************** -# * -# * Authors: Adrian Sansinena Rodriguez (adrian.sansinena@cnb.csic.es) -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow.tests import BaseTest, DataSet, setupTestProject - -from pwem.protocols import (ProtImportParticles, ProtSubSet, - exists) - -from pwem import emlib -from xmipp3.protocols import XmippProtDeepCenterPredict, XmippProtDeepCenter - - -class TestDeepCenterPredict(BaseTest): - @classmethod - def runImportParticles(cls): - """ Import Particles. - """ - args = {'importFrom': ProtImportParticles.IMPORT_FROM_SCIPION, - 'sqliteFile': cls.particles, - 'amplitudConstrast': 0.1, - 'sphericalAberration': 2.0, - 'voltage': 200, - 'samplingRate': 0.99, - 'haveDataBeenPhaseFlipped': True - } - - protImport = cls.newProtocol(ProtImportParticles, **args) - protImport.setObjLabel('import particles') - cls.launchProtocol(protImport) - return protImport - - @classmethod - def setUpClass(cls): - setupTestProject(cls) - - # Data - cls.dataset = DataSet.getDataSet('10010') - cls.particles = cls.dataset.getFile('particles') - - cls.protImportParts = cls.runImportParticles() - - def test(self): - subset = self.newProtocol(ProtSubSet, - inputFullSet=self.protImportParts.outputParticles, - chooseAtRandom=True, - nElements=400) - self.launchProtocol(subset) - - deepCenter = self.newProtocol(XmippProtDeepCenter, - inputTrainSet=subset.outputParticles, - Xdim=128, - modelPretrain=False, - numCenModels=5, - numEpochs_cen=1, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepCenter) - - deepCenterPredict = self.newProtocol(XmippProtDeepCenterPredict, - inputImageSet=subset.outputParticles, - Xdim=128, - inputModel=deepCenter, - numModels=5, - tolerance=2, - maxModels=1) - self.launchProtocol(deepCenterPredict) - - self.assertIsNotNone(deepCenterPredict.outputParticles, - "There was a problem with Deep Center Predict") - diff --git a/xmipp3/tests/test_protocol_deep_global_assignment.py b/xmipp3/tests/test_protocol_deep_global_assignment.py deleted file mode 100644 index 84a89a4b9..000000000 --- a/xmipp3/tests/test_protocol_deep_global_assignment.py +++ /dev/null @@ -1,109 +0,0 @@ -# ************************************************************************** -# * -# * Authors: Adrian Sansinena Rodriguez (adrian.sansinena@cnb.csic.es) -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow.tests import BaseTest, DataSet, setupTestProject - -from pwem.protocols import (ProtImportParticles, ProtSubSet, - exists) - -from pwem import emlib -from xmipp3.protocols import XmippProtDeepGlobalAssignment - - -class TestDeepGlobalAssignment(BaseTest): - @classmethod - def runImportParticles(cls): - """ Import Particles. - """ - args = {'importFrom': ProtImportParticles.IMPORT_FROM_SCIPION, - 'sqliteFile': cls.particles, - 'amplitudConstrast': 0.1, - 'sphericalAberration': 2.0, - 'voltage': 200, - 'samplingRate': 0.99, - 'haveDataBeenPhaseFlipped': True - } - - # Id's should be set increasing from 1 if ### is not in the - # pattern - protImport = cls.newProtocol(ProtImportParticles, **args) - protImport.setObjLabel('import particles') - cls.launchProtocol(protImport) - return protImport - - @classmethod - def setUpClass(cls): - setupTestProject(cls) - - # Data - cls.dataset = DataSet.getDataSet('10010') - cls.particles = cls.dataset.getFile('particles') - - cls.protImportParts = cls.runImportParticles() - - def test(self): - subset = self.newProtocol(ProtSubSet, - inputFullSet=self.protImportParts.outputParticles, - chooseAtRandom=True, - nElements=400) - self.launchProtocol(subset) - - deepGA = self.newProtocol(XmippProtDeepGlobalAssignment, - inputTrainSet=subset.outputParticles, - Xdim=128, - modelPretrain=False, - numAngModels=5, - numEpochs_ang=1, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepGA) - - fnModel0 = deepGA._getExtraPath("modelAngular0.h5") - fnModel2 = deepGA._getExtraPath("modelAngular2.h5") - - if not exists(fnModel0): - self.assertTrue(False, fnModel0 + " does not exist") - if not exists(fnModel2): - self.assertTrue(False, fnModel2 + " does not exist") - - deepGA2 = self.newProtocol(XmippProtDeepGlobalAssignment, - inputTrainSet=subset.outputParticles, - Xdim=128, - modelPretrain=True, - pretrainedModels=deepGA, - numAngModels=5, - numEpochs_ang=10, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepGA2) - - fnModel0 = deepGA2._getExtraPath("modelAngular0.h5") - if not exists(fnModel0): - self.assertTrue(False, fnModel0 + " does not exist") \ No newline at end of file diff --git a/xmipp3/tests/test_protocol_deep_global_assignment_predict.py b/xmipp3/tests/test_protocol_deep_global_assignment_predict.py deleted file mode 100644 index 2a41fd181..000000000 --- a/xmipp3/tests/test_protocol_deep_global_assignment_predict.py +++ /dev/null @@ -1,94 +0,0 @@ -# ************************************************************************** -# * -# * Authors: Adrian Sansinena Rodriguez (adrian.sansinena@cnb.csic.es) -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -from pyworkflow.tests import BaseTest, DataSet, setupTestProject - -from pwem.protocols import (ProtImportParticles, ProtSubSet, - exists) - -from pwem import emlib -from xmipp3.protocols import XmippProtDeepGlobalAssignmentPredict, XmippProtDeepGlobalAssignment - - -class TestDeepGlobalAssignmentPredict(BaseTest): - @classmethod - def runImportParticles(cls): - """ Import Particles. - """ - args = {'importFrom': ProtImportParticles.IMPORT_FROM_SCIPION, - 'sqliteFile': cls.particles, - 'amplitudConstrast': 0.1, - 'sphericalAberration': 2.0, - 'voltage': 200, - 'samplingRate': 0.99, - 'haveDataBeenPhaseFlipped': True - } - - protImport = cls.newProtocol(ProtImportParticles, **args) - protImport.setObjLabel('import particles') - cls.launchProtocol(protImport) - return protImport - - @classmethod - def setUpClass(cls): - setupTestProject(cls) - - # Data - cls.dataset = DataSet.getDataSet('10010') - cls.particles = cls.dataset.getFile('particles') - - cls.protImportParts = cls.runImportParticles() - - def test(self): - subset = self.newProtocol(ProtSubSet, - inputFullSet=self.protImportParts.outputParticles, - chooseAtRandom=True, - nElements=400) - self.launchProtocol(subset) - - deepGA = self.newProtocol(XmippProtDeepGlobalAssignment, - inputTrainSet=subset.outputParticles, - Xdim=128, - modelPretrain=False, - numAngModels=5, - numEpochs_ang=1, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepGA) - - deepGAPredict = self.newProtocol(XmippProtDeepGlobalAssignmentPredict, - inputImageSet=subset.outputParticles, - Xdim=128, - inputModel=deepGA, - numAngModels=5, - tolerance=45, - maxModels=3) - self.launchProtocol(deepGAPredict) - - self.assertIsNotNone(deepGAPredict.outputParticles, - "There was a problem with Deep Center Predict") \ No newline at end of file diff --git a/xmipp3/tests/test_protocols_mixed_movies.py b/xmipp3/tests/test_protocols_mixed_movies.py index 4f15c670b..cfc12ce6d 100644 --- a/xmipp3/tests/test_protocols_mixed_movies.py +++ b/xmipp3/tests/test_protocols_mixed_movies.py @@ -109,10 +109,6 @@ def test_CorrelationOpticalFlow(self): mc1.inputMovies.set(protMovieImport.outputMovies) self.launchProtocol(mc1) - f0, fN, ffi = mc1.outputMovies.getFramesRange() - # Here the first frame index should be 2, since we are not - # re-writing the movie - self.assertEqual((2, 10, 2), (f0, fN, ffi)) # Shifts should be different from zero self.assertNotAlmostEqual(0, self._sumShifts(mc1.outputMovies)) @@ -125,26 +121,9 @@ def test_CorrelationOpticalFlow(self): mc2.inputMovies.set(protMovieImport.outputMovies) self.launchProtocol(mc2) - f0, fN, ffi = mc2.outputMovies.getFramesRange() - # Here the first frame index should be 1, since we wrote a new movie - self.assertEqual((2, 10, 1), (f0, fN, ffi)) # All shifts should be zero, since we have written the move and the # shifts have been already applied self.assertAlmostEqual(0, self._sumShifts(mc2.outputMovies)) - - # test that global alignment has been properly stored - for movie in mc2.outputMovies: - args = '-i {} --save_image_stats'.format(movie.getFileName()) - Plugin.runXmippProgram("xmipp_image_statistics", args) # merge frames - root = movie.getBaseName().split('_movie.mrcs')[0] - mic = [x.getFileName() for x in mc1.outputMicrographs] # when I tried to get mic directly, I was getting wrong name for some strange reason - mic = [x for x in mic if root in x][0] - ih = ImageHandler() - img1 = ih.createImage() - img1.read('average.xmp') - img2 = ih.createImage() - img2.read(mic) - self.assertTrue(img1.equal(img2, 1.0)) of1 = self.newProtocol(XmippProtOFAlignment, objLabel='OF (1)', diff --git a/xmipp3/tests/test_protocols_xmipp_3d.py b/xmipp3/tests/test_protocols_xmipp_3d.py index 634357ae2..55982c2cc 100644 --- a/xmipp3/tests/test_protocols_xmipp_3d.py +++ b/xmipp3/tests/test_protocols_xmipp_3d.py @@ -2715,6 +2715,66 @@ def testXmippResolutionBfactor(self): fscResolution=8.35) self.launchProtocol(protbfactorResolution) +class TestXmippLocalVolAdjust(TestXmippBase): + + @classmethod + def setUpClass(cls): + setupTestProject(cls) + cls.dataset = DataSet.getDataSet(db_xmipp_tutorial) + cls.vol1 = cls.dataset.getFile(vol1_iter2) + cls.vol2 = cls.dataset.getFile(vol2_iter2) + + def testXmippVolSub(self): + print("Import Volume 1") + protImportVol1 = self.newProtocol(ProtImportVolumes, + objLabel='Volume', + filesPath=self.vol1, + samplingRate=7.08) + self.launchProtocol(protImportVol1) + self.assertIsNotNone(protImportVol1.getFiles(), + "There was a problem with the import 1") + + print("Import Volume 2") + protImportVol2 = self.newProtocol(ProtImportVolumes, + objLabel='Volume', + filesPath=self.vol2, + samplingRate=7.08) + self.launchProtocol(protImportVol2) + self.assertIsNotNone(protImportVol2.getFiles(), + "There was a problem with the import 2") + + print("Create mask") + protCreateMask = self.newProtocol(XmippProtCreateMask3D, + inputVolume=protImportVol1.outputVolume, + threshold=0.1) + self.launchProtocol(protCreateMask) + self.assertIsNotNone(protCreateMask.getFiles(), + "There was a problem with the 3D mask") + + print("Run volume local adjust") + protVolLocalAdj = self.newProtocol(XmippProtLocalVolAdj, + vol1=protImportVol1.outputVolume, + vol2=protImportVol2.outputVolume, + mask=protCreateMask.outputMask, + neighborhood=35) + self.launchProtocol(protVolLocalAdj) + self.assertIsNotNone(protVolLocalAdj.outputVolume, "There was a problem with Volumes local adjust") + self.assertEqual(protVolLocalAdj.outputVolume.getSamplingRate(), 7.08, (MSG_WRONG_SAMPLING, "volume")) + self.assertEqual(protVolLocalAdj.outputVolume.getDim(), (64, 64, 64), (MSG_WRONG_DIM, "volume")) + + print("Run volume local adjust with subtraction") + protVolLocalAdjSub = self.newProtocol(XmippProtLocalVolAdj, + vol1=protImportVol1.outputVolume, + vol2=protImportVol2.outputVolume, + mask=protCreateMask.outputMask, + neighborhood=35, + subtract=True) + self.launchProtocol(protVolLocalAdjSub) + self.assertIsNotNone(protVolLocalAdjSub.outputVolume, + "There was a problem with Volumes adjust without computing energy") + self.assertEqual(protVolLocalAdjSub.outputVolume.getSamplingRate(), 7.08, (MSG_WRONG_SAMPLING, "volume")) + self.assertEqual(protVolLocalAdjSub.outputVolume.getDim(), (64, 64, 64), (MSG_WRONG_DIM, "volume")) + if __name__ == "__main__": if len(sys.argv) > 1: className = sys.argv[1] diff --git a/xmipp3/tests/test_protocols_xmipp_mics.py b/xmipp3/tests/test_protocols_xmipp_mics.py index d69fd0058..098a65a0c 100644 --- a/xmipp3/tests/test_protocols_xmipp_mics.py +++ b/xmipp3/tests/test_protocols_xmipp_mics.py @@ -623,11 +623,11 @@ def compare(objId, delta=1.0): def testExtractOther(self): print("Run extract particles from original micrographs, with downsampling") - downFactor = 3.0 protExtract = self.newProtocol(XmippProtExtractParticles, - boxSize=183, downsampleType=OTHER, - doDownsample=True, - downFactor=downFactor, + boxSize=166, downsampleType=OTHER, + doResize=True, + resizeOption=1, + resizeDim=80, doInvert=False, doFlip=False) # Get all the micrographs ids to validate that all particles @@ -643,7 +643,7 @@ def testExtractOther(self): inputCoords = protExtract.inputCoordinates.get() outputParts = protExtract.outputParticles samplingCoords = self.protPP.outputCoordinates.getMicrographs().getSamplingRate() - samplingFinal = self.protImport.outputMicrographs.getSamplingRate() * downFactor + samplingFinal = self.protImport.outputMicrographs.getSamplingRate() * protExtract._getDownFactor() samplingMics = protExtract.inputMicrographs.get().getSamplingRate() factor = samplingFinal / samplingCoords self.assertIsNotNone(outputParts, @@ -665,20 +665,20 @@ def compare(objId, delta=1.0): outputSampling = outputParts.getSamplingRate() self.assertAlmostEqual(outputSampling/samplingMics, - downFactor, 1, + protExtract._getDownFactor(), 1, "There was a problem generating the output.") for particle in outputParts: self.assertTrue(particle.getCoordinate().getMicId() in micsId) self.assertAlmostEqual(outputSampling, particle.getSamplingRate()) - self._checkVarianceAndGiniCoeff(outputParts[170], 1.229023, 0.512485) + self._checkVarianceAndGiniCoeff(outputParts[170], 1.099442, 0.396918) def testExtractNoise(self): # here we will try a different patchSize than the default print("Run extract particles from original micrographs, with downsampling") - downFactor = 5.0 + downFactor = 3.0 protExtract = self.newProtocol(XmippProtExtractParticles, - boxSize=183, downsampleType=OTHER, - doDownsample=True, + boxSize=-1, downsampleType=OTHER, + doResize=True, downFactor=downFactor, doInvert=False, doFlip=False, @@ -692,8 +692,8 @@ def testExtractNoise(self): outputParts = protExtract.outputParticles self.assertIsNotNone(outputParts, "There was a problem generating the output.") - self.assertAlmostEquals(outputParts.getSize(), 395, delta=1) - self._checkVarianceAndGiniCoeff(outputParts[170], 1.1594, 0.5702) + self.assertAlmostEquals(outputParts.getSize(), 403, delta=1) + self._checkVarianceAndGiniCoeff(outputParts[170], 1.161262, 0.5702) def testExtractCTF(self): print("Run extract particles with CTF") diff --git a/xmipp3/viewers/viewer.py b/xmipp3/viewers/viewer.py index 2a9210103..f6ede2ae3 100644 --- a/xmipp3/viewers/viewer.py +++ b/xmipp3/viewers/viewer.py @@ -203,7 +203,7 @@ def _visualize(self, obj, **kwargs): viewParams={ORDER: labels, VISIBLE: labels, RENDER: labelRender, - ZOOM: 25, + ZOOM: 5, MODE: MODE_MD})) if obj.hasAttribute('outputMicrographs'): @@ -216,7 +216,7 @@ def _visualize(self, obj, **kwargs): viewParams={ORDER: labels, VISIBLE: labels, RENDER: labelRender, - ZOOM: 25, + ZOOM: 5, MODE: MODE_MD})) if not(obj.hasAttribute('discardedMicrographs')) and not(obj.hasAttribute('outputMicrographs')): diff --git a/xmipp3/viewers/viewer_resolution_directional.py b/xmipp3/viewers/viewer_resolution_directional.py index 80f5d963d..9e2d17411 100644 --- a/xmipp3/viewers/viewer_resolution_directional.py +++ b/xmipp3/viewers/viewer_resolution_directional.py @@ -158,13 +158,13 @@ def _defineParams(self, form): default=COLOR_JET, label='Color map', help='Select the color map to be applied ' - 'http://matplotlib.org/1.3.0/examples/color/colormaps_reference.html.') + 'https://matplotlib.org/1.3.0/examples/color/colormaps_reference.html.') group.addParam('otherColorMap', StringParam, default='jet', condition = binaryCondition, label='Customized Color map', help='Name of a color map to apply to be applied. Valid names can be found at ' - 'http://matplotlib.org/1.3.0/examples/color/colormaps_reference.html') + 'https://matplotlib.org/1.3.0/examples/color/colormaps_reference.html') group.addParam('sliceAxis', EnumParam, default=AX_Z, choices=['x', 'y', 'z'], display=EnumParam.DISPLAY_HLIST, @@ -525,4 +525,4 @@ def getColorMap(self): cmap = cm.get_cmap(COLOR_CHOICES[self.colorMap.get()]) if cmap is None: cmap = cm.jet - return cmap \ No newline at end of file + return cmap diff --git a/xmipp3/xmipp_logo_original.png b/xmipp3/xmipp_logo_original.png deleted file mode 100644 index 1ada98399..000000000 Binary files a/xmipp3/xmipp_logo_original.png and /dev/null differ