Source code for cryosparc2.protocols.protocol_cryosparc_homogeneous_reconstruction

# **************************************************************************
# *
# * Authors: Yunior C. Fonseca Reyna    (
# *
# *
# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 2 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307  USA
# *
# *  All comments concerning this program package may be sent to the
# *  e-mail address ''
# *
# **************************************************************************
import ast
import os

import emtable
from pkg_resources import parse_version

from pwem import ALIGN_PROJ

import pyworkflow.utils as pwutils
from pyworkflow import NEW
from pyworkflow.object import String
from pyworkflow.protocol.params import (PointerParam, LEVEL_ADVANCED, IntParam,
                                        BooleanParam, EnumParam, FloatParam)
from pwem.objects import Volume

from .protocol_base import ProtCryosparcBase
from ..convert import (defineArgs, convertCs2Star, createItemMatrix,
from ..utils import (addComputeSectionParams, calculateNewSamplingRate,
                     cryosparcValidate, gpusValidate, enqueueJob,
                     waitForCryosparc, clearIntermediateResults, fixVolume,
                     copyFiles, addSymmetryParam, getSymmetry,
                     getCryosparcVersion, get_job_streamlog)
from ..constants import *

[docs]class ProtCryoSparcHomogeneousReconstruct(ProtCryosparcBase): """ Create a 3D reconstruction from input particles that already have alignments in 3D. """ _label = 'homogeneous reconstruction' _className = "homo_reconstruct" _devStatus = NEW _fscColumns = 6 _protCompatibility = [V3_3_0, V3_3_1] ewsParamsName = [] def _initialize(self): self._defineFileNames() def _defineFileNames(self): """ Centralize how files are called. """ myDict = { 'input_particles': self._getTmpPath(''), 'out_particles': self._getExtraPath(''), 'stream_log': self._getPath() + '/stream.log' } self._updateFilenamesDict(myDict) def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputParticles', PointerParam, pointerClass='SetOfParticles', label="Input particles", important=True, help='Select the experimental particles.') form.addParam('refMask', PointerParam, pointerClass='VolumeMask', label='Mask to be applied to this map', allowsNull=True, help="Provide a soft mask where the protein density " "you wish to subtract from the experimental " "particles is white (1) and the rest of the " "protein and the solvent is black (0). " "That is: *the mask should INCLUDE the part of the " "volume that you wish to SUBTRACT.*") form.addSection(label='Homogeneous Reconstruction') form.addParam('refine_N_cmp', IntParam, default=None, allowsNull=True, label="Reconstruction box size (voxels)", help='The volume size to use for refinement. Default: raw particle size') addSymmetryParam(form, help="Symmetry String (C, D, I, O, T). E.g. C1, " "D7, C4, etc") form.addParam('refine_helical_twist', IntParam, default=None, allowsNull=True, label="Helical twist (degrees)", help='Helical twist to apply in degrees') form.addParam('refine_helical_shift', IntParam, default=None, allowsNull=True, label="Helical rise (Angstroms)", help='Helical rise to apply in Angstroms') form.addParam('refine_hsym_order', IntParam, default=None, allowsNull=True, label="Helical symmetry order to apply", help='The maximum amount of helical symmetry to impose ' 'during reconstruction; i.e., the maximum number ' 'of (twist, rise) pairs to backproject each ' 'particle image with. For particles picked outside ' 'of the filament tracer or template picker, this ' 'should be set to the distance between extracted ' 'boxes, divided by the helical rise. If left as ' 'None, will be calculated based on the inter-box ' 'distance (if available). Set to 1 for no helical ' 'symmetry during reconstruction.') form.addParam('refine_gs_resplit', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Force re-do GS split", help='Force re-splitting the particles into two random ' 'gold-standard halves. If this is not set, split is ' 'preserved from input alignments') form.addParam('refine_do_flip', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Flip the reconstruction hand", help='Flip the hand of the reconstruction, and correct ' 'the alignments. If flipping the hand and applying ' 'helical symmetry, ensure that the sign of the ' 'helical twist is inverted to match the hand of ' 'the new symmetry.') form.addParam('refine_ignore_tilt', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Ignore the tilt", help='Ignore the tilt') form.addParam('refine_ignore_trefoil', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Ignore trefoil", help='Ignore trefoil') form.addParam('refine_ignore_tetra', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Ignore tetra", help='Ignore tetra') form.addParam('refine_ignore_anisomag', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Ignore anisomag", help='Ignore the anisomag') csVersion = getCryosparcVersion() if parse_version(csVersion) >= parse_version(V3_3_1): form.addParam('refine_do_ews_correct', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Do EWS correction", help='Whether or not to correct for the curvature of the ' 'Ewald Sphere.') form.addParam('refine_ews_zsign', EnumParam, choices=['positive', 'negative'], default=0, expertLevel=LEVEL_ADVANCED, label="EWS curvature sign", help='Whether to use positive or negative curvature in ' 'Ewald Sphere correction.') form.addParam('refine_ews_simple', EnumParam, choices=['simple', 'iterative'], default=0, expertLevel=LEVEL_ADVANCED, label="EWS correction method", help='Whether to use the simple insertion method, or to ' 'use an iterative optimization method, for Ewald ' 'Sphere correction.') form.addParam('refine_fsc_mask_opt', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Optimize FSC mask", help='Whether or not to optimize the mask used for ' 'calculating FSCs') form.addParam('refine_override_filter', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Override FSC Filtering", help='Whether to override the FSC-filtering and use ' 'manual filtering and sharpening (set below) ' 'instead. FSC is still computed, just not used ' 'for filtering.') form.addParam('refine_override_filter_res', FloatParam, default=None, allowsNull=True, expertLevel=LEVEL_ADVANCED, label="Override Filtering Resolution", help='Override filter corner resolution (A)') form.addParam('refine_override_filter_order', IntParam, default=8, expertLevel=LEVEL_ADVANCED, label="Override Filtering Order", help='Override filter order (Butterworth)') form.addParam('refine_override_filter_bfactor', FloatParam, default=0, expertLevel=LEVEL_ADVANCED, label="Override Filtering Bfactor", help='Override sharpening B-factor. Negative to sharpen') self.ewsParamsName = ['refine_do_ews_correct', 'refine_ews_zsign', 'refine_fsc_mask_opt', 'refine_override_filter', 'refine_override_filter_res', 'refine_ews_simple', 'refine_override_filter_order', 'refine_override_filter_bfactor'] form.addParam('recon_do_expand_mask', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Wide mask final output", help='Apply a wide mask to the final map, to reduce file ' 'size after compression.') form.addParam('intermediate_plots', BooleanParam, default=False, expertLevel=LEVEL_ADVANCED, label="Show plots from intermediate steps", help='Hide plots from intermediate steps to speed up ' 'processing.') form.addParam('refine_compute_batch_size', IntParam, default=None, expertLevel=LEVEL_ADVANCED, allowsNull=True, label="GPU batch size of images", help='Batch size of images to process at a time on the ' 'GPU. If you run out of GPU memory, try setting ' 'this to a small number to override the ' 'auto-detect procedure.') # --------------[Compute settings]--------------------------- form.addSection(label="Compute settings") addComputeSectionParams(form, allowMultipleGPUs=False) # --------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): self._defineFileNames() self._defineParamsName() self._initializeCryosparcProject() self._insertFunctionStep(self.convertInputStep) self._insertFunctionStep(self.processStep) self._insertFunctionStep(self.createOutputStep)
[docs] def processStep(self): print(pwutils.yellowStr("Homogeneous Reconstruction started..."), flush=True) self.doHomogeneousReconstruction()
[docs] def createOutputStep(self): """ Create the protocol output. Convert cryosparc file to Relion file """ self._initializeUtilsVariables() idd = self.findLastIteration(self.runHomogeneousReconstruction.get()) csOutputFolder = os.path.join(self.projectPath, self.projectName.get(), self.runHomogeneousReconstruction.get()) csOutputPattern = "cryosparc_%s_%s" % (self.projectName.get(), self.runHomogeneousReconstruction.get()) csParticlesName = csOutputPattern + "_particles.cs" fnVolName = csOutputPattern + "_volume_map.mrc" half1Name = csOutputPattern + "_volume_map_half_A.mrc" half2Name = csOutputPattern + "_volume_map_half_B.mrc" # Copy the CS output to extra folder copyFiles(csOutputFolder, self._getExtraPath(), files=[csParticlesName, fnVolName, half1Name, half2Name]) csFile = os.path.join(self._getExtraPath(), csParticlesName) outputStarFn = self._getFileName('out_particles') argsList = [csFile, outputStarFn] parser = defineArgs() args = parser.parse_args(argsList) convertCs2Star(args) fnVol = os.path.join(self._getExtraPath(), fnVolName) half1 = os.path.join(self._getExtraPath(), half1Name) half2 = os.path.join(self._getExtraPath(), half2Name) imgSet = self._getInputParticles() vol = Volume() fixVolume([fnVol, half1, half2]) vol.setFileName(fnVol) vol.setSamplingRate(calculateNewSamplingRate(vol.getDim(), imgSet.getSamplingRate(), imgSet.getDim())) vol.setHalfMaps([half1, half2]) outImgSet = self._createSetOfParticles() outImgSet.copyInfo(imgSet) self._fillDataFromIter(outImgSet) self._defineOutputs(outputVolume=vol) self._defineSourceRelation(self.inputParticles.get(), vol) self._defineOutputs(outputParticles=outImgSet) self._defineTransformRelation(self.inputParticles.get(), outImgSet) self.createFSC(idd, imgSet, vol)
def _fillDataFromIter(self, imgSet): outImgsFn = 'particles@' + self._getFileName('out_particles') imgSet.setAlignmentProj() imgSet.copyItems(self._getInputParticles(), updateItemCallback=self._createItemMatrix, itemDataIterator=emtable.Table.iterRows(outImgsFn)) def _createItemMatrix(self, particle, row): createItemMatrix(particle, row, align=ALIGN_PROJ) setCryosparcAttributes(particle, row, RELIONCOLUMNS.rlnRandomSubset.value)
[docs] def findLastIteration(self, jobName): get_job_streamlog(self.projectName.get(), jobName, self._getFileName('stream_log')) # Get the metadata information from stream.log with open(self._getFileName('stream_log')) as f: data = f.readlines() x = ast.literal_eval(data[0]) # Find the ID of last iteration and the map resolution for y in x: if 'text' in y: z = str(y['text']) if z.startswith('FSC, after mask auto-tightening'): idd = y['imgfiles'][2]['fileid'] return idd
# --------------------------- INFO functions ------------------------------- def _validate(self): """ Should be overwritten in subclasses to return summary message for NORMAL EXECUTION. """ validateMsgs = cryosparcValidate() if not validateMsgs: csVersion = getCryosparcVersion() if [version for version in self._protCompatibility if parse_version(version) >= parse_version(csVersion)]: validateMsgs = gpusValidate(self.getGpuList(), checkSingleGPU=True) if not validateMsgs: particles = self._getInputParticles() if not particles.hasCTF(): validateMsgs.append("The Particles has not associated a " "CTF model") if not validateMsgs and not particles.hasAlignment3D(): validateMsgs.append("The Particles has not a 3D " "alignment") else: validateMsgs.append("The protocol is not compatible with the " "cryoSPARC version %s" % csVersion) return validateMsgs def _summary(self): summary = [] if (not hasattr(self, 'outputVolume') or not hasattr(self, 'outputParticles')): summary.append("Output objects not ready yet.") else: summary.append("Input Particles: %s" % self.getObjectTag('inputParticles')) summary.append("Input Mask: %s" % self.getObjectTag('refMask')) summary.append("Symmetry: %s" % getSymmetry(self.symmetryGroup.get(), self.symmetryOrder.get()) ) summary.append("------------------------------------------") summary.append("Output particles %s" % self.getObjectTag('outputParticles')) summary.append("Output volume %s" % self.getObjectTag('outputVolume')) if self.hasAttribute('mapResolution'): summary.append( "\nMap Resolution: %s" % self.mapResolution.get()) if self.hasAttribute('estBFactor'): summary.append( '\nEstimated Bfactor: %s' % self.estBFactor.get()) return summary def _defineParamsName(self): """ Define a list with all protocol parameters names""" self._paramsName = ['refine_N_cmp', 'refine_gs_resplit', 'refine_do_flip', 'refine_ignore_tilt', 'refine_ignore_trefoil', 'refine_ignore_tetra', 'refine_ignore_anisomag', 'refine_fsc_mask_opt', 'recon_do_expand_mask', 'intermediate_plots', 'refine_symmetry', 'refine_compute_batch_size', 'refine_helical_twist', 'refine_helical_shift', 'refine_hsym_order', 'compute_use_ssd'] + self.ewsParamsName self.lane = str(self.getAttributeValue('compute_lane'))
[docs] def doHomogeneousReconstruction(self): input_group_connect = {"particles": self.particles.get()} if self.mask.get() is not None: input_group_connect["mask"] = self.mask.get() params = {} for paramName in self._paramsName: if (paramName != 'refine_N_cmp' and paramName != 'refine_compute_batch_size' and paramName != 'refine_symmetry' and paramName != 'refine_helical_twist' and paramName != 'refine_helical_shift' and paramName != 'refine_hsym_order' and paramName != 'refine_fsc_mask_opt' and paramName != 'refine_ews_zsign' and paramName != 'refine_ews_simple' and paramName != 'refine_override_filter_res'): params[str(paramName)] = str(self.getAttributeValue(paramName)) elif paramName == 'refine_fsc_mask_opt': params[str(paramName)] = str("True") elif paramName == 'refine_N_cmp' and self.refine_N_cmp.get() is not None: params[str(paramName)] = str(self.getAttributeValue(paramName)) elif paramName == 'refine_compute_batch_size' and self.refine_compute_batch_size.get() is not None: params[str(paramName)] = str(self.getAttributeValue(paramName)) elif paramName == 'refine_symmetry': symetryValue = getSymmetry(self.symmetryGroup.get(), self.symmetryOrder.get()) params[str(paramName)] = symetryValue elif paramName == 'refine_helical_twist' and self.refine_helical_twist.get() is not None: params[str(paramName)] = str(self.getAttributeValue(paramName)) elif paramName == 'refine_helical_shift' and self.refine_helical_shift.get() is not None: params[str(paramName)] = str(self.getAttributeValue(paramName)) elif paramName == 'refine_hsym_order' and self.refine_hsym_order.get() is not None: params[str(paramName)] = str(self.getAttributeValue(paramName)) elif paramName == 'refine_ews_zsign': params[str(paramName)] = str(EWS_CURVATURE_SIGN[self.refine_ews_zsign.get()]) elif paramName == 'refine_ews_simple': params[str(paramName)] = str(EWS_CORRECTION_METHOD[self.refine_ews_simple.get()]) elif paramName == 'refine_override_filter_res' and self.refine_override_filter_res.get() is not None: params[str(paramName)] = str(self.getAttributeValue(paramName)) # Determinate the GPUs to use (in dependence of # the cryosparc version) try: gpusToUse = self.getGpuList() except Exception: gpusToUse = False runHomogeneousReconstructionJob = enqueueJob(self._className, self.projectName.get(), self.workSpaceName.get(), str(params).replace('\'', '"'), str(input_group_connect).replace('\'', '"'), self.lane, gpusToUse) self.runHomogeneousReconstruction = String(runHomogeneousReconstructionJob.get()) self.currenJob.set(runHomogeneousReconstructionJob.get()) self._store(self) waitForCryosparc(self.projectName.get(), self.runHomogeneousReconstruction.get(), "An error occurred in the homogeneous reconstruction process. " "Please, go to cryoSPARC software for more " "details.") clearIntermediateResults(self.projectName.get(), self.runHomogeneousReconstruction.get())