Source code for tomo.protocols.protocol_ts_import

# **************************************************************************
# *
# * Authors:     J.M. De la Rosa Trevin ( [1]
# *
# * [1] SciLifeLab, Stockholm University
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 2 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307  USA
# *
# *  All comments concerning this program package may be sent to the
# *  e-mail address ''
# *
# **************************************************************************

import os
import re
from glob import glob
import time
from datetime import timedelta, datetime
from collections import OrderedDict
from os.path import join
from statistics import mean

import numpy as np
from sqlite3 import OperationalError

import pyworkflow as pw
import pyworkflow.protocol.params as params
import pyworkflow.utils as pwutils
from pwem.objects import Transform
from pyworkflow.object import Integer
from pyworkflow.utils import yellowStr
from import Message
from pwem.emlib.image import ImageHandler
from pwem.protocols import ProtImport

from tomo.convert import (getAnglesFromHeader, getAnglesFromMdoc,
from tomo.convert.mdoc import normalizeTSId, MDoc
from tomo.objects import TomoAcquisition

from .protocol_base import ProtTomoBase

[docs]class ProtImportTsBase(ProtImport, ProtTomoBase): """ Base class for Tilt-Series and Tilt-SeriesMovies import protocols. """ IMPORT_FROM_FILES = 0 # How to handle the input files into the project IMPORT_COPY_FILES = 0 IMPORT_LINK_ABS = 1 IMPORT_LINK_REL = 2 ANGLES_FROM_FILENAME = 'Filename' ANGLES_FROM_HEADER = 'Header' ANGLES_FROM_MDOC = 'Mdoc' ANGLES_FROM_TLT = 'Tlt' ANGLES_FROM_RANGE = 'Range' NOT_MDOC_GUI_COND = ('filesPattern is None or ' + '(filesPattern is not None and ".mdoc" ' + 'not in filesPattern)') MDOC_DATA_SOURCE = False acquisitions = None sRates = None accumDoses = None incomingDose = None meanDosesPerFrame = None def __init__(self, **args): ProtImport.__init__(self, **args) ProtTomoBase.__init__(self) self.skippedMdocs = Integer() # -------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): form.addSection(label='Import') form.addParam('filesPath', params.PathParam, label="Files directory", help="Root directory of the tilt-series " "(or movies) files.") form.addParam('filesPattern', params.StringParam, label='Pattern', help="It determines if the tilt series are going to " "be imported using the mdoc file or the tilt " "series files. To import from the mdoc files, " "the word '.mdoc' must appear in the pattern, " "if not, a tilt series pattern is expected. " "In the first case, the angular and acquisition " "data are directly read from the corresponding " "mdoc file, while in the second it is read " "the base name of the matching files, according to " " the pattern introduced.\n\n" "*IMPORTING WITH MDOC FILES*\n\n" "For *tilt series movies*, a mdoc per tilt series " "movies is expected. " "The corresponding movie file/s must be located in " "the same path as the mdoc file. The tilt series " "id will be the base name of the mdoc files, " "so the names of the mdoc files must be different, " "even if they're located in " "different paths.\n\n" "For *tilt series*, the only difference is that a " "stack .mrcs file is expected for each " "mdoc, which means, per each tilt series desired " "to be imported.\n\n" "*IMPORTING WITH A PATTERN OF THE TILT SERIES FILE " "NAMES*\n\n" "The pattern can contain standard wildcards such " "as *, ?, etc.\n\n" "It should also contains the following special " "tags:\n" " *{TS}*: tilt series identifier, which can be " "any UNIQUE part of the path. This must be " "an alpha-numeric sequence (avoid symbols as -) " "that can not start with a number.\n" " *{TO}*: acquisition order, an integer value " "(important for dose).\n" " *{TA}*: tilt angle, a positive or negative " "float value.\n\n" "Examples:\n\n" "To import a set of image stacks (tilt-series " "or tilt-series movies) as: \n" "TiltSeries_a_001_0.0.mrc\n" "TiltSeries_a_002_3.0.mrc\n" "TiltSeries_a_003_-3.0.mrc\n" "...\n" "TiltSeries_b_001_0.0.mrc\n" "TiltSeries_b_002_3.0.mrc\n" "TiltSeries_b_003_-3.0.mrc\n" "...\n" "The pattern TiltSeries_{TS}_{TO}_{TA}.mrc will " "identify:\n" "{TS} as a, b, ...\n" "{TO} as 001, 002, 003, ...\n" "{TA} as 0.0, 3.0, -3.0, ...\n") form.addParam('exclusionWords', params.StringParam, label='Exclusion words:', help="List of words separated by a space that the path " "should not have", expertLevel=params.LEVEL_ADVANCED) form.addParam('mdocInfo', params.LabelParam, condition='not (%s)' % self.NOT_MDOC_GUI_COND, label='Acquisition values provided below will override ' 'the mdoc corresponding values', important=True) self._defineAngleParam(form) form.addParam('importAction', params.EnumParam, default=self.IMPORT_LINK_REL, choices=['Copy files', 'Absolute symlink', 'Relative symlink'], display=params.EnumParam.DISPLAY_HLIST, expertLevel=params.LEVEL_ADVANCED, label="Import action on files", help="This parameters determine how the project will " "deal with imported files. It can be: \n" "*Copy files*: Input files will be copied into " "your project. (this will duplicate the raw data)." "*Absolute symlink*: Create symbolic links to the " "absolute path of the files." "*Relative symlink*: Create symbolic links as " "relative path from the protocol run folder. ") self._defineAcquisitionParams(form) form.addSection('Streaming') form.addParam('dataStreaming', params.BooleanParam, default=False, label="Process data in streaming?", help="Select this option if you want import data as it " "is generated and process on the fly by next " "protocols. In this case the protocol will " "keep running to check new files and will " "update the output Set, which can " "be used right away by next steps.") form.addParam('timeout', params.IntParam, default=43200, condition='dataStreaming', label="Timeout (secs)", help="Interval of time (in seconds) after which, " "if no new file is detected, the protocol will " "end. When finished, the output Set will be " "closed and no more data will be " "added to it. \n" "Note 1: The default value is high (12 hours) to " "avoid the protocol finishes during the acq of the " "microscope. You can also stop it from right click " "and press STOP_STREAMING.\n" "Note 2: If you're using individual frames when " "importing movies, the timeout won't be refreshed" "until a whole movie is stacked.") form.addParam('fileTimeout', params.IntParam, default=30, condition='dataStreaming', label="File timeout (secs)", help="Interval of time (in seconds) after which, if a " "file has not changed, we consider it as a new " "file.\n") self._defineBlacklistParams(form) def _defineAngleParam(self, form): """ Used in subclasses to define the option to fetch tilt angles. """ pass def _defineAcquisitionParams(self, form): """ Define acq parameters, it can be overridden by subclasses to change what parameters to include. """ group = form.addGroup('Acquisition info - ' 'override mdoc values if provided') group.addParam('voltage', params.FloatParam, label=Message.LABEL_VOLTAGE, allowsNull=True, help=Message.TEXT_VOLTAGE) group.addParam('sphericalAberration', params.FloatParam, default=2.7, label=Message.LABEL_SPH_ABERRATION, expertLevel=params.LEVEL_ADVANCED, help=Message.TEXT_SPH_ABERRATION) group.addParam('amplitudeContrast', params.FloatParam, default=0.1, label=Message.LABEL_AMPLITUDE, expertLevel=params.LEVEL_ADVANCED, help=Message.TEXT_AMPLITUDE) group.addParam('magnification', params.IntParam, label=Message.LABEL_MAGNI_RATE, allowsNull=True, help=Message.TEXT_MAGNI_RATE) group.addParam('samplingRate', params.FloatParam, label=Message.LABEL_SAMP_RATE, allowsNull=True, help=Message.TEXT_SAMP_RATE) group.addParam('tiltAxisAngle', params.FloatParam, label='Tilt axis angle (deg.)', allowsNull=True) line = group.addLine('Dose (electrons/sq.Å)', help="Initial accumulated dose (usually 0) and " "dose per tilt image (electrons/sq.Å). ") line.addParam('doseInitial', params.FloatParam, default=0, label='Initial dose') line.addParam('dosePerFrame', params.FloatParam, allowsNull=True, label='Dose per tilt image') return group def _defineBlacklistParams(self, form): """ Override to add options related to blacklist info. """ pass # -------------------------- INSERT functions ----------------------------- def _insertAllSteps(self): self._initialize() self._insertFunctionStep(self.importStep) # -------------------------- STEPS functions ------------------------------
[docs] def importStep(self): """ Copy images matching the filename pattern Register other parameters. """ accumDoseList = [] incomingDoseList = [] samplingRate = self.samplingRate.get() counter = 0 if not self.MDOC_DATA_SOURCE:"Using glob pattern: '%s'" % self._globPattern)"Using regex pattern: '%s'" % self._regexPattern) outputSet = getattr(self, self._outputName, None) if outputSet is None: createSetFunc = getattr(self, self._createOutputName) outputSet = createSetFunc() elif outputSet.getSize() > 0: outputSet.loadAllProperties() for ts in outputSet: self._existingTs.add(ts.getTsId()) self._fillAcquisitionInfo(outputSet) tsClass = outputSet.ITEM_TYPE tiClass = tsClass.ITEM_TYPE finished = False lastDetectedChange = # Ignore the timeout variables if we are not really in streaming mode if self.dataStreaming: timeout = timedelta(seconds=self.timeout.get()) fileTimeout = timedelta(seconds=self.fileTimeout.get()) else: timeout = timedelta(seconds=5) fileTimeout = timedelta(seconds=5) while not finished: time.sleep(3) # wait 3 seconds before check for new files someNew = False # Check if some new TS has been found someAdded = False # Check if some new were added # incompleteTs = False # Check if there are incomplete TS matchingFiles = self.getMatchingFiles(fileTimeOut=fileTimeout) if self._existingTs: outputSet.enableAppend() for ts, tiltSeriesList in matchingFiles.items(): someNew = True tsObj = tsClass(tsId=ts) # Form value has higher priority than the mdoc values samplingRate =\ float(samplingRate if samplingRate else self.sRates[ts]) origin = Transform() tsObj.setOrigin(origin) tsObj.setAnglesCount(len(tiltSeriesList)) # we need this to set mapper before adding any item outputSet.append(tsObj) if self.MDOC_DATA_SOURCE: accumDoseList = self.accumDoses[ts] # tsObj.getAcquisition().setAccumDose(accumDoseList[-1]) incomingDoseList = self.incomingDose[ts] counter = 0 tiltSeriesObjList = [] toList = [ti[1] for ti in tiltSeriesList] # Add tilt images to the tiltSeries for f, to, ta in tiltSeriesList: try: # Link/move to extra if type(f) == tuple: imageFile = f[1] else: imageFile = f finalDestination =\ self._getExtraPath(os.path.basename(imageFile)) self.copyOrLink(imageFile, finalDestination) if type(f) == tuple: f = f[0], finalDestination else: f = finalDestination ti = tiClass(location=f, acquisitionOrder=to, tiltAngle=ta) ti.setAcquisition(tsObj.getAcquisition().clone()) if self.MDOC_DATA_SOURCE: dosePerFrame = incomingDoseList[counter] accumDose = accumDoseList[counter] else: dosePerFrame = self.dosePerFrame.get() accumDose =\ self.dosePerFrame.get() *\ int(to if min(toList) == 1 else (int(to) + 1)) # Incoming dose in current ti ti.getAcquisition().setDosePerFrame(dosePerFrame) # Accumulated dose in current ti ti.getAcquisition().setAccumDose(accumDose) tiltSeriesObjList.append(ti) counter += 1 # TODO: variable e is assigned but never used except OperationalError as e: raise Exception("%s is an invalid for the {TS} field, " "it must be an alpha-numeric sequence " "(avoid symbols as -) that can not " "start with a number." % ts) # Sort tilt image metadata if importing tilt series if not self._isImportingTsMovies(): tiltSeriesObjList.sort(key=lambda x: x.getTiltAngle(), reverse=False) for ti in tiltSeriesObjList: tsObj.append(ti) tsObjFirstItem = tsObj.getFirstItem() origin.setShifts(-tsObjFirstItem.getXDim() / 2 * samplingRate, -tsObjFirstItem.getYDim() / 2 * samplingRate, 0) if self.MDOC_DATA_SOURCE: tsObj.getAcquisition().setAccumDose(accumDoseList[-1]) # Tilt series object dose per frame has been updated each # time the tilt image dose per frame has # been updated before, so the mean value is used to be the # reference in the acquisition of the # whole tilt series movie tsObj.getAcquisition().setDosePerFrame( mean(incomingDoseList)) else: tsObj.getAcquisition().setDosePerFrame( self.dosePerFrame.get()) tsObj.getAcquisition().setAccumDose( self.dosePerFrame.get() * len(tiltSeriesList)) tsObj.getAcquisition().setTiltAxisAngle( self.tiltAxisAngle.get()) outputSet.update(tsObj) # update items and size info self._existingTs.add(ts) someAdded = True if someAdded: self.debug('Updating output...') outputSet.updateDim() self._updateOutputSet(self._outputName, outputSet, state=outputSet.STREAM_OPEN) self.debug('Update Done.') self.debug('Checking if finished...someNew: %s' % someNew) now = if not someNew: # If there are no new detected files, we should check the # inactivity time elapsed (from last event to now) and # if it is greater than the defined timeout, we conclude # the import and close the output set # Another option is to check if the protocol have some # special stop condition, this can be used to manually stop # some protocols such as import movies finished = (now - lastDetectedChange > timeout or self.streamingHasFinished()) self.debug("Checking if finished:") self.debug(" Now - Last Change: %s" % pwutils.prettyDelta(now - lastDetectedChange)) else: # If we have detected some files, we should update # the timestamp of the last event lastDetectedChange = now finished = not self.isInStreaming() self.debug("Finished: %s" % finished) # Close the output set self._updateOutputSet(self._outputName, outputSet, state=outputSet.STREAM_CLOSED)
# -------------------------- INFO functions ------------------------------- def _summary(self): summary = [] if self.getOutputsSize(): for key, output in self.iterOutputAttributes(): summary.append("Imported tilt-series %s from: %s" % ( "movies" if output.getLastName() == "outputTiltSeriesM" else "", self.filesPath.get())) summary.append("Using pattern: %s" % self.filesPattern.get()) summary.append(u"Sampling rate: *%0.2f* (Å/px)" % output.getSamplingRate()) else: summary.append(Message.TEXT_NO_OUTPUT_FILES) if self.skippedMdocs.get(): summary.append('*%i* mdoc files were skipped --> ' 'check the Output Log tab for more details.' % self.skippedMdocs.get()) return summary def _validate(self): self._initialize() try: matching = self.getMatchingFiles(isValidation=True) except Exception as e: errorStr = str(e) return [errorStr] if not matching: return ["There are no files matching the pattern %s" % self._globPattern] self._firstMatch = list(matching.items())[0] self._tiltAngleList = self._getSortedAngles(self._firstMatch[1]) errMsg = [] errorMsgAngles = self._validateAngles() if errorMsgAngles: errMsg.append(errorMsgAngles) # In the mdoc case, voltage, magnification and sampling # rate are optional inputs. In the user introduces # one of these values, it will be considered more # prior than the corresponding value read from the mdoc file if not self.MDOC_DATA_SOURCE: if not self.voltage.get(): errMsg.append('Voltage should be a float') if not self.magnification.get(): errMsg.append('Magnification should be an integer') if not self.samplingRate.get(): errMsg.append('Sampling rate should be a float') if not self.dosePerFrame.get(): errMsg.append('Dose per frame should be a float') if self.tiltAxisAngle.get() is None: errMsg.append('Tilt axis angle should be a float') return errMsg def _validateAngles(self): """ Function to be implemented in subclass to validate the angles range. """ return None # -------------------------- BASE methods to be overridden ---------------- # def _getImportChoices(self): # """ Return a list of possible choices # from which the import can be done. # (usually packages form such as: xmipp3, eman2, relion...etc. # """ # return ['files'] # -------------------------- UTILS functions ------------------------------ def _initialize(self): """ Initialize some internal variables such as: - patterns: Expand the pattern using environ vars or username and also replacing special character # by digit matching. - outputs: Output variable names """ self.MDOC_DATA_SOURCE = 'mdoc' in self.filesPattern.get() if not self.MDOC_DATA_SOURCE: path = self.filesPath.get('').strip() pattern = self.filesPattern.get('').strip() self._pattern = os.path.join(path, pattern) if pattern else path def _replace(p, ts, to, ta): p = p.replace('{TS}', ts) p = p.replace('{TO}', to) p = p.replace('{TA}', ta) return p self._regexPattern = _replace(self._pattern.replace('*', '(.*)'), r'(?P<TS>.*)', r'(?P<TO>\d+)', r'(?P<TA>[+-]?\d+(\.\d+)?)') self._regex = re.compile(self._regexPattern) self._globPattern = _replace(self._pattern, '*', '*', '*') # Set output names depending on the import type # (either movies or images) self._outputName = 'outputTiltSeries' self._createOutputName = '_createSetOfTiltSeries' # Keep track of which existing tilt-series has already been found self._existingTs = set() def _anglesInPattern(self): """ This function should be called after a call to _initialize""" return '{TA}' in self._pattern and '{TO}' in self._pattern def _getMatchingFilesFromMdoc(self, isValidation): """If the list of files provided by the user is a list of mdoc files, then the tilt series movies are built from them, following the considerations listed below: - For each mdoc file, it and the corresponding movie files must be in the same directory. - The tilt series id will be the base name of the mdoc file, by default, so the mdocs must have different base name. If another name is desired, the user can introduce the name structure (see advanced parameter) """ fpath = self.filesPath.get() mdocList = glob(join(fpath, self.filesPattern.get())) hasDoseList = [] if not mdocList: raise Exception('No mdoc files were found in the ' 'introduced path:\n%s' % fpath) matchingFiles = OrderedDict() self.acquisitions = OrderedDict() self.sRates = OrderedDict() self.accumDoses = OrderedDict() self.incomingDose = OrderedDict() warningHeadMsg = yellowStr('The following mdoc files were skipped. ' 'See details below:\n\n') warningDetailedMsg = [] skippedMdocs = 0 for mdoc in mdocList: # Note: voltage, magnification and sampling rate values are the # ones introduced by the user in the protocol's form. # Otherwise, the corresponding values considered will be the ones # read from the mdoc. # This is because because you can't trust mdoc # (often dose is not calibrated in serialem, so you get 0; # pixel size might be binned as mdoc comes from a binned record # not movie and there are no Cs and amp # contrast fields in mdoc) mdocObj = MDoc( mdoc, voltage=self.voltage.get() if self.voltage.get() else None, magnification=(self.magnification.get() if self.magnification.get() else None), samplingRate=(self.samplingRate.get() if self.samplingRate.get() else None), doseProvidedByUser=(self.dosePerFrame.get() if self.dosePerFrame.get() else None), tiltAngleProvidedByUser=(self.tiltAxisAngle.get() if self.tiltAxisAngle.get() else None)) validationError = isImportingTsMovies=self._isImportingTsMovies()) hasDoseList.append(mdocObj.mdocHasDose) if validationError: warningHeadMsg += yellowStr('\t- %s\n' % mdoc) warningDetailedMsg.append(validationError) skippedMdocs += 1 # validationErrors.append(validationError) # Continue parsing the remaining mdoc files to # provide a fully detailed error message continue acquisition = self._genTsAcquisitionFromMdoc(mdocObj) tsId = mdocObj.getTsId() fileOrderAngleList = [] accumulatedDoseList = [] incomingDoseList = [] for tiltMetadata in mdocObj.getTiltsMetadata(): fileOrderAngleList.append(( tiltMetadata.getAngleMovieFile(), # Filename '{:03d}'.format(tiltMetadata.getAcqOrder()), # Acquisition # order tiltMetadata.getTiltAngle(), # Tilt angle )) accumulatedDoseList.append(tiltMetadata.getAccumDose()) incomingDoseList.append(tiltMetadata.getIncomingDose()) # self._getTsIdFromMdocData(fileList) matchingFiles[tsId] = fileOrderAngleList self.acquisitions[tsId] = acquisition self.sRates[tsId] = mdocObj.getSamplingRate() self.accumDoses[tsId] = accumulatedDoseList self.incomingDose[tsId] = incomingDoseList if isValidation: if matchingFiles: if warningDetailedMsg: self.skippedMdocs.set(skippedMdocs) self._store(self.skippedMdocs) print(warningHeadMsg + ' '.join(warningDetailedMsg)) return matchingFiles else: # If the only info missing is the dose related data, # it's suggested to introduce it manually if not any(hasDoseList): raise Exception('*The dose was not possible to be ' 'obtained from any of the provided ' 'mdoc files.\n' 'Please check the data of your mdoc ' 'files or introduce a dose value in the ' 'protocol form.\n\n' 'Dose related mdoc labels are:\n\n' '- ExposureDose or\n' '- FrameDosesAndNumber or\n' '- DoseRate and ExposureTime or\n' '- MinMaxMean and CountsPerElectron' ) else: raise Exception('*All the mdoc files introduced present ' 'validation errors.*\n\n%s' % (warningHeadMsg + ' '.join(warningDetailedMsg))) else: return matchingFiles def _isImportingTsMovies(self): return True if type(self) is ProtImportTsMovies else False def _genTsAcquisitionFromMdoc(self, mdocObj): acq =\ TomoAcquisition(voltage=mdocObj.getVoltage(), sphericalAberration=self.sphericalAberration.get(), amplitudeContrast=self.amplitudeContrast.get(), magnification=mdocObj.getMagnification(), tiltAxisAngle=mdocObj.getTiltAxisAngle() ) # This field is only present in the form for TsM import if hasattr(self, 'doseInitial'): acq.setDoseInitial(self.doseInitial.get()) return acq def _excludeByWords(self, files): exclusionWords = self.exclusionWords.get() if exclusionWords is None: return files exclusionWordList = exclusionWords.split() allowedFiles = [] for file in files: if any(bannedWord in file for bannedWord in exclusionWordList): print("%s excluded. Contains any of %s" % (file, exclusionWords)) continue allowedFiles.append(file) return allowedFiles
[docs] def getMatchingFiles(self, fileTimeOut=None, isValidation=False): """ Return an ordered dict with TiltSeries found in the files as key and a list of all tilt images of that series as value. """ if self.MDOC_DATA_SOURCE: return self._getMatchingFilesFromMdoc(isValidation=isValidation) else: return self._getMatchingFilesFromRegExPattern( fileTimeOut=fileTimeOut)
def _getMatchingFilesFromRegExPattern(self, fileTimeOut): filePaths = glob(self._globPattern) filePaths = self._excludeByWords(filePaths) filePaths.sort(key=lambda fn: os.path.getmtime(fn)) fileTimeOut = fileTimeOut or timedelta(seconds=5) matchingFiles = OrderedDict() def _getTsId(match): """ Retrieve the TiltSerie ID from the matching object. We need to have tsId that starts with character, so let's add a prefix if it is not the case """ tsId ='TS') return normalizeTSId(tsId) def _addOne(fileList, file, match): """ Add one file matching to the list. """ fileList.append((file, int('TO')), float('TA')))) def _addMany(fileList, file, match): """ Add many 'files' (when angles in header or mdoc) to the list. """ anglesFrom = self.getEnumText('anglesFrom') if anglesFrom == self.ANGLES_FROM_HEADER: angles = getAnglesFromHeader(file) elif anglesFrom == self.ANGLES_FROM_MDOC: mdocFn = os.path.splitext(file)[0] + '.mdoc' if not os.path.exists(mdocFn): raise Exception("Missing angles file: %s" % mdocFn) angles = getAnglesFromMdoc(mdocFn) elif anglesFrom == self.ANGLES_FROM_TLT: tltFn = os.path.splitext(file)[0] + '.tlt' if not os.path.exists(tltFn): raise Exception("Missing angles file: %s" % tltFn) angles = getAnglesFromTlt(tltFn) elif anglesFrom == self.ANGLES_FROM_RANGE: angles = self._getTiltAngleRange() else: raise Exception('Invalid angles option: %s' % anglesFrom) for i, a in enumerate(angles): fileList.append(((i + 1, file), i + 1, a)) addFunc = _addOne if self._anglesInPattern() else _addMany # Handle special case of just one TiltSeries, to avoid # the user the need to specify {TS} if len(filePaths) == 1 and not self.isInStreaming(): f = filePaths[0] ts = pwutils.removeBaseExt(f) # Base name without extension matchingFiles[ts] = [] _addMany(matchingFiles[ts], f, None) else: for f in filePaths: if self.fileModified(f, fileTimeOut): continue m = self._regex.match(f) if m is not None: ts = _getTsId(m) # Only report files of new tilt-series if ts not in self._existingTs: if ts not in matchingFiles: matchingFiles[ts] = [] addFunc(matchingFiles[ts], f, m) return matchingFiles def _getCopyOrLink(self): """ Returns a function to copy or link files based on user selected option""" if self.importAction.get() == self.IMPORT_COPY_FILES: return pw.utils.copyFile elif self.importAction.get() == self.IMPORT_LINK_REL: return pw.utils.createLink else: return pw.utils.createAbsLink
[docs] def fileModified(self, fileName, fileTimeout): """ Check if the fileName modification time is less than a given timeout. Params: fileName: input filename that will be checked. fileTimeout: timeout """ self.debug('Checking file: %s' % fileName) mTime = datetime.fromtimestamp(os.path.getmtime(fileName)) delta = - mTime self.debug(' Modification time: %s' % pw.utils.prettyTime(mTime)) self.debug(' Delta: %s' % pw.utils.prettyDelta(delta)) return delta < fileTimeout
[docs] def isBlacklisted(self, fileName): """ Overwrite in subclasses """ return False
[docs] @classmethod def worksInStreaming(cls): # Import protocols always work in streaming return True
def _fillAcquisitionInfo(self, inputTs): # TODO Acquisition is historically expected as # one per set of tilt series movies, # so the first one is the one used, at least for now if self.MDOC_DATA_SOURCE: firstTsId = list(self.acquisitions.keys())[0] acq = self.acquisitions[firstTsId] sRate = self.sRates[firstTsId] inputTs.setAcquisition(acq) inputTs.setSamplingRate(sRate) else: inputTs.setSamplingRate(self.samplingRate.get()) acq = inputTs.getAcquisition() acq.setVoltage(self.voltage.get()) acq.setSphericalAberration(self.sphericalAberration.get()) acq.setAmplitudeContrast(self.amplitudeContrast.get()) acq.setMagnification(self.magnification.get()) def _getTiltAngleRange(self): """ Return the list with all expected tilt angles. """ offset = 1 if self.minAngle.get() > self.maxAngle.get(): offset *= -1 return np.arange( self.minAngle.get(), self.maxAngle.get() + offset, # also include last angle self.stepAngle.get()) @staticmethod def _getSortedAngles(tiltSeriesList): """ Return the sorted angles from a given tiltSeriesList. """ return sorted(item[2] for item in tiltSeriesList) def _sameTiltAngleRange(self, tiltAngleRange, tiltSeriesList): # allow some tolerance when comparing tilt-angles return np.allclose(tiltAngleRange, self._getSortedAngles(tiltSeriesList), atol=0.1) # --------------- Streaming special functions ----------------------- def _getStopStreamingFilename(self): return self._getExtraPath("STOP_STREAMING.TXT")
[docs] def getActions(self): """ This method will allow that the 'Stop import' action to appears in the GUI when the user right-click in the protocol import box. It will allow a user to manually stop the streaming. """ # Only allow to stop if running and in streaming mode if self.dataStreaming and self.isRunning(): return [('STOP STREAMING', self.stopImport)] else: return []
[docs] def stopImport(self): """ Since the actual protocol that is running is in a different process that the one that this method will be invoked from the GUI, we will use a simple mechanism to place an special file to stop the streaming. """ # Just place an special file into the run folder f = open(self._getStopStreamingFilename(), 'w') f.close()
[docs] def streamingHasFinished(self): return (not self.isInStreaming() or os.path.exists(self._getStopStreamingFilename()))
[docs] def isInStreaming(self): return self.dataStreaming.get()
[docs]class ProtImportTs(ProtImportTsBase): """Protocol to import tilt series.""" _label = 'import tilt-series' _devStatus = pw.BETA def _defineAngleParam(self, form): """ Used in subclasses to define the option to fetch tilt angles. """ group = form.addGroup('Tilt info', condition=self.NOT_MDOC_GUI_COND) group.addParam('anglesFrom', params.EnumParam, default=0, choices=[self.ANGLES_FROM_RANGE, self.ANGLES_FROM_HEADER, # self.ANGLES_FROM_MDOC, self.ANGLES_FROM_TLT], display=params.EnumParam.DISPLAY_HLIST, label='Import angles from', help="Choose how the tilt angles will be inferred. " "It can be taken from a range: Min, Max, Step " "or from the image header, or from an" "mdoc or tlt file (should have the SAME file name " "but with the .mdoc or .tlt " "extension at the end).") line = group.addLine('Tilt angular range', condition='anglesFrom==0', # ANGLES_FROM_RANGE help="Specify the tilting angular range. " "Depending on the collection schema, the " "order of the acquisition does not need to " "be the same order of the angular range. ") line.addParam('minAngle', params.FloatParam, default=-60, label='min') line.addParam('maxAngle', params.FloatParam, default=60, label='max') line.addParam('stepAngle', params.FloatParam, default=3, label='step') def _validateAngles(self): if not self.MDOC_DATA_SOURCE: # If importing from pattern and getting the tilt data from header, # it has to check if IMOD's extract tilts is installed if self.getEnumText('anglesFrom') == self.ANGLES_FROM_HEADER: from pwem import Domain imod = Domain.importFromPlugin('imod') if imod is None: return ['Imod plugin is needed to import angles from header.' 'Please install it'] ts, tiltSeriesList = self._firstMatch i, fileName = tiltSeriesList[0][0] x, y, z, n = ImageHandler().getDimensions(fileName) nImages = max(z, n) # Just handle ambiguity with mrc format nAngles = len(self._tiltAngleList) if nAngles != nImages: return 'Tilt-series %s stack has different number of images '\ '(%d) than the expected number of tilt angles (%d). '\ % (fileName, nImages, nAngles) else: return None
[docs]class ProtImportTsMovies(ProtImportTsBase): """Protocol to import tilt series movies.""" _label = 'import tilt-series movies' _devStatus = pw.BETA def _defineAngleParam(self, form): """ Used in subclasses to define the option to fetch tilt angles. """ group = form.addGroup('Tilt info', condition=self.NOT_MDOC_GUI_COND) group.addParam('anglesFrom', params.EnumParam, default=0, choices=[self.ANGLES_FROM_FILENAME], display=params.EnumParam.DISPLAY_HLIST, label='Import angles from', help='Angles will be parsed from the filename pattern.' 'The special token {TA} should be specified as ' ' part of the pattern, that will be used to match ' ' the value of the angle for each TiltSeriesMovie.' ) def _defineAcquisitionParams(self, form): """ Add movie specific options to the acquisition section. """ group = ProtImportTsBase._defineAcquisitionParams(self, form) group.addParam('gainFile', params.FileParam, label='Gain image', help='A gain reference related to a set of movies ' 'for gain correction') group.addParam('darkFile', params.FileParam, label='Dark image', help='A dark image related to a set of movies') return group def _fillAcquisitionInfo(self, inputTs): ProtImportTsBase._fillAcquisitionInfo(self, inputTs) inputTs.setGain(self.gainFile.get()) inputTs.setDark(self.darkFile.get()) if not self.MDOC_DATA_SOURCE: acq = inputTs.getAcquisition() acq.setDoseInitial(self.doseInitial.get()) acq.setDosePerFrame(self.dosePerFrame.get()) def _initialize(self): ProtImportTsBase._initialize(self) self._outputName += 'M' self._createOutputName += 'M' def _validateAngles(self): """ Function to be implemented in subclass to validate the angles range. """ if not self.MDOC_DATA_SOURCE and \ self.getEnumText('anglesFrom') == self.ANGLES_FROM_FILENAME: if not self._anglesInPattern(): return 'When importing movies, {TA} and {TO} ' \ 'should be in the files pattern.' else: return None