Source code for emfacilities.protocols.report_html

# -*- coding: utf-8 -*-
# **************************************************************************
# *
# * Authors:     J.M. De la Rosa Trevin (
# *              Pablo Conesa (
# *
# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 2 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307  USA
# *
# *  All comments concerning this program package may be sent to the
# *  e-mail address ''
# *
# **************************************************************************

import json
import os
from os.path import join, exists, abspath, basename
import numpy as np
import multiprocessing
from datetime import datetime

from pyworkflow.protocol import getUpdatedProtocol
import pyworkflow.utils as pwutils

from pwem.emlib.image import ImageHandler

from emfacilities import Plugin
from .summary_provider import SummaryProvider

# --------------------- CONSTANTS -----------------------------------
# These constants are the keys used in the ctfMonitor function
# getData() to store paths to micrographs, psd files and shift plots
MIC_PATH = 'imgMicPath'
PSD_PATH = 'imgPsdPath'
SHIFT_PATH = 'imgShiftPath'
# These constants are the name of the folders where thumbnails
# for the html report will be stored. They are also the keys to
# used in the execution.summary.template.html to read data (where
# they will need to be changed if they're changed here)
MIC_THUMBS = 'imgMicThumbs'
PSD_THUMBS = 'imgPsdThumbs'
SHIFT_THUMBS = 'imgShiftThumbs'
MIC_ID = 'micId'

[docs]class ReportHtml: """ Create an html report with a summary of the processing. The report will be updated with a given frequency. """ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCmd=None, **kwargs): # The CTF protocol to monitor self.protocol = protocol self.ctfProtocol = protocol._getCtfProtocol() self.alignProtocol = protocol._getAlignProtocol() self.micThumbSymlinks = False self.reportPath = protocol.reportPath self.reportDir = protocol.reportDir self.provider = SummaryProvider(protocol) self.ctfMonitor = ctfMonitor self.sysMonitor = sysMonitor self.movieGainMonitor = movieGainMonitor self.lastThumbIndex = 0 self.thumbsReady = 0 self.thumbPaths = {MIC_THUMBS: [], PSD_THUMBS: [], SHIFT_THUMBS: [], MIC_PATH: [], SHIFT_PATH: [], PSD_PATH: [], MIC_ID: []} # Get the html template to be used, by default use the one # in scipion/config/templates self.template = self._getHTMLTemplatePath() self.publishCmd = publishCmd self.refreshSecs = kwargs.get('refreshSecs', 60) def _getHTMLTemplatePath(self): """ Returns the path of the customized template at config/execution.summary.html or the standard scipion HTML template""" # Try if there is a customized template template = os.path.join(basename(pwutils.Config.SCIPION_CONFIG), 'execution.summary.html') if not os.path.exists(template): template = os.path.join(Plugin.getPluginTemplateDir(), 'execution.summary.template.html') else: print("Customized HTML template found at %s." % template) return template
[docs] def getHTMLReportText(self): if exists(self.template): return open(self.template, encoding="utf-8").read() else: return ""
[docs] def info(self, msg): if self.protocol._log is not None: else: print(msg)
[docs] def checkNewThumbsReady(self): thumbKeys = [MIC_THUMBS] if PSD_THUMBS in self.thumbPaths.keys(): lastThumb = min(len(self.thumbPaths[PSD_THUMBS]), len(self.thumbPaths[MIC_THUMBS])) thumbKeys.append(PSD_THUMBS) else: lastThumb = len(self.thumbPaths[MIC_THUMBS]) if SHIFT_THUMBS in self.thumbPaths.keys(): thumbKeys.append(SHIFT_THUMBS) for i in range(self.thumbsReady, lastThumb): pathsReady = [exists(join(self.reportDir, self.thumbPaths[k][i])) for k in thumbKeys] if all(pathsReady): self.thumbsReady += 1 return self.thumbsReady
[docs] def setUp(self): """Setup actions for the html report: create directories to store thumbnails, check if thumbnails are already generated in the alignment protocol. """ # make report folders pwutils.makePath(join(self.reportDir, MIC_THUMBS), join(self.reportDir, PSD_THUMBS), join(self.reportDir, SHIFT_THUMBS)) # check if align protocol already has thumbnails if (hasattr(self.alignProtocol, 'doComputeMicThumbnail') and self.alignProtocol._doComputeMicThumbnail()): self.micThumbSymlinks = True
[docs] def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='png', micIdSet=None): """Adds to self.thumbPaths the paths to the report thumbnails that come from the alignment and/or ctf protocol. ===== Params ===== - thumbsDone: how many thumbnails have already been generated. we will get paths starting from this index - ctfData: dict resulting from ctfMonitor.getData() - ext: extension of the thumbnail images. Defaults to png. - micIdSet: mic indexes to use """ # get psd thumbs from ctfData if ctfData is not None: for i in range(thumbsDone, len(ctfData[PSD_PATH])): psdPath = ctfData[PSD_PATH][i] movie = basename(os.path.dirname(psdPath)) psdThumb = join(PSD_THUMBS, "%s_%s" % (movie, pwutils.replaceExt(basename(psdPath), ext))) self.thumbPaths[PSD_THUMBS].append(psdThumb) self.thumbPaths[PSD_PATH].append(psdPath) # get alignment and mic thumbs if self.alignProtocol is not None: getMicFromCTF = False updatedProt = getUpdatedProtocol(self.alignProtocol) if hasattr(updatedProt, 'outputMicrographs'): outputSet = updatedProt.outputMicrographs if micIdSet is None: micIdSet = list(outputSet.getIdSet()) else: return elif self.ctfProtocol is not None: getMicFromCTF = True updatedProt = getUpdatedProtocol(self.ctfProtocol) if hasattr(updatedProt, 'outputCTF'): outputSet = updatedProt.outputCTF if micIdSet is None: micIdSet = list(outputSet.getIdSet()) else: return else: return for micId in micIdSet[thumbsDone:]: mic = outputSet[micId] if getMicFromCTF: mic = mic.getMicrograph() if hasattr(mic, 'thumbnail'): srcMicFn = abspath(mic.thumbnail.getFileName()) else: srcMicFn = abspath(mic.getFileName()) micThumbFn = join(MIC_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext)) self.thumbPaths[MIC_PATH].append(srcMicFn) self.thumbPaths[MIC_THUMBS].append(micThumbFn) shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None)) if shiftPlot is not None: shiftPath = "" if shiftPlot is None else abspath(shiftPlot.getFileName()) shiftCopy = "" if shiftPlot is None else join(SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext)) self.thumbPaths[SHIFT_PATH].append(shiftPath) self.thumbPaths[SHIFT_THUMBS].append(shiftCopy) else: if SHIFT_PATH in self.thumbPaths: self.thumbPaths.pop(SHIFT_PATH, None) if SHIFT_THUMBS in self.thumbPaths: self.thumbPaths.pop(SHIFT_THUMBS, None) self.thumbPaths[MIC_ID].append(micId) if self.ctfProtocol is None: def getMicPSDPath(mic): if hasattr(mic, 'psdJpeg'): return mic.psdJpeg.getFileName() elif hasattr(mic, 'psdCorr'): return mic.psdCorr.getFileName() else: return None psdPath = getMicPSDPath(mic) psdThumb = None if psdPath is None: psdThumb = join(PSD_THUMBS, pwutils.replaceExt(basename(str(psdPath)), ext)) self.thumbPaths[PSD_THUMBS].append(psdThumb) self.thumbPaths[PSD_PATH].append(psdPath) else: if PSD_THUMBS in self.thumbPaths: self.thumbPaths.pop(PSD_THUMBS, None) if PSD_PATH in self.thumbPaths: self.thumbPaths.pop(PSD_PATH, None)
[docs] def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from self.thumbPaths. ===== Params ===== - firstThumbIndex: index from which we start generating thumbnails - micScaleFactor: how much to reduce in size the micrographs. """ ih = ImageHandler() numMics = len(self.thumbPaths[MIC_PATH]) for i in range(firstThumbIndex, numMics): print('Generating images for mic %d' % (i+1)) # mic thumbnails dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) if not exists(dstImgPath): if self.micThumbSymlinks: pwutils.copyFile(self.thumbPaths[MIC_PATH][i], dstImgPath) else: ih.computeThumbnail(self.thumbPaths[MIC_PATH][i], dstImgPath, scaleFactor=micScaleFactor, flipOnY=True) # shift plots if SHIFT_THUMBS in self.thumbPaths: dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i]) if not exists(dstImgPath): pwutils.copyFile(self.thumbPaths[SHIFT_PATH][i], dstImgPath) # Psd thumbnails # If there ARE thumbnail for the PSD (no ctf protocol and # moviealignment hasn't computed it if PSD_THUMBS in self.thumbPaths: if self.ctfProtocol is None: srcImgPath = self.thumbPaths[PSD_PATH][i] dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath) and srcImgPath is not None: if srcImgPath.endswith('psd'): psdImg1 = psdImg1.convertPSD() psdImg1.write(dstImgPath) ih.computeThumbnail(dstImgPath, dstImgPath, scaleFactor=1, flipOnY=True) else: pwutils.copyFile(srcImgPath, dstImgPath) else: dstImgPath = join(self.reportDir, self.thumbPaths[PSD_THUMBS][i]) if not exists(dstImgPath): ih.computeThumbnail(self.thumbPaths[PSD_PATH][i], dstImgPath, scaleFactor=1, flipOnY=True) return
[docs] def processDefocusValues(self, defocusList): maxDefocus = self.protocol.maxDefocus.get()*1e-4 minDefocus = self.protocol.minDefocus.get()*1e-4 # Convert defocus values to microns defocusList = [i*1e-4 for i in defocusList] edges = np.arange(0, maxDefocus+DEFOCUS_HIST_BIN_WIDTH, DEFOCUS_HIST_BIN_WIDTH) edges = np.insert(edges[edges > minDefocus], 0, minDefocus) values, binEdges = np.histogram(defocusList, bins=edges, range=(minDefocus, maxDefocus)) belowThresh = 0 aboveThresh = 0 labels = ["%0.1f-%0.1f" % (x[0], x[1]) for x in zip(binEdges, binEdges[1:])] for v in defocusList: if v < minDefocus: belowThresh += 1 elif v > maxDefocus: aboveThresh += 1 zipped = list(zip(values, labels)) zipped[:0] = [(belowThresh, "0-%0.1f" % minDefocus)] # TODO unresolved method for class Iterator in python3 # zipped.append((aboveThresh, "> %0.1f" % (maxDefocus))) return zipped
[docs] def getTimeSeries(self, data): from .protocol_monitor_ctf import (PHASE_SHIFT, TIME_STAMP, DEFOCUS_U, RESOLUTION) # Get timeStamp ts = data[TIME_STAMP] timeSeries = dict() # Get phaseShift phaseShiftSerie = data[PHASE_SHIFT] phaseShiftSerie = list(zip(ts, phaseShiftSerie)) # Add it to the series timeSeries[PHASE_SHIFT] = phaseShiftSerie # Get defocusU is coming in Å, reduce it to μm defocusSerie = data[DEFOCUS_U] defocusSerie = [i * 1e-4 for i in defocusSerie] defocusSerie = list(zip(ts, defocusSerie)) # Add it to the series timeSeries[DEFOCUS_U] = defocusSerie # Get Resolution resSerie = data[RESOLUTION] resSerie = list(zip(ts, resSerie)) # Add it to the series timeSeries[RESOLUTION] = resSerie return timeSeries
[docs] def getResolutionHistogram(self, resolutionValues): if len(resolutionValues) == 0: return [] maxValue = int(np.ceil(max(resolutionValues))) edges = np.append(np.arange(0, maxValue, RESOLUTION_HIST_BIN_WIDTH), maxValue) values, binEdges = np.histogram(resolutionValues, bins=edges, range=(0, maxValue)) return list(zip(values, binEdges))
[docs] def generate(self, finished): reportTemplate = self.getHTMLReportText() if not reportTemplate: raise Exception("HTML template file '%s' not found. " % self.template) project = self.protocol.getProject() projName = project.getShortName() acquisitionLines = '' self.provider.refreshObjects() for item in self.provider.acquisition: if not acquisitionLines == '': acquisitionLines += ',' acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item runLines = '' wasProtocol = None for obj in self.provider.getObjects(): # If it's a protocol isProtocol = True if else False if isProtocol: if runLines != '': runLines += ']},' runLines += '{protocolName: "%s", output:[' % else: if not wasProtocol: runLines += ',' runLines += '{name: "%s", size:"%s"}' % (obj.output, obj.outSize) wasProtocol = isProtocol # End the runLines JSON object runLines += ']}' # Ctf monitor chart data data = {} if self.ctfMonitor is None else self.ctfMonitor.getData() if data: numMicsDone = len(self.thumbPaths[PSD_THUMBS]) numMics = len(data[PSD_PATH]) numMicsToDo = numMics - numMicsDone self.getThumbPaths(ctfData=data, thumbsDone=numMicsDone, micIdSet=data['idValues']) if len(data['defocusU']) < 100: data['defocusCoverage'] = self.processDefocusValues(data['defocusU']) else: data['defocusCoverage'] = self.processDefocusValues(data['defocusU'][:-50]) data['defocusCoverageLast50'] = self.processDefocusValues(data['defocusU'][-50:]) data['resolutionHistogram'] = self.getResolutionHistogram(data['resolution']) data['timeSeries'] = self.getTimeSeries(data) else: # Thumbnails for Micrograph Table numMicsDone = len(self.thumbPaths[MIC_THUMBS]) self.getThumbPaths(thumbsDone=numMicsDone) numMics = len(self.thumbPaths[MIC_PATH]) numMicsToDo = numMics - numMicsDone if numMicsToDo <= 10: # we have few new images, eg streaming mode, generate thumbnails now self.generateReportImages(firstThumbIndex=numMicsDone) else: # we have many images, generate thumbs in a separate process process = multiprocessing.Process(target=self.generateReportImages, args=(numMicsDone,)) process.start() # send over only thumbnails of the mics that have been fully processed self.thumbsReady = self.checkNewThumbsReady() thumbsLoading = numMics - self.thumbsReady for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]: if k in self.thumbPaths: data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading data[MIC_ID] = self.thumbPaths[MIC_ID] reportFinished = self.thumbsReady == numMics def convert(o): if isinstance(o, np.int64): return int(o) raise TypeError ctfData = json.dumps(data, default=convert) # Movie gain monitor chart data data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData() movieGainData = json.dumps(data) # system monitor chart data data = self.sysMonitor.getData() systemData = json.dumps(data, default=convert) tnow = args = {'projectName': projName, 'startTime': pwutils.dateStr(project.getCreationTime(), secs=True), 'dateStr': pwutils.prettyTime(dt=tnow, secs=True), 'projectDuration': pwutils.prettyDelta(tnow-project.getCreationTime()), 'projectStatus': "FINISHED" if finished else "RUNNING", 'scipionVersion': os.environ['SCIPION_VERSION'], 'acquisitionLines': acquisitionLines, 'runLines': runLines, 'ctfData': ctfData, 'movieGainData': movieGainData, 'systemData': systemData, 'refresh': self.refreshSecs }"Writing report html to: %s" % abspath(self.reportPath)) pwutils.cleanPath(self.reportPath) reportFile = open(self.reportPath, 'w', encoding="utf-8") reportTemplate = reportTemplate % args reportFile.write(reportTemplate) reportFile.close() if self.publishCmd:"Publishing the report:") cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir} os.system(cmd) return reportFinished