# Copyright (c) 2011-2014, B.I.Stepanov Institute of Physics, National Academy # of Sciences of Belarus. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software # Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import math import numpy from common.utils import txt from common.DataRecord import * __all__ = ['LiricOutput'] # ***************************************************************************** class LiricOutput(DataRecord): """Output data for a single aerosol profile retrieval. Attributes to be read from a database: - 'latitude', 'longitude', 'altitude', 'startDate', 'stopDate', 'startTime', 'stopTime': same as in 'PreparedLidarInput'. - 'photometerDate', 'photometerTime': date and time of the photometer measurement (same as in 'PhotometerInput'). - 'retrievalDate', 'retrievalTime': date and time of the retrieval; - 'gridStep', 'gridZenithAngle': parameters of the data grid used for output data profiles (similar to those of 'PreparedLidarInput'). - 'joinIndex': first index of the data grid that belongs to the far measurement zone, as opposed to the near one. This is used in automated aerosol retrieval only. - 'profileFine', 'profileSpherical', 'profileSpheroid', 'profileCoarse': retrieved aerosol concentration profiles for different aerosol modes, in ppb. Index zero of the arrays corresponds to index zero of the data grid. For polarimetric retrievals, 'Coarse' mode would not be directly available, and should be obtained as a sum of 'Spherical' and 'Spheroid' modes. For not-polarimetric retrievals, 'Spherical' and 'Spheroid' aerosol modes would not be available at all. - 'measuredSignal<...>', 'calculatedSignal<...>': measured and calculated lidar signals for different lidar channels, divided by their average values around the reference points, with molecular components removed. - 'signalId<...>': 'localId' fields of lidar inputs for the main (far) measurement zone. - 'signalId<...>Near': 'localId' fields of lidar inputs for the near measurement zone or 'None' for outputs generated by the manual version of the algorithm. - 'vFine', 'vCoarse': corrected values for 'vFine' and 'vCoarse' from the 'PhotometerInput' instance. Correction factor is 'aot675Corr / aot675'. - 'sphericity': same as in 'PhotometerInput'. - 'atmosphereModel': copy of 'atmoModel' from the 'PreparedLidarInput' instances (this value should be the same among the inputs). - 'polarization<...>': 'polarization' values from the lidar inputs. - 'dispersion<...>': 'lidarDispersion' arrays from the lidar inputs. - 'b<...><...>': aerosol backscatter coefficients used in the retrieval equations, with parallel signal leakage not taken into account for cross-polarized channels. Used to calculate integrated backscatters displayed in the plot widgets. - 'a<...><...>': aerosol attenuation coefficients used in the retrieval equations. Used to calculate aerosol optical thicknesses displayed in the plot widgets. - 'weighting<...>' 'weightSmooth<...>', 'molDepolarization<...>', 'parallelLeakage<...>': a copy of some of the algorithm parameters that had been used in the retrieval. - 'molThicknessError<...>', 'aerosolThicknessError<...>', 'totalBackscatterError<...>': estimations of the atmospheric relative errors copied from the corresponding 'PreparedLidarInput' instances. - 'ratio<...>': retrieved backscatter ratios at the reference points. - 'residual': square root of the squared lidar equation residual, integrated over the retrieval interval. - 'residual': absolute value of the photometer equation residual for the given aerosol mode. - 'resSmooth': square root of the squared approximated second order derivative of the retrieved concentration profile, integrated over the retrieval interval. - 'profileDispersion<...>', 'ratioDispersion<...>': dispersions of the corresponding 'profile<...>' and 'ratio<...>' retrieved parameters that had been obtained while applying a predefined set of perturbances to the input data. Attributes that are calculated manually in 'onDataLoaded': - 'gridHeightStep': height step of the data grid (similar to 'PreparedLidarInput'). - 'vSpherical', 'vSpheroid': the value of 'vCoarse', divided into two parts according to the value of 'sphericity' attribute. - 'calculatedVFine', 'calculatedVCoarse', 'calculatedVSpherical', 'calculatedVSpheroid': volumetric concentrations of retrieved aerosol modes, in micrometers, obtained by summation of the corresponding 'profile<...>' arrays (i.e., the values retrieved by the aerosol profile retrieval algorithm, in contrast to 'vFine' and 'vCoarse' etc. taken from the photometer data). Note: for non-polarimetric retrievals, 'calculatedVSpherical' and 'calculatedVSpheroid' are obtained from 'profileCoarse' using 'sphericity' attribute. - 'calculatedSphericity': ratio of 'calculatedVSpherical' to the sum of 'calculatedVSpherical' and 'calculatedVSpheroid', in percent. For non-polarimetric retrievals, all the attributes for 'Spherical' and 'Spheroid' aerosol modes (with an exception for 'calculatedVSpherical' and 'calculatedVSpheroid') will be 'None', as well as attributes related to the cross-polarized lidar channel. For polarimetric retrievals, all the attributes for 'Coarse' aerosol mode will be 'None'.""" # ---- Class attributes --------------------------------------------------- # Name of the aerosol profile output database table to read the data from. tableName = 'AerosolProfiles' fields = [ DataField('latitude', 'Latitude', 'SINGLE' ), DataField('longitude', 'Longitude', 'SINGLE' ), DataField('altitude', 'Altitude', 'SINGLE' ), DataField('startDate', 'StartDate', 'DATE'), DataField('stopDate', 'StopDate', 'DATE'), DataField('startTime', 'StartTime', 'TIME'), DataField('stopTime', 'StopTime', 'TIME'), DataField('photometerDate', 'PhotometerDate', 'DATE'), DataField('photometerTime', 'PhotometerTime', 'TIME'), DataField('retrievalDate', 'RetrievalDate', 'DATE'), DataField('retrievalTime', 'RetrievalTime', 'TIME'), DataField('gridStep', 'Step', 'SINGLE'), DataField('gridZenithAngle', 'Zenith', 'SINGLE'), DataField('joinIndex', 'JoinIndex', 'SMALLINT'), DataField('firstInputIndex355', 'Left_355', 'SMALLINT', required = False), DataField('firstInputIndex532', 'Left_532', 'SMALLINT', required = False), DataField('firstInputIndex1064', 'Left_1064', 'SMALLINT', required = False), DataField('firstInputIndex532C', 'Left_532C', 'SMALLINT', required = False), DataField('profileFine', 'FineProfile', 'LONGBINARY', numpy.float32), DataField('profileSpherical', 'CoarseSphericalProfile', 'LONGBINARY', numpy.float32), DataField('profileSpheroid', 'CoarseSpheroidProfile', 'LONGBINARY', numpy.float32), DataField('profileCoarse', 'CoarseProfile', 'LONGBINARY', numpy.float32), DataField('measuredSignal355', 'Measured_355', 'LONGBINARY', numpy.float32), DataField('measuredSignal532', 'Measured_532', 'LONGBINARY', numpy.float32), DataField('measuredSignal1064', 'Measured_1064', 'LONGBINARY', numpy.float32), DataField('measuredSignal532C', 'Measured_532C', 'LONGBINARY', numpy.float32), DataField('calculatedSignal355', 'Calculated_355', 'LONGBINARY', numpy.float32), DataField('calculatedSignal532', 'Calculated_532', 'LONGBINARY', numpy.float32), DataField('calculatedSignal1064', 'Calculated_1064', 'LONGBINARY', numpy.float32), DataField('calculatedSignal532C', 'Calculated_532C', 'LONGBINARY', numpy.float32), DataField('signalId355', 'IDLocal_355', 'VARCHAR'), DataField('signalId532', 'IDLocal_532', 'VARCHAR'), DataField('signalId1064', 'IDLocal_1064', 'VARCHAR'), DataField('signalId532C', 'IDLocal_532C', 'VARCHAR'), DataField('signalId355Near', 'IDLocalNear_355', 'VARCHAR'), DataField('signalId532Near', 'IDLocalNear_532', 'VARCHAR'), DataField('signalId1064Near', 'IDLocalNear_1064', 'VARCHAR'), DataField('signalId532CNear', 'IDLocalNear_532C', 'VARCHAR'), DataField('vFine', 'PhotometerVFine', 'SINGLE'), DataField('vCoarse', 'PhotometerVCoarse', 'SINGLE'), DataField('sphericity', 'Sphericity', 'SINGLE'), DataField('atmosphereModel', 'AtmoModel', 'VARCHAR'), DataField('polarization355', 'Polarization_355', 'SMALLINT', required = False), DataField('polarization532', 'Polarization_532', 'SMALLINT', required = False), DataField('polarization1064', 'Polarization_1064', 'SMALLINT', required = False), DataField('dispersion355', 'StdDeviation_355', 'LONGBINARY', numpy.float32, required = False), DataField('dispersion532', 'StdDeviation_532', 'LONGBINARY', numpy.float32, required = False), DataField('dispersion1064', 'StdDeviation_1064', 'LONGBINARY', numpy.float32, required = False), DataField('dispersion532C', 'StdDeviation_532C', 'LONGBINARY', numpy.float32, required = False), DataField('bFine355', 'BFine_355', 'SINGLE'), DataField('bFine355P', 'BFine_355P', 'SINGLE', required = False), DataField('bFine355C', 'BFine_355C', 'SINGLE', required = False), DataField('bFine532', 'BFine_532', 'SINGLE'), DataField('bFine532P', 'BFine_532P', 'SINGLE', required = False), DataField('bFine532C', 'BFine_532C', 'SINGLE'), DataField('bFine1064', 'BFine_1064', 'SINGLE'), DataField('bFine1064P', 'BFine_1064P', 'SINGLE', required = False), DataField('bFine1064C', 'BFine_1064C', 'SINGLE', required = False), DataField('bSpherical355', 'BSpherical_355', 'SINGLE'), DataField('bSpherical355P', 'BSpherical_355P', 'SINGLE', required = False), DataField('bSpherical355C', 'BSpherical_355C', 'SINGLE', required = False), DataField('bSpherical532', 'BSpherical_532', 'SINGLE'), DataField('bSpherical532P', 'BSpherical_532P', 'SINGLE', required = False), DataField('bSpherical532C', 'BSpherical_532C', 'SINGLE'), DataField('bSpherical1064', 'BSpherical_1064', 'SINGLE'), DataField('bSpherical1064P', 'BSpherical_1064P', 'SINGLE', required = False), DataField('bSpherical1064C', 'BSpherical_1064C', 'SINGLE', required = False), DataField('bSpheroid355', 'BSpheroid_355', 'SINGLE'), DataField('bSpheroid355P', 'BSpheroid_355P', 'SINGLE', required = False), DataField('bSpheroid355C', 'BSpheroid_355C', 'SINGLE', required = False), DataField('bSpheroid532', 'BSpheroid_532', 'SINGLE'), DataField('bSpheroid532P', 'BSpheroid_532P', 'SINGLE', required = False), DataField('bSpheroid532C', 'BSpheroid_532C', 'SINGLE'), DataField('bSpheroid1064', 'BSpheroid_1064', 'SINGLE'), DataField('bSpheroid1064P', 'BSpheroid_1064P', 'SINGLE', required = False), DataField('bSpheroid1064C', 'BSpheroid_1064C', 'SINGLE', required = False), DataField('bCoarse355', 'BCoarse_355', 'SINGLE'), DataField('bCoarse355P', 'BCoarse_355P', 'SINGLE', required = False), DataField('bCoarse355C', 'BCoarse_355C', 'SINGLE', required = False), DataField('bCoarse532', 'BCoarse_532', 'SINGLE'), DataField('bCoarse532P', 'BCoarse_532P', 'SINGLE', required = False), DataField('bCoarse532C', 'BCoarse_532C', 'SINGLE'), DataField('bCoarse1064', 'BCoarse_1064', 'SINGLE'), DataField('bCoarse1064P', 'BCoarse_1064P', 'SINGLE', required = False), DataField('bCoarse1064C', 'BCoarse_1064C', 'SINGLE', required = False), DataField('aFine355', 'AFine_355', 'SINGLE'), DataField('aFine532', 'AFine_532', 'SINGLE'), DataField('aFine1064', 'AFine_1064', 'SINGLE'), DataField('aSpherical355', 'ASpherical_355', 'SINGLE'), DataField('aSpherical532', 'ASpherical_532', 'SINGLE'), DataField('aSpherical1064', 'ASpherical_1064', 'SINGLE'), DataField('aSpheroid355', 'ASpheroid_355', 'SINGLE'), DataField('aSpheroid532', 'ASpheroid_532', 'SINGLE'), DataField('aSpheroid1064', 'ASpheroid_1064', 'SINGLE'), DataField('aCoarse355', 'ACoarse_355', 'SINGLE'), DataField('aCoarse532', 'ACoarse_532', 'SINGLE'), DataField('aCoarse1064', 'ACoarse_1064', 'SINGLE'), DataField('weighting355', 'Weight_355', 'SINGLE'), DataField('weighting532', 'Weight_532', 'SINGLE'), DataField('weighting1064', 'Weight_1064', 'SINGLE'), DataField('weighting532C', 'Weight_532C', 'SINGLE'), DataField('weightingFine', 'Weight_Fine', 'SINGLE'), DataField('weightingSpherical', 'Weight_Spherical', 'SINGLE'), DataField('weightingSpheroid', 'Weight_Spheroid', 'SINGLE'), DataField('weightingCoarse', 'Weight_Coarse', 'SINGLE'), DataField('weightSmoothFine', 'WeightSmooth_Fine', 'SINGLE'), DataField('weightSmoothSpherical', 'WeightSmooth_Spherical', 'SINGLE'), DataField('weightSmoothSpheroid', 'WeightSmooth_Spheroid', 'SINGLE'), DataField('weightSmoothCoarse', 'WeightSmooth_Coarse', 'SINGLE'), DataField('molDepolarization532', 'MolDepolar_532', 'SINGLE'), DataField('parallelLeakage532', 'Leakage_532', 'SINGLE'), DataField('molThicknessError355', 'RE OMT_355', 'SINGLE'), DataField('molThicknessError532', 'RE OMT_532', 'SINGLE'), DataField('molThicknessError1064', 'RE OMT_1064', 'SINGLE'), DataField('molThicknessError532C', 'RE OMT_532C', 'SINGLE'), DataField('aerosolThicknessError355', 'RE OAT_355', 'SINGLE'), DataField('aerosolThicknessError532', 'RE OAT_532', 'SINGLE'), DataField('aerosolThicknessError1064', 'RE OAT_1064', 'SINGLE'), DataField('aerosolThicknessError532C', 'RE OAT_532C', 'SINGLE'), DataField('totalBackscatterError355', 'RE GBS_355', 'SINGLE'), DataField('totalBackscatterError532', 'RE GBS_532', 'SINGLE'), DataField('totalBackscatterError1064', 'RE GBS_1064', 'SINGLE'), DataField('totalBackscatterError532C', 'RE GBS_532C', 'SINGLE'), DataField('ratio355', 'Ratio_355', 'SINGLE'), DataField('ratio532', 'Ratio_532', 'SINGLE'), DataField('ratio1064', 'Ratio_1064', 'SINGLE'), DataField('ratio532C', 'Ratio_532C', 'SINGLE'), DataField('residual355', 'Residual_355', 'SINGLE'), DataField('residual532', 'Residual_532', 'SINGLE'), DataField('residual1064', 'Residual_1064', 'SINGLE'), DataField('residual532C', 'Residual_532C', 'SINGLE'), DataField('residualFine', 'Residual_Fine', 'SINGLE'), DataField('residualSpherical', 'Residual_Spherical', 'SINGLE'), DataField('residualSpheroid', 'Residual_Spheroid', 'SINGLE'), DataField('residualCoarse', 'Residual_Coarse', 'SINGLE'), DataField('resSmoothFine', 'ResSmooth_Fine', 'SINGLE'), DataField('resSmoothSpherical', 'ResSmooth_Spherical', 'SINGLE'), DataField('resSmoothSpheroid', 'ResSmooth_Spheroid', 'SINGLE'), DataField('resSmoothCoarse', 'ResSmooth_Coarse', 'SINGLE'), DataField('profileDispersionFine', 'FineDispersion', 'LONGBINARY', numpy.float32, required = False), DataField('profileDispersionSpherical', 'CoarseSphericalDispersion', 'LONGBINARY', numpy.float32, required = False), DataField('profileDispersionSpheroid', 'CoarseSpheroidDispersion', 'LONGBINARY', numpy.float32, required = False), DataField('profileDispersionCoarse', 'CoarseDispersion', 'LONGBINARY', numpy.float32, required = False), DataField('ratioDispersion355', 'RatioDispersion_355', 'SINGLE', required = False), DataField('ratioDispersion532', 'RatioDispersion_532', 'SINGLE', required = False), DataField('ratioDispersion1064', 'RatioDispersion_1064', 'SINGLE', required = False), DataField('ratioDispersion532C', 'RatioDispersion_532C', 'SINGLE', required = False), DataField('perturbanceEvaluations', 'PerturbanceEvaluations', 'SMALLINT', required = False), DataField('whiteNoise355', 'WhiteNoise_355', 'SINGLE', required = False), DataField('whiteNoise532', 'WhiteNoise_532', 'SINGLE', required = False), DataField('whiteNoise1064', 'WhiteNoise_1064', 'SINGLE', required = False), DataField('whiteNoise532C', 'WhiteNoise_532C', 'SINGLE', required = False), DataField('linearNoise355', 'LinearNoise_355', 'SINGLE', required = False), DataField('linearNoise532', 'LinearNoise_532', 'SINGLE', required = False), DataField('linearNoise1064', 'LinearNoise_1064', 'SINGLE', required = False), DataField('linearNoise532C', 'LinearNoise_532C', 'SINGLE', required = False), DataField('vFineNoise', 'PhotometerVFineNoise', 'SINGLE', required = False), DataField('vCoarseNoise', 'PhotometerVCoarseNoise', 'SINGLE', required = False), DataField('sphericityNoise', 'SphericityNoise', 'SINGLE', required = False), DataField('f11NoiseFine355', 'F11NoiseFine_355', 'SINGLE', required = False), DataField('f11NoiseFine532', 'F11NoiseFine_532', 'SINGLE', required = False), DataField('f11NoiseFine1064', 'F11NoiseFine_1064', 'SINGLE', required = False), DataField('f11NoiseCoarse355', 'F11NoiseCoarse_355', 'SINGLE', required = False), DataField('f11NoiseCoarse532', 'F11NoiseCoarse_532', 'SINGLE', required = False), DataField('f11NoiseCoarse1064', 'F11NoiseCoarse_1064', 'SINGLE', required = False), DataField('f11NoiseSpherical355', 'F11NoiseSpherical_355', 'SINGLE', required = False), DataField('f11NoiseSpherical532', 'F11NoiseSpherical_532', 'SINGLE', required = False), DataField('f11NoiseSpherical1064', 'F11NoiseSpherical_1064', 'SINGLE', required = False), DataField('f11NoiseSpheroid355', 'F11NoiseSpheroid_355', 'SINGLE', required = False), DataField('f11NoiseSpheroid532', 'F11NoiseSpheroid_532', 'SINGLE', required = False), DataField('f11NoiseSpheroid1064', 'F11NoiseSpheroid_1064', 'SINGLE', required = False), DataField('f22NoiseSpheroid532', 'F22NoiseSpheroid_532', 'SINGLE', required = False), ] createDatabaseErrorMessage = ( 'Aerosol profile output database file could not be created') openDatabaseErrorMessage = ( 'Aerosol profile output file is not a valid Access database or may ' 'not be opened for reading') queryFailureErrorMessage = ( 'Aerosol profile output: failed to query data from the database file') invalidFormatErrorPrefix = ( 'Aerosol profile output file format is invalid: ') writeDataErrorMessage = ( 'Failed to write the data to the aerosol profile output file') importDataFromExcelErrorPrefix = ( 'Failed to read the data from the Excel file') exportDataToExcelErrorPrefix = ( 'Failed to write the data to the Excel file') # ---- Public methods ----------------------------------------------------- def onDataLoaded(self): # Calculate the vertical height step of the data grid. if self.gridStep is None or self.gridZenithAngle is None: self.gridHeigthStep = None else: self.gridHeightStep = (self.gridStep * math.cos(self.gridZenithAngle * math.pi / 180.0)) # Store auxiliary aerosol concentration attributes. if self.sphericity is not None: self.vSpherical = self.vCoarse * (self.sphericity * 0.01) self.vSpheroid = self.vCoarse * (1.0 - self.sphericity * 0.01) else: self.vSpherical = None self.vSpheroid = None # Store retrieved total aerosol concentrations to be displayed by the # table widget. self.calculatedVFine = self.getCalculatedThickness(self.profileFine) if self.profileCoarse is None: # For polarimetric retrievals, use the retrieved profiles. self.calculatedVSpherical = self.getCalculatedThickness( self.profileSpherical) self.calculatedVSpheroid = self.getCalculatedThickness( self.profileSpheroid) self.calculatedVCoarse = ( self.calculatedVSpherical + self.calculatedVSpheroid) else: # For non-polarimetric retrievals, rely upon 'Coarse' profile only. self.calculatedVCoarse = self.getCalculatedThickness( self.profileCoarse) if self.sphericity is not None: self.calculatedVSpherical = self.calculatedVCoarse * ( self.sphericity * 0.01) self.calculatedVSpheroid = self.calculatedVCoarse * ( 1.0 - self.sphericity * 0.01) else: self.calculatedVSpherical = None self.calculatedVSpheroid = None if (self.calculatedVSpherical is None or self.calculatedVSpheroid is None): self.calculatedSphericity = None else: self.calculatedSphericity = self.calculatedVSpherical / ( self.calculatedVSpherical + self.calculatedVSpheroid) * 100.0 def getErrorMessage(self): """Return an error message describing the malformed data or 'None' if all the attribute values are plausible and consistent.""" isPolarimetric = self.profileCoarse is None # Check if some of the values were missing in the database. for name in self.getRequiredAttributeNames(): if getattr(self, name) is not None: continue if not isPolarimetric and name == 'sphericity': continue if isPolarimetric and name.find('Coarse') != -1: continue if not isPolarimetric and (name.find('532C') != -1 or name.find('Spherical') != -1 or name.find('Spheroid') != -1): continue if not isPolarimetric and name in ('molDepolarization532', 'parallelLeakage532'): continue if name.endswith('Near') or name in ( 'joinIndex', 'fineError', 'coarseError', 'atmosphereModel'): continue return ('%s database field contains no data' % txt.quote(self.getFieldName(name))) def lidarChannelIds(self): """Return a list of attribute suffixes for lidar channels used in the retrieval.""" isPolarimetric = self.profileCoarse is None if isPolarimetric: return ('355', '532', '1064', '532C') else: return ('355', '532', '1064') def aerosolModeSuffixes(self): """Return a list of attribute suffixes for aerosol modes that are available in this retrieval.""" isPolarimetric = self.profileCoarse is None if isPolarimetric: return ('Fine', 'Spherical', 'Spheroid') else: return ('Fine', 'Coarse') # ---- Private methods ---------------------------------------------------- def getCalculatedThickness(self, aerosolProfile): # Use 0.001 to convert meters * ppb to micrometers. if aerosolProfile is None: return None return aerosolProfile.sum() * 0.001 * self.gridHeightStep