""" MEX/ELS with the data center support.
A data access to MEX/ELS using the DataCenter approach.
*How to read the data?*
First, prepare the data center
>>> from irfpy.mels import rawdata
>>> dc = rawdata.DataCenterElsCounts()
Specify the time range
>>> import datetime
>>> t0 = datetime.datetime(2014, 10, 19, 17, 54, 55)
>>> t1 = datetime.datetime(2014, 10, 19, 19, 8)
Get the data as two arrays. First for the start time of the session, and the latter for the data.
>>> time_array, data_array = dc.get_array(t0, t1)
>>> print(len(data_array))
954
>>> print(time_array[0])
2014-10-19 17:54:55.271000
>>> print(data_array[0].shape)
(128, 16)
>>> print(data_array[0].max())
20
For practical use, you may create a new numpy array as follows:
>>> import numpy as np
>>> data_array = np.array(data_array)
>>> print(data_array.shape)
(954, 128, 16)
"""
from irfpy.util import datacenter as _datacenter
from irfpy.util import irfpyrc as _irfpyrc
import os as _os
import irfpy.util.exception as _ex
import logging as _logging
import datetime as _datetime
[docs]class DataCenterElsCounts(_datacenter.BaseDataCenter):
""" MEX/ELS count rate data center.
The returned data is in numpy array with a shape of (E128, A16).
(TBC; What if other mode??)
>>> from irfpy.mels import rawdata
>>> import datetime
>>> t0 = datetime.datetime(2010, 10, 30, 23, 15)
>>> mels_datacenter = rawdata.DataCenterElsCounts()
>>> tobs, cnts = mels_datacenter.nearest(t0)
>>> print(tobs, cnts.shape, cnts.max())
2010-10-30 23:15:01.803000 (128, 16) 9
"""
def __init__(self):
_datacenter.BaseDataCenter.__init__(self)
[docs] def search_files(self):
rc = _irfpyrc.Rc()
basepath = rc.get('mels', 'elsmatbase')
if not _os.path.exists(basepath):
msg = 'The path {} not found in ELS data center.'.format(basepath)
raise _ex.IrfpyException(msg)
filelist = []
for pp, dd, ff in _os.walk(basepath):
for f in ff:
pp_split = _os.path.split(pp)
bf = _os.path.basename(f)
if len(bf) != 18:
continue
if not f.startswith(pp_split[-1][-6:]):
continue
if not f.endswith('.mat'):
continue
try:
int(f[:14])
except ValueError:
continue
full_f = _os.path.join(pp, f)
filelist.append(full_f)
return filelist
[docs] def read_file(self, filename):
""" Read the file contents.
From a given file, electron data is read.
The format is in the :class:`irfpy.vels.scidata.ElsCountData` object.
"""
_logger = _logging.getLogger(self.__class__.__name__)
import scipy.io as _spio
try:
mat = _spio.loadmat(filename)
except IOError as e:
emsg = 'Failed to load the matlba file: %s' % filename
_logger.error(emsg)
_logger.error('... Continue with ignoring the file %s' % filename)
return (), ()
obstime_mat = mat['elstimes'].flatten() # (N,) data
from irfpy.util import utc as _utc
obstime_dt = [_utc.mat2dt(_t) for _t in obstime_mat]
ecnts = mat['elsmatrix'] # 128 x 16 x N
# A special case. If the number of data is only one,
# the data is stored as in 2-D data! Annoying feature
# of the dataset, but need to be checked and handled.
if len(obstime_dt) == 1 and ecnts.shape == (128, 16):
import numpy as _np
ecnts = ecnts[:, :, _np.newaxis]
ecnts = list(ecnts.transpose((2, 0, 1)))
return (obstime_dt, ecnts)
[docs] def approximate_starttime(self, filename):
f = _os.path.basename(filename)
yr, mo, dy, hr = int(f[:4]), int(f[4:6]), int(f[6:8]), int(f[8:10])
return _datetime.datetime(yr, mo, dy, hr)