""" Another (new) implementation of raw data access.
A new implementation of raw data access using the
data center (:class:`irfpy.util.datacenter.BaseDataCenter`) approach.
.. seealso:: :mod:`irfpy.vima.scidata_util`
*For developer*
This is almost identical to MEX version. So if one change the contents,
it is also recommended to change MEX version.
"""
import os as _os
import logging as _logging
from irfpy.util import datacenter as _dc
from irfpy.util import timeseries as _ts
from irfpy.util import exception as _ex
from irfpy.util import irfpyrc as _irfpyrc
from irfpy.vima import SENSOR_NAME
from irfpy.vima import scidata as _scidata
from irfpy.vima import scidata_tree as _st
from irfpy.vima import scidata_util as _vsu
_logger = _logging.getLogger(__name__)
[docs]class DataCenterCount2d(_dc.BaseDataCenter):
""" Raw count data center for 2D, emulated matrix.
>>> dc = DataCenterCount2d()
>>> import datetime
>>> t0 = datetime.datetime(2007, 3, 25, 7)
>>> t, d = dc.nearest(t0)
>>> print(t)
2007-03-25 06:59:48.864080
>>> print(d)
<<class 'irfpy.imacommon.imascipac.CntMatrix2D'>(VEX/IMA)@2007-03-25T06:59:48.864080:MOD=25 >>25<<:POL=12/13:CNTmax=2>
>>> t1 = datetime.datetime(2007, 3, 25, 8)
>>> tlist, dlist = dc.get_array_strict(t0, t1)
>>> from pprint import pprint
>>> pprint(tlist) # doctest: +ELLIPSIS
[datetime.datetime(2007, 3, 25, 7, 0, 12, 864080),
datetime.datetime(2007, 3, 25, 7, 0, 36, 864120),
...
datetime.datetime(2007, 3, 25, 7, 54, 37, 239500)]
>>> for t, d in dc.iter_wide(t0, t1):
... print(t) # doctest: +ELLIPSIS
2007-03-25 06:59:48.864080
2007-03-25 07:00:12.864080
...
2007-03-25 07:54:13.239500
2007-03-25 07:54:37.239500
2007-03-25 15:02:14.711540
"""
def __init__(self, emulate_full=True):
rc = _irfpyrc.Rc()
self.basepath = rc.get('vima', 'imamatbase')
self.emulate_full = emulate_full
if not _os.path.isdir(self.basepath):
msg = 'The path {} not found for VIMA data center.'.format(self.basepath)
_logger.error(msg)
raise ValueError(msg)
_dc.BaseDataCenter.__init__(self)
[docs] def search_files(self):
filelist = []
for pp, dd, ff in _os.walk(self.basepath):
for f in ff:
if not f.endswith('.mat'):
_logger.info('{} is not mat file. Skipped.'.format(f))
continue
fn = _os.path.join(pp, f)
filelist.append(fn)
return filelist
[docs] def read_file(self, filename):
return read_mat_file(filename, emulate_full=self.emulate_full)
[docs] def approximate_starttime(self, filename):
return _st._guess(filename)
[docs]class DataCenterCount3d(_dc.BaseDataCenter):
""" Raw count data center for IMA 3D data.
The datacenter approach is used:
>>> dc = DataCenterCount3d()
>>> import datetime
>>> t0 = datetime.datetime(2007, 3, 25, 7)
>>> t, d = dc.nearest(t0)
>>> print(t)
2007-03-25 07:00:36.864120
>>> print(d)
<<class 'irfpy.imacommon.imascipac.CntMatrix'>(VEX/IMA)@2007-03-25T07:00:36.864120:MOD=24 >>25<<:CNTmax=192>
For raw data (withouut emulation to mode 24), you can use different datacenter as
>>> dc = DataCenterCount3d(emulate_full=False)
>>> t, d = dc.nearest(t0)
>>> print(t) # This should be the same
2007-03-25 07:00:36.864120
>>> print(d)
<<class 'irfpy.imacommon.imascipac.CntMatrix'>(VEX/IMA)@2007-03-25T07:00:36.864120:MOD=25 >>25<<:CNTmax=384>
>>> print(d.matrix.shape)
(32, 16, 96, 8)
>>> t1 = datetime.datetime(2007, 3, 25, 8)
>>> tlist, dlist = dc.get_array_strict(t0, t1)
>>> from pprint import pprint
>>> pprint(tlist) # doctest: +ELLIPSIS
[datetime.datetime(2007, 3, 25, 7, 0, 36, 864120),
datetime.datetime(2007, 3, 25, 7, 3, 48, 832880),
...
datetime.datetime(2007, 3, 25, 7, 48, 37, 239480),
datetime.datetime(2007, 3, 25, 7, 51, 49, 239500)]
>>> for t, d in dc.iter_wide(t0, t1):
... print(t) # doctest: +ELLIPSIS
2007-03-25 06:57:24.864080
2007-03-25 07:00:36.864120
...
2007-03-25 07:51:49.239500
2007-03-25 15:02:14.711540
"""
def __init__(self, emulate_full=True):
rc = _irfpyrc.Rc()
self.basepath = rc.get('vima', 'imamatbase')
self.emulate_full = emulate_full
if not _os.path.isdir(self.basepath):
msg = 'The path {} not found for VIMA data center.'.format(self.basepath)
_logger.error(msg)
raise ValueError(msg)
_dc.BaseDataCenter.__init__(self, cache_size=10)
[docs] def search_files(self):
filelist = []
for pp, dd, ff in _os.walk(self.basepath):
for f in ff:
if not f.endswith('.mat'):
_logger.info('{} is not mat file. Skipped.'.format(f))
continue
fn = _os.path.join(pp, f)
filelist.append(fn)
return filelist
[docs] def read_file(self, filename):
return read_mat_file_3d(filename, emulate_full=self.emulate_full)
[docs] def approximate_starttime(self, filename):
return _st._guess(filename)
[docs]def read_mat_file(filename, emulate_full=True):
try:
arranged_matrix = _vsu._arrange_matlab_file(filename, emulate_full=emulate_full)
except _ex.IrfpyException:
return _ts.ObjectSeries([], []) # No data to return
tlist = []
dlist = []
for t, d in arranged_matrix:
d.name = SENSOR_NAME
tlist.append(t)
dlist.append(d)
return _ts.ObjectSeries(tlist, dlist)
[docs]def read_mat_file_3d(filename, emulate_full=True):
""" Return the 3D data series.
:param filename: The matlab file name
:return: (tlist, dlist), where tlist is the observation time and dlist is for the data.
"""
try:
arranged_matrix = _vsu._arrange_matlab_file(filename, emulate_full=emulate_full)
except _ex.IrfpyException:
return _ts.ObjectSeries([], []) # No data to return
from irfpy.imacommon import imascipac as _imascipac
arranged_matrix = _imascipac.convert2to3(arranged_matrix, emulate_full=emulate_full)
tlist = []
dlist = []
for t, d in arranged_matrix:
d.name = SENSOR_NAME
tlist.append(t)
dlist.append(d)
return tlist, dlist