''' This module provides an implementation of time series of database index.
.. codeauthor:: Yoshifumi Futaana
Frequently the need of database in timeseries order.
Main use is a dataset under a specific folder.
For example, there is a database like::
rootfolder -+- 200501 -+- 20050101.dat # data from 2005-01-01 00:00:00
| +- 20050102.dat # data from 2005-01-02 00:00:00
| +- 20050103.dat # data from 2005-01-03 00:00:00
| +- ...
|
+- 200502 -+- 20050215.dat # data from 2005-02-15 00:00:00
| +- 20050216.dat # data from 2005-02-16 00:00:00
| +- 20050217.dat # data from 2005-02-17 00:00:00
| +- ...
|
+- ...
The ``timeseriesdb`` module provides to connect between the time and the file name.
The user can find the file name from time. :class:`DB` class provides an implementation
of such database.
In case you do not know the exact start time of the data file, you can
use :class:`FazzyDB` class.
Usage follows. Use :meth:`append` method to connect the file name and the start time
of the data file.
>>> db = DB() # Instance the DB object.
>>> db.append('rootfolder/200501/20050101.dat', datetime.datetime(2005, 1, 1))
>>> db.append('rootfolder/200501/20050102.dat', datetime.datetime(2005, 1, 2))
>>> db.append('rootfolder/200501/20050103.dat', datetime.datetime(2005, 1, 3))
>>> db.append('rootfolder/200502/20050215.dat', datetime.datetime(2005, 2, 15))
>>> db.append('rootfolder/200502/20050216.dat', datetime.datetime(2005, 2, 16))
>>> db.append('rootfolder/200502/20050217.dat', datetime.datetime(2005, 2, 17))
>>> print(db.get(datetime.datetime(2005, 1, 1, 12, 0, 0)))
rootfolder/200501/20050101.dat
>>> print(db.get(datetime.datetime(2005, 1, 3, 0, 0, 0)))
rootfolder/200501/20050103.dat
>>> print(db.get(datetime.datetime(2005, 5, 3, 0, 0, 0))) # later than last data
rootfolder/200502/20050217.dat
Another example is as follows.
::
rootfolder -+- orb0001 # Data from 2004-01-05 15:30:00
+- orb0002 # Data from 2004-01-05 18:30:00
+- orb0003 # Data from 2004-01-06 00:30:00
+- orb0004 # Data from 2004-01-06 09:30:00
+- orb0011 # Data from 2004-01-12 11:30:00
| # no data and file between 5 and 10.
+- orb0012 # Data from 2004-01-12 21:30:00
>>> db = DB()
>>> db.append('rootfolder/orb0001', datetime.datetime(2004, 1, 5, 15, 30))
>>> db.append('rootfolder/orb0002', datetime.datetime(2004, 1, 5, 18, 30))
>>> db.append('rootfolder/orb0003', datetime.datetime(2004, 1, 6, 0, 30))
>>> db.append('rootfolder/orb0004', datetime.datetime(2004, 1, 6, 9, 30))
>>> db.append('rootfolder/orb0011', datetime.datetime(2004, 1, 12, 11, 30))
>>> db.append('rootfolder/orb0012', datetime.datetime(2004, 1, 12, 21, 30))
>>> # If you give the date before the data start, DataNotInDbError is returned.
>>> print(db.get(datetime.datetime(2004, 1, 1))) #doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
irfpy.util.timeseriesdb.DataNotInDbError: message
>>> print(db.get(datetime.datetime(2004, 1, 5, 15, 30)))
rootfolder/orb0001
>>> # Most likely the data at 2004-01-10T00:00:00 is not in the dataset,
>>> # but still returns as included in orb0004, because the epoch is between
>>> # the starts of 0004 and 0011.
>>> print(db.get(datetime.datetime(2004, 1, 10, 0, 0)))
rootfolder/orb0004
>>> # As the end of the time is not included in the database,
>>> # the last file is always returned if you give later than the database coverage.
>>> print(db.get(datetime.datetime(2100, 1, 1, 0, 0, 0)))
rootfolder/orb0012
See :ref:`cookbook_timeseriesdb` for more information.
**Fazzy database**
Sometimes, files in the database do not provide exact start times
because the computational cost to get the exact start time are expensive.
In such cases, "fazzy database" may provide a better solution rather
than surveying all the files.
To use this fazzy database strategy, still one must know
the chronological order of the files, and estimate the start times
for all files.
A rough estimates of start times is usually fine.
They can be from the file name and other resources (orbit number), for example.
Even "evenly-distributed" times may be acceptable.
You can see sample in :class:`FazzyDB`.
.. warning::
Before the version 4.2.6a3, the :class:`DB` and :class:`FazzyDB` had a critical error.
The bug was reported in https://gitlab.irf.se/irfpy/util/issues/2
and has been fixed by the commit ac33aade.
'''
import datetime
import bisect
import logging
_logger = logging.getLogger(__name__)
[docs]class DataNotInDbError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
[docs]class DuplicatedError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
[docs]class DB:
''' Implementation of the timeseries database.
'''
# Note for developers
# If your new method will change the data, the sorted key should be
# flushed with _flush_sortedkeys() method.
# If your new method will refer to the sorted key, the sorted key
# should be always produced with _make_sortedkeys() method.
logger = logging.getLogger(__name__ + '.DB')
def __init__(self):
self.dbdict = {}
self.invdict = {}
self._flush_sortedkeys()
def _flush_sortedkeys(self):
''' For consistency, remove sorted key.
'''
self.invdict_sortedkey = None
self.dbdict_sortedkey = None
def _make_sortedkeys(self):
''' For performance sorted key is generated.
For performance sorted key is generated only
when sortedkey is None. If you want to re-generate sorted keys,
first flush with :meth:`_flush_sortedkey`.
'''
# if self.invdict_sortedkey == None:
# self.invdict_sortedkey = sorted(self.invdict.keys())
if self.dbdict_sortedkey is None or self.invdict_sortedkey is None:
self.dbdict_sortedkey = sorted(self.dbdict.keys())
self.invdict_sortedkey = []
for t in self.dbdict_sortedkey:
self.invdict_sortedkey.append(self.dbdict[t])
@classmethod
def _get_sample_database(cls):
''' Return a sample data base.
>>> db = DB._get_sample_database()
>>> print(len(db))
6
'''
db = DB()
db.append('rootfolder/orb0001', datetime.datetime(2004, 1, 5, 15, 30))
db.append('rootfolder/orb0002', datetime.datetime(2004, 1, 5, 18, 30))
db.append('rootfolder/orb0003', datetime.datetime(2004, 1, 6, 0, 30))
db.append('rootfolder/orb0004', datetime.datetime(2004, 1, 6, 9, 30))
db.append('rootfolder/orb0011', datetime.datetime(2004, 1, 12, 11, 30))
db.append('rootfolder/orb0012', datetime.datetime(2004, 1, 12, 21, 30))
return db
[docs] def append(self, filename, starttime):
''' Append the file into the database together with the start time.
>>> db = DB()
>>> db.append('file1', datetime.datetime(2009, 1, 1, 0, 0, 0))
>>> try:
... db.append('file1', datetime.datetime(2009, 2, 1, 0, 0, 0))
... print("!!!! Should not reach here")
... except DuplicatedError as e:
... print("Duplicated error correctly caught!")
Duplicated error correctly caught!
'''
if starttime in self.dbdict:
raise DuplicatedError('Time %s exists in DB (%s).' % (starttime, filename))
if filename in self.invdict:
raise DuplicatedError('File %s exists in DB.' % filename)
self.dbdict[starttime] = filename
self.invdict[filename] = starttime
# If you added an entry, the stored sorted keys are flushed.
self._flush_sortedkeys()
[docs] def remove(self, filename):
""" Remove the file from the database
:param filename: The file name.
:return: None
If the filename does not exist, ``ValueError`` is raised.
"""
if filename not in self.invdict:
raise ValueError('Filename {} not in the database.')
starttime = self.gettime(filename)
self.dbdict.pop(starttime)
self.invdict.pop(filename)
self._flush_sortedkeys()
[docs] def t0(self):
''' Return the first time
Note that the "last time" cannot be identified, because
the dataset is only for start time.
>>> db = DB()
>>> db.append('a', datetime.datetime(2009, 1, 10))
>>> print(db.t0())
2009-01-10 00:00:00
>>> db.append('b', datetime.datetime(2008, 1, 25, 12))
>>> print(db.t0())
2008-01-25 12:00:00
>>> db.append('c', datetime.datetime(2012, 1, 25, 12))
>>> print(db.t0())
2008-01-25 12:00:00
'''
self._make_sortedkeys()
return self.dbdict_sortedkey[0]
[docs] def get(self, t):
''' Return the filename that contains the date of the specified time.
Return the filename that contains the data of the specified time.
If the specified time is before the DB start time,
:class:`DataNotInDbError` is raised.
First, load the sample data base.
>>> db = DB._get_sample_database()
2004-01-05T15:30:00 contains in orb0001.
>>> t0 = datetime.datetime(2004, 1, 5, 15, 30)
>>> print(db.get(t0))
rootfolder/orb0001
2004-01-05T17:00:00 is also in orb001.
>>> t1 = datetime.datetime(2004, 1, 5, 17)
>>> print(db.get(t1))
rootfolder/orb0001
2004-01-05T00:00:00 is before the database.
>>> t2 = datetime.datetime(2004, 1, 5, 0)
>>> try:
... print(db.get(t2))
... print("!!!! Should not reach here")
... except DataNotInDbError as e:
... print("Exception caught")
Exception caught
2010-01-01T00:00:00 is, in a common sense, not included
in this data base, but it returns the last file.
>>> t3 = datetime.datetime(2010, 1, 1)
>>> print(db.get(t3))
rootfolder/orb0012
'''
self._make_sortedkeys()
keys = self.dbdict_sortedkey
idx = bisect.bisect(keys, t)
if idx == 0:
raise DataNotInDbError("Time %s is before the DB time." % t)
key = keys[idx - 1]
return self.dbdict[key]
[docs] def getfiles(self, t0, t1):
''' Return the filenames that covers the specified range
:param t0: Start. (``datetime.datetime``)
:param t1: End. (``datetime.datetime``)
:returns: Tuple of the file names.
>>> db = DB()
>>> db.append('orb0001', datetime.datetime(2004, 1, 5, 15, 30))
>>> db.append('orb0002', datetime.datetime(2004, 1, 5, 18, 30))
>>> db.append('orb0003', datetime.datetime(2004, 1, 6, 0, 30))
>>> db.append('orb0004', datetime.datetime(2004, 1, 6, 9, 30))
>>> db.append('orb0005', datetime.datetime(2004, 1, 6, 18, 30))
>>> db.append('orb0006', datetime.datetime(2004, 1, 7, 3, 30))
>>> db.append('orb0011', datetime.datetime(2004, 1, 12, 11, 30))
>>> db.append('orb0012', datetime.datetime(2004, 1, 12, 21, 30))
>>> print(db.getfiles(datetime.datetime(2004, 1, 6), datetime.datetime(2004, 1, 7)))
('orb0002', 'orb0003', 'orb0004', 'orb0005')
>>> print(db.getfiles(datetime.datetime(2004, 1, 1), datetime.datetime(2004, 1, 6)))
('orb0001', 'orb0002')
'''
if t0 > t1:
raise ValueError('t0 must be earlier than t1.')
try:
f1 = self.get(t1)
except DataNotInDbError:
raise
try:
f0 = self.get(t0)
except DataNotInDbError:
f0 = self.get(self.t0())
f = f0
fnames = [f]
while f != f1:
f = self.nextof(f)
fnames.append(f)
return tuple(fnames)
[docs] def nextof(self, filename):
''' Return the next data of the given filename.
If the given filename is not found in the database,
ValueError is returned.
If the given file is the last file, DataNotInDbError is raised.
>>> db = DB._get_sample_database()
>>> print(db.nextof('rootfolder/orb0001'))
rootfolder/orb0002
>>> try:
... print(db.nextof('rootfolder/orb0005'))
... print("!!!! Should not reach here")
... except ValueError as e:
... print("Exception correctly caught")
Exception correctly caught
>>> try:
... print(db.nextof('rootfolder/orb0012'))
... print("!!!! Should not reach here")
... except DataNotInDbError as e:
... print("Exception correctly caught")
Exception correctly caught
'''
self._make_sortedkeys()
files = self.invdict_sortedkey
try:
idx = files.index(filename)
except ValueError as e:
self.logger.warning('No given file in the DB (%s).' % filename)
raise
try:
next = files[idx + 1]
except IndexError as e:
# self.logger.warning('This is the last file in the DB (%s)' % filename)
raise DataNotInDbError('This is the last file in the DB (%s)' % filename)
return next
[docs] def previousof(self, filename):
''' Return the previous data of the given filename
If the given filename is not found in the database,
KeyError is returned.
If the given file is the last file, DataNotInDbError is raised.
>>> db = DB._get_sample_database()
>>> try:
... print(db.previousof('rootfolder/orb0001'))
... print("!!!! Should not reach here")
... except DataNotInDbError as e:
... print("Exception correctly caught")
Exception correctly caught
'''
self._make_sortedkeys()
files = self.invdict_sortedkey
try:
idx = files.index(filename)
except ValueError as e:
self.logger.warning('No given file in the DB (%s).' % filename)
raise
if idx == 0:
# self.logger.warning('This is the first file in the DB (%s).'
# % filename)
raise DataNotInDbError('This is the first file in the DB (%s).'
% filename)
prev = files[idx - 1]
return prev
[docs] def gettime(self, filename):
''' Return the registered time for corresponding filename.
>>> db = DB._get_sample_database()
>>> print(db.gettime('rootfolder/orb0004'))
2004-01-06 09:30:00
'''
return self.invdict[filename]
def __len__(self):
''' Return the number of entries of the data base.
>>> db = DB()
>>> db.append('rootfolder/orb0001', datetime.datetime(2004, 1, 5, 15, 30))
>>> db.append('rootfolder/orb0002', datetime.datetime(2004, 1, 5, 18, 30))
>>> print(len(db))
2
'''
return len(self.dbdict)
[docs] def clear(self):
self.dbdict = {}
self.invdict = {}
self._flush_sortedkeys()
def __str__(self):
self._make_sortedkeys()
ndata = len(self)
def _fmt(t):
return " {:%FT%T.%f} : {}\n".format(t, self.get(t))
s = "<timeseriesdb.DB object with {} entry>\n".format(len(self))
if ndata == 0:
return s
if ndata <= 10:
for t in self.dbdict_sortedkey:
s += _fmt(t)
else:
for t in self.dbdict_sortedkey[:9]:
s += _fmt(t)
s += '...\n'
for t in self.dbdict_sortedkey[-2:]:
s += _fmt(t)
return s
[docs] def print_all(self):
self._make_sortedkeys()
for i, t in enumerate(self.dbdict_sortedkey):
print("[{:4d}] {:%FT%T.%f} : {}".format(i, t, self.get(t)))
[docs] def print_invall(self):
self._make_sortedkeys()
for i, fn in enumerate(self.invdict_sortedkey):
print("[{:4d}] {} : {:%FT%T.%f}".format(i, fn, self.invdict[fn]))
[docs]class FazzyDB:
''' Time series database with a fazzy start time definition.
Sample of fazzy database:
If you have a dataset file0, file1, ... file4.
Assume you do not know the exact start times of these data files without
getting surveying all the data files, which may take time.
Now, assume you have guessed the start times as follows.::
file0 1996-01-01
file1 1997-01-01
file2 1998-01-01
file3 1999-01-01
file4 2000-01-01
Using the guessed times, first create :class:`DB` object.
>>> guessdb = DB()
>>> guessdb.append('file0', datetime.datetime(1996, 1, 1))
>>> guessdb.append('file1', datetime.datetime(1997, 1, 1))
>>> guessdb.append('file2', datetime.datetime(1998, 1, 1))
>>> guessdb.append('file3', datetime.datetime(1999, 1, 1))
>>> guessdb.append('file4', datetime.datetime(2000, 1, 1))
However, suppose the real start times are indeed as follows.::
file0 1996-06-01
file1 1996-08-01
file2 1996-10-01
file3 1999-08-01
file4 1999-10-01
Note that all the real start times of the files are not known a-pri-ori.
Suppose you need to call the following function.
>>> def real_start(filename): # Emulating the start time retrieval function
... st={'file0': datetime.datetime(1996, 6, 1),
... 'file1': datetime.datetime(1996, 8, 1),
... 'file2': datetime.datetime(1996,10, 1),
... 'file3': datetime.datetime(1999, 8, 1),
... 'file4': datetime.datetime(1999,10, 1),}
... # sleep(100) # Very heavy processing :)
... return st[filename]
Ok. Now preparation is ready. Make :class:`FazzyDB` object.
>>> fdb = FazzyDB(guessdb, real_start)
>>> print(len(fdb))
5
>>> fdb.get(datetime.datetime(1999, 3, 1))
'file2'
>>> fdb.get(datetime.datetime(1996, 6, 1))
'file0'
>>> fdb.get(datetime.datetime(1996, 7, 31))
'file0'
>>> fdb.get(datetime.datetime(1996, 8, 1))
'file1'
>>> fdb.get(datetime.datetime(1996, 10, 1))
'file2'
>>> fdb.get(datetime.datetime(1999, 9, 9))
'file3'
>>> fdb.get(datetime.datetime(1999, 10, 1))
'file4'
>>> fdb.get(datetime.datetime(1999, 10, 21))
'file4'
>>> fdb.get(datetime.datetime(2050, 10, 1))
'file4'
>>> fdb.get(datetime.datetime(1996, 3, 1)) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
irfpy.util.timeseriesdb.DataNotInDbError: 'This is the first file in the DB (file0).'
>>> fdb.get(datetime.datetime(1990, 3, 1)) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
irfpy.util.timeseriesdb.DataNotInDbError: 'This is the first file in the DB (file0).'
*Getting the first time*
>>> print(fdb.t0())
1996-06-01 00:00:00
*Getting files*
>>> print(fdb.getfiles(datetime.datetime(1996, 1, 1), datetime.datetime(1997, 1, 1)))
('file0', 'file1', 'file2')
'''
logger = logging.getLogger(__name__ + '.FazzyDB')
def __init__(self, guessed_db, func_getstart):
''' Create fazzy DB.
:param guessed_db: Guessed database (:class:`DB` object)
:param func_getstart: A function that eats filename and returns
the start time.
'''
self.guessed_db = guessed_db
self.func_getstart = func_getstart
self.exact_time_cache = {}
[docs] def get_filename_from_database(self, t):
''' Return the filename from the guessed databse
'''
return self.guessed_db.get(t)
[docs] def get_exactstart(self, filename):
''' Return the exact time range
The exact time range is returned by the real evaluation of
the data, or from the cache.
timerangefunc specified in the __init__ is used.
'''
if not (filename in self.exact_time_cache):
# self.logger.debug('#### GETTING START FOR {}'.format(filename))
t0 = self.func_getstart(filename)
self.exact_time_cache[filename] = t0
return self.exact_time_cache[filename]
[docs] def get(self, t):
''' Return the filename
'''
# self.logger.debug('# Time: {}'.format(str(t)))
# First, get the filename from the guessed time, as a start point.
try:
filename0 = self.get_filename_from_database(t)
except DataNotInDbError as e:
# If the given time, t, is before the guessed db start,
# the guessed db raises the exeption but still,
# we need to verify from the first file.
filename0 = self.get_filename_from_database(self.guessed_db.t0())
# Second, find out the filename of the next file.
try:
filename1 = self.guessed_db.nextof(filename0)
except DataNotInDbError:
# If the guessed file is the end, rewind one file, since still
# we need to verify it.
filename1 = filename0
filename0 = self.guessed_db.previousof(filename1)
# Third, main loop
while True:
# Get the exact time.
t0 = self.get_exactstart(filename0)
if t0 is None:
self.guessed_db.remove(filename0)
return self.get(t) # Recursive call.
t1 = self.get_exactstart(filename1)
if t1 is None:
self.guessed_db.remove(filename1)
return self.get(t)
# self.logger.debug('# Start file 0: {} {}'.format(filename0, str(t0)))
# self.logger.debug('# Start file 1: {} {}'.format(filename1, str(t1)))
if t0 <= t < t1:
# self.logger.debug('# ---(%s)-*-(%s)---' % (t0, t1))
return filename0
elif t < t0: # The given time is earlier than the expected file's start
# self.logger.debug('# -*-(%s)---(%s)---' % (t0, t1))
filename1 = filename0
filename0 = self.guessed_db.previousof(filename1)
# Here exception, DataNotInDbError, will be raised
# if the filename0 is the fist of the dataset
else: # The given time is later than the expected file's start
# self.logger.debug('# ---(%s)---(%s)-*-' % (t0, t1))
filename0 = filename1
try:
filename1 = self.guessed_db.nextof(filename0)
except DataNotInDbError: # This means that the file examined is the last.
return filename1
[docs] def getfiles(self, t0, t1):
''' Return the filenames that covers the specified range
:param t0: Start. (``datetime.datetime``)
:param t1: End. (``datetime.datetime``)
:returns: Tuple of the file names.
'''
if t0 > t1:
raise ValueError('t0 must be earlier than t1.')
try:
f1 = self.get(t1)
except DataNotInDbError:
raise
try:
f0 = self.get(t0)
except DataNotInDbError:
f0 = self.get(self.t0())
f = f0
fnames = [f]
while f != f1:
f = self.nextof(f)
fnames.append(f)
return tuple(fnames)
[docs] def nextof(self, filename):
'''
'''
return self.guessed_db.nextof(filename)
[docs] def previousof(self, filename):
'''
'''
return self.guessed_db.previousof(filename)
[docs] def t0(self):
'''
'''
t0g = self.guessed_db.t0()
f0g = self.guessed_db.get(t0g) # First file name
return self.gettime(f0g)
def __len__(self):
'''
'''
return len(self.guessed_db)
[docs] def gettime(self, filename):
'''
'''
return self.get_exactstart(filename)
def __str__(self):
dbstr = str(self.guessed_db).split('\n')[0] # The first line of guessed_db output
s = "<timeseriesdb.FassyDB object wrapping {}>".format(dbstr)
return s