""" Functions to syncronize data from the ICA pipeline and ICA related spice kernels
The functions here support the new key exchange based access method to the pipeline.
For key exchange contact Leif Kalla (leif@irf.se).
.. codeauthor:: Martin Wieser
Module: irfpy.ica.pipeline
You need to have our ssh key public registered with Leif Kalla (leif@irf.se)!
Usage examples::
updateproc('~/icadata/','20150901')
print the progress to stdout::
updateproc('~/icadata/', '2015', privatekey='myicakey', progress=print)
Syncronize latest SPICE kernels:
Get an updated metakernel with associated kernel files::
updatemetakernel('~/icadata/','ros_kernels_3dtool.txt', privatekey='myicakey', progress=print)
.. autosummary::
irfpy.ica.pipeline.updatekernels
irfpy.ica.pipeline.updatemetakernel
irfpy.ica.pipeline.updateproc
irfpy.ica.pipeline.updatespecial
irfpy.ica.pipeline.updateaux
irfpy.ica.pipeline.updatebestc
irfpy.ica.pipeline.updatecmd
irfpy.ica.pipeline.updatemag
irfpy.ica.pipeline.updatecops
"""
import pexpect
import os
import re
import glob
import logging
logging.basicConfig()
_logger = logging.getLogger('ica.pipeline')
_logger.setLevel(logging.DEBUG)
try:
if __name__ == '__main__':
import tools as icatools
else:
from . import tools as icatools
except:
import irfpy.ica.tools as icatools
[docs]def updatekernels(datarootpath, metakernel='ros_kernels_3dtool.txt',
privatekey='~/.ssh/icapipelinekey', progress=None):
r"""Fetches all kernel files referenced by local file metakernel using
using using public key authentication from data.irf.se.
The private key file is specified by privatekey.
PARAMETERS
metakernel:
Filename of local metakernel
datarootpath:
The kernels are copied to the local path datarootpath/kernels/xxx
mirroring the directory structure on data.irf.se.
progress:
Function with one parameter, usially used to print the progress
of the sync process
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
LIMITATIONS
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Authenticate using specific key::
updatekernels('~/icadata/','ros_kernels_3dtool.txt',
'~/.ssh/ica_pipeline_key', progress=print)
or short by making use of defaults::
updatekernels('~/icadata/', progress=print)
Note that there are some restrictions regarding the metakernel
format for this to work. It is requried that the following is on one line:
PATH_VALUES = ( '/irf/data/comet/rosetta/spice/kernels' )
and that there is only one PATH_SYMBOL called $KERNELS. See
ros_kernels_3dtool.txt for an example.
The metakernel 'ros_kernels_3dtool.txt' automatically generated by the
pipeline fulfills these requiremetns automatically.
"""
kernels = []
# expand ~ in filenames
thekey = os.path.expanduser(privatekey)
if not os.path.exists(thekey):
print("Can not find the privatekey file: " + thekey)
return -1
tp = os.path.expanduser(datarootpath)
if tp[-1] != "/":
tp = tp + "/"
if not os.path.exists(tp):
ValueError("Data directory root does not exist: " + tp)
checkdirectory(tp)
tp = tp + 'spice/'
theuser = 'icapy'
srcpath = ''
# find out what kernels are needed
with open(tp + metakernel, "r") as metafile: # read a metakernel
lines = metafile.readlines()
for line in lines:
# if the line contains the string $KERNELS then extract the
# filename from the end of the line
if '$KERNELS' in line:
m = re.search('\$KERNELS/[^/]*/([^/\']*)', line)
if m:
kernels.append(m.group(1))
rules = "--include */ "
for k in kernels:
rules = rules + "--include " + k + " "
rules = rules + "--exclude * --prune-empty-dirs"
# make sure datarootpath/kernels exists...
if not os.path.exists(tp + '/kernels'):
os.makedirs(tp + '/kernels')
if progress is not None:
progress('rsync''ing spice kernels used in "' + metakernel +
'" from ' + theuser + '@data.irf.se...')
ssh_newkey = 'Are you sure you want to continue connecting'
command = "rsync -rLtzv -e 'ssh -p 2200 -i " + thekey + " -o \"PasswordAuthentication no\" -C' " + rules + ' ' + \
theuser + '@adata.irf.se:' + srcpath + '/spice/kernels/ ' + tp + 'kernels'
p = pexpect.spawn(command)
i = p.expect([ssh_newkey, 'password:', 'Permission denied',
'or directory', pexpect.EOF, pexpect.TIMEOUT], 2)
if i == 0:
p.sendline('yes')
i = p.expect([ssh_newkey, 'password:', 'Permission denied',
'or directory', pexpect.EOF, pexpect.TIMEOUT])
if i == 1 or i == 2:
if progress is not None:
progress('Access denied to ' + theuser +
'@data.irf.se. Maybe wrong private key used?')
p.kill(0)
return -2
# run until rsync is complete
while True:
p.timeout = 300 # 5 minutes
i = p.expect(['\r\n', 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
if progress is not None:
progress(bytes.decode(p.before))
if i == 1:
break
elif i == 2:
break
elif i == 3:
if progress is not None:
progress('rsync timeout.')
p.kill(0)
break
if progress is not None:
progress('Kernel syncronization complete.')
return 0
def __fetchdata(level, datarootpath, mask, privatekey, progress,
dataformat='mat', onlynewfiles=False,
**kwargs):
# expand ~ in filenames
tp = os.path.expanduser(datarootpath)
thekey = os.path.expanduser(privatekey)
if not os.path.exists(thekey):
print("Can not find the privatekey file: " + thekey)
return -1
if tp[-1] != "/":
tp = tp + "/"
# sanitize the mask
msk = ''.join(c for c in mask if c.isalnum() or c == '_')
if not os.path.exists(tp):
ValueError("Data directory root does not exist: " + tp)
theuser = 'icapy'
private = kwargs.pop('private', False)
if private:
theuser='icapy_private'
srcpath = ''
checkdirectory(tp)
subdir, extension = icatools.get_data_path_info(level, dataformat)
tp = tp + subdir
if not os.path.exists(tp):
if progress is not None:
progress("Created data directory: " + tp)
os.makedirs(tp)
if progress is not None:
progress('rsync''ing ' + level + ' files matching "' +
msk + '" from ' + theuser + '@data.irf.se...')
ssh_newkey = 'Are you sure you want to continue connecting'
rules = '--include */ --include ' + msk + '* --exclude * '
if onlynewfiles:
rules = "--ignore-existing " + rules
command = "rsync -rLtzv -e 'ssh -p 2200 -i " + thekey + " -o \"PasswordAuthentication no\" -C' " + \
theuser + '@data.irf.se:' + srcpath + '/' + subdir + ' ' + rules + ' ' + tp
if progress is not None:
progress(command)
p = pexpect.spawn(command)
i = p.expect([ssh_newkey, 'password:', 'Permission denied',
'or directory', pexpect.EOF, pexpect.TIMEOUT], 3)
if i == 0:
p.sendline('yes')
i = p.expect([ssh_newkey, 'password:', 'Permission denied',
'or directory', pexpect.EOF, pexpect.TIMEOUT])
if i == 1 or i == 2:
if progress is not None:
progress('Access denied to ' + theuser +
'@data.irf.se. Maybe wrong private key used?')
p.kill(0)
return -2
# run until rsync is complete
while True:
p.timeout = 300
i = p.expect(['\r\n', 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
if progress is not None:
progress(bytes.decode(p.before))
if i == 1:
break
elif i == 2:
break
elif i == 3:
break
return 0
[docs]def updatelevel1(datarootpath, mask='proc', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches level1 proc files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
Consider using updateproc(), updateaux(), updatespecial(), etc. instead.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'proc' : fetch all proc files, all existing dates, default
'proc201502' : fetch all proc files from february 2015
'special20141227T00' : fetch special from 2014-12-27 00h
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read, if equal to 'mat' then the
matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all proc from 2015 using the key icakey::
updatelevel1('/home/myuser/icadataroot/', mask='proc2015', privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
__fetchdata('level1', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updateraw(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches raw files (level0) from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'' : fetch all level0 files, all existing dates, default
'201502' : fetch all level0 files from february 2015
'20141227' : fetch level0 files from 2014-12-27
'20141227T00' : fetch level0 files from 2014-12-27 00h
progress:
Function with one parameter usually used to print the progress
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read, if
equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all raw files from 2015 using the key icakey::
updateraw('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
__fetchdata('level0', datarootpath, 'RPC_ICA_' + mask,
privatekey, progress, dataformat, onlynewfiles, **kwargs)
# compatibilty with old code:
updatelevel0 = updateraw
[docs]def updatecmd(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches command log files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
Note that by default the cmd files are part of the level0 data tree.
mask:
mask defines what files are fetched:
mask defines what files are fetched:
'' : fetch all cmd files, all existing dates, default
'201502' : fetch all cmd files from february 2015
'20141227' : fetch cmd files from 2014-12-27
'20141227T00' : fetch cmd files from 2014-12-27
progress:
Function with one parameter usually used to print the progress
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read,
if equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all raw files from 2015 using the key icakey::
updatecmd('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
__fetchdata('level0', datarootpath, 'rpc' + mask[2:8], privatekey,
progress, dataformat, onlynewfiles,
**kwargs)
[docs]def updateproc(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches level1 proc files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'' : fetch all proc files, all existing dates, default
'201502' : fetch all proc files from february 2015
'20141227T00' : fetch special from 2014-12-27 00h
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read, if
equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all proc from 2015 using the key icakey::
updateproc('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey',progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
if mask[0:4] != 'proc':
mask = 'proc' + mask
__fetchdata('level1', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updatespecial(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat=None, onlynewfiles=False,
**kwargs):
r"""
Fetches special files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all special from 2015 using the key icakey::
updatespecial('/home/myuser/icadataroot/',
privatekey='icakey',progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
__fetchdata('special', datarootpath, '', privatekey, progress, None,
onlynewfiles, **kwargs)
[docs]def updateaux(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False, **kwargs):
r"""
Fetches aux files from data using public key authentication from data.irf.se.
The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'20141227' : fetch aux files from 2014-12-27
'201412' : fetch aux files from 2014-12-01 to 2014-12-31
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read,
if equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all aux from 2015 using the key icakey::
updatelevel1('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
# below is for compatibility with old code
if mask[0:3] != 'aux':
mask = 'aux' + mask
__fetchdata('aux', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updatelap(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches lap files from data using public key authentication from
data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'20141227' : fetch lap files from 2014-12-27
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read,
if equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all lap from 2015 using the key icakey::
updatelevel1('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
# below is for compatibility with old code
if mask[0:3] != 'lap':
mask = 'lap' + mask
__fetchdata('lap', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updatecops(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches cops files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'20141227' : fetch cops files from 2014-12-27
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read,
if equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all cops from 2015 using the key icakey::
updatecops('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
# below is for compatibility with old code
if mask[0:4] != 'cops':
mask = 'cops' + mask
__fetchdata('cops', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updatemag(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches mag files from data using public key authentication from
data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'20141227' : fetch mag files from 2014-12-27
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read,
if equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all cops from 2015 using the key icakey::
updatemag('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
# below is for compatibility with old code
if mask[0:3] != 'mag':
mask = 'mag' + mask
__fetchdata('mag', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updatebestc(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches bestc (level2) files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'20141227' : fetch bestc files from 2014-12-27
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if equal to 'h5' then the hdf5 data tree is read,
if equal to 'mat' then the matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all bestc from 2015 using the key icakey::
updatebestc('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
# below is for compatibility with old code
if mask[0:5] != 'bestc':
mask = 'bestc' + mask
__fetchdata('level2', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def updateflux(datarootpath, mask='', privatekey='~/.ssh/icapipelinekey',
progress=None, dataformat='mat', onlynewfiles=False,
**kwargs):
r"""
Fetches flux (level3) files from data using public key authentication
from data.irf.se. The private key file is specified by privatekey.
datarootpath:
The data is copied to the local directory datarootpath.
mask:
mask defines what files are fetched:
'20141227' : fetch flux files from 2014-12-27
privatekey:
Path to the private key file, default '~/.ssh/icapipelinekey'
progress:
Function with one parameter usually used to print the progress
dataformat (str):
'mat' or 'h5'. if 'h5' then the hdf5 data tree is read if 'mat' then the
matlab data tree is read.
onlynewfiles (bool):
If set to True, only files not existing locally will be fetched.
Files that exist locally but in an outdated version are not updated.
This is useful for slow links. Note this may result in inconsistent
data files. Default is False.
Limitations:
It is assumed that a key exchange has happened previously for
user icapy@data.irf.se.
Contact Leif Kalla (leif@irf.se) for key exchange.
Fetch all flux from 2015 using the key icakey::
updatebestc('/home/myuser/icadataroot/', mask='2015',
privatekey='icakey' progress=p)
progress is a function with one string parameter that is called for
each line rsync returns. Example::
def p(line):
print(line)
"""
# below is for compatibility with old code
if mask[0:4] != 'flux':
mask = 'flux' + mask
__fetchdata('level3', datarootpath, mask, privatekey, progress, dataformat,
onlynewfiles, **kwargs)
[docs]def checkdirectory(d):
""" Verifies that the given directory points to the root of an ICA data tree """
if d[-1] != "/":
d = d + "/"
pathendswith = os.path.basename(os.path.normpath(d))
if (len(glob.glob(d + '20*')) > 0) or \
(pathendswith in ['aux', 'bestc', 'proc', 'mag', 'cops',
'lap', 'level1', 'level2', 'level3', 'physmass']):
raise ValueError("\n" +
"*********************************************************************\n" +
"The given path to ICA data files '" + d +
"'\npoints to the wrong place: " +
"Starting from version 3.5.0 of irfpy.ica,\n" +
"the data path should point ot the top of a subdirectory structure\n" +
"as shown in \n" +
"https://rosetta-wiki.irf.se/doku.php?id=pipeline_directory_structure\n" +
"For the given path '" + d +
"', \nthe data files were expected to be in: \n" +
"'" + d + "xxxxx/matlab/20yy/mm/dd/*.mat'\n" +
"(with xxxxx one of level0, level1, level2, mag, cops, aux, etc.),\n" +
"but your data is apparently located in: \n" +
"'" + d + "20yy/mm/dd/*.mat'\n" +
"Your path must not contain the 'xxxxx' or 'matlab' parts.\n" +
"Please update your local data structure by moving files.\n" +
"Contact wieser@irf.se if you need support in this matter.\n" +
"*********************************************************************\n" +
"\n")