Commit 85bdd50f authored by Caillat Michel's avatar Caillat Michel
Browse files

Continued to work on the refactoring

parent 04a1ee27
......@@ -485,12 +485,6 @@ class DataBlock:
#===========================================================================
# Public getters and accessors.
#
def getInfo(self):
return {
'relFITSFilePath': self.__relFITSFilePath,
'timestamp' : datetime.fromtimestamp(timestamp).strftime("%m/%d/%Y, %H:%M:%S")
}
def getTimestamp(self):
return self.timestamp
......
......@@ -24,6 +24,8 @@ import dask.array as da
import traceback
import DataBlock
mpl.use('Agg') # to avoid problem when DISPLAY isn't defined.
from matplotlib import pyplot as pp
......@@ -31,6 +33,17 @@ DataRoot = "dataroot"
FITSFilePrefix = DataRoot + '/FITS/'
PNGFilePrefix = DataRoot + '/PNG/'
#
#
# Tell DataBlock where the files will be located
#
# The FITS files
DataBlock.setFITSFilePrefix(FITSFilePrefix)
# The PNG files
DataBlock.setPNGFilePrefix(PNGFilePrefix)
def __cmap2palette (name):
x = pp.get_cmap(palette_name)
return [tuple(map(lambda x: int(round(x*255)), x(i)[0:3])) for i in range(x.N)]
......@@ -82,6 +95,53 @@ class DataManagerImpl :
else:
return {status: True, message: 'FITS file "{relFITSFilePath}" is not present in memory. Call "setData" first'}
def __convert_size(self, size_bytes):
if size_bytes == 0:
return "0B"
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
i = int(math.floor(math.log(size_bytes, 1024)))
p = math.pow(1024, i)
s = round(size_bytes / p, 2)
return "%s %s" % (s, size_name[i])
def __getEntries_0(self, relKey):
self.__logger.debug("__getEntries_0: entering")
try :
absFITSFilePrefix = FITSFilePrefix + relKey
entries = (os.listdir(absFITSFilePrefix))
self.__logger.debug("type of entries %s" % (type(entries)))
sortedEntries = entries.sort()
children = []
for entry in entries:
p = absFITSFilePrefix + '/' + entry
if entry.startswith(".") or (not os.path.isdir(p) and not p.endswith(".fits")) :
continue
elif entry in ["log", "NOFITS", "IGNORE"]:
continue
else:
d = dict()
d["key"] = relKey + '/' + entry
d["folder"] = os.path.isdir(p)
d["lazy"] = d["folder"]
if entry.endswith(".fits") :
size = self.__convert_size(os.path.getsize(p))
d["title"] = "<a href = 'visit/?path=%s/%s' target = '_blank'>%s %s</a>" % (relKey, entry, entry, size)
else:
d["title"] = entry
children.append(d)
result = {"status": True, "message": "", "result": json.dumps(children)}
except Exception as e :
result = {"status": False, "message": "Problem while looking for entries under '%s'. Error message was '%s'" % ( relKey, e)}
self.__logger.debug("%r" % result)
self.__logger.debug("__getEntries_0: exiting")
return result
#
# Return the keys of the dataBlocks dictionary as if it was ordered by timestamp ( asc or desc)
#
def __getKeysOrderedByTimestamp (self, reverse=True):
return [x[0] for x in sorted({x: self.__dataBlocks[x]["timestamp"] for x in self.__dataBlocks.keys()}.items(), key=lambda kv[1], reverse)]
#===========================================================================
# Public methods.
......@@ -91,7 +151,6 @@ class DataManagerImpl :
def __init__(self, logger):
self.__dataBlocks = dict()
self.__openedFiles = dict()
self.__logger = logger
#===========================================================================
......@@ -125,11 +184,6 @@ class DataManagerImpl :
def getDataBlocks(self):
return self.__dataBlocks
#
# Return the keys of the dataBlocks dictionary as if it was ordered by timestamp ( asc or desc)
#
def getKeysOrderedByTimestamp (self):
return [x[0] for x in sorted({x: self.__dataBlocks[x]["timestamp"] for x in self.__dataBlocks.keys()}.items(), key=lambda kv[1])]
#===========================================================================
# Getters on a DataBlock identified by its relFITSFilePath ( i.e. the dictionary keys in __dataBlocks )
......@@ -180,96 +234,34 @@ class DataManagerImpl :
def getPixelValueAtiFreqiRAiDEC(self, relFITSFilePath, iFreq, iRA, iDEC):
self.__logger.debug("getPixelValueAtiFreqiRAiDEC : entering")
try :
data = self.__data[relFITSFilePath]
shape=data.shape
if (len(shape) == 3 ) and (0 <= iFreq < shape[0]) and ( 0 <= iRA < shape[1]) and (0 <= iDEC < shape[2]) :
result = {"status": True, "message": "", "result": self.__getPixelValueAtiFreqiRAiDEC(relFITSFilePath, iFreq, iRA, iDEC)}
else:
result = {"status": False, "message" : "Invalid coordinates '{0}, {1}, {2}' or data with inappropriate shape '{3}' ".format(iFreq, iRA, iDEC, shape), "result": None}
except Exception as e:
result = {"status": False, "message" : "{0} - {1}".format(type(e), e.args), "result": None}
result = self.__checkPresence(relFITSFilePath)
if result["status"]:
result=self.__dataBlocks[relFITSFilePath].getPixelValueAtiFreqiRAiDEC(iFreq, iRA, iDEC)
self.__logger.debug("getPixelValueAtiFreqiRAiDEC : exiting")
return result
def getAverageSpectrum(self, relFITSFilePath, startY=None, endY=None, startX=None, endX=None, retFITS=False):
self.__logger.debug( "Getting frequency average.")
startZ = 0
endZ = self.__data[relFITSFilePath].shape[0]
if startX == None:
startX = 0
if endX == None:
endX = self.__data[relFITSFilePath].shape[2]
if startY == None:
startY = 0
if endY == None:
endY = self.__data[relFITSFilePath].shape[1]
self.__logger.debug("%d %d %d %d" % (startX, endX, startY, endY))
self.__logger.debug("%s %s %s %s" % (type(startX), type(endX), type(startY), type(endY)))
pi180 = math.pi / 180 / 4.86e-6
averageSpectrum = None
with_dask=True
if with_dask:
averageSpectrum = (dask.array.nansum(self.__data[relFITSFilePath][:, startY:endY, startX:endX], (1,2)) / self.__convert[relFITSFilePath] * self.__cdelt[relFITSFilePath]).compute().tolist()
else:
averageSpectrum = np.nansum(self.__data[relFITSFilePath][:, startY:endY, startX:endX], (1,2)) / self.__convert[relFITSFilePath] * self.__cdelt[relFITSFilePath]
if retFITS :
crds = self.__getFreqsAtiRAiDEC(relFITSFilePath, startX, startY)
averageSpectrumFits = self.__createFITSSpectrumFromData(relFITSFilePath, crds,self.__header[relFITSFilePath]["CUNIT3"], averageSpectrum, "Jy")
result = {"averageSpectrum": averageSpectrum, "averageSpectrumFits": averageSpectrumFits}
else :
result = {"averageSpectrum": averageSpectrum, "averageSpectrumFits": None}
return {"status": True, "message": "", "result": result} #.tolist()}
def __getSumOverSliceRectArea_0(self, relFITSFilePath, sliceIndex, RAPix0=None, RAPix1=None, DECPix0=None, DECPix1=None):
self.__logger.debug("__getSumOnSliceRectArea_0 : entering")
with_dask=True
if with_dask:
x = (dask.array.nansum(self.__data[relFITSFilePath][sliceIndex, DECPix0:DECPix1, RAPix0:RAPix1]) / self.__convert[relFITSFilePath] * self.__cdelt[relFITSFilePath]).compute().tolist()
else:
x = np.nansum(self.__data[relFITSFilePath][sliceIndex, DECPix0:DECPix1, RAPix0:RAPix1]) / self.__convert[relFITSFilePath] * self.__cdelt[relFITSFilePath]
self.__logger.debug("__getSumOnSliceRectArea__0 : exiting")
return {"status": True, "message": "", "result": x}
result = self.__checkPresence(relFITSFilePath)
if result["status"]:
result = self.__data[relFITSFilePath].getAverageSpectrum(startY, endY, startX, sliceIndex, retFITS)
return {"status": True, "message": "", "result": result}
def getSumOverSliceRectArea(self, relFITSFilePath, sliceIndex, RAPix0=None, RAPix1=None, DECPix0=None, DECPix1=None):
self.__logger.debug("getSumOnSliceRectArea : entering")
if (len(self.__data[relFITSFilePath].shape) == 2 and sliceIndex >= 1) or (len(self.__data[relFITSFilePath].shape) == 3 and sliceIndex >= self.__data[relFITSFilePath].shape[0]):
result = {"status": False, "message": "Invalid slice index '%d'." % (sliceIndex), "result": None }
else:
result = self.__getSumOverSliceRectArea_0( relFITSFilePath, sliceIndex, RAPix0, RAPix1, DECPix0, DECPix1)
result = self.__checkPresence(relFITSFilePath)
if result["status"]:
result=self.__dataBlocks[relFITSFilePath].getPixelValueAtiFreqiRAiDEC(iFreq, iRA, iDEC)
self.__logger.debug("getSumOnSliceRectArea : exiting")
return result
def getAverage(self, relFITSFilePath, step, startZ=None, endZ=None, startY=None, endY=None, startX=None, endX=None, retFITS=False):
self.__logger.debug("getAverage : entering")
result = self.__getAverage_0(relFITSFilePath, step, startZ, endZ, startY, endY, startX, endX)
result = self.__checkPresence(relFITSFilePath)
if result["status"]:
result = self.__dataBlocks[relFITSFilePath].getAverage(step, startZ, endZ, startY, endY, startX, endX, retFITS)
self.__logger.debug("getAverage : exiting")
return {"status": True, "message": "", "result": result}
def __squareSliceData (self, relFITSFilePath, sliceData):
sliceShape = sliceData.shape
if sliceShape[0] == sliceShape[1] :
squaredSliceData = sliceData
else:
tmparr = [sliceData, self.__sliceMargin[relFITSFilePath]]
if sliceShape[0] > sliceShape[1] :
axis = 1
else :
axis = 0
squaredSliceData = da.concatenate(tmparr, axis=axis)
return squaredSliceData
return result
def getOneSliceAsPNG (self, sliceIndex, relFITSFilePath, transformation_name=__default_transformation_name, palette_name=__default_palette_name, video_mode_name=__default_video_mode_name):
self.__logger.debug("getOneSliceAsPNG : entering.")
......@@ -278,7 +270,7 @@ class DataManagerImpl :
if result["status"]:
result = __dataBlocks[relFITSFilePath].getOneSliceAsPNG(sliceIndex, transformation_name, palette_name, video_mode_name)
self.__logger.debug("getOneSliceAsPNG : exiting.")
return {"status": True, "message": "", "result": result}
return result
def getSummedSliceRangeAsPNG( self, sliceIndex0, sliceIndex1, relFITSFilePath, transformation_name=__default_transformation_name, palette_name=__default_palette_name, video_mode_name=__default_video_mode_name):
self.__logger.debug("getSummedSliceRangeAsPNG : entering.")
......@@ -316,47 +308,6 @@ class DataManagerImpl :
return self.__getEntries_0(relKey)
self.__logger.debug("getEntries: exiting")
def __convert_size(self, size_bytes):
if size_bytes == 0:
return "0B"
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
i = int(math.floor(math.log(size_bytes, 1024)))
p = math.pow(1024, i)
s = round(size_bytes / p, 2)
return "%s %s" % (s, size_name[i])
def __getEntries_0(self, relKey):
self.__logger.debug("__getEntries_0: entering")
try :
absFITSFilePrefix = FITSFilePrefix + relKey
entries = (os.listdir(absFITSFilePrefix))
self.__logger.debug("type of entries %s" % (type(entries)))
sortedEntries = entries.sort()
children = []
for entry in entries:
p = absFITSFilePrefix + '/' + entry
if entry.startswith(".") or (not os.path.isdir(p) and not p.endswith(".fits")) :
continue
elif entry in ["log", "NOFITS", "IGNORE"]:
continue
else:
d = dict()
d["key"] = relKey + '/' + entry
d["folder"] = os.path.isdir(p)
d["lazy"] = d["folder"]
if entry.endswith(".fits") :
size = self.__convert_size(os.path.getsize(p))
d["title"] = "<a href = 'visit/?path=%s/%s' target = '_blank'>%s %s</a>" % (relKey, entry, entry, size)
else:
d["title"] = entry
children.append(d)
result = {"status": True, "message": "", "result": json.dumps(children)}
except Exception as e :
result = {"status": False, "message": "Problem while looking for entries under '%s'. Error message was '%s'" % ( relKey, e)}
self.__logger.debug("%r" % result)
self.__logger.debug("__getEntries_0: exiting")
return result
#
# Returns the capabilities of the server relative to the way the image pixels are calculated.
#
......@@ -375,7 +326,14 @@ class DataManagerImpl :
#
# Return the list of defined DataBlock s sorted ( asc or desc ) by timestamp
#
def getDataBlockInfos(self, reverse=True):
keys = self.__getKeysOrderedByTimestamp(reverse)
result = [
{"path": key,
"lastAccess": datetime.frometimestamp(self.__dataBlocks[key].getTimestamp()).strftime("%m/%d/%Y, %H:%M:%S")},
"size": self.__convert_size(os.path.getsize(FITSFilePrefix + "/" + key))
]
return {"status": True, message: "", "result": result}
#
#
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment