Commit 704f0e4a authored by Caillat Michel's avatar Caillat Michel
Browse files

Added statistics calculation and management

parent edd05a15
......@@ -19,6 +19,7 @@ import png
from PIL import Image, PngImagePlugin
import dask
import dask.array as da
import dask.delayed as dd
import traceback
from datetime import datetime, timedelta
......@@ -186,6 +187,8 @@ class DataBlock:
"idle": None
}
self.__statistics ={}
#===========================================================================
#
# Properties
......@@ -463,22 +466,24 @@ class DataBlock:
cdelt = 0.0
self.__logger.debug("slice sum begin")
with_dask = True
if with_dask:
if self.__header["NAXIS"] > 2:
res = (dask.array.nansum(self.__data[iFREQ0:iFREQ1, iDEC0:iDEC1, iRA0:iRA1], 0)*cdelt).compute()
else:
res = (dask.array.nansum(self.__data[iDEC0:iDEC1, iRA0:iRA1], 0)*cdelt).compute()
self.__collectStatistics("%d-%d"%(iFREQ0, iFREQ1), res.compute())
else:
if self.__header["NAXIS"] > 2:
res = np.nansum(self.__data[iFREQ0:iFREQ1, iDEC0:iDEC1, iRA0:iRA1], 0)*cdelt
else:
res = np.nansum(self.__data[iFREQ0:iFREQ1, iDEC0:iDEC1, iRA0:iRA1], 0)*cdelt
self.__collectStatistics("%d-%d"%(iFREQ0, iFREQ1), res)
self.__logger.debug("slice sum end")
self.__logger.debug("__getAverage_0: exiting")
self.__logger.debug(f"res={res}")
self.__logger.debug("__getAverage_0: exiting")
return res
def __getPercentile(self, a, percent):
......@@ -699,6 +704,30 @@ class DataBlock:
def getDimensions(self):
return {"status": True, "message": "", "result": self.__data.shape}
def __collectStatistics(self, key, data):
self.__logger.debug("__collectStatistics : entering")
if key in self.__statistics:
self.__logger.debug("Statistics are already collected")
else:
self.__logger.debug("Statistics have to be collected")
results = []
t = dd(np.nanmin)(data); results.append(t)
t = dd(np.nanmax)(data); results.append(t)
t = dd(np.nanmean)(data); results.append(t)
t = dd(np.nanstd)(data); results.append(t)
t = dd(np.histogram)(data[~np.isnan(data)],bins=100 ); results.append(t)
results = dask.compute(*results)
self.__statistics[key]={}
self.__statistics[key]["min"] = results[0].item()
self.__statistics[key]["max"] = results[1].item()
self.__statistics[key]["mean"] = results[2].item()
self.__statistics[key]["std"] = results[3].item()
self.__statistics[key]["histogram"] = [x.tolist() for x in results[4]]
self.__logger.debug(self.__statistics[key]);
self.__logger.debug("__collectStatistics : exiting")
def __getSlice(self, iFREQ, step=1):
self.__logger.debug("__getSlice: entering")
......@@ -711,8 +740,12 @@ class DataBlock:
if iFREQ == None:
iFREQ = self.__data.shape[0] / 2
x = self.__data[iFREQ, 0:self.__data.shape[1]:step, 0:self.__data.shape[2]:step]
self.__logger.debug("__getSlice: exiting")
#
# Collect some statistics if not already done
#
self.__collectStatistics("%d"%iFREQ, x.compute())
self.__logger.debug("__getSlice: exiting")
return x
def getSlice(self, iFREQ, step=1):
......@@ -737,7 +770,9 @@ class DataBlock:
self.__logger.debug("Apres isnan")
result["status"]=True
result["message"]=""
result["result"]=x.tolist()
result["result"]={}
result["result"]["slice"]=x.tolist()
result["result"]["statistics"]=self.__statistics[iFREQ]
except Exception as e:
result["message"]=f"An exception occurred with message {e}"
result["status"]= False
......@@ -835,7 +870,7 @@ class DataBlock:
status = None
sliceData = None
data = self.__data #self.__HDUList[0].data
data = self.__data
shape = data.shape
self.__logger.debug(shape)
......@@ -844,12 +879,7 @@ class DataBlock:
elif ( len(shape) > 4 ):
self.__logger.debug("Can't process data with more than 4 dimensions")
else:
if len(shape) == 2:
sliceData = data
elif len(shape) == 3:
sliceData = data[iFREQ,:,:]
elif len(shape) == 4:
sliceData = data[0, iFREQ,:,:]
sliceData = self.__getSlice(iFREQ)
relPNGFilePath = ("%s/%d.%s.%s.%s.png" % (relPNGFileDir, iFREQ, ittName, lutName, vmName))
absPNGFilePath = DataBlock.getPNGFilePrefix() + "/" + relPNGFilePath
......@@ -898,7 +928,7 @@ class DataBlock:
status = True
self.__logger.debug("Using cached file '%s'" % absPNGFilePath)
result = {"data_steps": data_steps, "path_to_png": relPNGFilePath}
result = {"data_steps": data_steps, "path_to_png": relPNGFilePath, "statistics": self.__statistics["%d"%iFREQ]}
self.__logger.debug("getOneSliceAsPNG: exiting.")
return {"status": True, "message": "", "result": result}
......@@ -959,7 +989,7 @@ class DataBlock:
status = True
self.__logger.debug("Using cached file '%s'" % absPNGFilePath)
result = {"data_steps": data_steps, "path_to_png": relPNGFilePath}
result = {"data_steps": data_steps, "path_to_png": relPNGFilePath, "statistics": self.__statistics["%d-%d"%(iFREQ0, iFREQ1)]}
self.__logger.debug("getSummedSliceRangeAsPNG: exiting.")
return {"status": True, "message": "", "result": result}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment