Commit 00c62a0d authored by Caillat Michel's avatar Caillat Michel
Browse files

Implemented the concept of idle duration and the purge mechanism based on the idle duration

parent 8e657ea4
......@@ -21,7 +21,7 @@ import dask
import dask.array as da
import traceback
from datetime import datetime
from datetime import datetime, timedelta
import os
import re
from functools import reduce
......@@ -64,9 +64,16 @@ class DataBlock:
"default_vmode_index": 0
}
# date time format
__dateTimeFormat = "%m/%d/%Y - %H:%M:%S"
# A regulare expression to catch the NAXISn
__naxisn_rexp = re.compile("NAXIS[1-9]")
# Maximum idleness duration for an instance of DataBlock
# retrieved from the env var YAFITSS_MAXIDLE
__max_idle = float(os.environ["YAFITSS_MAXIDLE"])
#===========================================================================
# Class methods - Setters and getters
@classmethod
......@@ -113,7 +120,15 @@ class DataBlock:
@classmethod
def getDataBlockInfoNames(cls):
return ["path", "creationTime", "lastAccess", "naxisn", "size"]
return ["path", "creationTime", "lastAccess", "naxisn", "size", "idle"]
@classmethod
def getMaxIdle(cls):
return DataBlock.__max_idle
@classmethod
def getDateTimeFormat(cls):
return DataBlock.__dateTimeFormat
#===========================================================================
# CTOR
......@@ -137,10 +152,11 @@ class DataBlock:
self.__infod = {
"path": lambda: self.__relFITSFilePath,
"creationTime": lambda: self.__creationTime.strftime("%m/%d/%Y - %H:%M:%S"),
"lastAccess": lambda: self.__lastAccess.strftime("%m/%d/%Y - %H:%M:%S"),
"creationTime": lambda: self.__creationTime.strftime(DataBlock.getDateTimeFormat()),
"lastAccess": lambda: self.__lastAccess.strftime(DataBlock.getDateTimeFormat()),
"naxisn": lambda: self.__naxisn,
"size": lambda: DataBlock.convert_size(self.__sizeInBytes)
"size": lambda: DataBlock.convert_size(self.__sizeInBytes),
"idle": None
}
#===========================================================================
......@@ -935,12 +951,15 @@ class DataBlock:
self.__logger.debug("rangeToDMS: exiting")
return {"status": True, "message": "", "result": result}
def getDataBlockInfos(self, infoNames):
def getDataBlockInfos(self, now, infoNames):
self.__logger.debug("getDataBlockInfos: entering")
result = []
for info in infoNames:
if info in self.__infod:
result.append(self.__infod[info]())
if info == 'idle' :
result.append(int((now-self.__lastAccess).total_seconds()))
else:
result.append(self.__infod[info]())
else:
result.append("_undefined_")
self.__logger.debug("getDataBlockInfos: exiting")
......
......@@ -23,6 +23,7 @@ from PIL import Image, PngImagePlugin
import math
import dask
import dask.array as da
from datetime import datetime
import traceback
......@@ -155,11 +156,13 @@ class DataManagerImpl :
# In any case return the header of the FITS data as a dictionary.
#
def setData(self, relFITSFilePath):
self.__logger.debug("setData : entering")
self.__logger.debug("setData : entering");
result = self.__checkPresence(relFITSFilePath)
if result["status"]:
self.__dataBlocks[relFITSFilePath].setLastAccess()
result = self.__dataBlocks[relFITSFilePath].getHeader()
else :
self.purgeDataBlocks()
db = DataBlock(self.__logger)
result = db.setData(relFITSFilePath)
if (result["status"]):
......@@ -343,6 +346,10 @@ class DataManagerImpl :
self.__logger.debug("getDataBlockInfos : entering")
result = dict()
#
# report time
#
result["when"] = datetime.now().strftime(DataBlock.getDateTimeFormat())
# Machine name
#
hostname = socket.gethostname()
......@@ -368,18 +375,35 @@ class DataManagerImpl :
result["memory"] = memoryInfos
# Maximum idleness duration
result["maxidle"] = DataBlock.getMaxIdle()
#
# DataBlocks informations
#
infoNames = DataBlock.getDataBlockInfoNames()
x = [infoNames]
now = datetime.now()
for k in self.__dataBlocks:
x.append(self.__dataBlocks[k].getDataBlockInfos(infoNames))
x.append(self.__dataBlocks[k].getDataBlockInfos(now, infoNames))
self.__logger.debug("getDataBlockInfos : exiting")
result["dataBlocks"] = x
self.__logger.debug("getDataBlockInfos : exiting")
return {"status": True, "message": "", "result": result}
#
# Delete the DataBlocks having their "idle" property
# greater than DataBlock.getMaxIdle()
#
def purgeDataBlocks(self):
self.__logger.debug("purgeDataBlocks : entering")
tobePurged = [db for db in self.__dataBlocks if (datetime.now() - self.__dataBlocks[db].lastAccess).total_seconds() > DataBlock.getMaxIdle()]
for db in tobePurged:
del self.__dataBlocks[db]
self.__logger.debug("purgeDataBlocks : exiting")
return {"status": True, "message": f"{len(tobePurged)} data set(s) erased.", "result": tobePurged}
#
#
# End of the DataManagerImpl class
......
......@@ -550,6 +550,18 @@ def getDataBlockInfos():
logger.debug("getDataBlockInfos wrapper : exiting")
return json.dumps(result)
"""
Erase the DataBlocks idle for a duration greater than DataBlock.getMaxIdle
(env var YAFITSS_MAXIDLE)
"""
@route( baseUrl+'/purgeDataBlocks', name='purgeDataBlocks', method='POST')
@enable_cors
def purgeDataBlocks():
logger.debug("purgeDataBlocks wrapper : entering")
result = dm.purgeDataBlocks()
logger.debug("purgeDataBlocks wrapper : exiting")
return json.dumps(result)
#
# End of the class
#
......@@ -610,7 +622,6 @@ def main(argv):
dm = dataManager.DataManagerImpl(logger)
app = default_app()
run(host=hostname, port=port, debug=True, quiet=False)
return
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment