dataManager_michel.py 17.3 KB
Newer Older
Caillat Michel's avatar
Caillat Michel committed
1
2
3
#!/usr/bin/env python

import os
4
5
6
import subprocess
import socket
import psutil
Caillat Michel's avatar
Caillat Michel committed
7
8
import json
import math
9
10
11
12
13
import numpy as np
import yt
from spectral_cube import SpectralCube
from astropy.io import fits
from astropy.stats import sigma_clip
14
from datetime import datetime
Caillat Michel's avatar
Caillat Michel committed
15

16
17
from DataBlock import DataBlock

18
FITSFilePrefix = os.getenv("YAFITS_FITSDIR")
19
20
21
# Ensure that FITSFilePrefix ends with a '/'
if not FITSFilePrefix.endswith("/"):
    FITSFilePrefix += "/"
22
23

DataRoot = "/home/partemix/dataroot"
24
PNGFilePrefix = DataRoot + '/PNG/'
25
OBJFilePrefix = DataRoot + '/OBJ/'
26
ApiDocPrefix = DataRoot + '/apidoc/'
Caillat Michel's avatar
Caillat Michel committed
27

28
29
30
31
32
33
#
#
# Tell DataBlock where the files will be located
#
# The FITS files
DataBlock.setFITSFilePrefix(FITSFilePrefix)
Caillat Michel's avatar
Caillat Michel committed
34

35
36
# The PNG files
DataBlock.setPNGFilePrefix(PNGFilePrefix)
Caillat Michel's avatar
Caillat Michel committed
37

38
39
40
# The PNG files
DataBlock.setOBJFilePrefix(OBJFilePrefix)

Caillat Michel's avatar
Caillat Michel committed
41
42
43
44
#
# An implementation of the class used in the serverWsgi-ish server.
#
class DataManagerImpl :
45
    __default_palette_name, __default_transformation_name, __default_video_mode_name = DataBlock.getDefaults()
Caillat Michel's avatar
Caillat Michel committed
46

47
48
    #===========================================================================
    # Private methods
Caillat Michel's avatar
Caillat Michel committed
49
50
    #

51
52
    # check if the FITS data identified by the relFITSFilePath are present in
    # memory. If yes update the corresponding DataBlock's timestamp.
Caillat Michel's avatar
Caillat Michel committed
53

54
55
56
57
58
59
60
61
    def __checkPresence(self, relFITSFilePath):
        self.__logger.debug("__checkPresence : entering")
        result = {"status": True, "message" : '', "result": None}
        if relFITSFilePath in self.__dataBlocks:
            self.__dataBlocks[relFITSFilePath].setLastAccess()
        else:
            result = {"status": False, "message": f'FITS file "{relFITSFilePath}" is not present in memory. Call "setData" first'}
        self.__logger.debug("__checkPresence : exiting")
Caillat Michel's avatar
Caillat Michel committed
62
63
        return result

64
65
66
67
68
    def __getEntries_0(self, relKey):
        self.__logger.debug("__getEntries_0: entering")
        try :
            absFITSFilePrefix = FITSFilePrefix + relKey
            entries = (os.listdir(absFITSFilePrefix))
69
            self.__logger.debug("%r" % entries)
70
71
72
73
            sortedEntries = entries.sort()
            children = []
            for entry in entries:
                p = absFITSFilePrefix + '/' + entry
74
75
76
77
78
79
80
81
82
83
                condition = (os.path.isfile(p) and p.endswith(".fits"))
                if not condition :
                    self.__logger.debug("%s link %r" % (p, os.path.islink(p)))
                    if os.path.islink(p):
                        target = os.path.realpath(p)
                        self.__logger.debug("%s is dir %r" %(target, os.path.isdir(target)))
                        condition  = os.path.islink(p) # and os.path.isdir(target)
                    else :
                        condition = (os.path.isdir(p))

84
                if not condition :
85
86
87
88
89
90
                    continue
                elif entry in ["log", "NOFITS", "IGNORE"]:
                    continue
                else:
                    d = dict()
                    d["key"]    = relKey + '/' + entry
91
                    d["folder"] = not os.path.isfile(p)
92
93
94
                    d["lazy"]   = d["folder"]
                    if entry.endswith(".fits") :
                        size = DataBlock.convert_size(os.path.getsize(p))
95
                        d["title"] = "<a href = 'visit/?relFITSFilePath=%s/%s' target = '_blank'>%s %s</a>" % (relKey, entry, entry, size)
96
97
98
99
100
101
102
                    else:
                        d["title"] = entry
                    children.append(d)
            result = {"status": True, "message": "", "result": json.dumps(children)}
        except Exception as e :
            result = {"status": False, "message": "Problem while looking for entries under '%s'. Error message was '%s'" % ( relKey, e)}
            self.__logger.debug("%r" % result)
103
        self.__logger.debug("%r" % result)
104
        self.__logger.debug("__getEntries_0: exiting")
Caillat Michel's avatar
Caillat Michel committed
105
106
        return result

107
108
109
110
111
    #
    # Return the keys of the dataBlocks dictionary as if it was ordered by timestamp ( asc or desc)
    #
    def __getKeysOrderedByTimestamp (self, reverse=True):
        return [x[0] for x in sorted({x: self.__dataBlocks[x]["timestamp"] for x in self.__dataBlocks.keys()}.items(), key=lambda kv: kv[1], reverse=reverse)]
Caillat Michel's avatar
Caillat Michel committed
112

113
114
    #===========================================================================
    # Public methods.
Caillat Michel's avatar
Caillat Michel committed
115

116
117
118
119
    #===========================================================================
    # CTOR
    #
    def __init__(self, logger):
Caillat Michel's avatar
Caillat Michel committed
120

121
122
123
        self.__dataBlocks = dict()
        self.__logger = logger
        self.__logger.debug(f"An instance of DataManagerImpl is created")
Caillat Michel's avatar
Caillat Michel committed
124

125
126
127
128
129
130
131
132
133
134
135
136
    #===========================================================================
    # THE data selector.
    # Its execution will trigger the creation of a DataBlock populated by
    # the content of a FITS file if it's not already loaded.
    # In any case return the header of the FITS data as a dictionary.
    #
    def setData(self, relFITSFilePath):
        self.__logger.debug("setData : entering");
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            self.__dataBlocks[relFITSFilePath].setLastAccess()
            result = self.__dataBlocks[relFITSFilePath].getHeader()
Caillat Michel's avatar
Caillat Michel committed
137
        else :
138
139
140
141
142
143
            self.purgeDataBlocks()
            db = DataBlock(self.__logger)
            result = db.setData(relFITSFilePath)
            if (result["status"]):
                self.__dataBlocks[relFITSFilePath]=db
        self.__logger.debug(f"About to return {result}")
Caillat Michel's avatar
Caillat Michel committed
144
145
146
        self.__logger.debug("setData : Exiting")
        return result

147
148
149
150
    #===========================================================================
    # General getters
    #
    #
Caillat Michel's avatar
Caillat Michel committed
151
152


153
154
155
156
157
158
159
160
161
162
    #===========================================================================
    # Getters on a DataBlock identified by its relFITSFilePath ( i.e. the dictionary keys in __dataBlocks )
    #
    def getDimensions(self, relFITSFilePath):
        self.__logger.debug( "getDimensions : entering");
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].getDimensions()
        self.__logger.debug( "getDimensions : exiting");
        return result
Caillat Michel's avatar
Caillat Michel committed
163

Caillat Michel's avatar
Caillat Michel committed
164
    def getSlice(self, relFITSFilePath, iFREQ, step=1 ):
165
        self.__logger.debug("getSlice : entering")
Caillat Michel's avatar
Caillat Michel committed
166
        result = self.__checkPresence(relFITSFilePath)
167
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
168
            result = self.__dataBlocks[relFITSFilePath].getSlice(iFREQ, step)
Caillat Michel's avatar
Caillat Michel committed
169
        self.__logger.debug("getSlice : exiting")
170
        return result
Caillat Michel's avatar
Caillat Michel committed
171

172
173
    def getSpectrum(self, relFITSFilePath, iRA=None, iDEC=None, iFREQ0=None, iFREQ1=None):
        self.__logger.debug( "getSpectrum : entering")
174
175
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
176
177
            result = self.__dataBlocks[relFITSFilePath].getSpectrum(iRA, iDEC, iFREQ0, iFREQ1)
        self.__logger.debug( "getSpectrum : exiting")
178
        return result
Caillat Michel's avatar
Caillat Michel committed
179

Caillat Michel's avatar
Caillat Michel committed
180
    def getAverageSpectrum(self, relFITSFilePath, iDEC0=None, iDEC1=None, iRA0=None, iRA1=None, retFITS=False):
181
182
183
        self.__logger.debug("getAverageSpectrum : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
184
            result = self.__dataBlocks[relFITSFilePath].getAverageSpectrum(iDEC0, iDEC1, iRA0, iRA1, retFITS)
185
186
        self.__logger.debug("getAverageSpectrum : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
187
188


189
190
191
192
193
194
195
    def getHeader(self, relFITSFilePath):
        self.__logger.debug("getHeader : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].getHeader()
        self.__logger.debug("getHeader : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
196

197
198
199
200
201
202
203
    def RADECRangeInDegrees( self, relFITSFilePath):
        self.__logger.debug("RADECRangeInDegrees : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].RADECRangeInDegrees()
        self.__logger.debug("RADECRangeInDegrees : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
204

205
    def degToHMSDMS(self, relFITSFilePath, RAinDD, DECinDD):
206
207
208
        self.__logger.debug("degToHMSDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
209
            result = self.__dataBlocks[relFITSFilePath].degToHMSDMS(RAinDD, DECinDD)
Caillat Michel's avatar
Caillat Michel committed
210
211
        self.__logger.debug("degToHMSDMS : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
212

213
    def rangeToHMS(self,  relFITSFilePath, iRA0, iRA1, iRAstep):
214
215
216
        self.__logger.debug("degToHMSDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
217
            result=self.__dataBlocks[relFITSFilePath].rangeToHMS(iRA0, iRA1, iRAstep)
218
        self.__logger.debug("degToHMSDMS : exiting")
Caillat Michel's avatar
Caillat Michel committed
219
        return result
Caillat Michel's avatar
Caillat Michel committed
220

221
    def rangeToDMS(self,  relFITSFilePath, iDEC0, iDEC1, iDECstep):
222
223
224
        self.__logger.debug("rangeToDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
225
            result = self.__dataBlocks[relFITSFilePath].rangeToDMS(iDEC0, iDEC1, iDECstep)
226
        self.__logger.debug("rangeToDMS : exiting")
Caillat Michel's avatar
Caillat Michel committed
227
        return result
Caillat Michel's avatar
Caillat Michel committed
228
229
230

    def getPixelValueAtiFreqiRAiDEC(self, relFITSFilePath, iFreq, iRA, iDEC):
        self.__logger.debug("getPixelValueAtiFreqiRAiDEC : entering")
231
232
233
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result=self.__dataBlocks[relFITSFilePath].getPixelValueAtiFreqiRAiDEC(iFreq, iRA, iDEC)
Caillat Michel's avatar
Caillat Michel committed
234
235
236
        self.__logger.debug("getPixelValueAtiFreqiRAiDEC : exiting")
        return result

237
    def getSumOverSliceRectArea(self, relFITSFilePath, iFREQ, iRA0=None, iRA1=None, iDEC0=None, iDEC1=None):
Caillat Michel's avatar
Caillat Michel committed
238
        self.__logger.debug("getSumOnSliceRectArea : entering")
239
240
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
241
            result=self.__dataBlocks[relFITSFilePath].getSumOverSliceRectArea(iFREQ, iRA0, iRA1, iDEC0, iDEC1)
Caillat Michel's avatar
Caillat Michel committed
242
243
244
        self.__logger.debug("getSumOnSliceRectArea : exiting")
        return result

Caillat Michel's avatar
Caillat Michel committed
245
    def getAverage(self, relFITSFilePath, iFREQ0, iFREQ1, iDEC0, iDEC1, iRA0, iRA1, retFITS):
Caillat Michel's avatar
Caillat Michel committed
246
        self.__logger.debug("getAverage : entering")
247
248
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
249
            result = self.__dataBlocks[relFITSFilePath].getAverage(iFREQ0, iFREQ1, iDEC0, iDEC1, iRA0, iRA1, retFITS)
Caillat Michel's avatar
Caillat Michel committed
250
251
252
        self.__logger.debug("getAverage : exiting")
        return result

Caillat Michel's avatar
Caillat Michel committed
253
    def getOneSliceAsPNG (self, iFREQ, relFITSFilePath, **kwargs):
Caillat Michel's avatar
Caillat Michel committed
254
        self.__logger.debug("getOneSliceAsPNG : entering.")
Caillat Michel's avatar
Caillat Michel committed
255
        self.__logger.debug("iFREQ = %r, relFITSFilePath = %r" % (iFREQ, relFITSFilePath))
256
257
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
258
            result = self.__dataBlocks[relFITSFilePath].getOneSliceAsPNG(iFREQ, **kwargs)
Caillat Michel's avatar
Caillat Michel committed
259
        self.__logger.debug("getOneSliceAsPNG : exiting.")
260
        return result
Caillat Michel's avatar
Caillat Michel committed
261

262
    def getSummedSliceRangeAsPNG( self, relFITSFilePath, iFREQ0, iFREQ1, **kwargs):
Caillat Michel's avatar
Caillat Michel committed
263
        self.__logger.debug("getSummedSliceRangeAsPNG : entering.")
264
265
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
266
            result = self.__dataBlocks[relFITSFilePath].getSummedSliceRangeAsPNG(iFREQ0, iFREQ1, **kwargs)
Caillat Michel's avatar
Caillat Michel committed
267
268
269
        self.__logger.debug("getSummedSliceRangeAsPNG : exiting.")
        return result

Caillat Michel's avatar
Caillat Michel committed
270
271
272
    def getContours(self, relFITSFilePath, iFREQ, iDEC0, iDEC1, iRA0, iRA1, **kwargs):
        self.__logger.debug("getContours : entering.")
        result = self.__checkPresence(relFITSFilePath)
273
        if result["status"] :
Caillat Michel's avatar
Caillat Michel committed
274
275
276
277
            result = self.__dataBlocks[relFITSFilePath].getContours(iFREQ, iDEC0, iDEC1, iRA0, iRA1, **kwargs)
        self.__logger.debug("getContours : exiting.")
        return result

278
    def measureContour(self, relFITSFilePath, iFREQ, contour, level):
279
280
281
        self.__logger.debug("measureContour - dispatcher : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"] :
282
            result = self.__dataBlocks[relFITSFilePath].measureContour(iFREQ, contour, level)
283
284
285
        self.__logger.debug("measureContour - dispatcher : exiting")
        return result

286
287
288
289
290
291
292
293
    def measureBox(self, relFITSFilePath, iFREQ, iRA0, iRA1, iDEC0, iDEC1):
        self.__logger.debug("measureBox - dispatcher : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"] :
            result = self.__dataBlocks[relFITSFilePath].measureBox(iFREQ, iRA0, iRA1, iDEC0, iDEC1)
        self.__logger.debug("measureBox - dispatcher : exiting")
        return result

294

Caillat Michel's avatar
Caillat Michel committed
295
    #
296
297
298
    # create fits file containing a spectrum at iRA, iDEC
    # The use case is interoperability via SAMP
    #
Caillat Michel's avatar
Caillat Michel committed
299
300
    def createFits(self, relFITSFilePath, iRA, iDEC):
        self.__logger.debug("createFITS : entering")
301
302
303
        result = self.__checkPresence(relFITSFilePath)
        if result :
            result = self.__dataBlocks[relFITSFilePath].createFits(iRA, iDEC)
Caillat Michel's avatar
Caillat Michel committed
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        self.__logger.debug("createFITS : exiting")
        return result

    #
    # Returns all the entries directly located under a given subdirectory.
    # The subdirectory must be passed as a full path implicitely
    # rooted at FITSFilePrefix.
    # i.e. :
    # "/" will be interpreted as FITSFilePrefix+"/"
    # "/2018" will be interpreted as FITSFilePrefix+"/2018"
    # etc...
    # The result is a JSON array of dictionaries , on the basis of one dictionary per inode:
    # [{'title': filename of the entry, 'key': the FITSFilePrefix rooted path of the entry, 'folder' : true (resp. false) if the inode is (resp. is not) a subdir, 'lazy': true}]
    def getEntries(self, relKey):
        self.__logger.debug("getEntries: entering")
        return self.__getEntries_0(relKey)
        self.__logger.debug("getEntries: exiting")

    #
    # Returns the capabilities of the server relative to the way the image pixels are calculated.
    #
    #  Intensity Transformations names are defined in a list itts
    #  Colors names are defined in a list luts
    #  Video modes are defined in a list vmode
    #
    #  For each of this rendering filters default names are defined.
    #
    def renderingCapabilities(self) :
        self.__logger.debug("renderingCapabilities: entering")
333
        result = {"status": True, "message": "", "result": DataBlock.getRenderingCapabilities()}
Caillat Michel's avatar
Caillat Michel committed
334
335
        self.__logger.debug("renderingCapabilities: exiting")
        return result
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401

    #
    # Return the list of defined DataBlock s sorted ( asc or desc ) by timestamp
    #
    def getDataBlockInfos(self, reverse=True):
        self.__logger.debug("getDataBlockInfos : entering")
        result = dict()
        #
        # report time
        #
        result["when"] = datetime.now().strftime(DataBlock.getDateTimeFormat())

        # Machine name
        #
        hostname = socket.gethostname()
        result["hostname"] = hostname

        #
        # Machine informations
        #
        numsockets =  int(subprocess.check_output('cat /proc/cpuinfo | grep "physical id" | sort -u | wc -l', shell=True))
        cpucount = psutil.cpu_count()
        cpuInfos = {"numsockets": numsockets, "cpucount": cpucount}
        result["cpu"] = cpuInfos

        #
        # Physical memory informations
        #
        physMemoryInBytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
        usedMemoryInBytes = sum([self.__dataBlocks[dbId].sizeInBytes for dbId in self.__dataBlocks])
        usedMemoryPercentage = math.floor((usedMemoryInBytes / physMemoryInBytes) * 10000) / 100.
        physMemory = DataBlock.convert_size(physMemoryInBytes)
        usedMemory = DataBlock.convert_size(usedMemoryInBytes)
        memoryInfos = {"physMemory": physMemory, "usedMemory": usedMemory, "usedMemoryPercentage": usedMemoryPercentage}

        result["memory"] = memoryInfos

        # Maximum idleness duration
        result["maxidle"] = DataBlock.getMaxIdle()

        #
        # DataBlocks informations
        #
        infoNames = DataBlock.getDataBlockInfoNames()
        x = [infoNames]
        now = datetime.now()
        for k in self.__dataBlocks:
            x.append(self.__dataBlocks[k].getDataBlockInfos(now, infoNames))
        self.__logger.debug("getDataBlockInfos : exiting")

        result["dataBlocks"] = x

        self.__logger.debug("getDataBlockInfos : exiting")
        return {"status": True, "message": "", "result": result}

    #
    # Delete the DataBlocks having their "idle" property
    # greater than DataBlock.getMaxIdle()
    #
    def purgeDataBlocks(self):
        self.__logger.debug("purgeDataBlocks : entering")
        tobePurged = [db for db in self.__dataBlocks if (datetime.now() - self.__dataBlocks[db].lastAccess).total_seconds() > DataBlock.getMaxIdle()]
        for db in tobePurged:
            del self.__dataBlocks[db]
        self.__logger.debug("purgeDataBlocks : exiting")
        return {"status": True, "message": f"{len(tobePurged)} data set(s) erased.", "result": tobePurged}
Caillat Michel's avatar
Caillat Michel committed
402

403
404
405
    #
    #
    #
406
    def getYtObj(self,relFITSFilePath,product,coord):
ba yaye-awa's avatar
ba yaye-awa committed
407
        self.__logger.debug("getYtObj : entering")
408
        self.__logger.debug("relFITSFilePath = %r, product= %r, (%i,%i,%i,%i,%i,%i)" % (relFITSFilePath,product,coord['iRA0'],coord['iRA1'],coord['iDEC0'],coord['iDEC1'],coord['iFREQ0'],coord['iFREQ1']))
ba yaye-awa's avatar
ba yaye-awa committed
409
410
        result = self.__checkPresence(relFITSFilePath)
        if result :
411
            result = self.__dataBlocks[relFITSFilePath].getYtObj(relFITSFilePath,product,coord)
ba yaye-awa's avatar
ba yaye-awa committed
412
413
        self.__logger.debug("getYtObj : exiting")
        return result
414

Caillat Michel's avatar
Caillat Michel committed
415
416
417
418
419
#
#
#    End of the DataManagerImpl class
#
#