dataManager_michel.py 17.2 KB
Newer Older
Caillat Michel's avatar
Caillat Michel committed
1
2
3
#!/usr/bin/env python

import os
4
5
6
import subprocess
import socket
import psutil
Caillat Michel's avatar
Caillat Michel committed
7
8
import json
import math
9
10
11
12
13
import numpy as np
import yt
from spectral_cube import SpectralCube
from astropy.io import fits
from astropy.stats import sigma_clip
14
from datetime import datetime
Caillat Michel's avatar
Caillat Michel committed
15

16
17
from DataBlock import DataBlock

18
19
20
FITSFilePrefix = os.getenv("YAFITS_FITSDIR")

DataRoot = "/home/partemix/dataroot"
21
PNGFilePrefix = DataRoot + '/PNG/'
22
OBJFilePrefix = DataRoot + '/OBJ/'
23
ApiDocPrefix = DataRoot + '/apidoc/'
Caillat Michel's avatar
Caillat Michel committed
24

25
26
27
28
29
30
#
#
# Tell DataBlock where the files will be located
#
# The FITS files
DataBlock.setFITSFilePrefix(FITSFilePrefix)
Caillat Michel's avatar
Caillat Michel committed
31

32
33
# The PNG files
DataBlock.setPNGFilePrefix(PNGFilePrefix)
Caillat Michel's avatar
Caillat Michel committed
34

35
36
37
# The PNG files
DataBlock.setOBJFilePrefix(OBJFilePrefix)

Caillat Michel's avatar
Caillat Michel committed
38
39
40
41
#
# An implementation of the class used in the serverWsgi-ish server.
#
class DataManagerImpl :
42
    __default_palette_name, __default_transformation_name, __default_video_mode_name = DataBlock.getDefaults()
Caillat Michel's avatar
Caillat Michel committed
43

44
45
    #===========================================================================
    # Private methods
Caillat Michel's avatar
Caillat Michel committed
46
47
    #

48
49
    # check if the FITS data identified by the relFITSFilePath are present in
    # memory. If yes update the corresponding DataBlock's timestamp.
Caillat Michel's avatar
Caillat Michel committed
50

51
52
53
54
55
56
57
58
    def __checkPresence(self, relFITSFilePath):
        self.__logger.debug("__checkPresence : entering")
        result = {"status": True, "message" : '', "result": None}
        if relFITSFilePath in self.__dataBlocks:
            self.__dataBlocks[relFITSFilePath].setLastAccess()
        else:
            result = {"status": False, "message": f'FITS file "{relFITSFilePath}" is not present in memory. Call "setData" first'}
        self.__logger.debug("__checkPresence : exiting")
Caillat Michel's avatar
Caillat Michel committed
59
60
        return result

61
62
63
64
65
    def __getEntries_0(self, relKey):
        self.__logger.debug("__getEntries_0: entering")
        try :
            absFITSFilePrefix = FITSFilePrefix + relKey
            entries = (os.listdir(absFITSFilePrefix))
66
            self.__logger.debug("%r" % entries)
67
68
69
70
            sortedEntries = entries.sort()
            children = []
            for entry in entries:
                p = absFITSFilePrefix + '/' + entry
71
72
73
74
75
76
77
78
79
80
                condition = (os.path.isfile(p) and p.endswith(".fits"))
                if not condition :
                    self.__logger.debug("%s link %r" % (p, os.path.islink(p)))
                    if os.path.islink(p):
                        target = os.path.realpath(p)
                        self.__logger.debug("%s is dir %r" %(target, os.path.isdir(target)))
                        condition  = os.path.islink(p) # and os.path.isdir(target)
                    else :
                        condition = (os.path.isdir(p))

81
                if not condition :
82
83
84
85
86
87
                    continue
                elif entry in ["log", "NOFITS", "IGNORE"]:
                    continue
                else:
                    d = dict()
                    d["key"]    = relKey + '/' + entry
88
                    d["folder"] = not os.path.isfile(p)
89
90
91
                    d["lazy"]   = d["folder"]
                    if entry.endswith(".fits") :
                        size = DataBlock.convert_size(os.path.getsize(p))
92
                        d["title"] = "<a href = 'visit/?relFITSFilePath=%s/%s' target = '_blank'>%s %s</a>" % (relKey, entry, entry, size)
93
94
95
96
97
98
99
                    else:
                        d["title"] = entry
                    children.append(d)
            result = {"status": True, "message": "", "result": json.dumps(children)}
        except Exception as e :
            result = {"status": False, "message": "Problem while looking for entries under '%s'. Error message was '%s'" % ( relKey, e)}
            self.__logger.debug("%r" % result)
100
        self.__logger.debug("%r" % result)
101
        self.__logger.debug("__getEntries_0: exiting")
Caillat Michel's avatar
Caillat Michel committed
102
103
        return result

104
105
106
107
108
    #
    # Return the keys of the dataBlocks dictionary as if it was ordered by timestamp ( asc or desc)
    #
    def __getKeysOrderedByTimestamp (self, reverse=True):
        return [x[0] for x in sorted({x: self.__dataBlocks[x]["timestamp"] for x in self.__dataBlocks.keys()}.items(), key=lambda kv: kv[1], reverse=reverse)]
Caillat Michel's avatar
Caillat Michel committed
109

110
111
    #===========================================================================
    # Public methods.
Caillat Michel's avatar
Caillat Michel committed
112

113
114
115
116
    #===========================================================================
    # CTOR
    #
    def __init__(self, logger):
Caillat Michel's avatar
Caillat Michel committed
117

118
119
120
        self.__dataBlocks = dict()
        self.__logger = logger
        self.__logger.debug(f"An instance of DataManagerImpl is created")
Caillat Michel's avatar
Caillat Michel committed
121

122
123
124
125
126
127
128
129
130
131
132
133
    #===========================================================================
    # THE data selector.
    # Its execution will trigger the creation of a DataBlock populated by
    # the content of a FITS file if it's not already loaded.
    # In any case return the header of the FITS data as a dictionary.
    #
    def setData(self, relFITSFilePath):
        self.__logger.debug("setData : entering");
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            self.__dataBlocks[relFITSFilePath].setLastAccess()
            result = self.__dataBlocks[relFITSFilePath].getHeader()
Caillat Michel's avatar
Caillat Michel committed
134
        else :
135
136
137
138
139
140
            self.purgeDataBlocks()
            db = DataBlock(self.__logger)
            result = db.setData(relFITSFilePath)
            if (result["status"]):
                self.__dataBlocks[relFITSFilePath]=db
        self.__logger.debug(f"About to return {result}")
Caillat Michel's avatar
Caillat Michel committed
141
142
143
        self.__logger.debug("setData : Exiting")
        return result

144
145
146
147
    #===========================================================================
    # General getters
    #
    #
Caillat Michel's avatar
Caillat Michel committed
148
149


150
151
152
153
154
155
156
157
158
159
    #===========================================================================
    # Getters on a DataBlock identified by its relFITSFilePath ( i.e. the dictionary keys in __dataBlocks )
    #
    def getDimensions(self, relFITSFilePath):
        self.__logger.debug( "getDimensions : entering");
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].getDimensions()
        self.__logger.debug( "getDimensions : exiting");
        return result
Caillat Michel's avatar
Caillat Michel committed
160

Caillat Michel's avatar
Caillat Michel committed
161
    def getSlice(self, relFITSFilePath, iFREQ, step=1 ):
162
        self.__logger.debug("getSlice : entering")
Caillat Michel's avatar
Caillat Michel committed
163
        result = self.__checkPresence(relFITSFilePath)
164
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
165
            result = self.__dataBlocks[relFITSFilePath].getSlice(iFREQ, step)
Caillat Michel's avatar
Caillat Michel committed
166
        self.__logger.debug("getSlice : exiting")
167
        return result
Caillat Michel's avatar
Caillat Michel committed
168

169
170
    def getSpectrum(self, relFITSFilePath, iRA=None, iDEC=None, iFREQ0=None, iFREQ1=None):
        self.__logger.debug( "getSpectrum : entering")
171
172
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
173
174
            result = self.__dataBlocks[relFITSFilePath].getSpectrum(iRA, iDEC, iFREQ0, iFREQ1)
        self.__logger.debug( "getSpectrum : exiting")
175
        return result
Caillat Michel's avatar
Caillat Michel committed
176

Caillat Michel's avatar
Caillat Michel committed
177
    def getAverageSpectrum(self, relFITSFilePath, iDEC0=None, iDEC1=None, iRA0=None, iRA1=None, retFITS=False):
178
179
180
        self.__logger.debug("getAverageSpectrum : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
181
            result = self.__dataBlocks[relFITSFilePath].getAverageSpectrum(iDEC0, iDEC1, iRA0, iRA1, retFITS)
182
183
        self.__logger.debug("getAverageSpectrum : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
184
185


186
187
188
189
190
191
192
    def getHeader(self, relFITSFilePath):
        self.__logger.debug("getHeader : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].getHeader()
        self.__logger.debug("getHeader : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
193

194
195
196
197
198
199
200
    def RADECRangeInDegrees( self, relFITSFilePath):
        self.__logger.debug("RADECRangeInDegrees : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].RADECRangeInDegrees()
        self.__logger.debug("RADECRangeInDegrees : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
201

202
    def degToHMSDMS(self, relFITSFilePath, RAinDD, DECinDD):
203
204
205
        self.__logger.debug("degToHMSDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
206
            result = self.__dataBlocks[relFITSFilePath].degToHMSDMS(RAinDD, DECinDD)
Caillat Michel's avatar
Caillat Michel committed
207
208
        self.__logger.debug("degToHMSDMS : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
209

210
    def rangeToHMS(self,  relFITSFilePath, iRA0, iRA1, iRAstep):
211
212
213
        self.__logger.debug("degToHMSDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
214
            result=self.__dataBlocks[relFITSFilePath].rangeToHMS(iRA0, iRA1, iRAstep)
215
        self.__logger.debug("degToHMSDMS : exiting")
Caillat Michel's avatar
Caillat Michel committed
216
        return result
Caillat Michel's avatar
Caillat Michel committed
217

218
    def rangeToDMS(self,  relFITSFilePath, iDEC0, iDEC1, iDECstep):
219
220
221
        self.__logger.debug("rangeToDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
222
            result = self.__dataBlocks[relFITSFilePath].rangeToDMS(iDEC0, iDEC1, iDECstep)
223
        self.__logger.debug("rangeToDMS : exiting")
Caillat Michel's avatar
Caillat Michel committed
224
        return result
Caillat Michel's avatar
Caillat Michel committed
225
226
227

    def getPixelValueAtiFreqiRAiDEC(self, relFITSFilePath, iFreq, iRA, iDEC):
        self.__logger.debug("getPixelValueAtiFreqiRAiDEC : entering")
228
229
230
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result=self.__dataBlocks[relFITSFilePath].getPixelValueAtiFreqiRAiDEC(iFreq, iRA, iDEC)
Caillat Michel's avatar
Caillat Michel committed
231
232
233
        self.__logger.debug("getPixelValueAtiFreqiRAiDEC : exiting")
        return result

234
    def getSumOverSliceRectArea(self, relFITSFilePath, iFREQ, iRA0=None, iRA1=None, iDEC0=None, iDEC1=None):
Caillat Michel's avatar
Caillat Michel committed
235
        self.__logger.debug("getSumOnSliceRectArea : entering")
236
237
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
238
            result=self.__dataBlocks[relFITSFilePath].getSumOverSliceRectArea(iFREQ, iRA0, iRA1, iDEC0, iDEC1)
Caillat Michel's avatar
Caillat Michel committed
239
240
241
        self.__logger.debug("getSumOnSliceRectArea : exiting")
        return result

Caillat Michel's avatar
Caillat Michel committed
242
    def getAverage(self, relFITSFilePath, iFREQ0, iFREQ1, iDEC0, iDEC1, iRA0, iRA1, retFITS):
Caillat Michel's avatar
Caillat Michel committed
243
        self.__logger.debug("getAverage : entering")
244
245
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
246
            result = self.__dataBlocks[relFITSFilePath].getAverage(iFREQ0, iFREQ1, iDEC0, iDEC1, iRA0, iRA1, retFITS)
Caillat Michel's avatar
Caillat Michel committed
247
248
249
        self.__logger.debug("getAverage : exiting")
        return result

Caillat Michel's avatar
Caillat Michel committed
250
    def getOneSliceAsPNG (self, iFREQ, relFITSFilePath, **kwargs):
Caillat Michel's avatar
Caillat Michel committed
251
        self.__logger.debug("getOneSliceAsPNG : entering.")
Caillat Michel's avatar
Caillat Michel committed
252
        self.__logger.debug("iFREQ = %r, relFITSFilePath = %r" % (iFREQ, relFITSFilePath))
253
254
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
255
            result = self.__dataBlocks[relFITSFilePath].getOneSliceAsPNG(iFREQ, **kwargs)
Caillat Michel's avatar
Caillat Michel committed
256
        self.__logger.debug("getOneSliceAsPNG : exiting.")
257
        return result
Caillat Michel's avatar
Caillat Michel committed
258

259
    def getSummedSliceRangeAsPNG( self, relFITSFilePath, iFREQ0, iFREQ1, **kwargs):
Caillat Michel's avatar
Caillat Michel committed
260
        self.__logger.debug("getSummedSliceRangeAsPNG : entering.")
261
262
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
263
            result = self.__dataBlocks[relFITSFilePath].getSummedSliceRangeAsPNG(iFREQ0, iFREQ1, **kwargs)
Caillat Michel's avatar
Caillat Michel committed
264
265
266
        self.__logger.debug("getSummedSliceRangeAsPNG : exiting.")
        return result

Caillat Michel's avatar
Caillat Michel committed
267
268
269
    def getContours(self, relFITSFilePath, iFREQ, iDEC0, iDEC1, iRA0, iRA1, **kwargs):
        self.__logger.debug("getContours : entering.")
        result = self.__checkPresence(relFITSFilePath)
270
        if result["status"] :
Caillat Michel's avatar
Caillat Michel committed
271
272
273
274
            result = self.__dataBlocks[relFITSFilePath].getContours(iFREQ, iDEC0, iDEC1, iRA0, iRA1, **kwargs)
        self.__logger.debug("getContours : exiting.")
        return result

275
    def measureContour(self, relFITSFilePath, iFREQ, contour, level):
276
277
278
        self.__logger.debug("measureContour - dispatcher : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"] :
279
            result = self.__dataBlocks[relFITSFilePath].measureContour(iFREQ, contour, level)
280
281
282
        self.__logger.debug("measureContour - dispatcher : exiting")
        return result

283
284
285
286
287
288
289
290
    def measureBox(self, relFITSFilePath, iFREQ, iRA0, iRA1, iDEC0, iDEC1):
        self.__logger.debug("measureBox - dispatcher : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"] :
            result = self.__dataBlocks[relFITSFilePath].measureBox(iFREQ, iRA0, iRA1, iDEC0, iDEC1)
        self.__logger.debug("measureBox - dispatcher : exiting")
        return result

291

Caillat Michel's avatar
Caillat Michel committed
292
    #
293
294
295
    # create fits file containing a spectrum at iRA, iDEC
    # The use case is interoperability via SAMP
    #
Caillat Michel's avatar
Caillat Michel committed
296
297
    def createFits(self, relFITSFilePath, iRA, iDEC):
        self.__logger.debug("createFITS : entering")
298
299
300
        result = self.__checkPresence(relFITSFilePath)
        if result :
            result = self.__dataBlocks[relFITSFilePath].createFits(iRA, iDEC)
Caillat Michel's avatar
Caillat Michel committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
        self.__logger.debug("createFITS : exiting")
        return result

    #
    # Returns all the entries directly located under a given subdirectory.
    # The subdirectory must be passed as a full path implicitely
    # rooted at FITSFilePrefix.
    # i.e. :
    # "/" will be interpreted as FITSFilePrefix+"/"
    # "/2018" will be interpreted as FITSFilePrefix+"/2018"
    # etc...
    # The result is a JSON array of dictionaries , on the basis of one dictionary per inode:
    # [{'title': filename of the entry, 'key': the FITSFilePrefix rooted path of the entry, 'folder' : true (resp. false) if the inode is (resp. is not) a subdir, 'lazy': true}]
    def getEntries(self, relKey):
        self.__logger.debug("getEntries: entering")
        return self.__getEntries_0(relKey)
        self.__logger.debug("getEntries: exiting")

    #
    # Returns the capabilities of the server relative to the way the image pixels are calculated.
    #
    #  Intensity Transformations names are defined in a list itts
    #  Colors names are defined in a list luts
    #  Video modes are defined in a list vmode
    #
    #  For each of this rendering filters default names are defined.
    #
    def renderingCapabilities(self) :
        self.__logger.debug("renderingCapabilities: entering")
330
        result = {"status": True, "message": "", "result": DataBlock.getRenderingCapabilities()}
Caillat Michel's avatar
Caillat Michel committed
331
332
        self.__logger.debug("renderingCapabilities: exiting")
        return result
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398

    #
    # Return the list of defined DataBlock s sorted ( asc or desc ) by timestamp
    #
    def getDataBlockInfos(self, reverse=True):
        self.__logger.debug("getDataBlockInfos : entering")
        result = dict()
        #
        # report time
        #
        result["when"] = datetime.now().strftime(DataBlock.getDateTimeFormat())

        # Machine name
        #
        hostname = socket.gethostname()
        result["hostname"] = hostname

        #
        # Machine informations
        #
        numsockets =  int(subprocess.check_output('cat /proc/cpuinfo | grep "physical id" | sort -u | wc -l', shell=True))
        cpucount = psutil.cpu_count()
        cpuInfos = {"numsockets": numsockets, "cpucount": cpucount}
        result["cpu"] = cpuInfos

        #
        # Physical memory informations
        #
        physMemoryInBytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
        usedMemoryInBytes = sum([self.__dataBlocks[dbId].sizeInBytes for dbId in self.__dataBlocks])
        usedMemoryPercentage = math.floor((usedMemoryInBytes / physMemoryInBytes) * 10000) / 100.
        physMemory = DataBlock.convert_size(physMemoryInBytes)
        usedMemory = DataBlock.convert_size(usedMemoryInBytes)
        memoryInfos = {"physMemory": physMemory, "usedMemory": usedMemory, "usedMemoryPercentage": usedMemoryPercentage}

        result["memory"] = memoryInfos

        # Maximum idleness duration
        result["maxidle"] = DataBlock.getMaxIdle()

        #
        # DataBlocks informations
        #
        infoNames = DataBlock.getDataBlockInfoNames()
        x = [infoNames]
        now = datetime.now()
        for k in self.__dataBlocks:
            x.append(self.__dataBlocks[k].getDataBlockInfos(now, infoNames))
        self.__logger.debug("getDataBlockInfos : exiting")

        result["dataBlocks"] = x

        self.__logger.debug("getDataBlockInfos : exiting")
        return {"status": True, "message": "", "result": result}

    #
    # Delete the DataBlocks having their "idle" property
    # greater than DataBlock.getMaxIdle()
    #
    def purgeDataBlocks(self):
        self.__logger.debug("purgeDataBlocks : entering")
        tobePurged = [db for db in self.__dataBlocks if (datetime.now() - self.__dataBlocks[db].lastAccess).total_seconds() > DataBlock.getMaxIdle()]
        for db in tobePurged:
            del self.__dataBlocks[db]
        self.__logger.debug("purgeDataBlocks : exiting")
        return {"status": True, "message": f"{len(tobePurged)} data set(s) erased.", "result": tobePurged}
Caillat Michel's avatar
Caillat Michel committed
399

400
401
402
    #
    #
    #
403
    def getYtObj(self,relFITSFilePath,product,coord):
ba yaye-awa's avatar
ba yaye-awa committed
404
        self.__logger.debug("getYtObj : entering")
405
        self.__logger.debug("relFITSFilePath = %r, product= %r, (%i,%i,%i,%i,%i,%i)" % (relFITSFilePath,product,coord['iRA0'],coord['iRA1'],coord['iDEC0'],coord['iDEC1'],coord['iFREQ0'],coord['iFREQ1']))
ba yaye-awa's avatar
ba yaye-awa committed
406
407
        result = self.__checkPresence(relFITSFilePath)
        if result :
408
            result = self.__dataBlocks[relFITSFilePath].getYtObj(relFITSFilePath,product,coord)
ba yaye-awa's avatar
ba yaye-awa committed
409
410
        self.__logger.debug("getYtObj : exiting")
        return result
411

Caillat Michel's avatar
Caillat Michel committed
412
413
414
415
416
#
#
#    End of the DataManagerImpl class
#
#