dataManager_michel.py 16.9 KB
Newer Older
Caillat Michel's avatar
Caillat Michel committed
1
2
3
#!/usr/bin/env python

import os
4
5
6
import subprocess
import socket
import psutil
Caillat Michel's avatar
Caillat Michel committed
7
8
import json
import math
9
10
11
12
13
import numpy as np
import yt
from spectral_cube import SpectralCube
from astropy.io import fits
from astropy.stats import sigma_clip
14
from datetime import datetime
Caillat Michel's avatar
Caillat Michel committed
15

16
17
from DataBlock import DataBlock

18
19
20
DataRoot = "dataroot"
FITSFilePrefix = DataRoot + '/FITS/'
PNGFilePrefix = DataRoot + '/PNG/'
21
OBJFilePrefix = DataRoot + '/OBJ/'
22
ApiDocPrefix = DataRoot + '/apidoc/'
Caillat Michel's avatar
Caillat Michel committed
23

24
25
26
27
28
29
#
#
# Tell DataBlock where the files will be located
#
# The FITS files
DataBlock.setFITSFilePrefix(FITSFilePrefix)
Caillat Michel's avatar
Caillat Michel committed
30

31
32
# The PNG files
DataBlock.setPNGFilePrefix(PNGFilePrefix)
Caillat Michel's avatar
Caillat Michel committed
33

34
35
36
# The PNG files
DataBlock.setOBJFilePrefix(OBJFilePrefix)

Caillat Michel's avatar
Caillat Michel committed
37
38
39
40
#
# An implementation of the class used in the serverWsgi-ish server.
#
class DataManagerImpl :
41
    __default_palette_name, __default_transformation_name, __default_video_mode_name = DataBlock.getDefaults()
Caillat Michel's avatar
Caillat Michel committed
42

43
44
    #===========================================================================
    # Private methods
Caillat Michel's avatar
Caillat Michel committed
45
46
    #

47
48
    # check if the FITS data identified by the relFITSFilePath are present in
    # memory. If yes update the corresponding DataBlock's timestamp.
Caillat Michel's avatar
Caillat Michel committed
49

50
51
52
53
54
55
56
57
    def __checkPresence(self, relFITSFilePath):
        self.__logger.debug("__checkPresence : entering")
        result = {"status": True, "message" : '', "result": None}
        if relFITSFilePath in self.__dataBlocks:
            self.__dataBlocks[relFITSFilePath].setLastAccess()
        else:
            result = {"status": False, "message": f'FITS file "{relFITSFilePath}" is not present in memory. Call "setData" first'}
        self.__logger.debug("__checkPresence : exiting")
Caillat Michel's avatar
Caillat Michel committed
58
59
        return result

60
61
62
63
64
65
66
67
68
69
    def __getEntries_0(self, relKey):
        self.__logger.debug("__getEntries_0: entering")
        try :
            absFITSFilePrefix = FITSFilePrefix + relKey
            entries = (os.listdir(absFITSFilePrefix))
            self.__logger.debug("type of entries %s" % (type(entries)))
            sortedEntries = entries.sort()
            children = []
            for entry in entries:
                p = absFITSFilePrefix + '/' + entry
70
71
72
73
                condition = os.path.isfile(p) and p.endswith(".fits")
                condition  = condition or (os.path.islink(p) and os.path.isdir(os.readlink(p)))
                condition = condition or (os.path.isdir(p))
                if not condition :
74
75
76
77
78
79
                    continue
                elif entry in ["log", "NOFITS", "IGNORE"]:
                    continue
                else:
                    d = dict()
                    d["key"]    = relKey + '/' + entry
80
                    d["folder"] = not os.path.isfile(p) 
81
82
83
                    d["lazy"]   = d["folder"]
                    if entry.endswith(".fits") :
                        size = DataBlock.convert_size(os.path.getsize(p))
84
                        d["title"] = "<a href = 'visit/?relFITSFilePath=%s/%s' target = '_blank'>%s %s</a>" % (relKey, entry, entry, size)
85
86
87
88
89
90
91
92
                    else:
                        d["title"] = entry
                    children.append(d)
            result = {"status": True, "message": "", "result": json.dumps(children)}
        except Exception as e :
            result = {"status": False, "message": "Problem while looking for entries under '%s'. Error message was '%s'" % ( relKey, e)}
            self.__logger.debug("%r" % result)
        self.__logger.debug("__getEntries_0: exiting")
Caillat Michel's avatar
Caillat Michel committed
93
94
        return result

95
96
97
98
99
    #
    # Return the keys of the dataBlocks dictionary as if it was ordered by timestamp ( asc or desc)
    #
    def __getKeysOrderedByTimestamp (self, reverse=True):
        return [x[0] for x in sorted({x: self.__dataBlocks[x]["timestamp"] for x in self.__dataBlocks.keys()}.items(), key=lambda kv: kv[1], reverse=reverse)]
Caillat Michel's avatar
Caillat Michel committed
100

101
102
    #===========================================================================
    # Public methods.
Caillat Michel's avatar
Caillat Michel committed
103

104
105
106
107
    #===========================================================================
    # CTOR
    #
    def __init__(self, logger):
Caillat Michel's avatar
Caillat Michel committed
108

109
110
111
        self.__dataBlocks = dict()
        self.__logger = logger
        self.__logger.debug(f"An instance of DataManagerImpl is created")
Caillat Michel's avatar
Caillat Michel committed
112

113
114
115
116
117
118
119
120
121
122
123
124
    #===========================================================================
    # THE data selector.
    # Its execution will trigger the creation of a DataBlock populated by
    # the content of a FITS file if it's not already loaded.
    # In any case return the header of the FITS data as a dictionary.
    #
    def setData(self, relFITSFilePath):
        self.__logger.debug("setData : entering");
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            self.__dataBlocks[relFITSFilePath].setLastAccess()
            result = self.__dataBlocks[relFITSFilePath].getHeader()
Caillat Michel's avatar
Caillat Michel committed
125
        else :
126
127
128
129
130
131
            self.purgeDataBlocks()
            db = DataBlock(self.__logger)
            result = db.setData(relFITSFilePath)
            if (result["status"]):
                self.__dataBlocks[relFITSFilePath]=db
        self.__logger.debug(f"About to return {result}")
Caillat Michel's avatar
Caillat Michel committed
132
133
134
        self.__logger.debug("setData : Exiting")
        return result

135
136
137
138
    #===========================================================================
    # General getters
    #
    #
Caillat Michel's avatar
Caillat Michel committed
139
140


141
142
143
144
145
146
147
148
149
150
    #===========================================================================
    # Getters on a DataBlock identified by its relFITSFilePath ( i.e. the dictionary keys in __dataBlocks )
    #
    def getDimensions(self, relFITSFilePath):
        self.__logger.debug( "getDimensions : entering");
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].getDimensions()
        self.__logger.debug( "getDimensions : exiting");
        return result
Caillat Michel's avatar
Caillat Michel committed
151

Caillat Michel's avatar
Caillat Michel committed
152
    def getSlice(self, relFITSFilePath, iFREQ, step=1 ):
153
        self.__logger.debug("getSlice : entering")
Caillat Michel's avatar
Caillat Michel committed
154
        result = self.__checkPresence(relFITSFilePath)
155
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
156
            result = self.__dataBlocks[relFITSFilePath].getSlice(iFREQ, step)
Caillat Michel's avatar
Caillat Michel committed
157
        self.__logger.debug("getSlice : exiting")
158
        return result
Caillat Michel's avatar
Caillat Michel committed
159

160
161
    def getSpectrum(self, relFITSFilePath, iRA=None, iDEC=None, iFREQ0=None, iFREQ1=None):
        self.__logger.debug( "getSpectrum : entering")
162
163
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
164
165
            result = self.__dataBlocks[relFITSFilePath].getSpectrum(iRA, iDEC, iFREQ0, iFREQ1)
        self.__logger.debug( "getSpectrum : exiting")
166
        return result
Caillat Michel's avatar
Caillat Michel committed
167

Caillat Michel's avatar
Caillat Michel committed
168
    def getAverageSpectrum(self, relFITSFilePath, iDEC0=None, iDEC1=None, iRA0=None, iRA1=None, retFITS=False):
169
170
171
        self.__logger.debug("getAverageSpectrum : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
172
            result = self.__dataBlocks[relFITSFilePath].getAverageSpectrum(iDEC0, iDEC1, iRA0, iRA1, retFITS)
173
174
        self.__logger.debug("getAverageSpectrum : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
175
176


177
178
179
180
181
182
183
    def getHeader(self, relFITSFilePath):
        self.__logger.debug("getHeader : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].getHeader()
        self.__logger.debug("getHeader : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
184

185
186
187
188
189
190
191
    def RADECRangeInDegrees( self, relFITSFilePath):
        self.__logger.debug("RADECRangeInDegrees : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result = self.__dataBlocks[relFITSFilePath].RADECRangeInDegrees()
        self.__logger.debug("RADECRangeInDegrees : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
192

193
    def degToHMSDMS(self, relFITSFilePath, RAinDD, DECinDD):
194
195
196
        self.__logger.debug("degToHMSDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
197
            result = self.__dataBlocks[relFITSFilePath].degToHMSDMS(RAinDD, DECinDD)
Caillat Michel's avatar
Caillat Michel committed
198
199
        self.__logger.debug("degToHMSDMS : exiting")
        return result
Caillat Michel's avatar
Caillat Michel committed
200

201
    def rangeToHMS(self,  relFITSFilePath, iRA0, iRA1, iRAstep):
202
203
204
        self.__logger.debug("degToHMSDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
205
            result=self.__dataBlocks[relFITSFilePath].rangeToHMS(iRA0, iRA1, iRAstep)
206
        self.__logger.debug("degToHMSDMS : exiting")
Caillat Michel's avatar
Caillat Michel committed
207
        return result
Caillat Michel's avatar
Caillat Michel committed
208

209
    def rangeToDMS(self,  relFITSFilePath, iDEC0, iDEC1, iDECstep):
210
211
212
        self.__logger.debug("rangeToDMS : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
213
            result = self.__dataBlocks[relFITSFilePath].rangeToDMS(iDEC0, iDEC1, iDECstep)
214
        self.__logger.debug("rangeToDMS : exiting")
Caillat Michel's avatar
Caillat Michel committed
215
        return result
Caillat Michel's avatar
Caillat Michel committed
216
217
218

    def getPixelValueAtiFreqiRAiDEC(self, relFITSFilePath, iFreq, iRA, iDEC):
        self.__logger.debug("getPixelValueAtiFreqiRAiDEC : entering")
219
220
221
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
            result=self.__dataBlocks[relFITSFilePath].getPixelValueAtiFreqiRAiDEC(iFreq, iRA, iDEC)
Caillat Michel's avatar
Caillat Michel committed
222
223
224
        self.__logger.debug("getPixelValueAtiFreqiRAiDEC : exiting")
        return result

225
    def getSumOverSliceRectArea(self, relFITSFilePath, iFREQ, iRA0=None, iRA1=None, iDEC0=None, iDEC1=None):
Caillat Michel's avatar
Caillat Michel committed
226
        self.__logger.debug("getSumOnSliceRectArea : entering")
227
228
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
229
            result=self.__dataBlocks[relFITSFilePath].getSumOverSliceRectArea(iFREQ, iRA0, iRA1, iDEC0, iDEC1)
Caillat Michel's avatar
Caillat Michel committed
230
231
232
        self.__logger.debug("getSumOnSliceRectArea : exiting")
        return result

Caillat Michel's avatar
Caillat Michel committed
233
    def getAverage(self, relFITSFilePath, iFREQ0, iFREQ1, iDEC0, iDEC1, iRA0, iRA1, retFITS):
Caillat Michel's avatar
Caillat Michel committed
234
        self.__logger.debug("getAverage : entering")
235
236
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
237
            result = self.__dataBlocks[relFITSFilePath].getAverage(iFREQ0, iFREQ1, iDEC0, iDEC1, iRA0, iRA1, retFITS)
Caillat Michel's avatar
Caillat Michel committed
238
239
240
        self.__logger.debug("getAverage : exiting")
        return result

Caillat Michel's avatar
Caillat Michel committed
241
    def getOneSliceAsPNG (self, iFREQ, relFITSFilePath, **kwargs):
Caillat Michel's avatar
Caillat Michel committed
242
        self.__logger.debug("getOneSliceAsPNG : entering.")
Caillat Michel's avatar
Caillat Michel committed
243
        self.__logger.debug("iFREQ = %r, relFITSFilePath = %r" % (iFREQ, relFITSFilePath))
244
245
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
246
            result = self.__dataBlocks[relFITSFilePath].getOneSliceAsPNG(iFREQ, **kwargs)
Caillat Michel's avatar
Caillat Michel committed
247
        self.__logger.debug("getOneSliceAsPNG : exiting.")
248
        return result
Caillat Michel's avatar
Caillat Michel committed
249

250
    def getSummedSliceRangeAsPNG( self, relFITSFilePath, iFREQ0, iFREQ1, **kwargs):
Caillat Michel's avatar
Caillat Michel committed
251
        self.__logger.debug("getSummedSliceRangeAsPNG : entering.")
252
253
        result = self.__checkPresence(relFITSFilePath)
        if result["status"]:
Caillat Michel's avatar
Caillat Michel committed
254
            result = self.__dataBlocks[relFITSFilePath].getSummedSliceRangeAsPNG(iFREQ0, iFREQ1, **kwargs)
Caillat Michel's avatar
Caillat Michel committed
255
256
257
        self.__logger.debug("getSummedSliceRangeAsPNG : exiting.")
        return result

Caillat Michel's avatar
Caillat Michel committed
258
259
260
    def getContours(self, relFITSFilePath, iFREQ, iDEC0, iDEC1, iRA0, iRA1, **kwargs):
        self.__logger.debug("getContours : entering.")
        result = self.__checkPresence(relFITSFilePath)
261
        if result["status"] :
Caillat Michel's avatar
Caillat Michel committed
262
263
264
265
            result = self.__dataBlocks[relFITSFilePath].getContours(iFREQ, iDEC0, iDEC1, iRA0, iRA1, **kwargs)
        self.__logger.debug("getContours : exiting.")
        return result

266
    def measureContour(self, relFITSFilePath, iFREQ, contour, level):
267
268
269
        self.__logger.debug("measureContour - dispatcher : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"] :
270
            result = self.__dataBlocks[relFITSFilePath].measureContour(iFREQ, contour, level)
271
272
273
        self.__logger.debug("measureContour - dispatcher : exiting")
        return result

274
275
276
277
278
279
280
281
    def measureBox(self, relFITSFilePath, iFREQ, iRA0, iRA1, iDEC0, iDEC1):
        self.__logger.debug("measureBox - dispatcher : entering")
        result = self.__checkPresence(relFITSFilePath)
        if result["status"] :
            result = self.__dataBlocks[relFITSFilePath].measureBox(iFREQ, iRA0, iRA1, iDEC0, iDEC1)
        self.__logger.debug("measureBox - dispatcher : exiting")
        return result

282

Caillat Michel's avatar
Caillat Michel committed
283
    #
284
285
286
    # create fits file containing a spectrum at iRA, iDEC
    # The use case is interoperability via SAMP
    #
Caillat Michel's avatar
Caillat Michel committed
287
288
    def createFits(self, relFITSFilePath, iRA, iDEC):
        self.__logger.debug("createFITS : entering")
289
290
291
        result = self.__checkPresence(relFITSFilePath)
        if result :
            result = self.__dataBlocks[relFITSFilePath].createFits(iRA, iDEC)
Caillat Michel's avatar
Caillat Michel committed
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
        self.__logger.debug("createFITS : exiting")
        return result

    #
    # Returns all the entries directly located under a given subdirectory.
    # The subdirectory must be passed as a full path implicitely
    # rooted at FITSFilePrefix.
    # i.e. :
    # "/" will be interpreted as FITSFilePrefix+"/"
    # "/2018" will be interpreted as FITSFilePrefix+"/2018"
    # etc...
    # The result is a JSON array of dictionaries , on the basis of one dictionary per inode:
    # [{'title': filename of the entry, 'key': the FITSFilePrefix rooted path of the entry, 'folder' : true (resp. false) if the inode is (resp. is not) a subdir, 'lazy': true}]
    def getEntries(self, relKey):
        self.__logger.debug("getEntries: entering")
        return self.__getEntries_0(relKey)
        self.__logger.debug("getEntries: exiting")

    #
    # Returns the capabilities of the server relative to the way the image pixels are calculated.
    #
    #  Intensity Transformations names are defined in a list itts
    #  Colors names are defined in a list luts
    #  Video modes are defined in a list vmode
    #
    #  For each of this rendering filters default names are defined.
    #
    def renderingCapabilities(self) :
        self.__logger.debug("renderingCapabilities: entering")
321
        result = {"status": True, "message": "", "result": DataBlock.getRenderingCapabilities()}
Caillat Michel's avatar
Caillat Michel committed
322
323
        self.__logger.debug("renderingCapabilities: exiting")
        return result
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389

    #
    # Return the list of defined DataBlock s sorted ( asc or desc ) by timestamp
    #
    def getDataBlockInfos(self, reverse=True):
        self.__logger.debug("getDataBlockInfos : entering")
        result = dict()
        #
        # report time
        #
        result["when"] = datetime.now().strftime(DataBlock.getDateTimeFormat())

        # Machine name
        #
        hostname = socket.gethostname()
        result["hostname"] = hostname

        #
        # Machine informations
        #
        numsockets =  int(subprocess.check_output('cat /proc/cpuinfo | grep "physical id" | sort -u | wc -l', shell=True))
        cpucount = psutil.cpu_count()
        cpuInfos = {"numsockets": numsockets, "cpucount": cpucount}
        result["cpu"] = cpuInfos

        #
        # Physical memory informations
        #
        physMemoryInBytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
        usedMemoryInBytes = sum([self.__dataBlocks[dbId].sizeInBytes for dbId in self.__dataBlocks])
        usedMemoryPercentage = math.floor((usedMemoryInBytes / physMemoryInBytes) * 10000) / 100.
        physMemory = DataBlock.convert_size(physMemoryInBytes)
        usedMemory = DataBlock.convert_size(usedMemoryInBytes)
        memoryInfos = {"physMemory": physMemory, "usedMemory": usedMemory, "usedMemoryPercentage": usedMemoryPercentage}

        result["memory"] = memoryInfos

        # Maximum idleness duration
        result["maxidle"] = DataBlock.getMaxIdle()

        #
        # DataBlocks informations
        #
        infoNames = DataBlock.getDataBlockInfoNames()
        x = [infoNames]
        now = datetime.now()
        for k in self.__dataBlocks:
            x.append(self.__dataBlocks[k].getDataBlockInfos(now, infoNames))
        self.__logger.debug("getDataBlockInfos : exiting")

        result["dataBlocks"] = x

        self.__logger.debug("getDataBlockInfos : exiting")
        return {"status": True, "message": "", "result": result}

    #
    # Delete the DataBlocks having their "idle" property
    # greater than DataBlock.getMaxIdle()
    #
    def purgeDataBlocks(self):
        self.__logger.debug("purgeDataBlocks : entering")
        tobePurged = [db for db in self.__dataBlocks if (datetime.now() - self.__dataBlocks[db].lastAccess).total_seconds() > DataBlock.getMaxIdle()]
        for db in tobePurged:
            del self.__dataBlocks[db]
        self.__logger.debug("purgeDataBlocks : exiting")
        return {"status": True, "message": f"{len(tobePurged)} data set(s) erased.", "result": tobePurged}
Caillat Michel's avatar
Caillat Michel committed
390

391
392
393
    #
    #
    #
394
    def getYtObj(self,relFITSFilePath,product,coord):
ba yaye-awa's avatar
ba yaye-awa committed
395
        self.__logger.debug("getYtObj : entering")
396
        self.__logger.debug("relFITSFilePath = %r, product= %r, (%i,%i,%i,%i,%i,%i)" % (relFITSFilePath,product,coord['iRA0'],coord['iRA1'],coord['iDEC0'],coord['iDEC1'],coord['iFREQ0'],coord['iFREQ1']))
ba yaye-awa's avatar
ba yaye-awa committed
397
398
        result = self.__checkPresence(relFITSFilePath)
        if result :
399
            result = self.__dataBlocks[relFITSFilePath].getYtObj(relFITSFilePath,product,coord)
ba yaye-awa's avatar
ba yaye-awa committed
400
401
        self.__logger.debug("getYtObj : exiting")
        return result
402

Caillat Michel's avatar
Caillat Michel committed
403
404
405
406
407
#
#
#    End of the DataManagerImpl class
#
#