Skip to content

API Class CDFmag

CDFmag

Source code in leocdf/classCDFmag.py 
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
class CDFmag:
    defaultCDFdir = "/home/jovyan/leodata/data_cdf"
    #defaultCDFdir = "/mnt/c/Users/inaw2/Documents/leomagnetics/GIT/leo-pythontools"
    #defaultCDFdir = "/home/ina/GIT/cdf"
    configFilesDir = Path(__file__).parents[1]/'configFiles'
    zVarAttFile = configFilesDir/'CDFzVarAttribute.json'
    iagaHeaderFile = configFilesDir/'IAGA-headers.json'

    varGroups = { "Base" : ["H0", "D0", "Z0", "F0", "HNscv", "HEscv", "Zscv", "Basetrig"],
        "Data" : ["time", "HNvar", "HEvar", "Zvar", "T1", "T2", "Fsc"],
        "Time" : ["timeppm", "timegps", "timefge"],
        "Flags" : ["HNflag", "HEflag", "Zflag", "Fscflag", "FscQP", "T1flag", "T2flag"],
        "Error" : ["Timeerr", "Timeerrtrig"],
        "Filter" : ["filter_type", "filter_winlength", "filter_steplength"] }

    flags = {
        "HNvar": "HNflag",
        "HEvar": "HEflag",
        "Zvar": "Zflag",
        "Fsc": "Fscflag",
        "T1": "T1flag",
        "T2": "T2flag"
    }

    julianday = datetime(2000,1,1).replace(tzinfo=timezone.utc)

    # c-library
    path = os.getcwd()
    libmagcdf = ctypes.CDLL( Path(__file__).parents[0]/"libmagcdf/libmagcdf.so" )
    #libmagcdf = ctypes.CDLL(os.path.join(path,"leocdf/libmagcdf/libmagcdf.so"))
    py_createcdf = libmagcdf.createCDF
    py_createcdf.restype = None

    py_updatecdf = libmagcdf.updateCDF
    #py_createcdf.argtypes = [ctypes.c_char_p, c_double_p] 
    py_updatecdf.restype = None

    def __init__(self, datum: datetime | str = '', code: str = '', file: Path | str = ''):
        '''Class of daily cdf file

        Object with data (xarray), date, station code and filename and directory.\n
        Enter of datetime and station code or filename.
        Default directory is "/home/jovyan/leodata/data_cdf".

        Parameters
        ----------
        datum : datetime | str, optional
            Day of the cdf-file, by default ''
        code : str, optional
            Station code, by default ''
        file : Path | str, optional
            Cdf-file name to be read, by default ''

        Raises
        ------
        RunError
            Incorrect input parameter.
        FileError
            Wrong filename format.
        '''
        if datum == '' and code == '' and file == '':
            raise Exception("Error! Give datetime and code or a filename.") 

        else:
            if datum == '' and code =='':
                file_str = file.name if isinstance(file, Path) else file
                if len(file_str.replace('.cdf', '').split('_')) == 2:
                    self.code = file_str.replace('.cdf', '').split('_')[0]
                    self.datum = checkdateformat(file_str.replace('.cdf', '').split('_')[1])
                else:
                    raise Exception("Error! Wrong filename format. CODE_YYYYMMDD.cdf") 
            else:
                self.datum = datum if isinstance(datum, datetime) else checkdateformat(datum)
                self.code = code

            self.file = Path(file) if isinstance(file, str) else file
            if self.file.parent == Path('.') and self.file == Path(''):
                self.cdfdir = Path(f"{CDFmag.defaultCDFdir}/{self.code}/cdf/{self.datum.strftime('%Y')}/")
            else:
                self.cdfdir = self.file.parent

            if self.file == Path(''):
                self.file = Path(f"{self.code}_{self.datum.strftime('%Y%m%d')}.cdf") 
            self.file = self.file.name

            # ds - xarray with data
            if Path(self.cdfdir/self.file).exists():
                self.ds = cdflib.xarray.cdf_to_xarray(self.cdfdir/self.file)
            else:
                self.ds = self.emptyCDF()

    def emptyCDF(self) -> xr.DataSet:
        '''Creates an empty xarray.Dataset with labeled xarray.DataArrays and attributes.\n
        To-Do: pass title and header of cdf-file

        Returns
        -------
        ds : xr.DataSet
            empty xarray.Dataset with labeled xarray.DataArrays and attributes 
        '''

        basetrig = pd.date_range(start=self.datum, periods=8, freq='3h', tz='UTC')
        basetrig = basetrig.append(basetrig).sort_values()

        time = pd.date_range(start=self.datum, periods=86400, freq='1s', tz='UTC')

        ds = xr.Dataset(
            data_vars=dict(
                H0=( ["record0"], np.zeros(16).astype(np.float64) ),
                D0=( ["record0"], np.zeros(16).astype(np.float64) ),
                Z0=( ["record0"], np.zeros(16).astype(np.float64) ),
                F0=( ["record0"], np.zeros(16).astype(np.float64) ),
                HNscv=( ["record0"], np.ones(16).astype(np.float64) ),
                HEscv=( ["record0"], np.ones(16).astype(np.float64) ),
                Zscv=( ["record0"], np.ones(16).astype(np.float64) ),
                Basetrig=( ["record0"], np.array(mjd2000(basetrig), dtype=np.float64) ),
                time=(["record1"], np.array(mjd2000(time), dtype=np.float64) ),
                HNvar=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
                HEvar=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
                Zvar=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
                T1=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
                T2=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
                timeppm=( ["record2"], np.full(1, np.nan).astype(np.float64) ),
                timegps=( ["record2"], np.full(1, np.nan).astype(np.float64) ),
                timefge=( ["record2"], np.full(1, np.nan).astype(np.float64) ),
                Fsc=( ["record1"], np.full(86400, np.nan).astype(np.float64) ),
                HNflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
                HEflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
                Zflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
                Fscflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
                FscQP=( ["record1"], np.ones(86400).astype(np.int8)*0 ),
                T1flag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
                T2flag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
                Timeerr= ( ["record2"], np.zeros(1).astype(np.float64) ),
                Timeerrtrig = ( ["record2"], np.array( [mjd2000(time[0])] ).astype(np.float64) ),
                filter_type = ( ["record3"], np.zeros(4).astype(np.uint16) ),
                filter_winlength = ( ["record3"], np.zeros(4).astype(np.uint16) ),
                filter_steplength = ( ["record3"], np.zeros(4).astype(np.uint16) )
            ),
            attrs=dict(TITLE="Magnetometer data.",
                HEADER="The header           ', 'has to be determined.",
                DATE=self.datum.strftime('%Y%m%d'),
                CODE=self.code
                    )
        )

        return ds

    def save(self, pathdir: str | Path = None) -> None:
        '''Saves the cdf-file under the default directory */home/jovyan/leodata/data_cdf*. 

        Parameters
        ----------
        pathdir : str | Path, optional
            Changes the directory, by default None
        '''

        if pathdir == None:
            pathdir = self.cdfdir 
        elif isinstance(pathdir, str):
            pathdir = Path(pathdir)

        # check overwrite
        if Path(pathdir/self.file).exists():
            filename = questionoverwrite(Path(pathdir/self.file))
        else:
            filename = self.file
        att = readJson(CDFmag.zVarAttFile)

        zvarTuple = lib.createCdata( self.ds, att, [i for i in self.ds.data_vars] )
        # Convert to ctypes array
        arrayCtypes = (lib.zVariable * len(zvarTuple))(*zvarTuple)

        pathdir.mkdir(parents=True, exist_ok=True)

        # gesamter Pfad ohne .cdf

        #filename = f"{str(pathdir)}/{filename[:-4]}".encode('ASCII') # Anagbe ohne .cdf
        filename = lib.getCDFfilename(pathdir, filename)

        CDFmag.py_createcdf(filename, arrayCtypes, len(arrayCtypes))

    def updateCDF(self, vargroup: str | list) -> None:
        '''Updates the cdf-file. Only the submitted variables are updated.

        Parameters
        ----------
        vargroup : str | list
            Variables to be updated.

        Raises
        ------
        FileError
            cdf-file doesn't exist.
        VariableError
            Variable name doesn't exist.
        '''

        # check files exist
        if not Path(self.cdfdir/self.file).exists():
            raise Exception(f"Error! File {self.cdfdir}/{self.file} doesn't exist.")
        # check zvar in CDFmag.varGroups

        att = readJson(CDFmag.zVarAttFile)

        if var in list(att.keys()):
            var = list(var)
        elif var == 'Baseline':
            var = ["H0", "D0", "Z0", "F0", "HNscv", "HEscv", "Zscv", "Basetrig"]
        elif var == 'data':
            var = ["time", "HNvar", "HEvar", "Zvar", "T1", "T2", "Fsc"]
        elif var == 'flags':
            var = ["HNflag", "HEflag", "Zflag", "Fscflag", "FscQP", "T1flag", "T2flag"]
        else:
            raise Exception(f"Error! {var} not valid.") 

        zvarTuple = lib.createCdata(self.ds, att, var)
        # Convert to ctypes array
        arrayCtypes = (lib.zVariable * len(zvarTuple))(*zvarTuple)

        #filename = f"{str(self.cdfdir)}/{self.file[:-4]}".encode('ASCII')
        filename = lib.getCDFfilename(self.cdfdir, self.file)
        CDFmag.py_updatecdf(filename, arrayCtypes, len(arrayCtypes))         

    def setVar(self, arr: np.array, name: str) -> bool:
        '''Sets new data to variable. 

        Parameters
        ----------
        arr : np.array
            Array with new data.
        name : str
            Variable name receiving new data.

        Returns
        -------
        bool
            Successful update.

        Raises
        ------
        ArrayError
            Array has not the right length.
        '''

        # check array size
        if arr.shape != self.ds[ name ].data.shape:
            raise Exception(f"Array has not the right length. Shape {self.ds[ name ].data.shape} is used.") 
            return False

        self.ds[name].data = arr

        if name in CDFmag.flags.keys():
            i = np.argwhere( ~np.isnan( arr ) )
            self.ds[ CDFmag.flags[name] ].data[i] = 0

        return True

    def readBaselineCSV():
        pass

    def plotCDFold(self) -> None:
        xtime = [ CDFmag.julianday + timedelta(days = i) for i in self.ds['time'].values ]

        fig, ax = plt.subplots(1,figsize=(15, 5.5), layout="constrained")

        for var, c in zip(['Zvar', 'HNvar', 'HEvar', 'Fsc'], ['k', 'b', 'g', 'r']):
            ax.plot(xtime, 
                    self.ds[var].where( (self.ds[ CDFmag.flags[var] ] >= 0) & (self.ds[ CDFmag.flags[var] ] <= 60) ) - self.ds[var].mean(skipna=True), 
                    color=c, label=var, linewidth=0.5)

        ax.set_xlabel('time')
        ax.set_ylabel('variometer')
        ax.grid(True)
        ax.set_xlim( (xtime[0],xtime[-1]) )
        ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
        ax.legend()
        ax.set_title(f"{self.code} - {self.datum.strftime('%Y-%m-%d')}")
        plt.show()

    def plotCDF(self) -> None:
        '''Plot raw values Zvar, HNvar, HEvar, Fsc
        '''
        # ueberfuehren in df
        df = pd.DataFrame(index=[ CDFmag.julianday + timedelta(days = i) for i in self.ds['Basetrig'].values ], 
                    data = { 'Z0': self.ds['Z0'].values, 'Zscv': self.ds['Zscv'].values,
                      'H0': self.ds['H0'].values, 'HEscv': self.ds['HEscv'].values,
                      'D0': self.ds['D0'].values, 'HNscv': self.ds['HNscv'].values,
                      'F0': self.ds['F0'].values  })
        df = df.drop_duplicates()
        df = df.reindex( pd.date_range(start=df.index[0], periods=86400, freq='1s', tz='UTC') )
        df = df.ffill()

        df['Zscale'] = df['Z0'].values + df['Zscv'].values * self.ds['Zvar'].values
        df['HNscale'] = df['H0'].values + df['HNscv'].values * self.ds['HNvar'].values
        df['HEscale'] = df['HEscv'].values * self.ds['HEvar'].values
        df['Fscale'] = df['F0'].values + self.ds['Fsc'].values

        # plot 
        fig, ax = plt.subplots(1,figsize=(15, 5.5), layout="constrained")

        for var, lvar, c in zip(['Zscale', 'HNscale', 'HEscale', 'Fscale'], ['Zvar', 'HNvar', 'HEvar', 'Fsc'], ['k', 'b', 'g', 'r']):
            df[var] = df[var] - df[var].mean()  # mean abziehen
            df[var].plot(ax=ax, color=c, label=lvar, linewidth=0.5)
        ax.set_xlabel('time')
        ax.set_ylabel('variometer')
        ax.grid('on', which='minor', axis='x' )
        ax.grid('on', which='major', axis='y' )
        #ax.set_xlim( (df.index[0],df.index[-1]) )
        ax.legend()
        #ax.set_title(f"{self.code} - {self.datum.strftime('%Y-%m-%d')}")
        plt.show()



    def createIAGA(self, file: str, sec: int = 1, flagDelete: int = -1) -> None:
        '''Created ascii IAGA file. Only use definitiv values ( 0 < flags < 61).  

        Parameters
        ----------
        file : str
            Name of the new file
        sec : int, optional
            Sampling rate in seconds, by default 1
        flagDelete : int, optional
            Delete values with this flag, by default -1
        '''

        #if file.exists():
        #    pass

        # iagaCode = loadIagaCode(self.code)
        iagaCode = 'JRU'

        # Header
        writeHeader = []

        header = readJson(CDFmag.iagaHeaderFile)

        if iagaCode in header:
            header = header[iagaCode]
        else:
            print(f"Error. No station code {iagaCode} in iaga-header.json.")

        headerKey = {'Format': 'IAGA-2002', 
                        'Source of Data': ' ', 
                        'Station Name': ' ', 
                        'IAGA Code': iagaCode, 
                        'Geodetic Latitude': ' ', 
                        'Geodetic Longitude': ' ', 
                        'Elevation': ' ', 
                        'Reported': ' ', 
                        'Sensor Orientation': ' ',
                        'Digital Sampling': '5 seconds', 
                        'Data Interval Type': '1-second spot readings', 
                        'Data Type': 'Provisional'} 

        for h in headerKey:
            if h in header:
                headerValue = header[h] if isinstance(header[h], str) else str(header[h])
            else:
                headerValue = headerKey[h] if isinstance(headerKey[h], str) else str(headerKey[h])
            writeHeader.append(f" {h.ljust(23)}{headerValue.ljust(45)}|\n")

        infoText = "Hallo"
        for i in range(0,3):
            writeHeader.append(f" # {infoText.ljust(65)} |\n")

        columnRow = ""
        for c, i in zip(['DATE', 'TIME', 'DOY', f"{iagaCode}X", f"{iagaCode}Y", f"{iagaCode}Z", f"{iagaCode}F"], 
            [11, 13, 8, 10, 10, 10, 7]):
            columnRow = columnRow + c.ljust(i)
        writeHeader.append(f"{columnRow}|\n")

        # Data values
        di = pd.DataFrame(index=[ CDFmag.julianday + timedelta(days = i) for i in self.ds['time'].data ])

        for i, flag, var in zip(['X', 'Y', 'Z', 'F'], ['HNflag', 'HEflag', 'Zflag', 'Fscflag'], ['HNvar', 'HEvar', 'Zvar', 'Fsc']):
            di[f'{iagaCode}{i}'] = np.where((self.ds[flag].data >= 0) & (self.ds[flag].data <= 60), 
                                                self.ds[var], 99999.0)

        if flagDelete != -1:
            for f in [flagDelete]:
                for i, flag in zip(['X', 'Y', 'Z', 'F'], ['HNflag', 'HEflag', 'Zflag', 'Fscflag']):
                    di[f'{iagaCode}{i}'] = np.where(self.ds[flag].data == f, 99999.0, di[f'{iagaCode}{i}'])

        for c in [f'{iagaCode}X', f'{iagaCode}Y', f'{iagaCode}Z', f'{iagaCode}F']: 
            di[c] = di[c].apply(lambda x: format(x, '.2f').rjust(9))

            di['DOY'] = di.index.strftime('%j')
            di['DOY'] = di['DOY'].apply(lambda x: x.ljust(6))
            di['TIME'] = di.index.strftime("%H:%M:%S.000")
            di['DATE'] = di.index.strftime("%Y-%m-%d")

        # write to file

        with open(file, 'w') as writer:
            for lines in writeHeader:
                writer.writelines(lines)
            np.savetxt(writer, di[['DATE', 'TIME', 'DOY', f'{iagaCode}X', f'{iagaCode}Y', f'{iagaCode}Z', f'{iagaCode}F']].values, fmt = "%s")

__init__(datum='', code='', file='')

Class of daily cdf file

Object with data (xarray), date, station code and filename and directory.

Enter of datetime and station code or filename. Default directory is "/home/jovyan/leodata/data_cdf".

Parameters:

Name Type Description Default
datum datetime | str

Day of the cdf-file, by default ''

 ''
code str

Station code, by default ''

 ''
file Path | str

Cdf-file name to be read, by default ''

 ''

Raises:

Type Description
RunError

Incorrect input parameter.
FileError

Wrong filename format.
Source code in leocdf/classCDFmag.py 
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def __init__(self, datum: datetime | str = '', code: str = '', file: Path | str = ''):
    '''Class of daily cdf file

    Object with data (xarray), date, station code and filename and directory.\n
    Enter of datetime and station code or filename.
    Default directory is "/home/jovyan/leodata/data_cdf".

    Parameters
    ----------
    datum : datetime | str, optional
        Day of the cdf-file, by default ''
    code : str, optional
        Station code, by default ''
    file : Path | str, optional
        Cdf-file name to be read, by default ''

    Raises
    ------
    RunError
        Incorrect input parameter.
    FileError
        Wrong filename format.
    '''
    if datum == '' and code == '' and file == '':
        raise Exception("Error! Give datetime and code or a filename.") 

    else:
        if datum == '' and code =='':
            file_str = file.name if isinstance(file, Path) else file
            if len(file_str.replace('.cdf', '').split('_')) == 2:
                self.code = file_str.replace('.cdf', '').split('_')[0]
                self.datum = checkdateformat(file_str.replace('.cdf', '').split('_')[1])
            else:
                raise Exception("Error! Wrong filename format. CODE_YYYYMMDD.cdf") 
        else:
            self.datum = datum if isinstance(datum, datetime) else checkdateformat(datum)
            self.code = code

        self.file = Path(file) if isinstance(file, str) else file
        if self.file.parent == Path('.') and self.file == Path(''):
            self.cdfdir = Path(f"{CDFmag.defaultCDFdir}/{self.code}/cdf/{self.datum.strftime('%Y')}/")
        else:
            self.cdfdir = self.file.parent

        if self.file == Path(''):
            self.file = Path(f"{self.code}_{self.datum.strftime('%Y%m%d')}.cdf") 
        self.file = self.file.name

        # ds - xarray with data
        if Path(self.cdfdir/self.file).exists():
            self.ds = cdflib.xarray.cdf_to_xarray(self.cdfdir/self.file)
        else:
            self.ds = self.emptyCDF()

createIAGA(file, sec=1, flagDelete=-1)

Created ascii IAGA file. Only use definitiv values ( 0 < flags < 61).

Parameters:

Name Type Description Default
file str

Name of the new file

 required
sec int

Sampling rate in seconds, by default 1

 1
flagDelete int

Delete values with this flag, by default -1

 -1
Source code in leocdf/classCDFmag.py 
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
def createIAGA(self, file: str, sec: int = 1, flagDelete: int = -1) -> None:
    '''Created ascii IAGA file. Only use definitiv values ( 0 < flags < 61).  

    Parameters
    ----------
    file : str
        Name of the new file
    sec : int, optional
        Sampling rate in seconds, by default 1
    flagDelete : int, optional
        Delete values with this flag, by default -1
    '''

    #if file.exists():
    #    pass

    # iagaCode = loadIagaCode(self.code)
    iagaCode = 'JRU'

    # Header
    writeHeader = []

    header = readJson(CDFmag.iagaHeaderFile)

    if iagaCode in header:
        header = header[iagaCode]
    else:
        print(f"Error. No station code {iagaCode} in iaga-header.json.")

    headerKey = {'Format': 'IAGA-2002', 
                    'Source of Data': ' ', 
                    'Station Name': ' ', 
                    'IAGA Code': iagaCode, 
                    'Geodetic Latitude': ' ', 
                    'Geodetic Longitude': ' ', 
                    'Elevation': ' ', 
                    'Reported': ' ', 
                    'Sensor Orientation': ' ',
                    'Digital Sampling': '5 seconds', 
                    'Data Interval Type': '1-second spot readings', 
                    'Data Type': 'Provisional'} 

    for h in headerKey:
        if h in header:
            headerValue = header[h] if isinstance(header[h], str) else str(header[h])
        else:
            headerValue = headerKey[h] if isinstance(headerKey[h], str) else str(headerKey[h])
        writeHeader.append(f" {h.ljust(23)}{headerValue.ljust(45)}|\n")

    infoText = "Hallo"
    for i in range(0,3):
        writeHeader.append(f" # {infoText.ljust(65)} |\n")

    columnRow = ""
    for c, i in zip(['DATE', 'TIME', 'DOY', f"{iagaCode}X", f"{iagaCode}Y", f"{iagaCode}Z", f"{iagaCode}F"], 
        [11, 13, 8, 10, 10, 10, 7]):
        columnRow = columnRow + c.ljust(i)
    writeHeader.append(f"{columnRow}|\n")

    # Data values
    di = pd.DataFrame(index=[ CDFmag.julianday + timedelta(days = i) for i in self.ds['time'].data ])

    for i, flag, var in zip(['X', 'Y', 'Z', 'F'], ['HNflag', 'HEflag', 'Zflag', 'Fscflag'], ['HNvar', 'HEvar', 'Zvar', 'Fsc']):
        di[f'{iagaCode}{i}'] = np.where((self.ds[flag].data >= 0) & (self.ds[flag].data <= 60), 
                                            self.ds[var], 99999.0)

    if flagDelete != -1:
        for f in [flagDelete]:
            for i, flag in zip(['X', 'Y', 'Z', 'F'], ['HNflag', 'HEflag', 'Zflag', 'Fscflag']):
                di[f'{iagaCode}{i}'] = np.where(self.ds[flag].data == f, 99999.0, di[f'{iagaCode}{i}'])

    for c in [f'{iagaCode}X', f'{iagaCode}Y', f'{iagaCode}Z', f'{iagaCode}F']: 
        di[c] = di[c].apply(lambda x: format(x, '.2f').rjust(9))

        di['DOY'] = di.index.strftime('%j')
        di['DOY'] = di['DOY'].apply(lambda x: x.ljust(6))
        di['TIME'] = di.index.strftime("%H:%M:%S.000")
        di['DATE'] = di.index.strftime("%Y-%m-%d")

    # write to file

    with open(file, 'w') as writer:
        for lines in writeHeader:
            writer.writelines(lines)
        np.savetxt(writer, di[['DATE', 'TIME', 'DOY', f'{iagaCode}X', f'{iagaCode}Y', f'{iagaCode}Z', f'{iagaCode}F']].values, fmt = "%s")

emptyCDF()

Creates an empty xarray.Dataset with labeled xarray.DataArrays and attributes.

To-Do: pass title and header of cdf-file

Returns:

Name Type Description
ds DataSet

empty xarray.Dataset with labeled xarray.DataArrays and attributes
Source code in leocdf/classCDFmag.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def emptyCDF(self) -> xr.DataSet:
    '''Creates an empty xarray.Dataset with labeled xarray.DataArrays and attributes.\n
    To-Do: pass title and header of cdf-file

    Returns
    -------
    ds : xr.DataSet
        empty xarray.Dataset with labeled xarray.DataArrays and attributes 
    '''

    basetrig = pd.date_range(start=self.datum, periods=8, freq='3h', tz='UTC')
    basetrig = basetrig.append(basetrig).sort_values()

    time = pd.date_range(start=self.datum, periods=86400, freq='1s', tz='UTC')

    ds = xr.Dataset(
        data_vars=dict(
            H0=( ["record0"], np.zeros(16).astype(np.float64) ),
            D0=( ["record0"], np.zeros(16).astype(np.float64) ),
            Z0=( ["record0"], np.zeros(16).astype(np.float64) ),
            F0=( ["record0"], np.zeros(16).astype(np.float64) ),
            HNscv=( ["record0"], np.ones(16).astype(np.float64) ),
            HEscv=( ["record0"], np.ones(16).astype(np.float64) ),
            Zscv=( ["record0"], np.ones(16).astype(np.float64) ),
            Basetrig=( ["record0"], np.array(mjd2000(basetrig), dtype=np.float64) ),
            time=(["record1"], np.array(mjd2000(time), dtype=np.float64) ),
            HNvar=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
            HEvar=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
            Zvar=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
            T1=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
            T2=(["record1"], np.full(86400, np.nan).astype(np.float64) ),
            timeppm=( ["record2"], np.full(1, np.nan).astype(np.float64) ),
            timegps=( ["record2"], np.full(1, np.nan).astype(np.float64) ),
            timefge=( ["record2"], np.full(1, np.nan).astype(np.float64) ),
            Fsc=( ["record1"], np.full(86400, np.nan).astype(np.float64) ),
            HNflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
            HEflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
            Zflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
            Fscflag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
            FscQP=( ["record1"], np.ones(86400).astype(np.int8)*0 ),
            T1flag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
            T2flag=( ["record1"], np.ones(86400).astype(np.int8)*(-1) ),
            Timeerr= ( ["record2"], np.zeros(1).astype(np.float64) ),
            Timeerrtrig = ( ["record2"], np.array( [mjd2000(time[0])] ).astype(np.float64) ),
            filter_type = ( ["record3"], np.zeros(4).astype(np.uint16) ),
            filter_winlength = ( ["record3"], np.zeros(4).astype(np.uint16) ),
            filter_steplength = ( ["record3"], np.zeros(4).astype(np.uint16) )
        ),
        attrs=dict(TITLE="Magnetometer data.",
            HEADER="The header           ', 'has to be determined.",
            DATE=self.datum.strftime('%Y%m%d'),
            CODE=self.code
                )
    )

    return ds

plotCDF()

Plot raw values Zvar, HNvar, HEvar, Fsc

Source code in leocdf/classCDFmag.py 
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
def plotCDF(self) -> None:
    '''Plot raw values Zvar, HNvar, HEvar, Fsc
    '''
    # ueberfuehren in df
    df = pd.DataFrame(index=[ CDFmag.julianday + timedelta(days = i) for i in self.ds['Basetrig'].values ], 
                data = { 'Z0': self.ds['Z0'].values, 'Zscv': self.ds['Zscv'].values,
                  'H0': self.ds['H0'].values, 'HEscv': self.ds['HEscv'].values,
                  'D0': self.ds['D0'].values, 'HNscv': self.ds['HNscv'].values,
                  'F0': self.ds['F0'].values  })
    df = df.drop_duplicates()
    df = df.reindex( pd.date_range(start=df.index[0], periods=86400, freq='1s', tz='UTC') )
    df = df.ffill()

    df['Zscale'] = df['Z0'].values + df['Zscv'].values * self.ds['Zvar'].values
    df['HNscale'] = df['H0'].values + df['HNscv'].values * self.ds['HNvar'].values
    df['HEscale'] = df['HEscv'].values * self.ds['HEvar'].values
    df['Fscale'] = df['F0'].values + self.ds['Fsc'].values

    # plot 
    fig, ax = plt.subplots(1,figsize=(15, 5.5), layout="constrained")

    for var, lvar, c in zip(['Zscale', 'HNscale', 'HEscale', 'Fscale'], ['Zvar', 'HNvar', 'HEvar', 'Fsc'], ['k', 'b', 'g', 'r']):
        df[var] = df[var] - df[var].mean()  # mean abziehen
        df[var].plot(ax=ax, color=c, label=lvar, linewidth=0.5)
    ax.set_xlabel('time')
    ax.set_ylabel('variometer')
    ax.grid('on', which='minor', axis='x' )
    ax.grid('on', which='major', axis='y' )
    #ax.set_xlim( (df.index[0],df.index[-1]) )
    ax.legend()
    #ax.set_title(f"{self.code} - {self.datum.strftime('%Y-%m-%d')}")
    plt.show()

save(pathdir=None)

Saves the cdf-file under the default directory /home/jovyan/leodata/data_cdf.

Parameters:

Name Type Description Default
pathdir str | Path

Changes the directory, by default None

 None
Source code in leocdf/classCDFmag.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def save(self, pathdir: str | Path = None) -> None:
    '''Saves the cdf-file under the default directory */home/jovyan/leodata/data_cdf*. 

    Parameters
    ----------
    pathdir : str | Path, optional
        Changes the directory, by default None
    '''

    if pathdir == None:
        pathdir = self.cdfdir 
    elif isinstance(pathdir, str):
        pathdir = Path(pathdir)

    # check overwrite
    if Path(pathdir/self.file).exists():
        filename = questionoverwrite(Path(pathdir/self.file))
    else:
        filename = self.file
    att = readJson(CDFmag.zVarAttFile)

    zvarTuple = lib.createCdata( self.ds, att, [i for i in self.ds.data_vars] )
    # Convert to ctypes array
    arrayCtypes = (lib.zVariable * len(zvarTuple))(*zvarTuple)

    pathdir.mkdir(parents=True, exist_ok=True)

    # gesamter Pfad ohne .cdf

    #filename = f"{str(pathdir)}/{filename[:-4]}".encode('ASCII') # Anagbe ohne .cdf
    filename = lib.getCDFfilename(pathdir, filename)

    CDFmag.py_createcdf(filename, arrayCtypes, len(arrayCtypes))

setVar(arr, name)

Sets new data to variable.

Parameters:

Name Type Description Default
arr array

Array with new data.

 required
name str

Variable name receiving new data.

 required

Returns:

Type Description
bool

Successful update.

Raises:

Type Description
ArrayError

Array has not the right length.
Source code in leocdf/classCDFmag.py 
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
def setVar(self, arr: np.array, name: str) -> bool:
    '''Sets new data to variable. 

    Parameters
    ----------
    arr : np.array
        Array with new data.
    name : str
        Variable name receiving new data.

    Returns
    -------
    bool
        Successful update.

    Raises
    ------
    ArrayError
        Array has not the right length.
    '''

    # check array size
    if arr.shape != self.ds[ name ].data.shape:
        raise Exception(f"Array has not the right length. Shape {self.ds[ name ].data.shape} is used.") 
        return False

    self.ds[name].data = arr

    if name in CDFmag.flags.keys():
        i = np.argwhere( ~np.isnan( arr ) )
        self.ds[ CDFmag.flags[name] ].data[i] = 0

    return True

updateCDF(vargroup)

Updates the cdf-file. Only the submitted variables are updated.

Parameters:

Name Type Description Default
vargroup str | list

Variables to be updated.

 required

Raises:

Type Description
FileError

cdf-file doesn't exist.
VariableError

Variable name doesn't exist.
Source code in leocdf/classCDFmag.py 
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
def updateCDF(self, vargroup: str | list) -> None:
    '''Updates the cdf-file. Only the submitted variables are updated.

    Parameters
    ----------
    vargroup : str | list
        Variables to be updated.

    Raises
    ------
    FileError
        cdf-file doesn't exist.
    VariableError
        Variable name doesn't exist.
    '''

    # check files exist
    if not Path(self.cdfdir/self.file).exists():
        raise Exception(f"Error! File {self.cdfdir}/{self.file} doesn't exist.")
    # check zvar in CDFmag.varGroups

    att = readJson(CDFmag.zVarAttFile)

    if var in list(att.keys()):
        var = list(var)
    elif var == 'Baseline':
        var = ["H0", "D0", "Z0", "F0", "HNscv", "HEscv", "Zscv", "Basetrig"]
    elif var == 'data':
        var = ["time", "HNvar", "HEvar", "Zvar", "T1", "T2", "Fsc"]
    elif var == 'flags':
        var = ["HNflag", "HEflag", "Zflag", "Fscflag", "FscQP", "T1flag", "T2flag"]
    else:
        raise Exception(f"Error! {var} not valid.") 

    zvarTuple = lib.createCdata(self.ds, att, var)
    # Convert to ctypes array
    arrayCtypes = (lib.zVariable * len(zvarTuple))(*zvarTuple)

    #filename = f"{str(self.cdfdir)}/{self.file[:-4]}".encode('ASCII')
    filename = lib.getCDFfilename(self.cdfdir, self.file)
    CDFmag.py_updatecdf(filename, arrayCtypes, len(arrayCtypes))