tools/archiving_extraction/data_portioning.py

   1 #!/usr/bin/env python
   2 #
   3 import os, sys, time
   4
   5 from PyTango import *
   6
   7
   8
   9 #########################
  10 def hour2sec(hour_str):
  11     splitted_hour = hour_str.split(':')
  12     return float(splitted_hour[0])*3600 + float(splitted_hour[1])*60 + float(splitted_hour[2])
  13
  14
  15 #########################
  16 def str2sec(str):
  17     return time.mktime(time.strptime(str, "%Y-%m-%d %H:%M:%S"))
  18
  19
  20 #######################################
  21 def periodsStr2periodsSec(periods_str):
  22     periods_sec = []
  23
  24     for p in range (len(periods_str)):
  25
  26         startPeriod_sec = str2sec(periods_str[p][0])
  27         endPeriod_sec = str2sec(periods_str[p][1])
  28
  29         periods_sec.append( (startPeriod_sec, endPeriod_sec) )
  30
  31     return periods_sec
  32
  33
  34 #################
  35 def sec2str(sec):
  36     return time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(sec) )
  37
  38
  39 ########################################
  40 def periodsSec2periodsStr(periods_sec):
  41     periods_str = []
  42
  43     for p in range (len(periods_sec)):
  44
  45         startPeriod_str = sec2str(periods_sec[p][0])
  46         endPeriod_str = sec2str(periods_sec[p][1])
  47
  48         periods_str.append( (startPeriod_str, endPeriod_str) )
  49
  50     return periods_str
  51
  52
  53
  54 ####################################################
  55 def select_longIntervals(intervals_sec, minLength_sec):
  56
  57     long_intervals = []
  58     for i in intervals_sec:
  59         if i[1] - i[0] >= minLength_sec:
  60             long_intervals.append(i)
  61
  62     return long_intervals
  63
  64 ####################################################
  65 def split_longInterval(interval_str, maxLength_sec):
  66
  67     interval_sec = periodsStr2periodsSec([interval_str])[0]
  68     interval_length = interval_sec[1] - interval_sec[0]
  69
  70     if interval_length <= maxLength_sec:
  71         return [interval_str]
  72
  73     times = int(interval_length/maxLength_sec)
  74     rest = interval_length - times*maxLength_sec
  75
  76     shorter_intervals = []
  77     for i in range (times-1):
  78         shorter_intervals.append((interval_sec[0] + maxLength_sec*i, interval_sec[0]+ (i+1)*maxLength_sec))
  79
  80     if rest < float(maxLength_sec)/3:
  81         shorter_intervals.append((interval_sec[0]+ maxLength_sec*(times-1), interval_sec[0]+ maxLength_sec*times+ rest))
  82     else:
  83         shorter_intervals.append((interval_sec[0]+ maxLength_sec*(times-1), interval_sec[0]+ maxLength_sec*times))
  84         shorter_intervals.append((interval_sec[0]+ maxLength_sec*times, interval_sec[0]+ maxLength_sec*times+ rest))
  85
  86     return periodsSec2periodsStr(shorter_intervals)
  87
  88
  89 #################################################################################
  90 def get_okIntervals_indxs(param, param_data):
  91
  92     if len(param_data) == 0:
  93         return [] # it is not really correct but we are not supposed to get here
  94
  95     #timeInterval_sec = periodsStr2periodsSec([timeInterval_str])[0]
  96
  97     curr_startInx = -1
  98     curr_endInx = -1
  99     ok_intervals = []
 100     was_ok = False
 101
 102     for i in range(len(param_data)):
 103
 104         ok = param.is_ok(param_data[i][1])
 105         if ok and not was_ok:
 106             curr_startInx = i
 107             was_ok = True
 108         elif not ok and was_ok:
 109             curr_endInx = i-1
 110             was_ok = False
 111             ok_intervals.append((curr_startInx, curr_endInx))
 112
 113     #if curr_startInx != -1 and curr_endInx == -1:
 114     if curr_startInx > curr_endInx:
 115         ok_intervals.append((curr_startInx, len(param_data)-1))
 116
 117     #print ok_intervals # debug
 118
 119 ##    if len(ok_intervals) == 0:
 120 ##        return []
 121
 122     return ok_intervals
 123
 124
 125 #################################################################################
 126 def get_extraPeriods(param, param_data, timeInterval_str, minLength_sec = 0):
 127
 128
 129     okIntervals_indxs = get_okIntervals_indxs(param, param_data)
 130
 131     if len(okIntervals_indxs) == 0:
 132         return []
 133
 134     subIntervals_sec = []
 135
 136     timeInterval_sec = periodsStr2periodsSec([timeInterval_str])[0]
 137
 138     if okIntervals_indxs[0][0] == 0:
 139         first_start = timeInterval_sec[0]
 140     else: first_start = param_data[okIntervals_indxs[0][0]][0]
 141
 142     #print sec2str(first_start) # debug
 143
 144     if okIntervals_indxs[-1][1] == len(param_data)-1:
 145         last_end = timeInterval_sec[1]
 146     else: last_end = param_data[okIntervals_indxs[-1][1]][0]
 147
 148     #print sec2str(last_end) # debug
 149
 150     if len(okIntervals_indxs) == 1:
 151         subIntervals_sec.append( (first_start,  last_end) )
 152     else:
 153         subIntervals_sec.append((first_start,  param_data[okIntervals_indxs[0][1]][0]))
 154         for i in range(1, len(okIntervals_indxs)-1):
 155             subIntervals_sec.append((param_data[okIntervals_indxs[i][0]][0],  param_data[okIntervals_indxs[i][1]][0]))
 156         subIntervals_sec.append((param_data[okIntervals_indxs[-1][0]][0],  last_end))
 157
 158
 159
 160     return periodsSec2periodsStr( select_longIntervals(subIntervals_sec, minLength_sec) )
 161
 162
 163 #################################################################################
 164 def get_steadyPeriods(param, param_data, timeInterval_str, minLength_sec = 0):
 165
 166     #subIntervals_sec = periodsStr2periodsSec([timeInterval_str])
 167
 168     var_data = param.get_variation(param_data, periodsStr2periodsSec([timeInterval_str])[0])
 169
 170     okIntervals_indxs = get_okIntervals_indxs(param, var_data)
 171
 172     subIntervals_sec = []
 173
 174     for i in okIntervals_indxs:
 175         subIntervals_sec.append( (var_data[i[0]][0][0], var_data[i[1]][0][1]) )
 176
 177     return periodsSec2periodsStr( select_longIntervals(subIntervals_sec, minLength_sec) )
 178
 179
 180
 181 #################################################################################
 182 def takeWholeOrEmpty_extendingExtraction(param, timeInterval_str, extractor, dest_dir):
 183     # ones called it would not work as we don't know startShift_sec !!!!!!!
 184     # so let's take all
 185     print "WARNING: data_portionning.takeWholeOrEmpty_extendingExtraction called"
 186     print param.name
 187     print timeInterval_str
 188
 189     return [timeInterval_str]
 190
 191
 192     if ( str2sec(timeInterval_str[1]) - str2sec(timeInterval_str[0]) <=
 193              float(param.archiving_period)/1000 ):
 194
 195             extended_timeInterval_sec = ( str2sec(timeInterval_str[1])- float(param.archiving_period)/1000,
 196                                           str2sec(timeInterval_str[1]) )
 197
 198
 199
 200             param_data = extractAttr_relativeTime( param.attr_fullName, startShift_sec,
 201                                                    periodsSec2periodsStr([extended_timeInterval_sec])[0],
 202                                                    extractor, dest_dir )
 203
 204             if len(param_data) == 0:
 205                 print "No data for ", param.attr_fullName, " in the ", db, \
 206                       " for the period ", periodsSec2periodsStr([extended_timeInterval_sec])[0]
 207                 sys.exit(0)
 208
 209             if param.is_ok(param_data[-1][1]):
 210                 return [timeInterval_str]      # whole interval
 211             else:
 212                 return []                      # empty list
 213     else:
 214         print "No data for ", param.attr_fullName, " in the ", db, \
 215               " for the period ", timeInterval_str
 216         sys.exit(0)
 217
 218
 219 ##################################################
 220 def get_dataPortion(data, timeInterval_sec):
 221
 222     if timeInterval_sec[0] < data[0][0] or timeInterval_sec[1] > data[len(data)-1][0]:
 223         print "Invalid parameters in data_portionnig.get_dataPortion"
 224         sys.exit(0)
 225
 226     i =0
 227     while(data[i][0] < timeInterval_sec[0]):
 228         i +=1
 229     start_inx = i
 230
 231     while(data[i][0] <= timeInterval_sec[1]):
 232         i += 1
 233     end_inx = i
 234
 235     return data[start_inx:end_inx]
 236
 237
 238 #################################################################################
 239 def takeWholeOrEmpty_extendingTime(param, timeInterval_str, param_data):
 240
 241     if ( str2sec(timeInterval_str[1]) - str2sec(timeInterval_str[0]) <=
 242              float(param.archiving_period)/1000 ):
 243
 244             extended_timeInterval_sec = ( max( str2sec(timeInterval_str[1])- float(param.archiving_period)/1000,  param_data[0][0]),
 245                                           str2sec(timeInterval_str[1]) )
 246
 247             # take  the corresponding portion of param_data
 248             data_portion = get_dataPortion(param_data, extended_timeInterval_sec)
 249
 250             if len(data_portion) == 0:
 251                 print "No data for ", param.attr_fullName, " in the ", db, \
 252                       " for the period ", periodsSec2periodsStr([extended_timeInterval_sec])[0]
 253                 sys.exit(0)
 254
 255             if param.is_ok(data_portion[-1][1]):
 256                 return [timeInterval_str]      # whole interval
 257             else:
 258                 return []                      # empty list
 259     else:
 260         print "No data for ", param.attr_fullName, " in the ", db, \
 261               " for the period ", timeInterval_str
 262         sys.exit(0)
 263
 264
 265 ##############################################
 266 def split_paramData(param_data, periods_str):
 267
 268     data_piecewise = []
 269
 270     periods_sec = periodsStr2periodsSec(periods_str)
 271
 272     i =0
 273
 274     for period in periods_sec:
 275         #print period # debug
 276
 277 ##        if period[0] < param_data[0][0] or period[1] > param_data[len(param_data)-1][0]:
 278 ##            print "Invalid parameters in data_portionnig.split_paramData"
 279 ##            sys.exit(0)
 280
 281         if period[0] < param_data[0][0] or period[1] > param_data[len(param_data)-1][0]:
 282             return [param_data]
 283
 284         while(param_data[i][0] < period[0]):
 285             i +=1
 286         start_inx = i
 287
 288         while(param_data[i][0] <= period[1]):
 289             i += 1
 290         end_inx = i
 291
 292         data_piecewise.append(param_data[start_inx:end_inx])
 293
 294     return data_piecewise
 295
 296 ####################################################
 297 ##def split_longPeriod( startTime_str, endTime_str ):
 298
 299 ##    #start_time = str2time(startTime_str)
 300 ##    #end_time = str2time(endTime_str)
 301 ##    period_length = str2time(endTime_str) - str2time(startTime_str)
 302
 303 ##    fourHours = 4*3600
 304 ##    threeHours = 3*3600
 305
 306 ##    sub_periods = []
 307 ##    if period_length <= fourHours:
 308 ##        sub_periods.append( (startTime_str, endTime_str) )
 309 ##    else:
 310 ##        nb_perodsBy4hours = int(period_length)/fourHours
 311 ##        rest4 = period_length - nb_perodsBy4hours*fourHours
 312
 313 ##        nb_perodsBy3hours = int(period_length)/threeHours
 314 ##        rest3 = period_length - nb_perodsBy3hours*threeHours
 315
 316
 317 ##        startTime_i_tuple = time.strptime(startTime_str, "%Y-%m-%d %H:%M:%S")
 318 ##        endTime_i_tuple = startTime_i_tuple
 319
 320 ##        if fourHours-rest4 < threeHours-rest3:
 321 ##            endTime_i_tuple[3] += 4
 322 ##            for i in range (1, nb_perodsBy4hours):
 323 ##                startTime_i_str = strftime("%Y-%m-%d %H:%M:%S", startTime_i_tuple)
 324 ##                endTime_i_str = strftime("%Y-%m-%d %H:%M:%S", endTime_i_tuple)
 325 ##                sub_periods.append((startTime_i_str, endTime_i_str))
 326 ##                startTime_i_tuple[3] += 4
 327 ##                endTime_i_tuple[3] += 4
 328 ##            startTime_i_str = strftime("%Y-%m-%d %H:%M:%S", startTime_i_tuple)
 329 ##            sub_periods.append((startTime_i_str, endTime_str))
 330 ##        else:
 331 ##            endTime_i_tuple[3] += 3
 332 ##            for i in range (1, nb_perodsBy3hours):
 333 ##                startTime_i_str = strftime("%Y-%m-%d %H:%M:%S", startTime_i_tuple)
 334 ##                endTime_i_str = strftime("%Y-%m-%d %H:%M:%S", endTime_i_tuple)
 335 ##                sub_periods.append((startTime_i_str, endTime_i_str))
 336 ##                startTime_i_tuple[3] += 3
 337 ##                endTime_i_tuple[3] += 3
 338 ##            startTime_i_str = strftime("%Y-%m-%d %H:%M:%S", startTime_i_tuple)
 339 ##            sub_periods.append((startTime_i_str, endTime_str))
 340
 341 ##    return sub_periods
 342
 343 ##################################################################################################
 344
 345 ##################################################################################################
 346 if __name__ == '__main__':
 347
 348
 349     if len(sys.argv) < 4:
 350         print "Correct use: " + sys.argv[0] + " archi_filename  startTime_str ('Y-m-d H:M:S')  endTime_str ('Y-m-d H:M:S')  dest_dir [hdb/tdb]"
 351         sys.exit(0)
 352
 353     archi_filename = sys.argv[1]
 354     startTime_str = sys.argv[2]
 355     endTime_str = sys.argv[3]
 356     dest_dir = sys.argv[4]
 357
 358
 359     db = "TDB"
 360     if len(sys.argv) == 6 and (sys.argv[5] == "hdb" or sys.argv[5] == "HDB"):
 361         extractor = DeviceProxy("archiving/hdb/hdbextractor.1")
 362         db = "HDB"
 363     else:
 364         extractor = DeviceProxy("archiving/tdb/tdbextractor.1")
 365     #print db #debug
 366
 367
 368     extractor.set_timeout_millis(100000)
 369
 370     # TEST 1 -------------------------------------------
 371     attr_list = reading_txtac.attrs_fromDevList(archi_filename)
 372     #print attr_list # DEBUG
 373     date_string = startTime_str[:10]
 374     startHour_string = startTime_str[11:].replace(':', '.')
 375     endHour_string = endTime_str[11:].replace(':', '.')
 376
 377     baseOutputName = dest_dir + os.sep + date_string + '_' +  startHour_string + '-' + endHour_string + '_'
 378
 379     extractData_fromTimeToTime( attr_list, startTime_str, endTime_str, extractor, baseOutputName)
 380
 381
 382     # TEST 2 -------------------------------------------
 383     #extractData_archivedPeriods(archi_filename,  startTime_str, endTime_str, extractor, dest_dir)
 384
 385