process.py

   1 # General python for any case
   2 from JPSSD import read_fire_mesh, retrieve_af_data, sdata2json, json2kml, time_iso2num
   3 from interpolation import sort_dates
   4 from setup import process_satellite_detections
   5 from svm import preprocess_data_svm, SVM3
   6 from contline import get_contour_verts
   7 from contour2kml import contour2kml
   8 import saveload as sl
   9 from scipy.io import loadmat, savemat
  10 import numpy as np
  11 import datetime as dt
  12 import sys
  13 import os
  14 from time import time
  15
  16 satellite_file = 'data'
  17 fire_file = 'fire_detections.kml'
  18 ground_file = 'nofire.kml'
  19 bounds_file = 'result.mat'
  20 svm_file = 'svm.mat'
  21 contour_file = 'perimeters_svm.kml'
  22
  23 def exist(path):
  24         return (os.path.isfile(path) and os.access(path,os.R_OK))
  25
  26 satellite_exists = exist(satellite_file)
  27 fire_exists = exist(fire_file)
  28 ground_exists = exist(ground_file)
  29 bounds_exists = exist(bounds_file)
  30
  31 if len(sys.argv) != 4 and (not bounds_exists) and (not satellite_exists):
  32         print 'Error: python %s wrfout start_time days' % sys.argv[0]
  33         print ' * wrfout - string, wrfout file of WRF-SFIRE simulation'
  34         print ' * start_time - string, YYYYMMDDHHMMSS where: '
  35         print '         YYYY - year'
  36         print '         MM - month'
  37         print '         DD - day'
  38         print '         HH - hour'
  39         print '         MM - minute'
  40         print '         SS - second'
  41         print ' * days - float, number of days of simulation (can be less than a day)'
  42         sys.exit(0)
  43
  44 t_init = time()
  45
  46 print ''
  47 if bounds_exists:
  48         print '>> File %s already created! Skipping all satellite processing <<' % bounds_file
  49         print 'Loading from %s...' % bounds_file
  50         result = loadmat(bounds_file)
  51         # Taking necessary variables from result dictionary
  52         scale = result['time_scale_num'][0]
  53         time_num_granules = result['time_num_granules'][0]
  54         time_num_interval = result['time_num'][0]
  55 else:
  56         if satellite_exists:
  57                 print '>> File %s already created! Skipping satellite retrieval <<' % satellite_file
  58                 print 'Loading from %s...' % satellite_file
  59                 data,fxlon,fxlat,time_num = sl.load(satellite_file)
  60                 bbox = [fxlon.min(),fxlon.max(),fxlat.min(),fxlat.max()]
  61         else:
  62                 print '>> Reading the fire mesh <<'
  63                 sys.stdout.flush()
  64                 fxlon,fxlat,bbox,time_esmf = read_fire_mesh(sys.argv[1])
  65                 # converting times to ISO
  66                 dti = dt.datetime.strptime(sys.argv[2],'%Y%m%d%H%M%S')
  67                 time_start_iso = '%d-%02d-%02dT%02d:%02d:%02dZ' % (dti.year,dti.month,dti.day,dti.hour,dti.minute,dti.second)
  68                 dtf = dti+dt.timedelta(days=float(sys.argv[3]))
  69                 time_final_iso = '%d-%02d-%02dT%02d:%02d:%02dZ' % (dtf.year,dtf.month,dtf.day,dtf.hour,dtf.minute,dtf.second)
  70                 time_iso = (time_start_iso,time_final_iso)
  71
  72                 print ''
  73                 print '>> Retrieving satellite data <<'
  74                 sys.stdout.flush()
  75                 data = retrieve_af_data(bbox,time_iso)
  76
  77                 print ''
  78                 print '>> Saving satellite data file (data) <<'
  79                 sys.stdout.flush()
  80                 time_num = map(time_iso2num,time_iso)
  81                 sl.save((data,fxlon,fxlat,time_num),satellite_file)
  82                 print 'data file saved correctly!'
  83
  84         print ''
  85         if (not fire_exists) or (not ground_exists):
  86                 print '>> Generating KML of fire and ground detections <<'
  87                 sys.stdout.flush()
  88                 # sort the granules by dates
  89                 sdata=sort_dates(data)
  90                 tt = [ dd[1]['time_num'] for dd in sdata ]  # array of times
  91         if fire_exists:
  92                 print '>> File %s already created! <<' % fire_file
  93         else:
  94                 # writting fire detections file
  95                 print 'writting KML with fire detections'
  96                 keys = ['latitude','longitude','brightness','scan','track','acq_date','acq_time','satellite','instrument','confidence','bright_t31','frp','scan_angle']
  97                 dkeys = ['lat_fire','lon_fire','brig_fire','scan_fire','track_fire','acq_date','acq_time','sat_fire','instrument','conf_fire','t31_fire','frp_fire','scan_angle_fire']
  98                 prods = {'AF':'Active Fires','FRP':'Fire Radiative Power'}
  99                 N = [len(d[1]['lat_fire']) for d in sdata]
 100                 json = sdata2json(sdata,keys,dkeys,N)
 101                 json2kml(json,fire_file,bbox,prods)
 102         if ground_exists:
 103                 print ''
 104                 print '>> File %s already created! <<' % ground_file
 105         else:
 106                 # writting ground detections file
 107                 print 'writting KML with ground'
 108                 keys = ['latitude','longitude','scan','track','acq_date','acq_time','satellite','instrument','scan_angle']
 109                 dkeys = ['lat_nofire','lon_nofire','scan_nofire','track_nofire','acq_date','acq_time','sat_fire','instrument','scan_angle_nofire']
 110                 prods = {'NF':'No Fire'}
 111                 N = [len(d[1]['lat_nofire']) for d in sdata]
 112                 json = sdata2json(sdata,keys,dkeys,N)
 113                 json2kml(json,ground_file,bbox,prods)
 114
 115         print ''
 116         print '>> Processing satellite data <<'
 117         sys.stdout.flush()
 118         result = process_satellite_detections(data,fxlon,fxlat,time_num)
 119         # Taking necessary variables from result dictionary
 120         scale = result['time_scale_num']
 121         time_num_granules = result['time_num_granules']
 122         time_num_interval = result['time_num']
 123
 124 lon = result['fxlon']
 125 lat = result['fxlat']
 126 U = np.array(result['U']).astype(float)
 127 L = np.array(result['L']).astype(float)
 128 T = np.array(result['T']).astype(float)
 129
 130 print ''
 131 print '>> Preprocessing the data <<'
 132 sys.stdout.flush()
 133 X,y = preprocess_data_svm(lon,lat,U,L,T,scale,time_num_granules)
 134
 135 print ''
 136 print '>> Running Support Vector Machine <<'
 137 sys.stdout.flush()
 138 C = 10.
 139 kgam = 10.
 140 F = SVM3(X,y,C=C,kgam=kgam,fire_grid=(lon,lat))
 141
 142 print ''
 143 print '>> Saving the results <<'
 144 sys.stdout.flush()
 145 tscale = 24*3600
 146 # Fire arrival time in seconds from the begining of the simulation
 147 tign_g = F[2]*float(tscale)+scale[0]-time_num_interval[0]
 148 # Creating the dictionary with the results
 149 svm = {'dxlon': lon, 'dxlat': lat, 'U': U/tscale, 'L': L/tscale,
 150         'fxlon': F[0], 'fxlat': F[1], 'Z': F[2],
 151         'tign_g': tign_g,
 152         'tscale': tscale, 'time_num_granules': time_num_granules,
 153         'time_scale_num': scale, 'time_num': time_num_interval}
 154 # Save resulting file
 155 savemat(svm_file, mdict=svm)
 156 print 'The results are saved in svm.mat file'
 157
 158 print ''
 159 print '>> Computing contour lines of the fire arrival time <<'
 160 print 'Computing the contours...'
 161 # Fire arrival time in seconds from an old date
 162 Z = F[2]*tscale+scale[0]
 163 # Granules numeric times
 164 contour_data = get_contour_verts(F[0], F[1], Z, time_num_granules, contour_dt_hours=6, contour_dt_init=6, contour_dt_final=6)
 165 print 'Creating the KML file...'
 166 # Creating the KML file
 167 contour2kml(contour_data,contour_file)
 168 print 'The resulting contour lines are saved in perimeters_svm.kml file'
 169
 170 print ''
 171 print '>> DONE <<'
 172 t_final = time()
 173 print 'Elapsed time for all the process: %ss.' % str(abs(t_final-t_init))
 174 sys.exit()