src/join_catalog.py

   1 from __future__ import absolute_import
   2 import json,glob,logging
   3 import os.path as osp
   4 import collections
   5 from utils import lock
   6
   7
   8 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
   9
  10 root=osp.abspath('fdds/simulations')
  11 files = glob.glob(osp.join(root,'*/catalog.json'))
  12 catalog_path=osp.join(root,'catalog.json')
  13 lock_path=catalog_path + '.lock'
  14
  15 catalog = {}
  16 for f in files:
  17     try:
  18         f_json = json.load(open(f))
  19     except:
  20         logging.error('loading file %s failed, ignoring' % f)
  21     catalog.update(f_json)
  22
  23 # add blank fields if not present
  24 for i in catalog:
  25     catalog[i]['processed_utc']=catalog[i].get('processed_utc',None)
  26     catalog[i]['run_utc']=catalog[i].get('run_utc',None)
  27     catalog[i]['kml_url']=catalog[i].get('kml_url',None)
  28     catalog[i]['kml_size']=catalog[i].get('kml_size',None)
  29     catalog[i]['zip_url']=catalog[i].get('zip_url',None)
  30     catalog[i]['zip_size']=catalog[i].get('zip_size',None)
  31     catalog[i]['job_id']=i
  32
  33 catalog_sorted=collections.OrderedDict(sorted(list(catalog.items()), reverse=True))
  34
  35 l=lock(lock_path)
  36 l.acquire()
  37 json.dump(catalog_sorted, open(catalog_path, 'w'), indent=1, separators=(',',':'))
  38 l.release()
  39
  40 logging.info('Created catalog at %s',catalog_path)
  41
  42