src/make_zip.py

   1 from __future__ import absolute_import
   2 from __future__ import print_function
   3 import os.path as osp
   4 import glob
   5 import json
   6 import sys
   7 import logging
   8 from utils import load_sys_cfg
   9 import posixpath as pxp
  10 import requests
  11 import subprocess
  12
  13 sys_cfg = load_sys_cfg()
  14 sys_cfg.sims_path = 'fdds/simulations'
  15 sys_cfg.sims_url_path = 'simulations'
  16
  17 def make_zip(job_id):
  18     """
  19     Create ZIP file from visualization stored in wrfxweb.
  20     :param job_id: string, the name of job directory
  21     """
  22
  23     logging.info('make_zip: job_id=%s' % job_id)
  24     job_path = osp.join(osp.abspath(sys_cfg.sims_path),job_id)
  25     url_prefix = pxp.join(sys_cfg.url_root,sys_cfg.sims_url_path,job_id)
  26     logging.debug('make_zip: job_path %s' % job_path)
  27     logging.debug('make_zip: url_prefix %s' % url_prefix)
  28
  29     # read the catalog and the manifest
  30     cat_path = osp.join(job_path,'catalog.json')
  31     cat = json.load(open(cat_path))
  32     if job_id not in cat:
  33          logging.error('job id %s not in the catalog' % job_id)
  34          sys.exit(1)
  35     cat_entry = cat[job_id]
  36     mf = json.load(open(osp.join(sys_cfg.sims_path,cat_entry['manifest_path'])))
  37
  38     paths = [fn for fn in glob.glob(osp.join(job_path,'*.csv'))]
  39     logging.info('make_zip: files to compress %s' % paths)
  40
  41     description = cat_entry['description']
  42     logging.info('make_zip: job description: %s' % description)
  43
  44     # build output file
  45     zip_filename = job_id + '.zip'
  46     zip_path = osp.join(job_path,zip_filename)
  47     logging.info('make_zip: creating file %s' % zip_path)
  48     command = ['zip -jr', zip_path] + paths
  49     logging.info('make_zip: command - %s' % ' '.join(command))
  50     subprocess.call(' '.join(command),shell=True)
  51     url = pxp.join(url_prefix,zip_filename)
  52     logging.info('make_zip: file created at %s' % url)
  53     try:
  54         r = requests.get(url, stream=True)
  55         content_size = int(r.headers['Content-Length'])
  56         logging.info('make_zip: file size is %s' % content_size)
  57         cat[job_id]['zip_url']=url
  58         cat[job_id]['zip_size']=content_size
  59         json.dump(cat, open(cat_path,'w'), indent=4, separators=(',', ': '))
  60     except:
  61         logging.warning('make_zip: accessing the file over the web failed')
  62
  63
  64 if __name__ == '__main__':
  65
  66     sys.argv
  67
  68     if len(sys.argv) != 2:
  69         print('usage: make_zip.sh job_id')
  70         print('job_id: the name of job directory in ' + sys_cfg.sims_path)
  71         sys.exit(1)
  72
  73     logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
  74
  75     job_id = sys.argv[1]
  76     make_zip(job_id)
  77