src/misc/series_mpi.py

   1 #!/usr/bin/env mpipython
   2
   3
   4 #   gpiv_series - Processes a set of numbered input data
   5
   6 #   Copyright (C) 2008 Gerber van der Graaf
   7
   8 #   This program is free software; you can redistribute it and/or modify
   9 #   it under the terms of the GNU General Public License as published by
  10 #   the Free Software Foundation; either version 2, or (at your option)
  11 #   any later version.
  12
  13 #   This program is distributed in the hope that it will be useful,
  14 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #   GNU General Public License for more details.
  17
  18 #   You should have received a copy of the GNU General Public License
  19 #   along with this program; if not, write to the Free Software Foundation,
  20 #   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  21
  22
  23 #--------------------------------------------------------------------
  24
  25 #
  26 # This version is MPI enabled for parallel processing
  27 #
  28 import os, re
  29 from Scientific import MPI
  30 communicator = MPI.world.duplicate()
  31
  32 #
  33 #----------- Command line arguments parser
  34 #
  35 from optparse import OptionParser
  36
  37 usage = "%prog [options] \"process\""
  38 parser = OptionParser(usage)
  39 parser.add_option("-a", "--arg_n",
  40                   action="store_true", dest="arg_n", default=False,
  41                   help="ignore composing of the numbered filename.")
  42 parser.add_option("-b", "--basename", type='string', dest="basename",
  43                   help="File basename for reading", metavar="FILE")
  44 parser.add_option("-e", "--ext", type='string', dest="ext", metavar="EXT",
  45                   help="add an extension after the file basename + number (without leading \".\")")
  46 parser.add_option("-f", "--first", type='int', dest="first_nr", default=0,
  47                   help="first numbered file (default: 0)", metavar="N")
  48 parser.add_option("-l", "--last", type='int', dest="last_nr", default=0,
  49                   help="last numbered file(default: 0)", metavar="N")
  50 parser.add_option("-i", "--incr", type='int', dest="incr_nr", default=1,
  51                   help="increment file number (default: 1)", metavar="N")
  52 parser.add_option("-p", "--print",
  53                   action="store_true", dest="pri", default=False,
  54                   help="prints process parameters/variables to stdout")
  55 parser.add_option("--pad", type='int', dest="pad0", default=0,
  56                   help="padding number with zero's (default: 0)", metavar="N")
  57 parser.add_option("-n", "--none",
  58                   action="store_true", dest="none", default=False,
  59                   help="suppresses real execution")
  60 parser.add_option("-x", "--prefix", action="store_true", dest="prefix", default=False,
  61                   help="prefix numbering to file basename")
  62
  63 (options, args) = parser.parse_args()
  64 if len(args) != 1:
  65         parser.error("incorrect number of arguments")
  66 else:
  67         process = args[0]
  68
  69 if not "FNAME" in process:
  70         parser.error("process does not contain the required string 'FNAME'")
  71
  72 #
  73 #----------- Function definitions
  74 #
  75 def pri_date(msg = "Time stamp at start of series processing:"):
  76         """Prints time stamp.
  77
  78         Keyword arguments:
  79         msg -- message to be printed before time stamp
  80         """
  81         if options.pri == True:
  82                 print msg
  83                 os.system('date')
  84         elif options.none:
  85                 print msg
  86                 os.system('date')
  87
  88
  89
  90 def count_digits(nr):
  91         """Counts number of digits from a number
  92
  93         Keyword arguments:
  94         nr -- number to be questioned
  95         """
  96         count=0
  97         while nr/10 !=0:
  98                 nr=nr/10
  99                 count=count+1
 100         return count
 101
 102
 103 def pad0(nr):
 104         """Created a string for zero padding
 105
 106         Keyword arguments:
 107         nr -- number of zeros to be padded
 108         """
 109         pd0=""
 110         for i in range(0, nr):
 111                 pd0 = str(pd0)+"0"
 112
 113         return pd0
 114
 115
 116 def compose_name_nr(name, nr, ext):
 117         """Creates proper name from basename and number.
 118
 119         Keyword arguments:
 120         nr -- number of filename to be processed
 121         """
 122         if options.pad0 > 0:
 123                 ndig = count_digits(nr)
 124                 null_str = pad0(options.pad0 - ndig)
 125                 nr_str = null_str+str(nr)
 126         else:
 127                 nr_str = str(nr)
 128
 129         if options.prefix:
 130                 if options.arg_n:
 131                         name=str(name)
 132                 else:
 133                         name=nr_str+str(name)
 134         else:
 135                 if options.arg_n:
 136                         name=str(name)
 137                 else:
 138                         name=str(name)+nr_str
 139
 140         if str(ext) != "None":
 141                 name=str(name)+str(".")+str(options.ext)
 142
 143         return(name)
 144
 145
 146 def compose_cmd(name, nr, ext):
 147         """Creates proper command.
 148
 149         Keyword arguments:
 150         name -- complete filename
 151         """
 152         command=str(process)
 153
 154         # Eventually, substitutes "-f" with: "nr -f"
 155         #if options.arg_n:
 156         #       if "-f" in command:
 157         #               command=re.sub("-f", str(nr)+" -f", command)
 158         #       else:
 159         #               command=re.sub("FNAME", str(nr)+" FNAME", command)
 160
 161         # Substitutes macros
 162         command=re.sub("FNAME", str(name), str(command))
 163         command=re.sub("NR", str(nr), str(command))
 164         command=re.sub("EXT", str(ext), str(command))
 165
 166         return command
 167
 168
 169 def proc_series_par():
 170         """Processes a series on identic numbered files in parallel
 171         environment using mpipython.
 172
 173         """
 174         #
 175         # Total number of data and number of data per node
 176         #
 177         Nt = (options.last_nr+1 - options.first_nr) / options.incr_nr
 178         Nn = Nt / communicator.size
 179
 180         #
 181         # Bail out if number of nodes is larger than number of data
 182         #
 183         if communicator.size > Nt:
 184                 print 'nprocs (',communicator.size,\
 185                       ') larger then number of data (',Nt,")"
 186                 exit()
 187
 188
 189         for i in range(options.first_nr, options.last_nr+1, options.incr_nr):
 190                 for j in range(0, communicator.size, 1):
 191                         if communicator.rank == j:
 192                                 if i >= options.first_nr + j*Nn*options.incr_nr:
 193                                         if i < options.first_nr + (j+1)*Nn*options.incr_nr:
 194                                                 name_nr = compose_name_nr(i)
 195                                                 command = compose_cmd(name_nr, i)
 196                                                 if options.pri == True: print "rank=",communicator.rank,command
 197                                                 elif options.none == True: print "rank=",communicator.rank,command
 198                                                 if options.none == False: os.system(command)
 199
 200                                 #
 201                                 # The fraction of data left, if Nn is not an integer,
 202                                 # is processed at the highest node
 203                                 #
 204                                 if j == communicator.size - 1:
 205                                         if i >=  options.first_nr + (j+1)*Nn*options.incr_nr:
 206                                                 name_nr = compose_name_nr(options.basename, i, options.ext)
 207                                                 command = compose_cmd(name_nr, i, options.ext)
 208                                                 if options.pri == True: print "rank=",communicator.rank,command
 209                                                 elif options.none == True: print "rank=",communicator.rank,command
 210                                                 if options.none == False: os.system(command)
 211
 212
 213 #
 214 #----------- Calling functions
 215 #
 216 #if communicator.rank == 0:
 217 #       if options.pri == True: pri_date()
 218 #       elif options.none == True: pri_date()
 219
 220 proc_series_par()
 221
 222 #communicator.rank == 0:
 223 #       if options.pri == True: pri_date(msg = "Time stamp at end of series processing:")
 224 #       elif options.none == True: pri_date(msg = "Time stamp at end of series processing:")
 225
 226 #
 227 #----------- That's all folks
 228 #