Lib/pipes.py

   1 """Conversion pipeline templates.
   2
   3 The problem:
   4 ------------
   5
   6 Suppose you have some data that you want to convert to another format
   7 (e.g. from GIF image format to PPM image format).  Maybe the
   8 conversion involves several steps (e.g. piping it through compress or
   9 uuencode).  Some of the conversion steps may require that their input
  10 is a disk file, others may be able to read standard input; similar for
  11 their output.  The input to the entire conversion may also be read
  12 from a disk file or from an open file, and similar for its output.
  13
  14 The module lets you construct a pipeline template by sticking one or
  15 more conversion steps together.  It will take care of creating and
  16 removing temporary files if they are necessary to hold intermediate
  17 data.  You can then use the template to do conversions from many
  18 different sources to many different destinations.  The temporary
  19 file names used are different each time the template is used.
  20
  21 The templates are objects so you can create templates for many
  22 different conversion steps and store them in a dictionary, for
  23 instance.
  24
  25
  26 Directions:
  27 -----------
  28
  29 To create a template:
  30     t = Template()
  31
  32 To add a conversion step to a template:
  33    t.append(command, kind)
  34 where kind is a string of two characters: the first is '-' if the
  35 command reads its standard input or 'f' if it requires a file; the
  36 second likewise for the output. The command must be valid /bin/sh
  37 syntax.  If input or output files are required, they are passed as
  38 $IN and $OUT; otherwise, it must be  possible to use the command in
  39 a pipeline.
  40
  41 To add a conversion step at the beginning:
  42    t.prepend(command, kind)
  43
  44 To convert a file to another file using a template:
  45   sts = t.copy(infile, outfile)
  46 If infile or outfile are the empty string, standard input is read or
  47 standard output is written, respectively.  The return value is the
  48 exit status of the conversion pipeline.
  49
  50 To open a file for reading or writing through a conversion pipeline:
  51    fp = t.open(file, mode)
  52 where mode is 'r' to read the file, or 'w' to write it -- just like
  53 for the built-in function open() or for os.popen().
  54
  55 To create a new template object initialized to a given one:
  56    t2 = t.clone()
  57
  58 For an example, see the function test() at the end of the file.
  59 """                                     # '
  60
  61
  62 import sys
  63 import re
  64
  65 import os
  66 import tempfile
  67 import string
  68
  69 __all__ = ["Template"]
  70
  71 # Conversion step kinds
  72
  73 FILEIN_FILEOUT = 'ff'                   # Must read & write real files
  74 STDIN_FILEOUT  = '-f'                   # Must write a real file
  75 FILEIN_STDOUT  = 'f-'                   # Must read a real file
  76 STDIN_STDOUT   = '--'                   # Normal pipeline element
  77 SOURCE         = '.-'                   # Must be first, writes stdout
  78 SINK           = '-.'                   # Must be last, reads stdin
  79
  80 stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \
  81              SOURCE, SINK]
  82
  83
  84 class Template:
  85     """Class representing a pipeline template."""
  86
  87     def __init__(self):
  88         """Template() returns a fresh pipeline template."""
  89         self.debugging = 0
  90         self.reset()
  91
  92     def __repr__(self):
  93         """t.__repr__() implements `t`."""
  94         return '<Template instance, steps=' + `self.steps` + '>'
  95
  96     def reset(self):
  97         """t.reset() restores a pipeline template to its initial state."""
  98         self.steps = []
  99
 100     def clone(self):
 101         """t.clone() returns a new pipeline template with identical
 102         initial state as the current one."""
 103         t = Template()
 104         t.steps = self.steps[:]
 105         t.debugging = self.debugging
 106         return t
 107
 108     def debug(self, flag):
 109         """t.debug(flag) turns debugging on or off."""
 110         self.debugging = flag
 111
 112     def append(self, cmd, kind):
 113         """t.append(cmd, kind) adds a new step at the end."""
 114         if type(cmd) is not type(''):
 115             raise TypeError, \
 116                   'Template.append: cmd must be a string'
 117         if kind not in stepkinds:
 118             raise ValueError, \
 119                   'Template.append: bad kind ' + `kind`
 120         if kind == SOURCE:
 121             raise ValueError, \
 122                   'Template.append: SOURCE can only be prepended'
 123         if self.steps and self.steps[-1][1] == SINK:
 124             raise ValueError, \
 125                   'Template.append: already ends with SINK'
 126         if kind[0] == 'f' and not re.search('\$IN\b', cmd):
 127             raise ValueError, \
 128                   'Template.append: missing $IN in cmd'
 129         if kind[1] == 'f' and not re.search('\$OUT\b', cmd):
 130             raise ValueError, \
 131                   'Template.append: missing $OUT in cmd'
 132         self.steps.append((cmd, kind))
 133
 134     def prepend(self, cmd, kind):
 135         """t.prepend(cmd, kind) adds a new step at the front."""
 136         if type(cmd) is not type(''):
 137             raise TypeError, \
 138                   'Template.prepend: cmd must be a string'
 139         if kind not in stepkinds:
 140             raise ValueError, \
 141                   'Template.prepend: bad kind ' + `kind`
 142         if kind == SINK:
 143             raise ValueError, \
 144                   'Template.prepend: SINK can only be appended'
 145         if self.steps and self.steps[0][1] == SOURCE:
 146             raise ValueError, \
 147                   'Template.prepend: already begins with SOURCE'
 148         if kind[0] == 'f' and not re.search('\$IN\b', cmd):
 149             raise ValueError, \
 150                   'Template.prepend: missing $IN in cmd'
 151         if kind[1] == 'f' and not re.search('\$OUT\b', cmd):
 152             raise ValueError, \
 153                   'Template.prepend: missing $OUT in cmd'
 154         self.steps.insert(0, (cmd, kind))
 155
 156     def open(self, file, rw):
 157         """t.open(file, rw) returns a pipe or file object open for
 158         reading or writing; the file is the other end of the pipeline."""
 159         if rw == 'r':
 160             return self.open_r(file)
 161         if rw == 'w':
 162             return self.open_w(file)
 163         raise ValueError, \
 164               'Template.open: rw must be \'r\' or \'w\', not ' + `rw`
 165
 166     def open_r(self, file):
 167         """t.open_r(file) and t.open_w(file) implement
 168         t.open(file, 'r') and t.open(file, 'w') respectively."""
 169         if not self.steps:
 170             return open(file, 'r')
 171         if self.steps[-1][1] == SINK:
 172             raise ValueError, \
 173                   'Template.open_r: pipeline ends width SINK'
 174         cmd = self.makepipeline(file, '')
 175         return os.popen(cmd, 'r')
 176
 177     def open_w(self, file):
 178         if not self.steps:
 179             return open(file, 'w')
 180         if self.steps[0][1] == SOURCE:
 181             raise ValueError, \
 182                   'Template.open_w: pipeline begins with SOURCE'
 183         cmd = self.makepipeline('', file)
 184         return os.popen(cmd, 'w')
 185
 186     def copy(self, infile, outfile):
 187         return os.system(self.makepipeline(infile, outfile))
 188
 189     def makepipeline(self, infile, outfile):
 190         cmd = makepipeline(infile, self.steps, outfile)
 191         if self.debugging:
 192             print cmd
 193             cmd = 'set -x; ' + cmd
 194         return cmd
 195
 196
 197 def makepipeline(infile, steps, outfile):
 198     # Build a list with for each command:
 199     # [input filename or '', command string, kind, output filename or '']
 200
 201     list = []
 202     for cmd, kind in steps:
 203         list.append(['', cmd, kind, ''])
 204     #
 205     # Make sure there is at least one step
 206     #
 207     if not list:
 208         list.append(['', 'cat', '--', ''])
 209     #
 210     # Take care of the input and output ends
 211     #
 212     [cmd, kind] = list[0][1:3]
 213     if kind[0] == 'f' and not infile:
 214         list.insert(0, ['', 'cat', '--', ''])
 215     list[0][0] = infile
 216     #
 217     [cmd, kind] = list[-1][1:3]
 218     if kind[1] == 'f' and not outfile:
 219         list.append(['', 'cat', '--', ''])
 220     list[-1][-1] = outfile
 221     #
 222     # Invent temporary files to connect stages that need files
 223     #
 224     garbage = []
 225     for i in range(1, len(list)):
 226         lkind = list[i-1][2]
 227         rkind = list[i][2]
 228         if lkind[1] == 'f' or rkind[0] == 'f':
 229             temp = tempfile.mktemp()
 230             garbage.append(temp)
 231             list[i-1][-1] = list[i][0] = temp
 232     #
 233     for item in list:
 234         [inf, cmd, kind, outf] = item
 235         if kind[1] == 'f':
 236             cmd = 'OUT=' + quote(outf) + '; ' + cmd
 237         if kind[0] == 'f':
 238             cmd = 'IN=' + quote(inf) + '; ' + cmd
 239         if kind[0] == '-' and inf:
 240             cmd = cmd + ' <' + quote(inf)
 241         if kind[1] == '-' and outf:
 242             cmd = cmd + ' >' + quote(outf)
 243         item[1] = cmd
 244     #
 245     cmdlist = list[0][1]
 246     for item in list[1:]:
 247         [cmd, kind] = item[1:3]
 248         if item[0] == '':
 249             if 'f' in kind:
 250                 cmd = '{ ' + cmd + '; }'
 251             cmdlist = cmdlist + ' |\n' + cmd
 252         else:
 253             cmdlist = cmdlist + '\n' + cmd
 254     #
 255     if garbage:
 256         rmcmd = 'rm -f'
 257         for file in garbage:
 258             rmcmd = rmcmd + ' ' + quote(file)
 259         trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'
 260         cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
 261     #
 262     return cmdlist
 263
 264
 265 # Reliably quote a string as a single argument for /bin/sh
 266
 267 _safechars = string.letters + string.digits + '!@%_-+=:,./'     # Safe unquoted
 268 _funnychars = '"`$\\'                           # Unsafe inside "double quotes"
 269
 270 def quote(file):
 271     for c in file:
 272         if c not in _safechars:
 273             break
 274     else:
 275         return file
 276     if '\'' not in file:
 277         return '\'' + file + '\''
 278     res = ''
 279     for c in file:
 280         if c in _funnychars:
 281             c = '\\' + c
 282         res = res + c
 283     return '"' + res + '"'
 284
 285
 286 # Small test program and example
 287
 288 def test():
 289     print 'Testing...'
 290     t = Template()
 291     t.append('togif $IN $OUT', 'ff')
 292     t.append('giftoppm', '--')
 293     t.append('ppmtogif >$OUT', '-f')
 294     t.append('fromgif $IN $OUT', 'ff')
 295     t.debug(1)
 296     FILE = '/usr/local/images/rgb/rogues/guido.rgb'
 297     t.copy(FILE, '@temp')
 298     print 'Done.'