home *** CD-ROM | disk | FTP | other *** search
- # Conversion pipeline templates
- # =============================
-
-
- # The problem:
- # ------------
- #
- # Suppose you have some data that you want to convert to another format
- # (e.g. from GIF image format to PPM image format). Maybe the
- # conversion involves several steps (e.g. piping it through compress or
- # uuencode). Some of the conversion steps may require that their input
- # is a disk file, others may be able to read standard input; similar for
- # their output. The input to the entire conversion may also be read
- # from a disk file or from an open file, and similar for its output.
- #
- # The module lets you construct a pipeline template by sticking one or
- # more conversion steps together. It will take care of creating and
- # removing temporary files if they are necessary to hold intermediate
- # data. You can then use the template to do conversions from many
- # different sources to many different destinations. The temporary
- # file names used are different each time the template is used.
- #
- # The templates are objects so you can create templates for many
- # different conversion steps and store them in a dictionary, for
- # instance.
-
-
- # Directions:
- # -----------
- #
- # To create a template:
- # t = Template()
- #
- # To add a conversion step to a template:
- # t.append(command, kind)
- # where kind is a string of two characters: the first is '-' if the
- # command reads its standard input or 'f' if it requires a file; the
- # second likewise for the output. The command must be valid /bin/sh
- # syntax. If input or output files are required, they are passed as
- # $IN and $OUT; otherwise, it must be possible to use the command in
- # a pipeline.
- #
- # To add a conversion step at the beginning:
- # t.prepend(command, kind)
- #
- # To convert a file to another file using a template:
- # sts = t.copy(infile, outfile)
- # If infile or outfile are the empty string, standard input is read or
- # standard output is written, respectively. The return value is the
- # exit status of the conversion pipeline.
- #
- # To open a file for reading or writing through a conversion pipeline:
- # fp = t.open(file, mode)
- # where mode is 'r' to read the file, or 'w' to write it -- just like
- # for the built-in function open() or for os.popen().
- #
- # To create a new template object initialized to a given one:
- # t2 = t.clone()
- #
- # For an example, see the function test() at the end of the file.
-
-
- import sys
- import re
-
- import os
- import tempfile
- import string
-
-
- # Conversion step kinds
-
- FILEIN_FILEOUT = 'ff' # Must read & write real files
- STDIN_FILEOUT = '-f' # Must write a real file
- FILEIN_STDOUT = 'f-' # Must read a real file
- STDIN_STDOUT = '--' # Normal pipeline element
- SOURCE = '.-' # Must be first, writes stdout
- SINK = '-.' # Must be last, reads stdin
-
- stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \
- SOURCE, SINK]
-
-
- # A pipeline template is a Template object:
-
- class Template:
-
- # Template() returns a fresh pipeline template
- def __init__(self):
- self.debugging = 0
- self.reset()
-
- # t.__repr__() implements `t`
- def __repr__(self):
- return '<Template instance, steps=' + `self.steps` + '>'
-
- # t.reset() restores a pipeline template to its initial state
- def reset(self):
- self.steps = []
-
- # t.clone() returns a new pipeline template with identical
- # initial state as the current one
- def clone(self):
- t = Template()
- t.steps = self.steps[:]
- t.debugging = self.debugging
- return t
-
- # t.debug(flag) turns debugging on or off
- def debug(self, flag):
- self.debugging = flag
-
- # t.append(cmd, kind) adds a new step at the end
- def append(self, cmd, kind):
- if type(cmd) <> type(''):
- raise TypeError, \
- 'Template.append: cmd must be a string'
- if kind not in stepkinds:
- raise ValueError, \
- 'Template.append: bad kind ' + `kind`
- if kind == SOURCE:
- raise ValueError, \
- 'Template.append: SOURCE can only be prepended'
- if self.steps <> [] and self.steps[-1][1] == SINK:
- raise ValueError, \
- 'Template.append: already ends with SINK'
- if kind[0] == 'f' and not re.search('\$IN\b', cmd):
- raise ValueError, \
- 'Template.append: missing $IN in cmd'
- if kind[1] == 'f' and not re.search('\$OUT\b', cmd):
- raise ValueError, \
- 'Template.append: missing $OUT in cmd'
- self.steps.append((cmd, kind))
-
- # t.prepend(cmd, kind) adds a new step at the front
- def prepend(self, cmd, kind):
- if type(cmd) <> type(''):
- raise TypeError, \
- 'Template.prepend: cmd must be a string'
- if kind not in stepkinds:
- raise ValueError, \
- 'Template.prepend: bad kind ' + `kind`
- if kind == SINK:
- raise ValueError, \
- 'Template.prepend: SINK can only be appended'
- if self.steps <> [] and self.steps[0][1] == SOURCE:
- raise ValueError, \
- 'Template.prepend: already begins with SOURCE'
- if kind[0] == 'f' and not re.search('\$IN\b', cmd):
- raise ValueError, \
- 'Template.prepend: missing $IN in cmd'
- if kind[1] == 'f' and not re.search('\$OUT\b', cmd):
- raise ValueError, \
- 'Template.prepend: missing $OUT in cmd'
- self.steps.insert(0, (cmd, kind))
-
- # t.open(file, rw) returns a pipe or file object open for
- # reading or writing; the file is the other end of the pipeline
- def open(self, file, rw):
- if rw == 'r':
- return self.open_r(file)
- if rw == 'w':
- return self.open_w(file)
- raise ValueError, \
- 'Template.open: rw must be \'r\' or \'w\', not ' + `rw`
-
- # t.open_r(file) and t.open_w(file) implement
- # t.open(file, 'r') and t.open(file, 'w') respectively
-
- def open_r(self, file):
- if self.steps == []:
- return open(file, 'r')
- if self.steps[-1][1] == SINK:
- raise ValueError, \
- 'Template.open_r: pipeline ends width SINK'
- cmd = self.makepipeline(file, '')
- return os.popen(cmd, 'r')
-
- def open_w(self, file):
- if self.steps == []:
- return open(file, 'w')
- if self.steps[0][1] == SOURCE:
- raise ValueError, \
- 'Template.open_w: pipeline begins with SOURCE'
- cmd = self.makepipeline('', file)
- return os.popen(cmd, 'w')
-
- def copy(self, infile, outfile):
- return os.system(self.makepipeline(infile, outfile))
-
- def makepipeline(self, infile, outfile):
- cmd = makepipeline(infile, self.steps, outfile)
- if self.debugging:
- print cmd
- cmd = 'set -x; ' + cmd
- return cmd
-
-
- def makepipeline(infile, steps, outfile):
- # Build a list with for each command:
- # [input filename or '', command string, kind, output filename or '']
-
- list = []
- for cmd, kind in steps:
- list.append(['', cmd, kind, ''])
- #
- # Make sure there is at least one step
- #
- if list == []:
- list.append(['', 'cat', '--', ''])
- #
- # Take care of the input and output ends
- #
- [cmd, kind] = list[0][1:3]
- if kind[0] == 'f' and not infile:
- list.insert(0, ['', 'cat', '--', ''])
- list[0][0] = infile
- #
- [cmd, kind] = list[-1][1:3]
- if kind[1] == 'f' and not outfile:
- list.append(['', 'cat', '--', ''])
- list[-1][-1] = outfile
- #
- # Invent temporary files to connect stages that need files
- #
- garbage = []
- for i in range(1, len(list)):
- lkind = list[i-1][2]
- rkind = list[i][2]
- if lkind[1] == 'f' or rkind[0] == 'f':
- temp = tempfile.mktemp()
- garbage.append(temp)
- list[i-1][-1] = list[i][0] = temp
- #
- for item in list:
- [inf, cmd, kind, outf] = item
- if kind[1] == 'f':
- cmd = 'OUT=' + quote(outf) + '; ' + cmd
- if kind[0] == 'f':
- cmd = 'IN=' + quote(inf) + '; ' + cmd
- if kind[0] == '-' and inf:
- cmd = cmd + ' <' + quote(inf)
- if kind[1] == '-' and outf:
- cmd = cmd + ' >' + quote(outf)
- item[1] = cmd
- #
- cmdlist = list[0][1]
- for item in list[1:]:
- [cmd, kind] = item[1:3]
- if item[0] == '':
- if 'f' in kind:
- cmd = '{ ' + cmd + '; }'
- cmdlist = cmdlist + ' |\n' + cmd
- else:
- cmdlist = cmdlist + '\n' + cmd
- #
- if garbage:
- rmcmd = 'rm -f'
- for file in garbage:
- rmcmd = rmcmd + ' ' + quote(file)
- trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'
- cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
- #
- return cmdlist
-
-
- # Reliably quote a string as a single argument for /bin/sh
-
- _safechars = string.letters + string.digits + '!@%_-+=:,./' # Safe unquoted
- _funnychars = '"`$\\' # Unsafe inside "double quotes"
-
- def quote(file):
- for c in file:
- if c not in _safechars:
- break
- else:
- return file
- if '\'' not in file:
- return '\'' + file + '\''
- res = ''
- for c in file:
- if c in _funnychars:
- c = '\\' + c
- res = res + c
- return '"' + res + '"'
-
-
- # Small test program and example
-
- def test():
- import os
- print 'Testing...'
- t = Template()
- t.append('togif $IN $OUT', 'ff')
- t.append('giftoppm', '--')
- t.append('ppmtogif >$OUT', '-f')
- t.append('fromgif $IN $OUT', 'ff')
- t.debug(1)
- FILE = '/usr/local/images/rgb/rogues/guido.rgb'
- t.copy(FILE, '@temp')
- print 'Done.'
-