# Part of the A-A-P recipe executive: Aap commands in a recipe
# Copyright (C) 2002 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here:
http://www.a-a-p.org/COPYING
#
# These are functions available to the recipe.
#
# Some functions are used for translated items, such as dependencies and ":"
# commands.
#
# It's OK to do "from Commands import *", these things are supposed to be
# global.
#
import os
import os.path
import re
import string
import copy
from Depend import Depend
from Dictlist import string2dictlist, get_attrdict, listitem2str
from DoRead import read_recipe, recipe_dir
from Error import *
import Global
from Process import assert_var_name, recipe_error
from RecPos import rpdeepcopy
from Rule import Rule
from Util import *
from Work import getwork, getrpstack
import Cache
from Message import *
def aap_depend(line_nr, globals, targets, sources, cmd_line_nr, commands):
"""Add a dependency."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
# Expand the targets into dictlists.
targetlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, targets, Expand(1, Expand.quote_aap)))
# Parse build attributes {attr = value} zero or more times.
# Variables are not expanded now but when executing the build rules.
build_attr, i = get_attrdict(rpstack, None, sources, 0, 0)
# Expand the sources into dictlists.
sourcelist = string2dictlist(rpstack,
aap_eval(line_nr, globals, sources[i:], Expand(1, Expand.quote_aap)))
# Add to the global lists of dependencies.
# Make a copy of the RecPos stack, so that errors in "commands" can print
# the recipe stack. The last entry is going to be changed, thus it needs
# to be a copy, the rest can be referenced.
d = Depend(targetlist, build_attr, sourcelist, work,
rpdeepcopy(getrpstack(globals), cmd_line_nr), commands)
work.add_dependency(rpstack, d, commands != '')
def aap_autodepend(line_nr, globals, arg, cmd_line_nr, commands):
"""Add a dependency check."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
if not commands:
recipe_error(rpstack, _(":autodepend requires build commands"))
# Parse build attributes {attr = value} zero or more times.
# Variables are not expanded now but when executing the build rules.
build_attr, i = get_attrdict(rpstack, None, arg, 0, 0)
# Expand the other arguments into a dictlist.
arglist = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg[i:], Expand(1, Expand.quote_aap)))
# Use a rule object to store the info, a rule is just like a autodepend,
# except that the a autodepend uses filetype names instead of patterns.
rule = Rule([], build_attr, arglist,
rpdeepcopy(getrpstack(globals), cmd_line_nr), commands)
work.add_autodepend(rule)
def aap_rule(line_nr, globals, targets, sources, cmd_line_nr, commands):
"""Add a rule."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
# Expand the targets into dictlists.
targetlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, targets, Expand(1, Expand.quote_aap)))
# Parse build attributes {attr = value} zero or more times.
# Variables are not expanded now but when executing the build rules.
build_attr, i = get_attrdict(rpstack, None, sources, 0, 0)
# Expand the sources into dictlists.
sourcelist = string2dictlist(rpstack,
aap_eval(line_nr, globals, sources[i:], Expand(1, Expand.quote_aap)))
rule = Rule(targetlist, build_attr, sourcelist,
rpdeepcopy(getrpstack(globals), cmd_line_nr), commands)
work.add_rule(rule)
def aap_update(line_nr, globals, arg):
"""Handle ":update target ...": update target(s) now."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
from DoBuild import target_update
targets = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg, Expand(1, Expand.quote_aap)))
if len(targets) == 0:
recipe_error(rpstack, _("Missing argument for :update"))
for t in targets:
target_update(work, work.get_node(t["name"]), 1, t)
def aap_error(line_nr, globals, arg):
"""Handle: ":error foo bar"."""
rpstack = getrpstack(globals, line_nr)
recipe_error(rpstack, aap_eval(line_nr, globals, arg,
Expand(0, Expand.quote_aap)))
def aap_unknown(line_nr, globals, arg):
"""Handle: ":xxx arg". Postponed until executing the line, so that an
"@if aapversion > nr" can be used."""
rpstack = getrpstack(globals, line_nr)
recipe_error(rpstack, _('Unknown command: "%s"') % arg)
def aap_nothere(line_nr, globals, arg):
"""Handle using a toplevel command in build commands. Postponed until
executing the line, so that an "@if aapversion > nr" can be used."""
rpstack = getrpstack(globals, line_nr)
recipe_error(fp.rpstack, _('Command cannot be used here: "%s"') % arg)
#
############## start of commands used in a pipe
#
def _get_redir(line_nr, globals, raw_arg):
"""Get the redirection and pipe from the argument "raw_arg".
Returns these items:
1. the argument with $VAR expanded and redirection removed
2. the file name for redirection or None
3. the mode for redirection or None ("a" for append, "w" for write).
4. a command following '|' or None
When using ">file" also checks if the file doesn't exist yet."""
rpstack = getrpstack(globals, line_nr)
mode = None
fname = None
nextcmd = None
# Loop over the argument, getting one token at a time. Each token is
# either non-white (possibly with quoting) or a span of white space.
raw_arg_len = len(raw_arg)
i = 0 # current index in raw_arg
new_arg = '' # argument without redirection so far
while i < raw_arg_len:
t, i = get_token(raw_arg, i)
# Ignore trailing white space.
if i == raw_arg_len and is_white(t[0]):
break
# After (a span of) white space, check for redirection or pipe.
# Also at the start of the argument.
if new_arg == '' or is_white(t[0]):
if new_arg == '':
# Use the token at the start of the argument.
nt = t
t = ''
else:
# Get the token after the white space
nt, i = get_token(raw_arg, i)
if nt[0] == '>':
# Redirection: >, >> or >!
if mode:
recipe_error(rpstack, _('redirection appears twice'))
nt_len = len(nt)
ni = 1 # index after the ">", ">>" or ">!"
mode = 'w'
overwrite = 0
if nt_len > 1:
if nt[1] == '>':
mode = 'a'
ni = 2
elif nt[1] == '!':
overwrite = 1
ni = 2
if ni >= nt_len:
# white space after ">", get next token for fname
redir = nt[:ni]
if i < raw_arg_len:
# Get the separating white space.
nt, i = get_token(raw_arg, i)
if i == raw_arg_len:
recipe_error(rpstack, _('Missing file name after %s')
% redir)
# Get the file name
nt, i = get_token(raw_arg, i)
else:
# fname follows immediately after ">"
nt = nt[ni:]
# Expand $VAR in the file name. No attributes are added.
# Remove quotes from the result, it's used as a filename.
fname = unquote(aap_eval(line_nr, globals, nt,
Expand(0, Expand.quote_aap)))
if mode == "w" and not overwrite:
check_exists(rpstack, fname)
# When redirection is at the start, ignore the white space
# after it.
if new_arg == '' and i < raw_arg_len:
t, i = get_token(raw_arg, i)
elif nt[0] == '|':
# Pipe: the rest of the argument is another command
if mode:
recipe_error(rpstack, _("both redirection and '|'"))
if len(nt) > 1:
nextcmd = nt[1:] + raw_arg[i:]
else:
i = skip_white(raw_arg, i)
nextcmd = raw_arg[i:]
if not nextcmd:
# Can't have an empty command.
recipe_error(rpstack, _("Nothing follows '|'"))
if nextcmd[0] != ':':
# Must have an aap command here.
recipe_error(rpstack, _("Missing ':' after '|'"))
break
else:
# No redirection or pipe: add to argument
new_arg = new_arg + t + nt
else:
# Normal token: add to argument
new_arg = new_arg + t
if new_arg:
arg = aap_eval(line_nr, globals, new_arg, Expand(0, Expand.quote_aap))
else:
arg = new_arg
return arg, fname, mode, nextcmd
def _aap_pipe(line_nr, globals, cmd, pipein):
"""Handle the command that follows a '|'."""
rpstack = getrpstack(globals, line_nr)
items = string.split(cmd, None, 1)
if len(items) == 1: # command without argument, append empty argument.
items.append('')
if items[0] == ":assign":
_pipe_assign(line_nr, globals, items[1], pipein)
elif items[0] == ":cat":
aap_cat(line_nr, globals, items[1], pipein)
elif items[0] == ":filter":
_pipe_filter(line_nr, globals, items[1], pipein)
elif items[0] == ":print":
aap_print(line_nr, globals, items[1], pipein)
elif items[0] == ":tee":
_pipe_tee(line_nr, globals, items[1], pipein)
else:
recipe_error(rpstack,
(_('Invalid command after \'|\': "%s"') % items[0]))
def _pipe_assign(line_nr, globals, raw_arg, pipein):
"""Handle: ":assign var". Can only be used in a pipe."""
rpstack = getrpstack(globals, line_nr)
assert_var_name(raw_arg, rpstack)
globals[raw_arg] = pipein
def aap_cat(line_nr, globals, raw_arg, pipein = None):
"""Handle: ":cat >foo $bar"."""
rpstack = getrpstack(globals, line_nr)
# get the special items out of the argument
arg, fname, mode, nextcmd = _get_redir(line_nr, globals, raw_arg)
# get the list of files from the remaining argument
filelist = string2dictlist(rpstack, arg)
if len(filelist) == 0:
if pipein is None:
recipe_error(rpstack,
_(':cat command requires at least one file name argument'))
filelist = [ {"name" : "-"} ]
result = ''
if mode:
# Open the output file for writing
try:
wf = open(fname, mode)
except IOError, e:
recipe_error(rpstack,
(_('Cannot open "%s" for writing') % fname) + str(e))
# Loop over all arguments
for item in filelist:
fn = item["name"]
if fn == '-':
# "-" argument: use pipe input
if pipein is None:
recipe_error(rpstack, _('Using - not after a pipe'))
if nextcmd:
result = result + pipein
else:
lines = string.split(pipein, '\n')
else:
# file name argument: read the file
try:
rf = open(fn, "r")
except IOError, e:
recipe_error(rpstack,
(_('Cannot open "%s" for reading') % fn) + str(e))
try:
lines = rf.readlines()
rf.close()
except IOError, e:
recipe_error(rpstack,
(_('Cannot read from "%s"') % fn) + str(e))
if nextcmd:
# pipe output: append lines to the result
for l in lines:
result = result + l
if mode:
# file output: write lines to the file
try:
wf.writelines(lines)
except IOError, e:
recipe_error(rpstack,
(_('Cannot write to "%s"') % fname) + str(e))
elif not nextcmd:
# output to the terminal: print lines
msg_print(lines)
if mode:
# close the output file
try:
wf.close()
except IOError, e:
recipe_error(rpstack, (_('Error closing "%s"') % fname) + str(e))
if nextcmd:
# pipe output: execute the following command
_aap_pipe(line_nr, globals, nextcmd, result)
elif mode:
msg_info(_('Concatenated files into "%s"') % fname)
def _pipe_filter(line_nr, globals, raw_arg, pipein):
"""Handle: ":filter function ...". Can only be used in a pipe."""
rpstack = getrpstack(globals, line_nr)
arg, fname, mode, nextcmd = _get_redir(line_nr, globals, raw_arg)
# Replace "%s" with "_pipein".
# TODO: make it possible to escape the %s somehow?
s = string.find(arg, "%s")
if s < 0:
recipe_error(rpstack, _('%s missing in :filter argument'))
cmd = arg[:s] + "_pipein" + arg[s + 2:]
# Evaluate the expression.
globals["_pipein"] = pipein
try:
result = str(eval(cmd, globals, globals))
except StandardError, e:
recipe_error(rpstack, _(':filter command failed') + str(e))
del globals["_pipein"]
if mode:
# redirection: write output to a file
_write2file(rpstack, fname, result, mode)
elif nextcmd:
# pipe output: execute next command
_aap_pipe(line_nr, globals, nextcmd, result)
else:
# output to terminal: print the result
msg_print(result)
def aap_print(line_nr, globals, raw_arg, pipein = None):
"""Handle: ":print foo $bar"."""
rpstack = getrpstack(globals, line_nr)
arg, fname, mode, nextcmd = _get_redir(line_nr, globals, raw_arg)
if pipein:
if arg:
recipe_error(rpstack,
_(':print cannot have both pipe input and an argument'))
arg = pipein
if mode:
if len(arg) == 0 or arg[-1] != '\n':
arg = arg + '\n'
_write2file(rpstack, fname, arg, mode)
elif nextcmd:
if len(arg) == 0 or arg[-1] != '\n':
arg = arg + '\n'
_aap_pipe(line_nr, globals, nextcmd, arg)
else:
msg_print(arg)
def _pipe_tee(line_nr, globals, raw_arg, pipein):
"""Handle: ":tee fname ...". Can only be used in a pipe."""
rpstack = getrpstack(globals, line_nr)
arg, fname, mode, nextcmd = _get_redir(line_nr, globals, raw_arg)
# get the list of files from the remaining argument
filelist = string2dictlist(rpstack, arg)
if len(filelist) == 0:
recipe_error(rpstack,
_(':tee command requires at least one file name argument'))
for f in filelist:
fn = f["name"]
check_exists(rpstack, fn)
_write2file(rpstack, fn, pipein, "w")
if mode:
# redirection: write output to a file
_write2file(rpstack, fname, pipein, mode)
elif nextcmd:
# pipe output: execute next command
_aap_pipe(line_nr, globals, nextcmd, pipein)
else:
# output to terminal: print the result
msg_print(pipein)
def _write2file(rpstack, fname, str, mode):
"""Write string "str" to file "fname" opened with mode "mode"."""
try:
f = open(fname, mode)
except IOError, e:
recipe_error(rpstack,
(_('Cannot open "%s" for writing') % fname) + str(e))
try:
f.write(str)
f.close()
except IOError, e:
recipe_error(rpstack, (_('Cannot write to "%s"') % fname) + str(e))
#
############## end of commands used in a pipe
#
def aap_child(line_nr, globals, arg):
"""Handle ":child filename": execute a recipe."""
rpstack = getrpstack(globals, line_nr)
work = getwork(globals)
# Get the argument and attributes.
varlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg, Expand(1, Expand.quote_aap)))
if len(varlist) == 0:
recipe_error(rpstack, _(":child requires an argument"))
if len(varlist) > 1:
recipe_error(rpstack, _(":child only accepts one argument"))
name = varlist[0]["name"]
force_refresh = Global.cmd_args.has_option("refresh-recipe")
if ((force_refresh or not os.path.exists(name))
and varlist[0].has_key("refresh")):
# Need to create a node to refresh it.
# Ignore errors, a check for existence is below.
# Use a cached file when no forced refresh.
from VersCont import refresh_node
refresh_node(rpstack, globals,
work.get_node(name, 0, varlist[0]), not force_refresh)
if not os.path.exists(name):
if varlist[0].has_key("refresh"):
recipe_error(rpstack, _('Cannot download recipe "%s"') % name)
recipe_error(rpstack, _('Recipe "%s" does not exist') % name)
try:
cwd = os.getcwd()
except OSError:
recipe_error(rpstack, _("Cannot obtain current directory"))
name = recipe_dir(os.path.abspath(name))
# Execute the child recipe. Make a copy of the globals to avoid
# the child modifies them.
new_globals = globals.copy()
new_globals["exports"] = {}
new_globals["dependencies"] = []
read_recipe(rpstack, name, new_globals)
# TODO: move dependencies from the child to the current recipe,
# using the rules from the child
# Move the exported variables to the globals of the current recipe
exports = new_globals["exports"]
for e in exports.keys():
globals[e] = exports[e]
# go back to the previous current directory
try:
if cwd != os.getcwd():
# Note: This message is not translated, so that a parser
# for the messages isn't confused by various languages.
msg_changedir('Entering directory "' + cwd + '"')
try:
os.chdir(cwd)
except OSError:
recipe_error(rpstack,
_('Cannot change to directory "%s"') % cwd)
except OSError:
recipe_error(rpstack, _("Cannot obtain current directory"))
def aap_export(line_nr, globals, arg):
"""Export a variable to the parent recipe (if any)."""
rpstack = getrpstack(globals, line_nr)
varlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg, Expand(1, Expand.quote_aap)))
for i in varlist:
n = i["name"]
assert_var_name(n, rpstack)
globals["exports"][n] = get_var_val(line_nr, globals, n)
def aap_attr(line_nr, globals, arg):
"""Add attributes to nodes."""
aap_attribute(line_nr, globals, arg)
def aap_attribute(line_nr, globals, arg):
"""Add attributes to nodes."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
# Get the optional leading attributes.
if not arg:
recipe_error(rpstack, _(":attr command requires an argument"))
if arg[0] == '{':
attrdict, i = get_attrdict(rpstack, globals, arg, 0, 1)
else:
attrdict = {}
i = 0
# Get the list of items.
varlist = string2dictlist(rpstack, aap_eval(line_nr, globals,
arg[i:], Expand(1, Expand.quote_aap)))
if not varlist:
recipe_error(rpstack, _(":attr command requires a file argument"))
# Loop over all items, adding attributes to the node.
for i in varlist:
node = work.get_node(i["name"], 1, i)
node.set_attributes(attrdict)
def aap_assign(line_nr, globals, varname, arg, dollar, extra):
"""Assignment command in a recipe.
"varname" is the name of the variable.
"arg" is the argument value (Python expression already expanded).
When "dollar" is '$' don't expand $VAR items.
When "extra" is '?' only assign when "varname" wasn't set yet.
When "extra" is '+' append to "varname"."""
# Skip the whole assignment for "var ?= val" if var was already set.
if extra != '?' or not globals.has_key(varname):
if dollar != '$':
# Expand variables in "arg".
val = aap_eval(line_nr, globals, arg, Expand(1, Expand.quote_aap))
else:
# Postpone expanding variables in "arg". Set "$var$" to remember
# it has to be done when using the variable.
val = arg
# append or set the value
if extra == '+' and globals.has_key(varname):
globals[varname] = (get_var_val(line_nr, globals, varname)
+ ' ' + val)
else:
globals[varname] = val
exn = '$' + varname
if dollar != '$':
# Stop postponing variable expansion.
if globals.has_key(exn):
del globals[exn]
else:
# Postpone expanding variables in "arg". Set "$var" to remember
# it has to be done when using the variable.
globals[exn] = 1
def aap_eval(line_nr, globals, arg, expand, startquote = '', skip_errors = 0):
"""Evaluate $VAR, $(VAR) and ${VAR} in "arg", which is a string.
$VAR is expanded and the resulting string is combined with what comes
before and after $VAR. text$VAR -> "textval1" "textval2".
"expand" is an Expand object that specifies the way $VAR is expanded.
When "startquote" isn't empty, work like "arg" was prededed by it.
When "skip_errors" is non-zero, leave items with errors unexpanded,
never fail.
"""
rpstack = getrpstack(globals, line_nr)
res = '' # resulting string so far
inquote = startquote # ' or " when inside quotes
itemstart = 0 # index where a white separated item starts
arg_len = len(arg)
idx = 0
while idx < arg_len:
if arg[idx] == '$':
idx = idx + 1
if arg[idx] == '$':
res = res + '$' # reduce $$ to a single $
idx = idx + 1
elif arg[idx] == '#':
res = res + '#' # reduce $# to a single #
idx = idx + 1
else:
# Remember what non-white text came before the $.
before = res[itemstart:]
exp = copy.copy(expand) # make a copy so that we can change it
# Check for type of quoting.
if arg[idx] == '-':
idx = idx + 1
exp.attr = 0
elif arg[idx] == '+':
idx = idx + 1
exp.attr = 1
# Check for '+' (include attributes) or '-' (exclude
# attributes)
if arg[idx] == '=':
idx = idx + 1
exp.quote = Expand.quote_none
elif arg[idx] == "'":
idx = idx + 1
exp.quote = Expand.quote_aap
elif arg[idx] == '"':
idx = idx + 1
exp.quote = Expand.quote_double
elif arg[idx] == '\\':
idx = idx + 1
exp.quote = Expand.quote_bs
elif arg[idx] == '!':
idx = idx + 1
exp.quote = Expand.quote_shell
# Check for $(VAR) and ${VAR}.
if arg[idx] == '(' or arg[idx] == '{':
s = skip_white(arg, idx + 1)
else:
s = idx
# get the variable name
e = s
while e < arg_len and varchar(arg[e]):
e = e + 1
if e == s:
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack, _("Invalid character after $"))
varname = arg[s:e]
if not globals.has_key(varname):
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack, _('Unknown variable: "%s"') % varname)
index = -1
if s > idx:
# Inside () or {}
e = skip_white(arg, e)
if e < arg_len and arg[e] == '[':
# get index for $(var[n])
b = e
brak = 0
e = e + 1
# TODO: ignore [] inside quotes?
while e < arg_len and (arg[e] != ']' or brak > 0):
if arg[e] == '[':
brak = brak + 1
elif arg[e] == ']':
brak = brak - 1
e = e + 1
if e == arg_len or arg[e] != ']':
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack, _("Missing ']'"))
v = aap_eval(line_nr, globals, arg[b+1:e],
Expand(0, Expand.quote_none), '', skip_errors)
try:
index = int(v)
except:
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack,
_('index does not evaluate to a number: "%s"')
% v)
if index < 0:
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack,
_('index evaluates to a negative number: "%d"')
% index)
e = skip_white(arg, e + 1)
# Check for matching () and {}
if (e == arg_len
or (arg[idx] == '(' and arg[e] != ')')
or (arg[idx] == '{' and arg[e] != '}')):
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack, _('No match for "%s"') % arg[idx])
# Continue after the () or {}
idx = e + 1
else:
# Continue after the varname
idx = e
# Find what comes after $VAR.
# Need to remember whether it starts inside quotes.
after_inquote = inquote
s = idx
while idx < arg_len:
if inquote:
if arg[idx] == inquote:
inquote = ''
elif arg[idx] == '"' or arg[idx] == "'":
inquote = arg[idx]
elif string.find(string.whitespace + "{", arg[idx]) != -1:
break
idx = idx + 1
after = arg[s:idx]
if exp.attr:
# Obtain any following attributes, advance to after them.
# Also expand $VAR inside the attributes.
attrdict, idx = get_attrdict(rpstack, globals, arg, idx, 1)
else:
attrdict = {}
if before == '' and after == '' and len(attrdict) == 0:
if index < 0:
# No rc-style expansion or index, use the value of
# $VAR as specified with quote-expansion
try:
res = res + get_var_val(line_nr, globals,
varname, exp)
except TypeError:
if skip_errors:
res = res + '$'
continue
recipe_error(rpstack,
_('Type of variable "%s" must be a string')
% varname)
else:
# No rc-style expansion but does have an index.
# Get the Dictlist of the referred variable.
varlist = string2dictlist(rpstack,
get_var_val(line_nr, globals, varname))
if len(varlist) < index + 1:
msg_warning(
_('Index "%d" is out of range for variable "%s"')
% (index, varname))
else:
res = res + expand_item(varlist[index], exp)
idx = s
else:
# rc-style expansion of a variable
# Get the Dictlist of the referred variable.
# When an index is specified us that entry of the list.
# When index is out of range or the list is empty, use a
# list with one empty entry.
varlist1 = string2dictlist(rpstack,
get_var_val(line_nr, globals, varname))
if (len(varlist1) == 0
or (index >= 0 and len(varlist1) < index + 1)):
if index >= 0:
msg_warning(
_('Index "%d" is out of range for variable "%s"')
% (index, varname))
varlist1 = [{"name": ""}]
elif index >= 0:
varlist1 = [ varlist1[index] ]
# Evaluate the "after" of $(VAR)after {attr = val}.
varlist2 = string2dictlist(rpstack,
aap_eval(line_nr, globals, after,
Expand(1, Expand.quote_aap),
startquote = after_inquote),
startquote = after_inquote)
if len(varlist2) == 0:
varlist2 = [{"name": ""}]
# Remove quotes from "before", they are put back when
# needed.
lead = ''
q = ''
for c in before:
if q:
if c == q:
q = ''
else:
lead = lead + c
elif c == '"' or c == "'":
q = c
else:
lead = lead + c
# Combine "before", the list from $VAR, the list from
# "after" and the following attributes.
# Put "startquote" in front, because the terminalting quote
# will have been removed.
rcs = startquote
startquote = ''
for d1 in varlist1:
for d2 in varlist2:
if rcs:
rcs = rcs + ' '
s = lead + d1["name"] + d2["name"]
# If $VAR was in quotes put the result in quotes.
if after_inquote:
rcs = rcs + enquote(s, quote = after_inquote)
else:
rcs = rcs + expand_itemstr(s, exp)
if exp.attr:
for k in d1.keys():
if k != "name":
rcs = rcs + "{%s = %s}" % (k, d1[k])
for k in d2.keys():
if k != "name":
rcs = rcs + "{%s = %s}" % (k, d2[k])
for k in attrdict.keys():
rcs = rcs + "{%s = %s}" % (k, attrdict[k])
res = res[0:itemstart] + rcs
else:
# No '$' at this position, include the character in the result.
# Check if quoting starts or ends and whether white space separates
# an item, this is used for expanding $VAR.
if inquote:
if arg[idx] == inquote:
inquote = ''
elif arg[idx] == '"' or arg[idx] == "'":
inquote = arg[idx]
elif is_white(arg[idx]):
itemstart = len(res) + 1
res = res + arg[idx]
idx = idx + 1
return res
def expr2str(item):
"""Used to turn the result of a Python expression into a string.
For a list the elements are separated with a space.
Dollars are doubled to avoid them being recognized as variables."""
import types
if type(item) == types.ListType:
s = ''
for i in item:
if s:
s = s + ' '
s = s + listitem2str(str(i))
else:
s = str(item)
return string.replace(s, '$', '$$')
def aap_sufreplace(suffrom, sufto, string):
"""Replace suffixes in "string" from "suffrom" to "sufto"."""
return re.sub(string.replace(suffrom, ".", "\\.") + "\\b", sufto, string)
def aap_shell(line_nr, globals, cmds):
"""Execute shell commands from the recipe."""
s = aap_eval(line_nr, globals, cmds, Expand(0, Expand.quote_shell))
if globals.has_key("target"):
msg_extra(_('Shell commands for updating "%s":') % globals["target"])
n = logged_system(s)
if n != 0:
recipe_error(getrpstack(globals, line_nr),
_("Shell returned %d when executing:\n%s") % (n, s))
def aap_system(line_nr, globals, cmds):
"""Implementation of ":system cmds". Almost the same as aap_shell()."""
aap_shell(line_nr, globals, cmds + '\n')
def aap_sys(line_nr, globals, cmds):
"""Implementation of ":sys cmds". Almost the same as aap_shell()."""
aap_shell(line_nr, globals, cmds + '\n')
def aap_copy(line_nr, globals, arg):
"""Implementation of ":copy -x from to"."""
# It's in a separate module, it's quite a bit of stuff.
from CopyMove import copy_move
copy_move(line_nr, globals, arg, 1)
def aap_move(line_nr, globals, arg):
"""Implementation of ":move -x from to"."""
# It's in a separate module, it's quite a bit of stuff.
from CopyMove import copy_move
copy_move(line_nr, globals, arg, 0)
def aap_delete(line_nr, globals, raw_arg):
"""Alias for aap_del()."""
aap_del(line_nr, globals, raw_arg)
def aap_del(line_nr, globals, raw_arg):
"""Implementation of ":del -r file1 file2"."""
# Evaluate $VAR things
arg = aap_eval(line_nr, globals, raw_arg, Expand(0, Expand.quote_aap))
rpstack = getrpstack(globals, line_nr)
# flags:
# -f don't fail when not exists
# -q quiet
# -r -R recursive, delete directories
try:
flags, i = get_flags(arg, 0, "fqrR")
except UserError, e:
recipe_error(rpstack, e)
if 'r' in flags or 'R' in flags:
recursive = 1
else:
recursive = 0
# Get the remaining arguments, should be at least one.
arglist = string2dictlist(rpstack, arg[i:])
if not arglist:
recipe_error(rpstack, _(":del command requires an argument"))
import glob
from urlparse import urlparse
def deltree(dir):
"""Recursively delete a directory or a file."""
if os.path.isdir(dir):
fl = glob.glob(os.path.join(dir, "*"))
for f in fl:
deltree(f)
os.rmdir(dir)
else:
os.remove(dir)
for a in arglist:
fname = a["name"]
scheme, mach, path, parm, query, frag = urlparse(fname, '', 0)
if scheme != '':
recipe_error(rpstack, _('Cannot delete remotely yet: "%s"') % fname)
# Expand ~user and wildcards.
fl = glob.glob(os.path.expanduser(fname))
if len(fl) == 0 and not 'f' in flags:
recipe_error(rpstack, _('No match for "%s"') % fname)
for f in fl:
try:
if recursive:
deltree(f)
else:
os.remove(f)
except EnvironmentError, e:
recipe_error(rpstack, (_('Could not delete "%s"') % f) + str(e))
else:
if os.path.exists(f):
recipe_error(rpstack, _('Could not delete "%s"') % f)
if not 'q' in flags:
msg_info(_('Deleted "%s"') % fname)
def aap_mkdir(line_nr, globals, raw_arg):
"""Implementation of ":mkdir dir1 dir2"."""
# Evaluate $VAR things
arg = aap_eval(line_nr, globals, raw_arg, Expand(0, Expand.quote_aap))
rpstack = getrpstack(globals, line_nr)
# flags:
# -f create file when it does not exist
try:
flags, i = get_flags(arg, 0, "f")
except UserError, e:
recipe_error(rpstack, e)
# Get the arguments, should be at least one.
arglist = string2dictlist(rpstack, arg[i:])
if not arglist:
recipe_error(rpstack, _(":mkdir command requires an argument"))
from urlparse import urlparse
for a in arglist:
name = a["name"]
scheme, mach, path, parm, query, frag = urlparse(name, '', 0)
if scheme != '':
recipe_error(rpstack, _('Cannot create remote directory yet: "%s"')
% name)
# Expand ~user, create directory
dir = os.path.expanduser(name)
# Skip creation when it already exists.
if not ('f' in flags and os.path.isdir(dir)):
try:
os.mkdir(dir)
except EnvironmentError, e:
recipe_error(rpstack, (_('Could not create directory "%s"')
% dir) + str(e))
def aap_touch(line_nr, globals, raw_arg):
"""Implementation of ":touch file1 file2"."""
# Evaluate $VAR things
arg = aap_eval(line_nr, globals, raw_arg, Expand(0, Expand.quote_aap))
rpstack = getrpstack(globals, line_nr)
# flags:
# -f create file when it does not exist
try:
flags, i = get_flags(arg, 0, "f")
except UserError, e:
recipe_error(rpstack, e)
# Get the arguments, should be at least one.
arglist = string2dictlist(rpstack, arg[i:])
if not arglist:
recipe_error(rpstack, _(":touch command requires an argument"))
from urlparse import urlparse
import time
for a in arglist:
name = a["name"]
scheme, mach, path, parm, query, frag = urlparse(name, '', 0)
if scheme != '':
recipe_error(rpstack, _('Cannot touch remote file yet: "%s"')
% name)
# Expand ~user, touch file
name = os.path.expanduser(name)
if os.path.exists(name):
now = time.time()
try:
os.utime(name, (now, now))
except EnvironmentError, e:
recipe_error(rpstack, (_('Could not update time of "%s"')
% name) + str(e))
else:
if not 'f' in flags:
recipe_error(rpstack,
_('"%s" does not exist (use :touch -f to create it)')
% name)
try:
# create an empty file
f = os.open(name, os.O_WRONLY + os.O_CREAT + os.O_EXCL)
os.close(f)
except EnvironmentError, e:
recipe_error(rpstack, (_('Could not create "%s"')
% name) + str(e))
def flush_cache():
"""Called just before setting $CACHE."""
# It's here so that only this module has to be imported in Process.py.
Cache.dump_cache()
# dictionary of recipes that have been refreshed (using full path name).
recipe_refreshed = {}
def aap_include(line_nr, globals, arg):
"""Handle ":include filename": read the recipe into the current globals."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
# Evaluate the arguments
args = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg, Expand(1, Expand.quote_aap)))
if len(args) != 1:
recipe_error(rpstack, _(":include requires one argument"))
recname = args[0]["name"]
# Refresh the recipe when invoked with the "-R" argument.
if ((Global.cmd_args.has_option("refresh-recipe")
or not os.path.exists(recname))
and args[0].has_key("refresh")):
fullname = full_fname(recname)
if not recipe_refreshed.has_key(fullname):
from VersCont import refresh_node
# Create a node for the recipe and refresh it.
node = work.get_node(recname, 0, args[0])
if not refresh_node(rpstack, globals, node, 0):
msg_warning(_('Could not update recipe "%s"') % recname)
# Also mark it as updated when it failed, don't try again.
recipe_refreshed[fullname] = 1
read_recipe(rpstack, recname, globals)
def do_recipe_cmd(rpstack):
"""Return non-zero if a ":recipe" command in the current recipe may be
executed."""
# Return right away when not invoked with the "-R" argument.
if not Global.cmd_args.has_option("refresh-recipe"):
return 0
# Skip when this recipe was already updated.
recname = full_fname(rpstack[-1].name)
if recipe_refreshed.has_key(recname):
return 0
return 1
def aap_recipe(line_nr, globals, arg):
"""Handle ":recipe {refresh = name_list}": may download this recipe."""
work = getwork(globals)
rpstack = getrpstack(globals, line_nr)
# Return right away when not to be executed.
if not do_recipe_cmd(rpstack):
return
# Register the recipe to have been updated. Also when it failed, don't
# want to try again.
recname = full_fname(rpstack[-1].name)
recipe_refreshed[recname] = 1
short_name = shorten_name(recname)
msg_info(_('Updating recipe "%s"') % short_name)
orgdict, i = get_attrdict(rpstack, globals, arg, 0, 1)
if not orgdict.has_key("refresh"):
recipe_error(rpstack, _(":recipe requires a refresh attribute"))
# TODO: warning for trailing characters?
from VersCont import refresh_node
# Create a node for the recipe and refresh it.
node = work.get_node(short_name, 0, orgdict)
if refresh_node(rpstack, globals, node, 0):
# TODO: check if the recipe was completely read
# TODO: no need for restart if the recipe didn't change
# Restore the globals to the values from when starting to read the
# recipe.
start_globals = globals["_start_globals"]
for k in globals.keys():
if not start_globals.has_key(k):
del globals[k]
for k in start_globals.keys():
globals[k] = start_globals[k]
# read the new recipe file
read_recipe(rpstack, recname, globals, reread = 1)
# Throw an exception to cancel executing the rest of the script
# generated from the old recipe. This is catched in read_recipe()
raise OriginUpdate
msg_warning(_('Could not update recipe "%s"') % node.name)
#
# Generic function for getting the arguments of :refresh, :checkout, :commit,
# :checkin, :unlock and :publish
#
def get_verscont_args(line_nr, globals, arg, cmd):
""""Handle ":cmd {attr = } file ..."."""
rpstack = getrpstack(globals, line_nr)
# Get the optional attributes that apply to all arguments.
attrdict, i = get_attrdict(rpstack, globals, arg, 0, 1)
# evaluate the arguments into a dictlist
varlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg[i:], Expand(1, Expand.quote_aap)))
if not varlist:
recipe_error(rpstack, _(':%s requires an argument') % cmd)
return attrdict, varlist
def do_verscont_cmd(rpstack, globals, action, attrdict, varlist):
"""Perform "action" on items in "varlist", using attributes in
"attrdict"."""
from VersCont import verscont_node, refresh_node
work = getwork(globals)
for item in varlist:
node = work.get_node(item["name"], 1, item)
node.set_attributes(attrdict)
if action == "refresh":
r = (node.may_refresh()
and refresh_node(rpstack, globals, node, 0) == 0)
elif action == "publish":
r = publish_node(rpstack, globals, node)
else:
r = verscont_node(rpstack, globals, node, action)
if not r:
msg_warning(_('%s failed for "%s"') % (action, item["name"]))
def verscont_cmd(line_nr, globals, arg, action):
"""Perform "action" for each item "varlist"."""
rpstack = getrpstack(globals, line_nr)
attrdict, varlist = get_verscont_args(line_nr, globals, arg, action)
do_verscont_cmd(rpstack, globals, action, attrdict, varlist)
def aap_refresh(line_nr, globals, arg):
""""Handle ":refresh {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "refresh")
def aap_checkout(line_nr, globals, arg):
""""Handle ":checkout {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "checkout")
def aap_commit(line_nr, globals, arg):
""""Handle ":commit {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "commit")
def aap_checkin(line_nr, globals, arg):
""""Handle ":checkin {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "checkin")
def aap_unlock(line_nr, globals, arg):
""""Handle ":unlock {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "unlock")
def aap_publish(line_nr, globals, arg):
""""Handle ":publish {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "publish")
def aap_add(line_nr, globals, arg):
""""Handle ":add {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "add")
def aap_remove(line_nr, globals, arg):
""""Handle ":remove {attr = val} file ..."."""
verscont_cmd(line_nr, globals, arg, "remove")
def aap_verscont(line_nr, globals, arg):
""""Handle ":verscont action {attr = val} [file ...]"."""
rpstack = getrpstack(globals, line_nr)
# evaluate the arguments into a dictlist
varlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg, Expand(1, Expand.quote_aap)))
if not varlist:
recipe_error(rpstack, _(':verscont requires an argument'))
if len(varlist) > 1:
arglist = varlist[1:]
else:
arglist = []
do_verscont_cmd(rpstack, globals, varlist[0]["name"], varlist[0], arglist)
def aap_commitall(line_nr, globals, arg):
""""Handle ":commitall {attr = val} file ..."."""
attrdict, varlist = get_verscont_args(line_nr, globals, arg, "commitall")
recipe_error(rpstack, _('Sorry, :commitall is not implemented yet'))
def aap_publishall(line_nr, globals, arg):
""""Handle ":publishall {attr = val} file ..."."""
attrdict, varlist = get_verscont_args(line_nr, globals, arg, "publishall")
recipe_error(rpstack, _('Sorry, :publishall is not implemented yet'))
def aap_removeall(line_nr, globals, arg):
""""Handle ":removeall {attr = val} [dir ...]"."""
rpstack = getrpstack(globals, line_nr)
# flags:
# -l local (non-recursive)
# -r recursive
try:
flags, i = get_flags(arg, 0, "lr")
except UserError, e:
recipe_error(rpstack, e)
# Get the optional attributes that apply to all arguments.
attrdict, i = get_attrdict(rpstack, globals, arg, i, 1)
# evaluate the arguments into a dictlist
varlist = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg[i:], Expand(1, Expand.quote_aap)))
from VersCont import verscont_removeall
if varlist:
# Directory name arguments: Do each directory non-recursively
for dir in varlist:
for k in attrdict.keys():
dir[k] = attrdict[k]
verscont_removeall(rpstack, globals, dir, 'r' in flags)
else:
# No arguments: Do current directory recursively
attrdict["name"] = "."
verscont_removeall(rpstack, globals, attrdict, not 'l' in flags)
def aap_filetype(line_nr, globals, arg, cmd_line_nr, commands):
"""Add filetype detection from a file or in-line detection rules."""
from Filetype import ft_add_rules, ft_read_file, DetectError
rpstack = getrpstack(globals, line_nr)
# look through the arguments
args = string2dictlist(rpstack,
aap_eval(line_nr, globals, arg, Expand(0, Expand.quote_aap)))
if len(args) > 1:
recipe_error(rpstack, _('Too many arguments for :filetype'))
if len(args) == 1 and commands:
recipe_error(rpstack,
_('Cannot have file name and commands for :filetype'))
if len(args) == 0 and not commands:
recipe_error(rpstack,
_('Must have file name or commands for :filetype'))
try:
if commands:
what = "lines"
ft_add_rules(commands)
else:
fname = args[0]["name"]
what = 'file "%s"' % fname
ft_read_file(fname)
except DetectError, e:
recipe_error(rpstack, (_('Error in detection %s: ') % what) + str(e))
# vim: set sw=4 sts=4 tw=79 fo+=l:
