# Part of the A-A-P recipe executive: Parse and execute recipe commands
# Copyright (C) 2002 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here:
http://www.a-a-p.org/COPYING
import string
import sys
import traceback
from Util import *
from Work import setrpstack
from Error import *
from RecPos import rpcopy
import Global
# ":" commands that only work at the toplevel
aap_cmd_toplevel = [
"autodepend",
"recipe",
"rule",
"variant",
]
# All possible names of ":" commands in a recipe, including toplevel-only ones.
aap_cmd_names = [
"add",
"attr",
"attribute",
"cat",
"checkin",
"checkout",
"child",
"commit",
"commitall",
"copy",
"del",
"delete",
"error",
"export",
"filetype",
"include",
"mkdir",
"move",
"print",
"publish",
"publishall",
"refresh",
"remove",
"removeall",
"require",
"sys",
"system",
"touch",
"unlock",
"update",
"verscont",
] + aap_cmd_toplevel
# marker for recipe line number in Python script
line_marker = '#@recipe='
line_marker_len = len(line_marker)
def assert_var_name(name, rpstack):
"""Check if "name" is a valid variable name.
If it isn't, throw a user exception."""
for c in name:
if not varchar(c):
recipe_error(rpstack, _("Invalid character in variable name"))
def get_var_name(fp):
"""Get the name of a variable from the current position at "fp" and
get the index of the next non-white after it. Returns an empty string if
there is no valid variable name."""
idx = fp.idx
while idx < fp.line_len and varchar(fp.line[idx]):
idx = idx + 1
return fp.line[fp.idx:idx], skip_white(fp.line, idx)
class ArgItem:
"""Object used as smallest part in the arglist."""
def __init__(self, isexpr, str):
self.isexpr = isexpr # 0 for string, 1 for Python expression
self.str = str # the string itself
def getarg(fp, stop, globals):
"""Get an argument starting at fp.line[fp.idx] and ending at a character in
stop[].
Quotes are used to include stop characters in the argument.
Backticks are handled here. `` is reduced to a single `. A Python
expression `python` is translated to '" + expr2str(python) + "'.
Returns the resulting string and fp.idx is updated to the character
after the argument (the stop character or past the end of line).
"""
res = '' # argument collected so far
inquote = '' # quote we're inside of
inbraces = 0 # inside {} count
while 1:
if fp.idx >= fp.line_len: # end of line
break
# Python `expression`?
if fp.line[fp.idx] == '`':
# `` isn't the start of an expression, reduce it to a single `.
if fp.idx + 1 < fp.line_len and fp.line[fp.idx + 1] == '`':
res = res + '`'
fp.idx = fp.idx + 2
continue
# Append the Python expression.
res = res + '" + expr2str(' + get_py_expr(fp) + ') + "'
continue
# End of quoted string?
if inquote:
if fp.line[fp.idx] == inquote:
inquote = ''
# Start of quoted string?
elif fp.line[fp.idx] == '"' or fp.line[fp.idx] == "'":
inquote = fp.line[fp.idx]
else:
# start/end of {}?
if fp.line[fp.idx] == '{':
inbraces = inbraces + 1
elif fp.line[fp.idx] == '}':
inbraces = inbraces - 1
if inbraces < 0:
# TODO: recipe_error(fp.rpstack, _("Unmatched }"))
inbraces = 0
# Stop character found?
# A ':' must be followed by white space to be recongized.
# A '=' must not be inside {}.
if string.find(stop, fp.line[fp.idx]) != -1 \
and (fp.line[fp.idx] != ':'
or fp.idx + 1 == fp.line_len
or fp.line[fp.idx + 1] == ' '
or fp.line[fp.idx + 1] == '\t') \
and (fp.line[fp.idx] != '=' or inbraces == 0):
break
# Need to escape backslash and double quote.
c = fp.line[fp.idx]
if c == '"' or c == '\\':
res = res + '\\'
res = res + c
fp.idx = fp.idx + 1
# Skip over $$ and $#.
if (c == '$' and fp.idx < fp.line_len
and (fp.line[fp.idx] == '$' or fp.line[fp.idx] == '#')):
res = res + fp.line[fp.idx]
fp.idx = fp.idx + 1
# Remove trailing white space.
e = len(res)
while e > 0 and is_white(res[e - 1]):
e = e - 1
return res[:e]
def get_func_args(fp, indent, globals):
"""Get the arguments for an aap_ function or assignment from the recipe
line(s).
Stop at a line with an indent of "indent" or less.
Input lines: cmdname arg ` <python-expr> `arg
arg
Result: "arg " + expr2str(<python-expr>) + "arg arg"
Return the argument string, advance fp to the following line."""
res = ''
fp.idx = skip_white(fp.line, fp.idx)
while 1:
if fp.idx >= fp.line_len or fp.line[fp.idx] == '#':
# Read the next line
fp.nextline()
if fp.line is None:
break # end of file
fp.idx = skip_white(fp.line, 0)
if get_indent(fp.line) > indent:
continue
# A line with less indent finishes the list of arguments
break
# Get the argument, stop at a comment, handle python expression.
# A line break is changed into a space.
if res:
res = res + ' '
res = res + getarg(fp, "#", globals)
return '"' + res + '"'
def esc_quote(s):
"""Escape double quotes and backslash with a backslash."""
return string.replace(string.replace(s, '\\', '\\\\'), '"', '\\"')
def get_commands(fp, indent):
"""Read command lines for a dependency or a rule.
Stop when the indent is at or below "indent".
Returns the string of commands, each line ending in '\n'."""
s = ''
while 1:
if fp.line is None or get_indent(fp.line) <= indent:
break # end of commands reached
s = s + fp.line + '\n'
fp.nextline()
return '"""' + esc_quote(s) + '"""'
def get_py_expr(fp):
"""Get a Python expression from ` to matching `.
Reduce `` to `.
fp.idx points to the ` and is advanced to after the matching `.
Returns the expression excluding the ` before and after."""
# Remember the RecPos where the first ` was found; need to make a copy,
# because fp.nextline() will change it.
rpstack = rpcopy(fp.rpstack, fp.rpstack[-1].line_nr)
res = ''
fp.idx = fp.idx + 1
while 1:
if fp.idx >= fp.line_len:
# Python expression continues in the next line.
fp.nextline()
if fp.line is None:
recipe_error(rpstack, _("Missing `"))
res = res + '\n'
if fp.line[fp.idx] == '`':
# Either the matching ` or `` that stands for a single `.
fp.idx = fp.idx + 1
if fp.idx >= fp.line_len or fp.line[fp.idx] != '`':
break # found matching `
res = res + '`'
else:
# Append a character to the Python expression.
res = res + fp.line[fp.idx]
fp.idx = fp.idx + 1
return res
def recipe_error(rpstack, msg):
"""Throw an exception for an error in a recipe:
Error: Unknown command
in recipe "main.aap" line 88: :foobar asdf
included from "main.aap" line 33
When "rpstack" is empty it's not mentioned, useful for errors
not related to a specific line."""
# Note: These messages is not translated, so that a parser for the
# messages isn't confused by various languages.
if len(rpstack) == 0:
e = 'Error in recipe: %s\n' % msg
else:
e = 'Error in recipe "%s" line %d: %s\n' \
% (rpstack[-1].name, rpstack[-1].line_nr, msg)
if len(rpstack) > 1:
for i in range(len(rpstack) - 2, 0, -1):
e = e + 'included from "%s" line %d\n' \
% (rpstack[i].name, rpstack[i].line_nr)
e = e[:-1] # remove trailing \n
raise UserError, e
def script_error(rpstack, script, e):
"""Handle an error raised while executing the Python script produced for a
recipe. The error is probably in the recipe, try to give a useful error
message."""
etype, evalue, tb = sys.exc_info()
# A SyntaxError is special: it's not the last frame in the traceback but
# only in the "etype" and "evalue". When there is a filename it must have
# been an internal error, otherwise it's an error in the converted recipe.
py_line_nr = -1
if etype is SyntaxError:
try:
msg, (filename, py_line_nr, offset, line) = evalue
if not filename is None:
py_line_nr = -2
except:
pass
if py_line_nr < 0:
# Find the line number in the last traceback frame.
while tb.tb_next:
tb = tb.tb_next
fname = tb.tb_frame.f_code.co_filename
if py_line_nr == -2 or (fname and not fname == "<string>"):
# If there is a filename, it's not an error in the script.
from Main import error_msg
error_msg(_("Internal Error"))
traceback.print_exc()
sys.exit(1)
py_line_nr = traceback.tb_lineno(tb)
# Translate the line number in the Python script to the line number
# in the recipe.
i = 0
script_len = len(script)
rec_line_nr = 1
while 1:
if py_line_nr == 1:
break
while i < script_len:
if script[i] == '\n':
break
i = i + 1
i = i + 1
if i >= script_len:
break
if script[i : i + line_marker_len] == line_marker:
i = i + line_marker_len
j = i
while script[j] in string.digits:
j = j + 1
rec_line_nr = string.atoi(script[i:j])
py_line_nr = py_line_nr - 1
# Give the exception error with the line number in the recipe.
recipe_py_error(rpcopy(rpstack, rec_line_nr), '')
def recipe_py_error(rpstack, msg):
"""Turn the list from format_exception_only() into a simple string and pass
it to recipe_error()."""
etype, evalue, tb = sys.exc_info()
lines = traceback.format_exception_only(etype, evalue)
# For a syntax error remove the "<string>" and line number that the
# Python script causes.
if etype is SyntaxError:
try:
emsg, (filename, lineno, offset, line) = evalue
if filename is None:
lines[0] = '\n'
except:
pass
str = msg
for line in lines[:-1]:
str = str + line + ' '
str = str + lines[-1]
recipe_error(rpstack, str)
def Process(fp, globals):
"""Read all the lines in ParsePos "fp", convert it into a Python script and
execute it.
When "fp.string" is empty, the source is a recipe file, otherwise it is
a string (commands from a dependency or rule)."""
# Need to be able to find the RecPos stack in globals.
setrpstack(globals, fp.rpstack)
class Variant:
"""Class used to remember nested ":variant" commands in
variant_stack."""
def __init__(self, name, indent):
self.name = name
self.min_indent = indent
self.val_indent = 0
self.had_star = 0 # encountered * item
#
# At the start of the loop "fp.line" contains the next line to be
# processsed. "fp.rpstack[-1].line_nr" is the number of this line in the
# recipe.
#
script = ""
shell_cmd = "" # shell command collected so far
variant_stack = [] # nested ":variant" commands
had_recipe_cmd = 0 # encountered ":recipe" command
fp.nextline() # read the first line
while 1:
# Skip leading white space (unless at end of file).
if not fp.line is None:
indent = get_indent(fp.line)
fp.idx = skip_white(fp.line, 0)
# If it's not a shell command and the previous line was, generate the
# collected shell commands now.
if shell_cmd:
if (fp.line is None \
or indent < shell_cmd_indent \
or fp.line[fp.idx:fp.idx + 4] != ":sys"):
script = script + (' ' * shell_cmd_indent) \
+ ('aap_shell(%d, globals(), "%s")\n'
% (shell_cmd_line_nr, shell_cmd))
shell_cmd = ''
elif not fp.line is None:
# Append the recipe line number, used for error messages.
script = script + ("%s%d\n" % (line_marker, fp.rpstack[-1].line_nr))
#
# Handle the end of commands in a variant or the end of a variant
#
if len(variant_stack) > 0:
v = variant_stack[-1]
if fp.line is None or indent <= v.min_indent:
# End of the :variant command.
if v.val_indent == 0:
recipe_error(fp.rpstack,
_("Exepected list of values after :variant"))
script = script + (' ' * v.min_indent) + (
"BDIR = BDIR + '-' + %s\n" % v.name)
del variant_stack[-1]
if len(variant_stack) > 0:
continue # another may end here as well
else:
if v.val_indent == 0:
v.val_indent = indent
first = 1
else:
first = 0
if indent <= v.val_indent:
# Start of a variant value: "debug [ condition ]"
# We simply ignore the condition here.
if v.had_star:
recipe_error(fp.rpstack,
_("Variant item * must be last one"))
if fp.idx < fp.line_len and fp.line[fp.idx] == '*':
if (fp.idx + 1 < fp.line_len
and not is_white(fp.line[fp.idx + 1])):
recipe_error(fp.rpstack, _("* must be by itself"))
if not first:
script = script + (' ' * v.min_indent) + "else:\n"
v.had_star = 1
else:
val, n = get_var_name(fp)
if val == '':
recipe_error(fp.rpstack,
_("Exepected variant value"))
if first:
# Specify the default value
script = script + (' ' * v.min_indent) + (
'if not globals().has_key("%s"):\n' % v.name)
script = script + (' ' * v.min_indent) + (
' %s = "%s"\n' % (v.name, val))
script = script + (' ' * v.min_indent) + (
"if %s == '%s':\n" % (v.name, val))
fp.nextline()
if fp.line is None or get_indent(fp.line) <= v.val_indent:
script = script + (' ' * v.min_indent) + "pass\n"
continue
#
# Stop at the end of the file.
#
if fp.line is None:
break
#
# A Python block
#
# recipe: :python <<<
# command
# command
# <<<
# Python: if 1:
# command
# command
#
if fp.line[fp.idx:fp.idx + 7] == ":python":
fp.idx = skip_white(fp.line, fp.idx + 7)
if fp.idx >= fp.line_len or fp.line[fp.idx] == '#':
term = None
else:
n = skip_to_white(fp.line, fp.idx)
term = fp.line[fp.idx:n]
term_len = len(term)
n = skip_white(fp.line, n)
if n < fp.line_len and fp.line[n] != '#':
recipe_error(fp.rpstack,
_("Too many arguments for :python"))
start_line_nr = fp.rpstack[-1].line_nr
first = 1
while 1:
fp.nextline()
if fp.line is None:
if not term:
break
fp.rpstack[-1].line_nr = start_line_nr
recipe_error(fp.rpstack, _("Unterminated :python block"))
if first:
first = 0
# If the indent of the Python block is more than the
# current indent, insert an ":if 1".
if get_indent(fp.line) > indent:
script = script + (indent * ' ') + "if 1:" + '\n'
if not term:
# No terminator defined: end when indent is smaller.
if get_indent(fp.line) <= indent:
break
else:
# Terminator defined: end when it's found.
n = skip_white(fp.line, 0)
if n < fp.line_len and fp.line[n:n + term_len] == term:
n = skip_white(fp.line, n + term_len)
if n >= fp.line_len or fp.line[n] == "#":
fp.nextline()
break
# Append the recipe line number, used for error messages.
script = script + ("%s%d\n%s\n"
% (line_marker, fp.rpstack[-1].line_nr, fp.line))
continue
#
# An A-A-P command
#
# recipe: :cmd arg arg
# arg
# Python: aap_cmd(123, globals(), "arg arg arg")
#
if fp.line[fp.idx] == ":":
s = fp.idx
fp.idx = fp.idx + 1
e = skip_to_white(fp.line, fp.idx)
cmd_name = fp.line[fp.idx:e]
fp.idx = skip_white(fp.line, e)
# Check if this is a valid command name. The error is postponed
# until executing the line, so that "@if aapversion > nr" can be
# used before it.
if cmd_name not in aap_cmd_names:
cmd_name = "unknown"
fp.idx = s
if fp.string and cmd_name in aap_cmd_toplevel:
cmd_name = "nothere"
fp.idx = s
#
# To avoid starting a shell for every single command, collect
# system commands until encountering another command.
#
# recipe: :system one-shell-command
# :sys two-shell-command
# Python: aap_shell(123, globals(),
# "one-shell_command\ntwo_shell_command\n")
#
if cmd_name == "system" or cmd_name == "sys":
if not shell_cmd:
shell_cmd_line_nr = fp.rpstack[-1].line_nr
shell_cmd_indent = indent
shell_cmd = shell_cmd + getarg(fp, "#", globals) + '\\n'
# get the next line
fp.nextline()
continue
# recipe: :variant VAR
# foo [ condition ]
# cmds
# * [ condition ]
# cmds
# Python: if VAR == "foo":
# cmds
# else:
# cmds
# BDIR = BDIR + '-' + VAR
# This is complicated, because "cmds" can be any command, and
# variants may nest. Store the info about the variant in
# variant_stack and continue, the rest is handled above.
if cmd_name == "variant":
var_name, n = get_var_name(fp)
if var_name == '' or (n < fp.line_len and fp.line[n] != '#'):
recipe_error(fp.rpstack,
_("Expected variable name after :variant"))
variant_stack.append(Variant(var_name, indent))
# get the next line
fp.nextline()
continue
# Generate a call to the Python function for this command.
script = script + (indent * ' ') + ('aap_%s(%d, globals(), '
% (cmd_name, fp.rpstack[-1].line_nr))
if cmd_name == "rule" or cmd_name == "autodepend":
# recipe: :rule target : {attr} source
# commands
# Python: aap_rule(123, globals(), "target", "source",
# 124, """commands""")
#
# recipe: :autodepend {attr} source
# commands
# Python: aap_autodepend(123, globals(), "source",
# 124, """commands""")
if cmd_name == "rule":
target = getarg(fp, ":#", globals)
if fp.idx >= fp.line_len or fp.line[fp.idx] != ':':
recipe_error(fp.rpstack, _("Missing ':' after :%s")
% cmd_name)
fp.idx = fp.idx + 1
script = script + ('"%s", ' % target)
source = getarg(fp, "#", globals)
cmd_line_nr = fp.rpstack[-1].line_nr
fp.nextline()
cmds = get_commands(fp, indent)
script = script + ('"%s", %d, %s)\n'
% (source, cmd_line_nr, cmds))
elif cmd_name == "filetype":
# recipe: :filetype [filename]
# detection-lines
# Python: aap_filetype_python(123, globals(), "arg",
# """detection-lines""")
#
arg = getarg(fp, "#", globals)
cmd_line_nr = fp.rpstack[-1].line_nr
fp.nextline()
cmds = get_commands(fp, indent)
script = script + '"%s", %d, %s)\n' % (arg, cmd_line_nr, cmds)
else:
# get arguments that may continue on the next line
script = script + get_func_args(fp, indent, globals) + ")\n"
# When a ":recipe" command is encountered that will probably
# be executed, make a copy of the globals at the start, so that
# this can be restored when executing the updated recipe.
# This is a "heavy" command, only do it when needed.
from Commands import do_recipe_cmd
if (cmd_name == "recipe" and not had_recipe_cmd
and do_recipe_cmd(fp.rpstack)):
had_recipe_cmd = 1
script = ('globals()["_start_globals"] = globals().copy()\n'
+ script)
continue
#
# A Python command
#
# recipe: @command args
# Python: command args
#
if fp.line[fp.idx] == "@":
if fp.idx + 1 < fp.line_len:
if fp.line[fp.idx + 1] == ' ' \
or fp.line[fp.idx + 1] == '\t':
# followed by white space: replace @ with a space
script = script \
+ string.replace(fp.line, '@', ' ', 1) + '\n'
else:
# followed by text: remove the @
script = script \
+ string.replace(fp.line, '@', '', 1) + '\n'
# get the next line
fp.nextline()
continue
#
# Assignment
#
# recipe: name = $VAR {attr=val} ` glob("*.c") `
# two
# Python: name = aap_eval(123, globals(),
# "$VAR {attr=val} " + glob("*.c") + " two", 1)
#
# var = value assign
# var += value append (assign if not set yet)
# var ?= value only assign when not set yet
# var $= value evaluate when used
# var $+= value append, evaluate when used
# var $?= value only when not set, evaluate when used
var_name, n = get_var_name(fp)
if n < fp.line_len:
nc = fp.line[n]
ec = nc
if ec == '$' and n + 1 < fp.line_len:
ne = n + 1
ec = fp.line[ne]
else:
ne = n
if (ec == '+' or ec == '?') and ne + 1 < fp.line_len:
lc = ec
ne = ne + 1
ec = fp.line[ne]
else:
lc = ''
if var_name != '' and ec == '=':
# When changing $CACHE need to flush the cache and reload it.
if var_name == "CACHE":
script = script + (indent * ' ') + "flush_cache()\n"
fp.idx = skip_white(fp.line, ne + 1)
script = script + (indent * ' ') + (
"aap_assign(%d, globals(), '%s', "
% (fp.rpstack[-1].line_nr, var_name))
args = get_func_args(fp, indent, globals)
script = script + args + (", '%s', '%s')\n" % (nc, lc))
continue
#
# If there is no ":" following we don't know what it is.
#
targets = getarg(fp, ":#", globals)
if fp.idx >= fp.line_len or fp.line[fp.idx] != ':':
recipe_error(fp.rpstack, _("No recognized item"))
if fp.string:
recipe_error(fp.rpstack, _("Dependency not allowed here"))
#
# Dependency
#
# recipe: target target : source source
# commands
# Python: aap_depend(123, globals(), list-of-targets,
# list-of-sources, "commands")
#
else:
# Skip the ':' and get the list of sources.
fp.idx = skip_white(fp.line, fp.idx + 1)
sources = getarg(fp, '#', globals)
nr = fp.rpstack[-1].line_nr
fp.nextline()
script = script + ('aap_depend(%d, globals(), "%s", "%s", %d, '
% (nr, targets, sources, fp.rpstack[-1].line_nr))
# get the commands and the following line
cmds = get_commands(fp, indent)
script = script + cmds + ')\n'
#
# End of loop over all lines in recipe.
#
if fp.string:
# When parsing a string need to take care of the indent.
if is_white(fp.string[0]):
script = "if 1:\n" + script
else:
# Close the file before executing the script, so that ":recipe" can
# overwrite the file.
fp.file.close()
# Prepend the default imports.
script = "from Commands import *\n" \
+ "from glob import glob\n" \
+ script
# DEBUG
#print script
#
# Execute the resulting Python script.
# Give a useful error message when something is wrong.
#
try:
exec script in globals, globals
except StandardError, e:
script_error(fp.rpstack, script, e)
# vim: set sw=4 sts=4 tw=79 fo+=l:
