#
# Automated Testing Framework (atf)
#
# Copyright (c) 2007 The NetBSD Foundation, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
atf_test_case config
config_head()
{
atf_set "descr" "Tests that the config files are read in the correct" \
"order"
}
config_body()
{
create_helper config
atf_test_case vflag
vflag_head()
{
atf_set "descr" "Tests that the -v flag works and that it properly" \
"overrides the values in configuration files"
}
vflag_body()
{
create_helper testvar
echo "Checking that 'testvar' is not defined."
atf_check -s eq:1 -o ignore -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run helper"
echo "Checking that defining 'testvar' through '-v' works."
atf_check -s eq:0 -o match:'testvar: a value' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run -v testvar='a value' helper"
echo "Checking that defining 'testvar' through the configuration" \
"file works."
mkdir etc
cat >etc/common.conf <<EOF
Content-Type: application/X-atf-config; version="1"
testvar = "value in conf file"
EOF
atf_check -s eq:0 -o match:'testvar: value in conf file' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run helper"
echo "Checking that defining 'testvar' through -v overrides the" \
"configuration file."
atf_check -s eq:0 -o match:'testvar: a value' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run -v testvar='a value' helper"
}
atf_test_case atffile
atffile_head()
{
atf_set "descr" "Tests that the variables defined by the Atffile" \
"are recognized and that they take the lowest priority"
}
atffile_body()
{
create_helper testvar
echo "Checking that 'testvar' is not defined."
atf_check -s eq:1 -o ignore -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run helper"
echo "Checking that defining 'testvar' through the Atffile works."
echo 'conf: testvar = "a value"' >>Atffile
atf_check -s eq:0 -o match:'testvar: a value' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run helper"
echo "Checking that defining 'testvar' through the configuration" \
"file overrides the one in the Atffile."
mkdir etc
cat >etc/common.conf <<EOF
Content-Type: application/X-atf-config; version="1"
testvar = "value in conf file"
EOF
atf_check -s eq:0 -o match:'testvar: value in conf file' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run helper"
rm -rf etc
echo "Checking that defining 'testvar' through -v overrides the" \
"one in the Atffile."
atf_check -s eq:0 -o match:'testvar: new value' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run -v testvar='new value' helper"
}
atf_test_case atffile_recursive
atffile_recursive_head()
{
atf_set "descr" "Tests that variables defined by an Atffile are not" \
"inherited by other Atffiles."
}
atffile_recursive_body()
{
create_helper testvar
mkdir dir
mv Atffile helper dir
echo "Checking that 'testvar' is not inherited."
create_atffile dir
echo 'conf: testvar = "a value"' >> Atffile
atf_check -s eq:1 -o ignore -e ignore -x "ATF_CONFDIR=$(pwd)/etc atf-run"
echo "Checking that defining 'testvar' in the correct Atffile works."
echo 'conf: testvar = "a value"' >>dir/Atffile
atf_check -s eq:0 -o match:'testvar: a value' -e ignore -x \
"ATF_CONFDIR=$(pwd)/etc atf-run"
}
atf_test_case fds
fds_head()
{
atf_set "descr" "Tests that all streams are properly captured"
}
fds_body()
{
create_helper fds
atf_check -s eq:0 \
-o match:'^tc-so:msg1 to stdout$' \
-o match:'^tc-so:msg2 to stdout$' \
-o match:'^tc-se:msg1 to stderr$' \
-o match:'^tc-se:msg2 to stderr$' \
-e empty atf-run
}
atf_test_case mux_streams
mux_streams_head()
{
atf_set "descr" "Tests for a race condition in stream multiplexing"
}
mux_streams_body()
{
create_helper mux_streams
for i in 1 2 3 4 5; do
echo "Attempt ${i}"
atf_check -s eq:0 -o match:'stdout 9999' -o match:'stderr 9999' atf-run
done
}
atf_test_case expect
expect_head()
{
atf_set "descr" "Tests the processing of test case results and the" \
"expect features"
}
expect_body()
{
ln -s "$(atf_get_srcdir)/expect_helpers" .
create_atffile expect_helpers
atf_test_case broken_results
broken_results_head()
{
atf_set "descr" "Ensures that atf-run reports test programs that" \
"provide a bogus results output as broken programs"
}
broken_results_body()
{
# We produce two errors from the header to ensure that the parse
# errors are printed on a single line on the output file. Printing
# them on separate lines would be incorrect.
create_helper_stdin helper 1 <<EOF
echo 'line 1' >\${resfile}
echo 'line 2' >>\${resfile}
exit 0
EOF
chmod +x helper
atf_test_case broken_tp_list
broken_tp_list_head()
{
atf_set "descr" "Ensures that atf-run reports test programs that" \
"provide a bogus test case list"
}
broken_tp_list_body()
{
cat >helper <<EOF
#! $(atf-config -t atf_shell)
while [ \${#} -gt 0 ]; do
if [ \${1} = -l ]; then
echo 'Content-Type: application/X-atf-tp; version="1"'
echo
echo 'foo: bar'
exit 0
else
shift
fi
done
exit 0
EOF
chmod +x helper
create_atffile helper
re='^tp-end: [0-9][0-9]*\.[0-9]*, helper,'
re="${re} Invalid format for test case list:.*First property.*ident"
atf_check -s eq:1 -o match:"${re}" -e empty atf-run
}
atf_test_case hooks
hooks_head()
{
atf_set "descr" "Checks that the default hooks work and that they" \
"can be overriden by the user"
}
hooks_body()
{
cp $(atf_get_srcdir)/pass_helper helper
create_atffile helper
atf_test_case isolation_env
isolation_env_head()
{
atf_set "descr" "Tests that atf-run sets a set of environment variables" \
"to known sane values"
}
isolation_env_body()
{
undef_vars="LANG LC_ALL LC_COLLATE LC_CTYPE LC_MESSAGES LC_MONETARY \
LC_NUMERIC LC_TIME"
def_vars="HOME TZ"
mangleenv="env"
for v in ${undef_vars} ${def_vars}; do
mangleenv="${mangleenv} ${v}=bogus-value"
done
create_helper env_list
create_atffile helper
# We must ignore stderr in this call (instead of specifying -e empty)
# because, when atf-run invokes the shell to run the hooks, we may get
# error messages about an invalid locale. This happens, at least, when
# the shell is bash 4.x.
atf_check -s eq:0 -o save:stdout -e ignore ${mangleenv} atf-run helper
for v in ${undef_vars}; do
atf_check -s eq:1 -o empty -e empty grep "^tc-so:${v}=" stdout
done
for v in ${def_vars}; do
atf_check -s eq:0 -o ignore -e empty grep "^tc-so:${v}=" stdout
done
atf_test_case isolation_home
isolation_home_head()
{
atf_set "descr" "Tests that atf-run sets HOME to a sane and valid value"
}
isolation_home_body()
{
create_helper env_home
create_atffile helper
atf_check -s eq:0 -o ignore -e ignore env HOME=foo atf-run helper
}
atf_test_case isolation_stdin
isolation_stdin_head()
{
atf_set "descr" "Tests that atf-run nullifies the stdin of test cases"
}
isolation_stdin_body()
{
create_helper read_stdin
create_atffile helper
atf_check -s eq:0 -o ignore -e ignore -x 'echo hello world | atf-run helper'
}
atf_test_case isolation_umask
isolation_umask_head()
{
atf_set "descr" "Tests that atf-run sets the umask to a known value"
}
isolation_umask_body()
{
create_helper umask
create_atffile helper
atf_test_case cleanup_pass
cleanup_pass_head()
{
atf_set "descr" "Tests that atf-run calls the cleanup routine of the test" \
"case when the test case result is passed"
}
cleanup_pass_body()
{
create_helper cleanup_states
create_atffile helper
atf_check -s eq:0 -o match:'cleanup_states, passed' -e ignore atf-run \
-v state=pass -v statedir=$(pwd) helper
test -f to-stay || atf_fail "Test case body did not run correctly"
if [ -f to-delete ]; then
atf_fail "Test case cleanup did not run correctly"
fi
}
atf_test_case cleanup_fail
cleanup_fail_head()
{
atf_set "descr" "Tests that atf-run calls the cleanup routine of the test" \
"case when the test case result is failed"
}
cleanup_fail_body()
{
create_helper cleanup_states
create_atffile helper
atf_check -s eq:1 -o match:'cleanup_states, failed' -e ignore atf-run \
-v state=fail -v statedir=$(pwd) helper
test -f to-stay || atf_fail "Test case body did not run correctly"
if [ -f to-delete ]; then
atf_fail "Test case cleanup did not run correctly"
fi
}
atf_test_case cleanup_skip
cleanup_skip_head()
{
atf_set "descr" "Tests that atf-run calls the cleanup routine of the test" \
"case when the test case result is skipped"
}
cleanup_skip_body()
{
create_helper cleanup_states
create_atffile helper
atf_check -s eq:0 -o match:'cleanup_states, skipped' -e ignore atf-run \
-v state=skip -v statedir=$(pwd) helper
test -f to-stay || atf_fail "Test case body did not run correctly"
if [ -f to-delete ]; then
atf_fail "Test case cleanup did not run correctly"
fi
}
atf_test_case cleanup_curdir
cleanup_curdir_head()
{
atf_set "descr" "Tests that atf-run calls the cleanup routine in the same" \
"work directory as the body so that they can share data"
}
cleanup_curdir_body()
{
create_helper cleanup_curdir
create_atffile helper
atf_test_case cleanup_signal
cleanup_signal_head()
{
atf_set "descr" "Tests that atf-run calls the cleanup routine if it gets" \
"a termination signal while running the body"
}
cleanup_signal_body()
{
: # TODO: Write this.
}
atf_test_case cleanup_mount
cleanup_mount_head()
{
atf_set "descr" "Tests that the removal algorithm does not cross" \
"mount points"
atf_set "require.user" "root"
}
cleanup_mount_body()
{
ROOT="$(pwd)/root"; export ROOT
platform=$(uname)
case ${platform} in
Linux|FreeBSD|NetBSD|SunOS)
;;
*)
# XXX Possibly specify in meta-data too.
atf_skip "Test unimplemented in this platform (${platform})"
;;
esac
atf_test_case cleanup_symlink
cleanup_symlink_head()
{
atf_set "descr" "Tests that the removal algorithm does not follow" \
"symlinks, which may live in another device and thus" \
"be treated as mount points"
atf_set "require.user" "root"
}
cleanup_symlink_body()
{
ROOT="$(pwd)/root"; export ROOT
platform=$(uname)
case ${platform} in
Linux|FreeBSD|NetBSD|SunOS)
;;
*)
# XXX Possibly specify in meta-data too.
atf_skip "Test unimplemented in this platform (${platform})"
;;
esac
atf_test_case require_user_root
require_user_root_head()
{
atf_set "descr" "Tests that atf-run validates the require.user property" \
"when it is set to 'root'"
}
require_user_root_body()
{
create_helper require_user
create_atffile helper
if [ $(id -u) -eq 0 ]; then
exp=passed
else
exp=skipped
fi
atf_check -s eq:0 -o match:"${TESTCASE}, ${exp}" -e ignore atf-run \
-v user=root helper
}
atf_test_case require_user_unprivileged
require_user_unprivileged_head()
{
atf_set "descr" "Tests that atf-run validates the require.user property" \
"when it is set to 'root'"
}
require_user_unprivileged_body()
{
create_helper require_user
create_atffile helper
if [ $(id -u) -eq 0 ]; then
exp=skipped
else
exp=passed
fi
atf_check -s eq:0 -o match:"${TESTCASE}, ${exp}" -e ignore atf-run \
-v user=unprivileged helper
}
atf_test_case require_user_bad
require_user_bad_head()
{
atf_set "descr" "Tests that atf-run validates the require.user property" \
"when it is set to 'root'"
}
require_user_bad_body()
{
create_helper require_user
create_atffile helper
atf_test_case timeout
timeout_head()
{
atf_set "descr" "Tests that atf-run kills a test case that times out"
}
timeout_body()
{
create_helper timeout
create_atffile helper
atf_check -s eq:1 \
-o match:"${TESTCASE}, failed, .*timed out after 1 second" -e ignore \
atf-run -v statedir=$(pwd) helper
if [ -f finished ]; then
atf_fail "Test case was not killed after time out"
fi
}
atf_test_case timeout_forkexit
timeout_forkexit_head()
{
atf_set "descr" "Tests that atf-run deals gracefully with a test program" \
"that forks, exits, but the child process hangs"
}
timeout_forkexit_body()
{
create_helper timeout_forkexit
create_atffile helper
atf_check -s eq:0 -o match:"${TESTCASE}, passed" -e ignore atf-run \
-v statedir=$(pwd) helper
test -f parent-finished || atf_fail "Parent did not exit as expected"
test -f child-finished && atf_fail "Subprocess exited but it should have" \
"been forcibly terminated" || true
}