[netfilter-cvslog] r3537 - in trunk: nfsim/core nfsim-testsuite
rusty at netfilter.org
rusty at netfilter.org
Sat Jan 1 09:28:42 CET 2005
Author: rusty at netfilter.org
Date: 2005-01-01 09:28:42 +0100 (Sat, 01 Jan 2005)
New Revision: 3537
Added:
trunk/nfsim-testsuite/.gdbinit
trunk/nfsim-testsuite/README
Modified:
trunk/nfsim/core/failtest.c
Log:
Rename "failpattern" to "failpath" for consistency.
Add .gdbinit file for easy debugging.
Add README to describe writing and running tests.
Modified: trunk/nfsim/core/failtest.c
===================================================================
--- trunk/nfsim/core/failtest.c 2005-01-01 08:20:47 UTC (rev 3536)
+++ trunk/nfsim/core/failtest.c 2005-01-01 08:28:42 UTC (rev 3537)
@@ -41,8 +41,8 @@
static LIST_HEAD(decisions);
-/* Failure pattern to follow (initialized by --failpattern). */
-static const char *failpattern = NULL, *orig_failpattern;
+/* Failure path to follow (initialized by --failpath). */
+static const char *failpath = NULL, *orig_failpath;
/*** XML Argument:
<section id="a:failtest">
@@ -55,8 +55,8 @@
failure and success will be simulated: where we fail, the script
will no doubt fail later which is OK, but we ensure that the
entire system doesn't fall over or leak memory. If an error is
- found, the pattern of successes/failures will be printed out,
- which can be replayed using <option>--failpattern</option></para>
+ found, the path of successes/failures will be printed out,
+ which can be replayed using <option>--failpath</option></para>
</section>
*/
static void cmdline_failtest(struct option *opt)
@@ -67,11 +67,11 @@
/*** XML Argument:
- <section id="a:failpattern">
- <title><option>--failpattern
- <replaceable>pattern</replaceable></option></title>
- <subtitle>Replay a failure pattern</subtitle>
- <para>Given a failure pattern, (from <option>--failtest</option>), this will
+ <section id="a:failpath">
+ <title><option>--failpath
+ <replaceable>path</replaceable></option></title>
+ <subtitle>Replay a failure path</subtitle>
+ <para>Given a failure path, (from <option>--failtest</option>), this will
replay the sequence of sucesses/failures, allowing debugging. The input
should be the same as the original which caused the failure.
</para>
@@ -81,58 +81,64 @@
automatically.</para>
</section>
*/
-static void cmdline_failpattern(struct option *opt)
+static void cmdline_failpath(struct option *opt)
{
extern char *optarg;
if (!optarg)
barf("failtest option requires an argument");
- orig_failpattern = failpattern = optarg;
+ orig_failpath = failpath = optarg;
}
-cmdline_opt("failpattern", 1, 0, cmdline_failpattern);
+cmdline_opt("failpath", 1, 0, cmdline_failpath);
bool get_failtest(void)
{
return failtest;
}
-static bool do_failpattern(const char *func)
+/* Separate function to make .gdbinit easier */
+static bool failpath_fail(void)
{
- if (*failpattern == '[') {
- failpattern++;
- if (strncmp(failpattern, func, strlen(func)) != 0
- || failpattern[strlen(func)] != ']')
- barf("Failpattern expected %.*s not %s\n",
- strcspn(failpattern, "]"), failpattern, func);
- failpattern += strlen(func) + 1;
+ return false;
+}
+
+static bool do_failpath(const char *func)
+{
+ if (*failpath == '[') {
+ failpath++;
+ if (strncmp(failpath, func, strlen(func)) != 0
+ || failpath[strlen(func)] != ']')
+ barf("Failpath expected %.*s not %s\n",
+ strcspn(failpath, "]"), failpath, func);
+ failpath += strlen(func) + 1;
}
- if (*failpattern == ':') {
+ if (*failpath == ':') {
unsigned long line;
char *after;
- failpattern++;
- line = strtoul(failpattern, &after, 10);
+ failpath++;
+ line = strtoul(failpath, &after, 10);
if (*after != ':')
- barf("Bad failure pattern line number %s\n",
- failpattern);
+ barf("Bad failure path line number %s\n",
+ failpath);
if (line != tui_linenum)
barf("Unexpected line number %lu vs %u\n",
line, tui_linenum);
- failpattern = after+1;
+ failpath = after+1;
}
- switch ((failpattern++)[0]) {
+ switch ((failpath++)[0]) {
case 'F':
case 'f':
- return true;
+ return failpath_fail();
case 'S':
case 's':
return false;
case 0:
- failpattern = NULL;
+ failpath = NULL;
return false;
default:
- barf("Failpattern '%c' failed to pattern",
- failpattern[-1]);
+ barf("Failpath '%c' failed to path",
+ failpath[-1]);
}
}
@@ -199,13 +205,13 @@
if (suppress_failtest)
return false;
- if (failpattern) {
- p = strstr(orig_failpattern ?: "", location);
- if (p && p <= failpattern
+ if (failpath) {
+ p = strstr(orig_failpath ?: "", location);
+ if (p && p <= failpath
&& p[-1] == '[' && p[strlen(location)] == ']')
return false;
- return do_failpattern(location);
+ return do_failpath(location);
}
list_for_each_entry(i, &decisions, list)
@@ -229,8 +235,8 @@
if (suppress_failtest)
return false;
- if (failpattern)
- return do_failpattern(func);
+ if (failpath)
+ return do_failpath(func);
failpoints++;
if (!failtest)
Added: trunk/nfsim-testsuite/.gdbinit
===================================================================
--- trunk/nfsim-testsuite/.gdbinit 2005-01-01 08:20:47 UTC (rev 3536)
+++ trunk/nfsim-testsuite/.gdbinit 2005-01-01 08:28:42 UTC (rev 3537)
@@ -0,0 +1,5 @@
+# Break on FATAL
+break barf
+break barf_perror
+# Break on failure return from failpath.
+break failpath_fail
Added: trunk/nfsim-testsuite/README
===================================================================
--- trunk/nfsim-testsuite/README 2005-01-01 08:20:47 UTC (rev 3536)
+++ trunk/nfsim-testsuite/README 2005-01-01 08:28:42 UTC (rev 3537)
@@ -0,0 +1,168 @@
+Welcome to the Testsuite for IPv4 Netfilter and iptables!
+
+To run the testsuite from the repository, simply enter:
+
+ ./test-kernel-source <kernel-directory>
+
+This will compile up nfsim (expected to be found in ../nfsim/) for
+that kernel, run all the tests using the iptables binary in
+../iptables/ if found, and print out the coverage results if gcov is
+available.
+
+Running Tests
+=============
+
+Here is a list of options to test-kernel-source:
+
+Environment variables:
+ CC=<ccompiler>: The compiler to compile nfsim with (default: gcc)
+ GCOV=<gcov>: The gcov to run (default: gcov)
+
+Arguments:
+ --failtest
+ Test kernel failure paths. Much slower, but better
+ coverage. nfsim will fork, and the child will test
+ the failure case.
+
+ -v
+ Verbose mode: print out test names, and errors from
+ tests we are expecting to fail.
+
+ -vv
+ Very verbose mode: print out all the lines of the test
+ as it is run. Useful for debugging exactly what went
+ wrong with a test.
+
+ --time
+ Indicate the time that each test took.
+
+ [testname]
+ Start at this test. Often combined with -vv to
+ analyze exactly what happened in a test.
+
+Debugging a Test
+----------------
+
+If a test fails, run just that test in very verbose mode with:
+
+ ./test-kernel-source -vv <kernel-directory> <testname>
+
+If it's not a bug in the test itself, you can start a single test
+under the debugger like so:
+
+ gdb ../nfsim/simulator
+ (gdb) set args -ex 00simulator/01modules.sim
+
+The ".gdbinit" file in this directory will set breakpoints at various
+interesting points.
+
+If the test crashed or exited with a "FATAL:" message, this is
+sufficient for debugging. You can choose not to specify <testname>,
+and enter the commands manually.
+
+If the test problem was found with "--failtest", it will print out a
+path of failures which lead to the problem, eg:
+
+ Child signalled 11 on failure path: [ip_conntrack_core.c:1342]:2:S[core/netfilter.c:100]:2:S[__malloc]:2:S[kmem_cache_create]:2:S[kmem_cache_create]:2:S[proc_create]:2:S[proc_create]:2:S[proc_create]:2:S[register_sysctl_table]:2:F
+
+You can invoke nfsim with this failure path "--failpath" argument, eg:
+ (gdb) set args -ex --failpath=[ip_conntrack_core.c:1342]:2:S[core/netfilter.c:100]:2:S[__malloc]:2:S[kmem_cache_create]:2:S[kmem_cache_create]:2:S[proc_create]:2:S[proc_create]:2:S[proc_create]:2:S[register_sysctl_table]:2:F <testname 00simulator/01modules.sim
+
+This will cause the same path of failures which lead to the failure,
+so you can debug it: in this case, ip_conntrack_core was erroneously
+trying to unregister_sysctl_table(NULL) when register_sysctl_table()
+failed.
+
+
+Writing Tests
+=============
+
+Tests are a series of nfsim commands and comments (lines which begin
+with a #). In non-interactive mode, nfsim is usually invoked with
+"-e" meaning that a failure of any command will cause the script to
+fail. Tests usually consist of one or more "expect" commands followed
+by the command itself.
+
+A special comment tells test-kernel-sources that a test is expected to
+fail on some iptables or kernel versions:
+ # XFAIL:linux:2.4*
+ # XFAIL:iptables:1.2*
+
+Tests should test all aspects of one feature at a time, even if it
+means they are longer. This ensure that when something breaks,
+someone can tell exactly what failed. Tests should be organized, so
+that they test each feature by itself before becoming more
+sophisticated and testing features which are expecting to interact.
+
+For example, consider testing the UDP match:
+
+ iptables -A FORWARD -p udp --sport 2 -j DROP
+ expect gen_ip send:eth1 *
+ gen_ip IF=eth0 192.168.0.3 192.168.1.2 0 17 1 2
+ expect gen_ip hook:NF_IP_FORWARD * NF_DROP *
+ gen_ip IF=eth0 192.168.0.3 192.168.1.2 0 17 2 2
+ iptables -D FORWARD -p udp --source-port 2 -j DROP
+
+This tests the --sport feature. First that the rule doesn't match a
+packet it shouldn't, then that it does match a packet it should. It
+also checks that --sport and --source-port are equivalent by using the
+second form to delete the rule (I was a little lazy).
+
+The next test would test the range feature of --sport, using "--sport
+2-5". This allows us to test the boundary cases (1 and 2, 5 and 6) as
+well as a normal case inside the range (3). Then, I cut and pasted
+these tests to test the inverse case, by reversing every "expect"
+line.
+
+Tests Which Ignore Failures
+---------------------------
+
+Because --failtest tries every combination of failures until a test
+terminates, some tests can cause it to take a very long time if a test
+ignores the failures which --failtest inserts. There are, for
+example, around 11 ways a simple "iptables -A" command can fail:
+several calls to copy_from_user, copy_to_user, down_interruptible, and
+kmalloc. This means 2^11, or 2048 different paths. Fortunately, most
+of these failures cause iptables to fail immedately, so it's nowhere
+near this bad, only about 20 paths.
+
+Consider the following:
+ expect iptables iptables: command failed
+ iptables --some-bogus-option
+
+ expect iptables iptables: command failed
+ iptables --some-other-bogus-option
+
+Because the "expect" line is not sufficiently precise about how the
+command should fail, those 20 failure paths tried by --failtest do not
+cause the script to terminate. The second command also has 20 failure
+paths which do not terminate, leading to 400 combinations which must
+be attempted!
+
+In fact, the failtest code will warn if the test seems to be
+continuing after three or more failures, and will print out the line
+on which the failure occurred, and the failpath, like so:
+
+ WARNING: test /tmp/test.sim ignores failures
+ Line 2: [call_elem_hook:0]S[kmem_cache_alloc]F
+ Line 3: [kmem_cache_alloc]F
+ Line 10: [kmem_cache_alloc]S[kernelenv/kernelenv.c:116]F
+
+You can debug this by entering that just that one line of the test
+(say line 2) by hand into gdb like so:
+
+ (gdb) run -e --failpath=[call_elem_hook:0]S[kmem_cache_alloc]F
+
+The solution is two-fold:
+
+1) have an "expect" for every gen_ip line: gen_ip doesn't fail just
+ because the packet is dropped.
+
+2) if you're expecting failure, have a precise "expect" which will
+ only catch that failure, not others which --failtest will insert.
+ The "strace" command in nfsim is your friend here.
+
+
+Finally, have fun, and thanks for writing tests!
+
+Rusty.
More information about the netfilter-cvslog
mailing list