[netfilter-cvslog] r3537 - in trunk: nfsim/core nfsim-testsuite

rusty at netfilter.org rusty at netfilter.org
Sat Jan 1 09:28:42 CET 2005


Author: rusty at netfilter.org
Date: 2005-01-01 09:28:42 +0100 (Sat, 01 Jan 2005)
New Revision: 3537

Added:
   trunk/nfsim-testsuite/.gdbinit
   trunk/nfsim-testsuite/README
Modified:
   trunk/nfsim/core/failtest.c
Log:
Rename "failpattern" to "failpath" for consistency.
Add .gdbinit file for easy debugging.
Add README to describe writing and running tests.


Modified: trunk/nfsim/core/failtest.c
===================================================================
--- trunk/nfsim/core/failtest.c	2005-01-01 08:20:47 UTC (rev 3536)
+++ trunk/nfsim/core/failtest.c	2005-01-01 08:28:42 UTC (rev 3537)
@@ -41,8 +41,8 @@
 
 static LIST_HEAD(decisions);
 
-/* Failure pattern to follow (initialized by --failpattern). */
-static const char *failpattern = NULL, *orig_failpattern;
+/* Failure path to follow (initialized by --failpath). */
+static const char *failpath = NULL, *orig_failpath;
 
 /*** XML Argument:
     <section id="a:failtest">
@@ -55,8 +55,8 @@
      failure and success will be simulated: where we fail, the script
      will no doubt fail later which is OK, but we ensure that the
      entire system doesn't fall over or leak memory.  If an error is
-     found, the pattern of successes/failures will be printed out,
-     which can be replayed using <option>--failpattern</option></para>
+     found, the path of successes/failures will be printed out,
+     which can be replayed using <option>--failpath</option></para>
     </section>
 */
 static void cmdline_failtest(struct option *opt)
@@ -67,11 +67,11 @@
 
 
 /*** XML Argument:
-    <section id="a:failpattern">
-     <title><option>--failpattern
-      <replaceable>pattern</replaceable></option></title>
-     <subtitle>Replay a failure pattern</subtitle>
-     <para>Given a failure pattern, (from <option>--failtest</option>), this will
+    <section id="a:failpath">
+     <title><option>--failpath
+      <replaceable>path</replaceable></option></title>
+     <subtitle>Replay a failure path</subtitle>
+     <para>Given a failure path, (from <option>--failtest</option>), this will
       replay the sequence of sucesses/failures, allowing debugging.  The input
       should be the same as the original which caused the failure.
       </para>
@@ -81,58 +81,64 @@
      automatically.</para>
     </section>
 */
-static void cmdline_failpattern(struct option *opt)
+static void cmdline_failpath(struct option *opt)
 {
 	extern char *optarg;
 	if (!optarg)
 		barf("failtest option requires an argument");
-	orig_failpattern = failpattern = optarg;
+	orig_failpath = failpath = optarg;
 }
-cmdline_opt("failpattern", 1, 0, cmdline_failpattern);
+cmdline_opt("failpath", 1, 0, cmdline_failpath);
 
 bool get_failtest(void)
 {
 	return failtest;
 }
 
-static bool do_failpattern(const char *func)
+/* Separate function to make .gdbinit easier */
+static bool failpath_fail(void)
 {
-	if (*failpattern == '[') {
-		failpattern++;
-		if (strncmp(failpattern, func, strlen(func)) != 0
-		    || failpattern[strlen(func)] != ']')
-			barf("Failpattern expected %.*s not %s\n",
-			     strcspn(failpattern, "]"), failpattern, func);
-		failpattern += strlen(func) + 1;
+	return false;
+}
+
+static bool do_failpath(const char *func)
+{
+	if (*failpath == '[') {
+		failpath++;
+		if (strncmp(failpath, func, strlen(func)) != 0
+		    || failpath[strlen(func)] != ']')
+			barf("Failpath expected %.*s not %s\n",
+			     strcspn(failpath, "]"), failpath, func);
+		failpath += strlen(func) + 1;
 	}
 
-	if (*failpattern == ':') {
+	if (*failpath == ':') {
 		unsigned long line;
 		char *after;
-		failpattern++;
-		line = strtoul(failpattern, &after, 10);
+		failpath++;
+		line = strtoul(failpath, &after, 10);
 		if (*after != ':')
-			barf("Bad failure pattern line number %s\n",
-			     failpattern);
+			barf("Bad failure path line number %s\n",
+			     failpath);
 		if (line != tui_linenum)
 			barf("Unexpected line number %lu vs %u\n",
 			     line, tui_linenum);
-		failpattern = after+1;
+		failpath = after+1;
 	}
 
-	switch ((failpattern++)[0]) {
+	switch ((failpath++)[0]) {
 	case 'F':
 	case 'f':
-		return true;
+		return failpath_fail();
 	case 'S':
 	case 's':
 		return false;
 	case 0:
-		failpattern = NULL;
+		failpath = NULL;
 		return false;
 	default:
-		barf("Failpattern '%c' failed to pattern",
-		     failpattern[-1]);
+		barf("Failpath '%c' failed to path",
+		     failpath[-1]);
 	}
 }
 
@@ -199,13 +205,13 @@
 	if (suppress_failtest)
 		return false;
 
-	if (failpattern) {
-		p = strstr(orig_failpattern ?: "", location);
-		if (p && p <= failpattern
+	if (failpath) {
+		p = strstr(orig_failpath ?: "", location);
+		if (p && p <= failpath
 		    && p[-1] == '[' && p[strlen(location)] == ']')
 			return false;
 
-		return do_failpattern(location);
+		return do_failpath(location);
 	}
 
 	list_for_each_entry(i, &decisions, list)
@@ -229,8 +235,8 @@
 	if (suppress_failtest)
 		return false;
 
-	if (failpattern)
-		return do_failpattern(func);
+	if (failpath)
+		return do_failpath(func);
 
 	failpoints++;
 	if (!failtest)

Added: trunk/nfsim-testsuite/.gdbinit
===================================================================
--- trunk/nfsim-testsuite/.gdbinit	2005-01-01 08:20:47 UTC (rev 3536)
+++ trunk/nfsim-testsuite/.gdbinit	2005-01-01 08:28:42 UTC (rev 3537)
@@ -0,0 +1,5 @@
+# Break on FATAL
+break barf
+break barf_perror
+# Break on failure return from failpath.
+break failpath_fail

Added: trunk/nfsim-testsuite/README
===================================================================
--- trunk/nfsim-testsuite/README	2005-01-01 08:20:47 UTC (rev 3536)
+++ trunk/nfsim-testsuite/README	2005-01-01 08:28:42 UTC (rev 3537)
@@ -0,0 +1,168 @@
+Welcome to the Testsuite for IPv4 Netfilter and iptables!
+
+To run the testsuite from the repository, simply enter:
+
+	./test-kernel-source <kernel-directory>
+
+This will compile up nfsim (expected to be found in ../nfsim/) for
+that kernel, run all the tests using the iptables binary in
+../iptables/ if found, and print out the coverage results if gcov is
+available.
+
+Running Tests
+=============
+
+Here is a list of options to test-kernel-source:
+
+Environment variables:
+	CC=<ccompiler>: The compiler to compile nfsim with (default: gcc)
+	GCOV=<gcov>: The gcov to run (default: gcov)
+
+Arguments:
+	--failtest
+		Test kernel failure paths.  Much slower, but better
+		coverage.  nfsim will fork, and the child will test
+		the failure case.
+
+	-v
+		Verbose mode: print out test names, and errors from
+		tests we are expecting to fail.
+
+	-vv
+		Very verbose mode: print out all the lines of the test
+		as it is run.  Useful for debugging exactly what went
+		wrong with a test.
+
+	--time
+		Indicate the time that each test took.
+
+	[testname]
+		Start at this test.  Often combined with -vv to
+		analyze exactly what happened in a test.
+
+Debugging a Test
+----------------
+
+If a test fails, run just that test in very verbose mode with:
+
+	./test-kernel-source -vv <kernel-directory> <testname>
+
+If it's not a bug in the test itself, you can start a single test
+under the debugger like so:
+
+	gdb ../nfsim/simulator
+	(gdb) set args -ex 00simulator/01modules.sim
+
+The ".gdbinit" file in this directory will set breakpoints at various
+interesting points.
+
+If the test crashed or exited with a "FATAL:" message, this is
+sufficient for debugging.  You can choose not to specify <testname>,
+and enter the commands manually.
+
+If the test problem was found with "--failtest", it will print out a
+path of failures which lead to the problem, eg:
+
+	Child signalled 11 on failure path: [ip_conntrack_core.c:1342]:2:S[core/netfilter.c:100]:2:S[__malloc]:2:S[kmem_cache_create]:2:S[kmem_cache_create]:2:S[proc_create]:2:S[proc_create]:2:S[proc_create]:2:S[register_sysctl_table]:2:F
+
+You can invoke nfsim with this failure path "--failpath" argument, eg:
+	(gdb) set args -ex --failpath=[ip_conntrack_core.c:1342]:2:S[core/netfilter.c:100]:2:S[__malloc]:2:S[kmem_cache_create]:2:S[kmem_cache_create]:2:S[proc_create]:2:S[proc_create]:2:S[proc_create]:2:S[register_sysctl_table]:2:F <testname 00simulator/01modules.sim
+
+This will cause the same path of failures which lead to the failure,
+so you can debug it: in this case, ip_conntrack_core was erroneously
+trying to unregister_sysctl_table(NULL) when register_sysctl_table()
+failed.
+
+
+Writing Tests
+=============
+
+Tests are a series of nfsim commands and comments (lines which begin
+with a #).  In non-interactive mode, nfsim is usually invoked with
+"-e" meaning that a failure of any command will cause the script to
+fail.  Tests usually consist of one or more "expect" commands followed
+by the command itself.
+
+A special comment tells test-kernel-sources that a test is expected to
+fail on some iptables or kernel versions:
+	# XFAIL:linux:2.4*
+	# XFAIL:iptables:1.2*
+
+Tests should test all aspects of one feature at a time, even if it
+means they are longer.  This ensure that when something breaks,
+someone can tell exactly what failed.  Tests should be organized, so
+that they test each feature by itself before becoming more
+sophisticated and testing features which are expecting to interact.
+
+For example, consider testing the UDP match:
+
+	iptables -A FORWARD -p udp --sport 2 -j DROP
+	expect gen_ip send:eth1 *
+	gen_ip IF=eth0 192.168.0.3 192.168.1.2 0 17 1 2
+	expect gen_ip hook:NF_IP_FORWARD * NF_DROP *
+	gen_ip IF=eth0 192.168.0.3 192.168.1.2 0 17 2 2
+	iptables -D FORWARD -p udp --source-port 2 -j DROP
+
+This tests the --sport feature.  First that the rule doesn't match a
+packet it shouldn't, then that it does match a packet it should.  It
+also checks that --sport and --source-port are equivalent by using the
+second form to delete the rule (I was a little lazy).
+
+The next test would test the range feature of --sport, using "--sport
+2-5".  This allows us to test the boundary cases (1 and 2, 5 and 6) as
+well as a normal case inside the range (3).  Then, I cut and pasted
+these tests to test the inverse case, by reversing every "expect"
+line.
+
+Tests Which Ignore Failures
+---------------------------
+
+Because --failtest tries every combination of failures until a test
+terminates, some tests can cause it to take a very long time if a test
+ignores the failures which --failtest inserts.  There are, for
+example, around 11 ways a simple "iptables -A" command can fail:
+several calls to copy_from_user, copy_to_user, down_interruptible, and
+kmalloc.  This means 2^11, or 2048 different paths.  Fortunately, most
+of these failures cause iptables to fail immedately, so it's nowhere
+near this bad, only about 20 paths.
+
+Consider the following:
+	expect iptables iptables: command failed
+	iptables --some-bogus-option
+
+	expect iptables iptables: command failed
+	iptables --some-other-bogus-option
+
+Because the "expect" line is not sufficiently precise about how the
+command should fail, those 20 failure paths tried by --failtest do not
+cause the script to terminate.  The second command also has 20 failure
+paths which do not terminate, leading to 400 combinations which must
+be attempted!
+
+In fact, the failtest code will warn if the test seems to be
+continuing after three or more failures, and will print out the line
+on which the failure occurred, and the failpath, like so:
+
+	WARNING: test /tmp/test.sim ignores failures
+	 Line 2: [call_elem_hook:0]S[kmem_cache_alloc]F
+	 Line 3: [kmem_cache_alloc]F
+	 Line 10: [kmem_cache_alloc]S[kernelenv/kernelenv.c:116]F
+
+You can debug this by entering that just that one line of the test
+(say line 2) by hand into gdb like so:
+
+	(gdb) run -e --failpath=[call_elem_hook:0]S[kmem_cache_alloc]F
+
+The solution is two-fold:
+
+1) have an "expect" for every gen_ip line: gen_ip doesn't fail just
+   because the packet is dropped.
+
+2) if you're expecting failure, have a precise "expect" which will
+   only catch that failure, not others which --failtest will insert.
+   The "strace" command in nfsim is your friend here.
+
+
+Finally, have fun, and thanks for writing tests!
+
+Rusty.




More information about the netfilter-cvslog mailing list