Make it possible to run individual tests.

Fixes #100.

Makefile

@@ -97,15 +97,21 @@ test/echo.o: test/echo.c test/echo.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c test/echo.c -o test/echo.o
 
 
-.PHONY: clean clean-platform distclean distclean-platform test benchmark
+.PHONY: clean clean-platform distclean distclean-platform test bench
 
 
 test: test/run-tests$(E)
 	test/run-tests
 
+test-%:	test/run-tests$(E)
+	test/run-tests $(@:test-%=%)
+
 bench: test/run-benchmarks$(E)
 	test/run-benchmarks
 
+bench-%:	test/run-benchmarks$(E)
+	test/run-benchmarks $(@:bench-%=%)
+
 clean: clean-platform
 	$(RM) -f src/*.o *.a test/run-tests$(E) test/run-benchmarks$(E)
 

test/run-benchmarks.c

@@ -34,26 +34,14 @@
 
 
 int main(int argc, char **argv) {
-  task_entry_t *task;
-
   platform_init(argc, argv);
 
-  if (argc > 1) {
+  switch (argc) {
-    /* A specific process was requested. */
+  case 1: return run_tests(BENCHMARK_TIMEOUT, 1);
-    return run_process(argv[1]);
+  case 2: return run_test(argv[1], BENCHMARK_TIMEOUT, 1);
-
+  case 3: return run_test_part(argv[1], argv[2]);
-  } else {
+  default:
-    /* Run all benchmarks. */
+    LOGF("Too many arguments.\n");
-    task = (task_entry_t*)&TASKS;
+    return 1;
-    for (task = (task_entry_t*)&TASKS; task->main; task++) {
-      if (task->is_helper) {
-        continue;
-      }
-
-      run_task(task, BENCHMARK_TIMEOUT, 1);
-    }
-    LOG("Done.\n");
-
-    return 0;
   }
 }

test/run-tests.c

@@ -33,54 +33,15 @@
 #define TEST_TIMEOUT  5000
 
 
-static void log_progress(int total, int passed, int failed, char* name) {
-  LOGF("[%% %3d|+ %3d|- %3d]: %s", (passed + failed) / total * 100,
-      passed, failed, name);
-}
-
-
 int main(int argc, char **argv) {
-  int total, passed, failed;
-  task_entry_t* task;
-
   platform_init(argc, argv);
 
-  if (argc > 1) {
+  switch (argc) {
-    /* A specific process was requested. */
+  case 1: return run_tests(TEST_TIMEOUT, 0);
-    return run_process(argv[1]);
+  case 2: return run_test(argv[1], TEST_TIMEOUT, 0);
-
+  case 3: return run_test_part(argv[1], argv[2]);
-  } else {
+  default:
-    /* Count the number of tests. */
+    LOGF("Too many arguments.\n");
-    total = 0;
+    return 1;
-    task = (task_entry_t*)&TASKS;
-    for (task = (task_entry_t*)&TASKS; task->main; task++) {
-      if (!task->is_helper) {
-        total++;
-      }
-    }
-
-    /* Run all tests. */
-    passed = 0;
-    failed = 0;
-    task = (task_entry_t*)&TASKS;
-    for (task = (task_entry_t*)&TASKS; task->main; task++) {
-      if (task->is_helper) {
-        continue;
-      }
-
-      rewind_cursor();
-      log_progress(total, passed, failed, task->task_name);
-
-      if (run_task(task, TEST_TIMEOUT, 0)) {
-        passed++;
-      } else {
-        failed++;
-      }
-    }
-
-    rewind_cursor();
-    log_progress(total, passed, failed, "Done.\n");
-
-    return 0;
   }
 }

test/runner-unix.c

@@ -68,9 +68,9 @@ void platform_init(int argc, char **argv) {
 }
 
 
-/* Invoke "arv[0] test-name". Store process info in *p. */
+/* Invoke "argv[0] test-name [test-part]". Store process info in *p. */
 /* Make sure that all stdio output of the processes is buffered up. */
-int process_start(char* name, process_info_t* p) {
+int process_start(char* name, char* part, process_info_t* p) {
   FILE* stdout_file = tmpfile();
   if (!stdout_file) {
     perror("tmpfile");
@@ -92,7 +92,7 @@ int process_start(char* name, process_info_t* p) {
     dup2(fileno(stdout_file), STDOUT_FILENO);
     dup2(fileno(stdout_file), STDERR_FILENO);
 
-    char* args[3] = { executable_path, name, NULL };
+    char* args[] = { executable_path, name, part, NULL };
     execvp(executable_path, args);
     perror("execvp()");
     _exit(127);

test/runner-win.c

@@ -52,7 +52,7 @@ void platform_init(int argc, char **argv) {
 }
 
 
-int process_start(char *name, process_info_t *p) {
+int process_start(char *name, char *part, process_info_t *p) {
   HANDLE file = INVALID_HANDLE_VALUE;
   HANDLE nul = INVALID_HANDLE_VALUE;
   WCHAR path[MAX_PATH], filename[MAX_PATH];
@@ -97,12 +97,24 @@ int process_start(char *name, process_info_t *p) {
   if (result == 0 || result == sizeof(image))
     goto error;
 
-  if (_snwprintf((wchar_t*)&args,
+  if (part) {
-                 sizeof(args) / sizeof(wchar_t),
+    if (_snwprintf((wchar_t*)args,
-                 L"\"%s\" %S",
+                   sizeof(args) / sizeof(wchar_t),
-                 image,
+                   L"\"%s\" %S %S",
-                 name) < 0)
+                   image,
-    goto error;
+                   name,
+                   part) < 0) {
+      goto error;
+    }
+  } else {
+    if (_snwprintf((wchar_t*)args,
+                   sizeof(args) / sizeof(wchar_t),
+                   L"\"%s\" %S",
+                   image,
+                   name) < 0) {
+      goto error;
+    }
+  }
 
   memset((void*)&si, 0, sizeof(si));
   si.cb = sizeof(si);

test/runner.c

@@ -26,102 +26,155 @@
 
 char executable_path[PATHMAX] = { '\0' };
 
-/* Start a specific process declared by TEST_ENTRY or TEST_HELPER. */
-/* Returns the exit code of the specific process. */
-int run_process(char* name) {
-  task_entry_t *test;
 
-  for (test = (task_entry_t*)&TASKS; test->main; test++) {
+static void log_progress(int total, int passed, int failed, const char* name) {
-    if (strcmp(name, test->process_name) == 0) {
+  LOGF("[%% %3d|+ %3d|- %3d]: %s", (passed + failed) / total * 100,
-      return test->main();
+      passed, failed, name);
-    }
-  }
-
-  LOGF("Test process %s not found!\n", name);
-  return 255;
 }
 
 
-/*
+int run_tests(int timeout, int benchmark_output) {
- * Runs all processes associated with a particular test or benchmark.
+  int total, passed, failed;
- * It returns 1 if the test succeeded, 0 if it failed.
+  task_entry_t* task;
- * If the test fails it prints diagnostic information.
- * If benchmark_output is nonzero, the output from the main process is
- * always shown.
- */
-int run_task(task_entry_t *test, int timeout, int benchmark_output) {
-  int i, result, success;
-  char errmsg[256];
-  task_entry_t *helper;
-  int process_count;
-  process_info_t processes[MAX_PROCESSES];
-  process_info_t *main_process;
 
-  success = 0;
+  /* Count the number of tests. */
-
+  total = 0;
-  process_count = 0;
+  for (task = TASKS; task->main; task++) {
-
+    if (!task->is_helper) {
-  /* Start all helpers for this test first. */
+      total++;
-  for (helper = (task_entry_t*)&TASKS; helper->main; helper++) {
-    if (helper->is_helper &&
-        strcmp(test->task_name, helper->task_name) == 0) {
-      if (process_start(helper->process_name, &processes[process_count]) == -1) {
-        snprintf((char*)&errmsg,
-                 sizeof(errmsg),
-                 "process `%s` failed to start.",
-                 helper->process_name);
-        goto finalize;
-      }
-      process_count++;
     }
   }
 
-  /* Wait a little bit to allow servers to start. Racy. */
+  /* Run all tests. */
-  uv_sleep(100);
+  passed = 0;
+  failed = 0;
+  for (task = TASKS; task->main; task++) {
+    if (task->is_helper) {
+      continue;
+    }
 
-  /* Start the main test process. */
+    rewind_cursor();
-  if (process_start(test->process_name, &processes[process_count]) == -1) {
+    log_progress(total, passed, failed, task->task_name);
-    snprintf((char*)&errmsg, sizeof(errmsg), "process `%s` failed to start.",
+
-        test->process_name);
+    if (run_test(task->task_name, timeout, benchmark_output) == 0) {
-    goto finalize;
+      passed++;
+    } else {
+      failed++;
+    }
   }
-  main_process = &processes[process_count];
-  process_count++;
 
-  /* Wait for the main process to terminate. */
+  rewind_cursor();
-  result = process_wait(main_process, 1, timeout);
+  log_progress(total, passed, failed, "Done.\n");
+
+  return 0;
+}
+
+
+int run_test(const char* test, int timeout, int benchmark_output) {
+  char errmsg[1024] = "no error";
+  process_info_t processes[1024];
+  process_info_t *main_proc;
+  task_entry_t* task;
+  int process_count;
+  int result;
+  int status;
+  int i;
+
+  status = 255;
+  process_count = 0;
+
+  /* Start the helpers first. */
+  for (task = TASKS; task->main; task++) {
+    if (strcmp(test, task->task_name) != 0) {
+      continue;
+    }
+
+    /* Skip the test itself. */
+    if (!task->is_helper) {
+      continue;
+    }
+
+    if (process_start(task->task_name,
+                      task->process_name,
+                      &processes[process_count]) == -1) {
+      snprintf(errmsg,
+               sizeof errmsg,
+               "Process `%s` failed to start.",
+               task->process_name);
+      goto out;
+    }
+
+    process_count++;
+  }
+
+  /* Give the helpers time to settle. Race-y, fix this. */
+  uv_sleep(250);
+
+  /* Now start the test itself. */
+  for (main_proc = NULL, task = TASKS; task->main; task++) {
+    if (strcmp(test, task->task_name) != 0) {
+      continue;
+    }
+
+    if (task->is_helper) {
+      continue;
+    }
+
+    if (process_start(task->task_name,
+                      task->process_name,
+                      &processes[process_count]) == -1) {
+      snprintf(errmsg,
+               sizeof errmsg,
+               "Process `%s` failed to start.",
+               task->process_name);
+      goto out;
+    }
+
+    main_proc = &processes[process_count];
+    process_count++;
+    break;
+  }
+
+  if (main_proc == NULL) {
+    snprintf(errmsg,
+             sizeof errmsg,
+             "No test with that name: %s",
+             test);
+    goto out;
+  }
+
+  result = process_wait(main_proc, 1, timeout);
   if (result == -1) {
     FATAL("process_wait failed");
   } else if (result == -2) {
-    snprintf((char*)&errmsg, sizeof(errmsg), "timeout.");
+    /* Don't have to clean up the process, process_wait() has killed it. */
-    goto finalize;
+    snprintf(errmsg,
+             sizeof errmsg,
+             "timeout");
+    goto out;
   }
 
-  /* Reap the main process. */
+  status = process_reap(main_proc);
-  result = process_reap(main_process);
+  if (status != 0) {
-  if (result != 0) {
+    snprintf(errmsg,
-    snprintf((char*)&errmsg, sizeof(errmsg), "exit code %d.", result);
+             sizeof errmsg,
-    goto finalize;
+             "exit code %d",
+             status);
   }
 
-  /* Yes! did it. */
+out:
-  success = 1;
+  /* Reap running processes except the main process, it's already dead. */
-
+  for (i = 0; i < process_count - 1; i++) {
-finalize:
-  /* Kill all (helper) processes that are still running. */
-  for (i = 0; i < process_count; i++) {
-    /* If terminate fails the process is probably already closed. */
     process_terminate(&processes[i]);
   }
 
-  /* Wait until all processes have really terminated. */
+  if (process_wait(processes, process_count - 1, -1) < 0) {
-  if (process_wait((process_info_t*)&processes, process_count, -1) < 0) {
     FATAL("process_wait failed");
   }
 
   /* Show error and output from processes if the test failed. */
-  if (!success) {
+  if (status != 0) {
-    LOGF("\n`%s` failed: %s\n", test->task_name, errmsg);
+    LOGF("\n`%s` failed: %s\n", test, errmsg);
 
     for (i = 0; i < process_count; i++) {
       switch (process_output_size(&processes[i])) {
@@ -142,30 +195,25 @@ finalize:
       }
     }
     LOG("=============================================================\n");
+  }
 
-  /* In benchmark mode show concise output from the main process. */
+  return status;
-  } else if (benchmark_output) {
+}
-    switch (process_output_size(main_process)) {
-     case -1:
-      LOGF("%s: (unavailabe)\n", test->task_name);
-      break;
 
-     case 0:
-      LOGF("%s: (no output)\n", test->task_name);
-      break;
 
-     default:
+/* Returns the status code of the task part
-      for (i = 0; i < process_count; i++) {
+ * or 255 if no matching task was not found.
-        process_copy_output(&processes[i], fileno(stderr));
+ */
-      }
+int run_test_part(const char* test, const char* part) {
-      break;
+  task_entry_t* task;
+
+  for (task = TASKS; task->main; task++) {
+    if (strcmp(test, task->task_name) == 0
+        && strcmp(part, task->process_name) == 0) {
+      return task->main();
     }
   }
 
-  /* Clean up all process handles. */
+  LOGF("No test part with that name: %s:%s\n", test, part);
-  for (i = 0; i < process_count; i++) {
+  return 255;
-    process_cleanup(&processes[i]);
-  }
-
-  return success;
 }

test/runner.h

@@ -41,13 +41,6 @@ typedef struct {
 } task_entry_t, bench_entry_t;
 
 
-/* Runs an individual task; returns 1 if the test succeeded, 0 if it failed. */
-/* If the test fails it prints diagnostic information. */
-/* If benchmark_output is nonzero, the output from the main process is
-/* always shown. */
-int run_task(task_entry_t *test, int timeout, int benchmark_output);
-
-
 /*
  * Macros used by test-list.h and benchmark-list.h.
  */
@@ -95,13 +88,20 @@ extern char executable_path[PATHMAX];
 /* The array that is filled by test-list.h or benchmark-list.h */
 extern task_entry_t TASKS[];
 
-/* Start a specific process declared by TEST_ENTRY or TEST_HELPER. */
+/*
-/* Returns the exit code of the specific process. */
+ * Run all tests.
-int run_task(task_entry_t *test, int timeout, int benchmark_output);
+ */
+int run_tests(int timeout, int benchmark_output);
 
-/* Start a specific process declared by TEST_ENTRY or TEST_HELPER. */
+/*
-/* Returns the exit code of the specific process. */
+ * Run a single test. Starts up any helpers.
-int run_process(char* name);
+ */
+int run_test(const char* test, int timeout, int benchmark_output);
+
+/*
+ * Run a test part, i.e. the test or one of its helpers.
+ */
+int run_test_part(const char* test, const char* part);
 
 
 /*
@@ -113,9 +113,9 @@ int run_process(char* name);
 /* Do platform-specific initialization. */
 void platform_init();
 
-/* Invoke "arv[0] test-name". Store process info in *p. */
+/* Invoke "argv[0] test-name [test-part]". Store process info in *p. */
 /* Make sure that all stdio output of the processes is buffered up. */
-int process_start(char *name, process_info_t *p);
+int process_start(char *name, char* part, process_info_t *p);
 
 /* Wait for all `n` processes in `vec` to terminate. */
 /* Time out after `timeout` msec, or never if timeout == -1 */