1 /* tinytest.c -- Copyright 2009-2010 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include <stdio.h> 27 #include <stdlib.h> 28 #include <string.h> 29 #include <assert.h> 30 31 #ifdef WIN32 32 #include <windows.h> 33 #else 34 #include <sys/types.h> 35 #include <sys/wait.h> 36 #include <unistd.h> 37 #endif 38 39 #ifndef __GNUC__ 40 #define __attribute__(x) 41 #endif 42 43 #ifdef TINYTEST_LOCAL 44 #include "tinytest_local.h" 45 #endif 46 #include "tinytest.h" 47 #include "tinytest_macros.h" 48 49 #define LONGEST_TEST_NAME 16384 50 51 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 52 static int n_ok = 0; /**< Number of tests that have passed */ 53 static int n_bad = 0; /**< Number of tests that have failed. */ 54 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 55 56 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 57 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 58 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 59 const char *verbosity_flag = ""; 60 61 enum outcome { SKIP=2, OK=1, FAIL=0 }; 62 static enum outcome cur_test_outcome = 0; 63 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 64 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 65 const char *cur_test_name = NULL; 66 67 #ifdef WIN32 68 /** Pointer to argv[0] for win32. */ 69 static const char *commandname = NULL; 70 #endif 71 72 static void usage(struct testgroup_t *groups, int list_groups) 73 __attribute__((noreturn)); 74 75 static enum outcome 76 _testcase_run_bare(const struct testcase_t *testcase) 77 { 78 void *env = NULL; 79 int outcome; 80 if (testcase->setup) { 81 env = testcase->setup->setup_fn(testcase); 82 if (!env) 83 return FAIL; 84 else if (env == (void*)TT_SKIP) 85 return SKIP; 86 } 87 88 cur_test_outcome = OK; 89 testcase->fn(env); 90 outcome = cur_test_outcome; 91 92 if (testcase->setup) { 93 if (testcase->setup->cleanup_fn(testcase, env) == 0) 94 outcome = FAIL; 95 } 96 97 return outcome; 98 } 99 100 #define MAGIC_EXITCODE 42 101 102 static enum outcome 103 _testcase_run_forked(const struct testgroup_t *group, 104 const struct testcase_t *testcase) 105 { 106 #ifdef WIN32 107 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 108 we'll invoke our own exe (whose name we recall from the command 109 line) with a command line that tells it to run just the test we 110 want, and this time without forking. 111 112 (No, threads aren't an option. The whole point of forking is to 113 share no state between tests.) 114 */ 115 int ok; 116 char buffer[LONGEST_TEST_NAME+256]; 117 STARTUPINFOA si; 118 PROCESS_INFORMATION info; 119 DWORD exitcode; 120 121 if (!in_tinytest_main) { 122 printf("\nERROR. On Windows, _testcase_run_forked must be" 123 " called from within tinytest_main.\n"); 124 abort(); 125 } 126 if (opt_verbosity>0) 127 printf("[forking] "); 128 129 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 130 commandname, verbosity_flag, group->prefix, testcase->name); 131 132 memset(&si, 0, sizeof(si)); 133 memset(&info, 0, sizeof(info)); 134 si.cb = sizeof(si); 135 136 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 137 0, NULL, NULL, &si, &info); 138 if (!ok) { 139 printf("CreateProcess failed!\n"); 140 return 0; 141 } 142 WaitForSingleObject(info.hProcess, INFINITE); 143 GetExitCodeProcess(info.hProcess, &exitcode); 144 CloseHandle(info.hProcess); 145 CloseHandle(info.hThread); 146 if (exitcode == 0) 147 return OK; 148 else if (exitcode == MAGIC_EXITCODE) 149 return SKIP; 150 else 151 return FAIL; 152 #else 153 int outcome_pipe[2]; 154 pid_t pid; 155 (void)group; 156 157 if (pipe(outcome_pipe)) 158 perror("opening pipe"); 159 160 if (opt_verbosity>0) 161 printf("[forking] "); 162 pid = fork(); 163 if (!pid) { 164 /* child. */ 165 int test_r, write_r; 166 char b[1]; 167 close(outcome_pipe[0]); 168 test_r = _testcase_run_bare(testcase); 169 assert(0<=(int)test_r && (int)test_r<=2); 170 b[0] = "NYS"[test_r]; 171 write_r = (int)write(outcome_pipe[1], b, 1); 172 if (write_r != 1) { 173 perror("write outcome to pipe"); 174 exit(1); 175 } 176 exit(0); 177 } else { 178 /* parent */ 179 int status, r; 180 char b[1]; 181 /* Close this now, so that if the other side closes it, 182 * our read fails. */ 183 close(outcome_pipe[1]); 184 r = (int)read(outcome_pipe[0], b, 1); 185 if (r == 0) { 186 printf("[Lost connection!] "); 187 return 0; 188 } else if (r != 1) { 189 perror("read outcome from pipe"); 190 } 191 waitpid(pid, &status, 0); 192 close(outcome_pipe[0]); 193 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 194 } 195 #endif 196 } 197 198 int 199 testcase_run_one(const struct testgroup_t *group, 200 const struct testcase_t *testcase) 201 { 202 enum outcome outcome; 203 204 if (testcase->flags & TT_SKIP) { 205 if (opt_verbosity>0) 206 printf("%s%s: SKIPPED\n", 207 group->prefix, testcase->name); 208 ++n_skipped; 209 return SKIP; 210 } 211 212 if (opt_verbosity>0 && !opt_forked) { 213 printf("%s%s: ", group->prefix, testcase->name); 214 } else { 215 if (opt_verbosity==0) printf("."); 216 cur_test_prefix = group->prefix; 217 cur_test_name = testcase->name; 218 } 219 220 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 221 outcome = _testcase_run_forked(group, testcase); 222 } else { 223 outcome = _testcase_run_bare(testcase); 224 } 225 226 if (outcome == OK) { 227 ++n_ok; 228 if (opt_verbosity>0 && !opt_forked) 229 puts(opt_verbosity==1?"OK":""); 230 } else if (outcome == SKIP) { 231 ++n_skipped; 232 if (opt_verbosity>0 && !opt_forked) 233 puts("SKIPPED"); 234 } else { 235 ++n_bad; 236 if (!opt_forked) 237 printf("\n [%s FAILED]\n", testcase->name); 238 } 239 240 if (opt_forked) { 241 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 242 } else { 243 return (int)outcome; 244 } 245 } 246 247 int 248 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) 249 { 250 int i, j; 251 size_t length = LONGEST_TEST_NAME; 252 char fullname[LONGEST_TEST_NAME]; 253 int found=0; 254 if (strstr(arg, "..")) 255 length = strstr(arg,"..")-arg; 256 for (i=0; groups[i].prefix; ++i) { 257 for (j=0; groups[i].cases[j].name; ++j) { 258 snprintf(fullname, sizeof(fullname), "%s%s", 259 groups[i].prefix, groups[i].cases[j].name); 260 if (!flag) /* Hack! */ 261 printf(" %s\n", fullname); 262 if (!strncmp(fullname, arg, length)) { 263 groups[i].cases[j].flags |= flag; 264 ++found; 265 } 266 } 267 } 268 return found; 269 } 270 271 static void 272 usage(struct testgroup_t *groups, int list_groups) 273 { 274 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 275 puts(" Specify tests by name, or using a prefix ending with '..'"); 276 puts(" To skip a test, list give its name prefixed with a colon."); 277 puts(" Use --list-tests for a list of tests."); 278 if (list_groups) { 279 puts("Known tests are:"); 280 _tinytest_set_flag(groups, "..", 0); 281 } 282 exit(0); 283 } 284 285 int 286 tinytest_main(int c, const char **v, struct testgroup_t *groups) 287 { 288 int i, j, n=0; 289 290 #ifdef WIN32 291 commandname = v[0]; 292 #endif 293 for (i=1; i<c; ++i) { 294 if (v[i][0] == '-') { 295 if (!strcmp(v[i], "--RUNNING-FORKED")) { 296 opt_forked = 1; 297 } else if (!strcmp(v[i], "--no-fork")) { 298 opt_nofork = 1; 299 } else if (!strcmp(v[i], "--quiet")) { 300 opt_verbosity = -1; 301 verbosity_flag = "--quiet"; 302 } else if (!strcmp(v[i], "--verbose")) { 303 opt_verbosity = 2; 304 verbosity_flag = "--verbose"; 305 } else if (!strcmp(v[i], "--terse")) { 306 opt_verbosity = 0; 307 verbosity_flag = "--terse"; 308 } else if (!strcmp(v[i], "--help")) { 309 usage(groups, 0); 310 } else if (!strcmp(v[i], "--list-tests")) { 311 usage(groups, 1); 312 } else { 313 printf("Unknown option %s. Try --help\n",v[i]); 314 return -1; 315 } 316 } else { 317 const char *test = v[i]; 318 int flag = _TT_ENABLED; 319 if (test[0] == ':') { 320 ++test; 321 flag = TT_SKIP; 322 } else { 323 ++n; 324 } 325 if (!_tinytest_set_flag(groups, test, flag)) { 326 printf("No such test as %s!\n", v[i]); 327 return -1; 328 } 329 } 330 } 331 if (!n) 332 _tinytest_set_flag(groups, "..", _TT_ENABLED); 333 334 setvbuf(stdout, NULL, _IONBF, 0); 335 336 ++in_tinytest_main; 337 for (i=0; groups[i].prefix; ++i) 338 for (j=0; groups[i].cases[j].name; ++j) 339 if (groups[i].cases[j].flags & _TT_ENABLED) 340 testcase_run_one(&groups[i], 341 &groups[i].cases[j]); 342 343 --in_tinytest_main; 344 345 if (opt_verbosity==0) 346 puts(""); 347 348 if (n_bad) 349 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 350 n_bad+n_ok,n_skipped); 351 else if (opt_verbosity >= 1) 352 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 353 354 return (n_bad == 0) ? 0 : 1; 355 } 356 357 int 358 _tinytest_get_verbosity(void) 359 { 360 return opt_verbosity; 361 } 362 363 void 364 _tinytest_set_test_failed(void) 365 { 366 if (opt_verbosity <= 0 && cur_test_name) { 367 if (opt_verbosity==0) puts(""); 368 printf("%s%s: ", cur_test_prefix, cur_test_name); 369 cur_test_name = NULL; 370 } 371 cur_test_outcome = 0; 372 } 373 374 void 375 _tinytest_set_test_skipped(void) 376 { 377 if (cur_test_outcome==OK) 378 cur_test_outcome = SKIP; 379 } 380 381