1 /* tinytest.c -- Copyright 2009-2010 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include <stdio.h> 27 #include <stdlib.h> 28 #include <string.h> 29 #include <assert.h> 30 31 #ifdef TINYTEST_LOCAL 32 #include "tinytest_local.h" 33 #endif 34 35 #ifdef WIN32 36 #include <windows.h> 37 #else 38 #include <sys/types.h> 39 #include <sys/wait.h> 40 #include <unistd.h> 41 #endif 42 43 #ifndef __GNUC__ 44 #define __attribute__(x) 45 #endif 46 47 #include "tinytest.h" 48 #include "tinytest_macros.h" 49 50 #define LONGEST_TEST_NAME 16384 51 52 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 53 static int n_ok = 0; /**< Number of tests that have passed */ 54 static int n_bad = 0; /**< Number of tests that have failed. */ 55 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 56 57 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 58 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 59 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 60 const char *verbosity_flag = ""; 61 62 enum outcome { SKIP=2, OK=1, FAIL=0 }; 63 static enum outcome cur_test_outcome = 0; 64 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 65 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 66 const char *cur_test_name = NULL; 67 68 #ifdef WIN32 69 /** Pointer to argv[0] for win32. */ 70 static const char *commandname = NULL; 71 #endif 72 73 static void usage(struct testgroup_t *groups, int list_groups) 74 __attribute__((noreturn)); 75 76 static enum outcome 77 _testcase_run_bare(const struct testcase_t *testcase) 78 { 79 void *env = NULL; 80 int outcome; 81 if (testcase->setup) { 82 env = testcase->setup->setup_fn(testcase); 83 if (!env) 84 return FAIL; 85 else if (env == (void*)TT_SKIP) 86 return SKIP; 87 } 88 89 cur_test_outcome = OK; 90 testcase->fn(env); 91 outcome = cur_test_outcome; 92 93 if (testcase->setup) { 94 if (testcase->setup->cleanup_fn(testcase, env) == 0) 95 outcome = FAIL; 96 } 97 98 return outcome; 99 } 100 101 #define MAGIC_EXITCODE 42 102 103 static enum outcome 104 _testcase_run_forked(const struct testgroup_t *group, 105 const struct testcase_t *testcase) 106 { 107 #ifdef WIN32 108 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 109 we'll invoke our own exe (whose name we recall from the command 110 line) with a command line that tells it to run just the test we 111 want, and this time without forking. 112 113 (No, threads aren't an option. The whole point of forking is to 114 share no state between tests.) 115 */ 116 int ok; 117 char buffer[LONGEST_TEST_NAME+256]; 118 STARTUPINFOA si; 119 PROCESS_INFORMATION info; 120 DWORD exitcode; 121 122 if (!in_tinytest_main) { 123 printf("\nERROR. On Windows, _testcase_run_forked must be" 124 " called from within tinytest_main.\n"); 125 abort(); 126 } 127 if (opt_verbosity>0) 128 printf("[forking] "); 129 130 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 131 commandname, verbosity_flag, group->prefix, testcase->name); 132 133 memset(&si, 0, sizeof(si)); 134 memset(&info, 0, sizeof(info)); 135 si.cb = sizeof(si); 136 137 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 138 0, NULL, NULL, &si, &info); 139 if (!ok) { 140 printf("CreateProcess failed!\n"); 141 return 0; 142 } 143 WaitForSingleObject(info.hProcess, INFINITE); 144 GetExitCodeProcess(info.hProcess, &exitcode); 145 CloseHandle(info.hProcess); 146 CloseHandle(info.hThread); 147 if (exitcode == 0) 148 return OK; 149 else if (exitcode == MAGIC_EXITCODE) 150 return SKIP; 151 else 152 return FAIL; 153 #else 154 int outcome_pipe[2]; 155 pid_t pid; 156 (void)group; 157 158 if (pipe(outcome_pipe)) 159 perror("opening pipe"); 160 161 if (opt_verbosity>0) 162 printf("[forking] "); 163 pid = fork(); 164 if (!pid) { 165 /* child. */ 166 int test_r, write_r; 167 char b[1]; 168 close(outcome_pipe[0]); 169 test_r = _testcase_run_bare(testcase); 170 assert(0<=(int)test_r && (int)test_r<=2); 171 b[0] = "NYS"[test_r]; 172 write_r = (int)write(outcome_pipe[1], b, 1); 173 if (write_r != 1) { 174 perror("write outcome to pipe"); 175 exit(1); 176 } 177 exit(0); 178 return FAIL; /* unreachable */ 179 } else { 180 /* parent */ 181 int status, r; 182 char b[1]; 183 /* Close this now, so that if the other side closes it, 184 * our read fails. */ 185 close(outcome_pipe[1]); 186 r = (int)read(outcome_pipe[0], b, 1); 187 if (r == 0) { 188 printf("[Lost connection!] "); 189 return 0; 190 } else if (r != 1) { 191 perror("read outcome from pipe"); 192 } 193 waitpid(pid, &status, 0); 194 close(outcome_pipe[0]); 195 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 196 } 197 #endif 198 } 199 200 int 201 testcase_run_one(const struct testgroup_t *group, 202 const struct testcase_t *testcase) 203 { 204 enum outcome outcome; 205 206 if (testcase->flags & TT_SKIP) { 207 if (opt_verbosity>0) 208 printf("%s%s: SKIPPED\n", 209 group->prefix, testcase->name); 210 ++n_skipped; 211 return SKIP; 212 } 213 214 if (opt_verbosity>0 && !opt_forked) { 215 printf("%s%s: ", group->prefix, testcase->name); 216 } else { 217 if (opt_verbosity==0) printf("."); 218 cur_test_prefix = group->prefix; 219 cur_test_name = testcase->name; 220 } 221 222 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 223 outcome = _testcase_run_forked(group, testcase); 224 } else { 225 outcome = _testcase_run_bare(testcase); 226 } 227 228 if (outcome == OK) { 229 ++n_ok; 230 if (opt_verbosity>0 && !opt_forked) 231 puts(opt_verbosity==1?"OK":""); 232 } else if (outcome == SKIP) { 233 ++n_skipped; 234 if (opt_verbosity>0 && !opt_forked) 235 puts("SKIPPED"); 236 } else { 237 ++n_bad; 238 if (!opt_forked) 239 printf("\n [%s FAILED]\n", testcase->name); 240 } 241 242 if (opt_forked) { 243 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 244 return 1; /* unreachable */ 245 } else { 246 return (int)outcome; 247 } 248 } 249 250 int 251 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) 252 { 253 int i, j; 254 size_t length = LONGEST_TEST_NAME; 255 char fullname[LONGEST_TEST_NAME]; 256 int found=0; 257 if (strstr(arg, "..")) 258 length = strstr(arg,"..")-arg; 259 for (i=0; groups[i].prefix; ++i) { 260 for (j=0; groups[i].cases[j].name; ++j) { 261 snprintf(fullname, sizeof(fullname), "%s%s", 262 groups[i].prefix, groups[i].cases[j].name); 263 if (!flag) /* Hack! */ 264 printf(" %s\n", fullname); 265 if (!strncmp(fullname, arg, length)) { 266 groups[i].cases[j].flags |= flag; 267 ++found; 268 } 269 } 270 } 271 return found; 272 } 273 274 static void 275 usage(struct testgroup_t *groups, int list_groups) 276 { 277 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 278 puts(" Specify tests by name, or using a prefix ending with '..'"); 279 puts(" To skip a test, list give its name prefixed with a colon."); 280 puts(" Use --list-tests for a list of tests."); 281 if (list_groups) { 282 puts("Known tests are:"); 283 _tinytest_set_flag(groups, "..", 0); 284 } 285 exit(0); 286 } 287 288 int 289 tinytest_main(int c, const char **v, struct testgroup_t *groups) 290 { 291 int i, j, n=0; 292 293 #ifdef WIN32 294 commandname = v[0]; 295 #endif 296 for (i=1; i<c; ++i) { 297 if (v[i][0] == '-') { 298 if (!strcmp(v[i], "--RUNNING-FORKED")) { 299 opt_forked = 1; 300 } else if (!strcmp(v[i], "--no-fork")) { 301 opt_nofork = 1; 302 } else if (!strcmp(v[i], "--quiet")) { 303 opt_verbosity = -1; 304 verbosity_flag = "--quiet"; 305 } else if (!strcmp(v[i], "--verbose")) { 306 opt_verbosity = 2; 307 verbosity_flag = "--verbose"; 308 } else if (!strcmp(v[i], "--terse")) { 309 opt_verbosity = 0; 310 verbosity_flag = "--terse"; 311 } else if (!strcmp(v[i], "--help")) { 312 usage(groups, 0); 313 } else if (!strcmp(v[i], "--list-tests")) { 314 usage(groups, 1); 315 } else { 316 printf("Unknown option %s. Try --help\n",v[i]); 317 return -1; 318 } 319 } else { 320 const char *test = v[i]; 321 int flag = _TT_ENABLED; 322 if (test[0] == ':') { 323 ++test; 324 flag = TT_SKIP; 325 } else { 326 ++n; 327 } 328 if (!_tinytest_set_flag(groups, test, flag)) { 329 printf("No such test as %s!\n", v[i]); 330 return -1; 331 } 332 } 333 } 334 if (!n) 335 _tinytest_set_flag(groups, "..", _TT_ENABLED); 336 337 setvbuf(stdout, NULL, _IONBF, 0); 338 339 ++in_tinytest_main; 340 for (i=0; groups[i].prefix; ++i) 341 for (j=0; groups[i].cases[j].name; ++j) 342 if (groups[i].cases[j].flags & _TT_ENABLED) 343 testcase_run_one(&groups[i], 344 &groups[i].cases[j]); 345 346 --in_tinytest_main; 347 348 if (opt_verbosity==0) 349 puts(""); 350 351 if (n_bad) 352 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 353 n_bad+n_ok,n_skipped); 354 else if (opt_verbosity >= 1) 355 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 356 357 return (n_bad == 0) ? 0 : 1; 358 } 359 360 int 361 _tinytest_get_verbosity(void) 362 { 363 return opt_verbosity; 364 } 365 366 void 367 _tinytest_set_test_failed(void) 368 { 369 if (opt_verbosity <= 0 && cur_test_name) { 370 if (opt_verbosity==0) puts(""); 371 printf("%s%s: ", cur_test_prefix, cur_test_name); 372 cur_test_name = NULL; 373 } 374 cur_test_outcome = 0; 375 } 376 377 void 378 _tinytest_set_test_skipped(void) 379 { 380 if (cur_test_outcome==OK) 381 cur_test_outcome = SKIP; 382 } 383 384