1 /* tinytest.c -- Copyright 2009 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include <stdio.h> 27 #include <stdlib.h> 28 #include <string.h> 29 #include <assert.h> 30 31 #ifdef WIN32 32 #include <winsock2.h> 33 #include <windows.h> 34 #else 35 #include <sys/types.h> 36 #include <sys/wait.h> 37 #include <unistd.h> 38 #endif 39 40 #include <event2/util.h> 41 42 #include "tinytest.h" 43 #include "tinytest_macros.h" 44 45 #define LONGEST_TEST_NAME 16384 46 47 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 48 static int n_ok = 0; /**< Number of tests that have passed */ 49 static int n_bad = 0; /**< Number of tests that have failed. */ 50 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 51 52 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 53 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 54 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 55 const char *verbosity_flag = ""; 56 57 enum outcome { SKIP=2, OK=1, FAIL=0 }; 58 static enum outcome cur_test_outcome = 0; 59 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 60 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 61 const char *cur_test_name = NULL; 62 63 #ifdef WIN32 64 /** Pointer to argv[0] for win32. */ 65 static const char *commandname = NULL; 66 #endif 67 68 static enum outcome 69 _testcase_run_bare(const struct testcase_t *testcase) 70 { 71 void *env = NULL; 72 int outcome; 73 if (testcase->setup) { 74 env = testcase->setup->setup_fn(testcase); 75 if (!env) 76 return FAIL; 77 else if (env == (void*)TT_SKIP) 78 return SKIP; 79 } 80 81 cur_test_outcome = OK; 82 testcase->fn(env); 83 outcome = cur_test_outcome; 84 85 if (testcase->setup) { 86 if (testcase->setup->cleanup_fn(testcase, env) == 0) 87 outcome = FAIL; 88 } 89 90 return outcome; 91 } 92 93 #define MAGIC_EXITCODE 42 94 95 static enum outcome 96 _testcase_run_forked(const struct testgroup_t *group, 97 const struct testcase_t *testcase) 98 { 99 #ifdef WIN32 100 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 101 we'll invoke our own exe (whose name we recall from the command 102 line) with a command line that tells it to run just the test we 103 want, and this time without forking. 104 105 (No, threads aren't an option. The whole point of forking is to 106 share no state between tests.) 107 */ 108 int ok; 109 char buffer[LONGEST_TEST_NAME+256]; 110 STARTUPINFOA si; 111 PROCESS_INFORMATION info; 112 DWORD exitcode; 113 114 if (!in_tinytest_main) { 115 printf("\nERROR. On Windows, _testcase_run_forked must be" 116 " called from within tinytest_main.\n"); 117 abort(); 118 } 119 if (opt_verbosity>0) 120 printf("[forking] "); 121 122 evutil_snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 123 commandname, verbosity_flag, group->prefix, testcase->name); 124 125 memset(&si, 0, sizeof(si)); 126 memset(&info, 0, sizeof(info)); 127 si.cb = sizeof(si); 128 129 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 130 0, NULL, NULL, &si, &info); 131 if (!ok) { 132 printf("CreateProcess failed!\n"); 133 return 0; 134 } 135 WaitForSingleObject(info.hProcess, INFINITE); 136 GetExitCodeProcess(info.hProcess, &exitcode); 137 CloseHandle(info.hProcess); 138 CloseHandle(info.hThread); 139 if (exitcode == 0) 140 return OK; 141 else if (exitcode == MAGIC_EXITCODE) 142 return SKIP; 143 else 144 return FAIL; 145 #else 146 int outcome_pipe[2]; 147 pid_t pid; 148 (void)group; 149 150 if (pipe(outcome_pipe)) 151 perror("opening pipe"); 152 153 if (opt_verbosity>0) 154 printf("[forking] "); 155 pid = fork(); 156 if (!pid) { 157 /* child. */ 158 int test_r, write_r; 159 char b[1]; 160 close(outcome_pipe[0]); 161 test_r = _testcase_run_bare(testcase); 162 assert(0<=(int)test_r && (int)test_r<=2); 163 b[0] = "NYS"[test_r]; 164 write_r = write(outcome_pipe[1], b, 1); 165 if (write_r != 1) { 166 perror("write outcome to pipe"); 167 exit(1); 168 } 169 exit(0); 170 } else { 171 /* parent */ 172 int status, r; 173 char b[1]; 174 /* Close this now, so that if the other side closes it, 175 * our read fails. */ 176 close(outcome_pipe[1]); 177 r = read(outcome_pipe[0], b, 1); 178 if (r == 0) { 179 printf("[Lost connection!] "); 180 return 0; 181 } else if (r != 1) { 182 perror("read outcome from pipe"); 183 } 184 waitpid(pid, &status, 0); 185 close(outcome_pipe[0]); 186 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 187 } 188 #endif 189 } 190 191 int 192 testcase_run_one(const struct testgroup_t *group, 193 const struct testcase_t *testcase) 194 { 195 enum outcome outcome; 196 197 if (testcase->flags & TT_SKIP) { 198 if (opt_verbosity>0) 199 printf("%s%s: SKIPPED\n", 200 group->prefix, testcase->name); 201 ++n_skipped; 202 return SKIP; 203 } 204 205 if (opt_verbosity>0 && !opt_forked) { 206 printf("%s%s: ", group->prefix, testcase->name); 207 } else { 208 if (opt_verbosity==0) printf("."); 209 cur_test_prefix = group->prefix; 210 cur_test_name = testcase->name; 211 } 212 213 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 214 outcome = _testcase_run_forked(group, testcase); 215 } else { 216 outcome = _testcase_run_bare(testcase); 217 } 218 219 if (outcome == OK) { 220 ++n_ok; 221 if (opt_verbosity>0 && !opt_forked) 222 puts(opt_verbosity==1?"OK":""); 223 } else if (outcome == SKIP) { 224 ++n_skipped; 225 if (opt_verbosity>0 && !opt_forked) 226 puts("SKIPPED"); 227 } else { 228 ++n_bad; 229 if (!opt_forked) 230 printf("\n [%s FAILED]\n", testcase->name); 231 } 232 233 if (opt_forked) { 234 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 235 } else { 236 return (int)outcome; 237 } 238 } 239 240 int 241 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) 242 { 243 int i, j; 244 int length = LONGEST_TEST_NAME; 245 char fullname[LONGEST_TEST_NAME]; 246 int found=0; 247 if (strstr(arg, "..")) 248 length = strstr(arg,"..")-arg; 249 for (i=0; groups[i].prefix; ++i) { 250 for (j=0; groups[i].cases[j].name; ++j) { 251 evutil_snprintf(fullname, sizeof(fullname), "%s%s", 252 groups[i].prefix, groups[i].cases[j].name); 253 if (!flag) /* Hack! */ 254 printf(" %s\n", fullname); 255 if (!strncmp(fullname, arg, length)) { 256 groups[i].cases[j].flags |= flag; 257 ++found; 258 } 259 } 260 } 261 return found; 262 } 263 264 static void 265 usage(struct testgroup_t *groups, int list_groups) 266 { 267 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 268 puts(" Specify tests by name, or using a prefix ending with '..'"); 269 puts(" Use --list-tests for a list of tests."); 270 if (list_groups) { 271 puts("Known tests are:"); 272 _tinytest_set_flag(groups, "..", 0); 273 } 274 exit(0); 275 } 276 277 int 278 tinytest_main(int c, const char **v, struct testgroup_t *groups) 279 { 280 int i, j, n=0; 281 282 #ifdef WIN32 283 commandname = v[0]; 284 #endif 285 for (i=1; i<c; ++i) { 286 if (v[i][0] == '-') { 287 if (!strcmp(v[i], "--RUNNING-FORKED")) { 288 opt_forked = 1; 289 } else if (!strcmp(v[i], "--no-fork")) { 290 opt_nofork = 1; 291 } else if (!strcmp(v[i], "--quiet")) { 292 opt_verbosity = -1; 293 verbosity_flag = "--quiet"; 294 } else if (!strcmp(v[i], "--verbose")) { 295 opt_verbosity = 2; 296 verbosity_flag = "--verbose"; 297 } else if (!strcmp(v[i], "--terse")) { 298 opt_verbosity = 0; 299 verbosity_flag = "--terse"; 300 } else if (!strcmp(v[i], "--help")) { 301 usage(groups, 0); 302 } else if (!strcmp(v[i], "--list-tests")) { 303 usage(groups, 1); 304 } else { 305 printf("Unknown option %s. Try --help\n",v[i]); 306 return -1; 307 } 308 } else { 309 ++n; 310 if (!_tinytest_set_flag(groups, v[i], _TT_ENABLED)) { 311 printf("No such test as %s!\n", v[i]); 312 return -1; 313 } 314 } 315 } 316 if (!n) 317 _tinytest_set_flag(groups, "..", _TT_ENABLED); 318 319 setvbuf(stdout, NULL, _IONBF, 0); 320 321 ++in_tinytest_main; 322 for (i=0; groups[i].prefix; ++i) 323 for (j=0; groups[i].cases[j].name; ++j) 324 if (groups[i].cases[j].flags & _TT_ENABLED) 325 testcase_run_one(&groups[i], 326 &groups[i].cases[j]); 327 328 --in_tinytest_main; 329 330 if (opt_verbosity==0) 331 puts(""); 332 333 if (n_bad) 334 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 335 n_bad+n_ok,n_skipped); 336 else if (opt_verbosity >= 1) 337 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 338 339 return (n_bad == 0) ? 0 : 1; 340 } 341 342 int 343 _tinytest_get_verbosity(void) 344 { 345 return opt_verbosity; 346 } 347 348 void 349 _tinytest_set_test_failed(void) 350 { 351 if (opt_verbosity <= 0 && cur_test_name) { 352 if (opt_verbosity==0) puts(""); 353 printf("%s%s: ", cur_test_prefix, cur_test_name); 354 cur_test_name = NULL; 355 } 356 cur_test_outcome = 0; 357 } 358 359 void 360 _tinytest_set_test_skipped(void) 361 { 362 if (cur_test_outcome==OK) 363 cur_test_outcome = SKIP; 364 } 365 366