1" This script is sourced while editing the .vim file with the tests. 2" When the script is successful the .res file will be created. 3" Errors are appended to the test.log file. 4" 5" To execute only specific test functions, add a second argument. It will be 6" matched against the names of the Test_ function. E.g.: 7" ../vim -u NONE -S runtest.vim test_channel.vim open_delay 8" The output can be found in the "messages" file. 9" 10" The test script may contain anything, only functions that start with 11" "Test_" are special. These will be invoked and should contain assert 12" functions. See test_assert.vim for an example. 13" 14" It is possible to source other files that contain "Test_" functions. This 15" can speed up testing, since Vim does not need to restart. But be careful 16" that the tests do not interfere with each other. 17" 18" If an error cannot be detected properly with an assert function add the 19" error to the v:errors list: 20" call add(v:errors, 'test foo failed: Cannot find xyz') 21" 22" If preparation for each Test_ function is needed, define a SetUp function. 23" It will be called before each Test_ function. 24" 25" If cleanup after each Test_ function is needed, define a TearDown function. 26" It will be called after each Test_ function. 27" 28" When debugging a test it can be useful to add messages to v:errors: 29" call add(v:errors, "this happened") 30 31 32" Without the +eval feature we can't run these tests, bail out. 33so small.vim 34 35" Check that the screen size is at least 24 x 80 characters. 36if &lines < 24 || &columns < 80 37 let error = 'Screen size too small! Tests require at least 24 lines with 80 characters, got ' .. &lines .. ' lines with ' .. &columns .. ' characters' 38 echoerr error 39 split test.log 40 $put =error 41 write 42 split messages 43 call append(line('$'), '') 44 call append(line('$'), 'From ' . expand('%') . ':') 45 call append(line('$'), error) 46 write 47 qa! 48endif 49 50if has('reltime') 51 let s:start_time = reltime() 52endif 53 54" Common with all tests on all systems. 55source setup.vim 56 57" For consistency run all tests with 'nocompatible' set. 58" This also enables use of line continuation. 59set nocp viminfo+=nviminfo 60 61" Use utf-8 by default, instead of whatever the system default happens to be. 62" Individual tests can overrule this at the top of the file and use 63" g:orig_encoding if needed. 64let g:orig_encoding = &encoding 65set encoding=utf-8 66 67" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for 68" the test_name.vim file itself. Replace it here with a more restrictive one, 69" so we still catch mistakes. 70let s:test_script_fname = expand('%') 71au! SwapExists * call HandleSwapExists() 72func HandleSwapExists() 73 " Ignore finding a swap file for the test script (the user might be 74 " editing it and do ":make test_name") and the output file. 75 " Report finding another swap file and chose 'q' to avoid getting stuck. 76 if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname 77 let v:swapchoice = 'e' 78 else 79 call assert_report('Unexpected swap file: ' .. v:swapname) 80 let v:swapchoice = 'q' 81 endif 82endfunc 83 84" Avoid stopping at the "hit enter" prompt 85set nomore 86 87" Output all messages in English. 88lang mess C 89 90" Always use forward slashes. 91set shellslash 92 93let s:srcdir = expand('%:p:h:h') 94 95" Prepare for calling test_garbagecollect_now(). 96let v:testing = 1 97 98" Support function: get the alloc ID by name. 99function GetAllocId(name) 100 exe 'split ' . s:srcdir . '/alloc.h' 101 let top = search('typedef enum') 102 if top == 0 103 call add(v:errors, 'typedef not found in alloc.h') 104 endif 105 let lnum = search('aid_' . a:name . ',') 106 if lnum == 0 107 call add(v:errors, 'Alloc ID ' . a:name . ' not defined') 108 endif 109 close 110 return lnum - top - 1 111endfunc 112 113func RunTheTest(test) 114 echo 'Executing ' . a:test 115 if has('reltime') 116 let func_start = reltime() 117 endif 118 119 " Avoid stopping at the "hit enter" prompt 120 set nomore 121 122 " Avoid a three second wait when a message is about to be overwritten by the 123 " mode message. 124 set noshowmode 125 126 " Clear any overrides. 127 call test_override('ALL', 0) 128 129 " Some tests wipe out buffers. To be consistent, always wipe out all 130 " buffers. 131 %bwipe! 132 133 " The test may change the current directory. Save and restore the 134 " directory after executing the test. 135 let save_cwd = getcwd() 136 137 if exists("*SetUp") 138 try 139 call SetUp() 140 catch 141 call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 142 endtry 143 endif 144 145 if a:test =~ 'Test_nocatch_' 146 " Function handles errors itself. This avoids skipping commands after the 147 " error. 148 exe 'call ' . a:test 149 else 150 try 151 let s:test = a:test 152 au VimLeavePre * call EarlyExit(s:test) 153 exe 'call ' . a:test 154 au! VimLeavePre 155 catch /^\cskipped/ 156 call add(s:messages, ' Skipped') 157 call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) 158 catch 159 call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 160 endtry 161 endif 162 163 " In case 'insertmode' was set and something went wrong, make sure it is 164 " reset to avoid trouble with anything else. 165 set noinsertmode 166 167 if exists("*TearDown") 168 try 169 call TearDown() 170 catch 171 call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 172 endtry 173 endif 174 175 " Clear any autocommands and put back the catch-all for SwapExists. 176 au! 177 au SwapExists * call HandleSwapExists() 178 179 " Close any stray popup windows 180 if has('popupwin') 181 call popup_clear() 182 endif 183 184 " Close any extra tab pages and windows and make the current one not modified. 185 while tabpagenr('$') > 1 186 quit! 187 endwhile 188 189 while 1 190 let wincount = winnr('$') 191 if wincount == 1 192 break 193 endif 194 bwipe! 195 if wincount == winnr('$') 196 " Did not manage to close a window. 197 only! 198 break 199 endif 200 endwhile 201 202 exe 'cd ' . save_cwd 203 204 let message = 'Executed ' . a:test 205 if has('reltime') 206 let message ..= ' in ' .. reltimestr(reltime(func_start)) .. ' seconds' 207 endif 208 call add(s:messages, message) 209 let s:done += 1 210endfunc 211 212func AfterTheTest() 213 if len(v:errors) > 0 214 let s:fail += 1 215 call add(s:errors, 'Found errors in ' . s:test . ':') 216 call extend(s:errors, v:errors) 217 let v:errors = [] 218 endif 219endfunc 220 221func EarlyExit(test) 222 " It's OK for the test we use to test the quit detection. 223 if a:test != 'Test_zz_quit_detected()' 224 call add(v:errors, 'Test caused Vim to exit: ' . a:test) 225 endif 226 227 call FinishTesting() 228endfunc 229 230" This function can be called by a test if it wants to abort testing. 231func FinishTesting() 232 call AfterTheTest() 233 234 " Don't write viminfo on exit. 235 set viminfo= 236 237 " Clean up files created by setup.vim 238 call delete('XfakeHOME', 'rf') 239 240 if s:fail == 0 241 " Success, create the .res file so that make knows it's done. 242 exe 'split ' . fnamemodify(g:testname, ':r') . '.res' 243 write 244 endif 245 246 if len(s:errors) > 0 247 " Append errors to test.log 248 split test.log 249 call append(line('$'), '') 250 call append(line('$'), 'From ' . g:testname . ':') 251 call append(line('$'), s:errors) 252 write 253 endif 254 255 if s:done == 0 256 if s:filtered > 0 257 let message = "NO tests match $TEST_FILTER: '" .. $TEST_FILTER .. "'" 258 else 259 let message = 'NO tests executed' 260 endif 261 else 262 if s:filtered > 0 263 call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER") 264 endif 265 let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') 266 endif 267 if s:done > 0 && has('reltime') 268 let message ..= ' in ' .. reltimestr(reltime(s:start_time)) .. ' seconds' 269 endif 270 echo message 271 call add(s:messages, message) 272 if s:fail > 0 273 let message = s:fail . ' FAILED:' 274 echo message 275 call add(s:messages, message) 276 call extend(s:messages, s:errors) 277 endif 278 279 " Add SKIPPED messages 280 call extend(s:messages, s:skipped) 281 282 " Append messages to the file "messages" 283 split messages 284 call append(line('$'), '') 285 call append(line('$'), 'From ' . g:testname . ':') 286 call append(line('$'), s:messages) 287 write 288 289 qall! 290endfunc 291 292" Source the test script. First grab the file name, in case the script 293" navigates away. g:testname can be used by the tests. 294let g:testname = expand('%') 295let s:done = 0 296let s:fail = 0 297let s:errors = [] 298let s:messages = [] 299let s:skipped = [] 300if expand('%') =~ 'test_vimscript.vim' 301 " this test has intentional s:errors, don't use try/catch. 302 source % 303else 304 try 305 source % 306 catch /^\cskipped/ 307 call add(s:messages, ' Skipped') 308 call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) 309 catch 310 let s:fail += 1 311 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) 312 endtry 313endif 314 315" Names of flaky tests. 316let s:flaky_tests = [ 317 \ 'Test_autocmd_SafeState()', 318 \ 'Test_call()', 319 \ 'Test_channel_handler()', 320 \ 'Test_client_server()', 321 \ 'Test_close_and_exit_cb()', 322 \ 'Test_close_callback()', 323 \ 'Test_close_handle()', 324 \ 'Test_close_lambda()', 325 \ 'Test_close_output_buffer()', 326 \ 'Test_close_partial()', 327 \ 'Test_collapse_buffers()', 328 \ 'Test_communicate()', 329 \ 'Test_cwd()', 330 \ 'Test_diff_screen()', 331 \ 'Test_exit_callback()', 332 \ 'Test_exit_callback_interval()', 333 \ 'Test_map_timeout_with_timer_interrupt()', 334 \ 'Test_nb_basic()', 335 \ 'Test_open_delay()', 336 \ 'Test_out_cb()', 337 \ 'Test_pipe_through_sort_all()', 338 \ 'Test_pipe_through_sort_some()', 339 \ 'Test_popup_and_window_resize()', 340 \ 'Test_quoteplus()', 341 \ 'Test_quotestar()', 342 \ 'Test_raw_one_time_callback()', 343 \ 'Test_reltime()', 344 \ 'Test_server_crash()', 345 \ 'Test_state()', 346 \ 'Test_terminal_ansicolors_default()', 347 \ 'Test_terminal_ansicolors_func()', 348 \ 'Test_terminal_ansicolors_global()', 349 \ 'Test_terminal_composing_unicode()', 350 \ 'Test_terminal_does_not_truncate_last_newlines()', 351 \ 'Test_terminal_env()', 352 \ 'Test_terminal_hide_buffer()', 353 \ 'Test_terminal_make_change()', 354 \ 'Test_terminal_no_cmd()', 355 \ 'Test_terminal_noblock()', 356 \ 'Test_terminal_redir_file()', 357 \ 'Test_terminal_response_to_control_sequence()', 358 \ 'Test_terminal_scrollback()', 359 \ 'Test_terminal_split_quit()', 360 \ 'Test_terminal_termwinkey()', 361 \ 'Test_terminal_termwinsize_minimum()', 362 \ 'Test_terminal_termwinsize_option_fixed()', 363 \ 'Test_terminal_termwinsize_option_zero()', 364 \ 'Test_terminal_tmap()', 365 \ 'Test_terminal_wall()', 366 \ 'Test_terminal_wipe_buffer()', 367 \ 'Test_terminal_wqall()', 368 \ 'Test_termwinscroll()', 369 \ 'Test_timer_oneshot()', 370 \ 'Test_timer_paused()', 371 \ 'Test_timer_repeat_many()', 372 \ 'Test_timer_repeat_three()', 373 \ 'Test_timer_stop_all_in_callback()', 374 \ 'Test_timer_stop_in_callback()', 375 \ 'Test_two_channels()', 376 \ 'Test_unlet_handle()', 377 \ 'Test_timer_with_partial_callback()', 378 \ 'Test_zero_reply()', 379 \ 'Test_zz1_terminal_in_gui()', 380 \ ] 381 382" Pattern indicating a common flaky test failure. 383let s:flaky_errors_re = 'StopVimInTerminal\|VerifyScreenDump' 384 385" Locate Test_ functions and execute them. 386redir @q 387silent function /^Test_ 388redir END 389let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) 390 391" If there is an extra argument filter the function names against it. 392if argc() > 1 393 let s:tests = filter(s:tests, 'v:val =~ argv(1)') 394endif 395 396" If the environment variable $TEST_FILTER is set then filter the function 397" names against it. 398let s:filtered = 0 399if $TEST_FILTER != '' 400 let s:filtered = len(s:tests) 401 let s:tests = filter(s:tests, 'v:val =~ $TEST_FILTER') 402 let s:filtered -= len(s:tests) 403endif 404 405" Execute the tests in alphabetical order. 406for s:test in sort(s:tests) 407 " Silence, please! 408 set belloff=all 409 let prev_error = '' 410 let total_errors = [] 411 let run_nr = 1 412 413 call RunTheTest(s:test) 414 415 " Repeat a flaky test. Give up when: 416 " - it fails again with the same message 417 " - it fails five times (with a different message) 418 if len(v:errors) > 0 419 \ && (index(s:flaky_tests, s:test) >= 0 420 \ || v:errors[0] =~ s:flaky_errors_re) 421 while 1 422 call add(s:messages, 'Found errors in ' . s:test . ':') 423 call extend(s:messages, v:errors) 424 425 call add(total_errors, 'Run ' . run_nr . ':') 426 call extend(total_errors, v:errors) 427 428 if run_nr == 5 || prev_error == v:errors[0] 429 call add(total_errors, 'Flaky test failed too often, giving up') 430 let v:errors = total_errors 431 break 432 endif 433 434 call add(s:messages, 'Flaky test failed, running it again') 435 436 " Flakiness is often caused by the system being very busy. Sleep a 437 " couple of seconds to have a higher chance of succeeding the second 438 " time. 439 sleep 2 440 441 let prev_error = v:errors[0] 442 let v:errors = [] 443 let run_nr += 1 444 445 call RunTheTest(s:test) 446 447 if len(v:errors) == 0 448 " Test passed on rerun. 449 break 450 endif 451 endwhile 452 endif 453 454 call AfterTheTest() 455endfor 456 457call FinishTesting() 458 459" vim: shiftwidth=2 sts=2 expandtab 460