1" This script is sourced while editing the .vim file with the tests. 2" When the script is successful the .res file will be created. 3" Errors are appended to the test.log file. 4" 5" To execute only specific test functions, add a second argument. It will be 6" matched against the names of the Test_ funtion. E.g.: 7" ../vim -u NONE -S runtest.vim test_channel.vim open_delay 8" The output can be found in the "messages" file. 9" 10" The test script may contain anything, only functions that start with 11" "Test_" are special. These will be invoked and should contain assert 12" functions. See test_assert.vim for an example. 13" 14" It is possible to source other files that contain "Test_" functions. This 15" can speed up testing, since Vim does not need to restart. But be careful 16" that the tests do not interfere with each other. 17" 18" If an error cannot be detected properly with an assert function add the 19" error to the v:errors list: 20" call add(v:errors, 'test foo failed: Cannot find xyz') 21" 22" If preparation for each Test_ function is needed, define a SetUp function. 23" It will be called before each Test_ function. 24" 25" If cleanup after each Test_ function is needed, define a TearDown function. 26" It will be called after each Test_ function. 27" 28" When debugging a test it can be useful to add messages to v:errors: 29" call add(v:errors, "this happened") 30 31 32" Without the +eval feature we can't run these tests, bail out. 33so small.vim 34 35" Check that the screen size is at least 24 x 80 characters. 36if &lines < 24 || &columns < 80 37 let error = 'Screen size too small! Tests require at least 24 lines with 80 characters' 38 echoerr error 39 split test.log 40 $put =error 41 w 42 cquit 43endif 44 45" Common with all tests on all systems. 46source setup.vim 47 48" For consistency run all tests with 'nocompatible' set. 49" This also enables use of line continuation. 50set nocp viminfo+=nviminfo 51 52" Use utf-8 by default, instead of whatever the system default happens to be. 53" Individual tests can overrule this at the top of the file and use 54" g:orig_encoding if needed. 55let g:orig_encoding = &encoding 56set encoding=utf-8 57 58" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for 59" the test_name.vim file itself. Replace it here with a more restrictive one, 60" so we still catch mistakes. 61let s:test_script_fname = expand('%') 62au! SwapExists * call HandleSwapExists() 63func HandleSwapExists() 64 " Only ignore finding a swap file for the test script (the user might be 65 " editing it and do ":make test_name") and the output file. 66 if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname 67 let v:swapchoice = 'e' 68 endif 69endfunc 70 71" Avoid stopping at the "hit enter" prompt 72set nomore 73 74" Output all messages in English. 75lang mess C 76 77" Always use forward slashes. 78set shellslash 79 80let s:srcdir = expand('%:p:h:h') 81 82" Prepare for calling test_garbagecollect_now(). 83let v:testing = 1 84 85" Support function: get the alloc ID by name. 86function GetAllocId(name) 87 exe 'split ' . s:srcdir . '/alloc.h' 88 let top = search('typedef enum') 89 if top == 0 90 call add(v:errors, 'typedef not found in alloc.h') 91 endif 92 let lnum = search('aid_' . a:name . ',') 93 if lnum == 0 94 call add(v:errors, 'Alloc ID ' . a:name . ' not defined') 95 endif 96 close 97 return lnum - top - 1 98endfunc 99 100func RunTheTest(test) 101 echo 'Executing ' . a:test 102 103 " Avoid stopping at the "hit enter" prompt 104 set nomore 105 106 " Avoid a three second wait when a message is about to be overwritten by the 107 " mode message. 108 set noshowmode 109 110 " Clear any overrides. 111 call test_override('ALL', 0) 112 113 " Some tests wipe out buffers. To be consistent, always wipe out all 114 " buffers. 115 %bwipe! 116 117 " The test may change the current directory. Save and restore the 118 " directory after executing the test. 119 let save_cwd = getcwd() 120 121 if exists("*SetUp") 122 try 123 call SetUp() 124 catch 125 call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 126 endtry 127 endif 128 129 call add(s:messages, 'Executing ' . a:test) 130 let s:done += 1 131 132 if a:test =~ 'Test_nocatch_' 133 " Function handles errors itself. This avoids skipping commands after the 134 " error. 135 exe 'call ' . a:test 136 else 137 try 138 let s:test = a:test 139 au VimLeavePre * call EarlyExit(s:test) 140 exe 'call ' . a:test 141 au! VimLeavePre 142 catch /^\cskipped/ 143 call add(s:messages, ' Skipped') 144 call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) 145 catch 146 call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 147 endtry 148 endif 149 150 " In case 'insertmode' was set and something went wrong, make sure it is 151 " reset to avoid trouble with anything else. 152 set noinsertmode 153 154 if exists("*TearDown") 155 try 156 call TearDown() 157 catch 158 call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 159 endtry 160 endif 161 162 " Clear any autocommands 163 au! 164 au SwapExists * call HandleSwapExists() 165 166 " Close any extra tab pages and windows and make the current one not modified. 167 while tabpagenr('$') > 1 168 quit! 169 endwhile 170 171 while 1 172 let wincount = winnr('$') 173 if wincount == 1 174 break 175 endif 176 bwipe! 177 if wincount == winnr('$') 178 " Did not manage to close a window. 179 only! 180 break 181 endif 182 endwhile 183 184 exe 'cd ' . save_cwd 185endfunc 186 187func AfterTheTest() 188 if len(v:errors) > 0 189 let s:fail += 1 190 call add(s:errors, 'Found errors in ' . s:test . ':') 191 call extend(s:errors, v:errors) 192 let v:errors = [] 193 endif 194endfunc 195 196func EarlyExit(test) 197 " It's OK for the test we use to test the quit detection. 198 if a:test != 'Test_zz_quit_detected()' 199 call add(v:errors, 'Test caused Vim to exit: ' . a:test) 200 endif 201 202 call FinishTesting() 203endfunc 204 205" This function can be called by a test if it wants to abort testing. 206func FinishTesting() 207 call AfterTheTest() 208 209 " Don't write viminfo on exit. 210 set viminfo= 211 212 " Clean up files created by setup.vim 213 call delete('XfakeHOME', 'rf') 214 215 if s:fail == 0 216 " Success, create the .res file so that make knows it's done. 217 exe 'split ' . fnamemodify(g:testname, ':r') . '.res' 218 write 219 endif 220 221 if len(s:errors) > 0 222 " Append errors to test.log 223 split test.log 224 call append(line('$'), '') 225 call append(line('$'), 'From ' . g:testname . ':') 226 call append(line('$'), s:errors) 227 write 228 endif 229 230 if s:done == 0 231 let message = 'NO tests executed' 232 else 233 let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') 234 endif 235 echo message 236 call add(s:messages, message) 237 if s:fail > 0 238 let message = s:fail . ' FAILED:' 239 echo message 240 call add(s:messages, message) 241 call extend(s:messages, s:errors) 242 endif 243 244 " Add SKIPPED messages 245 call extend(s:messages, s:skipped) 246 247 " Append messages to the file "messages" 248 split messages 249 call append(line('$'), '') 250 call append(line('$'), 'From ' . g:testname . ':') 251 call append(line('$'), s:messages) 252 write 253 254 qall! 255endfunc 256 257" Source the test script. First grab the file name, in case the script 258" navigates away. g:testname can be used by the tests. 259let g:testname = expand('%') 260let s:done = 0 261let s:fail = 0 262let s:errors = [] 263let s:messages = [] 264let s:skipped = [] 265if expand('%') =~ 'test_vimscript.vim' 266 " this test has intentional s:errors, don't use try/catch. 267 source % 268else 269 try 270 source % 271 catch 272 let s:fail += 1 273 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) 274 endtry 275endif 276 277" Names of flaky tests. 278let s:flaky_tests = [ 279 \ 'Test_call()', 280 \ 'Test_channel_handler()', 281 \ 'Test_client_server()', 282 \ 'Test_close_and_exit_cb()', 283 \ 'Test_close_callback()', 284 \ 'Test_close_handle()', 285 \ 'Test_close_lambda()', 286 \ 'Test_close_output_buffer()', 287 \ 'Test_close_partial()', 288 \ 'Test_collapse_buffers()', 289 \ 'Test_communicate()', 290 \ 'Test_cwd()', 291 \ 'Test_diff_screen()', 292 \ 'Test_exit_callback()', 293 \ 'Test_exit_callback_interval()', 294 \ 'Test_nb_basic()', 295 \ 'Test_oneshot()', 296 \ 'Test_open_delay()', 297 \ 'Test_out_cb()', 298 \ 'Test_paused()', 299 \ 'Test_pipe_through_sort_all()', 300 \ 'Test_pipe_through_sort_some()', 301 \ 'Test_popup_and_window_resize()', 302 \ 'Test_quoteplus()', 303 \ 'Test_quotestar()', 304 \ 'Test_raw_one_time_callback()', 305 \ 'Test_reltime()', 306 \ 'Test_repeat_three()', 307 \ 'Test_server_crash()', 308 \ 'Test_terminal_ansicolors_default()', 309 \ 'Test_terminal_ansicolors_func()', 310 \ 'Test_terminal_ansicolors_global()', 311 \ 'Test_terminal_composing_unicode()', 312 \ 'Test_terminal_does_not_truncate_last_newlines()', 313 \ 'Test_terminal_env()', 314 \ 'Test_terminal_hide_buffer()', 315 \ 'Test_terminal_make_change()', 316 \ 'Test_terminal_no_cmd()', 317 \ 'Test_terminal_noblock()', 318 \ 'Test_terminal_redir_file()', 319 \ 'Test_terminal_response_to_control_sequence()', 320 \ 'Test_terminal_scrollback()', 321 \ 'Test_terminal_split_quit()', 322 \ 'Test_terminal_termwinkey()', 323 \ 'Test_terminal_termwinsize_mininmum()', 324 \ 'Test_terminal_termwinsize_option_fixed()', 325 \ 'Test_terminal_termwinsize_option_zero()', 326 \ 'Test_terminal_tmap()', 327 \ 'Test_terminal_wall()', 328 \ 'Test_terminal_wipe_buffer()', 329 \ 'Test_terminal_wqall()', 330 \ 'Test_two_channels()', 331 \ 'Test_unlet_handle()', 332 \ 'Test_with_partial_callback()', 333 \ 'Test_zero_reply()', 334 \ 'Test_zz1_terminal_in_gui()', 335 \ ] 336 337" Pattern indicating a common flaky test failure. 338let s:flaky_errors_re = 'StopVimInTerminal\|VerifyScreenDump' 339 340" Locate Test_ functions and execute them. 341redir @q 342silent function /^Test_ 343redir END 344let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) 345 346" If there is an extra argument filter the function names against it. 347if argc() > 1 348 let s:tests = filter(s:tests, 'v:val =~ argv(1)') 349endif 350 351" Execute the tests in alphabetical order. 352for s:test in sort(s:tests) 353 " Silence, please! 354 set belloff=all 355 let prev_error = '' 356 let total_errors = [] 357 let run_nr = 1 358 359 call RunTheTest(s:test) 360 361 " Repeat a flaky test. Give up when: 362 " - it fails again with the same message 363 " - it fails five times (with a different mesage) 364 if len(v:errors) > 0 365 \ && (index(s:flaky_tests, s:test) >= 0 366 \ || v:errors[0] =~ s:flaky_errors_re) 367 while 1 368 call add(s:messages, 'Found errors in ' . s:test . ':') 369 call extend(s:messages, v:errors) 370 371 call add(total_errors, 'Run ' . run_nr . ':') 372 call extend(total_errors, v:errors) 373 374 if run_nr == 5 || prev_error == v:errors[0] 375 call add(total_errors, 'Flaky test failed too often, giving up') 376 let v:errors = total_errors 377 break 378 endif 379 380 call add(s:messages, 'Flaky test failed, running it again') 381 382 " Flakiness is often caused by the system being very busy. Sleep a 383 " couple of seconds to have a higher chance of succeeding the second 384 " time. 385 sleep 2 386 387 let prev_error = v:errors[0] 388 let v:errors = [] 389 let run_nr += 1 390 391 call RunTheTest(s:test) 392 393 if len(v:errors) == 0 394 " Test passed on rerun. 395 break 396 endif 397 endwhile 398 endif 399 400 call AfterTheTest() 401endfor 402 403call FinishTesting() 404 405" vim: shiftwidth=2 sts=2 expandtab 406