1" This script is sourced while editing the .vim file with the tests. 2" When the script is successful the .res file will be created. 3" Errors are appended to the test.log file. 4" 5" To execute only specific test functions, add a second argument. It will be 6" matched against the names of the Test_ funtion. E.g.: 7" ../vim -u NONE -S runtest.vim test_channel.vim open_delay 8" The output can be found in the "messages" file. 9" 10" The test script may contain anything, only functions that start with 11" "Test_" are special. These will be invoked and should contain assert 12" functions. See test_assert.vim for an example. 13" 14" It is possible to source other files that contain "Test_" functions. This 15" can speed up testing, since Vim does not need to restart. But be careful 16" that the tests do not interfere with each other. 17" 18" If an error cannot be detected properly with an assert function add the 19" error to the v:errors list: 20" call add(v:errors, 'test foo failed: Cannot find xyz') 21" 22" If preparation for each Test_ function is needed, define a SetUp function. 23" It will be called before each Test_ function. 24" 25" If cleanup after each Test_ function is needed, define a TearDown function. 26" It will be called after each Test_ function. 27" 28" When debugging a test it can be useful to add messages to v:errors: 29" call add(v:errors, "this happened") 30 31 32" Without the +eval feature we can't run these tests, bail out. 33so small.vim 34 35" Check that the screen size is at least 24 x 80 characters. 36if &lines < 24 || &columns < 80 37 let error = 'Screen size too small! Tests require at least 24 lines with 80 characters' 38 echoerr error 39 split test.log 40 $put =error 41 w 42 cquit 43endif 44 45" Common with all tests on all systems. 46source setup.vim 47 48" For consistency run all tests with 'nocompatible' set. 49" This also enables use of line continuation. 50set nocp viminfo+=nviminfo 51 52" Use utf-8 by default, instead of whatever the system default happens to be. 53" Individual tests can overrule this at the top of the file. 54set encoding=utf-8 55 56" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for 57" the test_name.vim file itself. Replace it here with a more restrictive one, 58" so we still catch mistakes. 59let s:test_script_fname = expand('%') 60au! SwapExists * call HandleSwapExists() 61func HandleSwapExists() 62 " Only ignore finding a swap file for the test script (the user might be 63 " editing it and do ":make test_name") and the output file. 64 if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname 65 let v:swapchoice = 'e' 66 endif 67endfunc 68 69" Avoid stopping at the "hit enter" prompt 70set nomore 71 72" Output all messages in English. 73lang mess C 74 75" Always use forward slashes. 76set shellslash 77 78let s:srcdir = expand('%:p:h:h') 79 80" Prepare for calling test_garbagecollect_now(). 81let v:testing = 1 82 83" Support function: get the alloc ID by name. 84function GetAllocId(name) 85 exe 'split ' . s:srcdir . '/alloc.h' 86 let top = search('typedef enum') 87 if top == 0 88 call add(v:errors, 'typedef not found in alloc.h') 89 endif 90 let lnum = search('aid_' . a:name . ',') 91 if lnum == 0 92 call add(v:errors, 'Alloc ID ' . a:name . ' not defined') 93 endif 94 close 95 return lnum - top - 1 96endfunc 97 98func RunTheTest(test) 99 echo 'Executing ' . a:test 100 101 " Avoid stopping at the "hit enter" prompt 102 set nomore 103 104 " Avoid a three second wait when a message is about to be overwritten by the 105 " mode message. 106 set noshowmode 107 108 " Clear any overrides. 109 call test_override('ALL', 0) 110 111 " Some tests wipe out buffers. To be consistent, always wipe out all 112 " buffers. 113 %bwipe! 114 115 " The test may change the current directory. Save and restore the 116 " directory after executing the test. 117 let save_cwd = getcwd() 118 119 if exists("*SetUp") 120 try 121 call SetUp() 122 catch 123 call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 124 endtry 125 endif 126 127 call add(s:messages, 'Executing ' . a:test) 128 let s:done += 1 129 130 if a:test =~ 'Test_nocatch_' 131 " Function handles errors itself. This avoids skipping commands after the 132 " error. 133 exe 'call ' . a:test 134 else 135 try 136 let s:test = a:test 137 au VimLeavePre * call EarlyExit(s:test) 138 exe 'call ' . a:test 139 au! VimLeavePre 140 catch /^\cskipped/ 141 call add(s:messages, ' Skipped') 142 call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) 143 catch 144 call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 145 endtry 146 endif 147 148 " In case 'insertmode' was set and something went wrong, make sure it is 149 " reset to avoid trouble with anything else. 150 set noinsertmode 151 152 if exists("*TearDown") 153 try 154 call TearDown() 155 catch 156 call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) 157 endtry 158 endif 159 160 " Clear any autocommands 161 au! 162 au SwapExists * call HandleSwapExists() 163 164 " Close any extra tab pages and windows and make the current one not modified. 165 while tabpagenr('$') > 1 166 quit! 167 endwhile 168 169 while 1 170 let wincount = winnr('$') 171 if wincount == 1 172 break 173 endif 174 bwipe! 175 if wincount == winnr('$') 176 " Did not manage to close a window. 177 only! 178 break 179 endif 180 endwhile 181 182 exe 'cd ' . save_cwd 183endfunc 184 185func AfterTheTest() 186 if len(v:errors) > 0 187 let s:fail += 1 188 call add(s:errors, 'Found errors in ' . s:test . ':') 189 call extend(s:errors, v:errors) 190 let v:errors = [] 191 endif 192endfunc 193 194func EarlyExit(test) 195 " It's OK for the test we use to test the quit detection. 196 if a:test != 'Test_zz_quit_detected()' 197 call add(v:errors, 'Test caused Vim to exit: ' . a:test) 198 endif 199 200 call FinishTesting() 201endfunc 202 203" This function can be called by a test if it wants to abort testing. 204func FinishTesting() 205 call AfterTheTest() 206 207 " Don't write viminfo on exit. 208 set viminfo= 209 210 " Clean up files created by setup.vim 211 call delete('XfakeHOME', 'rf') 212 213 if s:fail == 0 214 " Success, create the .res file so that make knows it's done. 215 exe 'split ' . fnamemodify(g:testname, ':r') . '.res' 216 write 217 endif 218 219 if len(s:errors) > 0 220 " Append errors to test.log 221 split test.log 222 call append(line('$'), '') 223 call append(line('$'), 'From ' . g:testname . ':') 224 call append(line('$'), s:errors) 225 write 226 endif 227 228 if s:done == 0 229 let message = 'NO tests executed' 230 else 231 let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') 232 endif 233 echo message 234 call add(s:messages, message) 235 if s:fail > 0 236 let message = s:fail . ' FAILED:' 237 echo message 238 call add(s:messages, message) 239 call extend(s:messages, s:errors) 240 endif 241 242 " Add SKIPPED messages 243 call extend(s:messages, s:skipped) 244 245 " Append messages to the file "messages" 246 split messages 247 call append(line('$'), '') 248 call append(line('$'), 'From ' . g:testname . ':') 249 call append(line('$'), s:messages) 250 write 251 252 qall! 253endfunc 254 255" Source the test script. First grab the file name, in case the script 256" navigates away. g:testname can be used by the tests. 257let g:testname = expand('%') 258let s:done = 0 259let s:fail = 0 260let s:errors = [] 261let s:messages = [] 262let s:skipped = [] 263if expand('%') =~ 'test_vimscript.vim' 264 " this test has intentional s:errors, don't use try/catch. 265 source % 266else 267 try 268 source % 269 catch 270 let s:fail += 1 271 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) 272 endtry 273endif 274 275" Names of flaky tests. 276let s:flaky_tests = [ 277 \ 'Test_call()', 278 \ 'Test_channel_handler()', 279 \ 'Test_client_server()', 280 \ 'Test_close_and_exit_cb()', 281 \ 'Test_close_callback()', 282 \ 'Test_close_handle()', 283 \ 'Test_close_lambda()', 284 \ 'Test_close_output_buffer()', 285 \ 'Test_close_partial()', 286 \ 'Test_collapse_buffers()', 287 \ 'Test_communicate()', 288 \ 'Test_cwd()', 289 \ 'Test_diff_screen()', 290 \ 'Test_exit_callback()', 291 \ 'Test_exit_callback_interval()', 292 \ 'Test_nb_basic()', 293 \ 'Test_oneshot()', 294 \ 'Test_open_delay()', 295 \ 'Test_out_cb()', 296 \ 'Test_paused()', 297 \ 'Test_pipe_through_sort_all()', 298 \ 'Test_pipe_through_sort_some()', 299 \ 'Test_popup_and_window_resize()', 300 \ 'Test_quoteplus()', 301 \ 'Test_quotestar()', 302 \ 'Test_raw_one_time_callback()', 303 \ 'Test_reltime()', 304 \ 'Test_repeat_three()', 305 \ 'Test_server_crash()', 306 \ 'Test_terminal_ansicolors_default()', 307 \ 'Test_terminal_ansicolors_func()', 308 \ 'Test_terminal_ansicolors_global()', 309 \ 'Test_terminal_composing_unicode()', 310 \ 'Test_terminal_env()', 311 \ 'Test_terminal_hide_buffer()', 312 \ 'Test_terminal_make_change()', 313 \ 'Test_terminal_noblock()', 314 \ 'Test_terminal_redir_file()', 315 \ 'Test_terminal_response_to_control_sequence()', 316 \ 'Test_terminal_scrollback()', 317 \ 'Test_terminal_split_quit()', 318 \ 'Test_terminal_termwinkey()', 319 \ 'Test_terminal_termwinsize_mininmum()', 320 \ 'Test_terminal_termwinsize_option_fixed()', 321 \ 'Test_terminal_termwinsize_option_zero()', 322 \ 'Test_terminal_tmap()', 323 \ 'Test_terminal_wall()', 324 \ 'Test_terminal_wipe_buffer()', 325 \ 'Test_terminal_wqall()', 326 \ 'Test_two_channels()', 327 \ 'Test_unlet_handle()', 328 \ 'Test_with_partial_callback()', 329 \ 'Test_zero_reply()', 330 \ 'Test_zz1_terminal_in_gui()', 331 \ ] 332 333" Pattern indicating a common flaky test failure. 334let s:flaky_errors_re = 'StopVimInTerminal\|VerifyScreenDump' 335 336" Locate Test_ functions and execute them. 337redir @q 338silent function /^Test_ 339redir END 340let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) 341 342" If there is an extra argument filter the function names against it. 343if argc() > 1 344 let s:tests = filter(s:tests, 'v:val =~ argv(1)') 345endif 346 347" Execute the tests in alphabetical order. 348for s:test in sort(s:tests) 349 " Silence, please! 350 set belloff=all 351 let prev_error = '' 352 let total_errors = [] 353 let run_nr = 1 354 355 call RunTheTest(s:test) 356 357 " Repeat a flaky test. Give up when: 358 " - it fails again with the same message 359 " - it fails five times (with a different mesage) 360 if len(v:errors) > 0 361 \ && (index(s:flaky_tests, s:test) >= 0 362 \ || v:errors[0] =~ s:flaky_errors_re) 363 while 1 364 call add(s:messages, 'Found errors in ' . s:test . ':') 365 call extend(s:messages, v:errors) 366 367 call add(total_errors, 'Run ' . run_nr . ':') 368 call extend(total_errors, v:errors) 369 370 if run_nr == 5 || prev_error == v:errors[0] 371 call add(total_errors, 'Flaky test failed too often, giving up') 372 let v:errors = total_errors 373 break 374 endif 375 376 call add(s:messages, 'Flaky test failed, running it again') 377 378 " Flakiness is often caused by the system being very busy. Sleep a 379 " couple of seconds to have a higher chance of succeeding the second 380 " time. 381 sleep 2 382 383 let prev_error = v:errors[0] 384 let v:errors = [] 385 let run_nr += 1 386 387 call RunTheTest(s:test) 388 389 if len(v:errors) == 0 390 " Test passed on rerun. 391 break 392 endif 393 endwhile 394 endif 395 396 call AfterTheTest() 397endfor 398 399call FinishTesting() 400 401" vim: shiftwidth=2 sts=2 expandtab 402