1#!/usr/bin/env python3 2 3# 4# units.py - Units test harness for ctags 5# 6# Copyright (C) 2019 Ken Takata 7# (Based on "units" written by Masatake YAMATO.) 8# 9# This program is free software; you can redistribute it and/or modify 10# it under the terms of the GNU General Public License as published by 11# the Free Software Foundation; either version 2 of the License, or 12# (at your option) any later version. 13# 14# This program is distributed in the hope that it will be useful, 15# but WITHOUT ANY WARRANTY; without even the implied warranty of 16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17# GNU General Public License for more details. 18# 19# You should have received a copy of the GNU General Public License 20# along with this program. If not, see <http://www.gnu.org/licenses/>. 21# 22 23# 24# Python 3.5 or later is required. 25# On Windows, unix-like shell (e.g. bash) and some unix tools (sed, 26# diff, etc.) are needed. 27# 28 29import time # for debugging 30import argparse 31import filecmp 32import glob 33import io 34import os 35import platform 36import queue 37import re 38import shutil 39import stat 40import subprocess 41import sys 42import threading 43 44# 45# Global Parameters 46# 47SHELL = '/bin/sh' 48CTAGS = './ctags' 49READTAGS = './readtags' 50OPTSCRIPT = './optscript' 51WITH_TIMEOUT = 0 52WITH_VALGRIND = False 53COLORIZED_OUTPUT = True 54CATEGORIES = [] 55UNITS = [] 56LANGUAGES = [] 57PRETENSE_OPTS = '' 58RUN_SHRINK = False 59SHOW_DIFF_OUTPUT = False 60NUM_WORKER_THREADS = 4 61DIFF_U_NUM = 0 62 63# 64# Internal variables and constants 65# 66_FEATURE_LIST = [] 67_PREPERE_ENV = '' 68_DEFAULT_CATEGORY = 'ROOT' 69_TIMEOUT_EXIT = 124 70_VG_TIMEOUT_FACTOR = 10 71_VALGRIND_EXIT = 58 72_STDERR_OUTPUT_NAME = 'STDERR.tmp' 73_DIFF_OUTPUT_NAME = 'DIFF.tmp' 74_VALGRIND_OUTPUT_NAME = 'VALGRIND.tmp' 75 76# 77# Results 78# 79L_PASSED = [] 80L_FIXED = [] 81L_FAILED_BY_STATUS = [] 82L_FAILED_BY_DIFF = [] 83L_SKIPPED_BY_FEATURES = [] 84L_SKIPPED_BY_LANGUAGES = [] 85L_SKIPPED_BY_ILOOP = [] 86L_KNOWN_BUGS = [] 87L_FAILED_BY_TIMEED_OUT = [] 88L_BROKEN_ARGS_CTAGS = [] 89L_VALGRIND = [] 90TMAIN_STATUS = True 91TMAIN_FAILED = [] 92 93def remove_prefix(string, prefix): 94 if string.startswith(prefix): 95 return string[len(prefix):] 96 else: 97 return string 98 99def is_cygwin(): 100 system = platform.system() 101 return system.startswith('CYGWIN_NT') or system.startswith('MINGW32_NT') 102 103def isabs(path): 104 if is_cygwin(): 105 import ntpath 106 if ntpath.isabs(path): 107 return True 108 return os.path.isabs(path) 109 110def action_help(parser, action, *args): 111 parser.print_help() 112 return 0 113 114def error_exit(status, msg): 115 print(msg, file=sys.stderr) 116 sys.exit(status) 117 118def line(*args, file=sys.stdout): 119 if len(args) > 0: 120 ch = args[0] 121 else: 122 ch = '-' 123 print(ch * 60, file=file) 124 125def remove_readonly(func, path, _): 126 # Clear the readonly bit and reattempt the removal 127 os.chmod(path, stat.S_IWRITE | stat.S_IREAD) 128 dname = os.path.dirname(path) 129 os.chmod(dname, os.stat(dname).st_mode | stat.S_IWRITE) 130 func(path) 131 132def clean_bundles(bundles): 133 if not os.path.isfile(bundles): 134 return 135 with open(bundles, 'r') as f: 136 for fn in f.read().splitlines(): 137 if os.path.isdir(fn): 138 shutil.rmtree(fn, onerror=remove_readonly) 139 elif os.path.isfile(fn): 140 os.remove(fn) 141 os.remove(bundles) 142 143def clean_tcase(d, bundles): 144 if os.path.isdir(d): 145 clean_bundles(bundles) 146 for fn in glob.glob(d + '/*.tmp'): 147 os.remove(fn) 148 for fn in glob.glob(d + '/*.TMP'): 149 os.remove(fn) 150 151def check_availability(cmd): 152 if not shutil.which(cmd): 153 error_exit(1, cmd + ' command is not available') 154 155def check_units(name, category): 156 if len(UNITS) == 0: 157 return True 158 159 for u in UNITS: 160 ret = re.match(r'(.+)/(.+)', u) 161 if ret: 162 if ret.group(1, 2) == (category, name): 163 return True 164 elif u == name: 165 return True 166 return False 167 168def init_features(): 169 global _FEATURE_LIST 170 ret = subprocess.run([CTAGS, '--quiet', '--options=NONE', '--list-features', '--with-list=no'], 171 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) 172 _FEATURE_LIST = re.sub(r'(?m)^([^ ]+).*$', r'\1', 173 ret.stdout.decode('utf-8')).splitlines() 174 175def check_features(feature, ffile): 176 features = [] 177 if feature: 178 features = [feature] 179 elif os.path.isfile(ffile): 180 with open(ffile, 'r') as f: 181 features = f.read().splitlines() 182 183 for expected in features: 184 if expected == '': 185 continue 186 found = False 187 found_unexpectedly = False 188 if expected[0] == '!': 189 if expected[1:] in _FEATURE_LIST: 190 found_unexpectedly = True 191 else: 192 if expected in _FEATURE_LIST: 193 found = True 194 if found_unexpectedly: 195 return (False, expected) 196 elif not found: 197 return (False, expected) 198 return (True, '') 199 200def check_languages(cmdline, lfile): 201 if not os.path.isfile(lfile): 202 return (True, '') 203 204 ret = subprocess.run(cmdline + ['--list-languages'], 205 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) 206 langs = ret.stdout.decode('utf-8').splitlines() 207 208 with open(lfile, 'r') as f: 209 for expected in f.read().splitlines(): 210 found = False 211 if expected in langs: 212 found = True 213 if not found: 214 return (False, expected) 215 return (True, '') 216 217def decorate(decorator, msg, colorized): 218 if decorator == 'red': 219 num = '31' 220 elif decorator == 'green': 221 num = '32' 222 elif decorator == 'yellow': 223 num = '33' 224 else: 225 error_exit(1, 'INTERNAL ERROR: wrong run_result function') 226 227 if colorized: 228 return "\x1b[" + num + 'm' + msg + "\x1b[m" 229 else: 230 return msg 231 232def run_result(result_type, msg, output, *args, file=sys.stdout): 233 func_dict = { 234 'skip': run_result_skip, 235 'error': run_result_error, 236 'ok': run_result_ok, 237 'known_error': run_result_known_error, 238 } 239 240 func_dict[result_type](msg, file, COLORIZED_OUTPUT, *args) 241 file.flush() 242 if output: 243 with open(output, 'w') as f: 244 func_dict[result_type](msg, f, False, *args) 245 246def run_result_skip(msg, f, colorized, *args): 247 s = msg + decorate('yellow', 'skipped', colorized) 248 if len(args) > 0: 249 s += ' (' + args[0] + ')' 250 print(s, file=f) 251 252def run_result_error(msg, f, colorized, *args): 253 s = msg + decorate('red', 'failed', colorized) 254 if len(args) > 0: 255 s += ' (' + args[0] + ')' 256 print(s, file=f) 257 258def run_result_ok(msg, f, colorized, *args): 259 s = msg + decorate('green', 'passed', colorized) 260 if len(args) > 0: 261 s += ' (' + args[0] + ')' 262 print(s, file=f) 263 264def run_result_known_error(msg, f, colorized, *args): 265 s = msg + decorate('yellow', 'failed', colorized) + ' (KNOWN bug)' 266 print(s, file=f) 267 268def run_shrink(cmdline_template, finput, foutput, lang): 269 script = sys.argv[0] 270 script = os.path.splitext(script)[0] # remove '.py' 271 272 print('Shrinking ' + finput + ' as ' + lang) 273 # fallback to the shell script version 274 subprocess.run([SHELL, script, 'shrink', 275 '--timeout=1', '--foreground', 276 cmdline_template, finput, foutput]) 277 278# return a filter for normalizing the basename 279# 280# If internal is True, return a pair of [pattern, replacement], 281# otherwise return a list of command line arguments. 282def basename_filter(internal, output_type): 283 filters_external = { 284 'ctags': 's%\(^[^\t]\{1,\}\t\)\(/\{0,1\}\([^/\t]\{1,\}/\)*\)%\\1%', 285 # "input" in the expresion is for finding input file names in the TAGS file. 286 # RAWOUT.tmp: 287 # 288 # ./Units/parser-ada.r/ada-etags-suffix.d/input_0.adb,238 289 # package body Input_0 is ^?Input_0/b^A1,0 290 # 291 # With the original expression, both "./Units/parser-ada.r/ada-etags-suffix.d/" 292 # and "package body Input_0 is Input_0/' are deleted. 293 # FILTERED.tmp: 294 # 295 # input_0.adb,238 296 # b^A1,0 297 # 298 # Adding "input" ot the expression is for deleting only the former one and for 299 # skpping the later one. 300 # 301 # FIXME: if "input" is included as a substring of tag entry names, filtering 302 # with this expression makes the test fail. 303 'etags': 's%.*\/\(input[-._][[:print:]]\{1,\}\),\([0-9]\{1,\}$\)%\\1,\\2%', 304 'xref': 's%\(.*[[:digit:]]\{1,\} \)\([^ ]\{1,\}[^ ]\{1,\}\)/\([^ ].\{1,\}.\{1,\}$\)%\\1\\3%', 305 'json': 's%\("path": \)"[^"]\{1,\}/\([^/"]\{1,\}\)"%\\1"\\2"%', 306 } 307 filters_internal = { 308 'ctags': [r'(^[^\t]+\t)(/?([^/\t]+/)*)', r'\1'], 309 # See above comments about "input". 310 'etags': [r'.*/(input[-._]\S+),([0-9]+$)', r'\1,\2'], 311 'xref': [r'(.*\d+ )([^ ]+[^ ]+)/([^ ].+.+$)', r'\1\3'], 312 'json': [r'("path": )"[^"]+/([^/"]+)"', r'\1"\2"'], 313 } 314 if internal: 315 return filters_internal[output_type] 316 else: 317 return ['sed', '-e', filters_external[output_type]] 318 319# convert a command line list to a command line string 320def join_cmdline(cmdline): 321 # surround with '' if an argument includes spaces or '\' 322 # TODO: use more robust way 323 return ' '.join("'" + x + "'" if (' ' in x) or ('\\' in x) else x 324 for x in cmdline) 325 326def run_record_cmdline(cmdline, ffilter, ocmdline, output_type): 327 with open(ocmdline, 'w') as f: 328 print("%s\n%s \\\n| %s \\\n| %s\n" % ( 329 _PREPERE_ENV, 330 join_cmdline(cmdline), 331 join_cmdline(basename_filter(False, output_type)), 332 ffilter), file=f) 333 334def prepare_bundles(frm, to, obundles): 335 for src in glob.glob(frm + '/*'): 336 fn = os.path.basename(src) 337 if fn.startswith('input.'): 338 continue 339 elif fn.startswith('expected.tags'): 340 continue 341 elif fn.startswith('README'): 342 continue 343 elif fn in ['features', 'languages', 'filters']: 344 continue 345 elif fn == 'args.ctags': 346 continue 347 else: 348 dist = to + '/' + fn 349 if os.path.isdir(src): 350 shutil.copytree(src, dist, copy_function=shutil.copyfile) 351 else: 352 shutil.copyfile(src, dist) 353 with open(obundles, 'a') as f: 354 print(dist, file=f) 355 356def anon_normalize_sub(internal, ctags, input_actual, *args): 357 # TODO: "Units" should not be hardcoded. 358 input_expected = './Units' + re.sub(r'^.*?/Units', r'', input_actual, 1) 359 360 ret = subprocess.run([CTAGS, '--quiet', '--options=NONE', '--_anonhash=' + input_actual], 361 stdout=subprocess.PIPE) 362 actual = ret.stdout.decode('utf-8').splitlines()[0] 363 ret = subprocess.run([CTAGS, '--quiet', '--options=NONE', '--_anonhash=' + input_expected], 364 stdout=subprocess.PIPE) 365 expected = ret.stdout.decode('utf-8').splitlines()[0] 366 367 if internal: 368 retlist = [[actual, expected]] 369 else: 370 retlist = ['-e', 's/' + actual + '/' + expected + '/g'] 371 if len(args) > 0: 372 return retlist + anon_normalize_sub(internal, ctags, *args) 373 else: 374 return retlist 375 376def is_anon_normalize_needed(rawout): 377 with open(rawout, 'r', errors='ignore') as f: 378 if re.search(r'[0-9a-f]{8}', f.read()): 379 return True 380 return False 381 382# return a list of filters for normalizing anonhash 383# 384# If internal is True, return a list of pairs of [pattern, replacement], 385# otherwise return a list of command line arguments. 386def anon_normalize(internal, rawout, ctags, input_actual, *args): 387 if is_anon_normalize_needed(rawout): 388 return anon_normalize_sub(internal, ctags, input_actual, *args) 389 else: 390 return [] 391 392def run_filter(finput, foutput, base_filter, anon_filters): 393 pat1 = [re.compile(base_filter[0]), base_filter[1]] 394 pat2 = [(re.compile(p[0]), p[1]) for p in anon_filters] 395 with open(finput, 'r', errors='surrogateescape') as fin, \ 396 open(foutput, 'w', errors='surrogateescape', newline='\n') as fout: 397 for l in fin: 398 l = pat1[0].sub(pat1[1], l, 1) 399 for p in pat2: 400 l = p[0].sub(p[1], l) 401 print(l, end='', file=fout) 402 403def guess_lang(cmdline, finput): 404 ret = subprocess.run(cmdline + ['--print-language', finput], 405 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) 406 return re.sub(r'^.*: ', r'', 407 ret.stdout.decode('utf-8').replace("\r\n", "\n").replace("\n", '')) 408 409def guess_lang_from_log(log): 410 with open(log, 'r', encoding='utf-8', errors='ignore') as f: 411 for l in f: 412 ret = re.match('OPENING.* as (.*) language .*file ', l) 413 if ret: 414 return ret.group(1) 415 return '' 416 417def run_tcase(finput, t, name, tclass, category, build_t, extra_inputs): 418 global L_PASSED 419 global L_FIXED 420 global L_FAILED_BY_STATUS 421 global L_FAILED_BY_DIFF 422 global L_SKIPPED_BY_FEATURES 423 global L_SKIPPED_BY_LANGUAGES 424 global L_SKIPPED_BY_ILOOP 425 global L_KNOWN_BUGS 426 global L_FAILED_BY_TIMEED_OUT 427 global L_BROKEN_ARGS_CTAGS 428 global L_VALGRIND 429 430 o = build_t 431 432 fargs = t + '/args.ctags' 433 ffeatures = t + '/features' 434 flanguages = t + '/languages' 435 ffilter = t + '/filter' 436 437 fexpected = t + '/expected.tags' 438 output_type = 'ctags' 439 output_label = '' 440 output_tflag = [] 441 output_feature = '' 442 output_lang_extras = '' 443 444 if os.path.isfile(fexpected): 445 pass 446 elif os.path.isfile(t + '/expected.tags-e'): 447 fexpected = t + '/expected.tags-e' 448 output_type = 'etags' 449 output_label = '/' + output_type 450 output_tflag = ['-e', '--tag-relative=no'] 451 elif os.path.isfile(t + '/expected.tags-x'): 452 fexpected = t + '/expected.tags-x' 453 output_type = 'xref' 454 output_label = '/' + output_type 455 output_tflag = ['-x'] 456 elif os.path.isfile(t + '/expected.tags-json'): 457 fexpected = t + '/expected.tags-json' 458 output_type = 'json' 459 output_label = '/' + output_type 460 output_tflag = ['--output-format=json'] 461 output_feature = 'json' 462 463 if len(extra_inputs) > 0: 464 output_lang_extras = ' (multi inputs)' 465 466 if not shutil.which(ffilter): 467 ffilter = 'cat' 468 469 ostderr = o + '/' + _STDERR_OUTPUT_NAME 470 orawout = o + '/RAWOUT.tmp' 471 ofiltered = o + '/FILTERED.tmp' 472 odiff = o + '/' + _DIFF_OUTPUT_NAME 473 ocmdline = o + '/CMDLINE.tmp' 474 ovalgrind = o + '/' + _VALGRIND_OUTPUT_NAME 475 oresult = o + '/RESULT.tmp' 476 oshrink_template = o + '/SHRINK-%s.tmp' 477 obundles = o + '/BUNDLES' 478 479 broken_args_ctags = False 480 481 # 482 # Filtered by UNIT 483 # 484 if not check_units(name, category): 485 return False 486 487 # 488 # Build cmdline 489 # 490 cmdline = [CTAGS, '--verbose', '--options=NONE', '--fields=-T'] 491 if PRETENSE_OPTS != '': 492 cmdline += [PRETENSE_OPTS] 493 cmdline += ['--optlib-dir=+' + t + '/optlib', '-o', '-'] 494 if os.path.isfile(fargs): 495 cmdline += ['--options=' + fargs] 496 ret = subprocess.run(cmdline + ['--_force-quit=0'], 497 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 498 if ret.returncode != 0: 499 broken_args_ctags = True 500 501 # 502 # make a backup (basedcmdline) of cmdline. basedcmdline is used 503 # as a command line template for running shrinker. basedcmdline 504 # should not include the name of the input file name. The 505 # shrinker makes a another cmdline by applying a real input file 506 # name to the template. On the other hand, cmdline is 507 # destructively updated by appending input file name in this 508 # function. The file name should not be included in the cmdline 509 # template. 510 # 511 # To avoid the updating in this function propagating to 512 # basecmdline, we copy the cmdline here. 513 # 514 basecmdline = cmdline[:] 515 516 # 517 # Filtered by LANGUAGES 518 # 519 guessed_lang = None 520 if len(LANGUAGES) > 0: 521 guessed_lang = guess_lang(basecmdline, finput) 522 if not guessed_lang in LANGUAGES: 523 return False 524 525 clean_tcase(o, obundles) 526 os.makedirs(o, exist_ok=True) 527 if not os.path.samefile(o, t): 528 prepare_bundles(t, o, obundles) 529 530 531 # helper function for building some strings based on guessed_lang 532 def build_strings(guessed_lang): 533 return ('%-59s ' % ('Testing ' + name + ' as ' + guessed_lang + output_lang_extras + output_label), 534 join_cmdline(basecmdline) + ' --language-force=' + guessed_lang + ' %s > /dev/null 2>&1', 535 oshrink_template % (guessed_lang.replace('/', '-'))) 536 537 (tmp, feat) = check_features(output_feature, ffeatures) 538 if not tmp: 539 if not guessed_lang: 540 guessed_lang = guess_lang(basecmdline, finput) 541 msg = build_strings(guessed_lang)[0] 542 L_SKIPPED_BY_FEATURES += [category + '/' + name] 543 if feat.startswith('!'): 544 run_result('skip', msg, oresult, 'unwanted feature "' + feat[1:] + '" is available') 545 else: 546 run_result('skip', msg, oresult, 'required feature "' + feat + '" is not available') 547 return False 548 (tmp, lang) = check_languages(basecmdline, flanguages) 549 if not tmp: 550 if not guessed_lang: 551 guessed_lang = guess_lang(basecmdline, finput) 552 msg = build_strings(guessed_lang)[0] 553 L_SKIPPED_BY_LANGUAGES += [category + '/' + name] 554 run_result('skip', msg, oresult, 'required language parser "' + lang + '" is not available') 555 return False 556 if WITH_TIMEOUT == 0 and tclass == 'i': 557 if not guessed_lang: 558 guessed_lang = guess_lang(basecmdline, finput) 559 msg = build_strings(guessed_lang)[0] 560 L_SKIPPED_BY_ILOOP += [category + '/' + name] 561 run_result('skip', msg, oresult, 'may cause an infinite loop') 562 return False 563 if broken_args_ctags: 564 if not guessed_lang: 565 guessed_lang = guess_lang(basecmdline, finput) 566 msg = build_strings(guessed_lang)[0] 567 L_BROKEN_ARGS_CTAGS += [category + '/' + name] 568 run_result('error', msg, None, 'broken args.ctags?') 569 return False 570 571 cmdline += output_tflag + [finput] 572 if len(extra_inputs) > 0: 573 cmdline += extra_inputs 574 575 timeout_value = WITH_TIMEOUT 576 if WITH_VALGRIND: 577 cmdline = ['valgrind', '--leak-check=full', '--track-origins=yes', 578 '--error-exitcode=' + str(_VALGRIND_EXIT), '--log-file=' + ovalgrind] + cmdline 579 timeout_value *= _VG_TIMEOUT_FACTOR 580 if timeout_value == 0: 581 timeout_value = None 582 583 start = time.time() 584 try: 585 with open(orawout, 'wb') as fo, \ 586 open(ostderr, 'wb') as fe: 587 ret = subprocess.run(cmdline, stdout=fo, stderr=fe, 588 timeout=timeout_value) 589 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 590 except subprocess.TimeoutExpired: 591 if not guessed_lang: 592 guessed_lang = guess_lang(basecmdline, finput) 593 (msg, cmdline_template, oshrink) = build_strings(guessed_lang) 594 L_FAILED_BY_TIMEED_OUT += [category + '/' + name] 595 run_result('error', msg, oresult, 'TIMED OUT') 596 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 597 if RUN_SHRINK and len(extra_inputs) == 0: 598 run_shrink(cmdline_template, finput, oshrink, guessed_lang) 599 return False 600 #print('execute time: %f' % (time.time() - start)) 601 602 guessed_lang = guess_lang_from_log(ostderr) 603 (msg, cmdline_template, oshrink) = build_strings(guessed_lang) 604 605 if ret.returncode != 0: 606 if WITH_VALGRIND and ret.returncode == _VALGRIND_EXIT and \ 607 tclass != 'v': 608 L_VALGRIND += [category + '/' + name] 609 run_result('error', msg, oresult, 'valgrind-error') 610 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 611 return False 612 elif tclass == 'b': 613 L_KNOWN_BUGS += [category + '/' + name] 614 run_result('known_error', msg, oresult) 615 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 616 if RUN_SHRINK and len(extra_inputs) == 0: 617 run_shrink(cmdline_template, finput, oshrink, guessed_lang) 618 return True 619 else: 620 L_FAILED_BY_STATUS += [category + '/' + name] 621 run_result('error', msg, oresult, 'unexpected exit status: ' + str(ret.returncode)) 622 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 623 if RUN_SHRINK and len(extra_inputs) == 0: 624 run_shrink(cmdline_template, finput, oshrink, guessed_lang) 625 return False 626 elif WITH_VALGRIND and tclass == 'v': 627 L_FIXED += [category + '/' + name] 628 629 if not os.path.isfile(fexpected): 630 clean_tcase(o, obundles) 631 if tclass == 'b': 632 L_FIXED += [category + '/' + name] 633 elif tclass == 'i': 634 L_FIXED += [category + '/' + name] 635 636 L_PASSED += [category + '/' + name] 637 run_result('ok', msg, None, '"expected.tags*" not found') 638 return True 639 640 start = time.time() 641 if ffilter != 'cat': 642 # Use external filter 643 filter_cmd = basename_filter(False, output_type) + \ 644 anon_normalize(False, orawout, CTAGS, finput, *extra_inputs) + \ 645 ['<', orawout] 646 filter_cmd += ['|', ffilter] 647 filter_cmd += ['>', ofiltered] 648 #print(filter_cmd) 649 subprocess.run([SHELL, '-c', join_cmdline(filter_cmd)]) 650 else: 651 # Use internal filter 652 run_filter(orawout, ofiltered, basename_filter(True, output_type), 653 anon_normalize(True, orawout, CTAGS, finput, *extra_inputs)) 654 #print('filter time: %f' % (time.time() - start)) 655 656 start = time.time() 657 if filecmp.cmp(fexpected, ofiltered): 658 ret.returncode = 0 659 else: 660 with open(odiff, 'wb') as f: 661 ret = subprocess.run(['diff', '-U', str(DIFF_U_NUM), 662 '-I', '^!_TAG', '--strip-trailing-cr', fexpected, ofiltered], 663 stdout=f) 664 #print('diff time: %f' % (time.time() - start)) 665 666 if ret.returncode == 0: 667 clean_tcase(o, obundles) 668 if tclass == 'b': 669 L_FIXED += [category + '/' + name] 670 elif WITH_TIMEOUT != 0 and tclass == 'i': 671 L_FIXED += [category + '/' + name] 672 673 L_PASSED += [category + '/' + name] 674 run_result('ok', msg, None) 675 return True 676 else: 677 if tclass == 'b': 678 L_KNOWN_BUGS += [category + '/' + name] 679 run_result('known_error', msg, oresult) 680 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 681 return True 682 else: 683 L_FAILED_BY_DIFF += [category + '/' + name] 684 run_result('error', msg, oresult, 'unexpected output') 685 run_record_cmdline(cmdline, ffilter, ocmdline, output_type) 686 return False 687 688def create_thread_queue(func): 689 q = queue.Queue() 690 threads = [] 691 for i in range(NUM_WORKER_THREADS): 692 t = threading.Thread(target=worker, args=(func, q), daemon=True) 693 t.start() 694 threads.append(t) 695 return (q, threads) 696 697def worker(func, q): 698 while True: 699 item = q.get() 700 if item is None: 701 break 702 try: 703 func(*item) 704 except: 705 import traceback 706 traceback.print_exc() 707 q.task_done() 708 709def join_workers(q, threads): 710 # block until all tasks are done 711 try: 712 q.join() 713 except KeyboardInterrupt: 714 # empty the queue 715 while True: 716 try: 717 q.get_nowait() 718 except queue.Empty: 719 break 720 # try to stop workers 721 for i in range(NUM_WORKER_THREADS): 722 q.put(None) 723 for t in threads: 724 t.join(timeout=2) 725 # exit regardless that workers are stopped 726 sys.exit(1) 727 728 # stop workers 729 for i in range(NUM_WORKER_THREADS): 730 q.put(None) 731 for t in threads: 732 t.join() 733 734def accepted_file(fname): 735 # Ignore backup files 736 return not fname.endswith('~') 737 738def run_dir(category, base_dir, build_base_dir): 739 # 740 # Filtered by CATEGORIES 741 # 742 if len(CATEGORIES) > 0 and not category in CATEGORIES: 743 return False 744 745 print("\nCategory: " + category) 746 line() 747 748 (q, threads) = create_thread_queue(run_tcase) 749 750 for finput in glob.glob(base_dir + '/*.[dbtiv]/input.*'): 751 finput = finput.replace('\\', '/') # for Windows 752 if not accepted_file(finput): 753 continue 754 755 dname = os.path.dirname(finput) 756 extra_inputs = sorted(map(lambda x: x.replace('\\', '/'), # for Windows 757 filter(accepted_file, 758 glob.glob(dname + '/input[-_][0-9].*') + 759 glob.glob(dname + '/input[-_][0-9][-_]*.*') 760 ))) 761 762 tcase_dir = dname 763 build_tcase_dir = build_base_dir + remove_prefix(tcase_dir, base_dir) 764 ret = re.match(r'^.*/(.*)\.([dbtiv])$', tcase_dir) 765 (name, tclass) = ret.group(1, 2) 766 q.put((finput, tcase_dir, name, tclass, category, build_tcase_dir, extra_inputs)) 767 768 join_workers(q, threads) 769 770def run_show_diff_output(units_dir, t): 771 print("\t", end='') 772 line('.') 773 for fn in glob.glob(units_dir + '/' + t + '.*/' + _DIFF_OUTPUT_NAME): 774 with open(fn, 'r') as f: 775 for l in f: 776 print("\t" + l, end='') 777 print() 778 779def run_show_stderr_output(units_dir, t): 780 print("\t", end='') 781 line('.') 782 for fn in glob.glob(units_dir + '/' + t + '.*/' + _STDERR_OUTPUT_NAME): 783 with open(fn, 'r') as f: 784 lines = f.readlines() 785 for l in lines[-50:]: 786 print("\t" + l, end='') 787 print() 788 789def run_show_valgrind_output(units_dir, t): 790 print("\t", end='') 791 line('.') 792 for fn in glob.glob(units_dir + '/' + t + '.*/' + _VALGRIND_OUTPUT_NAME): 793 with open(fn, 'r') as f: 794 for l in f: 795 print("\t" + l, end='') 796 print() 797 798def run_summary(build_dir): 799 print() 800 print('Summary (see CMDLINE.tmp to reproduce without test harness)') 801 line() 802 803 fmt = ' %-40s%d' 804 print(fmt % ('#passed:', len(L_PASSED))) 805 806 print(fmt % ('#FIXED:', len(L_FIXED))) 807 for t in L_FIXED: 808 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 809 810 print(fmt % ('#FAILED (broken args.ctags?):', len(L_BROKEN_ARGS_CTAGS))) 811 for t in L_BROKEN_ARGS_CTAGS: 812 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 813 814 print(fmt % ('#FAILED (unexpected-exit-status):', len(L_FAILED_BY_STATUS))) 815 for t in L_FAILED_BY_STATUS: 816 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 817 if SHOW_DIFF_OUTPUT: 818 run_show_stderr_output(build_dir, remove_prefix(t, _DEFAULT_CATEGORY + '/')) 819 820 print(fmt % ('#FAILED (unexpected-output):', len(L_FAILED_BY_DIFF))) 821 for t in L_FAILED_BY_DIFF: 822 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 823 if SHOW_DIFF_OUTPUT: 824 run_show_stderr_output(build_dir, remove_prefix(t, _DEFAULT_CATEGORY + '/')) 825 run_show_diff_output(build_dir, remove_prefix(t, _DEFAULT_CATEGORY + '/')) 826 827 if WITH_TIMEOUT != 0: 828 print(fmt % ('#TIMED-OUT (' + str(WITH_TIMEOUT) + 's):', len(L_FAILED_BY_TIMEED_OUT))) 829 for t in L_FAILED_BY_TIMEED_OUT: 830 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 831 832 print(fmt % ('#skipped (features):', len(L_SKIPPED_BY_FEATURES))) 833 for t in L_SKIPPED_BY_FEATURES: 834 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 835 836 print(fmt % ('#skipped (languages):', len(L_SKIPPED_BY_LANGUAGES))) 837 for t in L_SKIPPED_BY_LANGUAGES: 838 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 839 840 if WITH_TIMEOUT == 0: 841 print(fmt % ('#skipped (infinite-loop):', len(L_SKIPPED_BY_ILOOP))) 842 for t in L_SKIPPED_BY_ILOOP: 843 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 844 845 print(fmt % ('#known-bugs:', len(L_KNOWN_BUGS))) 846 for t in L_KNOWN_BUGS: 847 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 848 849 if WITH_VALGRIND: 850 print(fmt % ('#valgrind-error:', len(L_VALGRIND))) 851 for t in L_VALGRIND: 852 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 853 if SHOW_DIFF_OUTPUT: 854 print(fmt % ('##valgrind-error:', len(L_VALGRIND))) 855 for t in L_VALGRIND: 856 print("\t" + remove_prefix(t, _DEFAULT_CATEGORY + '/')) 857 run_show_valgrind_output(build_dir, remove_prefix(t, _DEFAULT_CATEGORY + '/')) 858 859def make_pretense_map(arg): 860 r = '' 861 for p in arg.split(','): 862 ret = re.match(r'(.*)/(.*)', p) 863 if not ret: 864 error_exit(1, 'wrong format of --_pretend option arg') 865 866 (newlang, oldlang) = ret.group(1, 2) 867 if newlang == '': 868 error_exit(1, 'newlang part of --_pretend option arg is empty') 869 if oldlang == '': 870 error_exit(1, 'oldlang part of --_pretend option arg is empty') 871 872 r += ' --_pretend-' + newlang + '=' + oldlang 873 874 return r 875 876def action_run(parser, action, *args): 877 global CATEGORIES 878 global CTAGS 879 global UNITS 880 global LANGUAGES 881 global WITH_TIMEOUT 882 global WITH_VALGRIND 883 global COLORIZED_OUTPUT 884 global RUN_SHRINK 885 global SHOW_DIFF_OUTPUT 886 global PRETENSE_OPTS 887 global NUM_WORKER_THREADS 888 global SHELL 889 890 parser.add_argument('--categories', metavar='CATEGORY1[,CATEGORY2,...]', 891 help='run only CATEGORY* related cases.') 892 parser.add_argument('--ctags', 893 help='ctags executable file for testing') 894 parser.add_argument('--units', metavar='UNITS1[,UNITS2,...]', 895 help='run only UNIT(S).') 896 parser.add_argument('--languages', metavar='PARSER1[,PARSER2,...]', 897 help='run only PARSER* related cases.') 898 parser.add_argument('--with-timeout', type=int, default=0, 899 metavar='DURATION', 900 help='run a test case with specified timeout in seconds. 0 means no timeout (default).') 901 parser.add_argument('--with-valgrind', action='store_true', default=False, 902 help='run a test case under valgrind') 903 parser.add_argument('--colorized-output', choices=['yes', 'no'], default='yes', 904 help='print the result in color.') 905 parser.add_argument('--run-shrink', action='store_true', default=False, 906 help='(TODO: NOT IMPLEMENTED YET)') 907 parser.add_argument('--show-diff-output', action='store_true', default=False, 908 help='show diff output (and valgrind errors) for failed test cases in the summary.') 909 parser.add_argument('--with-pretense-map', 910 metavar='NEWLANG0/OLDLANG0[,...]', 911 help='make NEWLANG parser pretend OLDLANG.') 912 parser.add_argument('--threads', type=int, default=NUM_WORKER_THREADS, 913 help='number of worker threads') 914 parser.add_argument('--shell', 915 help='shell to be used.') 916 parser.add_argument('units_dir', 917 help='Units directory.') 918 parser.add_argument('build_dir', nargs='?', default='', 919 help='Build directory. If not given, units_dir is used.') 920 921 res = parser.parse_args(args) 922 if res.categories: 923 CATEGORIES = [x if x == 'ROOT' or x.endswith('.r') else x + '.r' 924 for x in res.categories.split(',')] 925 if res.ctags: 926 CTAGS = res.ctags 927 if res.units: 928 UNITS = res.units.split(',') 929 if res.languages: 930 LANGUAGES = res.languages.split(',') 931 WITH_TIMEOUT = res.with_timeout 932 WITH_VALGRIND = res.with_valgrind 933 COLORIZED_OUTPUT = (res.colorized_output == 'yes') 934 RUN_SHRINK = res.run_shrink 935 SHOW_DIFF_OUTPUT = res.show_diff_output 936 if res.with_pretense_map: 937 PRETENSE_OPTS = make_pretense_map(res.with_pretense_map) 938 NUM_WORKER_THREADS = res.threads 939 if res.shell: 940 SHELL = res.shell 941 if res.build_dir == '': 942 res.build_dir = res.units_dir 943 944 if WITH_VALGRIND: 945 check_availability('valgrind') 946 check_availability('diff') 947 init_features() 948 949 if isabs(res.build_dir): 950 build_dir = res.build_dir 951 else: 952 build_dir = os.path.realpath(res.build_dir) 953 954 category = _DEFAULT_CATEGORY 955 if len(CATEGORIES) == 0 or (category in CATEGORIES): 956 run_dir(category, res.units_dir, build_dir) 957 958 for d in glob.glob(res.units_dir + '/*.r'): 959 d = d.replace('\\', '/') # for Windows 960 if not os.path.isdir(d): 961 continue 962 category = os.path.basename(d) 963 build_d = res.build_dir + '/' + category 964 run_dir(category, d, build_d) 965 966 run_summary(build_dir) 967 968 if L_FAILED_BY_STATUS or L_FAILED_BY_DIFF or \ 969 L_FAILED_BY_TIMEED_OUT or L_BROKEN_ARGS_CTAGS or \ 970 L_VALGRIND: 971 return 1 972 else: 973 return 0 974 975def action_clean(parser, action, *args): 976 parser.add_argument('units_dir', 977 help='Build directory for units testing.') 978 979 res = parser.parse_args(args) 980 units_dir = res.units_dir 981 982 if not os.path.isdir(units_dir): 983 error_exit(0, 'No such directory: ' + units_dir) 984 985 for bundles in glob.glob(units_dir + '/**/BUNDLES', recursive=True): 986 clean_bundles(bundles) 987 988 for fn in glob.glob(units_dir + '/**/*.tmp', recursive=True): 989 os.remove(fn) 990 for fn in glob.glob(units_dir + '/**/*.TMP', recursive=True): 991 os.remove(fn) 992 return 0 993 994def tmain_compare_result(build_topdir): 995 for fn in glob.glob(build_topdir + '/*/*-diff.txt'): 996 print(fn) 997 print() 998 with open(fn, 'r', errors='replace') as f: 999 for l in f: 1000 print("\t" + l, end='') 1001 print() 1002 1003 for fn in glob.glob(build_topdir + '/*/gdb-backtrace.txt'): 1004 with open(fn, 'r', errors='replace') as f: 1005 for l in f: 1006 print("\t" + l, end='') 1007 1008def tmain_compare(subdir, build_subdir, aspect, file): 1009 msg = '%-59s ' % (aspect) 1010 generated = build_subdir + '/' + aspect + '-diff.txt' 1011 actual = build_subdir + '/' + aspect + '-actual.txt' 1012 expected = subdir + '/' + aspect + '-expected.txt' 1013 if os.path.isfile(actual) and os.path.isfile(expected) and \ 1014 filecmp.cmp(actual, expected): 1015 run_result('ok', msg, None, file=file) 1016 # When successful, remove files generated in the last 1017 # failure to make the directory clean. 1018 # Unlike other generated files like gdb-backtrace.txt 1019 # misc/review script looks at the -diff.txt file. 1020 # Therefore we handle -diff.txt specially here. 1021 if os.path.isfile(generated): 1022 os.remove(generated) 1023 return True 1024 else: 1025 with open(generated, 'wb') as f: 1026 subprocess.run(['diff', '-U', 1027 str(DIFF_U_NUM), '--strip-trailing-cr', 1028 expected, actual], 1029 stdout=f, stderr=subprocess.STDOUT) 1030 run_result('error', msg, None, 'diff: ' + generated, file=file) 1031 return False 1032 1033def failed_git_marker(fn): 1034 if shutil.which('git'): 1035 ret = subprocess.run(['git', 'ls-files', '--', fn], 1036 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) 1037 if ret.returncode == 0 and ret.stdout == b'': 1038 return '<G>' 1039 return '' 1040 1041def is_crashed(fn): 1042 with open(fn, 'r') as f: 1043 if 'core dump' in f.read(): 1044 return True 1045 return False 1046 1047def print_backtraces(ctags_exe, cores, fn): 1048 with open(fn, 'wb') as f: 1049 for coref in cores: 1050 subprocess.run(['gdb', ctags_exe, '-c', coref, '-ex', 'where', '-batch'], 1051 stdout=f, stderr=subprocess.DEVNULL) 1052 1053def tmain_sub(test_name, basedir, subdir, build_subdir): 1054 global TMAIN_STATUS 1055 global TMAIN_FAILED 1056 1057 CODE_FOR_IGNORING_THIS_TMAIN_TEST = 77 1058 1059 os.makedirs(build_subdir, exist_ok=True) 1060 1061 for fn in glob.glob(build_subdir + '/*-actual.txt'): 1062 os.remove(fn) 1063 1064 strbuf = io.StringIO() 1065 print("\nTesting " + test_name, file=strbuf) 1066 line('-', file=strbuf) 1067 1068 if isabs(CTAGS): 1069 ctags_path = CTAGS 1070 else: 1071 ctags_path = os.path.join(basedir, CTAGS) 1072 1073 if isabs(READTAGS): 1074 readtags_path = READTAGS 1075 else: 1076 readtags_path = os.path.join(basedir, READTAGS) 1077 1078 if isabs(OPTSCRIPT): 1079 optscript_path = OPTSCRIPT 1080 else: 1081 optscript_path = os.path.join(basedir, OPTSCRIPT) 1082 1083 start = time.time() 1084 ret = subprocess.run([SHELL, 'run.sh', 1085 ctags_path, 1086 build_subdir, 1087 readtags_path, 1088 optscript_path], 1089 cwd=subdir, 1090 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1091 #print('execute time: %f' % (time.time() - start), file=strbuf) 1092 1093 encoding = 'utf-8' 1094 try: 1095 stdout = ret.stdout.decode(encoding).replace("\r\n", "\n") 1096 except UnicodeError: 1097 encoding = 'iso-8859-1' 1098 stdout = ret.stdout.decode(encoding).replace("\r\n", "\n") 1099 stderr = ret.stderr.decode('utf-8').replace("\r\n", "\n") 1100 if os.path.basename(CTAGS) != 'ctags': 1101 # program name needs to be canonicalized 1102 stderr = re.sub('(?m)^' + os.path.basename(CTAGS) + ':', 'ctags:', stderr) 1103 1104 if ret.returncode == CODE_FOR_IGNORING_THIS_TMAIN_TEST: 1105 run_result('skip', '', None, stdout.replace("\n", ''), file=strbuf) 1106 print(strbuf.getvalue(), end='') 1107 sys.stdout.flush() 1108 strbuf.close() 1109 return True 1110 1111 with open(build_subdir + '/exit-actual.txt', 'w', newline='\n') as f: 1112 print(ret.returncode, file=f) 1113 with open(build_subdir + '/stdout-actual.txt', 'w', newline='\n', encoding=encoding) as f: 1114 print(stdout, end='', file=f) 1115 with open(build_subdir + '/stderr-actual.txt', 'w', newline='\n') as f: 1116 print(stderr, end='', file=f) 1117 1118 if os.path.isfile(build_subdir + '/tags'): 1119 os.rename(build_subdir + '/tags', build_subdir + '/tags-actual.txt') 1120 1121 for aspect in ['stdout', 'stderr', 'exit', 'tags']: 1122 expected_txt = subdir + '/' + aspect + '-expected.txt' 1123 actual_txt = build_subdir + '/' + aspect + '-actual.txt' 1124 if os.path.isfile(expected_txt): 1125 if tmain_compare(subdir, build_subdir, aspect, strbuf): 1126 os.remove(actual_txt) 1127 else: 1128 TMAIN_FAILED += [test_name + '/' + aspect + '-compare' + 1129 failed_git_marker(expected_txt)] 1130 TMAIN_STATUS = False 1131 if aspect == 'stderr' and \ 1132 is_crashed(actual_txt) and \ 1133 shutil.which('gdb'): 1134 print_backtraces(ctags_path, 1135 glob.glob(build_subdir + '/core*'), 1136 build_subdir + '/gdb-backtrace.txt') 1137 elif os.path.isfile(actual_txt): 1138 os.remove(actual_txt) 1139 1140 print(strbuf.getvalue(), end='') 1141 sys.stdout.flush() 1142 strbuf.close() 1143 return True 1144 1145def tmain_run(topdir, build_topdir, units): 1146 global TMAIN_STATUS 1147 1148 TMAIN_STATUS = True 1149 1150 (q, threads) = create_thread_queue(tmain_sub) 1151 1152 basedir = os.getcwd() 1153 for subdir in glob.glob(topdir + '/*.d'): 1154 test_name = os.path.basename(subdir)[:-2] 1155 1156 if len(units) > 0 and not test_name in units: 1157 continue 1158 1159 build_subdir = build_topdir + '/' + os.path.basename(subdir) 1160 q.put((test_name, basedir, subdir, build_subdir)) 1161 1162 join_workers(q, threads) 1163 1164 print() 1165 if not TMAIN_STATUS: 1166 print('Failed tests') 1167 line('=') 1168 for f in TMAIN_FAILED: 1169 print(re.sub('<G>', ' (not committed/cached yet)', f)) 1170 print() 1171 1172 if SHOW_DIFF_OUTPUT: 1173 print('Detail [compare]') 1174 line('-') 1175 tmain_compare_result(build_topdir) 1176 1177 return TMAIN_STATUS 1178 1179def action_tmain(parser, action, *args): 1180 global CTAGS 1181 global COLORIZED_OUTPUT 1182 global WITH_VALGRIND 1183 global SHOW_DIFF_OUTPUT 1184 global READTAGS 1185 global OPTSCRIPT 1186 global UNITS 1187 global NUM_WORKER_THREADS 1188 global SHELL 1189 1190 parser.add_argument('--ctags', 1191 help='ctags executable file for testing') 1192 parser.add_argument('--colorized-output', choices=['yes', 'no'], default='yes', 1193 help='print the result in color.') 1194 parser.add_argument('--with-valgrind', action='store_true', default=False, 1195 help='(not implemented) run a test case under valgrind') 1196 parser.add_argument('--show-diff-output', action='store_true', default=False, 1197 help='how diff output for failed test cases in the summary.') 1198 parser.add_argument('--readtags', 1199 help='readtags executable file for testing') 1200 parser.add_argument('--optscript', 1201 help='optscript executable file for testing') 1202 parser.add_argument('--units', metavar='UNITS1[,UNITS2,...]', 1203 help='run only Tmain/UNIT*.d (.d is not needed)') 1204 parser.add_argument('--threads', type=int, default=NUM_WORKER_THREADS, 1205 help='number of worker threads') 1206 parser.add_argument('--shell', 1207 help='shell to be used.') 1208 parser.add_argument('tmain_dir', 1209 help='Tmain directory.') 1210 parser.add_argument('build_dir', nargs='?', default='', 1211 help='Build directory. If not given, tmain_dir is used.') 1212 1213 res = parser.parse_args(args) 1214 if res.ctags: 1215 CTAGS = res.ctags 1216 COLORIZED_OUTPUT = (res.colorized_output == 'yes') 1217 WITH_VALGRIND = res.with_valgrind 1218 SHOW_DIFF_OUTPUT = res.show_diff_output 1219 if res.readtags: 1220 READTAGS = res.readtags 1221 if res.optscript: 1222 OPTSCRIPT = res.optscript 1223 if res.units: 1224 UNITS = res.units.split(',') 1225 NUM_WORKER_THREADS = res.threads 1226 if res.shell: 1227 SHELL = res.shell 1228 if res.build_dir == '': 1229 res.build_dir = res.tmain_dir 1230 1231 #check_availability('awk') 1232 check_availability('diff') 1233 1234 if isabs(res.build_dir): 1235 build_dir = res.build_dir 1236 else: 1237 build_dir = os.path.realpath(res.build_dir) 1238 1239 ret = tmain_run(res.tmain_dir, build_dir, UNITS) 1240 if ret: 1241 return 0 1242 else: 1243 return 1 1244 1245def action_clean_tmain(parser, action, *args): 1246 parser.add_argument('tmain_dir', 1247 help='Build directory for tmain testing.') 1248 1249 res = parser.parse_args(args) 1250 tmain_dir = res.tmain_dir 1251 1252 if not os.path.isdir(tmain_dir): 1253 error_exit(0, 'No such directory: ' + tmain_dir) 1254 1255 for obj in ['stdout', 'stderr', 'exit', 'tags']: 1256 for typ in ['actual', 'diff']: 1257 for fn in glob.glob(tmain_dir + '/**/' + obj + '-' + typ + '.txt', recursive=True): 1258 os.remove(fn) 1259 for fn in glob.glob(tmain_dir + '/**/gdb-backtrace.txt', recursive=True): 1260 os.remove(fn) 1261 return 0 1262 1263def prepare_environment(): 1264 global _PREPERE_ENV 1265 1266 os.environ['LC_ALL'] = 'C' 1267 os.environ['MSYS2_ARG_CONV_EXCL'] = '--regex-;--_scopesep;--exclude;--exclude-exception' 1268 1269 _PREPERE_ENV = """LC_ALL="C"; export LC_ALL 1270MSYS2_ARG_CONV_EXCL='--regex-;--_scopesep;--exclude;--exclude-exception' export MSYS2_ARG_CONV_EXCL 1271""" 1272 1273# enable ANSI escape sequences on Windows 10 1511 (10.0.10586) or later 1274def enable_esc_sequence(): 1275 if os.name != 'nt': 1276 return 1277 1278 import ctypes 1279 1280 kernel32 = ctypes.windll.kernel32 1281 1282 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 1283 STD_OUTPUT_HANDLE = -11 1284 1285 out = kernel32.GetStdHandle(STD_OUTPUT_HANDLE) 1286 mode = ctypes.c_ulong() 1287 if kernel32.GetConsoleMode(out, ctypes.byref(mode)): 1288 kernel32.SetConsoleMode(out, 1289 mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) 1290 1291def main(): 1292 prepare_environment() 1293 enable_esc_sequence() 1294 1295 parser = argparse.ArgumentParser( 1296 description='Units test harness for ctags.') 1297 subparsers = parser.add_subparsers(dest='action', metavar='ACTION') 1298 cmdmap = {} 1299 cmdmap['run'] = [action_run, 1300 subparsers.add_parser('run', aliases=['units'], 1301 description='Run all tests case under units_dir.', 1302 help='Run all tests case')] 1303 cmdmap['units'] = cmdmap['run'] 1304 cmdmap['clean'] = [action_clean, 1305 subparsers.add_parser('clean', 1306 description='Clean all files created during units testing.', 1307 help='Clean all files created during units testing')] 1308 cmdmap['tmain'] = [action_tmain, 1309 subparsers.add_parser('tmain', 1310 description='Run tests for main part of ctags.', 1311 help='Run tests for main part of ctags')] 1312 cmdmap['clean-tmain'] = [action_clean_tmain, 1313 subparsers.add_parser('clean-tmain', 1314 description='Clean all files created during tmain testing.', 1315 help='Clean all files created during tmain testing')] 1316 subparsers.add_parser('help', 1317 help='show this help message and exit') 1318 cmdmap['help'] = [action_help, parser] 1319 1320 if len(sys.argv) < 2: 1321 parser.print_help() 1322 sys.exit(1) 1323 1324 res = parser.parse_args(sys.argv[1:2]) 1325 (func, subparser) = cmdmap[res.action] 1326 sys.exit(func(subparser, *sys.argv[1:])) 1327 1328if __name__ == '__main__': 1329 main() 1330