1from xnu import ( 2 kern, ArgumentError, unsigned, lldb_command, header, GetEnumValue, 3 GetEnumValues, GetEnumName, GetThreadName, GetProcStartAbsTimeForTask, 4 GetRecentTimestamp, GetProcNameForTask, FindTasksByName, IterateQueue) 5 6 7def validate_args(opts, valid_flags): 8 valid_flags = set(valid_flags) 9 for k in opts.keys(): 10 if k[1:] not in valid_flags: 11 raise ArgumentError('-{} not supported in subcommand'.format(k)) 12 13 14@lldb_command('recount', 'AF:MT', fancy=True) 15def Recount(cmd_args=None, cmd_options={}, O=None): # noqa: E741 16 """ Inspect counters maintained by the Recount subsystem on various resource 17 aggregators, like tasks or threads. 18 19 recount task [-TM] <task_t> [...] | -F <task_name> 20 recount thread [-M] <thread_t> [...] 21 recount coalition [-M] <coalition_t> [...] 22 recount processor [-ATM] [<processor_t-or-cpu-id>] [...] 23 24 Options: 25 -T : break out active threads for a task or processor 26 -M : show times in the Mach timebase 27 -A : show all processors 28 29 Diagnostic macros: 30 recount diagnose task <task_t> 31 - Ensure resource accounting consistency in a task. 32 recount triage 33 - Print out statistics useful for general panic triage. 34 35 """ 36 if not cmd_args: 37 raise ArgumentError('subcommand required') 38 39 if cmd_args[0] == 'coalition': 40 validate_args(cmd_options, ['M']) 41 RecountCoalition(cmd_args[1:], cmd_options=cmd_options, O=O) 42 elif cmd_args[0] == 'task': 43 validate_args(cmd_options, ['F', 'M', 'T']) 44 RecountTask(cmd_args[1:], cmd_options=cmd_options, O=O) 45 elif cmd_args[0] == 'thread': 46 validate_args(cmd_options, ['M']) 47 RecountThread(cmd_args[1:], cmd_options=cmd_options, O=O) 48 elif cmd_args[0] == 'processor': 49 validate_args(cmd_options, ['A', 'M', 'T']) 50 RecountProcessor(cmd_args[1:], cmd_options=cmd_options, O=O) 51 elif cmd_args[0] == 'diagnose': 52 RecountDiagnose(cmd_args[1:], cmd_options=cmd_options, O=O) 53 elif cmd_args[0] == 'triage': 54 validate_args(cmd_options, []) 55 RecountTriage(cmd_options=cmd_options, O=O) 56 else: 57 raise ArgumentError('{}: invalid subcommand'.format(cmd_args[0])) 58 59 60def scale_suffix(val, unit=''): 61 si_units = [ 62 (1e21, 'Z'), (1e18, 'E'), (1e15, 'P'), (1e12, 'T'), (1e9, 'B'), 63 (1e6, 'M'), (1e3, 'k'), (1, ' '), (1e-3, 'm'), (1e-6, 'u'), 64 (1e-9, 'n')] 65 scale, sfx = (1, '') 66 for (si_scale, si_sfx) in si_units: 67 if val >= si_scale: 68 scale, sfx = (si_scale, si_sfx) 69 break 70 return '{:>7.3f}{:<1s}{}'.format(val / scale, sfx, unit) 71 72 73class RecountSum(object): 74 """ 75 Accumulate usage counters. 76 """ 77 78 def __init__(self, mach_times=False): 79 self._mach_times = mach_times 80 self._levels = RecountPlan.levels() 81 self._times_mach = [0] * len(self._levels) 82 self._instructions = [0] * len(self._levels) 83 self._cycles = [0] * len(self._levels) 84 self._energy_nj = 0 85 self._valid_count = 0 86 87 def add_usage(self, usage): 88 for (_, level) in self._levels: 89 metrics = usage.ru_metrics[level] 90 self._times_mach[level] += unsigned(metrics.rm_time_mach) 91 if hasattr(metrics, 'rm_cycles'): 92 self._instructions[level] += unsigned(metrics.rm_instructions) 93 self._cycles[level] += unsigned(metrics.rm_cycles) 94 if unsigned(metrics.rm_cycles) != 0: 95 self._valid_count += 1 96 if hasattr(usage, 'ru_energy_nj'): 97 self._energy_nj += unsigned(usage.ru_energy_nj) 98 99 def user_sys_times(self): 100 user_level = GetEnumValue('recount_level_t', 'RCT_LVL_USER') 101 user_time = self._times_mach[user_level] 102 return (user_time, sum(self._times_mach) - user_time) 103 104 def div_valid(self, numer, denom): 105 if self._valid_count == 0 or denom == 0: 106 return 0 107 return numer / denom 108 109 def _convert_time(self, time): 110 if self._mach_times: 111 return time 112 return kern.GetNanotimeFromAbstime(time) / 1e9 113 114 def time(self): 115 time = sum(self._times_mach) 116 if self._mach_times: 117 return time 118 return kern.GetNanotimeFromAbstime(time) 119 120 def fmt_args(self): 121 level_args = [[ 122 level_name, 123 self._convert_time(self._times_mach[level]), 124 scale_suffix(self._cycles[level]), 125 self.div_valid( 126 self._cycles[level], 127 kern.GetNanotimeFromAbstime(self._times_mach[level])), 128 scale_suffix(self._instructions[level]), 129 self.div_valid(self._cycles[level], self._instructions[level]), 130 '-', 131 '-'] for (level_name, level) in 132 RecountPlan.levels()] 133 134 total_time_ns = kern.GetNanotimeFromAbstime(sum(self._times_mach)) 135 total_cycles = sum(self._cycles) 136 total_insns = sum(self._instructions) 137 power_w = self._energy_nj / total_time_ns if total_time_ns != 0 else 0 138 level_args.append([ 139 '*', 140 total_time_ns / 1e9, scale_suffix(total_cycles), 141 self.div_valid(total_cycles, total_time_ns), 142 scale_suffix(total_insns), 143 self.div_valid(total_cycles, total_insns), 144 scale_suffix(self._energy_nj / 1e9, 'J'), 145 scale_suffix(power_w, 'W')]) 146 return level_args 147 148 def fmt_basic_args(self): 149 return [[ 150 level_name, 151 self._convert_time(self._times_mach[level]), 152 self._cycles[level], 153 self._instructions[level], 154 '-'] for (level_name, level) in 155 RecountPlan.levels()] 156 157 158class RecountPlan(object): 159 """ 160 Format tracks and usage according to a plan. 161 """ 162 163 def __init__(self, name, mach_times=False): 164 self._mach_times = mach_times 165 self._group_names = [] 166 self._group_column = None 167 168 plan = kern.GetGlobalVariable('recount_' + name + '_plan') 169 topo = plan.rpl_topo 170 if topo == GetEnumValue('recount_topo_t', 'RCT_TOPO_CPU'): 171 self._group_column = 'cpu' 172 self._group_count = unsigned(kern.globals.real_ncpus) 173 self._group_names = [ 174 'cpu-{}'.format(i) for i in range(self._group_count)] 175 elif topo == GetEnumValue('recount_topo_t', 'RCT_TOPO_CPU_KIND'): 176 if kern.arch.startswith('arm64'): 177 self._group_column = 'cpu-kind' 178 cluster_mask = int(kern.globals.topology_info.cluster_types) 179 self._group_count = bin(cluster_mask).count('1') 180 self._group_names = [ 181 GetEnumName('recount_cpu_kind_t', i)[8:][:4] 182 for i in range(self._group_count)] 183 else: 184 self._group_count = 1 185 elif topo == GetEnumValue('recount_topo_t', 'RCT_TOPO_SYSTEM'): 186 self._group_count = 1 187 else: 188 raise RuntimeError('{}: Unexpected recount topography', topo) 189 190 def time_fmt(self): 191 return '{:>12d}' if self._mach_times else '{:>12.05f}' 192 193 def _usage_fmt(self): 194 prefix = '{n}{{:>6s}} {t} '.format( 195 t=self.time_fmt(), n='{:>8s} ' if self._group_column else '') 196 return prefix + '{:>8s} {:>7.3g} {:>8s} {:>5.03f} {:>9s} {:>9s}' 197 198 def usages(self, usages): 199 for i in range(self._group_count): 200 yield usages[i] 201 202 def track_usages(self, tracks): 203 for i in range(self._group_count): 204 yield tracks[i].rt_usage 205 206 def usage_header(self): 207 fmt = '{:>6s} {:>12s} {:>8s} {:>7s} {:>8s} {:>5s} {:>9s} {:>9s}'.format( # noqa: E501 208 'level', 'time', 'cycles', 'GHz', 'insns', 209 'CPI', 'energy', 'power',) 210 if self._group_column: 211 fmt = '{:>8s} '.format(self._group_column) + fmt 212 return fmt 213 214 def levels(): 215 names = ['kernel', 'user'] 216 levels = list(zip(names, GetEnumValues('recount_level_t', [ 217 'RCT_LVL_' + name.upper() for name in names]))) 218 try: 219 levels.append(('secure', 220 GetEnumValue('recount_level_t', 'RCT_LVL_SECURE'))) 221 except KeyError: 222 # RCT_LVL_SECURE is not defined on this system. 223 pass 224 return levels 225 226 def format_usage(self, usage, name=None, sum=None, O=None): 227 rows = [] 228 229 levels = RecountPlan.levels() 230 total_time = 0 231 total_time_ns = 0 232 total_cycles = 0 233 total_insns = 0 234 for (level_name, level) in levels: 235 metrics = usage.ru_metrics[level] 236 time = unsigned(metrics.rm_time_mach) 237 time_ns = kern.GetNanotimeFromAbstime(time) 238 total_time_ns += time_ns 239 if not self._mach_times: 240 time = time_ns / 1e9 241 total_time += time 242 if hasattr(metrics, 'rm_cycles'): 243 cycles = unsigned(metrics.rm_cycles) 244 total_cycles += cycles 245 freq = cycles / time_ns if time_ns != 0 else 0 246 insns = unsigned(metrics.rm_instructions) 247 total_insns += insns 248 cpi = cycles / insns if insns != 0 else 0 249 else: 250 cycles = 0 251 freq = 0 252 insns = 0 253 cpi = 0 254 rows.append([ 255 level_name, time, scale_suffix(cycles), freq, 256 scale_suffix(insns), cpi, '-', '-']) 257 258 if hasattr(usage, 'ru_energy_nj'): 259 energy_nj = unsigned(usage.ru_energy_nj) 260 if total_time_ns != 0: 261 power_w = energy_nj / total_time_ns 262 else: 263 power_w = 0 264 else: 265 energy_nj = 0 266 power_w = 0 267 if total_insns != 0: 268 total_freq = total_cycles / total_time_ns if total_time_ns != 0 else 0 269 total_cpi = total_cycles / total_insns 270 else: 271 total_freq = 0 272 total_cpi = 0 273 274 rows.append([ 275 '*', total_time, scale_suffix(total_cycles), total_freq, 276 scale_suffix(total_insns), total_cpi, 277 scale_suffix(energy_nj / 1e9, 'J'), 278 scale_suffix(power_w, 'W')]) 279 280 if sum: 281 sum.add_usage(usage) 282 283 if self._group_column: 284 for row in rows: 285 row.insert(0, name) 286 287 return [O.format(self._usage_fmt(), *row) for row in rows] 288 289 def format_sum(self, sum, O=None): 290 lines = [] 291 for line in sum.fmt_args(): 292 lines.append(O.format(self._usage_fmt(), '*', *line)) 293 return lines 294 295 def format_usages(self, usages, O=None): # noqa: E741 296 sum = RecountSum(self._mach_times) if self._group_count > 1 else None 297 str = '' 298 for (i, usage) in enumerate(self.usages(usages)): 299 name = self._group_names[i] if i < len(self._group_names) else None 300 lines = self.format_usage(usage, name=name, sum=sum, O=O) 301 str += '\n'.join(lines) + '\n' 302 if sum: 303 str += '\n'.join(self.format_sum(sum, O=O)) 304 return str 305 306 def format_tracks(self, tracks, O=None): # noqa: E741 307 sum = RecountSum(self._mach_times) if self._group_count > 1 else None 308 str = '' 309 for (i, usage) in enumerate(self.track_usages(tracks)): 310 name = self._group_names[i] if i < len(self._group_names) else None 311 lines = self.format_usage(usage, name=name, sum=sum, O=O) 312 str += '\n'.join(lines) + '\n' 313 if sum: 314 str += '\n'.join(self.format_sum(sum, O=O)) 315 return str 316 317 def sum_usages(self, usages, sum=None): 318 if sum is None: 319 sum = RecountSum(mach_times=self._mach_times) 320 for usage in self.usages(usages): 321 sum.add_usage(usage) 322 return sum 323 324 def sum_tracks(self, tracks, sum=None): 325 if sum is None: 326 sum = RecountSum(mach_times=self._mach_times) 327 for usage in self.track_usages(tracks): 328 sum.add_usage(usage) 329 return sum 330 331 332def GetTaskTerminatedUserSysTime(task): 333 plan = RecountPlan('task_terminated') 334 sum = RecountSum() 335 for usage in plan.usages(task.tk_recount.rtk_terminated): 336 sum.add_usage(usage) 337 return sum.user_sys_times() 338 339 340def GetThreadUserSysTime(thread): 341 plan = RecountPlan('thread') 342 sum = RecountSum() 343 for usage in plan.track_usages(thread.th_recount.rth_lifetime): 344 sum.add_usage(usage) 345 return sum.user_sys_times() 346 347 348def print_threads(plan, thread_ptrs, indent=False, O=None): # noqa: E741 349 for thread_ptr in thread_ptrs: 350 thread = kern.GetValueFromAddress(thread_ptr, 'thread_t') 351 print('{}thread 0x{:x} 0x{:x} {}'.format( 352 ' ' if indent else '', unsigned(thread.thread_id), 353 unsigned(thread), GetThreadName(thread))) 354 with O.table(plan.usage_header(), indent=indent): 355 print(plan.format_tracks(thread.th_recount.rth_lifetime, O=O)) 356 357 358def RecountThread( 359 thread_ptrs, cmd_options={}, indent=False, O=None): # noqa: E741 360 plan = RecountPlan('thread', mach_times='-M' in cmd_options) 361 print_threads(plan, thread_ptrs, indent=indent, O=O) 362 363 364def get_task_age_ns(task): 365 start_abs = GetProcStartAbsTimeForTask(task) 366 if start_abs is not None: 367 return kern.GetNanotimeFromAbstime(GetRecentTimestamp() - start_abs) 368 return None 369 370 371def print_task_description(task): 372 task_name = GetProcNameForTask(task) 373 task_age_ns = get_task_age_ns(task) 374 if task_age_ns is not None: 375 duration_desc = '{:.3f}s'.format(task_age_ns / 1e9) 376 else: 377 duration_desc = '-s' 378 print('task 0x{:x} {} ({} old)'.format( 379 unsigned(task), task_name, duration_desc)) 380 return task_name 381 382 383def RecountTask(task_ptrs, cmd_options={}, O=None): # noqa: E741 384 if '-F' in cmd_options: 385 tasks = FindTasksByName(cmd_options['-F']) 386 else: 387 tasks = [kern.GetValueFromAddress(t, 'task_t') for t in task_ptrs] 388 mach_times = '-M' in cmd_options 389 plan = RecountPlan('task', mach_times=mach_times) 390 terminated_plan = RecountPlan('task_terminated', mach_times=mach_times) 391 active_threads = '-T' in cmd_options 392 if active_threads: 393 thread_plan = RecountPlan('thread', mach_times=mach_times) 394 for task in tasks: 395 task_name = print_task_description(task) 396 with O.table(plan.usage_header()): 397 print(plan.format_tracks(task.tk_recount.rtk_lifetime, O=O)) 398 if active_threads: 399 threads = [unsigned(t) for t in IterateQueue( 400 task.threads, 'thread *', 'task_threads')] 401 print_threads(thread_plan, threads, indent=True, O=O) 402 print('task (terminated threads) 0x{:x} {}'.format( 403 unsigned(task), task_name)) 404 with O.table(terminated_plan.usage_header()): 405 print(terminated_plan.format_usages( 406 task.tk_recount.rtk_terminated, O=O)) 407 408 409def RecountCoalition(coal_ptrs, cmd_options={}, O=None): # noqa: E741 410 plan = RecountPlan('coalition', mach_times='-M' in cmd_options) 411 coals = [kern.GetValueFromAddress(c, 'coalition_t') for c in coal_ptrs] 412 for coal in coals: 413 print('coalition 0x{:x} {}'.format(unsigned(coal), unsigned(coal.id))) 414 with O.table(plan.usage_header()): 415 print(plan.format_usages(coal.r.co_recount.rco_exited, O=O)) 416 417 418def get_processor(ptr_or_id): 419 ptr_or_id = unsigned(ptr_or_id) 420 if ptr_or_id < 1024: 421 processor_list = kern.GetGlobalVariable('processor_list') 422 current_processor = processor_list 423 while unsigned(current_processor) > 0: 424 if unsigned(current_processor.cpu_id) == ptr_or_id: 425 return current_processor 426 current_processor = current_processor.processor_list 427 raise ArgumentError('no processor found with CPU ID {}'.format( 428 ptr_or_id)) 429 else: 430 return kern.GetValueFromAddress(ptr_or_id, 'processor_t') 431 432 433def get_all_processors(): 434 processors = [] 435 processor_list = kern.GetGlobalVariable('processor_list') 436 current_processor = processor_list 437 while unsigned(current_processor) > 0: 438 processors.append(current_processor) 439 current_processor = current_processor.processor_list 440 return sorted(processors, key=lambda p: p.cpu_id) 441 442 443def RecountProcessor(pr_ptrs_or_ids, cmd_options={}, O=None): # noqa: E741 444 mach_times = '-M' in cmd_options 445 plan = RecountPlan('processor', mach_times=mach_times) 446 if '-A' in cmd_options: 447 prs = get_all_processors() 448 else: 449 prs = [get_processor(p) for p in pr_ptrs_or_ids] 450 active_threads = '-T' in cmd_options 451 if active_threads: 452 thread_plan = RecountPlan('thread', mach_times=mach_times) 453 hdr_prefix = '{:>18s} {:>4s} {:>4s} '.format('processor', 'cpu', 'kind',) 454 header_fmt = ' {:>12s} {:>12s} {:>8s}' 455 hdr_suffix = header_fmt.format('idle-time', 'total-time', 'idle-pct') 456 null_suffix = header_fmt.format('-', '-', '-') 457 levels = RecountPlan.levels() 458 with O.table(hdr_prefix + plan.usage_header() + hdr_suffix): 459 for pr in prs: 460 usage = pr.pr_recount.rpr_active.rt_usage 461 idle_time = pr.pr_recount.rpr_idle_time_mach 462 times = [usage.ru_metrics[i].rm_time_mach for (_, i) in levels] 463 total_time = sum(times) + idle_time 464 if not mach_times: 465 idle_time = kern.GetNanotimeFromAbstime(idle_time) / 1e9 466 total_time = kern.GetNanotimeFromAbstime(total_time) / 1e9 467 pset = pr.processor_set 468 cluster_kind = 'SMP' 469 if unsigned(pset.pset_cluster_type) != 0: 470 cluster_kind = GetEnumName('pset_cluster_type_t', 471 pset.pset_cluster_type, 'PSET_AMP_') 472 prefix = '{:<#018x} {:>4d} {:>4s} '.format( 473 unsigned(pr), pr.cpu_id, cluster_kind) 474 suffix = ( 475 ' ' + plan.time_fmt().format(idle_time) + ' ' + 476 plan.time_fmt().format(total_time) + 477 ' {:>7.2f}%'.format(idle_time / total_time * 100)) 478 usage_lines = plan.format_usage(usage, O=O) 479 for (i, line) in enumerate(usage_lines): 480 line_suffix = null_suffix 481 if i + 1 == len(usage_lines): 482 line_suffix = suffix 483 O.write(prefix + line + line_suffix + '\n') 484 if active_threads: 485 active_thread = unsigned(pr.active_thread) 486 if active_thread != 0: 487 print_threads( 488 thread_plan, [active_thread], indent=True, O=O) 489 490 491@header('{:>4s} {:>20s} {:>20s} {:>20s}'.format( 492 'cpu', 'time-mach', 'cycles', 'insns')) 493def GetRecountSnapshot(cpu, snap, O=None): 494 (insns, cycles) = (0, 0) 495 if hasattr(snap, 'rsn_cycles'): 496 (insns, cycles) = (snap.rsn_insns, snap.rsn_cycles) 497 return O.format( 498 '{:4d} {:20d} {:20d} {:20d}', cpu, snap.rsn_time_mach, 499 cycles, insns) 500 501 502def GetRecountProcessorState(pr): 503 state_time = pr.pr_recount.rpr_state_last_abs_time 504 state = state_time >> 63 505 return ( 506 pr.pr_recount.rpr_snap, 507 'I' if state == 1 else 'A', 508 state_time & ~(0x1 << 63)) 509 510 511@header('{:>20s} {:>4s} {:>6s} {:>18s} {:>18s} {:>18s} {:>18s} {:>18s}'.format( 512 'processor', 'cpu', 'state', 'last-idle-change', 'last-user-change', 513 'last-disp', 'since-idle-change', 'since-user-change')) 514def GetRecountProcessorDiagnostics(pr, cur_time, O=None): 515 (snap, state, time) = GetRecountProcessorState(pr) 516 cpu_id = unsigned(pr.cpu_id) 517 last_usrchg = snap.rsn_time_mach 518 since_usrchg = cur_time - last_usrchg 519 last_disp = '{}{:>d}'.format( 520 '*' if cur_time == unsigned(pr.last_dispatch) else '', 521 pr.last_dispatch) 522 return O.format( 523 '{:>#20x} {:4d} {:>6s} {:>18d} {:>18d} {:>18s} {:>18d} {:>18d}', 524 unsigned(pr), cpu_id, state, time, last_usrchg, last_disp, 525 cur_time - time, since_usrchg) 526 527 528@header('{:>12s} {:>6s} {:>12s} {:>20s} {:>20s}'.format( 529 'group', 'level', 'time', 'cycles', 'insns')) 530def RecountDiagnoseTask(task_ptrs, cmd_options={}, O=None): # noqa: E74 531 if '-F' in cmd_options: 532 tasks = FindTasksByName(cmd_options['-F']) 533 else: 534 tasks = [kern.GetValueFromAddress(t, 'task_t') for t in task_ptrs] 535 536 line_fmt = '{:20s} = {:10.3f}' 537 row_fmt = '{:>12s} {:>6s} {:>12.3f} {:>20d} {:>20d}' 538 539 task_plan = RecountPlan('task', mach_times=False) 540 term_plan = RecountPlan('task_terminated', mach_times=False) 541 for task in tasks: 542 print_task_description(task) 543 with O.table(RecountDiagnoseTask.header): 544 task_sum = task_plan.sum_tracks(task.tk_recount.rtk_lifetime) 545 for line in task_sum.fmt_basic_args(): 546 line = line[:-1] 547 print(O.format(row_fmt, 'task', *line)) 548 549 term_sum = term_plan.sum_usages(task.tk_recount.rtk_terminated) 550 for line in term_sum.fmt_basic_args(): 551 print(O.format(row_fmt, 'terminated', *line)) 552 term_sum_ns = term_sum.time() 553 554 threads_sum = RecountSum(mach_times=True) 555 threads_time_mach = threads_sum.time() 556 for thread in IterateQueue( 557 task.threads, 'thread *', 'task_threads'): 558 usr_time, sys_time = GetThreadUserSysTime(thread) 559 threads_time_mach += usr_time + sys_time 560 561 threads_sum_ns = kern.GetNanotimeFromAbstime(threads_time_mach) 562 print(line_fmt.format('threads CPU', threads_sum_ns / 1e9)) 563 564 all_threads_sum_ns = threads_sum_ns + term_sum_ns 565 print(line_fmt.format('all threads CPU', all_threads_sum_ns / 1e9)) 566 567 print(line_fmt.format( 568 'discrepancy', task_sum.time() - all_threads_sum_ns)) 569 570 571def RecountDiagnose(cmd_args=[], cmd_options={}, O=None): # noqa: E741 572 if not cmd_args: 573 raise ArgumentError('diagnose subcommand required') 574 575 if cmd_args[0] == 'task': 576 validate_args(cmd_options, ['F']) 577 RecountDiagnoseTask(cmd_args[1:], cmd_options=cmd_options, O=O) 578 else: 579 raise ArgumentError('{}: invalid diagnose subcommand'.format( 580 cmd_args[0])) 581 582 583def RecountTriage(cmd_options={}, O=None): # noqa: E741 584 prs = get_all_processors() 585 print('processors') 586 with O.table(GetRecountProcessorDiagnostics.header, indent=True): 587 max_dispatch = max([unsigned(pr.last_dispatch) for pr in prs]) 588 for pr in prs: 589 print(GetRecountProcessorDiagnostics( 590 pr, cur_time=max_dispatch, O=O)) 591 592 print('snapshots') 593 with O.table(GetRecountSnapshot.header, indent=True): 594 for (i, pr) in enumerate(prs): 595 print(GetRecountSnapshot(i, pr.pr_recount.rpr_snap, O=O)) 596