1import lldb
2import json
3from lldbsuite.test.decorators import *
4from lldbsuite.test.lldbtest import *
5from lldbsuite.test import lldbutil
6
7class TestCase(TestBase):
8
9    mydir = TestBase.compute_mydir(__file__)
10
11    def setUp(self):
12        TestBase.setUp(self)
13        self.build()
14
15    NO_DEBUG_INFO_TESTCASE = True
16
17    def test_enable_disable(self):
18        """
19        Test "statistics disable" and "statistics enable". These don't do
20        anything anymore for cheap to gather statistics. In the future if
21        statistics are expensive to gather, we can enable the feature inside
22        of LLDB and test that enabling and disabling stops expesive information
23        from being gathered.
24        """
25        target = self.createTestTarget()
26
27        self.expect("statistics disable", substrs=['need to enable statistics before disabling'], error=True)
28        self.expect("statistics enable")
29        self.expect("statistics enable", substrs=['already enabled'], error=True)
30        self.expect("statistics disable")
31        self.expect("statistics disable", substrs=['need to enable statistics before disabling'], error=True)
32
33    def verify_key_in_dict(self, key, d, description):
34        self.assertEqual(key in d, True,
35            'make sure key "%s" is in dictionary %s' % (key, description))
36
37    def verify_key_not_in_dict(self, key, d, description):
38        self.assertEqual(key in d, False,
39            'make sure key "%s" is in dictionary %s' % (key, description))
40
41    def verify_keys(self, dict, description, keys_exist, keys_missing=None):
42        """
43            Verify that all keys in "keys_exist" list are top level items in
44            "dict", and that all keys in "keys_missing" do not exist as top
45            level items in "dict".
46        """
47        if keys_exist:
48            for key in keys_exist:
49                self.verify_key_in_dict(key, dict, description)
50        if keys_missing:
51            for key in keys_missing:
52                self.verify_key_not_in_dict(key, dict, description)
53
54    def verify_success_fail_count(self, stats, key, num_successes, num_fails):
55        self.verify_key_in_dict(key, stats, 'stats["%s"]' % (key))
56        success_fail_dict = stats[key]
57        self.assertEqual(success_fail_dict['successes'], num_successes,
58                         'make sure success count')
59        self.assertEqual(success_fail_dict['failures'], num_fails,
60                         'make sure success count')
61
62    def get_stats(self, options=None, log_path=None):
63        """
64            Get the output of the "statistics dump" with optional extra options
65            and return the JSON as a python dictionary.
66        """
67        # If log_path is set, open the path and emit the output of the command
68        # for debugging purposes.
69        if log_path is not None:
70            f = open(log_path, 'w')
71        else:
72            f = None
73        return_obj = lldb.SBCommandReturnObject()
74        command = "statistics dump "
75        if options is not None:
76            command += options
77        if f:
78            f.write('(lldb) %s\n' % (command))
79        self.ci.HandleCommand(command, return_obj, False)
80        metrics_json = return_obj.GetOutput()
81        if f:
82            f.write(metrics_json)
83        return json.loads(metrics_json)
84
85
86    def get_target_stats(self, debug_stats):
87        if "targets" in debug_stats:
88            return debug_stats["targets"][0]
89        return None
90
91    def test_expressions_frame_var_counts(self):
92        lldbutil.run_to_source_breakpoint(self, "// break here",
93                                          lldb.SBFileSpec("main.c"))
94
95        self.expect("expr patatino", substrs=['27'])
96        stats = self.get_target_stats(self.get_stats())
97        self.verify_success_fail_count(stats, 'expressionEvaluation', 1, 0)
98        self.expect("expr doesnt_exist", error=True,
99                    substrs=["undeclared identifier 'doesnt_exist'"])
100        # Doesn't successfully execute.
101        self.expect("expr int *i = nullptr; *i", error=True)
102        # Interpret an integer as an array with 3 elements is a failure for
103        # the "expr" command, but the expression evaluation will succeed and
104        # be counted as a success even though the "expr" options will for the
105        # command to fail. It is more important to track expression evaluation
106        # from all sources instead of just through the command, so this was
107        # changed. If we want to track command success and fails, we can do
108        # so using another metric.
109        self.expect("expr -Z 3 -- 1", error=True,
110                    substrs=["expression cannot be used with --element-count"])
111        # We should have gotten 3 new failures and the previous success.
112        stats = self.get_target_stats(self.get_stats())
113        self.verify_success_fail_count(stats, 'expressionEvaluation', 2, 2)
114
115        self.expect("statistics enable")
116        # 'frame var' with enabled statistics will change stats.
117        self.expect("frame var", substrs=['27'])
118        stats = self.get_target_stats(self.get_stats())
119        self.verify_success_fail_count(stats, 'frameVariable', 1, 0)
120
121    def test_default_no_run(self):
122        """Test "statistics dump" without running the target.
123
124        When we don't run the target, we expect to not see any 'firstStopTime'
125        or 'launchOrAttachTime' top level keys that measure the launch or
126        attach of the target.
127
128        Output expected to be something like:
129
130        (lldb) statistics dump
131        {
132          "modules" : [...],
133          "targets" : [
134            {
135                "targetCreateTime": 0.26566899599999999,
136                "expressionEvaluation": {
137                    "failures": 0,
138                    "successes": 0
139                },
140                "frameVariable": {
141                    "failures": 0,
142                    "successes": 0
143                },
144                "moduleIdentifiers": [...],
145            }
146          ],
147          "totalDebugInfoByteSize": 182522234,
148          "totalDebugInfoIndexTime": 2.33343,
149          "totalDebugInfoParseTime": 8.2121400240000071,
150          "totalSymbolTableParseTime": 0.123,
151          "totalSymbolTableIndexTime": 0.234,
152        }
153        """
154        target = self.createTestTarget()
155        debug_stats = self.get_stats()
156        debug_stat_keys = [
157            'modules',
158            'targets',
159            'totalSymbolTableParseTime',
160            'totalSymbolTableIndexTime',
161            'totalDebugInfoByteSize',
162            'totalDebugInfoIndexTime',
163            'totalDebugInfoParseTime',
164        ]
165        self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
166        stats = debug_stats['targets'][0]
167        keys_exist = [
168            'expressionEvaluation',
169            'frameVariable',
170            'moduleIdentifiers',
171            'targetCreateTime',
172        ]
173        keys_missing = [
174            'firstStopTime',
175            'launchOrAttachTime'
176        ]
177        self.verify_keys(stats, '"stats"', keys_exist, keys_missing)
178        self.assertGreater(stats['targetCreateTime'], 0.0)
179
180    def test_default_with_run(self):
181        """Test "statistics dump" when running the target to a breakpoint.
182
183        When we run the target, we expect to see 'launchOrAttachTime' and
184        'firstStopTime' top level keys.
185
186        Output expected to be something like:
187
188        (lldb) statistics dump
189        {
190          "modules" : [...],
191          "targets" : [
192                {
193                    "firstStopTime": 0.34164492800000001,
194                    "launchOrAttachTime": 0.31969605400000001,
195                    "moduleIdentifiers": [...],
196                    "targetCreateTime": 0.0040863039999999998
197                    "expressionEvaluation": {
198                        "failures": 0,
199                        "successes": 0
200                    },
201                    "frameVariable": {
202                        "failures": 0,
203                        "successes": 0
204                    },
205                }
206            ],
207            "totalDebugInfoByteSize": 182522234,
208            "totalDebugInfoIndexTime": 2.33343,
209            "totalDebugInfoParseTime": 8.2121400240000071,
210            "totalSymbolTableParseTime": 0.123,
211            "totalSymbolTableIndexTime": 0.234,
212        }
213
214        """
215        target = self.createTestTarget()
216        lldbutil.run_to_source_breakpoint(self, "// break here",
217                                          lldb.SBFileSpec("main.c"))
218        debug_stats = self.get_stats()
219        debug_stat_keys = [
220            'modules',
221            'targets',
222            'totalSymbolTableParseTime',
223            'totalSymbolTableIndexTime',
224            'totalDebugInfoByteSize',
225            'totalDebugInfoIndexTime',
226            'totalDebugInfoParseTime',
227        ]
228        self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
229        stats = debug_stats['targets'][0]
230        keys_exist = [
231            'expressionEvaluation',
232            'firstStopTime',
233            'frameVariable',
234            'launchOrAttachTime',
235            'moduleIdentifiers',
236            'targetCreateTime',
237        ]
238        self.verify_keys(stats, '"stats"', keys_exist, None)
239        self.assertGreater(stats['firstStopTime'], 0.0)
240        self.assertGreater(stats['launchOrAttachTime'], 0.0)
241        self.assertGreater(stats['targetCreateTime'], 0.0)
242
243    def find_module_in_metrics(self, path, stats):
244        modules = stats['modules']
245        for module in modules:
246            if module['path'] == path:
247                return module
248        return None
249
250    def test_modules(self):
251        """
252            Test "statistics dump" and the module information.
253        """
254        exe = self.getBuildArtifact("a.out")
255        target = self.createTestTarget(file_path=exe)
256        debug_stats = self.get_stats()
257        debug_stat_keys = [
258            'modules',
259            'targets',
260            'totalSymbolTableParseTime',
261            'totalSymbolTableIndexTime',
262            'totalDebugInfoParseTime',
263            'totalDebugInfoIndexTime',
264            'totalDebugInfoByteSize'
265        ]
266        self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
267        stats = debug_stats['targets'][0]
268        keys_exist = [
269            'moduleIdentifiers',
270        ]
271        self.verify_keys(stats, '"stats"', keys_exist, None)
272        exe_module = self.find_module_in_metrics(exe, debug_stats)
273        module_keys = [
274            'debugInfoByteSize',
275            'debugInfoIndexTime',
276            'debugInfoParseTime',
277            'identifier',
278            'path',
279            'symbolTableIndexTime',
280            'symbolTableParseTime',
281            'triple',
282            'uuid',
283        ]
284        self.assertNotEqual(exe_module, None)
285        self.verify_keys(exe_module, 'module dict for "%s"' % (exe), module_keys)
286
287    def test_breakpoints(self):
288        """Test "statistics dump"
289
290        Output expected to be something like:
291
292        {
293          "modules" : [...],
294          "targets" : [
295                {
296                    "firstStopTime": 0.34164492800000001,
297                    "launchOrAttachTime": 0.31969605400000001,
298                    "moduleIdentifiers": [...],
299                    "targetCreateTime": 0.0040863039999999998
300                    "expressionEvaluation": {
301                        "failures": 0,
302                        "successes": 0
303                    },
304                    "frameVariable": {
305                        "failures": 0,
306                        "successes": 0
307                    },
308                    "breakpoints": [
309                        {
310                            "details": {...},
311                            "id": 1,
312                            "resolveTime": 2.65438675
313                        },
314                        {
315                            "details": {...},
316                            "id": 2,
317                            "resolveTime": 4.3632581669999997
318                        }
319                    ]
320                }
321            ],
322            "totalDebugInfoByteSize": 182522234,
323            "totalDebugInfoIndexTime": 2.33343,
324            "totalDebugInfoParseTime": 8.2121400240000071,
325            "totalSymbolTableParseTime": 0.123,
326            "totalSymbolTableIndexTime": 0.234,
327            "totalBreakpointResolveTime": 7.0176449170000001
328        }
329
330        """
331        target = self.createTestTarget()
332        self.runCmd("b main.cpp:7")
333        self.runCmd("b a_function")
334        debug_stats = self.get_stats()
335        debug_stat_keys = [
336            'modules',
337            'targets',
338            'totalSymbolTableParseTime',
339            'totalSymbolTableIndexTime',
340            'totalDebugInfoParseTime',
341            'totalDebugInfoIndexTime',
342            'totalDebugInfoByteSize',
343        ]
344        self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
345        target_stats = debug_stats['targets'][0]
346        keys_exist = [
347            'breakpoints',
348            'expressionEvaluation',
349            'frameVariable',
350            'targetCreateTime',
351            'moduleIdentifiers',
352            'totalBreakpointResolveTime',
353        ]
354        self.verify_keys(target_stats, '"stats"', keys_exist, None)
355        self.assertGreater(target_stats['totalBreakpointResolveTime'], 0.0)
356        breakpoints = target_stats['breakpoints']
357        bp_keys_exist = [
358            'details',
359            'id',
360            'internal',
361            'numLocations',
362            'numResolvedLocations',
363            'resolveTime'
364        ]
365        for breakpoint in breakpoints:
366            self.verify_keys(breakpoint, 'target_stats["breakpoints"]',
367                             bp_keys_exist, None)
368