1#SPDX-License-Identifier: GPL-2.0
2import re
3import csv
4import json
5import argparse
6from pathlib import Path
7import subprocess
8
9class Validator:
10    def __init__(self, rulefname, reportfname='', t=5, debug=False, datafname='', fullrulefname='', workload='true', metrics=''):
11        self.rulefname = rulefname
12        self.reportfname = reportfname
13        self.rules = None
14        self.collectlist=metrics
15        self.metrics = set()
16        self.tolerance = t
17
18        self.workloads = [x for x in workload.split(",") if x]
19        self.wlidx = 0 # idx of current workloads
20        self.allresults = dict() # metric results of all workload
21        self.allignoremetrics = dict() # metrics with no results or negative results
22        self.allfailtests = dict()
23        self.alltotalcnt = dict()
24        self.allpassedcnt = dict()
25        self.allerrlist = dict()
26
27        self.results = dict() # metric results of current workload
28        # vars for test pass/failure statistics
29        self.ignoremetrics= set() # metrics with no results or negative results, neg result counts as a failed test
30        self.failtests = dict()
31        self.totalcnt = 0
32        self.passedcnt = 0
33        # vars for errors
34        self.errlist = list()
35
36        # vars for Rule Generator
37        self.pctgmetrics = set() # Percentage rule
38
39        # vars for debug
40        self.datafname = datafname
41        self.debug = debug
42        self.fullrulefname = fullrulefname
43
44    def read_json(self, filename: str) -> dict:
45        try:
46            with open(Path(filename).resolve(), "r") as f:
47                data = json.loads(f.read())
48        except OSError as e:
49            print(f"Error when reading file {e}")
50            sys.exit()
51
52        return data
53
54    def json_dump(self, data, output_file):
55        parent = Path(output_file).parent
56        if not parent.exists():
57            parent.mkdir(parents=True)
58
59        with open(output_file, "w+") as output_file:
60            json.dump(data,
61                      output_file,
62                      ensure_ascii=True,
63                      indent=4)
64
65    def get_results(self, idx:int = 0):
66        return self.results[idx]
67
68    def get_bounds(self, lb, ub, error, alias={}, ridx:int = 0) -> list:
69        """
70        Get bounds and tolerance from lb, ub, and error.
71        If missing lb, use 0.0; missing ub, use float('inf); missing error, use self.tolerance.
72
73        @param lb: str/float, lower bound
74        @param ub: str/float, upper bound
75        @param error: float/str, error tolerance
76        @returns: lower bound, return inf if the lower bound is a metric value and is not collected
77                  upper bound, return -1 if the upper bound is a metric value and is not collected
78                  tolerance, denormalized base on upper bound value
79        """
80        # init ubv and lbv to invalid values
81        def get_bound_value (bound, initval, ridx):
82            val = initval
83            if isinstance(bound, int) or isinstance(bound, float):
84                val = bound
85            elif isinstance(bound, str):
86                if bound == '':
87                    val = float("inf")
88                elif bound in alias:
89                    vall = self.get_value(alias[ub], ridx)
90                    if vall:
91                        val = vall[0]
92                elif bound.replace('.', '1').isdigit():
93                    val = float(bound)
94                else:
95                    print("Wrong bound: {0}".format(bound))
96            else:
97                print("Wrong bound: {0}".format(bound))
98            return val
99
100        ubv = get_bound_value(ub, -1, ridx)
101        lbv = get_bound_value(lb, float('inf'), ridx)
102        t = get_bound_value(error, self.tolerance, ridx)
103
104        # denormalize error threshold
105        denormerr = t * ubv / 100 if ubv != 100 and ubv > 0 else t
106
107        return lbv, ubv, denormerr
108
109    def get_value(self, name:str, ridx:int = 0) -> list:
110        """
111        Get value of the metric from self.results.
112        If result of this metric is not provided, the metric name will be added into self.ignoremetics and self.errlist.
113        All future test(s) on this metric will fail.
114
115        @param name: name of the metric
116        @returns: list with value found in self.results; list is empty when not value found.
117        """
118        results = []
119        data = self.results[ridx] if ridx in self.results else self.results[0]
120        if name not in self.ignoremetrics:
121            if name in data:
122                results.append(data[name])
123            elif name.replace('.', '1').isdigit():
124                results.append(float(name))
125            else:
126                self.errlist.append("Metric '%s' is not collected or the value format is incorrect"%(name))
127                self.ignoremetrics.add(name)
128        return results
129
130    def check_bound(self, val, lb, ub, err):
131        return True if val <= ub + err and val >= lb - err else False
132
133    # Positive Value Sanity check
134    def pos_val_test(self):
135        """
136        Check if metrics value are non-negative.
137        One metric is counted as one test.
138        Failure: when metric value is negative or not provided.
139        Metrics with negative value will be added into the self.failtests['PositiveValueTest'] and self.ignoremetrics.
140        """
141        negmetric = set()
142        missmetric = set()
143        pcnt = 0
144        tcnt = 0
145        for name, val in self.get_results().items():
146            if val is None or val == '':
147                missmetric.add(name)
148                self.errlist.append("Metric '%s' is not collected"%(name))
149            elif val < 0:
150                negmetric.add("{0}(={1:.4f})".format(name, val))
151            else:
152                pcnt += 1
153            tcnt += 1
154
155        self.failtests['PositiveValueTest']['Total Tests'] = tcnt
156        self.failtests['PositiveValueTest']['Passed Tests'] = pcnt
157        if len(negmetric) or len(missmetric)> 0:
158            self.ignoremetrics.update(negmetric)
159            self.ignoremetrics.update(missmetric)
160            self.failtests['PositiveValueTest']['Failed Tests'].append({'NegativeValue':list(negmetric), 'MissingValue':list(missmetric)})
161
162        return
163
164    def evaluate_formula(self, formula:str, alias:dict, ridx:int = 0):
165        """
166        Evaluate the value of formula.
167
168        @param formula: the formula to be evaluated
169        @param alias: the dict has alias to metric name mapping
170        @returns: value of the formula is success; -1 if the one or more metric value not provided
171        """
172        stack = []
173        b = 0
174        errs = []
175        sign = "+"
176        f = str()
177
178        #TODO: support parenthesis?
179        for i in range(len(formula)):
180            if i+1 == len(formula) or formula[i] in ('+', '-', '*', '/'):
181                s = alias[formula[b:i]] if i+1 < len(formula) else alias[formula[b:]]
182                v = self.get_value(s, ridx)
183                if not v:
184                    errs.append(s)
185                else:
186                    f = f + "{0}(={1:.4f})".format(s, v[0])
187                    if sign == "*":
188                        stack[-1] = stack[-1] * v
189                    elif sign == "/":
190                        stack[-1] = stack[-1] / v
191                    elif sign == '-':
192                        stack.append(-v[0])
193                    else:
194                        stack.append(v[0])
195                if i + 1 < len(formula):
196                    sign = formula[i]
197                    f += sign
198                    b = i + 1
199
200        if len(errs) > 0:
201            return -1, "Metric value missing: "+','.join(errs)
202
203        val = sum(stack)
204        return val, f
205
206    # Relationships Tests
207    def relationship_test(self, rule: dict):
208        """
209        Validate if the metrics follow the required relationship in the rule.
210        eg. lower_bound <= eval(formula)<= upper_bound
211        One rule is counted as ont test.
212        Failure: when one or more metric result(s) not provided, or when formula evaluated outside of upper/lower bounds.
213
214        @param rule: dict with metric name(+alias), formula, and required upper and lower bounds.
215        """
216        alias = dict()
217        for m in rule['Metrics']:
218            alias[m['Alias']] = m['Name']
219        lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'], alias, ridx=rule['RuleIndex'])
220        val, f = self.evaluate_formula(rule['Formula'], alias, ridx=rule['RuleIndex'])
221        if val == -1:
222            self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Description':f})
223        elif not self.check_bound(val, lbv, ubv, t):
224            lb = rule['RangeLower']
225            ub = rule['RangeUpper']
226            if isinstance(lb, str):
227                if lb in alias:
228                    lb = alias[lb]
229            if isinstance(ub, str):
230                if ub in alias:
231                    ub = alias[ub]
232            self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Formula':f,
233                                                                       'RangeLower': lb, 'LowerBoundValue': self.get_value(lb),
234                                                                       'RangeUpper': ub, 'UpperBoundValue':self.get_value(ub),
235                                                                       'ErrorThreshold': t, 'CollectedValue': val})
236        else:
237            self.passedcnt += 1
238            self.failtests['RelationshipTest']['Passed Tests'] += 1
239        self.totalcnt += 1
240        self.failtests['RelationshipTest']['Total Tests'] += 1
241
242        return
243
244
245    # Single Metric Test
246    def single_test(self, rule:dict):
247        """
248        Validate if the metrics are in the required value range.
249        eg. lower_bound <= metrics_value <= upper_bound
250        One metric is counted as one test in this type of test.
251        One rule may include one or more metrics.
252        Failure: when the metric value not provided or the value is outside the bounds.
253        This test updates self.total_cnt and records failed tests in self.failtest['SingleMetricTest'].
254
255        @param rule: dict with metrics to validate and the value range requirement
256        """
257        lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'])
258        metrics = rule['Metrics']
259        passcnt = 0
260        totalcnt = 0
261        faillist = []
262        for m in metrics:
263            totalcnt += 1
264            result = self.get_value(m['Name'])
265            if len(result) > 0 and self.check_bound(result[0], lbv, ubv, t):
266                passcnt += 1
267            else:
268                faillist.append({'MetricName':m['Name'], 'CollectedValue':result})
269
270        self.totalcnt += totalcnt
271        self.passedcnt += passcnt
272        self.failtests['SingleMetricTest']['Total Tests'] += totalcnt
273        self.failtests['SingleMetricTest']['Passed Tests'] += passcnt
274        if len(faillist) != 0:
275            self.failtests['SingleMetricTest']['Failed Tests'].append({'RuleIndex':rule['RuleIndex'],
276                                                                       'RangeLower': rule['RangeLower'],
277                                                                       'RangeUpper': rule['RangeUpper'],
278                                                                       'ErrorThreshold':rule['ErrorThreshold'],
279                                                                       'Failure':faillist})
280
281        return
282
283    def create_report(self):
284        """
285        Create final report and write into a JSON file.
286        """
287        alldata = list()
288        for i in range(0, len(self.workloads)):
289            reportstas = {"Total Rule Count": self.alltotalcnt[i], "Passed Rule Count": self.allpassedcnt[i]}
290            data = {"Metric Validation Statistics": reportstas, "Tests in Category": self.allfailtests[i],
291                    "Errors":self.allerrlist[i]}
292            alldata.append({"Workload": self.workloads[i], "Report": data})
293
294        json_str = json.dumps(alldata, indent=4)
295        print("Test validation finished. Final report: ")
296        print(json_str)
297
298        if self.debug:
299            allres = [{"Workload": self.workloads[i], "Results": self.allresults[i]} for i in range(0, len(self.workloads))]
300            self.json_dump(allres, self.datafname)
301
302    def check_rule(self, testtype, metric_list):
303        """
304        Check if the rule uses metric(s) that not exist in current platform.
305
306        @param metric_list: list of metrics from the rule.
307        @return: False when find one metric out in Metric file. (This rule should not skipped.)
308                 True when all metrics used in the rule are found in Metric file.
309        """
310        if testtype == "RelationshipTest":
311            for m in metric_list:
312                if m['Name'] not in self.metrics:
313                    return False
314        return True
315
316    # Start of Collector and Converter
317    def convert(self, data: list, idx: int):
318        """
319        Convert collected metric data from the -j output to dict of {metric_name:value}.
320        """
321        for json_string in data:
322            try:
323                result =json.loads(json_string)
324                if "metric-unit" in result and result["metric-unit"] != "(null)" and result["metric-unit"] != "":
325                    name = result["metric-unit"].split("  ")[1] if len(result["metric-unit"].split("  ")) > 1 \
326                        else result["metric-unit"]
327                    if idx not in self.results: self.results[idx] = dict()
328                    self.results[idx][name.lower()] = float(result["metric-value"])
329            except ValueError as error:
330                continue
331        return
332
333    def collect_perf(self, data_file: str, workload: str):
334        """
335        Collect metric data with "perf stat -M" on given workload with -a and -j.
336        """
337        self.results = dict()
338        tool = 'perf'
339        print(f"Starting perf collection")
340        print(f"Workload: {workload}")
341        collectlist = dict()
342        if self.collectlist != "":
343            collectlist[0] = {x for x in self.collectlist.split(",")}
344        else:
345            collectlist[0] = set(list(self.metrics))
346        # Create metric set for relationship rules
347        for rule in self.rules:
348            if rule["TestType"] == "RelationshipTest":
349                metrics = [m["Name"] for m in rule["Metrics"]]
350                if not any(m not in collectlist[0] for m in metrics):
351                    collectlist[rule["RuleIndex"]] = set(metrics)
352
353        for idx, metrics in collectlist.items():
354            if idx == 0: wl = "sleep 0.5".split()
355            else: wl = workload.split()
356            for metric in metrics:
357                command = [tool, 'stat', '-j', '-M', f"{metric}", "-a"]
358                command.extend(wl)
359                cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8')
360                data = [x+'}' for x in cmd.stderr.split('}\n') if x]
361                self.convert(data, idx)
362    # End of Collector and Converter
363
364    # Start of Rule Generator
365    def parse_perf_metrics(self):
366        """
367        Read and parse perf metric file:
368        1) find metrics with '1%' or '100%' as ScaleUnit for Percent check
369        2) create metric name list
370        """
371        command = ['perf', 'list', '-j', '--details', 'metrics']
372        cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8')
373        try:
374            data = json.loads(cmd.stdout)
375            for m in data:
376                if 'MetricName' not in m:
377                    print("Warning: no metric name")
378                    continue
379                name = m['MetricName']
380                self.metrics.add(name)
381                if 'ScaleUnit' in m and (m['ScaleUnit'] == '1%' or m['ScaleUnit'] == '100%'):
382                    self.pctgmetrics.add(name.lower())
383        except ValueError as error:
384            print(f"Error when parsing metric data")
385            sys.exit()
386
387        return
388
389    def create_rules(self):
390        """
391        Create full rules which includes:
392        1) All the rules from the "relationshi_rules" file
393        2) SingleMetric rule for all the 'percent' metrics
394
395        Reindex all the rules to avoid repeated RuleIndex
396        """
397        self.rules = self.read_json(self.rulefname)['RelationshipRules']
398        pctgrule = {'RuleIndex':0,
399                    'TestType':'SingleMetricTest',
400                    'RangeLower':'0',
401                    'RangeUpper': '100',
402                    'ErrorThreshold': self.tolerance,
403                    'Description':'Metrics in percent unit have value with in [0, 100]',
404                    'Metrics': [{'Name': m} for m in self.pctgmetrics]}
405        self.rules.append(pctgrule)
406
407        # Re-index all rules to avoid repeated RuleIndex
408        idx = 1
409        for r in self.rules:
410            r['RuleIndex'] = idx
411            idx += 1
412
413        if self.debug:
414            #TODO: need to test and generate file name correctly
415            data = {'RelationshipRules':self.rules, 'SupportedMetrics': [{"MetricName": name} for name in self.metrics]}
416            self.json_dump(data, self.fullrulefname)
417
418        return
419    # End of Rule Generator
420
421    def _storewldata(self, key):
422        '''
423        Store all the data of one workload into the corresponding data structure for all workloads.
424        @param key: key to the dictionaries (index of self.workloads).
425        '''
426        self.allresults[key] = self.results
427        self.allignoremetrics[key] = self.ignoremetrics
428        self.allfailtests[key] = self.failtests
429        self.alltotalcnt[key] = self.totalcnt
430        self.allpassedcnt[key] = self.passedcnt
431        self.allerrlist[key] = self.errlist
432
433    #Initialize data structures before data validation of each workload
434    def _init_data(self):
435
436        testtypes = ['PositiveValueTest', 'RelationshipTest', 'SingleMetricTest']
437        self.results = dict()
438        self.ignoremetrics= set()
439        self.errlist = list()
440        self.failtests = {k:{'Total Tests':0, 'Passed Tests':0, 'Failed Tests':[]} for k in testtypes}
441        self.totalcnt = 0
442        self.passedcnt = 0
443
444    def test(self):
445        '''
446        The real entry point of the test framework.
447        This function loads the validation rule JSON file and Standard Metric file to create rules for
448        testing and namemap dictionaries.
449        It also reads in result JSON file for testing.
450
451        In the test process, it passes through each rule and launch correct test function bases on the
452        'TestType' field of the rule.
453
454        The final report is written into a JSON file.
455        '''
456        self.parse_perf_metrics()
457        self.create_rules()
458        for i in range(0, len(self.workloads)):
459            self._init_data()
460            self.collect_perf(self.datafname, self.workloads[i])
461            # Run positive value test
462            self.pos_val_test()
463            for r in self.rules:
464                # skip rules that uses metrics not exist in this platform
465                testtype = r['TestType']
466                if not self.check_rule(testtype, r['Metrics']):
467                    continue
468                if  testtype == 'RelationshipTest':
469                    self.relationship_test(r)
470                elif testtype == 'SingleMetricTest':
471                    self.single_test(r)
472                else:
473                    print("Unsupported Test Type: ", testtype)
474                    self.errlist.append("Unsupported Test Type from rule: " + r['RuleIndex'])
475            self._storewldata(i)
476            print("Workload: ", self.workloads[i])
477            print("Total metrics collected: ", self.failtests['PositiveValueTest']['Total Tests'])
478            print("Non-negative metric count: ", self.failtests['PositiveValueTest']['Passed Tests'])
479            print("Total Test Count: ", self.totalcnt)
480            print("Passed Test Count: ", self.passedcnt)
481
482        self.create_report()
483        return sum(self.alltotalcnt.values()) != sum(self.allpassedcnt.values())
484# End of Class Validator
485
486
487def main() -> None:
488    parser = argparse.ArgumentParser(description="Launch metric value validation")
489
490    parser.add_argument("-rule", help="Base validation rule file", required=True)
491    parser.add_argument("-output_dir", help="Path for validator output file, report file", required=True)
492    parser.add_argument("-debug", help="Debug run, save intermediate data to files", action="store_true", default=False)
493    parser.add_argument("-wl", help="Workload to run while data collection", default="true")
494    parser.add_argument("-m", help="Metric list to validate", default="")
495    args = parser.parse_args()
496    outpath = Path(args.output_dir)
497    reportf = Path.joinpath(outpath, 'perf_report.json')
498    fullrule = Path.joinpath(outpath, 'full_rule.json')
499    datafile = Path.joinpath(outpath, 'perf_data.json')
500
501    validator = Validator(args.rule, reportf, debug=args.debug,
502                        datafname=datafile, fullrulefname=fullrule, workload=args.wl,
503                        metrics=args.m)
504    ret = validator.test()
505
506    return ret
507
508
509if __name__ == "__main__":
510    import sys
511    sys.exit(main())
512
513
514
515