diff options
Diffstat (limited to 'scripts/lib/build_perf')
| -rw-r--r-- | scripts/lib/build_perf/__init__.py | 31 | ||||
| -rw-r--r-- | scripts/lib/build_perf/html.py | 19 | ||||
| -rw-r--r-- | scripts/lib/build_perf/html/measurement_chart.html | 50 | ||||
| -rw-r--r-- | scripts/lib/build_perf/html/report.html | 209 | ||||
| -rw-r--r-- | scripts/lib/build_perf/report.py | 342 |
5 files changed, 651 insertions, 0 deletions
diff --git a/scripts/lib/build_perf/__init__.py b/scripts/lib/build_perf/__init__.py new file mode 100644 index 0000000000..1f8b729078 --- /dev/null +++ b/scripts/lib/build_perf/__init__.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance test library functions""" + +def print_table(rows, row_fmt=None): + """Print data table""" + if not rows: + return + if not row_fmt: + row_fmt = ['{:{wid}} '] * len(rows[0]) + + # Go through the data to get maximum cell widths + num_cols = len(row_fmt) + col_widths = [0] * num_cols + for row in rows: + for i, val in enumerate(row): + col_widths[i] = max(col_widths[i], len(str(val))) + + for row in rows: + print(*[row_fmt[i].format(col, wid=col_widths[i]) for i, col in enumerate(row)]) + diff --git a/scripts/lib/build_perf/html.py b/scripts/lib/build_perf/html.py new file mode 100644 index 0000000000..578bb162ee --- /dev/null +++ b/scripts/lib/build_perf/html.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Helper module for HTML reporting""" +from jinja2 import Environment, PackageLoader + + +env = Environment(loader=PackageLoader('build_perf', 'html')) + +template = env.get_template('report.html') diff --git a/scripts/lib/build_perf/html/measurement_chart.html b/scripts/lib/build_perf/html/measurement_chart.html new file mode 100644 index 0000000000..26fe1453c0 --- /dev/null +++ b/scripts/lib/build_perf/html/measurement_chart.html @@ -0,0 +1,50 @@ +<script type="text/javascript"> + google.charts.setOnLoadCallback(drawChart_{{ chart_elem_id }}); + function drawChart_{{ chart_elem_id }}() { + var data = new google.visualization.DataTable(); + + // Chart options + var options = { + theme : 'material', + legend: 'none', + hAxis: { format: '', title: 'Commit number', + minValue: {{ chart_opts.haxis.min }}, + maxValue: {{ chart_opts.haxis.max }} }, + {% if measurement.type == 'time' %} + vAxis: { format: 'h:mm:ss' }, + {% else %} + vAxis: { format: '' }, + {% endif %} + pointSize: 5, + chartArea: { left: 80, right: 15 }, + }; + + // Define data columns + data.addColumn('number', 'Commit'); + data.addColumn('{{ measurement.value_type.gv_data_type }}', + '{{ measurement.value_type.quantity }}'); + // Add data rows + data.addRows([ + {% for sample in measurement.samples %} + [{{ sample.commit_num }}, {{ sample.mean.gv_value() }}], + {% endfor %} + ]); + + // Finally, draw the chart + chart_div = document.getElementById('{{ chart_elem_id }}'); + var chart = new google.visualization.LineChart(chart_div); + google.visualization.events.addListener(chart, 'ready', function () { + //chart_div = document.getElementById('{{ chart_elem_id }}'); + //chart_div.innerHTML = '<img src="' + chart.getImageURI() + '">'; + png_div = document.getElementById('{{ chart_elem_id }}_png'); + png_div.outerHTML = '<a id="{{ chart_elem_id }}_png" href="' + chart.getImageURI() + '">PNG</a>'; + console.log("CHART READY: {{ chart_elem_id }}"); + {% if last_chart == true %} + console.log("ALL CHARTS READY"); + {% endif %} + //console.log(chart_div.innerHTML); + }); + chart.draw(data, options); +} +</script> + diff --git a/scripts/lib/build_perf/html/report.html b/scripts/lib/build_perf/html/report.html new file mode 100644 index 0000000000..e42871177d --- /dev/null +++ b/scripts/lib/build_perf/html/report.html @@ -0,0 +1,209 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +{# Scripts, for visualization#} +<!--START-OF-SCRIPTS--> +<script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script> +<script type="text/javascript"> +google.charts.load('current', {'packages':['corechart']}); +</script> + +{# Render measurement result charts #} +{% for test in test_data %} + {% set test_loop = loop %} + {% if test.status == 'SUCCESS' %} + {% for measurement in test.measurements %} + {% set chart_elem_id = test.name + '_' + measurement.name + '_chart' %} + {% if test_loop.last and loop.last %} + {% set last_chart = true %} + {% endif %} + {% include 'measurement_chart.html' %} + {% endfor %} + {% endif %} +{% endfor %} + +<!--END-OF-SCRIPTS--> + +{# Styles #} +<style> +.meta-table { + font-size: 14px; + text-align: left; + border-collapse: collapse; +} +.meta-table tr:nth-child(even){background-color: #f2f2f2} +meta-table th, .meta-table td { + padding: 4px; +} +.summary { + margin: 0; + font-size: 14px; + text-align: left; + border-collapse: collapse; +} +summary th, .meta-table td { + padding: 4px; +} +.measurement { + padding: 8px 0px 8px 8px; + border: 2px solid #f0f0f0; + margin-bottom: 10px; +} +.details { + margin: 0; + font-size: 12px; + text-align: left; + border-collapse: collapse; +} +.details th { + font-weight: normal; + padding-right: 8px; +} +.preformatted { + font-family: monospace; + white-space: pre-wrap; + background-color: #f0f0f0; + margin-left: 10px; +} +hr { + color: #f0f0f0; +} +h2 { + font-size: 20px; + margin-bottom: 0px; + color: #707070; +} +h3 { + font-size: 16px; + margin: 0px; + color: #707070; +} +</style> + +<title>{{ title }}</title> +</head> + +{% macro poky_link(commit) -%} + <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?id={{ commit }}">{{ commit[0:11] }}</a> +{%- endmacro %} + +<body><div style="width: 700px"> + {# Test metadata #} + <h2>General</h2> + <hr> + <table class="meta-table" style="width: 100%"> + <tr> + <th></th> + <th>Current commit</th> + <th>Comparing with</th> + </tr> + {% for key, item in metadata.items() %} + <tr> + <th>{{ item.title }}</th> + {%if key == 'commit' %} + <td>{{ poky_link(item.value) }}</td> + <td>{{ poky_link(item.value_old) }}</td> + {% else %} + <td>{{ item.value }}</td> + <td>{{ item.value_old }}</td> + {% endif %} + </tr> + {% endfor %} + </table> + + {# Test result summary #} + <h2>Test result summary</h2> + <hr> + <table class="summary" style="width: 100%"> + {% for test in test_data %} + {% if loop.index is even %} + {% set row_style = 'style="background-color: #f2f2f2"' %} + {% else %} + {% set row_style = 'style="background-color: #ffffff"' %} + {% endif %} + <tr {{ row_style }}><td>{{ test.name }}: {{ test.description }}</td> + {% if test.status == 'SUCCESS' %} + {% for measurement in test.measurements %} + {# add empty cell in place of the test name#} + {% if loop.index > 1 %}<td></td>{% endif %} + {% if measurement.absdiff > 0 %} + {% set result_style = "color: red" %} + {% elif measurement.absdiff == measurement.absdiff %} + {% set result_style = "color: green" %} + {% else %} + {% set result_style = "color: orange" %} + {%endif %} + <td>{{ measurement.description }}</td> + <td style="font-weight: bold">{{ measurement.value.mean }}</td> + <td style="{{ result_style }}">{{ measurement.absdiff_str }}</td> + <td style="{{ result_style }}">{{ measurement.reldiff }}</td> + </tr><tr {{ row_style }}> + {% endfor %} + {% else %} + <td style="font-weight: bold; color: red;">{{test.status }}</td> + <td></td> <td></td> <td></td> <td></td> + {% endif %} + </tr> + {% endfor %} + </table> + + {# Detailed test results #} + {% for test in test_data %} + <h2>{{ test.name }}: {{ test.description }}</h2> + <hr> + {% if test.status == 'SUCCESS' %} + {% for measurement in test.measurements %} + <div class="measurement"> + <h3>{{ measurement.description }}</h3> + <div style="font-weight:bold;"> + <span style="font-size: 23px;">{{ measurement.value.mean }}</span> + <span style="font-size: 20px; margin-left: 12px"> + {% if measurement.absdiff > 0 %} + <span style="color: red"> + {% elif measurement.absdiff == measurement.absdiff %} + <span style="color: green"> + {% else %} + <span style="color: orange"> + {% endif %} + {{ measurement.absdiff_str }} ({{measurement.reldiff}}) + </span></span> + </div> + <table style="width: 100%"> + <tr> + <td style="width: 75%"> + {# Linechart #} + <div id="{{ test.name }}_{{ measurement.name }}_chart"></div> + </td> + <td> + {# Measurement statistics #} + <table class="details"> + <tr> + <th>Test runs</th><td>{{ measurement.value.sample_cnt }}</td> + </tr><tr> + <th>-/+</th><td>-{{ measurement.value.minus }} / +{{ measurement.value.plus }}</td> + </tr><tr> + <th>Min</th><td>{{ measurement.value.min }}</td> + </tr><tr> + <th>Max</th><td>{{ measurement.value.max }}</td> + </tr><tr> + <th>Stdev</th><td>{{ measurement.value.stdev }}</td> + </tr><tr> + <th><div id="{{ test.name }}_{{ measurement.name }}_chart_png"></div></th> + </tr> + </table> + </td> + </tr> + </table> + </div> + {% endfor %} + {# Unsuccessful test #} + {% else %} + <span style="font-size: 150%; font-weight: bold; color: red;">{{ test.status }} + {% if test.err_type %}<span style="font-size: 75%; font-weight: normal">({{ test.err_type }})</span>{% endif %} + </span> + <div class="preformatted">{{ test.message }}</div> + {% endif %} + {% endfor %} +</div></body> +</html> + diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py new file mode 100644 index 0000000000..eb00ccca2d --- /dev/null +++ b/scripts/lib/build_perf/report.py @@ -0,0 +1,342 @@ +# +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Handling of build perf test reports""" +from collections import OrderedDict, Mapping +from datetime import datetime, timezone +from numbers import Number +from statistics import mean, stdev, variance + + +def isofmt_to_timestamp(string): + """Convert timestamp string in ISO 8601 format into unix timestamp""" + if '.' in string: + dt = datetime.strptime(string, '%Y-%m-%dT%H:%M:%S.%f') + else: + dt = datetime.strptime(string, '%Y-%m-%dT%H:%M:%S') + return dt.replace(tzinfo=timezone.utc).timestamp() + + +def metadata_xml_to_json(elem): + """Convert metadata xml into JSON format""" + assert elem.tag == 'metadata', "Invalid metadata file format" + + def _xml_to_json(elem): + """Convert xml element to JSON object""" + out = OrderedDict() + for child in elem.getchildren(): + key = child.attrib.get('name', child.tag) + if len(child): + out[key] = _xml_to_json(child) + else: + out[key] = child.text + return out + return _xml_to_json(elem) + + +def results_xml_to_json(elem): + """Convert results xml into JSON format""" + rusage_fields = ('ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', + 'ru_majflt', 'ru_inblock', 'ru_oublock', 'ru_nvcsw', + 'ru_nivcsw') + iostat_fields = ('rchar', 'wchar', 'syscr', 'syscw', 'read_bytes', + 'write_bytes', 'cancelled_write_bytes') + + def _read_measurement(elem): + """Convert measurement to JSON""" + data = OrderedDict() + data['type'] = elem.tag + data['name'] = elem.attrib['name'] + data['legend'] = elem.attrib['legend'] + values = OrderedDict() + + # SYSRES measurement + if elem.tag == 'sysres': + for subel in elem: + if subel.tag == 'time': + values['start_time'] = isofmt_to_timestamp(subel.attrib['timestamp']) + values['elapsed_time'] = float(subel.text) + elif subel.tag == 'rusage': + rusage = OrderedDict() + for field in rusage_fields: + if 'time' in field: + rusage[field] = float(subel.attrib[field]) + else: + rusage[field] = int(subel.attrib[field]) + values['rusage'] = rusage + elif subel.tag == 'iostat': + values['iostat'] = OrderedDict([(f, int(subel.attrib[f])) + for f in iostat_fields]) + elif subel.tag == 'buildstats_file': + values['buildstats_file'] = subel.text + else: + raise TypeError("Unknown sysres value element '{}'".format(subel.tag)) + # DISKUSAGE measurement + elif elem.tag == 'diskusage': + values['size'] = int(elem.find('size').text) + else: + raise Exception("Unknown measurement tag '{}'".format(elem.tag)) + data['values'] = values + return data + + def _read_testcase(elem): + """Convert testcase into JSON""" + assert elem.tag == 'testcase', "Expecting 'testcase' element instead of {}".format(elem.tag) + + data = OrderedDict() + data['name'] = elem.attrib['name'] + data['description'] = elem.attrib['description'] + data['status'] = 'SUCCESS' + data['start_time'] = isofmt_to_timestamp(elem.attrib['timestamp']) + data['elapsed_time'] = float(elem.attrib['time']) + measurements = OrderedDict() + + for subel in elem.getchildren(): + if subel.tag == 'error' or subel.tag == 'failure': + data['status'] = subel.tag.upper() + data['message'] = subel.attrib['message'] + data['err_type'] = subel.attrib['type'] + data['err_output'] = subel.text + elif subel.tag == 'skipped': + data['status'] = 'SKIPPED' + data['message'] = subel.text + else: + measurements[subel.attrib['name']] = _read_measurement(subel) + data['measurements'] = measurements + return data + + def _read_testsuite(elem): + """Convert suite to JSON""" + assert elem.tag == 'testsuite', \ + "Expecting 'testsuite' element instead of {}".format(elem.tag) + + data = OrderedDict() + if 'hostname' in elem.attrib: + data['tester_host'] = elem.attrib['hostname'] + data['start_time'] = isofmt_to_timestamp(elem.attrib['timestamp']) + data['elapsed_time'] = float(elem.attrib['time']) + tests = OrderedDict() + + for case in elem.getchildren(): + tests[case.attrib['name']] = _read_testcase(case) + data['tests'] = tests + return data + + # Main function + assert elem.tag == 'testsuites', "Invalid test report format" + assert len(elem) == 1, "Too many testsuites" + + return _read_testsuite(elem.getchildren()[0]) + + +def aggregate_metadata(metadata): + """Aggregate metadata into one, basically a sanity check""" + mutable_keys = ('pretty_name', 'version_id') + + def aggregate_obj(aggregate, obj, assert_str=True): + """Aggregate objects together""" + assert type(aggregate) is type(obj), \ + "Type mismatch: {} != {}".format(type(aggregate), type(obj)) + if isinstance(obj, Mapping): + assert set(aggregate.keys()) == set(obj.keys()) + for key, val in obj.items(): + aggregate_obj(aggregate[key], val, key not in mutable_keys) + elif isinstance(obj, list): + assert len(aggregate) == len(obj) + for i, val in enumerate(obj): + aggregate_obj(aggregate[i], val) + elif not isinstance(obj, str) or (isinstance(obj, str) and assert_str): + assert aggregate == obj, "Data mismatch {} != {}".format(aggregate, obj) + + if not metadata: + return {} + + # Do the aggregation + aggregate = metadata[0].copy() + for testrun in metadata[1:]: + aggregate_obj(aggregate, testrun) + aggregate['testrun_count'] = len(metadata) + return aggregate + + +def aggregate_data(data): + """Aggregate multiple test results JSON structures into one""" + + mutable_keys = ('status', 'message', 'err_type', 'err_output') + + class SampleList(list): + """Container for numerical samples""" + pass + + def new_aggregate_obj(obj): + """Create new object for aggregate""" + if isinstance(obj, Number): + new_obj = SampleList() + new_obj.append(obj) + elif isinstance(obj, str): + new_obj = obj + else: + # Lists and and dicts are kept as is + new_obj = obj.__class__() + aggregate_obj(new_obj, obj) + return new_obj + + def aggregate_obj(aggregate, obj, assert_str=True): + """Recursive "aggregation" of JSON objects""" + if isinstance(obj, Number): + assert isinstance(aggregate, SampleList) + aggregate.append(obj) + return + + assert type(aggregate) == type(obj), \ + "Type mismatch: {} != {}".format(type(aggregate), type(obj)) + if isinstance(obj, Mapping): + for key, val in obj.items(): + if not key in aggregate: + aggregate[key] = new_aggregate_obj(val) + else: + aggregate_obj(aggregate[key], val, key not in mutable_keys) + elif isinstance(obj, list): + for i, val in enumerate(obj): + if i >= len(aggregate): + aggregate[key] = new_aggregate_obj(val) + else: + aggregate_obj(aggregate[i], val) + elif isinstance(obj, str): + # Sanity check for data + if assert_str: + assert aggregate == obj, "Data mismatch {} != {}".format(aggregate, obj) + else: + raise Exception("BUG: unable to aggregate '{}' ({})".format(type(obj), str(obj))) + + if not data: + return {} + + # Do the aggregation + aggregate = data[0].__class__() + for testrun in data: + aggregate_obj(aggregate, testrun) + return aggregate + + +class MeasurementVal(float): + """Base class representing measurement values""" + gv_data_type = 'number' + + def gv_value(self): + """Value formatting for visualization""" + if self != self: + return "null" + else: + return self + + +class TimeVal(MeasurementVal): + """Class representing time values""" + quantity = 'time' + gv_title = 'elapsed time' + gv_data_type = 'timeofday' + + def hms(self): + """Split time into hours, minutes and seconeds""" + hhh = int(abs(self) / 3600) + mmm = int((abs(self) % 3600) / 60) + sss = abs(self) % 60 + return hhh, mmm, sss + + def __str__(self): + if self != self: + return "nan" + hh, mm, ss = self.hms() + sign = '-' if self < 0 else '' + if hh > 0: + return '{}{:d}:{:02d}:{:02.0f}'.format(sign, hh, mm, ss) + elif mm > 0: + return '{}{:d}:{:04.1f}'.format(sign, mm, ss) + elif ss > 1: + return '{}{:.1f} s'.format(sign, ss) + else: + return '{}{:.2f} s'.format(sign, ss) + + def gv_value(self): + """Value formatting for visualization""" + if self != self: + return "null" + hh, mm, ss = self.hms() + return [hh, mm, int(ss), int(ss*1000) % 1000] + + +class SizeVal(MeasurementVal): + """Class representing time values""" + quantity = 'size' + gv_title = 'size in MiB' + gv_data_type = 'number' + + def __str__(self): + if self != self: + return "nan" + if abs(self) < 1024: + return '{:.1f} kiB'.format(self) + elif abs(self) < 1048576: + return '{:.2f} MiB'.format(self / 1024) + else: + return '{:.2f} GiB'.format(self / 1048576) + + def gv_value(self): + """Value formatting for visualization""" + if self != self: + return "null" + return self / 1024 + +def measurement_stats(meas, prefix=''): + """Get statistics of a measurement""" + if not meas: + return {prefix + 'sample_cnt': 0, + prefix + 'mean': MeasurementVal('nan'), + prefix + 'stdev': MeasurementVal('nan'), + prefix + 'variance': MeasurementVal('nan'), + prefix + 'min': MeasurementVal('nan'), + prefix + 'max': MeasurementVal('nan'), + prefix + 'minus': MeasurementVal('nan'), + prefix + 'plus': MeasurementVal('nan')} + + stats = {'name': meas['name']} + if meas['type'] == 'sysres': + val_cls = TimeVal + values = meas['values']['elapsed_time'] + elif meas['type'] == 'diskusage': + val_cls = SizeVal + values = meas['values']['size'] + else: + raise Exception("Unknown measurement type '{}'".format(meas['type'])) + stats['val_cls'] = val_cls + stats['quantity'] = val_cls.quantity + stats[prefix + 'sample_cnt'] = len(values) + + mean_val = val_cls(mean(values)) + min_val = val_cls(min(values)) + max_val = val_cls(max(values)) + + stats[prefix + 'mean'] = mean_val + if len(values) > 1: + stats[prefix + 'stdev'] = val_cls(stdev(values)) + stats[prefix + 'variance'] = val_cls(variance(values)) + else: + stats[prefix + 'stdev'] = float('nan') + stats[prefix + 'variance'] = float('nan') + stats[prefix + 'min'] = min_val + stats[prefix + 'max'] = max_val + stats[prefix + 'minus'] = val_cls(mean_val - min_val) + stats[prefix + 'plus'] = val_cls(max_val - mean_val) + + return stats + |