meta/classes/testimage-auto.bbclass


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

# resulttool - regression analysis
#
# Copyright (c) 2019, Intel Corporation.
# Copyright (c) 2019, Linux Foundation
#
# SPDX-License-Identifier: GPL-2.0-only
#

import resulttool.resultutils as resultutils
import json

from oeqa.utils.git import GitRepo
import oeqa.utils.gitarchive as gitarchive

def compare_result(logger, base_name, target_name, base_result, target_result):
    base_result = base_result.get('result')
    target_result = target_result.get('result')
    result = {}
    if base_result and target_result:
        for k in base_result:
            base_testcase = base_result[k]
            base_status = base_testcase.get('status')
            if base_status:
                target_testcase = target_result.get(k, {})
                target_status = target_testcase.get('status')
                if base_status != target_status:
                    result[k] = {'base': base_status, 'target': target_status}
            else:
                logger.error('Failed to retrieved base test case status: %s' % k)
    if result:
        resultstring = "Regression: %s\n            %s\n" % (base_name, target_name)
        for k in sorted(result):
            resultstring += '    %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
    else:
        resultstring = "Match: %s\n       %s" % (base_name, target_name)
    return result, resultstring

def get_results(logger, source):
    return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)

def regression(args, logger):
    base_results = get_results(logger, args.base_result)
    target_results = get_results(logger, args.target_result)

    regression_common(args, logger, base_results, target_results)

def regression_common(args, logger, base_results, target_results):
    if args.base_result_id:
        base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
    if args.target_result_id:
        target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)

    matches = []
    regressions = []
    notfound = []

    for a in base_results:
        if a in target_results:
            base = list(base_results[a].keys())
            target = list(target_results[a].keys())
            # We may have multiple base/targets which are for different configurations. Start by
            # removing any pairs which match
            for c in base.copy():
                for b in target.copy():
                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
                    if not res:
                        matches.append(resstr)
                        base.remove(c)
                        target.remove(b)
                        break
            # Should only now see regressions, we may not be able to match multiple pairs directly
            for c in base:
                for b in target:
                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
                    if res:
                        regressions.append(resstr)
        else:
            notfound.append("%s not found in target" % a)
    print("\n".join(sorted(matches)))
    print("\n".join(sorted(regressions)))
    print("\n".join(sorted(notfound)))

    return 0

def regression_git(args, logger):
    base_results = {}
    target_results = {}

    tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
    repo = GitRepo(args.repo)

    revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)

    if args.branch2:
        revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
        if not len(revs2):
            logger.error("No revisions found to compare against")
            return 1
        if not len(revs):
            logger.error("No revision to report on found")
            return 1
    else:
        if len(revs) < 2:
            logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
            return 1

    # Pick revisions
    if args.commit:
        if args.commit_number:
            logger.warning("Ignoring --commit-number as --commit was specified")
        index1 = gitarchive.rev_find(revs, 'commit', args.commit)
    elif args.commit_number:
        index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
    else:
        index1 = len(revs) - 1

    if args.branch2:
        revs2.append(revs[index1])
        index1 = len(revs2) - 1
        revs = revs2

    if args.commit2:
        if args.commit_number2:
            logger.warning("Ignoring --commit-number2 as --commit2 was specified")
        index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
    elif args.commit_number2:
        index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
    else:
        if index1 > 0:
            index2 = index1 - 1
            # Find the closest matching commit number for comparision
            # In future we could check the commit is a common ancestor and
            # continue back if not but this good enough for now
            while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
                index2 = index2 - 1
        else:
            logger.error("Unable to determine the other commit, use "
                      "--commit2 or --commit-number2 to specify it")
            return 1

    logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))

    base_results = resultutils.git_get_result(repo, revs[index1][2])
    target_results = resultutils.git_get_result(repo, revs[index2][2])

    regression_common(args, logger, base_results, target_results)

    return 0

def register_commands(subparsers):
    """Register subcommands from this plugin"""

    parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
                                         description='regression analysis comparing the base set of results to the target results',
                                         group='analysis')
    parser_build.set_defaults(func=regression)
    parser_build.add_argument('base_result',
                              help='base result file/directory/URL for the comparison')
    parser_build.add_argument('target_result',
                              help='target result file/directory/URL to compare with')
    parser_build.add_argument('-b', '--base-result-id', default='',
                              help='(optional) filter the base results to this result ID')
    parser_build.add_argument('-t', '--target-result-id', default='',
                              help='(optional) filter the target results to this result ID')

    parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
                                         description='regression analysis comparing base result set to target '
                                                     'result set',
                                         group='analysis')
    parser_build.set_defaults(func=regression_git)
    parser_build.add_argument('repo',
                              help='the git repository containing the data')
    parser_build.add_argument('-b', '--base-result-id', default='',
                              help='(optional) default select regression based on configurations unless base result '
                                   'id was provided')
    parser_build.add_argument('-t', '--target-result-id', default='',
                              help='(optional) default select regression based on configurations unless target result '
                                   'id was provided')

    parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
    parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
    parser_build.add_argument('--commit', help="Revision to search for")
    parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
    parser_build.add_argument('--commit2', help="Revision to compare with")
    parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
Age	Commit message (Collapse)	Author
2018-06-21	image/testimage: Rework auto image test execution	Richard Purdie
	The TEST_IMAGE interface has never particularly worked and image testing currently gets configured manually. This reworks the interface to better serve the needs of its users, replacing it with TESTIMAGE_AUTO. This does away with the need for the separate class due to better bitbake APIs for changing tasks. TESTIMAGE_AUTO will automatically boot under qemu any image that is built. It also adds in dependencies so that any SDK for which testing is requested will automatically be built first. The code in bitbake.conf was error prone (e.g. testsdk wasn't considered), this improves it to standardise on IMAGE_CLASSES as the standard configuration mechanism. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-01-11	classes: Fix do_rootfs references	Richard Purdie
	After the separation of do_rootfs, some rootfs references need changing to image_complete. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2014-03-05	testimage: add task level lock	Stefan Stanacar
	For machines other than qemu it will not be okay to run multiple testimage tasks in parallel. Signed-off-by: Stefan Stanacar <stefanx.stanacar@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-12-03	testimage: use the new targetcontrol.py module for running tests	Stefan Stanacar
	This patch makes the necessary changes for using the targetcontrol.py module so that one can run the same tests on a qemu instance or a remote machine based on the value of TEST_TARGET variable: "qemu" or "simpleremote". The default value is "qemu" which starts a qemu instance and it's the with what we currently have. With "simpleremote", the remote machine must be up with network and ssh and you need to set TEST_TARGET_IP with the IP address of the remote machine (it can still be a qemu instance that was manually started). Basically testimage.bbclass now does something along the lines of: - load tests -> deploy (prepare) / start target -> run tests. There were a couple of changes necessary for tests and also some cleanups/renames that were needed to adjust this change. (use ip everywhere when refering to target and server_ip when refering to host/build machine) Also two unnecessary and unsed methods were dropped from sshcontrol. [ YOCTO #5554 ] Signed-off-by: Stefan Stanacar <stefanx.stanacar@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-09-20	classes/testimage-auto: add class to allow automatically running image tests	Paul Eggleton
	Setting TEST_IMAGE = "1" alone will now automatically run tests on the image immediately after the image is built instead of having to add INHERIT += "testimage" and run bitbake -c testimage <image> manually (but that will still work). This restores functionality that was present in the older imagetest-qemu class with IMAGETEST. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>