aboutsummaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/core/loader.py
blob: 7cc4d4c0cfefafd292f5e195952635d0f5fd982f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
# Copyright (C) 2016 Intel Corporation
# Released under the MIT license (see COPYING.MIT)

import os
import sys
import unittest

from oeqa.core.utils.path import findFile
from oeqa.core.utils.test import getSuiteModules, getCaseID

from oeqa.core.case import OETestCase
from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
        OETestFilter, OETestDiscover

def _make_failed_test(classname, methodname, exception, suiteClass):
    """
        When loading tests, the unittest framework stores any exceptions and
        displays them only when the 'run' method is called.

        For our purposes, it is better to raise the exceptions in the loading
        step rather than waiting to run the test suite.
    """
    raise exception
unittest.loader._make_failed_test = _make_failed_test

def _find_duplicated_modules(suite, directory):
    for module in getSuiteModules(suite):
        path = findFile('%s.py' % module, directory)
        if path:
            raise ImportError("Duplicated %s module found in %s" % (module, path))

def _built_modules_dict(modules):
    modules_dict = {}

    if modules == None:
        return modules_dict

    for m in modules:
        ms = m.split('.')

        if len(ms) == 1:
            module_name = ms[0]
            if not module_name in modules_dict:
                modules_dict[module_name] = {}
        elif len(ms) == 2:
            module_name = ms[0]
            class_name = ms[1]
            if not module_name in modules_dict:
                modules_dict[module_name] = {}
            if not class_name in modules_dict[module_name]:
                modules_dict[module_name][class_name] = []
        elif len(ms) == 3:
            module_name = ms[0]
            class_name = ms[1]
            test_name = ms[2]

            if not module_name in modules_dict:
                modules_dict[module_name] = {}
            if not class_name in modules_dict[module_name]:
                modules_dict[module_name][class_name] = []
            if not test_name in modules_dict[module_name][class_name]:
                modules_dict[module_name][class_name].append(test_name)
        elif len(ms) >= 4:
            module_name = '.'.join(ms[0:-2])
            class_name = ms[-2]
            test_name = ms[-1]

            if not module_name in modules_dict:
                modules_dict[module_name] = {}
            if not class_name in modules_dict[module_name]:
                modules_dict[module_name][class_name] = []
            if not test_name in modules_dict[module_name][class_name]:
                modules_dict[module_name][class_name].append(test_name)

    return modules_dict

class OETestLoader(unittest.TestLoader):
    caseClass = OETestCase

    kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass',
            '_top_level_dir']

    def __init__(self, tc, module_paths, modules, tests, modules_required,
            filters, *args, **kwargs):
        self.tc = tc

        self.modules = _built_modules_dict(modules)

        self.tests = tests
        self.modules_required = modules_required

        self.filters = filters
        self.decorator_filters = [d for d in decoratorClasses if \
                issubclass(d, OETestFilter)]
        self._validateFilters(self.filters, self.decorator_filters)
        self.used_filters = [d for d in self.decorator_filters
                             for f in self.filters
                             if f in d.attrs]

        if isinstance(module_paths, str):
            module_paths = [module_paths]
        elif not isinstance(module_paths, list):
            raise TypeError('module_paths must be a str or a list of str')
        self.module_paths = module_paths

        for kwname in self.kwargs_names:
            if kwname in kwargs:
                setattr(self, kwname, kwargs[kwname])

        self._patchCaseClass(self.caseClass)

        super(OETestLoader, self).__init__()

    def _patchCaseClass(self, testCaseClass):
        # Adds custom attributes to the OETestCase class
        setattr(testCaseClass, 'tc', self.tc)
        setattr(testCaseClass, 'td', self.tc.td)
        setattr(testCaseClass, 'logger', self.tc.logger)

    def _validateFilters(self, filters, decorator_filters):
        # Validate if filter isn't empty
        for key,value in filters.items():
            if not value:
                raise TypeError("Filter %s specified is empty" % key)

        # Validate unique attributes
        attr_filters = [attr for clss in decorator_filters \
                                for attr in clss.attrs]
        dup_attr = [attr for attr in attr_filters
                    if attr_filters.count(attr) > 1]
        if dup_attr:
            raise TypeError('Detected duplicated attribute(s) %s in filter'
                            ' decorators' % ' ,'.join(dup_attr))

        # Validate if filter is supported
        for f in filters:
            if f not in attr_filters:
                classes = ', '.join([d.__name__ for d in decorator_filters])
                raise TypeError('Found "%s" filter but not declared in any of '
                                '%s decorators' % (f, classes))

    def _registerTestCase(self, case):
        case_id = case.id()
        self.tc._registry['cases'][case_id] = case

    def _handleTestCaseDecorators(self, case):
        def _handle(obj):
            if isinstance(obj, OETestDecorator):
                if not obj.__class__ in decoratorClasses:
                    raise Exception("Decorator %s isn't registered" \
                            " in decoratorClasses." % obj.__name__)
                obj.bind(self.tc._registry, case)

        def _walk_closure(obj):
            if hasattr(obj, '__closure__') and obj.__closure__:
                for f in obj.__closure__:
                    obj = f.cell_contents
                    _handle(obj)
                    _walk_closure(obj)
        method = getattr(case, case._testMethodName, None)
        _walk_closure(method)

    def _filterTest(self, case):
        """
            Returns True if test case must be filtered, False otherwise.
        """
        # Filters by module.class.name
        module_name = case.__module__
        class_name = case.__class__.__name__
        test_name = case._testMethodName

        if self.modules:
            if not module_name in self.modules:
                return True

            if self.modules[module_name]:
                if not class_name in self.modules[module_name]:
                    return True

                if self.modules[module_name][class_name]:
                    if test_name not in self.modules[module_name][class_name]:
                        return True

        # Decorator filters
        if self.filters:
            filters = self.filters.copy()
            case_decorators = [cd for cd in case.decorators
                               if cd.__class__ in self.used_filters]

            # Iterate over case decorators to check if needs to be filtered.
            for cd in case_decorators:
                if cd.filtrate(filters):
                    return True

            # Case is missing one or more decorators for all the filters
            # being used, so filter test case.
            if filters:
                return True

        return False

    def _getTestCase(self, testCaseClass, tcName):
        if not hasattr(testCaseClass, '__oeqa_loader'):
            # In order to support data_vars validation
            # monkey patch the default setUp/tearDown{Class} to use
            # the ones provided by OETestCase
            setattr(testCaseClass, 'setUpClassMethod',
                    getattr(testCaseClass, 'setUpClass'))
            setattr(testCaseClass, 'tearDownClassMethod',
                    getattr(testCaseClass, 'tearDownClass'))
            setattr(testCaseClass, 'setUpClass',
                    testCaseClass._oeSetUpClass)
            setattr(testCaseClass, 'tearDownClass',
                    testCaseClass._oeTearDownClass)

            # In order to support decorators initialization
            # monkey patch the default setUp/tearDown to use
            # a setUpDecorators/tearDownDecorators that methods
            # will call setUp/tearDown original methods.
            setattr(testCaseClass, 'setUpMethod',
                    getattr(testCaseClass, 'setUp')) 
            setattr(testCaseClass, 'tearDownMethod',
                    getattr(testCaseClass, 'tearDown'))
            setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp)
            setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown)

            setattr(testCaseClass, '__oeqa_loader', True)

        case = testCaseClass(tcName)
        setattr(case, 'decorators', [])

        return case

    def loadTestsFromTestCase(self, testCaseClass):
        """
            Returns a suite of all tests cases contained in testCaseClass.
        """
        if issubclass(testCaseClass, unittest.suite.TestSuite):
            raise TypeError("Test cases should not be derived from TestSuite." \
                                " Maybe you meant to derive %s from TestCase?" \
                                % testCaseClass.__name__)
        if not issubclass(testCaseClass, self.caseClass):
            raise TypeError("Test %s is not derived from %s" % \
                    (testCaseClass.__name__, self.caseClass.__name__))

        testCaseNames = self.getTestCaseNames(testCaseClass)
        if not testCaseNames and hasattr(testCaseClass, 'runTest'):
            testCaseNames = ['runTest']

        suite = []
        for tcName in testCaseNames:
            case = self._getTestCase(testCaseClass, tcName)
            # Filer by case id
            if not (self.tests and not 'all' in self.tests
                    and not getCaseID(case) in self.tests):
                self._handleTestCaseDecorators(case)

                # Filter by decorators
                if not self._filterTest(case):
                    self._registerTestCase(case)
                    suite.append(case)

        return self.suiteClass(suite)

    def discover(self):
        big_suite = self.suiteClass()
        for path in self.module_paths:
            _find_duplicated_modules(big_suite, path)
            suite = super(OETestLoader, self).discover(path,
                    pattern='*.py', top_level_dir=path)
            big_suite.addTests(suite)

        cases = None
        discover_classes = [clss for clss in decoratorClasses
                            if issubclass(clss, OETestDiscover)]
        for clss in discover_classes:
            cases = clss.discover(self.tc._registry)

        return self.suiteClass(cases) if cases else big_suite

    # XXX After Python 3.5, remove backward compatibility hacks for
    # use_load_tests deprecation via *args and **kws.  See issue 16662.
    if sys.version_info >= (3,5):
        def loadTestsFromModule(self, module, *args, pattern=None, **kws):
            """
                Returns a suite of all tests cases contained in module.
            """
            if module.__name__ in sys.builtin_module_names:
                msg = 'Tried to import %s test module but is a built-in'
                raise ImportError(msg % module.__name__)

            # Normal test modules are loaded if no modules were specified,
            # if module is in the specified module list or if 'all' is in
            # module list.
            # Underscore modules are loaded only if specified in module list.
            load_module = True if not module.__name__.startswith('_') \
                                  and (not self.modules \
                                       or module.__name__ in self.modules \
                                       or 'all' in self.modules) \
                               else False

            load_underscore = True if module.__name__.startswith('_') \
                                      and module.__name__ in self.modules \
                                   else False

            if load_module or load_underscore:
                return super(OETestLoader, self).loadTestsFromModule(
                        module, *args, pattern=pattern, **kws)
            else:
                return self.suiteClass()
    else:
        def loadTestsFromModule(self, module, use_load_tests=True):
            """
                Returns a suite of all tests cases contained in module.
            """
            if module.__name__ in sys.builtin_module_names:
                msg = 'Tried to import %s test module but is a built-in'
                raise ImportError(msg % module.__name__)

            # Normal test modules are loaded if no modules were specified,
            # if module is in the specified module list or if 'all' is in
            # module list.
            # Underscore modules are loaded only if specified in module list.
            load_module = True if not module.__name__.startswith('_') \
                                  and (not self.modules \
                                       or module.__name__ in self.modules \
                                       or 'all' in self.modules) \
                               else False

            load_underscore = True if module.__name__.startswith('_') \
                                      and module.__name__ in self.modules \
                                   else False

            if load_module or load_underscore:
                return super(OETestLoader, self).loadTestsFromModule(
                        module, use_load_tests)
            else:
                return self.suiteClass()