summaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/__init__.py
blob: d39c474fbdc96a4f80f4a2de0cc20db242cce5f0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
#!/usr/bin/env python3

# Development tool - utility functions for plugins
#
# Copyright (C) 2014 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
"""Devtool plugins module"""

import os
import sys
import subprocess
import logging
import re
import codecs

logger = logging.getLogger('devtool')

class DevtoolError(Exception):
    """Exception for handling devtool errors"""
    def __init__(self, message, exitcode=1):
        super(DevtoolError, self).__init__(message)
        self.exitcode = exitcode


def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
    """Run a program in bitbake build context"""
    import bb
    if not 'cwd' in options:
        options["cwd"] = builddir
    if init_path:
        # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
        # and can't determine it when running under dash, we need to set
        # the executable to bash to correctly set things up
        if not 'executable' in options:
            options['executable'] = 'bash'
        logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
        init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
    else:
        logger.debug('Executing command "%s"' % cmd)
        init_prefix = ''
    if watch:
        if sys.stdout.isatty():
            # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
            cmd = 'script -e -q -c "%s" /dev/null' % cmd
        return exec_watch('%s%s' % (init_prefix, cmd), **options)
    else:
        return bb.process.run('%s%s' % (init_prefix, cmd), **options)

def exec_watch(cmd, **options):
    """Run program with stdout shown on sys.stdout"""
    import bb
    if isinstance(cmd, str) and not "shell" in options:
        options["shell"] = True

    process = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
    )

    reader = codecs.getreader('utf-8')(process.stdout)
    buf = ''
    while True:
        out = reader.read(1, 1)
        if out:
            sys.stdout.write(out)
            sys.stdout.flush()
            buf += out
        elif out == '' and process.poll() != None:
            break

    if process.returncode != 0:
        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)

    return buf, None

def exec_fakeroot(d, cmd, **kwargs):
    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
    # Grab the command and check it actually exists
    fakerootcmd = d.getVar('FAKEROOTCMD')
    if not os.path.exists(fakerootcmd):
        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
        return 2
    # Set up the appropriate environment
    newenv = dict(os.environ)
    fakerootenv = d.getVar('FAKEROOTENV')
    for varvalue in fakerootenv.split():
        if '=' in varvalue:
            splitval = varvalue.split('=', 1)
            newenv[splitval[0]] = splitval[1]
    return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)

def setup_tinfoil(config_only=False, basepath=None, tracking=False):
    """Initialize tinfoil api from bitbake"""
    import scriptpath
    orig_cwd = os.path.abspath(os.curdir)
    try:
        if basepath:
            os.chdir(basepath)
        bitbakepath = scriptpath.add_bitbake_lib_path()
        if not bitbakepath:
            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
            sys.exit(1)

        import bb.tinfoil
        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
        try:
            tinfoil.logger.setLevel(logger.getEffectiveLevel())
            tinfoil.prepare(config_only)
        except bb.tinfoil.TinfoilUIException:
            tinfoil.shutdown()
            raise DevtoolError('Failed to start bitbake environment')
        except:
            tinfoil.shutdown()
            raise
    finally:
        os.chdir(orig_cwd)
    return tinfoil

def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
    """Parse the specified recipe"""
    try:
        recipefile = tinfoil.get_recipe_file(pn)
    except bb.providers.NoProvider as e:
        logger.error(str(e))
        return None
    if appends:
        append_files = tinfoil.get_file_appends(recipefile)
        if filter_workspace:
            # Filter out appends from the workspace
            append_files = [path for path in append_files if
                            not path.startswith(config.workspace_path)]
    else:
        append_files = None
    try:
        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
    except Exception as e:
        logger.error(str(e))
        return None
    return rd

def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
    """
    Check that a recipe is in the workspace and (optionally) that source
    is present.
    """

    workspacepn = pn

    for recipe, value in workspace.items():
        if recipe == pn:
            break
        if bbclassextend:
            recipefile = value['recipefile']
            if recipefile:
                targets = get_bbclassextend_targets(recipefile, recipe)
                if pn in targets:
                    workspacepn = recipe
                    break
    else:
        raise DevtoolError("No recipe named '%s' in your workspace" % pn)

    if checksrc:
        srctree = workspace[workspacepn]['srctree']
        if not os.path.exists(srctree):
            raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
        if not os.listdir(srctree):
            raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))

    return workspacepn

def use_external_build(same_dir, no_same_dir, d):
    """
    Determine if we should use B!=S (separate build and source directories) or not
    """
    b_is_s = True
    if no_same_dir:
        logger.info('Using separate build directory since --no-same-dir specified')
        b_is_s = False
    elif same_dir:
        logger.info('Using source tree as build directory since --same-dir specified')
    elif bb.data.inherits_class('autotools-brokensep', d):
        logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
    elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
        logger.info('Using source tree as build directory since that would be the default for this recipe')
    else:
        b_is_s = False
    return b_is_s

def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
    """
    Set up the git repository for the source tree
    """
    import bb.process
    import oe.patch
    if not os.path.exists(os.path.join(repodir, '.git')):
        bb.process.run('git init', cwd=repodir)
        bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
        bb.process.run('git add .', cwd=repodir)
        commit_cmd = ['git']
        oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
        commit_cmd += ['commit', '-q']
        stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
        if not stdout:
            commit_cmd.append('--allow-empty')
            commitmsg = "Initial empty commit with no upstream sources"
        elif version:
            commitmsg = "Initial commit from upstream at version %s" % version
        else:
            commitmsg = "Initial commit from upstream"
        commit_cmd += ['-m', commitmsg]
        bb.process.run(commit_cmd, cwd=repodir)

    # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
    excludes = []
    excludefile = os.path.join(repodir, '.git', 'info', 'exclude')
    try:
        with open(excludefile, 'r') as f:
            excludes = f.readlines()
    except FileNotFoundError:
        pass
    if 'singletask.lock\n' not in excludes:
        excludes.append('singletask.lock\n')
    with open(excludefile, 'w') as f:
        for line in excludes:
            f.write(line)

    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
    bb.process.run('git tag -f %s' % basetag, cwd=repodir)

def recipe_to_append(recipefile, config, wildcard=False):
    """
    Convert a recipe file to a bbappend file path within the workspace.
    NOTE: if the bbappend already exists, you should be using
    workspace[args.recipename]['bbappend'] instead of calling this
    function.
    """
    appendname = os.path.splitext(os.path.basename(recipefile))[0]
    if wildcard:
        appendname = re.sub(r'_.*', '_%', appendname)
    appendpath = os.path.join(config.workspace_path, 'appends')
    appendfile = os.path.join(appendpath, appendname + '.bbappend')
    return appendfile

def get_bbclassextend_targets(recipefile, pn):
    """
    Cheap function to get BBCLASSEXTEND and then convert that to the
    list of targets that would result.
    """
    import bb.utils

    values = {}
    def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
        values[varname] = origvalue
        return origvalue, None, 0, True
    with open(recipefile, 'r') as f:
        bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)

    targets = []
    bbclassextend = values.get('BBCLASSEXTEND', '').split()
    if bbclassextend:
        for variant in bbclassextend:
            if variant == 'nativesdk':
                targets.append('%s-%s' % (variant, pn))
            elif variant in ['native', 'cross', 'crosssdk']:
                targets.append('%s-%s' % (pn, variant))
    return targets

def replace_from_file(path, old, new):
    """Replace strings on a file"""

    def read_file(path):
        data = None
        with open(path) as f:
            data = f.read()
        return data

    def write_file(path, data):
        if data is None:
            return
        wdata = data.rstrip() + "\n"
        with open(path, "w") as f:
            f.write(wdata)

    # In case old is None, return immediately
    if old is None:
        return
    try:
        rdata = read_file(path)
    except IOError as e:
        # if file does not exit, just quit, otherwise raise an exception
        if e.errno == errno.ENOENT:
            return
        else:
            raise

    old_contents = rdata.splitlines()
    new_contents = []
    for old_content in old_contents:
        try:
            new_contents.append(old_content.replace(old, new))
        except ValueError:
            pass
    write_file(path, "\n".join(new_contents))


def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
    """ This function will make unlocked-sigs.inc match the recipes in the
    workspace plus any extras we want unlocked. """

    if not fixed_setup:
        # Only need to write this out within the eSDK
        return

    if not extra:
        extra = []

    confdir = os.path.join(basepath, 'conf')
    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')

    # Get current unlocked list if any
    values = {}
    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
        values[varname] = origvalue
        return origvalue, None, 0, True
    if os.path.exists(unlockedsigs):
        with open(unlockedsigs, 'r') as f:
            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))

    # If the new list is different to the current list, write it out
    newunlocked = sorted(list(workspace.keys()) + extra)
    if unlocked != newunlocked:
        bb.utils.mkdirhier(confdir)
        with open(unlockedsigs, 'w') as f:
            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
                    "# This layer was created by the OpenEmbedded devtool" +
                    " utility in order to\n" +
                    "# contain recipes that are unlocked.\n")

            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
            for pn in newunlocked:
                f.write('    ' + pn)
            f.write('"')

def check_prerelease_version(ver, operation):
    if 'pre' in ver or 'rc' in ver:
        logger.warning('Version "%s" looks like a pre-release version. '
                       'If that is the case, in order to ensure that the '
                       'version doesn\'t appear to go backwards when you '
                       'later upgrade to the final release version, it is '
                       'recommmended that instead you use '
                       '<current version>+<pre-release version> e.g. if '
                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
                       'If you prefer not to reset and re-try, you can change '
                       'the version after %s succeeds using "devtool rename" '
                       'with -V/--version.' % (ver, operation))

def check_git_repo_dirty(repodir):
    """Check if a git repository is clean or not"""
    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
    return stdout

def check_git_repo_op(srctree, ignoredirs=None):
    """Check if a git repository is in the middle of a rebase"""
    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
    topleveldir = stdout.strip()
    if ignoredirs and topleveldir in ignoredirs:
        return
    gitdir = os.path.join(topleveldir, '.git')
    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)