aboutsummaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorRobert Yang <liezhi.yang@windriver.com>2014-03-24 13:39:26 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-03-24 10:18:25 +0000
commit0e798d5cbcf585535e19633828dc540a282261fc (patch)
treedab0ec291c1fe011423347aa2728b2b02b976284 /meta
parent312ca574e62cb0ac5fdad4943d7ff9d457caf8e3 (diff)
downloadopenembedded-core-contrib-0e798d5cbcf585535e19633828dc540a282261fc.tar.gz
archiver.bbclass: make it can filter the license
* Filter the license (default: no), the recipe whose license in COPYLEFT_LICENSE_INCLUDE will be included, and in COPYLEFT_LICENSE_EXCLUDE will be excluded. * The user can set the recipe type that would be archived (native, target, and so on), deafult to all. The copyleft_filter.bbclass is come from copyleft_compliance.bbclass, which is used by both copyleft_compliance.bbclass and archiver.bbclass. [YOCTO #5740] Signed-off-by: Robert Yang <liezhi.yang@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/archiver.bbclass27
-rw-r--r--meta/classes/copyleft_compliance.bbclass56
-rw-r--r--meta/classes/copyleft_filter.bbclass62
3 files changed, 88 insertions, 57 deletions
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 7780c71ade..9aa9335b94 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -12,9 +12,23 @@
# 5) The environment data, similar to 'bitbake -e recipe':
# ARCHIVER_MODE[dumpdata] = "1"
# 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1"
+# 7) Whether output the .src.rpm package:
+# ARCHIVER_MODE[srpm] = "1"
+# 8) Filter the license, the recipe whose license in
+# COPYLEFT_LICENSE_INCLUDE will be included, and in
+# COPYLEFT_LICENSE_EXCLUDE will be excluded.
+# COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*'
+# COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary'
+# 9) The recipe type that will be archived:
+# COPYLEFT_RECIPE_TYPES = 'target'
#
-# All of the above can be packed into a .src.rpm package: (when PACKAGES != "")
-# ARCHIVER_MODE[srpm] = "1"
+
+# Don't filter the license by default
+COPYLEFT_LICENSE_INCLUDE ?= ''
+COPYLEFT_LICENSE_EXCLUDE ?= ''
+# Create archive for all the recipe types
+COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian'
+inherit copyleft_filter
ARCHIVER_MODE[srpm] ?= "0"
ARCHIVER_MODE[src] ?= "patched"
@@ -38,6 +52,15 @@ do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}"
python () {
pn = d.getVar('PN', True)
+ if d.getVar('COPYLEFT_LICENSE_INCLUDE', True) or \
+ d.getVar('COPYLEFT_LICENSE_EXCLUDE', True):
+ included, reason = copyleft_should_include(d)
+ if not included:
+ bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason))
+ return
+ else:
+ bb.debug(1, 'archiver: %s is included: %s' % (pn, reason))
+
ar_src = d.getVarFlag('ARCHIVER_MODE', 'src', True)
ar_dumpdata = d.getVarFlag('ARCHIVER_MODE', 'dumpdata', True)
ar_recipe = d.getVarFlag('ARCHIVER_MODE', 'recipe', True)
diff --git a/meta/classes/copyleft_compliance.bbclass b/meta/classes/copyleft_compliance.bbclass
index b47c2c95ad..907c1836b3 100644
--- a/meta/classes/copyleft_compliance.bbclass
+++ b/meta/classes/copyleft_compliance.bbclass
@@ -2,66 +2,12 @@
# Defaults to using symlinks, as it's a quick operation, and one can easily
# follow the links when making use of the files (e.g. tar with the -h arg).
#
-# By default, includes all GPL and LGPL, and excludes CLOSED and Proprietary.
-#
# vi:sts=4:sw=4:et
-COPYLEFT_LICENSE_INCLUDE ?= 'GPL* LGPL*'
-COPYLEFT_LICENSE_INCLUDE[type] = 'list'
-COPYLEFT_LICENSE_INCLUDE[doc] = 'Space separated list of globs which include licenses'
-
-COPYLEFT_LICENSE_EXCLUDE ?= 'CLOSED Proprietary'
-COPYLEFT_LICENSE_EXCLUDE[type] = 'list'
-COPYLEFT_LICENSE_EXCLUDE[doc] = 'Space separated list of globs which exclude licenses'
-
-COPYLEFT_RECIPE_TYPE ?= '${@copyleft_recipe_type(d)}'
-COPYLEFT_RECIPE_TYPE[doc] = 'The "type" of the current recipe (e.g. target, native, cross)'
-
-COPYLEFT_RECIPE_TYPES ?= 'target'
-COPYLEFT_RECIPE_TYPES[type] = 'list'
-COPYLEFT_RECIPE_TYPES[doc] = 'Space separated list of recipe types to include'
-
-COPYLEFT_AVAILABLE_RECIPE_TYPES = 'target native nativesdk cross crosssdk cross-canadian'
-COPYLEFT_AVAILABLE_RECIPE_TYPES[type] = 'list'
-COPYLEFT_AVAILABLE_RECIPE_TYPES[doc] = 'Space separated list of available recipe types'
+inherit copyleft_filter
COPYLEFT_SOURCES_DIR ?= '${DEPLOY_DIR}/copyleft_sources'
-def copyleft_recipe_type(d):
- for recipe_type in oe.data.typed_value('COPYLEFT_AVAILABLE_RECIPE_TYPES', d):
- if oe.utils.inherits(d, recipe_type):
- return recipe_type
- return 'target'
-
-def copyleft_should_include(d):
- """
- Determine if this recipe's sources should be deployed for compliance
- """
- import ast
- import oe.license
- from fnmatch import fnmatchcase as fnmatch
-
- recipe_type = d.getVar('COPYLEFT_RECIPE_TYPE', True)
- if recipe_type not in oe.data.typed_value('COPYLEFT_RECIPE_TYPES', d):
- return False, 'recipe type "%s" is excluded' % recipe_type
-
- include = oe.data.typed_value('COPYLEFT_LICENSE_INCLUDE', d)
- exclude = oe.data.typed_value('COPYLEFT_LICENSE_EXCLUDE', d)
-
- try:
- is_included, reason = oe.license.is_included(d.getVar('LICENSE', True), include, exclude)
- except oe.license.LicenseError as exc:
- bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
- else:
- if is_included:
- if reason:
- return True, 'recipe has included licenses: %s' % ', '.join(reason)
- else:
- return False, 'recipe does not include a copyleft license'
- else:
- return False, 'recipe has excluded licenses: %s' % ', '.join(reason)
-
-
python do_prepare_copyleft_sources () {
"""Populate a tree of the recipe sources and emit patch series files"""
import os.path
diff --git a/meta/classes/copyleft_filter.bbclass b/meta/classes/copyleft_filter.bbclass
new file mode 100644
index 0000000000..2c1d8f1c90
--- /dev/null
+++ b/meta/classes/copyleft_filter.bbclass
@@ -0,0 +1,62 @@
+# Filter the license, the copyleft_should_include returns True for the
+# COPYLEFT_LICENSE_INCLUDE recipe, and False for the
+# COPYLEFT_LICENSE_EXCLUDE.
+#
+# By default, includes all GPL and LGPL, and excludes CLOSED and Proprietary.
+#
+# vi:sts=4:sw=4:et
+
+COPYLEFT_LICENSE_INCLUDE ?= 'GPL* LGPL*'
+COPYLEFT_LICENSE_INCLUDE[type] = 'list'
+COPYLEFT_LICENSE_INCLUDE[doc] = 'Space separated list of globs which include licenses'
+
+COPYLEFT_LICENSE_EXCLUDE ?= 'CLOSED Proprietary'
+COPYLEFT_LICENSE_EXCLUDE[type] = 'list'
+COPYLEFT_LICENSE_EXCLUDE[doc] = 'Space separated list of globs which exclude licenses'
+
+COPYLEFT_RECIPE_TYPE ?= '${@copyleft_recipe_type(d)}'
+COPYLEFT_RECIPE_TYPE[doc] = 'The "type" of the current recipe (e.g. target, native, cross)'
+
+COPYLEFT_RECIPE_TYPES ?= 'target'
+COPYLEFT_RECIPE_TYPES[type] = 'list'
+COPYLEFT_RECIPE_TYPES[doc] = 'Space separated list of recipe types to include'
+
+COPYLEFT_AVAILABLE_RECIPE_TYPES = 'target native nativesdk cross crosssdk cross-canadian'
+COPYLEFT_AVAILABLE_RECIPE_TYPES[type] = 'list'
+COPYLEFT_AVAILABLE_RECIPE_TYPES[doc] = 'Space separated list of available recipe types'
+
+def copyleft_recipe_type(d):
+ for recipe_type in oe.data.typed_value('COPYLEFT_AVAILABLE_RECIPE_TYPES', d):
+ if oe.utils.inherits(d, recipe_type):
+ return recipe_type
+ return 'target'
+
+def copyleft_should_include(d):
+ """
+ Determine if this recipe's sources should be deployed for compliance
+ """
+ import ast
+ import oe.license
+ from fnmatch import fnmatchcase as fnmatch
+
+ recipe_type = d.getVar('COPYLEFT_RECIPE_TYPE', True)
+ if recipe_type not in oe.data.typed_value('COPYLEFT_RECIPE_TYPES', d):
+ return False, 'recipe type "%s" is excluded' % recipe_type
+
+ include = oe.data.typed_value('COPYLEFT_LICENSE_INCLUDE', d)
+ exclude = oe.data.typed_value('COPYLEFT_LICENSE_EXCLUDE', d)
+
+ try:
+ is_included, reason = oe.license.is_included(d.getVar('LICENSE', True), include, exclude)
+ except oe.license.LicenseError as exc:
+ bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
+ else:
+ if is_included:
+ if reason:
+ return True, 'recipe has included licenses: %s' % ', '.join(reason)
+ else:
+ return False, 'recipe does not include a copyleft license'
+ else:
+ return False, 'recipe has excluded licenses: %s' % ', '.join(reason)
+
+
hlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
#!/usr/bin/env python3
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-

# Copyright (c) 2013 Wind River Systems, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import os
import sys
import getopt
import shutil
import re
import warnings
import subprocess
import argparse

scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
lib_path = scripts_path + '/lib'
sys.path = sys.path + [lib_path]

import scriptpath

# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
bitbakepath = scriptpath.add_bitbake_lib_path()
if not bitbakepath:
    sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
    sys.exit(1)
scriptpath.add_oe_lib_path()
import argparse_oe

import bb.siggen
import bb.process

# Match the stamp's filename
# group(1): PE_PV (may no PE)
# group(2): PR
# group(3): TASK
# group(4): HASH
stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)")
sigdata_re = re.compile(".*\.sigdata\..*")

def gen_dict(stamps):
    """
    Generate the dict from the stamps dir.
    The output dict format is:
    {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}}
    Where:
    fake_f: pv + task + hash
    path: the path to the stamp file
    """
    # The member of the sub dict (A "path" will be appended below)
    sub_mem = ("pv", "pr", "task")
    d = {}
    for dirpath, _, files in os.walk(stamps):
        for f in files:
            # The "bitbake -S" would generate ".sigdata", but no "_setscene".
            fake_f = re.sub('_setscene.', '.', f)
            fake_f = re.sub('.sigdata', '', fake_f)
            subdict = {}
            tmp = stamp_re.match(fake_f)
            if tmp:
                for i in sub_mem:
                    subdict[i] = tmp.group(i)
                if len(subdict) != 0:
                    pn = os.path.basename(dirpath)
                    subdict['pn'] = pn
                    # The path will be used by os.stat() and bb.siggen
                    subdict['path'] = dirpath + "/" + f
                    fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash')
                    d[fake_f] = subdict
    return d

# Re-construct the dict
def recon_dict(dict_in):
    """
    The output dict format is:
    {pn_task: {pv: PV, pr: PR, path: PATH}}
    """
    dict_out = {}
    for k in dict_in.keys():
        subdict = {}
        # The key
        pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task'))
        # If more than one stamps are found, use the latest one.
        if pn_task in dict_out:
            full_path_pre = dict_out.get(pn_task).get('path')
            full_path_cur = dict_in.get(k).get('path')
            if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime:
                continue
        subdict['pv'] = dict_in.get(k).get('pv')
        subdict['pr'] = dict_in.get(k).get('pr')
        subdict['path'] = dict_in.get(k).get('path')
        dict_out[pn_task] = subdict

    return dict_out

def split_pntask(s):
    """
    Split the pn_task in to (pn, task) and return it
    """
    tmp = re.match("(.*)_(do_.*)", s)
    return (tmp.group(1), tmp.group(2))


def print_added(d_new = None, d_old = None):
    """
    Print the newly added tasks
    """
    added = {}
    for k in list(d_new.keys()):
        if k not in d_old:
            # Add the new one to added dict, and remove it from
            # d_new, so the remaining ones are the changed ones
            added[k] = d_new.get(k)
            del(d_new[k])

    if not added:
        return 0

    # Format the output, the dict format is:
    # {pn: task1, task2 ...}
    added_format = {}
    counter = 0
    for k in added.keys():
        pn, task = split_pntask(k)
        if pn in added_format:
            # Append the value
            added_format[pn] = "%s %s" % (added_format.get(pn), task)
        else:
            added_format[pn] = task
        counter += 1
    print("=== Newly added tasks: (%s tasks)" % counter)
    for k in added_format.keys():
        print("  %s: %s" % (k, added_format.get(k)))

    return counter

def print_vrchanged(d_new = None, d_old = None, vr = None):
    """
    Print the pv or pr changed tasks.
    The arg "vr" is "pv" or "pr"
    """
    pvchanged = {}
    counter = 0
    for k in list(d_new.keys()):
        if d_new.get(k).get(vr) != d_old.get(k).get(vr):
            counter += 1
            pn, task = split_pntask(k)
            if pn not in pvchanged:
                # Format the output, we only print pn (no task) since
                # all the tasks would be changed when pn or pr changed,
                # the dict format is:
                # {pn: pv/pr_old -> pv/pr_new}
                pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr))
            del(d_new[k])

    if not pvchanged:
        return 0

    print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter))
    for k in pvchanged.keys():
        print("  %s: %s" % (k, pvchanged.get(k)))

    return counter

def print_depchanged(d_new = None, d_old = None, verbose = False):
    """
    Print the dependency changes
    """
    depchanged = {}
    counter = 0
    for k in d_new.keys():
        counter += 1
        pn, task = split_pntask(k)
        if (verbose):
            full_path_old = d_old.get(k).get("path")
            full_path_new = d_new.get(k).get("path")
            # No counter since it is not ready here
            if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new):
                output = bb.siggen.compare_sigfiles(full_path_old, full_path_new)
                if output:
                    print("\n=== The verbose changes of %s.%s:" % (pn, task))
                    print('\n'.join(output))
        else:
            # Format the output, the format is:
            # {pn: task1, task2, ...}
            if pn in depchanged:
                depchanged[pn] = "%s %s" % (depchanged.get(pn), task)
            else:
                depchanged[pn] = task

    if len(depchanged) > 0:
        print("\n=== Dependencies changed: (%s tasks)" % counter)
        for k in depchanged.keys():
            print("  %s: %s" % (k, depchanged[k]))

    return counter


def main():
    """
    Print what will be done between the current and last builds:
    1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps
    2) Figure out what are newly added and changed, can't figure out
       what are removed since we can't know the previous stamps
       clearly, for example, if there are several builds, we can't know
       which stamps the last build has used exactly.
    3) Use bb.siggen.compare_sigfiles to diff the old and new stamps
    """

    parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...]
print what will be done between the current and last builds, for example:

    $ bitbake core-image-sato
    # Edit the recipes
    $ bitbake-whatchanged core-image-sato

The changes will be printed"

Note:
    The amount of tasks is not accurate when the task is "do_build" since
    it usually depends on other tasks.
    The "nostamp" task is not included.
"""
)
    parser.add_argument("recipe", help="recipe to check")
    parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true")
    args = parser.parse_args()

    # Get the STAMPS_DIR
    print("Figuring out the STAMPS_DIR ...")
    cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'"
    try:
        stampsdir, err = bb.process.run(cmdline)
    except:
        raise
    if not stampsdir:
        print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr)
        return 2
    stampsdir = stampsdir.rstrip("\n")
    if not os.path.isdir(stampsdir):
        print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr)
        return 2

    # The new stamps dir
    new_stampsdir = stampsdir + ".bbs"
    if os.path.exists(new_stampsdir):
        print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr)
        return 2

    try:
        # Generate the new stamps dir
        print("Generating the new stamps ... (need several minutes)")
        cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe)
        # FIXME
        # The "bitbake -S" may fail, not fatal error, the stamps will still
        # be generated, this might be a bug of "bitbake -S".
        try:
            bb.process.run(cmdline)
        except Exception as exc:
            print(exc)

        # The dict for the new and old stamps.
        old_dict = gen_dict(stampsdir)
        new_dict = gen_dict(new_stampsdir)

        # Remove the same one from both stamps.
        cnt_unchanged = 0
        for k in list(new_dict.keys()):
            if k in old_dict:
                cnt_unchanged += 1
                del(new_dict[k])
                del(old_dict[k])

        # Re-construct the dict to easily find out what is added or changed.
        # The dict format is:
        # {pn_task: {pv: PV, pr: PR, path: PATH}}
        new_recon = recon_dict(new_dict)
        old_recon = recon_dict(old_dict)

        del new_dict
        del old_dict

        # Figure out what are changed, the new_recon would be changed
        # by the print_xxx function.
        # Newly added
        cnt_added = print_added(new_recon, old_recon)

        # PV (including PE) and PR changed
        # Let the bb.siggen handle them if verbose
        cnt_rv = {}
        if not args.verbose:
            for i in ('pv', 'pr'):
               cnt_rv[i] = print_vrchanged(new_recon, old_recon, i)

        # Dependencies changed (use bitbake-diffsigs)
        cnt_dep = print_depchanged(new_recon, old_recon, args.verbose)

        total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep

        print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged))
        if args.verbose:
            print("Newly added: %s\nDependencies changed: %s\n" % \
                (cnt_added, cnt_dep))
        else:
            print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \
                (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep))
    except:
        print("ERROR occurred!")
        raise
    finally:
        # Remove the newly generated stamps dir
        if os.path.exists(new_stampsdir):
            print("Removing the newly generated stamps dir ...")
            shutil.rmtree(new_stampsdir)

if __name__ == "__main__":
    sys.exit(main())