cve-check: backport rewrite from master

As detailed at [1] the XML feeds provided by NIST are being discontinued on
October 9th 2019.  As cve-check-tool uses these feeds, cve-check.bbclass will be
inoperable after this date.

To ensure that cve-check continues working, backport the following commits from
master to move away from the unmaintained cve-check-tool to our own Python code
that fetches the JSON:

546d14135c5 cve-update-db: New recipe to update CVE database
bc144b028f6 cve-check: Remove dependency to cve-check-tool-native
7f62a20b32a cve-check: Manage CVE_PRODUCT with more than one name
3bf63bc6084 cve-check: Consider CVE that affects versions with less than operator
c0eabd30d7b cve-update-db: Use std library instead of urllib3
27eb839ee65 cve-check: be idiomatic
09be21f4d17 cve-update-db: Manage proxy if needed.
975793e3825 cve-update-db: do_populate_cve_db depends on do_fetch
0325dd72714 cve-update-db: Catch request.urlopen errors.
4078da92b49 cve-check: Depends on cve-update-db-native
f7676e9a38d cve-update-db: Use NVD CPE data to populate PRODUCTS table
bc0195be1b1 cve-check: Update unpatched CVE matching
c807c2a6409 cve-update-db-native: Skip recipe when cve-check class is not loaded.
07bb8b25e17 cve-check: remove redundant readline CVE whitelisting
5388ed6d137 cve-check-tool: remove
270ac00cb43 cve-check.bbclass: initialize to_append
e6bf9000987 cve-check: allow comparison of Vendor as well as Product
91770338f76 cve-update-db-native: use SQL placeholders instead of format strings
7069302a4cc cve-check: Replace CVE_CHECK_CVE_WHITELIST by CVE_CHECK_WHITELIST
78de2cb39d7 cve-update-db-native: Remove hash column from database.
4b301030cf9 cve-update-db-native: use os.path.join instead of +
f0d822fad2a cve-update-db: actually inherit native
b309840b6aa cve-update-db-native: use executemany() to optimise CPE insertion
bb4e53af33d cve-update-db-native: improve metadata parsing
94227459792 cve-update-db-native: clean up JSON fetching
95438d52b73 cve-update-db-native: fix https proxy issues
1f9a963b9ff glibc: exclude child recipes from CVE scanning

[1] https://nvd.nist.gov/General/News/XML-Vulnerability-Feed-Retirement

(From OE-Core rev: 8c87e78547c598cada1bce92e7b25d85b994e2eb)

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Armin Kuster <akuster808@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>

4 files changed, 204 insertions, 0 deletions

diff --git a/meta/recipes-core/glibc/glibc-locale.inc b/meta/recipes-core/glibc/glibc-locale.inc
index 1b676dc26e..97d83cb856 100644
--- a/meta/recipes-core/glibc/glibc-locale.inc
+++ b/meta/recipes-core/glibc/glibc-locale.inc
@@ -95,3 +95,6 @@ do_install () {
 inherit libc-package
 
 BBCLASSEXTEND = "nativesdk"
+
+# Don't scan for CVEs as glibc will be scanned
+CVE_PRODUCT = ""
diff --git a/meta/recipes-core/glibc/glibc-mtrace.inc b/meta/recipes-core/glibc/glibc-mtrace.inc
index d703c14bdc..ef9d60ec23 100644
--- a/meta/recipes-core/glibc/glibc-mtrace.inc
+++ b/meta/recipes-core/glibc/glibc-mtrace.inc
@@ -11,3 +11,6 @@ do_install() {
 	install -d -m 0755 ${D}${bindir}
 	install -m 0755 ${SRC}/mtrace ${D}${bindir}/
 }
+
+# Don't scan for CVEs as glibc will be scanned
+CVE_PRODUCT = ""
diff --git a/meta/recipes-core/glibc/glibc-scripts.inc b/meta/recipes-core/glibc/glibc-scripts.inc
index 2a2b41507e..14a14e4512 100644
--- a/meta/recipes-core/glibc/glibc-scripts.inc
+++ b/meta/recipes-core/glibc/glibc-scripts.inc
@@ -18,3 +18,6 @@ do_install() {
 # sotruss script requires sotruss-lib.so (given by libsotruss package), 
 # to produce trace of the library calls.
 RDEPENDS_${PN} += "libsotruss"
+
+# Don't scan for CVEs as glibc will be scanned
+CVE_PRODUCT = ""
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
new file mode 100644
index 0000000000..2c427a5884
--- /dev/null
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -0,0 +1,195 @@
+SUMMARY = "Updates the NVD CVE database"
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+
+inherit native
+
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_populate_sysroot
+
+python () {
+    if not d.getVar("CVE_CHECK_DB_FILE"):
+        raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
+}
+
+python do_populate_cve_db() {
+    """
+    Update NVD database with json data feed
+    """
+
+    import sqlite3, urllib, urllib.parse, shutil, gzip
+    from datetime import date
+
+    BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
+    YEAR_START = 2002
+
+    db_dir = os.path.join(d.getVar("DL_DIR"), 'CVE_CHECK')
+    db_file = os.path.join(db_dir, 'nvdcve_1.0.db')
+    json_tmpfile = os.path.join(db_dir, 'nvd.json.gz')
+    proxy = d.getVar("https_proxy")
+
+    if proxy:
+        # instantiate an opener but do not install it as the global
+        # opener unless if we're really sure it's applicable for all
+        # urllib requests
+        proxy_handler = urllib.request.ProxyHandler({'https': proxy})
+        proxy_opener = urllib.request.build_opener(proxy_handler)
+    else:
+        proxy_opener = None
+
+    cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')
+
+    if not os.path.isdir(db_dir):
+        os.mkdir(db_dir)
+
+    # Connect to database
+    conn = sqlite3.connect(db_file)
+    c = conn.cursor()
+
+    initialize_db(c)
+
+    for year in range(YEAR_START, date.today().year + 1):
+        year_url = BASE_URL + str(year)
+        meta_url = year_url + ".meta"
+        json_url = year_url + ".json.gz"
+
+        # Retrieve meta last modified date
+
+        response = None
+
+        if proxy_opener:
+            response = proxy_opener.open(meta_url)
+        else:
+            req = urllib.request.Request(meta_url)
+            response = urllib.request.urlopen(req)
+
+        if response:
+            for l in response.read().decode("utf-8").splitlines():
+                key, value = l.split(":", 1)
+                if key == "lastModifiedDate":
+                    last_modified = value
+                    break
+            else:
+                bb.warn("Cannot parse CVE metadata, update failed")
+                return
+
+        # Compare with current db last modified date
+        c.execute("select DATE from META where YEAR = ?", (year,))
+        meta = c.fetchone()
+        if not meta or meta[0] != last_modified:
+            # Clear products table entries corresponding to current year
+            c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,))
+
+            # Update db with current year json file
+            try:
+                if proxy_opener:
+                    response = proxy_opener.open(json_url)
+                else:
+                    req = urllib.request.Request(json_url)
+                    response = urllib.request.urlopen(req)
+
+                if response:
+                    update_db(c, gzip.decompress(response.read()).decode('utf-8'))
+                c.execute("insert or replace into META values (?, ?)", [year, last_modified])
+            except urllib.error.URLError as e:
+                cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
+                bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
+                return
+
+        # Update success, set the date to cve_check file.
+        if year == date.today().year:
+            cve_f.write('CVE database update : %s\n\n' % date.today())
+
+    cve_f.close()
+    conn.commit()
+    conn.close()
+}
+
+def initialize_db(c):
+    c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+    c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+        SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
+    c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
+        VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
+        VERSION_END TEXT, OPERATOR_END TEXT)")
+
+def parse_node_and_insert(c, node, cveId):
+    # Parse children node if needed
+    for child in node.get('children', ()):
+        parse_node_and_insert(c, child, cveId)
+
+    def cpe_generator():
+        for cpe in node.get('cpe_match', ()):
+            if not cpe['vulnerable']:
+                return
+            cpe23 = cpe['cpe23Uri'].split(':')
+            vendor = cpe23[3]
+            product = cpe23[4]
+            version = cpe23[5]
+
+            if version != '*':
+                # Version is defined, this is a '=' match
+                yield [cveId, vendor, product, version, '=', '', '']
+            else:
+                # Parse start version, end version and operators
+                op_start = ''
+                op_end = ''
+                v_start = ''
+                v_end = ''
+
+                if 'versionStartIncluding' in cpe:
+                    op_start = '>='
+                    v_start = cpe['versionStartIncluding']
+
+                if 'versionStartExcluding' in cpe:
+                    op_start = '>'
+                    v_start = cpe['versionStartExcluding']
+
+                if 'versionEndIncluding' in cpe:
+                    op_end = '<='
+                    v_end = cpe['versionEndIncluding']
+
+                if 'versionEndExcluding' in cpe:
+                    op_end = '<'
+                    v_end = cpe['versionEndExcluding']
+
+                yield [cveId, vendor, product, v_start, op_start, v_end, op_end]
+
+    c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator())
+
+def update_db(c, jsondata):
+    import json
+    root = json.loads(jsondata)
+
+    for elt in root['CVE_Items']:
+        if not elt['impact']:
+            continue
+
+        cveId = elt['cve']['CVE_data_meta']['ID']
+        cveDesc = elt['cve']['description']['description_data'][0]['value']
+        date = elt['lastModifiedDate']
+        accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
+        cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
+
+        try:
+            cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
+        except:
+            cvssv3 = 0.0
+
+        c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
+                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
+
+        configurations = elt['configurations']['nodes']
+        for config in configurations:
+            parse_node_and_insert(c, config, cveId)
+
+
+addtask do_populate_cve_db before do_fetch
+do_populate_cve_db[nostamp] = "1"
+
+EXCLUDE_FROM_WORLD = "1"