openembedded-core-contrib - OpenEmbedded Core user contribution trees

Age	Commit message (Expand)	Author
2014-07-16	docbook-xml: add docbook-xsl-stylesheets 1.78.1	Hongxu Jia
SUMMARY = "Updates the NVD CVE database"
LICENSE = "MIT"

INHIBIT_DEFAULT_DEPS = "1"
PACKAGES = ""

inherit nopackages

deltask do_unpack
deltask do_patch
deltask do_configure
deltask do_compile
deltask do_install
deltask do_populate_sysroot

python () {
    if not d.getVar("CVE_CHECK_DB_FILE"):
        raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
}

python do_populate_cve_db() {
    """
    Update NVD database with json data feed
    """

    import sqlite3, urllib, shutil, gzip, re
    from datetime import date

    BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
    YEAR_START = 2002

    db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
    db_file = db_dir + '/nvdcve.db'
    json_tmpfile = db_dir + '/nvd.json.gz'
    proxy = d.getVar("https_proxy")
    cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')

    if not os.path.isdir(db_dir):
        os.mkdir(db_dir)

    # Connect to database
    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    initialize_db(c)

    for year in range(YEAR_START, date.today().year + 1):
        year_url = BASE_URL + str(year)
        meta_url = year_url + ".meta"
        json_url = year_url + ".json.gz"

        # Retrieve meta last modified date
        req = urllib.request.Request(meta_url)
        if proxy:
            req.set_proxy(proxy, 'https')
        try:
            with urllib.request.urlopen(req, timeout=1) as r:
                date_line = str(r.read().splitlines()[0])
                last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
        except:
            cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
            break

        # Compare with current db last modified date
        c.execute("select DATE from META where YEAR = ?", (year,))
        meta = c.fetchone()
        if not meta or meta[0] != last_modified:
            # Update db with current year json file
            req = urllib.request.Request(json_url)
            if proxy:
                req.set_proxy(proxy, 'https')
            try:
                with urllib.request.urlopen(req, timeout=1) as r, \
                     open(json_tmpfile, 'wb') as tmpfile:
                    shutil.copyfileobj(r, tmpfile)
            except:
                cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
                break

            with gzip.open(json_tmpfile, 'rt') as jsonfile:
                update_db(c, jsonfile)
            c.execute("insert or replace into META values (?, ?)",
                    [year, last_modified])

        # Update success, set the date to cve_check file.
        if year == date.today().year:
            cve_f.write('CVE database update : %s\n\n' % date.today())

    cve_f.close()
    conn.commit()
    conn.close()
}

# DJB2 hash algorithm
def hash_djb2(s):
    hash = 5381
    for x in s:
        hash = (( hash << 5) + hash) + ord(x)

    return hash & 0xFFFFFFFF

def initialize_db(c):
    c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
    c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
        SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
    c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
        VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
        VERSION_END TEXT, OPERATOR_END TEXT)")

def insert_elt(c, db_values):
    product_str = db_values[0] + db_values[1] + db_values[2] + db_values[3]
    hashstr = hash_djb2(product_str)
    db_values.insert(0, hashstr)
    query = "insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?, ?, ?)"
    c.execute(query, db_values)

def parse_node_and_insert(c, node, cveId):
    # Parse children node if needed
    try:
        for child in node['children']:
            parse_node_and_insert(c, child, cveId)
    except:
        pass

    # Exit if the cpe_match node does not exists
    try:
        cpe_match = node['cpe_match']
    except:
        return

    for cpe in cpe_match:
        if not cpe['vulnerable']:
            return
        cpe23 = cpe['cpe23Uri'].split(':')
        vendor = cpe23[3]
        product = cpe23[4]
        version = cpe23[5]

        if version != '*':
            # Version is defined, this is a '=' match
            db_values = [cveId, vendor, product, version, '=', '', '']
            insert_elt(c, db_values)
        else:
            # Parse start version, end version and operators
            op_start = ''
            op_end = ''
            v_start = ''
            v_end = ''

            try:
                if cpe['versionStartIncluding']:
                    op_start = '>='
                    v_start = cpe['versionStartIncluding']
            except:
                pass
            try:
                if cpe['versionStartExcluding']:
                    op_start = '>'
                    v_start = cpe['versionStartExcluding']
            except:
                pass
            try:
                if cpe['versionEndIncluding']:
                    op_end = '<='
                    v_end = cpe['versionEndIncluding']
            except:
                pass
            try:
                if cpe['versionEndExcluding']:
                    op_end = '<'
                    v_end = cpe['versionEndExcluding']
            except:
                pass

            db_values = [cveId, vendor, product, v_start, op_start, v_end, op_end]
            insert_elt(c, db_values)

def update_db(c, json_filename):
    import json
    root = json.load(json_filename)

    for elt in root['CVE_Items']:
        if not elt['impact']:
            continue

        cveId = elt['cve']['CVE_data_meta']['ID']
        cveDesc = elt['cve']['description']['description_data'][0]['value']
        date = elt['lastModifiedDate']
        accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
        cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']

        try:
            cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
        except:
            cvssv3 = 0.0

        c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])

        configurations = elt['configurations']['nodes']
        for config in configurations:
            parse_node_and_insert(c, config, cveId)


addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"

EXCLUDE_FROM_WORLD = "1"