aboutsummaryrefslogtreecommitdiffstats
path: root/meta/recipes-core/meta/cve-update-db-native.bb
blob: b3dc33734d6bbd3442bb241db49aff0bc338886d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
SUMMARY = "Updates the NVD CVE database"
LICENSE = "MIT"

INHIBIT_DEFAULT_DEPS = "1"

inherit native

deltask do_unpack
deltask do_patch
deltask do_configure
deltask do_compile
deltask do_install
deltask do_populate_sysroot

python () {
    if not bb.data.inherits_class("cve-check", d):
        raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
}

python do_fetch() {
    """
    Update NVD database with json data feed
    """
    import bb.utils
    import bb.progress
    import sqlite3, urllib, urllib.parse, gzip
    from datetime import date

    bb.utils.export_proxies(d)

    BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-"
    YEAR_START = 2002

    db_file = d.getVar("CVE_CHECK_DB_FILE")
    db_dir = os.path.dirname(db_file)

    if os.path.exists("{0}-journal".format(db_file)):
        # If a journal is present the last update might have been interrupted. In that case,
        # just wipe any leftovers and force the DB to be recreated.
        os.remove("{0}-journal".format(db_file))

        if os.path.exists(db_file):
            os.remove(db_file)

    # Don't refresh the database more than once an hour
    try:
        import time
        if time.time() - os.path.getmtime(db_file) < (60*60):
            bb.debug(2, "Recently updated, skipping")
            return
    except OSError:
        pass

    bb.utils.mkdirhier(db_dir)

    # Connect to database
    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    initialize_db(c)

    with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:
        total_years = date.today().year + 1 - YEAR_START
        for i, year in enumerate(range(YEAR_START, date.today().year + 1)):
            bb.debug(2, "Updating %d" % year)
            ph.update((float(i + 1) / total_years) * 100)
            year_url = BASE_URL + str(year)
            meta_url = year_url + ".meta"
            json_url = year_url + ".json.gz"

            # Retrieve meta last modified date
            try:
                response = urllib.request.urlopen(meta_url)
            except urllib.error.URLError as e:
                cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n')
                bb.warn("Failed to fetch CVE data (%s)" % e.reason)
                return

            if response:
                for l in response.read().decode("utf-8").splitlines():
                    key, value = l.split(":", 1)
                    if key == "lastModifiedDate":
                        last_modified = value
                        break
                else:
                    bb.warn("Cannot parse CVE metadata, update failed")
                    return

            # Compare with current db last modified date
            c.execute("select DATE from META where YEAR = ?", (year,))
            meta = c.fetchone()
            if not meta or meta[0] != last_modified:
                bb.debug(2, "Updating entries")
                # Clear products table entries corresponding to current year
                c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,))

                # Update db with current year json file
                try:
                    response = urllib.request.urlopen(json_url)
                    if response:
                        update_db(c, gzip.decompress(response.read()).decode('utf-8'))
                    c.execute("insert or replace into META values (?, ?)", [year, last_modified])
                except urllib.error.URLError as e:
                    cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
                    bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
                    return
            else:
                bb.debug(2, "Already up to date (last modified %s)" % last_modified)
            # Update success, set the date to cve_check file.
            if year == date.today().year:
                cve_f.write('CVE database update : %s\n\n' % date.today())

        conn.commit()
        conn.close()
}

do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
do_fetch[file-checksums] = ""
do_fetch[vardeps] = ""

def initialize_db(c):
    c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")

    c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
        SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")

    c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
        VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
        VERSION_END TEXT, OPERATOR_END TEXT)")
    c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")

def parse_node_and_insert(c, node, cveId):
    # Parse children node if needed
    for child in node.get('children', ()):
        parse_node_and_insert(c, child, cveId)

    def cpe_generator():
        for cpe in node.get('cpe_match', ()):
            if not cpe['vulnerable']:
                return
            cpe23 = cpe['cpe23Uri'].split(':')
            vendor = cpe23[3]
            product = cpe23[4]
            version = cpe23[5]

            if cpe23[6] == '*' or cpe23[6] == '-':
                version_suffix = ""
            else:
                version_suffix = "_" + cpe23[6]

            if version != '*' and version != '-':
                # Version is defined, this is a '=' match
                yield [cveId, vendor, product, version + version_suffix, '=', '', '']
            elif version == '-':
                # no version information is available
                yield [cveId, vendor, product, version, '', '', '']
            else:
                # Parse start version, end version and operators
                op_start = ''
                op_end = ''
                v_start = ''
                v_end = ''

                if 'versionStartIncluding' in cpe:
                    op_start = '>='
                    v_start = cpe['versionStartIncluding']

                if 'versionStartExcluding' in cpe:
                    op_start = '>'
                    v_start = cpe['versionStartExcluding']

                if 'versionEndIncluding' in cpe:
                    op_end = '<='
                    v_end = cpe['versionEndIncluding']

                if 'versionEndExcluding' in cpe:
                    op_end = '<'
                    v_end = cpe['versionEndExcluding']

                if op_start or op_end or v_start or v_end:
                    yield [cveId, vendor, product, v_start, op_start, v_end, op_end]
                else:
                    # This is no version information, expressed differently.
                    # Save processing by representing as -.
                    yield [cveId, vendor, product, '-', '', '', '']

    c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator())

def update_db(c, jsondata):
    import json
    root = json.loads(jsondata)

    for elt in root['CVE_Items']:
        if not elt['impact']:
            continue

        accessVector = None
        cveId = elt['cve']['CVE_data_meta']['ID']
        cveDesc = elt['cve']['description']['description_data'][0]['value']
        date = elt['lastModifiedDate']
        try:
            accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
            cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
        except KeyError:
            cvssv2 = 0.0
        try:
            accessVector = accessVector or elt['impact']['baseMetricV3']['cvssV3']['attackVector']
            cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
        except KeyError:
            accessVector = accessVector or "UNKNOWN"
            cvssv3 = 0.0

        c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])

        configurations = elt['configurations']['nodes']
        for config in configurations:
            parse_node_and_insert(c, config, cveId)


do_fetch[nostamp] = "1"

EXCLUDE_FROM_WORLD = "1"