1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
SUMMARY = "Updates the NVD CVE database"
LICENSE = "MIT"
INHIBIT_DEFAULT_DEPS = "1"
PACKAGES = ""
inherit nopackages
deltask do_unpack
deltask do_patch
deltask do_configure
deltask do_compile
deltask do_install
deltask do_populate_sysroot
python do_populate_cve_db() {
"""
Update NVD database with json data feed
"""
import sqlite3, urllib, shutil, gzip, re
from datetime import date
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
YEAR_START = 2002
db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
db_file = db_dir + '/nvd-json.db'
json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
if not os.path.isdir(db_dir):
os.mkdir(db_dir)
# Connect to database
conn = sqlite3.connect(db_file)
c = conn.cursor()
initialize_db(c)
for year in range(YEAR_START, date.today().year + 1):
year_url = BASE_URL + str(year)
meta_url = year_url + ".meta"
json_url = year_url + ".json.gz"
# Retrieve meta last modified date
req = urllib.request.Request(meta_url)
if proxy:
req.set_proxy(proxy, 'https')
with urllib.request.urlopen(req) as r:
date_line = str(r.read().splitlines()[0])
last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
# Compare with current db last modified date
c.execute("select DATE from META where YEAR = '%d'" % year)
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Update db with current year json file
req = urllib.request.Request(json_url)
if proxy:
req.set_proxy(proxy, 'https')
with urllib.request.urlopen(req) as r, open(json_tmpfile, 'wb') as tmpfile:
shutil.copyfileobj(r, tmpfile)
with gzip.open(json_tmpfile, 'rt') as jsonfile:
update_db(c, jsonfile)
c.execute("insert or replace into META values (?, ?)",
[year, last_modified])
conn.commit()
conn.close()
cve_check_tmp_file = d.getVar("TMPDIR") + '/cve_check'
with open(cve_check_tmp_file, 'a'):
os.utime(cve_check_tmp_file, None)
}
# DJB2 hash algorithm
def hash_djb2(s):
hash = 5381
for x in s:
hash = (( hash << 5) + hash) + ord(x)
return hash & 0xFFFFFFFF
def initialize_db(c):
c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
(PRODUCT, VERSION)")
def update_db(c, json_filename):
import json
root = json.load(json_filename)
for elt in root['CVE_Items']:
if not elt['impact']:
continue
cveId = elt['cve']['CVE_data_meta']['ID']
cveDesc = elt['cve']['description']['description_data'][0]['value']
date = elt['lastModifiedDate']
accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
try:
cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
except:
cvssv3 = 0.0
c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
[cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
for vendor in elt['cve']['affects']['vendor']['vendor_data']:
for product in vendor['product']['product_data']:
for version in product['version']['version_data']:
product_str = cveId+vendor['vendor_name']+product['product_name']+version['version_value']
hashstr = hash_djb2(product_str)
c.execute("insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?)",
[ hashstr, cveId, vendor['vendor_name'],
product['product_name'], version['version_value'],
version['version_affected']])
addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
|
