cve-update-db-native: fix https proxy issues

When https_proxy is set, use proxy opener to open CVE metadata and
database URLs, otherwise fallback to the urllib.request.urlopen.

Also fix a minor issue where the json database which has been gzip
decompressed as byte object should be decoded as utf-8 string as
expected by update_db.

Signed-off-by: Chin Huat Ang <chin.huat.ang@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 30 insertions, 11 deletions

diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
index 9c083bdc99..2c427a5884 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -22,7 +22,7 @@ python do_populate_cve_db() {
     Update NVD database with json data feed
     """
 
-    import sqlite3, urllib, shutil, gzip
+    import sqlite3, urllib, urllib.parse, shutil, gzip
     from datetime import date
 
     BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
@@ -32,6 +32,16 @@ python do_populate_cve_db() {
     db_file = os.path.join(db_dir, 'nvdcve_1.0.db')
     json_tmpfile = os.path.join(db_dir, 'nvd.json.gz')
     proxy = d.getVar("https_proxy")
+
+    if proxy:
+        # instantiate an opener but do not install it as the global
+        # opener unless if we're really sure it's applicable for all
+        # urllib requests
+        proxy_handler = urllib.request.ProxyHandler({'https': proxy})
+        proxy_opener = urllib.request.build_opener(proxy_handler)
+    else:
+        proxy_opener = None
+
     cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')
 
     if not os.path.isdir(db_dir):
@@ -49,11 +59,17 @@ python do_populate_cve_db() {
         json_url = year_url + ".json.gz"
 
         # Retrieve meta last modified date
-        req = urllib.request.Request(meta_url)
-        if proxy:
-            req.set_proxy(proxy, 'https')
-        with urllib.request.urlopen(req) as r:
-            for l in r.read().decode("utf-8").splitlines():
+
+        response = None
+
+        if proxy_opener:
+            response = proxy_opener.open(meta_url)
+        else:
+            req = urllib.request.Request(meta_url)
+            response = urllib.request.urlopen(req)
+
+        if response:
+            for l in response.read().decode("utf-8").splitlines():
                 key, value = l.split(":", 1)
                 if key == "lastModifiedDate":
                     last_modified = value
@@ -71,11 +87,14 @@ python do_populate_cve_db() {
 
             # Update db with current year json file
             try:
-                req = urllib.request.Request(json_url)
-                if proxy:
-                    req.set_proxy(proxy, 'https')
-                with urllib.request.urlopen(req) as r:
-                    update_db(c, gzip.decompress(r.read()))
+                if proxy_opener:
+                    response = proxy_opener.open(json_url)
+                else:
+                    req = urllib.request.Request(json_url)
+                    response = urllib.request.urlopen(req)
+
+                if response:
+                    update_db(c, gzip.decompress(response.read()).decode('utf-8'))
                 c.execute("insert or replace into META values (?, ?)", [year, last_modified])
             except urllib.error.URLError as e:
                 cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')