[RFC,CFH,sumo,11/47] cve-check: Depends on cve-update-db-native

Submitted by Mikko Rapeli on Nov. 6, 2019, 3:37 p.m. | Patch ID: 166649

Details

Message ID da44064b3ba33798636de84080dacf24913f856f.1573047194.git.mikko.rapeli@bmw.de
State New
Headers show

Commit Message

Mikko Rapeli Nov. 6, 2019, 3:37 p.m.
From: Pierre Le Magourou <pierre.lemagourou@softbankrobotics.com>

do_populate_cve_db is a native task.

(From OE-Core rev: 4078da92b49946848cddebe1735f301af161e162)

Signed-off-by: Pierre Le Magourou <pierre.lemagourou@softbankrobotics.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

Conflicts:
	meta/conf/distro/include/maintainers.inc
---
 meta/classes/cve-check.bbclass                 |   2 +-
 meta/conf/distro/include/maintainers.inc       |   2 +
 meta/recipes-core/meta/cve-update-db-native.bb | 143 +++++++++++++++++++++++++
 meta/recipes-core/meta/cve-update-db.bb        | 143 -------------------------
 4 files changed, 146 insertions(+), 144 deletions(-)
 create mode 100644 meta/recipes-core/meta/cve-update-db-native.bb
 delete mode 100644 meta/recipes-core/meta/cve-update-db.bb

Patch hide | download patch | download mbox

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 81071e3..6ffa0c4 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -63,7 +63,7 @@  python do_cve_check () {
 }
 
 addtask cve_check after do_unpack before do_build
-do_cve_check[depends] = "cve-update-db:do_populate_cve_db"
+do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db"
 do_cve_check[nostamp] = "1"
 
 python cve_check_cleanup () {
diff --git a/meta/conf/distro/include/maintainers.inc b/meta/conf/distro/include/maintainers.inc
index a299177..ee46a98 100644
--- a/meta/conf/distro/include/maintainers.inc
+++ b/meta/conf/distro/include/maintainers.inc
@@ -122,7 +122,9 @@  RECIPE_MAINTAINER_pn-cryptodev-module = "Robert Yang <liezhi.yang@windriver.com>
 RECIPE_MAINTAINER_pn-cryptodev-tests = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-cups = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-curl = "Armin Kuster <akuster@mvista.com>"
+RECIPE_MAINTAINER_pn-cve-check-tool = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-cve-update-db = "Ross Burton <ross.burton@intel.com>"
+RECIPE_MAINTAINER_pn-cve-update-db-native = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-cwautomacros = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-damageproto = "Armin Kuster <akuster@mvista.com>"
 RECIPE_MAINTAINER_pn-db = "Mark Hatle <mark.hatle@windriver.com>"
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
new file mode 100644
index 0000000..ae8f1a9
--- /dev/null
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -0,0 +1,143 @@ 
+SUMMARY = "Updates the NVD CVE database"
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+PACKAGES = ""
+
+inherit nopackages
+
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_populate_sysroot
+
+python do_populate_cve_db() {
+    """
+    Update NVD database with json data feed
+    """
+
+    import sqlite3, urllib, shutil, gzip, re
+    from datetime import date
+
+    BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
+    YEAR_START = 2002
+
+    db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
+    db_file = db_dir + '/nvd-json.db'
+    json_tmpfile = db_dir + '/nvd.json.gz'
+    proxy = d.getVar("https_proxy")
+    cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
+
+    if not os.path.isdir(db_dir):
+        os.mkdir(db_dir)
+
+    # Connect to database
+    conn = sqlite3.connect(db_file)
+    c = conn.cursor()
+
+    initialize_db(c)
+
+    for year in range(YEAR_START, date.today().year + 1):
+        year_url = BASE_URL + str(year)
+        meta_url = year_url + ".meta"
+        json_url = year_url + ".json.gz"
+
+        # Retrieve meta last modified date
+        req = urllib.request.Request(meta_url)
+        if proxy:
+            req.set_proxy(proxy, 'https')
+        try:
+            with urllib.request.urlopen(req, timeout=1) as r:
+                date_line = str(r.read().splitlines()[0])
+                last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
+        except:
+            cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
+            break
+
+        # Compare with current db last modified date
+        c.execute("select DATE from META where YEAR = '%d'" % year)
+        meta = c.fetchone()
+        if not meta or meta[0] != last_modified:
+            # Update db with current year json file
+            req = urllib.request.Request(json_url)
+            if proxy:
+                req.set_proxy(proxy, 'https')
+            try:
+                with urllib.request.urlopen(req, timeout=1) as r, \
+                     open(json_tmpfile, 'wb') as tmpfile:
+                    shutil.copyfileobj(r, tmpfile)
+            except:
+                cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
+                break
+
+            with gzip.open(json_tmpfile, 'rt') as jsonfile:
+                update_db(c, jsonfile)
+            c.execute("insert or replace into META values (?, ?)",
+                    [year, last_modified])
+
+        # Update success, set the date to cve_check file.
+        if year == date.today().year:
+            cve_f.write('CVE database update : %s\n\n' % date.today())
+
+    cve_f.close()
+    conn.commit()
+    conn.close()
+}
+
+# DJB2 hash algorithm
+def hash_djb2(s):
+    hash = 5381
+    for x in s:
+        hash = (( hash << 5) + hash) + ord(x)
+
+    return hash & 0xFFFFFFFF
+
+def initialize_db(c):
+    c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+    c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+        SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
+    c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
+        VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
+    c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
+        (PRODUCT, VERSION)")
+
+def update_db(c, json_filename):
+    import json
+    root = json.load(json_filename)
+
+    for elt in root['CVE_Items']:
+        if not elt['impact']:
+            continue
+
+        cveId = elt['cve']['CVE_data_meta']['ID']
+        cveDesc = elt['cve']['description']['description_data'][0]['value']
+        date = elt['lastModifiedDate']
+        accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
+        cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
+
+        try:
+            cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
+        except:
+            cvssv3 = 0.0
+
+        c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
+                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
+
+        for vendor in elt['cve']['affects']['vendor']['vendor_data']:
+            for product in vendor['product']['product_data']:
+                for version in product['version']['version_data']:
+                    product_str = cveId+vendor['vendor_name']+product['product_name']+version['version_value']
+                    hashstr = hash_djb2(product_str)
+                    c.execute("insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?)",
+                            [ hashstr, cveId, vendor['vendor_name'],
+                                product['product_name'], version['version_value'],
+                                version['version_affected']])
+
+
+
+addtask do_populate_cve_db before do_fetch
+do_populate_cve_db[nostamp] = "1"
+
+EXCLUDE_FROM_WORLD = "1"
diff --git a/meta/recipes-core/meta/cve-update-db.bb b/meta/recipes-core/meta/cve-update-db.bb
deleted file mode 100644
index ae8f1a9..0000000
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ /dev/null
@@ -1,143 +0,0 @@ 
-SUMMARY = "Updates the NVD CVE database"
-LICENSE = "MIT"
-
-INHIBIT_DEFAULT_DEPS = "1"
-PACKAGES = ""
-
-inherit nopackages
-
-deltask do_unpack
-deltask do_patch
-deltask do_configure
-deltask do_compile
-deltask do_install
-deltask do_populate_sysroot
-
-python do_populate_cve_db() {
-    """
-    Update NVD database with json data feed
-    """
-
-    import sqlite3, urllib, shutil, gzip, re
-    from datetime import date
-
-    BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
-    YEAR_START = 2002
-
-    db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
-    db_file = db_dir + '/nvd-json.db'
-    json_tmpfile = db_dir + '/nvd.json.gz'
-    proxy = d.getVar("https_proxy")
-    cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
-
-    if not os.path.isdir(db_dir):
-        os.mkdir(db_dir)
-
-    # Connect to database
-    conn = sqlite3.connect(db_file)
-    c = conn.cursor()
-
-    initialize_db(c)
-
-    for year in range(YEAR_START, date.today().year + 1):
-        year_url = BASE_URL + str(year)
-        meta_url = year_url + ".meta"
-        json_url = year_url + ".json.gz"
-
-        # Retrieve meta last modified date
-        req = urllib.request.Request(meta_url)
-        if proxy:
-            req.set_proxy(proxy, 'https')
-        try:
-            with urllib.request.urlopen(req, timeout=1) as r:
-                date_line = str(r.read().splitlines()[0])
-                last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
-        except:
-            cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
-            break
-
-        # Compare with current db last modified date
-        c.execute("select DATE from META where YEAR = '%d'" % year)
-        meta = c.fetchone()
-        if not meta or meta[0] != last_modified:
-            # Update db with current year json file
-            req = urllib.request.Request(json_url)
-            if proxy:
-                req.set_proxy(proxy, 'https')
-            try:
-                with urllib.request.urlopen(req, timeout=1) as r, \
-                     open(json_tmpfile, 'wb') as tmpfile:
-                    shutil.copyfileobj(r, tmpfile)
-            except:
-                cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
-                break
-
-            with gzip.open(json_tmpfile, 'rt') as jsonfile:
-                update_db(c, jsonfile)
-            c.execute("insert or replace into META values (?, ?)",
-                    [year, last_modified])
-
-        # Update success, set the date to cve_check file.
-        if year == date.today().year:
-            cve_f.write('CVE database update : %s\n\n' % date.today())
-
-    cve_f.close()
-    conn.commit()
-    conn.close()
-}
-
-# DJB2 hash algorithm
-def hash_djb2(s):
-    hash = 5381
-    for x in s:
-        hash = (( hash << 5) + hash) + ord(x)
-
-    return hash & 0xFFFFFFFF
-
-def initialize_db(c):
-    c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
-    c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
-        SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
-    c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
-        VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
-    c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
-        (PRODUCT, VERSION)")
-
-def update_db(c, json_filename):
-    import json
-    root = json.load(json_filename)
-
-    for elt in root['CVE_Items']:
-        if not elt['impact']:
-            continue
-
-        cveId = elt['cve']['CVE_data_meta']['ID']
-        cveDesc = elt['cve']['description']['description_data'][0]['value']
-        date = elt['lastModifiedDate']
-        accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
-        cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
-
-        try:
-            cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
-        except:
-            cvssv3 = 0.0
-
-        c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
-                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
-
-        for vendor in elt['cve']['affects']['vendor']['vendor_data']:
-            for product in vendor['product']['product_data']:
-                for version in product['version']['version_data']:
-                    product_str = cveId+vendor['vendor_name']+product['product_name']+version['version_value']
-                    hashstr = hash_djb2(product_str)
-                    c.execute("insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?)",
-                            [ hashstr, cveId, vendor['vendor_name'],
-                                product['product_name'], version['version_value'],
-                                version['version_affected']])
-
-
-
-addtask do_populate_cve_db before do_fetch
-do_populate_cve_db[nostamp] = "1"
-
-EXCLUDE_FROM_WORLD = "1"