cve_policy_filter.py: Get the filter data from nvd@nist.gov item

Now the latest json format result file includes the several items
in the set data["scannedCves"][cve_id]["cveContents"]["nvd"], so
the original usage is not available to filter CVE info anymore.

So it's time to drop the exception which is to raise this condition
that the length is greater than 1. It will be failed to throw the
exception. We are going to use the condition 'source=nvd@nist.gov'
to get the accurate CVE information instead.

Another update is to expand the function find_lp_assigned with
adding new condition to find the CVE id in the description section
of the LP page. As the length of title is limited, if one page is
used to track many CVE issues, the length may be not enough to
record all CVE ID items.

Closes-Bug: 2059996

Signed-off-by: Zhixiong Chi <zhixiong.chi@windriver.com>
Change-Id: Ia7dfee5db53baaa82a8e6dd9d5dde8a31da5bcc2

cve_support/cve_policy_filter.py

@@ -25,18 +25,6 @@ cves_to_omit = []
 cves_report = {}
 
 
-class NVDLengthException(Exception):
-    """
-    Throw the exception when the length of NVD list != 1
-    """
-    def __init__(self, length):
-        self.length = length
-
-    def __str__(self):
-        print("Warning: NVD length: %d, not 1, Please check again!" \
-            % self.length)
-
-
 def print_html_report(cves_report, title):
     """
     Print the html report
@@ -256,13 +244,29 @@ def cvssv3_parse_n_report(cves,title,data):
         cve_id = cve["id"]
         affectedpackages_list = []
         allfixed = "fixed"
-        try:
-            nvdlength = len(data["scannedCves"][cve_id]["cveContents"]["nvd"])
-            if nvdlength != 1:
-                raise NVDLengthException(nvdlength)
 
-            nvd3_score = data["scannedCves"][cve_id]["cveContents"]["nvd"][0]["cvss3Score"]
-            cvss3vector = data["scannedCves"][cve_id]["cveContents"]["nvd"][0]["cvss3Vector"]
+        if 'nvd' not in data['scannedCves'][cve_id]['cveContents'].keys():
+            continue
+
+        missing = False
+        use_l = {}
+        for l in data['scannedCves'][cve_id]['cveContents']['nvd']:
+            try:
+                if l["optional"]["source"] == "nvd@nist.gov":
+                    if not use_l:
+                        use_l = l
+                    else:
+                        print("Oops: two entries for nvd@nist.gov: %s" % k)
+            except KeyError:
+                # ignore missing ["optional"]["source"]
+                missing = True
+                pass
+        if missing and use_l:
+            print("CVE %s is example" % cve_id)
+
+        try:
+            nvd3_score = l["cvss3Score"]
+            cvss3vector = l["cvss3Vector"]
             if cvss3vector == "":
                 raise KeyError
         except KeyError:
@@ -304,13 +308,31 @@ def cvssv2_parse_n_report(cves,title,data):
         cve_id = cve["id"]
         affectedpackages_list = []
         allfixed = "fixed"
-        try:
-            nvdlength = len(data["scannedCves"][cve_id]["cveContents"]["nvd"])
-            if nvdlength != 1:
-                raise NVDLengthException(nvdlength)
 
-            nvd2_score = data["scannedCves"][cve_id]["cveContents"]["nvd"][0]["cvss2Score"]
-            cvss2vector = data["scannedCves"][cve_id]["cveContents"]["nvd"][0]["cvss2Vector"]
+        if 'nvd' not in data['scannedCves'][cve_id]['cveContents'].keys():
+            continue
+
+        missing = False
+        use_l = {}
+        for l in data['scannedCves'][cve_id]['cveContents']['nvd']:
+            try:
+                if l["optional"]["source"] == "nvd@nist.gov":
+                    if not use_l:
+                        use_l = l
+                    else:
+                        print("Oops: two entries for nvd@nist.gov: %s" % k)
+            except KeyError:
+                # ignore missing ["optional"]["source"]
+                missing = True
+                pass
+        if missing and use_l:
+            print("CVE %s is example" % cve_id)
+
+        try:
+            nvd2_score = l["cvss2Score"]
+            cvss2vector = l["cvss2Vector"]
+            if cvss2vector == "":
+                raise KeyError
         except KeyError:
             cves_w_errors.append(cve)
         else:

cve_support/lp.py

@@ -9,6 +9,7 @@ Implement system to detect if CVEs has launchpad assigned
 """
 import json
 import os
+import re
 from os import path
 from launchpadlib.launchpad import Launchpad
 
@@ -27,6 +28,7 @@ STATUSES = [
 
 CACHEDIR = path.join('/tmp', os.environ['USER'], '.launchpadlib/cache')
 CVES_FILE = path.join(CACHEDIR, 'cves_open.json')
+NVD_URL = 'https://nvd.nist.gov/vuln/detail'
 DATA = []
 
 
@@ -47,6 +49,7 @@ def search_upstrem_lps():
             bug_dic['status'] = task.status
             bug_dic['title'] = bug.title
             bug_dic['link'] = bug.self_link
+            bug_dic['description'] = bug.description
             DATA.append(bug_dic)
 
     with open(CVES_FILE, 'w') as outfile:
@@ -66,7 +69,8 @@ def find_lp_assigned(cve_id):
             search_upstrem_lps()
 
     for bug in DATA:
-        if cve_id in bug["title"]:
+        pattern = cve_id + ": " + path.join(NVD_URL, cve_id)
+        if re.search(cve_id, bug["title"]) or re.search(pattern, bug["description"]):
             return bug
 
     return None