From d91a1a12216e6e777ed4b588c14e237e78b25e52 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 21 Mar 2021 11:17:04 -0400 Subject: [PATCH] remove new matching strategy #3140 and modify existing matching strategy to account for the fact that CPE product names may contain the major version. This also resolves #3193 and #3183 --- .../dependencycheck/analyzer/CPEAnalyzer.java | 123 +++++++++--------- .../dependencycheck/data/nvdcve/CveDB.java | 49 ------- .../resources/data/dbStatements.properties | 1 - 3 files changed, 61 insertions(+), 112 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java index 09846da3aee..910cb8de2b9 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java @@ -256,9 +256,17 @@ public void closeAnalyzer() { protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException, AnalysisException { boolean identifierAdded; - if (simpleSearch(dependency)) { - return; - } + Set majorVersions = dependency.getSoftwareIdentifiers() + .stream() + .filter(i -> i instanceof PurlIdentifier) + .map(i -> { + PurlIdentifier p = (PurlIdentifier) i; + DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false); + if (depVersion != null) { + return depVersion.getVersionParts().get(0); + } + return null; + }).collect(Collectors.toSet()); final Map vendors = new HashMap<>(); final Map products = new HashMap<>(); @@ -268,6 +276,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException, collectTerms(vendors, dependency.getIterator(EvidenceType.VENDOR, confidence)); LOGGER.debug("vendor search: {}", vendors); collectTerms(products, dependency.getIterator(EvidenceType.PRODUCT, confidence)); + addMajorVersionToTerms(majorVersions, products); LOGGER.debug("product search: {}", products); if (!vendors.isEmpty() && !products.isEmpty()) { final List entries = searchCPE(vendors, products, @@ -283,7 +292,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException, continue; } previouslyFound.add(e.getDocumentId()); - if (verifyEntry(e, dependency)) { + if (verifyEntry(e, dependency, majorVersions)) { final String vendor = e.getVendor(); final String product = e.getProduct(); LOGGER.debug("identified vendor/product: {}/{}", vendor, product); @@ -310,6 +319,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException, * @param evidence an iterable set of evidence to concatenate */ @SuppressWarnings("null") + protected void collectTerms(Map terms, Iterable evidence) { for (Evidence e : evidence) { String value = cleanseText(e.getValue()); @@ -357,12 +367,38 @@ protected void collectTerms(Map terms, Iterable ev value = value.substring(0, 1000); } } - final MutableInt count = terms.get(value); - if (count == null) { - terms.put(value, new MutableInt(1)); - } else { - count.add(1); - } + addTerm(terms, value); + } + } + + private void addMajorVersionToTerms(Set majorVersions, Map products) { + Map temp = new HashMap<>(); + products.entrySet().stream() + .filter(term -> term.getKey() != null) + .forEach(term -> { + majorVersions.stream() + .filter(version -> (!term.getKey().endsWith(version) + && !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1)) + && !products.containsKey(term.getKey() + version))) + .forEach(version -> { + addTerm(temp, term.getKey() + version); + }); + }); + products.putAll(temp); + } + + /** + * Adds a term to the map of terms. + * + * @param terms the map of terms + * @param value the value of the term to add + */ + private void addTerm(Map terms, String value) { + final MutableInt count = terms.get(value); + if (count == null) { + terms.put(value, new MutableInt(1)); + } else { + count.add(1); } } @@ -587,7 +623,8 @@ private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) { * @param dependency the dependency that the CPE entries could be for. * @return whether or not the entry is valid. */ - private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) { + private boolean verifyEntry(final IndexEntry entry, final Dependency dependency, + final Set majorVersions) { boolean isValid = false; //TODO - does this nullify some of the fuzzy matching that happens in the lucene search? // for instance CPE some-component and in the evidence we have SomeComponent. @@ -602,9 +639,15 @@ private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) } } } - } else if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct()) - && collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) { - isValid = true; + } else if (collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) { + if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())) { + isValid = true; + } else { + isValid = majorVersions.stream().filter(version->entry.getProduct().endsWith(version) && entry.getProduct().length()>version.length()) + .anyMatch(version -> + collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct().substring(0, entry.getProduct().length()-version.length())) + ); + } } return isValid; } @@ -993,56 +1036,12 @@ private Set filterEcosystem(String ecosystem, Set entries) { } /** - * Performs a simplistic search for CPE entries based on the vendor and - * product from a Purl identifier. + * Add the given version to the CpeBuilder - this method attempts to parse + * out the update from the version and correctly set the value in the CPE. * - * @param dependency the dependency to perform the search for a CPE - * @return true if an identifier is found; otherwise - * false + * @param depVersion the version to add + * @param cpeBuilder a reference to the CPE Builder */ - private boolean simpleSearch(Dependency dependency) { - return dependency.getSoftwareIdentifiers() - .stream() - .filter(i -> i instanceof PurlIdentifier) - .map(i -> { - PurlIdentifier p = (PurlIdentifier) i; - String vendor = p.getNamespace(); - if (vendor == null) { - vendor = p.getName(); - } - String product = p.getName(); - DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false); - boolean identifierAdded = false; - if (depVersion != null) { - String majorVersion = depVersion.getVersionParts().get(0); - - Set> simpleMatches = cve.simpleCPESearch(vendor, product, majorVersion); - for (Pair match : simpleMatches) { - final Set cpePlusEntries = cve.getCPEs(vendor, product); - final Set cpes = filterEcosystem(dependency.getEcosystem(), cpePlusEntries); - if (cpes == null || cpes.isEmpty()) { - continue; - } - - final CpeBuilder cpeBuilder = new CpeBuilder(); - cpeBuilder.part(Part.APPLICATION).vendor(match.getLeft()).product(match.getRight()); - addVersionAndUpdate(depVersion, cpeBuilder); - try { - final Cpe cpeId = cpeBuilder.build(); - final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeId.getVendor(), UTF8), - URLEncoder.encode(cpeId.getProduct(), UTF8), URLEncoder.encode(cpeId.getVersion(), UTF8)); - CpeIdentifier identifier = new CpeIdentifier(cpeId, url, Confidence.HIGHEST); - dependency.addVulnerableSoftwareIdentifier(identifier); - identifierAdded = true; - } catch (CpeValidationException | UnsupportedEncodingException ex) { - LOGGER.debug("Error building CPE ", ex); - } - } - } - return identifierAdded; - }).anyMatch(p -> p); - } - private void addVersionAndUpdate(DependencyVersion depVersion, final CpeBuilder cpeBuilder) { final int idx = depVersion.getVersionParts().size() - 1; if (idx > 0 && depVersion.getVersionParts().get(idx) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 3200a2c74b7..76bd2fc3a14 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -39,7 +39,6 @@ import static org.apache.commons.collections.map.AbstractReferenceMap.HARD; import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT; -import org.apache.commons.lang3.StringUtils; import org.owasp.dependencycheck.analyzer.exception.LambdaExceptionWrapper; import org.owasp.dependencycheck.analyzer.exception.UnexpectedAnalysisException; import org.owasp.dependencycheck.data.nvd.json.BaseMetricV2; @@ -129,7 +128,6 @@ public final class CveDB implements AutoCloseable { */ private boolean isOracle = false; - /** * The enumeration value names must match the keys of the statements in the * statement bundles "dbStatements*.properties". @@ -203,10 +201,6 @@ enum PreparedStatementCveDb { * Key for SQL Statement. */ SELECT_VENDOR_PRODUCT_LIST, - /** - * Key for SQL Statement. - */ - SELECT_SIMPLE_CPE_SEARCH, /** * Key for SQL Statement. */ @@ -510,49 +504,6 @@ public synchronized Set getCPEs(String vendor, String product) { return cpe; } - - public synchronized Set> simpleCPESearch(String vendor, String product, String majorVersion) { - final Set> data = new HashSet<>(); - ResultSet rs = null; - try { - final PreparedStatement ps = getPreparedStatement(SELECT_SIMPLE_CPE_SEARCH); - if (ps == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + SELECT_VENDOR_PRODUCT_LIST); - } - String vendorSearch = vendor.replace("-","_"); - if (StringUtils.countMatches(vendorSearch, '.')>1) { - String[] parts = vendorSearch.split("\\."); - if ("org".equals(parts[0]) || "com".equals(parts[0])) { - vendorSearch = parts[1]; - } - } - final String productSearch = product.replace("-","_"); - - ps.setString(1, vendorSearch); - ps.setString(2, productSearch); - - ps.setString(3, vendorSearch + "_project"); - ps.setString(4, productSearch); - - ps.setString(5, vendorSearch); - ps.setString(6, productSearch + majorVersion); - - ps.setString(7, vendorSearch + "_project"); - ps.setString(8, productSearch + majorVersion); - - rs = ps.executeQuery(); - while (rs.next()) { - data.add(new Pair<>(rs.getString(1), rs.getString(2))); - } - } catch (SQLException ex) { - final String msg = "An unexpected SQL Exception occurred; please see the verbose log for more details."; - throw new DatabaseException(msg, ex); - } finally { - DBUtils.closeResultSet(rs); - } - return data; - } - /** * Returns the entire list of vendor/product combinations. * diff --git a/core/src/main/resources/data/dbStatements.properties b/core/src/main/resources/data/dbStatements.properties index ebc37c4def4..d94a7261ad9 100644 --- a/core/src/main/resources/data/dbStatements.properties +++ b/core/src/main/resources/data/dbStatements.properties @@ -41,7 +41,6 @@ DELETE_PROPERTY=DELETE FROM properties WHERE id = ? UPDATE_ECOSYSTEM=UPDATE cpeEntry e SET e.ecosystem=(SELECT cpeEcosystemCache.ecosystem FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND e.ecosystem IS NULL AND cpeEcosystemCache.ecosystem<>'MULTIPLE') WHERE e.ecosystem IS NULL; UPDATE_ECOSYSTEM2=UPDATE cpeEntry e SET e.ecosystem=null WHERE e.ecosystem IS NOT NULL AND EXISTS(SELECT * FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND cpeEcosystemCache.ecosystem='MULTIPLE'); -SELECT_SIMPLE_CPE_SEARCH=SELECT DISTINCT VENDOR, PRODUCT FROM CPEENTRY WHERE PART='a' AND ((REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?)) #the following two statements are unused and are only referenced in dead code #DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software) #ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true)