Skip to content

Commit

Permalink
Merge pull request #3207 from jeremylong/matchingStrategy
Browse files Browse the repository at this point in the history 
Matching Strategy - Product Names & Major Version Numbers
  • Loading branch information
@jeremylong
jeremylong authored Mar 21, 2021
2 parents 28dee17 + d91a1a1 commit 96c34b0
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 112 deletions.
123 changes: 61 additions & 62 deletions core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,9 +256,17 @@ public void closeAnalyzer() {
protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException, AnalysisException {
boolean identifierAdded;

if (simpleSearch(dependency)) {
return;
}
Set<String> majorVersions = dependency.getSoftwareIdentifiers()
.stream()
.filter(i -> i instanceof PurlIdentifier)
.map(i -> {
PurlIdentifier p = (PurlIdentifier) i;
DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false);
if (depVersion != null) {
return depVersion.getVersionParts().get(0);
}
return null;
}).collect(Collectors.toSet());

final Map<String, MutableInt> vendors = new HashMap<>();
final Map<String, MutableInt> products = new HashMap<>();
Expand All @@ -268,6 +276,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException,
collectTerms(vendors, dependency.getIterator(EvidenceType.VENDOR, confidence));
LOGGER.debug("vendor search: {}", vendors);
collectTerms(products, dependency.getIterator(EvidenceType.PRODUCT, confidence));
addMajorVersionToTerms(majorVersions, products);
LOGGER.debug("product search: {}", products);
if (!vendors.isEmpty() && !products.isEmpty()) {
final List<IndexEntry> entries = searchCPE(vendors, products,
Expand All @@ -283,7 +292,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException,
continue;
}
previouslyFound.add(e.getDocumentId());
if (verifyEntry(e, dependency)) {
if (verifyEntry(e, dependency, majorVersions)) {
final String vendor = e.getVendor();
final String product = e.getProduct();
LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
Expand All @@ -310,6 +319,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException,
* @param evidence an iterable set of evidence to concatenate
*/
@SuppressWarnings("null")

protected void collectTerms(Map<String, MutableInt> terms, Iterable<Evidence> evidence) {
for (Evidence e : evidence) {
String value = cleanseText(e.getValue());
Expand Down Expand Up @@ -357,12 +367,38 @@ protected void collectTerms(Map<String, MutableInt> terms, Iterable<Evidence> ev
value = value.substring(0, 1000);
}
}
final MutableInt count = terms.get(value);
if (count == null) {
terms.put(value, new MutableInt(1));
} else {
count.add(1);
}
addTerm(terms, value);
}
}

private void addMajorVersionToTerms(Set<String> majorVersions, Map<String, MutableInt> products) {
Map<String, MutableInt> temp = new HashMap<>();
products.entrySet().stream()
.filter(term -> term.getKey() != null)
.forEach(term -> {
majorVersions.stream()
.filter(version -> (!term.getKey().endsWith(version)
&& !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1))
&& !products.containsKey(term.getKey() + version)))
.forEach(version -> {
addTerm(temp, term.getKey() + version);
});
});
products.putAll(temp);
}

/**
* Adds a term to the map of terms.
*
* @param terms the map of terms
* @param value the value of the term to add
*/
private void addTerm(Map<String, MutableInt> terms, String value) {
final MutableInt count = terms.get(value);
if (count == null) {
terms.put(value, new MutableInt(1));
} else {
count.add(1);
}
}

Expand Down Expand Up @@ -587,7 +623,8 @@ private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
* @param dependency the dependency that the CPE entries could be for.
* @return whether or not the entry is valid.
*/
private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
private boolean verifyEntry(final IndexEntry entry, final Dependency dependency,
final Set<String> majorVersions) {
boolean isValid = false;
//TODO - does this nullify some of the fuzzy matching that happens in the lucene search?
// for instance CPE some-component and in the evidence we have SomeComponent.
Expand All @@ -602,9 +639,15 @@ private boolean verifyEntry(final IndexEntry entry, final Dependency dependency)
}
}
}
} else if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())
&& collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) {
isValid = true;
} else if (collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) {
if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())) {
isValid = true;
} else {
isValid = majorVersions.stream().filter(version->entry.getProduct().endsWith(version) && entry.getProduct().length()>version.length())
.anyMatch(version ->
collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct().substring(0, entry.getProduct().length()-version.length()))
);
}
}
return isValid;
}
Expand Down Expand Up @@ -993,56 +1036,12 @@ private Set<Cpe> filterEcosystem(String ecosystem, Set<CpePlus> entries) {
}

/**
* Performs a simplistic search for CPE entries based on the vendor and
* product from a Purl identifier.
* Add the given version to the CpeBuilder - this method attempts to parse
* out the update from the version and correctly set the value in the CPE.
*
* @param dependency the dependency to perform the search for a CPE
* @return <code>true</code> if an identifier is found; otherwise
* <code>false</code>
* @param depVersion the version to add
* @param cpeBuilder a reference to the CPE Builder
*/
private boolean simpleSearch(Dependency dependency) {
return dependency.getSoftwareIdentifiers()
.stream()
.filter(i -> i instanceof PurlIdentifier)
.map(i -> {
PurlIdentifier p = (PurlIdentifier) i;
String vendor = p.getNamespace();
if (vendor == null) {
vendor = p.getName();
}
String product = p.getName();
DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false);
boolean identifierAdded = false;
if (depVersion != null) {
String majorVersion = depVersion.getVersionParts().get(0);

Set<Pair<String, String>> simpleMatches = cve.simpleCPESearch(vendor, product, majorVersion);
for (Pair<String, String> match : simpleMatches) {
final Set<CpePlus> cpePlusEntries = cve.getCPEs(vendor, product);
final Set<Cpe> cpes = filterEcosystem(dependency.getEcosystem(), cpePlusEntries);
if (cpes == null || cpes.isEmpty()) {
continue;
}

final CpeBuilder cpeBuilder = new CpeBuilder();
cpeBuilder.part(Part.APPLICATION).vendor(match.getLeft()).product(match.getRight());
addVersionAndUpdate(depVersion, cpeBuilder);
try {
final Cpe cpeId = cpeBuilder.build();
final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeId.getVendor(), UTF8),
URLEncoder.encode(cpeId.getProduct(), UTF8), URLEncoder.encode(cpeId.getVersion(), UTF8));
CpeIdentifier identifier = new CpeIdentifier(cpeId, url, Confidence.HIGHEST);
dependency.addVulnerableSoftwareIdentifier(identifier);
identifierAdded = true;
} catch (CpeValidationException | UnsupportedEncodingException ex) {
LOGGER.debug("Error building CPE ", ex);
}
}
}
return identifierAdded;
}).anyMatch(p -> p);
}

private void addVersionAndUpdate(DependencyVersion depVersion, final CpeBuilder cpeBuilder) {
final int idx = depVersion.getVersionParts().size() - 1;
if (idx > 0 && depVersion.getVersionParts().get(idx)
Expand Down
49 changes: 0 additions & 49 deletions core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@

import static org.apache.commons.collections.map.AbstractReferenceMap.HARD;
import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT;
import org.apache.commons.lang3.StringUtils;
import org.owasp.dependencycheck.analyzer.exception.LambdaExceptionWrapper;
import org.owasp.dependencycheck.analyzer.exception.UnexpectedAnalysisException;
import org.owasp.dependencycheck.data.nvd.json.BaseMetricV2;
Expand Down Expand Up @@ -129,7 +128,6 @@ public final class CveDB implements AutoCloseable {
*/
private boolean isOracle = false;


/**
* The enumeration value names must match the keys of the statements in the
* statement bundles "dbStatements*.properties".
Expand Down Expand Up @@ -203,10 +201,6 @@ enum PreparedStatementCveDb {
* Key for SQL Statement.
*/
SELECT_VENDOR_PRODUCT_LIST,
/**
* Key for SQL Statement.
*/
SELECT_SIMPLE_CPE_SEARCH,
/**
* Key for SQL Statement.
*/
Expand Down Expand Up @@ -510,49 +504,6 @@ public synchronized Set<CpePlus> getCPEs(String vendor, String product) {
return cpe;
}


public synchronized Set<Pair<String,String>> simpleCPESearch(String vendor, String product, String majorVersion) {
final Set<Pair<String, String>> data = new HashSet<>();
ResultSet rs = null;
try {
final PreparedStatement ps = getPreparedStatement(SELECT_SIMPLE_CPE_SEARCH);
if (ps == null) {
throw new SQLException("Database query does not exist in the resource bundle: " + SELECT_VENDOR_PRODUCT_LIST);
}
String vendorSearch = vendor.replace("-","_");
if (StringUtils.countMatches(vendorSearch, '.')>1) {
String[] parts = vendorSearch.split("\\.");
if ("org".equals(parts[0]) || "com".equals(parts[0])) {
vendorSearch = parts[1];
}
}
final String productSearch = product.replace("-","_");

ps.setString(1, vendorSearch);
ps.setString(2, productSearch);

ps.setString(3, vendorSearch + "_project");
ps.setString(4, productSearch);

ps.setString(5, vendorSearch);
ps.setString(6, productSearch + majorVersion);

ps.setString(7, vendorSearch + "_project");
ps.setString(8, productSearch + majorVersion);

rs = ps.executeQuery();
while (rs.next()) {
data.add(new Pair<>(rs.getString(1), rs.getString(2)));
}
} catch (SQLException ex) {
final String msg = "An unexpected SQL Exception occurred; please see the verbose log for more details.";
throw new DatabaseException(msg, ex);
} finally {
DBUtils.closeResultSet(rs);
}
return data;
}

/**
* Returns the entire list of vendor/product combinations.
*
Expand Down
1 change: 0 additions & 1 deletion core/src/main/resources/data/dbStatements.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ DELETE_PROPERTY=DELETE FROM properties WHERE id = ?
UPDATE_ECOSYSTEM=UPDATE cpeEntry e SET e.ecosystem=(SELECT cpeEcosystemCache.ecosystem FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND e.ecosystem IS NULL AND cpeEcosystemCache.ecosystem<>'MULTIPLE') WHERE e.ecosystem IS NULL;
UPDATE_ECOSYSTEM2=UPDATE cpeEntry e SET e.ecosystem=null WHERE e.ecosystem IS NOT NULL AND EXISTS(SELECT * FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND cpeEcosystemCache.ecosystem='MULTIPLE');

SELECT_SIMPLE_CPE_SEARCH=SELECT DISTINCT VENDOR, PRODUCT FROM CPEENTRY WHERE PART='a' AND ((REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?))
#the following two statements are unused and are only referenced in dead code
#DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software)
#ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true)

0 comments on commit 96c34b0

Please sign in to comment.