Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Matching Strategy - Product Names & Major Version Numbers #3207

Merged
merged 1 commit into from
Mar 21, 2021
Merged

Matching Strategy - Product Names & Major Version Numbers #3207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 61 additions & 62 deletions core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,9 +256,17 @@ public void closeAnalyzer() {
protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException, AnalysisException {
boolean identifierAdded;

if (simpleSearch(dependency)) {
return;
}
Set<String> majorVersions = dependency.getSoftwareIdentifiers()
.stream()
.filter(i -> i instanceof PurlIdentifier)
.map(i -> {
PurlIdentifier p = (PurlIdentifier) i;
DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false);
if (depVersion != null) {
return depVersion.getVersionParts().get(0);
}
return null;
}).collect(Collectors.toSet());

final Map<String, MutableInt> vendors = new HashMap<>();
final Map<String, MutableInt> products = new HashMap<>();
Expand All @@ -268,6 +276,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException,
collectTerms(vendors, dependency.getIterator(EvidenceType.VENDOR, confidence));
LOGGER.debug("vendor search: {}", vendors);
collectTerms(products, dependency.getIterator(EvidenceType.PRODUCT, confidence));
addMajorVersionToTerms(majorVersions, products);
LOGGER.debug("product search: {}", products);
if (!vendors.isEmpty() && !products.isEmpty()) {
final List<IndexEntry> entries = searchCPE(vendors, products,
Expand All @@ -283,7 +292,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException,
continue;
}
previouslyFound.add(e.getDocumentId());
if (verifyEntry(e, dependency)) {
if (verifyEntry(e, dependency, majorVersions)) {
final String vendor = e.getVendor();
final String product = e.getProduct();
LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
Expand All @@ -310,6 +319,7 @@ protected void determineCPE(Dependency dependency) throws CorruptIndexException,
* @param evidence an iterable set of evidence to concatenate
*/
@SuppressWarnings("null")

protected void collectTerms(Map<String, MutableInt> terms, Iterable<Evidence> evidence) {
for (Evidence e : evidence) {
String value = cleanseText(e.getValue());
Expand Down Expand Up @@ -357,12 +367,38 @@ protected void collectTerms(Map<String, MutableInt> terms, Iterable<Evidence> ev
value = value.substring(0, 1000);
}
}
final MutableInt count = terms.get(value);
if (count == null) {
terms.put(value, new MutableInt(1));
} else {
count.add(1);
}
addTerm(terms, value);
}
}

private void addMajorVersionToTerms(Set<String> majorVersions, Map<String, MutableInt> products) {
Map<String, MutableInt> temp = new HashMap<>();
products.entrySet().stream()
.filter(term -> term.getKey() != null)
.forEach(term -> {
majorVersions.stream()
.filter(version -> (!term.getKey().endsWith(version)
&& !Character.isDigit(term.getKey().charAt(term.getKey().length() - 1))
&& !products.containsKey(term.getKey() + version)))
.forEach(version -> {
addTerm(temp, term.getKey() + version);
});
});
products.putAll(temp);
}

/**
* Adds a term to the map of terms.
*
* @param terms the map of terms
* @param value the value of the term to add
*/
private void addTerm(Map<String, MutableInt> terms, String value) {
final MutableInt count = terms.get(value);
if (count == null) {
terms.put(value, new MutableInt(1));
} else {
count.add(1);
}
}

Expand Down Expand Up @@ -587,7 +623,8 @@ private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
* @param dependency the dependency that the CPE entries could be for.
* @return whether or not the entry is valid.
*/
private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
private boolean verifyEntry(final IndexEntry entry, final Dependency dependency,
final Set<String> majorVersions) {
boolean isValid = false;
//TODO - does this nullify some of the fuzzy matching that happens in the lucene search?
// for instance CPE some-component and in the evidence we have SomeComponent.
Expand All @@ -602,9 +639,15 @@ private boolean verifyEntry(final IndexEntry entry, final Dependency dependency)
}
}
}
} else if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())
&& collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) {
isValid = true;
} else if (collectionContainsString(dependency.getEvidence(EvidenceType.VENDOR), entry.getVendor())) {
if (collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct())) {
isValid = true;
} else {
isValid = majorVersions.stream().filter(version->entry.getProduct().endsWith(version) && entry.getProduct().length()>version.length())
.anyMatch(version ->
collectionContainsString(dependency.getEvidence(EvidenceType.PRODUCT), entry.getProduct().substring(0, entry.getProduct().length()-version.length()))
);
}
}
return isValid;
}
Expand Down Expand Up @@ -993,56 +1036,12 @@ private Set<Cpe> filterEcosystem(String ecosystem, Set<CpePlus> entries) {
}

/**
* Performs a simplistic search for CPE entries based on the vendor and
* product from a Purl identifier.
* Add the given version to the CpeBuilder - this method attempts to parse
* out the update from the version and correctly set the value in the CPE.
*
* @param dependency the dependency to perform the search for a CPE
* @return <code>true</code> if an identifier is found; otherwise
* <code>false</code>
* @param depVersion the version to add
* @param cpeBuilder a reference to the CPE Builder
*/
private boolean simpleSearch(Dependency dependency) {
return dependency.getSoftwareIdentifiers()
.stream()
.filter(i -> i instanceof PurlIdentifier)
.map(i -> {
PurlIdentifier p = (PurlIdentifier) i;
String vendor = p.getNamespace();
if (vendor == null) {
vendor = p.getName();
}
String product = p.getName();
DependencyVersion depVersion = DependencyVersionUtil.parseVersion(p.getVersion(), false);
boolean identifierAdded = false;
if (depVersion != null) {
String majorVersion = depVersion.getVersionParts().get(0);

Set<Pair<String, String>> simpleMatches = cve.simpleCPESearch(vendor, product, majorVersion);
for (Pair<String, String> match : simpleMatches) {
final Set<CpePlus> cpePlusEntries = cve.getCPEs(vendor, product);
final Set<Cpe> cpes = filterEcosystem(dependency.getEcosystem(), cpePlusEntries);
if (cpes == null || cpes.isEmpty()) {
continue;
}

final CpeBuilder cpeBuilder = new CpeBuilder();
cpeBuilder.part(Part.APPLICATION).vendor(match.getLeft()).product(match.getRight());
addVersionAndUpdate(depVersion, cpeBuilder);
try {
final Cpe cpeId = cpeBuilder.build();
final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeId.getVendor(), UTF8),
URLEncoder.encode(cpeId.getProduct(), UTF8), URLEncoder.encode(cpeId.getVersion(), UTF8));
CpeIdentifier identifier = new CpeIdentifier(cpeId, url, Confidence.HIGHEST);
dependency.addVulnerableSoftwareIdentifier(identifier);
identifierAdded = true;
} catch (CpeValidationException | UnsupportedEncodingException ex) {
LOGGER.debug("Error building CPE ", ex);
}
}
}
return identifierAdded;
}).anyMatch(p -> p);
}

private void addVersionAndUpdate(DependencyVersion depVersion, final CpeBuilder cpeBuilder) {
final int idx = depVersion.getVersionParts().size() - 1;
if (idx > 0 && depVersion.getVersionParts().get(idx)
Expand Down
49 changes: 0 additions & 49 deletions core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@

import static org.apache.commons.collections.map.AbstractReferenceMap.HARD;
import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT;
import org.apache.commons.lang3.StringUtils;
import org.owasp.dependencycheck.analyzer.exception.LambdaExceptionWrapper;
import org.owasp.dependencycheck.analyzer.exception.UnexpectedAnalysisException;
import org.owasp.dependencycheck.data.nvd.json.BaseMetricV2;
Expand Down Expand Up @@ -129,7 +128,6 @@ public final class CveDB implements AutoCloseable {
*/
private boolean isOracle = false;


/**
* The enumeration value names must match the keys of the statements in the
* statement bundles "dbStatements*.properties".
Expand Down Expand Up @@ -203,10 +201,6 @@ enum PreparedStatementCveDb {
* Key for SQL Statement.
*/
SELECT_VENDOR_PRODUCT_LIST,
/**
* Key for SQL Statement.
*/
SELECT_SIMPLE_CPE_SEARCH,
/**
* Key for SQL Statement.
*/
Expand Down Expand Up @@ -510,49 +504,6 @@ public synchronized Set<CpePlus> getCPEs(String vendor, String product) {
return cpe;
}


public synchronized Set<Pair<String,String>> simpleCPESearch(String vendor, String product, String majorVersion) {
final Set<Pair<String, String>> data = new HashSet<>();
ResultSet rs = null;
try {
final PreparedStatement ps = getPreparedStatement(SELECT_SIMPLE_CPE_SEARCH);
if (ps == null) {
throw new SQLException("Database query does not exist in the resource bundle: " + SELECT_VENDOR_PRODUCT_LIST);
}
String vendorSearch = vendor.replace("-","_");
if (StringUtils.countMatches(vendorSearch, '.')>1) {
String[] parts = vendorSearch.split("\\.");
if ("org".equals(parts[0]) || "com".equals(parts[0])) {
vendorSearch = parts[1];
}
}
final String productSearch = product.replace("-","_");

ps.setString(1, vendorSearch);
ps.setString(2, productSearch);

ps.setString(3, vendorSearch + "_project");
ps.setString(4, productSearch);

ps.setString(5, vendorSearch);
ps.setString(6, productSearch + majorVersion);

ps.setString(7, vendorSearch + "_project");
ps.setString(8, productSearch + majorVersion);

rs = ps.executeQuery();
while (rs.next()) {
data.add(new Pair<>(rs.getString(1), rs.getString(2)));
}
} catch (SQLException ex) {
final String msg = "An unexpected SQL Exception occurred; please see the verbose log for more details.";
throw new DatabaseException(msg, ex);
} finally {
DBUtils.closeResultSet(rs);
}
return data;
}

/**
* Returns the entire list of vendor/product combinations.
*
Expand Down
1 change: 0 additions & 1 deletion core/src/main/resources/data/dbStatements.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ DELETE_PROPERTY=DELETE FROM properties WHERE id = ?
UPDATE_ECOSYSTEM=UPDATE cpeEntry e SET e.ecosystem=(SELECT cpeEcosystemCache.ecosystem FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND e.ecosystem IS NULL AND cpeEcosystemCache.ecosystem<>'MULTIPLE') WHERE e.ecosystem IS NULL;
UPDATE_ECOSYSTEM2=UPDATE cpeEntry e SET e.ecosystem=null WHERE e.ecosystem IS NOT NULL AND EXISTS(SELECT * FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND cpeEcosystemCache.ecosystem='MULTIPLE');

SELECT_SIMPLE_CPE_SEARCH=SELECT DISTINCT VENDOR, PRODUCT FROM CPEENTRY WHERE PART='a' AND ((REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?) OR (REPLACE(VENDOR,'-','_')=? AND REPLACE(PRODUCT,'-','_')=?))
#the following two statements are unused and are only referenced in dead code
#DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software)
#ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true)