Skip to content

Commit 9403808

Browse files
electricalwindelectricalwind
committed
update
1 parent e8d0e39 commit 9403808

File tree

7 files changed

+235
-156
lines changed

7 files changed

+235
-156
lines changed

‎.gitignore

+142
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,154 @@
1+
# Created by .ignore support plugin (hsz.mobi)
2+
### Java template
3+
# Compiled class file
14
*.class
25

6+
*.iml
7+
.idea
8+
# Log file
9+
*.log
10+
11+
# BlueJ files
12+
*.ctxt
13+
314
# Mobile Tools for Java (J2ME)
415
.mtj.tmp/
516

617
# Package Files #
718
*.jar
819
*.war
920
*.ear
21+
*.zip
22+
*.tar.gz
23+
*.rar
1024

1125
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
1226
hs_err_pid*
27+
### Eclipse template
28+
29+
.metadata
30+
bin/
31+
tmp/
32+
*.tmp
33+
*.bak
34+
*.swp
35+
*~.nib
36+
local.properties
37+
.settings/
38+
.loadpath
39+
.recommenders
40+
41+
# External tool builders
42+
.externalToolBuilders/
43+
44+
# Locally stored "Eclipse launch configurations"
45+
*.launch
46+
47+
# PyDev specific (Python IDE for Eclipse)
48+
*.pydevproject
49+
50+
# CDT-specific (C/C++ Development Tooling)
51+
.cproject
52+
53+
# Java annotation processor (APT)
54+
.factorypath
55+
56+
# PDT-specific (PHP Development Tools)
57+
.buildpath
58+
59+
# sbteclipse plugin
60+
.target
61+
62+
# Tern plugin
63+
.tern-project
64+
65+
# TeXlipse plugin
66+
.texlipse
67+
68+
# STS (Spring Tool Suite)
69+
.springBeans
70+
71+
# Code Recommenders
72+
.recommenders/
73+
74+
# Scala IDE specific (Scala & Java development for Eclipse)
75+
.cache-main
76+
.scala_dependencies
77+
.worksheet
78+
### JetBrains template
79+
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
80+
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
81+
82+
# User-specific stuff:
83+
.idea/**/workspace.xml
84+
.idea/**/tasks.xml
85+
.idea/dictionaries
86+
87+
# Sensitive or high-churn files:
88+
.idea/**/dataSources/
89+
.idea/**/dataSources.ids
90+
.idea/**/dataSources.xml
91+
.idea/**/dataSources.local.xml
92+
.idea/**/sqlDataSources.xml
93+
.idea/**/dynamic.xml
94+
.idea/**/uiDesigner.xml
95+
96+
# Gradle:
97+
.idea/**/gradle.xml
98+
.idea/**/libraries
99+
100+
# CMake
101+
cmake-build-debug/
102+
103+
# Mongo Explorer plugin:
104+
.idea/**/mongoSettings.xml
105+
106+
## File-based project format:
107+
*.iws
108+
109+
## Plugin-specific files:
110+
111+
# IntelliJ
112+
out/
113+
114+
# mpeltonen/sbt-idea plugin
115+
.idea_modules/
116+
117+
# JIRA plugin
118+
atlassian-ide-plugin.xml
119+
120+
# Cursive Clojure plugin
121+
.idea/replstate.xml
122+
123+
# Crashlytics plugin (for Android Studio and IntelliJ)
124+
com_crashlytics_export_strings.xml
125+
crashlytics.properties
126+
crashlytics-build.properties
127+
fabric.properties
128+
### Maven template
129+
target/
130+
pom.xml.tag
131+
pom.xml.releaseBackup
132+
pom.xml.versionsBackup
133+
pom.xml.next
134+
release.properties
135+
dependency-reduced-pom.xml
136+
buildNumber.properties
137+
.mvn/timing.properties
138+
139+
# Avoid ignoring Maven wrapper jar file (.jar files are usually ignored)
140+
!/.mvn/wrapper/maven-wrapper.jar
141+
### Kotlin template
142+
# Compiled class file
143+
144+
# Log file
145+
146+
# BlueJ files
147+
148+
# Mobile Tools for Java (J2ME)
149+
150+
# Package Files #
151+
152+
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
153+
154+
/cve/projects/projects.iml

‎src/main/java/framevpm/bugcollector/BugCollector.java

+39-16
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,27 @@
11
package framevpm.bugcollector;
22

33
import data7.model.Data7;
4+
import data7.model.change.Commit;
5+
import data7.model.vulnerability.Vulnerability;
46
import data7.project.CProjects;
57
import data7.project.Project;
68
import framevpm.Utils;
79
import framevpm.bugcollector.model.BugDataset;
810
import framevpm.bugcollector.model.BugIdDataset;
911
import framevpm.bugcollector.model.BugRegExpDataset;
12+
import framevpm.releasebalancer.project.CProjectsInfo;
1013
import gitUtilitaries.GitActions;
1114

1215
import java.io.IOException;
1316
import java.text.ParseException;
14-
import java.util.Set;
17+
import java.util.*;
18+
import java.util.stream.Collectors;
1519

1620

1721
import static data7.Importer.updateOrCreateDatasetFor;
1822
import static data7.Resources.PATH_TO_GIT;
1923
import static framevpm.Utils.listOfCommitsFromData7;
24+
import static framevpm.Utils.loadBugDataset;
2025

2126
/**
2227
* Bug Collector Class
@@ -68,29 +73,47 @@ public BugDataset updateOrCreateBugDataset() throws IOException, ClassNotFoundEx
6873

6974

7075
public static void main(String[] args) throws ParseException, IOException, ClassNotFoundException {
71-
long time = System.currentTimeMillis();
72-
/**System.out.println("Start Linux");
73-
Data7 dataset = updateOrCreateDatasetFor(CProjectsInfo.LINUX_KERNEL);
74-
BugCollector collector = new BugCollector(dataset);
75-
Utils.saveBugDataset(collector.updateOrCreateBugDataset());
76-
System.out.println("End Linux : " + (System.currentTimeMillis() - time));
77-
time = System.currentTimeMillis();
78-
System.out.println("Start SystemD");
79-
dataset = updateOrCreateDatasetFor(CProjectsInfo.SYSTEMD);
80-
collector = new BugCollector(dataset);
81-
Utils.saveBugDataset(collector.updateOrCreateBugDataset());
82-
System.out.println("End SystemD : " + (System.currentTimeMillis() - time));*/
76+
/**long time = System.currentTimeMillis();
77+
System.out.println("Start Linux");
78+
Data7 dataset = updateOrCreateDatasetFor(CProjects.LINUX_KERNEL);
79+
BugCollector collector = new BugCollector(dataset);
80+
Utils.saveBugDataset(collector.updateOrCreateBugDataset());
81+
System.out.println("End Linux : " + (System.currentTimeMillis() - time));
82+
time = System.currentTimeMillis();
83+
System.out.println("Start SystemD");
84+
dataset = updateOrCreateDatasetFor(CProjects.SYSTEMD);
85+
collector = new BugCollector(dataset);
86+
Utils.saveBugDataset(collector.updateOrCreateBugDataset());
87+
System.out.println("End SystemD : " + (System.currentTimeMillis() - time));
8388
time = System.currentTimeMillis();
8489
System.out.println("Start Wireshark");
85-
Data7 dataset = updateOrCreateDatasetFor(CProjects.WIRESHARK);
86-
BugCollector collector = new BugCollector(dataset);
90+
dataset = updateOrCreateDatasetFor(CProjects.WIRESHARK);
91+
collector = new BugCollector(dataset);
8792
Utils.saveBugDataset(collector.updateOrCreateBugDataset());
8893
System.out.println("End Wireshark : " + (System.currentTimeMillis() - time));
8994
time = System.currentTimeMillis();
9095
System.out.println("Start SSL");
9196
dataset = updateOrCreateDatasetFor(CProjects.OPEN_SSL);
9297
collector = new BugCollector(dataset);
9398
Utils.saveBugDataset(collector.updateOrCreateBugDataset());
94-
System.out.println("End SSL : " + (System.currentTimeMillis() - time));
99+
System.out.println("End SSL : " + (System.currentTimeMillis() - time));*/
100+
BugDataset bd = loadBugDataset(CProjects.OPEN_SSL.getName());
101+
final int[] filefixes = {0};
102+
Map<String, Integer> files = new HashMap<>();
103+
for (Map.Entry<String, Commit> bug : bd.getDataset().entrySet()) {
104+
bug.getValue().getFixes().forEach(fileFix -> {
105+
files.merge(fileFix.getFileAfter().getFilePath(), 1, Integer::sum);
106+
filefixes[0]++;
107+
}
108+
);
109+
}
110+
111+
Map<String, Integer> topTenfiles =
112+
files.entrySet().stream()
113+
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
114+
.limit(12)
115+
.collect(Collectors.toMap(
116+
Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
117+
int i =0;
95118
}
96119
}

‎src/main/java/framevpm/bugcollector/Readme.md

+19-19
Original file line numberDiff line numberDiff line change
@@ -88,30 +88,30 @@ Utils.loadBugDataset(CProjects.WIRESHARK.getName());
8888

8989
As of 5th June 2018
9090

91-
| Projects | Time to create | Dataset size |Number of BugId | Number of Commit | Number of FileFix| Number of Unique Buggy Files |
92-
|:----------:|:--------------:|:------------:|:--------------:|:----------------:|:----------------:|:----------------------------:|
93-
|Linux Kernel|56 mn |481.8 MB | | |5.41 |5.34 |
94-
|Wireshark |166mn |1.96 GB | | |4.99 |5.01 |
95-
|OpenSSL |19 mn |576 MB | | |5.34 |5.42 |
96-
|SystemD |11 mn |393,7 MB | | |5.76 |5.60 |
97-
|Total | mn | GB | | |5.375 |5.34 |
91+
| Projects | Time to create | Dataset size | Number of Commit | Number of FileFix| Number of Unique Buggy Files |
92+
|:----------:|:--------------:|:------------:|:----------------:|:----------------:|:----------------------------:|
93+
|Linux Kernel| 56 mn | 481.8 MB | 3160 | 5193 | 2428 |
94+
|Wireshark | 166 mn | 1.96 GB | 3871 | 8019 | 1907 |
95+
|OpenSSL | 19 mn | 576 MB | 2442 | 7741 | 1733 |
96+
|SystemD | 11 mn | 393.7 MB | 1868 | 3538 | 925 |
97+
|Total | 252 mn | 3.4 GB | 11341 | 24491 | 6993 |
9898

9999

100100

101101
### Top 10 Buggy Files
102102

103-
| Rank | Linux | Wireshark | OpenSSL | SystemD |
104-
|:----:|:--------------------------|:---------------------------------------|:---------------------------|:---------------------------------------|
105-
| 1 | arch/x86/kvm/x86.c (17) | epan/proto.c (24) | ssl/s3_clnt.c (53) | src/resolve/resolved-dns-packet.c (2) |
106-
| 2 | fs/namei.c (14) | epan/dissectors/packet-nbap.c (19) | ssl/s3_srvr.c (41) | src/resolve/resolved-dns-packet.h (1) |
107-
| 3 | net/socket.c (14) | epan/dissectors/packet-ncp2222.inc (19)| ssl/d1_both.c (35) | src/util.c (1) |
108-
| 4 | kernel/signal.c (14) | epan/dissectors/packet-wccp.c (18) | ssl/t1_lib.c (34) | src/nss-mymachines/nss-mymachines.c (1)|
109-
| 5 | fs/namespace.c (14) | epan/crypt/airpdcap.c (18) | ssl/d1_pkt.c (22) | src/tmpfiles/tmpfiles.c (1) |
110-
| 6 | kernel/bpf/verifier.c (13)| epan/dissectors/packet-ber.c (17) | crypto/asn1/tasn_dec.c (20)| |
111-
| 7 | net/sctp/socket.c (13) | epan/dissectors/packet-umts_fp.c (16) | ssl/ssl.h (20) | |
112-
| 8 | fs/exec.c (13) | epan/dissectors/packet-wbxml.c (15) | ssl/s3_pkt.c (20) | |
113-
| 9 | arch/x86/kvm/vmx.c (13) | epan/dissectors/packet-wsp.c (14) | ssl/ssl_err.c (17) | |
114-
| 10 | fs/open.c (12) | epan/dissectors/packet-wcp.c (13) | crypto/x509/x509_vfy.c (16)| |
103+
| Rank | Linux | Wireshark | OpenSSL | SystemD |
104+
|:----:|:------------------------------------------|:-----------------------------------------|:---------------------|:---------------------------------|
105+
| 1 | drivers/acpi/ec.c (96) | epan/dissectors/packet-ssl-utils.c (116) | ssl/s3_srvr.c (96) | src/systemctl/systemctl.c (62) |
106+
| 2 | sound/pci/hda/patch_realtek.c(79) | ui/qt/main_window_slots.cpp (103) | ssl/s3_clnt.c (88) | src/network/networkd-link.c (52) |
107+
| 3 | drivers/acpi/video.c (57) | ui/qt/main_window.cpp (94) | ssl/ssl_lib.c (79) | Makefile.am (46) |
108+
| 4 | drivers/acpi/osl.c (36) | epan/dissectors/packet-ieee80211.c (91) | apps/s_server.c (78) | man/systemd.exec.xml (45) |
109+
| 5 | drivers/gpu/drm/i915/intel_display.c (33) |"epan/dissectors/packet-ssl-utils.h (76) | ssl/ssl.h (67) | meson.build (44) |
110+
| 6 | drivers/acpi/scan.c (31) | epan/dissectors/packet-tcp.c (66) | apps/s_client.c (66) | man/systemd.network.xml (43) |
111+
| 7 | drivers/acpi/sleep.c (30) | ui/qt/packet_list.cpp (63) | ssl/t1_lib.c (62) | src/nspawn/nspawn.c (43) |
112+
| 8 | drivers/acpi/processor_idle.c (28) | epan/dissectors/packet-ssl.c (61) | ssl/s3_lib.c (62) | hwdb/60-evdev.hwdb (37) |
113+
| 9 | drivers/pci/quirks.c (26) | epan/proto.c (55) | ssl/ssl_locl.h (61) | hwdb/60-sensor.hwdb (35) |
114+
| 10 | drivers/ata/libata-core.c (24) | epan/dissectors/packet-bgp.c (54) | ssl/s3_pkt.c (51) | src/core/manager.c (34) |
115115

116116

117117

‎src/main/java/framevpm/releasebalancer/Balancer.java

+32-3
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
package framevpm.releasebalancer;
22

3+
import data7.Exporter;
34
import data7.Importer;
45
import data7.model.Data7;
56
import data7.model.change.Commit;
67
import data7.model.vulnerability.Vulnerability;
8+
import data7.project.CProjects;
79
import data7.project.ProjectFactory;
10+
import framevpm.Utils;
811
import framevpm.bugcollector.BugCollector;
912
import framevpm.bugcollector.model.BugDataset;
1013
import framevpm.releasebalancer.model.*;
14+
import framevpm.releasebalancer.project.CProjectsInfo;
1115
import framevpm.releasebalancer.project.ProjectInfoFactory;
1216

1317
import java.io.IOException;
1418
import java.text.ParseException;
1519
import java.util.*;
1620

21+
import static data7.Importer.updateOrCreateDatasetFor;
22+
1723
/**
1824
*
1925
*/
@@ -41,7 +47,7 @@ public Balancer(String project) {
4147

4248
public ProjectData balance(boolean bugs) throws IOException, ClassNotFoundException, ParseException {
4349
System.out.println("Starting processing: " + project);
44-
Data7 data7 = Importer.updateOrCreateDatasetFor(ProjectFactory.retrieveProjectInfo(project));
50+
Data7 data7 = Exporter.loadDataset(project);
4551
prepareVuln(data7);
4652
if (bugs) {
4753
prepareBug(data7);
@@ -70,13 +76,13 @@ private void prepareVuln(Data7 data7) {
7076

7177
private void prepareBug(Data7 data7) throws IOException, ClassNotFoundException {
7278
final int[] count = {0};
73-
BugDataset bugDataset = new BugCollector(data7).updateOrCreateBugDataset();
79+
BugDataset bugDataset = Utils.loadBugDataset(data7.getProject().getName());
7480
Set<Map.Entry<String, Commit>> bugds = bugDataset.getDataset().entrySet();
7581

7682
for (Map.Entry<String, Commit> bug : bugds) {
7783
if (!vulnerabilityHash.contains(bug.getKey()) && commitMessageFiltering.add(bug.getValue().getMessage().toLowerCase())) {
7884
Map.Entry<Long, String> release = releases.floorEntry(bug.getValue().getTimestamp());
79-
if (release != null) {
85+
if (releases.size() > 0) {
8086
String rel = release.getValue();
8187
ReleaseData releaseData = projectData.getOrCreateRelease(rel);
8288
bug.getValue().getFixes().forEach(fileFix -> {
@@ -260,4 +266,27 @@ private List<String> lookForCorrespondingRelease(long timestamp, Set<String> ver
260266
}
261267

262268

269+
public static void main(String[] args) throws ParseException, IOException, ClassNotFoundException {
270+
long time = System.currentTimeMillis();
271+
System.out.println("Start Linux");
272+
Balancer balancer = new Balancer(CProjects.LINUX_KERNEL.getName());
273+
Utils.saveProjectData(balancer.balance(true));
274+
System.out.println("End Linux : " + (System.currentTimeMillis() - time));
275+
time = System.currentTimeMillis();
276+
System.out.println("Start SystemD");
277+
balancer = new Balancer(CProjects.SYSTEMD.getName());
278+
Utils.saveProjectData(balancer.balance(true));
279+
System.out.println("End SystemD : " + (System.currentTimeMillis() - time));
280+
time = System.currentTimeMillis();
281+
System.out.println("Start Wireshark");
282+
balancer = new Balancer(CProjects.WIRESHARK.getName());
283+
Utils.saveProjectData(balancer.balance(true));
284+
System.out.println("End Wireshark : " + (System.currentTimeMillis() - time));
285+
time = System.currentTimeMillis();
286+
System.out.println("Start SSL");
287+
balancer = new Balancer(CProjects.OPEN_SSL.getName());
288+
Utils.saveProjectData(balancer.balance(true));
289+
System.out.println("End SSL : " + (System.currentTimeMillis() - time));
290+
}
291+
263292
}

‎src/main/java/framevpm/releasebalancer/Readme.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44

55
## What is it?
66

7-
The balancer package is the first mandatory package that have to be use in the frameVPM framework. It splits the differents vulnerability fixes (and bugs if chosen so) according to the release of the software they affected.
7+
The balancer package is the first mandatory package in the frameVPM framework.
8+
Its role is to splits the different vulnerability fixes according to the release they affected and is as well use to prune the fixes to only keep the relevant ones.
89

910
## Why?
1011

@@ -17,7 +18,7 @@ Maven will handle the dependancy.
1718

1819
## How does it work ?
1920

20-
21+
### heuristics
2122

2223
## Dataset Structure
2324

0 commit comments

Comments
 (0)