Skip to content

Commit 8cf4387

Browse files
o-shevchenkoo-shevchenko
authored
fix(test): Update schema for broken ConnImplBenchmark test (#3574)
* Fix ConnImplBenchmark test * refactor
1 parent 31fb15f commit 8cf4387

File tree

1 file changed

+81
-125
lines changed

1 file changed

+81
-125
lines changed

‎benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java

+81-125
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.sql.ResultSet;
2121
import java.sql.SQLException;
2222
import java.util.concurrent.TimeUnit;
23+
import java.util.function.Function;
2324
import java.util.logging.Level;
2425
import org.openjdk.jmh.annotations.Benchmark;
2526
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -48,31 +49,19 @@ public class ConnImplBenchmark {
4849
public int rowLimit;
4950

5051
private ConnectionSettings connectionSettingsReadAPIEnabled, connectionSettingsReadAPIDisabled;
51-
private long numBuffRows = 100000L;
52-
private final String DATASET = "new_york_taxi_trips";
5352
private final String QUERY =
5453
"SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s";
55-
public static final long NUM_PAGE_ROW_CNT_RATIO =
56-
10; // ratio of [records in the current page :: total rows] to be met to use read API
57-
public static final long NUM_MIN_RESULT_SIZE =
58-
200000; // min number of records to use to ReadAPI with
5954

6055
@Setup
6156
public void setUp() throws IOException {
6257
java.util.logging.Logger.getGlobal().setLevel(Level.ALL);
6358

64-
connectionSettingsReadAPIEnabled =
65-
ConnectionSettings.newBuilder()
66-
.setUseReadAPI(true) // enable read api
67-
.build();
59+
connectionSettingsReadAPIEnabled = ConnectionSettings.newBuilder().setUseReadAPI(true).build();
6860
connectionSettingsReadAPIDisabled =
69-
ConnectionSettings.newBuilder()
70-
.setUseReadAPI(false) // disable read api
71-
.build();
61+
ConnectionSettings.newBuilder().setUseReadAPI(false).build();
7262
}
7363

7464
@Benchmark
75-
// uses bigquery.query
7665
public void iterateRecordsWithBigQuery_Query(Blackhole blackhole) throws InterruptedException {
7766
String selectQuery = String.format(QUERY, rowLimit);
7867
BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
@@ -81,81 +70,35 @@ public void iterateRecordsWithBigQuery_Query(Blackhole blackhole) throws Interru
8170
TableResult result = bigQuery.query(config);
8271
long hash = 0L;
8372
int cnt = 0;
84-
System.out.print("\n Running");
85-
// iterate al the records and compute the hash
73+
long lastTime = System.currentTimeMillis();
74+
System.out.println("\n Running");
8675
for (FieldValueList row : result.iterateAll()) {
87-
hash +=
88-
row.get("vendor_id").getStringValue() == null
89-
? 0
90-
: row.get("vendor_id").getStringValue().hashCode();
91-
hash +=
92-
row.get("pickup_datetime").getStringValue() == null
93-
? 0
94-
: row.get("pickup_datetime").getStringValue().hashCode();
95-
hash +=
96-
row.get("dropoff_datetime").getStringValue() == null
97-
? 0
98-
: row.get("dropoff_datetime").getStringValue().hashCode();
99-
hash +=
100-
row.get("passenger_count").getValue() == null
101-
? 0
102-
: row.get("passenger_count").getLongValue();
103-
hash +=
104-
row.get("trip_distance").getValue() == null
105-
? 0
106-
: row.get("trip_distance").getDoubleValue();
107-
hash +=
108-
row.get("pickup_longitude").getValue() == null
109-
? 0
110-
: row.get("pickup_longitude").getDoubleValue();
111-
hash +=
112-
row.get("pickup_latitude").getValue() == null
113-
? 0
114-
: row.get("pickup_latitude").getDoubleValue();
115-
hash +=
116-
row.get("rate_code").getStringValue() == null
117-
? 0
118-
: row.get("rate_code").getStringValue().hashCode();
119-
hash +=
120-
row.get("store_and_fwd_flag").getStringValue() == null
121-
? 0
122-
: row.get("store_and_fwd_flag").getStringValue().hashCode();
123-
hash +=
124-
row.get("payment_type").getStringValue() == null
125-
? 0
126-
: row.get("payment_type").getStringValue().hashCode();
127-
hash +=
128-
row.get("pickup_location_id").getStringValue() == null
129-
? 0
130-
: row.get("pickup_location_id").getStringValue().hashCode();
131-
hash +=
132-
row.get("dropoff_location_id").getStringValue() == null
133-
? 0
134-
: row.get("dropoff_location_id").getStringValue().hashCode();
135-
hash +=
136-
row.get("dropoff_longitude").getValue() == null
137-
? 0
138-
: row.get("dropoff_longitude").getDoubleValue();
139-
hash +=
140-
row.get("dropoff_latitude").getValue() == null
141-
? 0
142-
: row.get("dropoff_latitude").getDoubleValue();
143-
hash +=
144-
row.get("fare_amount").getValue() == null ? 0 : row.get("fare_amount").getDoubleValue();
145-
hash += row.get("extra").getValue() == null ? 0 : row.get("extra").getDoubleValue();
146-
hash += row.get("mta_tax").getValue() == null ? 0 : row.get("mta_tax").getDoubleValue();
147-
hash += row.get("tip_amount").getValue() == null ? 0 : row.get("tip_amount").getDoubleValue();
148-
hash +=
149-
row.get("tolls_amount").getValue() == null ? 0 : row.get("tolls_amount").getDoubleValue();
150-
hash +=
151-
row.get("imp_surcharge").getValue() == null
152-
? 0
153-
: row.get("imp_surcharge").getDoubleValue();
154-
hash +=
155-
row.get("total_amount").getValue() == null ? 0 : row.get("total_amount").getDoubleValue();
76+
hash += computeHash(row.get("vendor_id"), FieldValue::getStringValue);
77+
hash += computeHash(row.get("pickup_datetime"), FieldValue::getStringValue);
78+
hash += computeHash(row.get("dropoff_datetime"), FieldValue::getStringValue);
79+
hash += computeHash(row.get("passenger_count"), FieldValue::getLongValue);
80+
hash += computeHash(row.get("trip_distance"), FieldValue::getDoubleValue);
81+
hash += computeHash(row.get("rate_code"), FieldValue::getStringValue);
82+
hash += computeHash(row.get("store_and_fwd_flag"), FieldValue::getStringValue);
83+
hash += computeHash(row.get("payment_type"), FieldValue::getStringValue);
84+
hash += computeHash(row.get("fare_amount"), FieldValue::getDoubleValue);
85+
hash += computeHash(row.get("extra"), FieldValue::getDoubleValue);
86+
hash += computeHash(row.get("mta_tax"), FieldValue::getDoubleValue);
87+
hash += computeHash(row.get("tip_amount"), FieldValue::getDoubleValue);
88+
hash += computeHash(row.get("tolls_amount"), FieldValue::getDoubleValue);
89+
hash += computeHash(row.get("imp_surcharge"), FieldValue::getDoubleValue);
90+
hash += computeHash(row.get("airport_fee"), FieldValue::getDoubleValue);
91+
hash += computeHash(row.get("total_amount"), FieldValue::getDoubleValue);
92+
hash += computeHash(row.get("pickup_location_id"), FieldValue::getStringValue);
93+
hash += computeHash(row.get("dropoff_location_id"), FieldValue::getStringValue);
94+
hash += computeHash(row.get("data_file_year"), FieldValue::getLongValue);
95+
hash += computeHash(row.get("data_file_month"), FieldValue::getLongValue);
15696

157-
if (++cnt % 100000 == 0) { // just to indicate the progress while long running benchmarks
158-
System.out.print(".");
97+
if (++cnt % 100_000 == 0) {
98+
long now = System.currentTimeMillis();
99+
long duration = now - lastTime;
100+
System.out.println("ROW " + cnt + " Time: " + duration + " ms");
101+
lastTime = now;
159102
}
160103
}
161104
System.out.println(cnt + " records processed using bigquery.query");
@@ -202,54 +145,67 @@ public void iterateRecordsWithoutUsingReadAPI(Blackhole blackhole)
202145
blackhole.consume(hash);
203146
}
204147

205-
// Hashes all the 20 columns of all the rows
206148
private long getResultHash(BigQueryResult bigQueryResultSet) throws SQLException {
207149
ResultSet rs = bigQueryResultSet.getResultSet();
208150
long hash = 0L;
209151
int cnt = 0;
210-
System.out.print("\n Running");
152+
long lastTime = System.currentTimeMillis();
153+
System.out.println("\n Running");
211154
while (rs.next()) {
212-
hash += rs.getString("vendor_id") == null ? 0 : rs.getString("vendor_id").hashCode();
213-
hash +=
214-
rs.getString("pickup_datetime") == null ? 0 : rs.getString("pickup_datetime").hashCode();
215-
hash +=
216-
rs.getString("dropoff_datetime") == null
217-
? 0
218-
: rs.getString("dropoff_datetime").hashCode();
219-
hash += rs.getLong("passenger_count");
220-
hash += rs.getDouble("trip_distance");
221-
hash += rs.getDouble("pickup_longitude");
222-
hash += rs.getDouble("pickup_latitude");
223-
hash += rs.getString("rate_code") == null ? 0 : rs.getString("rate_code").hashCode();
224-
hash +=
225-
rs.getString("store_and_fwd_flag") == null
226-
? 0
227-
: rs.getString("store_and_fwd_flag").hashCode();
228-
hash += rs.getDouble("dropoff_longitude");
229-
hash += rs.getDouble("dropoff_latitude");
230-
hash += rs.getString("payment_type") == null ? 0 : rs.getString("payment_type").hashCode();
231-
hash += rs.getDouble("fare_amount");
232-
hash += rs.getDouble("extra");
233-
hash += rs.getDouble("mta_tax");
234-
hash += rs.getDouble("tip_amount");
235-
hash += rs.getDouble("tolls_amount");
236-
hash += rs.getDouble("imp_surcharge");
237-
hash += rs.getDouble("total_amount");
238-
hash +=
239-
rs.getString("pickup_location_id") == null
240-
? 0
241-
: rs.getString("pickup_location_id").hashCode();
242-
hash +=
243-
rs.getString("dropoff_location_id") == null
244-
? 0
245-
: rs.getString("dropoff_location_id").hashCode();
246-
if (++cnt % 100000 == 0) { // just to indicate the progress while long running benchmarks
247-
System.out.print(".");
155+
hash += computeHash(rs, "vendor_id", ResultSet::getString);
156+
hash += computeHash(rs, "pickup_datetime", ResultSet::getString);
157+
hash += computeHash(rs, "dropoff_datetime", ResultSet::getString);
158+
hash += computeHash(rs, "passenger_count", ResultSet::getLong);
159+
hash += computeHash(rs, "trip_distance", ResultSet::getDouble);
160+
hash += computeHash(rs, "rate_code", ResultSet::getString);
161+
hash += computeHash(rs, "store_and_fwd_flag", ResultSet::getString);
162+
hash += computeHash(rs, "payment_type", ResultSet::getString);
163+
hash += computeHash(rs, "fare_amount", ResultSet::getDouble);
164+
hash += computeHash(rs, "extra", ResultSet::getDouble);
165+
hash += computeHash(rs, "mta_tax", ResultSet::getDouble);
166+
hash += computeHash(rs, "tip_amount", ResultSet::getDouble);
167+
hash += computeHash(rs, "tolls_amount", ResultSet::getDouble);
168+
hash += computeHash(rs, "imp_surcharge", ResultSet::getDouble);
169+
hash += computeHash(rs, "airport_fee", ResultSet::getDouble);
170+
hash += computeHash(rs, "total_amount", ResultSet::getDouble);
171+
hash += computeHash(rs, "pickup_location_id", ResultSet::getString);
172+
hash += computeHash(rs, "dropoff_location_id", ResultSet::getString);
173+
hash += computeHash(rs, "data_file_year", ResultSet::getLong);
174+
hash += computeHash(rs, "data_file_month", ResultSet::getLong);
175+
176+
if (++cnt % 100_000 == 0) {
177+
long now = System.currentTimeMillis();
178+
long duration = now - lastTime;
179+
System.out.println("ROW " + cnt + " Time: " + duration + " ms");
180+
lastTime = now;
248181
}
249182
}
250183
return hash;
251184
}
252185

186+
private <T> long computeHash(
187+
ResultSet rs, String columnName, SQLFunction<ResultSet, T> extractor) {
188+
try {
189+
T value = extractor.apply(rs, columnName);
190+
return (value == null) ? 0 : value.hashCode();
191+
} catch (SQLException e) {
192+
return 0;
193+
}
194+
}
195+
196+
@FunctionalInterface
197+
private interface SQLFunction<T, R> {
198+
R apply(T t, String columnName) throws SQLException;
199+
}
200+
201+
private <T> long computeHash(FieldValue fieldValue, Function<FieldValue, T> extractor) {
202+
if (fieldValue == null || fieldValue.isNull()) {
203+
return 0;
204+
}
205+
T value = extractor.apply(fieldValue);
206+
return (value == null) ? 0 : value.hashCode();
207+
}
208+
253209
public static void main(String[] args) throws Exception {
254210
Options opt = new OptionsBuilder().include(ConnImplBenchmark.class.getSimpleName()).build();
255211
new Runner(opt).run();

0 commit comments

Comments
 (0)