Skip to content

Commit a53d736

Browse files
thiagotnunesthiagotnunes
authored
fix: retry pdml transaction on EOS internal error (#360)
* fix: retries PDML transactions on EOS errors It is possible to have the stream closed with an EOS internal error. This should be retried by the client.
1 parent 6125c7d commit a53d736

7 files changed

+357
-43
lines changed

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/IsRetryableInternalError.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.api.gax.rpc.InternalException;
20+
import com.google.common.base.Predicate;
21+
import io.grpc.Status;
22+
import io.grpc.StatusRuntimeException;
23+
24+
public class IsRetryableInternalError implements Predicate<Throwable> {
25+
26+
private static final String HTTP2_ERROR_MESSAGE = "HTTP/2 error code: INTERNAL_ERROR";
27+
private static final String CONNECTION_CLOSED_ERROR_MESSAGE =
28+
"Connection closed with unknown cause";
29+
private static final String EOS_ERROR_MESSAGE =
30+
"Received unexpected EOS on DATA frame from server";
31+
32+
@Override
33+
public boolean apply(Throwable cause) {
34+
if (isInternalError(cause)) {
35+
if (cause.getMessage().contains(HTTP2_ERROR_MESSAGE)) {
36+
// See b/25451313.
37+
return true;
38+
} else if (cause.getMessage().contains(CONNECTION_CLOSED_ERROR_MESSAGE)) {
39+
// See b/27794742.
40+
return true;
41+
} else if (cause.getMessage().contains(EOS_ERROR_MESSAGE)) {
42+
return true;
43+
}
44+
}
45+
return false;
46+
}
47+
48+
private boolean isInternalError(Throwable cause) {
49+
return (cause instanceof InternalException)
50+
|| (cause instanceof StatusRuntimeException
51+
&& ((StatusRuntimeException) cause).getStatus().getCode() == Status.Code.INTERNAL);
52+
}
53+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/IsSslHandshakeException.java

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.common.base.Predicate;
20+
import javax.net.ssl.SSLHandshakeException;
21+
22+
public class IsSslHandshakeException implements Predicate<Throwable> {
23+
24+
@Override
25+
public boolean apply(Throwable input) {
26+
return input instanceof SSLHandshakeException;
27+
}
28+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/PartitionedDmlTransaction.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2019 Google LLC
2+
* Copyright 2020 Google LLC
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
2121
import com.google.api.gax.grpc.GrpcStatusCode;
2222
import com.google.api.gax.rpc.AbortedException;
2323
import com.google.api.gax.rpc.DeadlineExceededException;
24+
import com.google.api.gax.rpc.InternalException;
2425
import com.google.api.gax.rpc.ServerStream;
2526
import com.google.api.gax.rpc.UnavailableException;
2627
import com.google.cloud.spanner.spi.v1.SpannerRpc;
@@ -43,17 +44,20 @@
4344
import org.threeten.bp.temporal.ChronoUnit;
4445

4546
public class PartitionedDmlTransaction implements SessionImpl.SessionTransaction {
47+
4648
private static final Logger LOGGER = Logger.getLogger(PartitionedDmlTransaction.class.getName());
4749

4850
private final SessionImpl session;
4951
private final SpannerRpc rpc;
5052
private final Ticker ticker;
53+
private final IsRetryableInternalError isRetryableInternalErrorPredicate;
5154
private volatile boolean isValid = true;
5255

5356
PartitionedDmlTransaction(SessionImpl session, SpannerRpc rpc, Ticker ticker) {
5457
this.session = session;
5558
this.rpc = rpc;
5659
this.ticker = ticker;
60+
this.isRetryableInternalErrorPredicate = new IsRetryableInternalError();
5761
}
5862

5963
/**
@@ -95,6 +99,14 @@ long executeStreamingPartitionedUpdate(final Statement statement, final Duration
9599
LOGGER.log(
96100
Level.FINER, "Retrying PartitionedDml transaction after UnavailableException", e);
97101
request = resumeOrRestartRequest(resumeToken, statement, request);
102+
} catch (InternalException e) {
103+
if (!isRetryableInternalErrorPredicate.apply(e)) {
104+
throw e;
105+
}
106+
107+
LOGGER.log(
108+
Level.FINER, "Retrying PartitionedDml transaction after InternalException - EOS", e);
109+
request = resumeOrRestartRequest(resumeToken, statement, request);
98110
} catch (AbortedException e) {
99111
LOGGER.log(Level.FINER, "Retrying PartitionedDml transaction after AbortedException", e);
100112
resumeToken = ByteString.EMPTY;

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java

+4-32
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,10 @@
2626
import io.grpc.Context;
2727
import io.grpc.Metadata;
2828
import io.grpc.Status;
29-
import io.grpc.StatusRuntimeException;
3029
import io.grpc.protobuf.ProtoUtils;
3130
import java.util.concurrent.CancellationException;
3231
import java.util.concurrent.TimeoutException;
3332
import javax.annotation.Nullable;
34-
import javax.net.ssl.SSLHandshakeException;
3533

3634
/**
3735
* A factory for creating instances of {@link SpannerException} and its subtypes. All creation of
@@ -40,6 +38,7 @@
4038
* ErrorCode#ABORTED} are always represented by {@link AbortedException}.
4139
*/
4240
public final class SpannerExceptionFactory {
41+
4342
static final String SESSION_RESOURCE_TYPE = "type.googleapis.com/google.spanner.v1.Session";
4443
static final String DATABASE_RESOURCE_TYPE =
4544
"type.googleapis.com/google.spanner.admin.database.v1.Database";
@@ -257,35 +256,8 @@ private static boolean hasCauseMatching(
257256
}
258257

259258
private static class Matchers {
260-
static final Predicate<Throwable> isRetryableInternalError =
261-
new Predicate<Throwable>() {
262-
@Override
263-
public boolean apply(Throwable cause) {
264-
if (cause instanceof StatusRuntimeException
265-
&& ((StatusRuntimeException) cause).getStatus().getCode() == Status.Code.INTERNAL) {
266-
if (cause.getMessage().contains("HTTP/2 error code: INTERNAL_ERROR")) {
267-
// See b/25451313.
268-
return true;
269-
}
270-
if (cause.getMessage().contains("Connection closed with unknown cause")) {
271-
// See b/27794742.
272-
return true;
273-
}
274-
if (cause
275-
.getMessage()
276-
.contains("Received unexpected EOS on DATA frame from server")) {
277-
return true;
278-
}
279-
}
280-
return false;
281-
}
282-
};
283-
static final Predicate<Throwable> isSSLHandshakeException =
284-
new Predicate<Throwable>() {
285-
@Override
286-
public boolean apply(Throwable input) {
287-
return input instanceof SSLHandshakeException;
288-
}
289-
};
259+
260+
static final Predicate<Throwable> isRetryableInternalError = new IsRetryableInternalError();
261+
static final Predicate<Throwable> isSSLHandshakeException = new IsSslHandshakeException();
290262
}
291263
}

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/IsRetryableInternalErrorTest.java

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
21+
import com.google.api.gax.grpc.GrpcStatusCode;
22+
import com.google.api.gax.rpc.InternalException;
23+
import com.google.common.base.Predicate;
24+
import io.grpc.Status;
25+
import io.grpc.Status.Code;
26+
import io.grpc.StatusRuntimeException;
27+
import org.junit.Before;
28+
import org.junit.Test;
29+
import org.junit.runner.RunWith;
30+
import org.junit.runners.JUnit4;
31+
32+
@SuppressWarnings("unchecked")
33+
@RunWith(JUnit4.class)
34+
public class IsRetryableInternalErrorTest {
35+
36+
private Predicate<Throwable> predicate;
37+
38+
@Before
39+
public void setUp() {
40+
predicate = new IsRetryableInternalError();
41+
}
42+
43+
@Test
44+
public void http2ErrorStatusRuntimeExceptionIsRetryable() {
45+
final StatusRuntimeException e =
46+
new StatusRuntimeException(
47+
Status.fromCode(Code.INTERNAL)
48+
.withDescription("INTERNAL: HTTP/2 error code: INTERNAL_ERROR."));
49+
50+
assertThat(predicate.apply(e)).isTrue();
51+
}
52+
53+
@Test
54+
public void http2ErrorInternalExceptionIsRetryable() {
55+
final InternalException e =
56+
new InternalException(
57+
"INTERNAL: HTTP/2 error code: INTERNAL_ERROR.",
58+
null,
59+
GrpcStatusCode.of(Code.INTERNAL),
60+
false);
61+
62+
assertThat(predicate.apply(e)).isTrue();
63+
}
64+
65+
@Test
66+
public void connectionClosedStatusRuntimeExceptionIsRetryable() {
67+
final StatusRuntimeException e =
68+
new StatusRuntimeException(
69+
Status.fromCode(Code.INTERNAL)
70+
.withDescription("INTERNAL: Connection closed with unknown cause."));
71+
72+
assertThat(predicate.apply(e)).isTrue();
73+
}
74+
75+
@Test
76+
public void connectionClosedInternalExceptionIsRetryable() {
77+
final InternalException e =
78+
new InternalException(
79+
"INTERNAL: Connection closed with unknown cause.",
80+
null,
81+
GrpcStatusCode.of(Code.INTERNAL),
82+
false);
83+
84+
assertThat(predicate.apply(e)).isTrue();
85+
}
86+
87+
@Test
88+
public void eosStatusRuntimeExceptionIsRetryable() {
89+
final StatusRuntimeException e =
90+
new StatusRuntimeException(
91+
Status.fromCode(Code.INTERNAL)
92+
.withDescription("INTERNAL: Received unexpected EOS on DATA frame from server."));
93+
94+
assertThat(predicate.apply(e)).isTrue();
95+
}
96+
97+
@Test
98+
public void eosInternalExceptionIsRetryable() {
99+
final InternalException e =
100+
new InternalException(
101+
"INTERNAL: Received unexpected EOS on DATA frame from server.",
102+
null,
103+
GrpcStatusCode.of(Code.INTERNAL),
104+
false);
105+
106+
assertThat(predicate.apply(e)).isTrue();
107+
}
108+
109+
@Test
110+
public void genericInternalStatusRuntimeExceptionIsRetryable() {
111+
final StatusRuntimeException e =
112+
new StatusRuntimeException(
113+
Status.fromCode(Code.INTERNAL).withDescription("INTERNAL: Generic."));
114+
115+
assertThat(predicate.apply(e)).isFalse();
116+
}
117+
118+
@Test
119+
public void genericInternalExceptionIsNotRetryable() {
120+
final InternalException e =
121+
new InternalException("INTERNAL: Generic.", null, GrpcStatusCode.of(Code.INTERNAL), false);
122+
123+
assertThat(predicate.apply(e)).isFalse();
124+
}
125+
126+
@Test
127+
public void nullIsNotRetryable() {
128+
assertThat(predicate.apply(null)).isFalse();
129+
}
130+
131+
@Test
132+
public void genericExceptionIsNotRetryable() {
133+
assertThat(predicate.apply(new Exception())).isFalse();
134+
}
135+
}

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/IsSslHandshakeExceptionTest.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
21+
import com.google.common.base.Predicate;
22+
import javax.net.ssl.SSLHandshakeException;
23+
import org.junit.Before;
24+
import org.junit.Test;
25+
import org.junit.runner.RunWith;
26+
import org.junit.runners.JUnit4;
27+
28+
@SuppressWarnings("unchecked")
29+
@RunWith(JUnit4.class)
30+
public class IsSslHandshakeExceptionTest {
31+
32+
private Predicate<Throwable> predicate;
33+
34+
@Before
35+
public void setUp() {
36+
predicate = new IsSslHandshakeException();
37+
}
38+
39+
@Test
40+
public void sslHandshakeExceptionIsTrue() {
41+
assertThat(predicate.apply(new SSLHandshakeException("test"))).isTrue();
42+
}
43+
44+
@Test
45+
public void genericExceptionIsNotSslHandshakeException() {
46+
assertThat(predicate.apply(new Exception("test"))).isFalse();
47+
}
48+
49+
@Test
50+
public void nullIsNotSslHandshakeException() {
51+
assertThat(predicate.apply(null)).isFalse();
52+
}
53+
}

0 commit comments

Comments
 (0)