Skip to content

Commit

Permalink
Merge pull request #76 from benjchristensen/properties
Browse files Browse the repository at this point in the history 
New Properties (fallbacks and rolling percentile)
  • Loading branch information
@benjchristensen
benjchristensen committed Jan 7, 2013
2 parents e723beb + 4f4262d commit 950430d
Show file tree
Hide file tree
Showing 4 changed files with 182 additions and 46 deletions.
128 changes: 97 additions & 31 deletions hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1095,40 +1095,59 @@ private R getFallbackOrThrowException(HystrixEventType eventType, FailureType fa
*/
private R getFallbackOrThrowException(HystrixEventType eventType, FailureType failureType, String message, Exception e) {
try {
// retrieve the fallback
R fallback = getFallbackWithProtection();
// mark fallback on counter
metrics.markFallbackSuccess();
// record the executionResult
executionResult = executionResult.addEvents(eventType, HystrixEventType.FALLBACK_SUCCESS);
return executionHook.onComplete(this, fallback);
} catch (UnsupportedOperationException fe) {
logger.debug("No fallback for HystrixCommand. ", fe); // debug only since we're throwing the exception and someone higher will do something with it
// record the executionResult
executionResult = executionResult.addEvents(eventType);

/* executionHook for all errors */
try {
e = executionHook.onError(this, failureType, e);
} catch (Exception hookException) {
logger.warn("Error calling ExecutionHook.onError", hookException);
}
if (properties.fallbackEnabled().get()) {
/* fallback behavior is permitted so attempt */
try {
// retrieve the fallback
R fallback = getFallbackWithProtection();
// mark fallback on counter
metrics.markFallbackSuccess();
// record the executionResult
executionResult = executionResult.addEvents(eventType, HystrixEventType.FALLBACK_SUCCESS);
return executionHook.onComplete(this, fallback);
} catch (UnsupportedOperationException fe) {
logger.debug("No fallback for HystrixCommand. ", fe); // debug only since we're throwing the exception and someone higher will do something with it
// record the executionResult
executionResult = executionResult.addEvents(eventType);

/* executionHook for all errors */
try {
e = executionHook.onError(this, failureType, e);
} catch (Exception hookException) {
logger.warn("Error calling ExecutionHook.onError", hookException);
}

throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and no fallback available.", e, fe);
} catch (Exception fe) {
logger.error("Error retrieving fallback for HystrixCommand. ", fe);
metrics.markFallbackFailure();
// record the executionResult
executionResult = executionResult.addEvents(eventType, HystrixEventType.FALLBACK_FAILURE);
throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and no fallback available.", e, fe);
} catch (Exception fe) {
logger.error("Error retrieving fallback for HystrixCommand. ", fe);
metrics.markFallbackFailure();
// record the executionResult
executionResult = executionResult.addEvents(eventType, HystrixEventType.FALLBACK_FAILURE);

/* executionHook for all errors */
try {
e = executionHook.onError(this, failureType, e);
} catch (Exception hookException) {
logger.warn("Error calling ExecutionHook.onError", hookException);
}
/* executionHook for all errors */
try {
e = executionHook.onError(this, failureType, e);
} catch (Exception hookException) {
logger.warn("Error calling ExecutionHook.onError", hookException);
}

throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and failed retrieving fallback.", e, fe);
throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and failed retrieving fallback.", e, fe);
}
} else {
/* fallback is disabled so throw HystrixRuntimeException */

logger.debug("Fallback disabled for HystrixCommand so will throw HystrixRuntimeException. ", e); // debug only since we're throwing the exception and someone higher will do something with it
// record the executionResult
executionResult = executionResult.addEvents(eventType);

/* executionHook for all errors */
try {
e = executionHook.onError(this, failureType, e);
} catch (Exception hookException) {
logger.warn("Error calling ExecutionHook.onError", hookException);
}
throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and fallback disabled.", e, null);
}
} finally {
// record that we're completed (to handle non-successful events we do it here as well as at the end of executeCommand
isExecutionComplete.set(true);
Expand Down Expand Up @@ -5046,6 +5065,48 @@ public void testExecutionHookFailureWithSemaphoreIsolation() {
assertEquals(0, command.builder.executionHook.threadComplete.get());
}

/**
* Test a command execution that fails but has a fallback.
*/
@Test
public void testExecutionFailureWithFallbackImplementedButDisabled() {
TestHystrixCommand<Boolean> commandEnabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), true);
try {
assertEquals(false, commandEnabled.execute());
} catch (Exception e) {
e.printStackTrace();
fail("We should have received a response from the fallback.");
}

TestHystrixCommand<Boolean> commandDisabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), false);
try {
assertEquals(false, commandDisabled.execute());
fail("expect exception thrown");
} catch (Exception e) {
// expected
}

assertEquals("we failed with a simulated issue", commandDisabled.getFailedExecutionException().getMessage());

assertTrue(commandDisabled.isFailedExecution());

assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS));
assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN));
assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT));
assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));

assertEquals(100, commandDisabled.builder.metrics.getHealthCounts().getErrorPercentage());

assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size());
}

/* ******************************************************************************** */
/* ******************************************************************************** */
/* private HystrixCommand class implementations for unit testing */
Expand Down Expand Up @@ -5236,6 +5297,11 @@ public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker) {
super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics));
}

public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker, boolean fallbackEnabled) {
super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)
.setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withFallbackEnabled(fallbackEnabled)));
}

@Override
protected Boolean run() {
System.out.println("*** simulated failed execution ***");
Expand Down
12 changes: 4 additions & 8 deletions hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ public static Collection<HystrixCommandMetrics> getInstances() {
this.group = commandGroup;
this.properties = properties;
this.counter = new HystrixRollingNumber(properties.metricsRollingStatisticalWindowInMilliseconds(), properties.metricsRollingStatisticalWindowBuckets());
this.percentileExecution = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize());
this.percentileTotal = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize());
this.percentileExecution = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize(), properties.metricsRollingPercentileEnabled());
this.percentileTotal = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize(), properties.metricsRollingPercentileEnabled());
this.eventNotifier = eventNotifier;
}

Expand Down Expand Up @@ -364,9 +364,7 @@ public int getCurrentConcurrentExecutionCount() {
* Execution time of {@link HystrixCommand#run()}.
*/
/* package */void addCommandExecutionTime(long duration) {
if (properties.metricsRollingPercentileEnabled().get()) {
percentileExecution.addValue((int) duration);
}
percentileExecution.addValue((int) duration);
}

/**
Expand All @@ -376,9 +374,7 @@ public int getCurrentConcurrentExecutionCount() {
* This differs from {@link #addCommandExecutionTime} in that this covers all of the threading and scheduling overhead, not just the execution of the {@link HystrixCommand#run()} method.
*/
/* package */void addUserThreadExecutionTime(long duration) {
if (properties.metricsRollingPercentileEnabled().get()) {
percentileTotal.addValue((int) duration);
}
percentileTotal.addValue((int) duration);
}

private volatile HealthCounts healthCountsSnapshot = new HealthCounts(0, 0, 0);
Expand Down
30 changes: 30 additions & 0 deletions hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public abstract class HystrixCommandProperties {
private static final Boolean default_metricsRollingPercentileEnabled = true;
private static final Boolean default_requestCacheEnabled = true;
private static final Integer default_fallbackIsolationSemaphoreMaxConcurrentRequests = 10;
private static final Boolean default_fallbackEnabled = true;
private static final Integer default_executionIsolationSemaphoreMaxConcurrentRequests = 10;
private static final Boolean default_requestLogEnabled = true;
private static final Boolean default_circuitBreakerEnabled = true;
Expand All @@ -77,6 +78,7 @@ public abstract class HystrixCommandProperties {
private final HystrixProperty<String> executionIsolationThreadPoolKeyOverride; // What thread-pool this command should run in (if running on a separate thread).
private final HystrixProperty<Integer> executionIsolationSemaphoreMaxConcurrentRequests; // Number of permits for execution semaphore
private final HystrixProperty<Integer> fallbackIsolationSemaphoreMaxConcurrentRequests; // Number of permits for fallback semaphore
private final HystrixProperty<Boolean> fallbackEnabled; // Whether fallback should be attempted.
private final HystrixProperty<Boolean> executionIsolationThreadInterruptOnTimeout; // Whether an underlying Future/Thread (when runInSeparateThread == true) should be interrupted after a timeout
private final HystrixProperty<Integer> metricsRollingStatisticalWindowInMilliseconds; // milliseconds back that will be tracked
private final HystrixProperty<Integer> metricsRollingStatisticalWindowBuckets; // number of buckets in the statisticalWindow
Expand Down Expand Up @@ -122,6 +124,7 @@ protected HystrixCommandProperties(HystrixCommandKey key, HystrixCommandProperti
this.executionIsolationThreadInterruptOnTimeout = getProperty(propertyPrefix, key, "execution.isolation.thread.interruptOnTimeout", builder.getExecutionIsolationThreadInterruptOnTimeout(), default_executionIsolationThreadInterruptOnTimeout);
this.executionIsolationSemaphoreMaxConcurrentRequests = getProperty(propertyPrefix, key, "execution.isolation.semaphore.maxConcurrentRequests", builder.getExecutionIsolationSemaphoreMaxConcurrentRequests(), default_executionIsolationSemaphoreMaxConcurrentRequests);
this.fallbackIsolationSemaphoreMaxConcurrentRequests = getProperty(propertyPrefix, key, "fallback.isolation.semaphore.maxConcurrentRequests", builder.getFallbackIsolationSemaphoreMaxConcurrentRequests(), default_fallbackIsolationSemaphoreMaxConcurrentRequests);
this.fallbackEnabled = getProperty(propertyPrefix, key, "fallback.enabled", builder.getFallbackEnabled(), default_fallbackEnabled);
this.metricsRollingStatisticalWindowInMilliseconds = getProperty(propertyPrefix, key, "metrics.rollingStats.timeInMilliseconds", builder.getMetricsRollingStatisticalWindowInMilliseconds(), default_metricsRollingStatisticalWindow);
this.metricsRollingStatisticalWindowBuckets = getProperty(propertyPrefix, key, "metrics.rollingStats.numBuckets", builder.getMetricsRollingStatisticalWindowBuckets(), default_metricsRollingStatisticalWindowBuckets);
this.metricsRollingPercentileEnabled = getProperty(propertyPrefix, key, "metrics.rollingPercentile.enabled", builder.getMetricsRollingPercentileEnabled(), default_metricsRollingPercentileEnabled);
Expand Down Expand Up @@ -273,6 +276,17 @@ public HystrixProperty<Integer> fallbackIsolationSemaphoreMaxConcurrentRequests(
return fallbackIsolationSemaphoreMaxConcurrentRequests;
}

/**
* Whether {@link HystrixCommand#getFallback()} should be attempted when failure occurs.
*
* @return {@code HystrixProperty<Boolean>}
*
* @since 1.2
*/
public HystrixProperty<Boolean> fallbackEnabled() {
return fallbackEnabled;
}

/**
* Time in milliseconds to wait between allowing health snapshots to be taken that calculate success and error percentages and affect {@link HystrixCircuitBreaker#isOpen()} status.
* <p>
Expand Down Expand Up @@ -474,6 +488,7 @@ public static class Setter {
private Boolean executionIsolationThreadInterruptOnTimeout = null;
private Integer executionIsolationThreadTimeoutInMilliseconds = null;
private Integer fallbackIsolationSemaphoreMaxConcurrentRequests = null;
private Boolean fallbackEnabled = null;
private Integer metricsHealthSnapshotIntervalInMilliseconds = null;
private Integer metricsRollingPercentileBucketSize = null;
private Boolean metricsRollingPercentileEnabled = null;
Expand Down Expand Up @@ -532,6 +547,10 @@ public Integer getFallbackIsolationSemaphoreMaxConcurrentRequests() {
return fallbackIsolationSemaphoreMaxConcurrentRequests;
}

public Boolean getFallbackEnabled() {
return fallbackEnabled;
}

public Integer getMetricsHealthSnapshotIntervalInMilliseconds() {
return metricsHealthSnapshotIntervalInMilliseconds;
}
Expand Down Expand Up @@ -623,6 +642,11 @@ public Setter withFallbackIsolationSemaphoreMaxConcurrentRequests(int value) {
return this;
}

public Setter withFallbackEnabled(boolean value) {
this.fallbackEnabled = value;
return this;
}

public Setter withMetricsHealthSnapshotIntervalInMilliseconds(int value) {
this.metricsHealthSnapshotIntervalInMilliseconds = value;
return this;
Expand Down Expand Up @@ -686,6 +710,7 @@ public Setter withRequestLogEnabled(boolean value) {
.withRequestLogEnabled(true)
.withExecutionIsolationSemaphoreMaxConcurrentRequests(20)
.withFallbackIsolationSemaphoreMaxConcurrentRequests(10)
.withFallbackEnabled(true)
.withCircuitBreakerForceClosed(false)
.withMetricsRollingPercentileEnabled(true)
.withRequestCacheEnabled(true)
Expand Down Expand Up @@ -765,6 +790,11 @@ public HystrixProperty<Integer> fallbackIsolationSemaphoreMaxConcurrentRequests(
return HystrixProperty.Factory.asProperty(builder.fallbackIsolationSemaphoreMaxConcurrentRequests);
}

@Override
public HystrixProperty<Boolean> fallbackEnabled() {
return HystrixProperty.Factory.asProperty(builder.fallbackEnabled);
}

@Override
public HystrixProperty<Integer> metricsHealthSnapshotIntervalInMilliseconds() {
return HystrixProperty.Factory.asProperty(builder.metricsHealthSnapshotIntervalInMilliseconds);
Expand Down
Loading

0 comments on commit 950430d

Please sign in to comment.