From d00e9e283f9b56951e7a37a35fcfb30f90375a26 Mon Sep 17 00:00:00 2001 From: Matt Jacobs Date: Mon, 23 Feb 2015 16:54:44 -0800 Subject: [PATCH 1/2] Added unit test to help with understanding the test written in #685 Also added a missing execution event for FALLBACK_REJECTION --- .../com/netflix/hystrix/AbstractCommand.java | 4 +- .../concurrency/HystrixContextScheduler.java | 2 + .../netflix/hystrix/HystrixCommandTest.java | 44 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java index 3e1695b0d..d80ff4a25 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java @@ -686,6 +686,8 @@ public void call() { private Observable getFallbackOrThrowException(final HystrixEventType eventType, final FailureType failureType, final String message, final Exception originalException) { final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); + //System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " getFallback in : " + this); + if (properties.fallbackEnabled().get()) { /* fallback behavior is permitted so attempt */ // record the executionResult @@ -720,7 +722,7 @@ public void call() { }); } else { metrics.markFallbackRejection(); - + executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_REJECTION); logger.debug("HystrixCommand Fallback Rejection."); // debug only since we're throwing the exception and someone higher will do something with it // if we couldn't acquire a permit, we "fail fast" by throwing an exception return Observable.error(new HystrixRuntimeException(FailureType.REJECTED_SEMAPHORE_FALLBACK, this.getClass(), getLogMessagePrefix() + " fallback execution rejected.", null, null)); diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java index 5c870c63a..d887fda8f 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java @@ -171,7 +171,9 @@ public Subscription schedule(final Action0 action) { subscription.add(sa); sa.addParent(subscription); + //System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " about to submit to pool : (" + threadPool.getExecutor().getActiveCount() + "/" + threadPool.getExecutor().getCorePoolSize() + ")"); Future f = threadPool.getExecutor().submit(sa); + //System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " pool submission succeeded : (" + threadPool.getExecutor().getActiveCount() + "/" + threadPool.getExecutor().getCorePoolSize() + ")"); sa.add(new FutureCompleterWithConfigurableInterrupt(f, shouldInterruptThread)); return sa; diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java index 161c4c608..76786aba4 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java @@ -3634,6 +3634,50 @@ public void run() { assertEquals(0, circuitBreaker.metrics.getCumulativeCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); } + static class EventCommand extends HystrixCommand { + public EventCommand() { + super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("eventGroup")).andCommandPropertiesDefaults(new HystrixCommandProperties.Setter().withFallbackIsolationSemaphoreMaxConcurrentRequests(3))); + } + + @Override + protected String run() throws Exception { + System.out.println(Thread.currentThread().getName() + " : In run()"); + throw new RuntimeException("run_exception"); + } + + @Override + public String getFallback() { + try { + System.out.println(Thread.currentThread().getName() + " : In fallback => " + getExecutionEvents()); + Thread.sleep(30000L); + } catch (InterruptedException e) { + System.out.println(Thread.currentThread().getName() + " : Interruption occurred"); + } + System.out.println(Thread.currentThread().getName() + " : CMD Success Result"); + return "fallback"; + } + } + + //if I set fallback semaphore to same as threadpool (10), I set up a race. + //instead, I set fallback sempahore to much less (3). This should guarantee that all fallbacks only happen in the threadpool, and main thread does not block + @Test(timeout=5000) + public void testFallbackRejection() throws InterruptedException, ExecutionException { + for (int i = 0; i < 1000; i++) { + EventCommand cmd = new EventCommand(); + + try { + if (i == 500) { + Thread.sleep(100L); + } + cmd.queue(); + System.out.println("queued: " + i); + } catch (Exception e) { + System.out.println("Fail Fast on queue() : " + cmd.getExecutionEvents()); + + } + } + } + @Test public void testNonBlockingCommandQueueFiresTimeout() { //see https://github.com/Netflix/Hystrix/issues/514 final TestHystrixCommand cmd = getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 200, AbstractTestHystrixCommand.FallbackResult.SUCCESS, 50); From c75b8a2211e5e9a275c38ffefae95a0804574344 Mon Sep 17 00:00:00 2001 From: Matt Jacobs Date: Mon, 23 Feb 2015 16:56:26 -0800 Subject: [PATCH 2/2] Removed printlns --- .../src/main/java/com/netflix/hystrix/AbstractCommand.java | 2 -- .../hystrix/strategy/concurrency/HystrixContextScheduler.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java index d80ff4a25..5cccab932 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java @@ -686,8 +686,6 @@ public void call() { private Observable getFallbackOrThrowException(final HystrixEventType eventType, final FailureType failureType, final String message, final Exception originalException) { final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); - //System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " getFallback in : " + this); - if (properties.fallbackEnabled().get()) { /* fallback behavior is permitted so attempt */ // record the executionResult diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java index d887fda8f..5c870c63a 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java @@ -171,9 +171,7 @@ public Subscription schedule(final Action0 action) { subscription.add(sa); sa.addParent(subscription); - //System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " about to submit to pool : (" + threadPool.getExecutor().getActiveCount() + "/" + threadPool.getExecutor().getCorePoolSize() + ")"); Future f = threadPool.getExecutor().submit(sa); - //System.out.println(System.currentTimeMillis() + " : " + Thread.currentThread().getName() + " pool submission succeeded : (" + threadPool.getExecutor().getActiveCount() + "/" + threadPool.getExecutor().getCorePoolSize() + ")"); sa.add(new FutureCompleterWithConfigurableInterrupt(f, shouldInterruptThread)); return sa;