diff --git a/src/ml_optimiser_mpi.cpp b/src/ml_optimiser_mpi.cpp
index e025fae5d..c6938394e 100644
--- a/src/ml_optimiser_mpi.cpp
+++ b/src/ml_optimiser_mpi.cpp
@@ -97,11 +97,6 @@ void MlOptimiserMpi::initialise()
 
 	// Print information about MPI nodes:
 	printMpiNodesMachineNames(*node, nr_threads);
-#ifdef _CUDA_ENABLED
-    /************************************************************************/
-	//Setup GPU related resources
-	int devCount, deviceAffinity;
-	bool is_split(false);
 
 	if (gradient_refine && !do_split_random_halves) {
 		if (node->isLeader())
@@ -111,6 +106,11 @@ void MlOptimiserMpi::initialise()
 	}
 
 	grad_pseudo_halfsets = gradient_refine && !do_split_random_halves;
+#ifdef _CUDA_ENABLED
+    /************************************************************************/
+	//Setup GPU related resources
+	int devCount, deviceAffinity;
+	bool is_split(false);
 
 	if (do_gpu)
 	{