If a slurmd does not have a node listed in it's slurm.conf (slurm.conf's...

If a slurmd does not have a node listed in it's slurm.conf (slurm.conf's should be kept the same on all nodes) an error message is printed in the slurmctld log along with the message already being printed in the slurmd log for easier debugging.

NEWS

@@ -87,6 +87,10 @@ documents those changes that are of interest to users and admins.
     plugins.
  -- QOS support added with limits, priority and preemption
     (no documentation yet).
+ -- If a slurmd does not have a node listed in it's slurm.conf (slurm.conf's 
+    should be kept the same on all nodes) an error message is printed in the 
+    slurmctld log along with the message already being printed in the slurmd 
+    log for easier debugging.
 
 * Changes in SLURM 2.1.0-pre2
 =============================

slurm/slurm_errno.h

@@ -99,6 +99,7 @@ enum {
 	SLURM_MPI_PLUGIN_NAME_INVALID,
 	SLURM_MPI_PLUGIN_PRELAUNCH_SETUP_FAILED,
 	SLURM_PLUGIN_NAME_INVALID,
+	SLURM_UNKNOWN_FORWARD_ADDR,
 
 	/* communication failures to/from slurmctld */
 	SLURMCTLD_COMMUNICATIONS_CONNECTION_ERROR =     1800,

src/common/forward.c

@@ -102,7 +102,7 @@ void *_forward_thread(void *arg)
 			      "%s, check slurm.conf", name);
 			slurm_mutex_lock(fwd_msg->forward_mutex);
 			mark_as_failed_forward(&fwd_msg->ret_list, name,
-					SLURM_COMMUNICATIONS_CONNECTION_ERROR);
+					       SLURM_UNKNOWN_FORWARD_ADDR);
  			free(name);
 			if (hostlist_count(hl) > 0) {
 				slurm_mutex_unlock(fwd_msg->forward_mutex);
@@ -114,8 +114,9 @@ void *_forward_thread(void *arg)
 			error("forward_thread to %s: %m", name);
 
 			slurm_mutex_lock(fwd_msg->forward_mutex);
-			mark_as_failed_forward(&fwd_msg->ret_list, name,
-					SLURM_COMMUNICATIONS_CONNECTION_ERROR);
+			mark_as_failed_forward(
+				&fwd_msg->ret_list, name,
+				SLURM_COMMUNICATIONS_CONNECTION_ERROR);
 			free(name);
 			if (hostlist_count(hl) > 0) {
 				slurm_mutex_unlock(fwd_msg->forward_mutex);
@@ -321,7 +322,7 @@ void *_fwd_tree_thread(void *arg)
 			      "%s, check slurm.conf", name);
 			slurm_mutex_lock(fwd_tree->tree_mutex);
 			mark_as_failed_forward(&fwd_tree->ret_list, name,
-					SLURM_COMMUNICATIONS_CONNECTION_ERROR);
+					       SLURM_UNKNOWN_FORWARD_ADDR);
  			pthread_cond_signal(fwd_tree->notify);
 			slurm_mutex_unlock(fwd_tree->tree_mutex);
 			free(name);
@@ -361,8 +362,9 @@ void *_fwd_tree_thread(void *arg)
 			error("fwd_tree_thread: no return list given from "
 			      "slurm_send_addr_recv_msgs", name);
 			slurm_mutex_lock(fwd_tree->tree_mutex);
-			mark_as_failed_forward(&fwd_tree->ret_list, name,
-					SLURM_COMMUNICATIONS_CONNECTION_ERROR);
+			mark_as_failed_forward(
+				&fwd_tree->ret_list, name,
+				SLURM_COMMUNICATIONS_CONNECTION_ERROR);
  			pthread_cond_signal(fwd_tree->notify);
 			slurm_mutex_unlock(fwd_tree->tree_mutex);
 			free(name);

src/common/slurm_errno.c

@@ -70,7 +70,7 @@ static slurm_errtab_t slurm_errtab[] = {
 	{-1, "Unspecified error"},
 	{EINPROGRESS, "Operation now in progress"},
 
-	/*General Message error codes */
+	/* General Message error codes */
 	{ SLURM_UNEXPECTED_MSG_ERROR, 
 	  "Unexpected message received" 			},
 	{ SLURM_COMMUNICATIONS_CONNECTION_ERROR,
@@ -95,6 +95,8 @@ static slurm_errtab_t slurm_errtab[] = {
 	  "MPI plugin's pre-launch setup failed"                },
 	{ SLURM_PLUGIN_NAME_INVALID,
 	  "Plugin initialization failed"			},
+	{ SLURM_UNKNOWN_FORWARD_ADDR,
+	  "Can't find an address, check slurm.conf"		},
 
 	/* communication failures to/from slurmctld */
 	{ SLURMCTLD_COMMUNICATIONS_CONNECTION_ERROR,

src/slurmctld/agent.c

@@ -939,6 +939,13 @@ static void *_thread_per_group_rpc(void *args)
 			  ret_data_info->node_name); */
 			thread_state = DSH_DONE;
 			break;
+		case SLURM_UNKNOWN_FORWARD_ADDR:
+			error("We were unable to forward message to '%s'.  "
+			      "Make sure the slurm.conf for each slurmd "
+			      "contain all other nodes in your system.",
+			      ret_data_info->node_name);
+			thread_state = DSH_NO_RESP;
+			break;
 		case ESLURMD_EPILOG_FAILED:
 			error("Epilog failure on host %s, "
 			      "setting DOWN",