Include node name in agent communication failures.

Slurmctld priority of zero treated like job hold (not scheduled).
Preserve priority order even if incomming job could use idle resources.

src/slurmctld/agent.c

@@ -442,7 +442,9 @@ static void *_thread_per_node_rpc(void *args)
 	/* init message connection for message communication */
 	if ((sockfd = slurm_open_msg_conn(&thread_ptr->slurm_addr))
 	    == SLURM_SOCKET_ERROR) {
-		error("_thread_per_node_rpc/slurm_open_msg_conn error %m");
+		error(
+		    "_thread_per_node_rpc/slurm_open_msg_conn to host %s: %m",
+		    thread_ptr->node_name);
 		goto cleanup;
 	}
 
@@ -451,25 +453,31 @@ static void *_thread_per_node_rpc(void *args)
 	request_msg.data = task_ptr->msg_args_ptr;
 	if ((rc = slurm_send_node_msg(sockfd, &request_msg))
 	    == SLURM_SOCKET_ERROR) {
-		error("_thread_per_node_rpc/slurm_send_node_msg error %m");
+		error(
+		    "_thread_per_node_rpc/slurm_send_node_msg to host %s: %m",
+		    thread_ptr->node_name);
 		goto cleanup;
 	}
 
 	/* receive message */
 	if ((msg_size = slurm_receive_msg(sockfd, &response_msg))
 	    == SLURM_SOCKET_ERROR) {
-		error("_thread_per_node_rpc/slurm_receive_msg error %m");
+		error(
+		    "_thread_per_node_rpc/slurm_receive_msg to host %s: %m",
+		    thread_ptr->node_name);
 		goto cleanup;
 	}
 
 	/* shutdown message connection */
 	if ((rc = slurm_shutdown_msg_conn(sockfd)) == SLURM_SOCKET_ERROR) {
-		error
-		    ("_thread_per_node_rpc/slurm_shutdown_msg_conn error %m");
+		error(
+		    "_thread_per_node_rpc/slurm_shutdown_msg_conn to host %s: %m",
+		    thread_ptr->node_name);
 		goto cleanup;
 	}
 	if (msg_size) {
-		error("_thread_per_node_rpc/msg_size error %d", msg_size);
+		error("_thread_per_node_rpc/msg_size to host %s error %d", 
+		      thread_ptr->node_name, msg_size);
 		goto cleanup;
 	}
 
@@ -479,7 +487,8 @@ static void *_thread_per_node_rpc(void *args)
 		rc = slurm_rc_msg->return_code;
 		slurm_free_return_code_msg(slurm_rc_msg);
 		if (rc)
-			error("_thread_per_node_rpc/rc error: %s", 
+			error("_thread_per_node_rpc/rc error from host %s: %s",
+			      thread_ptr->node_name,
 			      slurm_strerror(rc));	/* Don't use %m */
 		else {
 			debug3
@@ -489,8 +498,8 @@ static void *_thread_per_node_rpc(void *args)
 		thread_state = DSH_DONE;
 		break;
 	default:
-		error("_thread_per_node_rpc bad msg_type %d",
-		      response_msg.msg_type);
+		error("_thread_per_node_rpc from host %s, bad msg_type %d",
+		      thread_ptr->node_name, response_msg.msg_type);
 		break;
 	}
 

src/slurmctld/job_mgr.c

@@ -123,11 +123,11 @@ static void _set_job_id(struct job_record *job_ptr);
 static void _set_job_prio(struct job_record *job_ptr);
 static void _signal_job_on_node(uint32_t job_id, uint16_t step_id,
 				int signum, char *node_name);
-static int _top_priority(struct job_record *job_ptr);
-static int _validate_job_desc(job_desc_msg_t * job_desc_msg, int allocate);
-static int _validate_job_create_req(job_desc_msg_t * job_desc);
-static int _write_data_to_file(char *file_name, char *data);
-static int _write_data_array_to_file(char *file_name, char **data,
+static bool _top_priority(struct job_record *job_ptr);
+static int  _validate_job_desc(job_desc_msg_t * job_desc_msg, int allocate);
+static int  _validate_job_create_req(job_desc_msg_t * job_desc);
+static int  _write_data_to_file(char *file_name, char *data);
+static int  _write_data_array_to_file(char *file_name, char **data,
 				     uint16_t size);
 
 /* 
@@ -1032,7 +1032,8 @@ int job_allocate(job_desc_msg_t * job_specs, uint32_t * new_job_id,
 		 uid_t submit_uid, uint16_t * node_cnt,
 		 slurm_addr ** node_addr)
 {
-	int error_code, test_only;
+	int error_code;
+	bool no_alloc, top_prio, test_only;
 	struct job_record *job_ptr;
 
 	error_code = _job_create(job_specs, new_job_id, allocate, will_run,
@@ -1043,14 +1044,15 @@ int job_allocate(job_desc_msg_t * job_specs, uint32_t * new_job_id,
 		fatal("job_allocate: allocated job %u lacks record",
 		      new_job_id);
 
-	if (immediate && _top_priority(job_ptr) != 1) {
+	top_prio = _top_priority(job_ptr);
+	if (immediate && (!top_prio)) {
 		job_ptr->job_state = JOB_FAILED;
 		job_ptr->end_time = 0;
 		return ESLURM_NOT_TOP_PRIORITY;
 	}
 
 	test_only = will_run || (allocate == 0);
-	if (test_only == 0) {
+	if (!test_only) {
 		/* Some of these pointers are NULL on submit */
 		if (num_cpu_groups)
 			*num_cpu_groups = 0;
@@ -1067,7 +1069,8 @@ int job_allocate(job_desc_msg_t * job_specs, uint32_t * new_job_id,
 		last_job_update = time(NULL);
 	}
 
-	error_code = select_nodes(job_ptr, test_only);
+	no_alloc = test_only || (!top_prio);
+	error_code = select_nodes(job_ptr, no_alloc);
 	if (error_code == ESLURM_NODES_BUSY) {
 		if (immediate) {
 			job_ptr->job_state = JOB_FAILED;
@@ -1088,7 +1091,7 @@ int job_allocate(job_desc_msg_t * job_specs, uint32_t * new_job_id,
 		job_ptr->end_time = 0;
 	}
 
-	if (test_only == 0) {
+	if (!no_alloc) {
 		if (node_list)
 			*node_list = job_ptr->nodes;
 		if (num_cpu_groups)
@@ -1102,6 +1105,7 @@ int job_allocate(job_desc_msg_t * job_specs, uint32_t * new_job_id,
 		if (node_addr)
 			*node_addr = job_ptr->node_addr;
 	}
+
 	return SLURM_SUCCESS;
 }
 
@@ -2210,15 +2214,18 @@ static void _set_job_prio(struct job_record *job_ptr)
  * _top_priority - determine if any other job for this partition has a 
  *	higher priority than specified job
  * IN job_ptr - pointer to selected partition
- * RET 1 if selected job has highest priority, 0 otherwise
+ * RET true if selected job has highest priority
  */
-static int _top_priority(struct job_record *job_ptr)
+static bool _top_priority(struct job_record *job_ptr)
 {
 	ListIterator job_record_iterator;
 	struct job_record *job_record_point;
-	int top;
+	bool top;
+
+	if (job_ptr->priority == 0)	/* held */
+		return false;
 
-	top = 1;		/* assume top priority until found otherwise */
+	top = true;		/* assume top priority until found otherwise */
 	job_record_iterator = list_iterator_create(job_list);
 	while ((job_record_point =
 		(struct job_record *) list_next(job_record_iterator))) {
@@ -2228,9 +2235,9 @@ static int _top_priority(struct job_record *job_ptr)
 			continue;
 		if (job_record_point->job_state != JOB_PENDING)
 			continue;
-		if (job_record_point->priority > job_ptr->priority &&
-		    job_record_point->part_ptr == job_ptr->part_ptr) {
-			top = 0;
+		if ((job_record_point->priority > job_ptr->priority) &&
+		    (job_record_point->part_ptr == job_ptr->part_ptr)) {
+			top = false;
 			break;
 		}
 	}

src/slurmctld/job_scheduler.c

@@ -36,6 +36,7 @@
 #include <unistd.h>
 
 #include "src/common/list.h"
+#include "src/common/macros.h"
 #include "src/common/xstring.h"
 #include "src/slurmctld/agent.h"
 #include "src/slurmctld/locks.h"
@@ -74,6 +75,8 @@ static int _build_job_queue(struct job_queue **job_queue)
 		(struct job_record *) list_next(job_record_iterator))) {
 		if (job_record_point->job_state != JOB_PENDING)
 			continue;
+		if (job_record_point->priority == 0)	/* held */
+			continue;
 		if (job_record_point->magic != JOB_MAGIC)
 			fatal("prio_order_job: data integrity is bad");
 		if (job_buffer_size <= job_queue_size) {
@@ -133,7 +136,7 @@ int schedule(void)
 		}
 		if (j < failed_part_cnt)
 			continue;
-		error_code = select_nodes(job_ptr, 0);
+		error_code = select_nodes(job_ptr, false);
 		if (error_code == ESLURM_NODES_BUSY) {
 			xrealloc(failed_parts,
 				 (failed_part_cnt +

src/slurmctld/node_scheduler.c

@@ -650,8 +650,8 @@ _pick_best_nodes(struct node_set *node_set_ptr, int node_set_size,
 /*
  * select_nodes - select and allocate nodes to a specific job
  * IN job_ptr - pointer to the job record
- * IN test_only - do not allocate nodes, just confirm they could be 
- *	allocated now
+ * IN test_only - if set do not allocate nodes, just confirm they  
+ *	could be allocated now
  * RET 0 on success, ESLURM code from slurm_errno.h otherwise
  * globals: list_part - global list of partition info
  *	default_part_loc - pointer to default partition 
@@ -664,7 +664,7 @@ _pick_best_nodes(struct node_set *node_set_ptr, int node_set_size,
  *	   the request, (e.g. best-fit or other criterion)
  *	3) Call allocate_nodes() to perform the actual allocation
  */
-int select_nodes(struct job_record *job_ptr, int test_only)
+int select_nodes(struct job_record *job_ptr, bool test_only)
 {
 	int error_code, i, node_set_index, node_set_size = 0;
 	bitstr_t *req_bitmap, *scratch_bitmap;

src/slurmctld/slurmctld.h

@@ -234,7 +234,8 @@ struct job_record {
 	time_t end_time;		/* time of termination, 
 					   actual or expected */
 	time_t time_last_active;	/* time of last job activity */
-	uint32_t priority;		/* relative priority of the job */
+	uint32_t priority;		/* relative priority of the job,
+					   zero == held (don't initiate) */
 	struct job_details *details;	/* job details */
 	uint16_t num_cpu_groups;	/* record count in cpus_per_node and 
 					   cpu_count_reps */
@@ -832,8 +833,8 @@ extern int schedule (void);
 /*
  * select_nodes - select and allocate nodes to a specific job
  * IN job_ptr - pointer to the job record
- * IN test_only - do not allocate nodes, just confirm they could be 
- *	allocated now
+ * IN test_only - if set do not allocate nodes, just confirm they  
+ *	could be allocated now
  * RET 0 on success, ESLURM code from slurm_errno.h otherwise
  * globals: list_part - global list of partition info
  *	default_part_loc - pointer to default partition 
@@ -846,7 +847,7 @@ extern int schedule (void);
  *	   the request, (e.g. best-fit or other criterion)
  *	3) Call allocate_nodes() to perform the actual allocation
  */
-extern int select_nodes (struct job_record *job_ptr, int test_only);
+extern int select_nodes (struct job_record *job_ptr, bool test_only);
 
 /* set_slurmd_addr - establish the slurm_addr for the slurmd on each node
  *	Uses common data structures. */