svn merge -r21044:21078 https://eris.llnl.gov/svn/slurm/branches/slurm-2.1

NEWS

@@ -467,7 +467,8 @@ documents those changes that are of interest to users and admins.
 
 * Changes in SLURM 2.1.13
 =========================
- -- Improve logging of invalid sinfo and squeue print options.
+ -- Fix race condition which can set a node state to IDLE on slurmctld startup
+    even if it has running jobs.
 
 * Changes in SLURM 2.1.12
 =========================
@@ -498,6 +499,10 @@ documents those changes that are of interest to users and admins.
  -- Fix to node in correct state in accounting when updating it to drain from
     scontrol/sview.
  -- BLUEGENE - Removed incorrect unlocking on error cases when starting jobs.
+ -- Improve logging of invalid sinfo and squeue print options.
+ -- BLUEGENE - Added check to libsched_if to allow root to run even outside of
+    SLURM.  This is needed when running certain blocks outside of SLURM in HTC
+    mode.
 
 * Changes in SLURM 2.1.11-2
 ===========================

src/plugins/select/bluegene/plugin/libsched_if64.c

@@ -40,13 +40,21 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
 
 int get_parameters(void *params)
 {
-	char *partition = getenv("MPIRUN_PARTITION"); /* get MPIRUN env
-						   * var to see if we
-						   * are inside slurm
-						   * or not */
+	char *partition;
+
+	/* Always allow root to run no matter what.  This is needed
+	   for HTC mode where it is common to run outside of SLURM.
+	*/
+	if (getuid() == 0)
+		return 0;
+
+	/* get MPIRUN env var to see if we are inside slurm or not */
+	partition = getenv("MPIRUN_PARTITION");
 	if (!partition || (strlen(partition) < 3)) {
 		printf("YOU ARE OUTSIDE OF SLURM!!!! NOT RUNNING MPIRUN!\n");
 		return 1;

src/slurmctld/node_mgr.c

@@ -1684,6 +1684,14 @@ extern int validate_node_specs(slurm_node_registration_status_msg_t *reg_msg)
 			   (reg_msg->job_count == 0)) {	/* job already done */
 			last_node_update = now;
 			node_ptr->node_state &= (~NODE_STATE_COMPLETING);
+		} else if (IS_NODE_IDLE(node_ptr) &&
+			   (reg_msg->job_count != 0)) {
+			last_node_update = now;
+			node_ptr->node_state = NODE_STATE_ALLOCATED |
+					       node_flags;
+			error("Invalid state for node %s, was IDLE with %u "
+			      "running jobs",
+			      node_ptr->name, reg_msg->job_count);
 		}
 
 		select_g_update_node_config((node_ptr-node_record_table_ptr));
@@ -1928,6 +1936,14 @@ extern int validate_nodes_via_front_end(
 				updated_job = true;
 				node_ptr->node_state &=
 					(~NODE_STATE_COMPLETING);
+			} else if (IS_NODE_IDLE(node_ptr) &&
+				   (jobs_on_node != 0)) {
+				updated_job = true;
+				node_ptr->node_state = NODE_STATE_ALLOCATED |
+						       node_flags;
+				error("Invalid state for node %s, was IDLE "
+				      "with %u running jobs",
+				      node_ptr->name, reg_msg->job_count);
 			}
 
 			select_g_update_node_config(
@@ -2035,7 +2051,11 @@ static void _node_did_resp(struct node_record *node_ptr)
 	if (IS_NODE_UNKNOWN(node_ptr)) {
 		last_node_update = now;
 		node_ptr->last_idle = now;
-		node_ptr->node_state = NODE_STATE_IDLE | node_flags;
+		if (node_ptr->run_job_cnt) {
+			node_ptr->node_state = NODE_STATE_ALLOCATED |
+					       node_flags;
+		} else
+			node_ptr->node_state = NODE_STATE_IDLE | node_flags;
 		if (!IS_NODE_DRAIN(node_ptr) && !IS_NODE_FAIL(node_ptr)) {
 			clusteracct_storage_g_node_up(acct_db_conn,
 						      node_ptr, now);