BLUEGENE - Added block node cnt to be able to differentiate between a...

BLUEGENE - Added block node cnt to be able to differentiate between a sub-block job and a regular full block job.

RELEASE_NOTES

@@ -129,6 +129,7 @@ select_jobdata_type
 	SELECT_JOBDATA_PTR	 /* data-> select_jobinfo_t *jobinfo */
 	SELECT_JOBDATA_BLOCK_PTR /* data-> bg_record_t *bg_record */
 	SELECT_JOBDATA_DIM_CNT   /* data-> uint16_t dim_cnt */
+	SELECT_JOBDATA_BLOCK_NODE_CNT	/* data-> uint32_t block_cnode_cnt */
 
 select_nodedata_type
 	SELECT_NODEDATA_PTR     /* data-> select_nodeinfo_t *nodeinfo */

slurm/slurm.h.in

@@ -331,7 +331,7 @@ enum select_jobdata_type {
 	SELECT_JOBDATA_BLOCK_ID,	/* data-> char *bg_block_id */
 	SELECT_JOBDATA_NODES,	/* data-> char *nodes */
 	SELECT_JOBDATA_IONODES,	/* data-> char *ionodes */
-	SELECT_JOBDATA_NODE_CNT,	/* data-> uint32_t node_cnt */
+	SELECT_JOBDATA_NODE_CNT,	/* data-> uint32_t cnode_cnt */
 	SELECT_JOBDATA_ALTERED,    /* data-> uint16_t altered */
 	SELECT_JOBDATA_BLRTS_IMAGE,/* data-> char *blrtsimage */
 	SELECT_JOBDATA_LINUX_IMAGE,/* data-> char *linuximage */
@@ -343,6 +343,7 @@ enum select_jobdata_type {
 	SELECT_JOBDATA_PTR,	/* data-> select_jobinfo_t *jobinfo */
 	SELECT_JOBDATA_BLOCK_PTR, /* data-> bg_record_t *bg_record */
 	SELECT_JOBDATA_DIM_CNT, /* data-> uint16_t dim_cnt */
+	SELECT_JOBDATA_BLOCK_NODE_CNT,	/* data-> uint32_t block_cnode_cnt */
 };
 
 enum select_nodedata_type {

src/common/env.c

@@ -90,6 +90,7 @@ static int _setup_particulars(uint32_t cluster_flags,
 	int rc = SLURM_SUCCESS;
 	if (cluster_flags & CLUSTER_FLAG_BG) {
 		char *bg_part_id = NULL;
+		uint32_t node_cnt = 0;
 		select_g_select_jobinfo_get(select_jobinfo,
 					    SELECT_JOBDATA_BLOCK_ID,
 					    &bg_part_id);
@@ -108,6 +109,13 @@ static int _setup_particulars(uint32_t cluster_flags,
 						bg_part_id);
 				}
 			}
+			select_g_select_jobinfo_get(
+				select_jobinfo,
+				SELECT_JOBDATA_BLOCK_NODE_CNT,
+				&node_cnt);
+			if (node_cnt)
+				setenvf(dest, "SLURM_BLOCK_NUM_NODES",
+					"%u", node_cnt);
 
 			setenvf(dest, "MPIRUN_PARTITION", "%s", bg_part_id);
 			setenvf(dest, "MPIRUN_NOFREE", "%d", 1);

src/plugins/select/bluegene/bg_job_info.c

@@ -68,6 +68,7 @@ extern select_jobinfo_t *alloc_select_jobinfo()
 	jobinfo->reboot = (uint16_t) NO_VAL;
 	jobinfo->rotate = (uint16_t) NO_VAL;
 	jobinfo->magic = JOBINFO_MAGIC;
+	jobinfo->block_cnode_cnt = NO_VAL;
 	jobinfo->cnode_cnt = NO_VAL;
 
 	/* This bitstr is created when used. */
@@ -158,18 +159,28 @@ extern int set_select_jobinfo(select_jobinfo_t *jobinfo,
 		jobinfo->rotate = *uint16;
 		break;
 	case SELECT_JOBDATA_CONN_TYPE:
-		for (i=0; i<jobinfo->dim_cnt; i++) {
+		for (i=0; i<jobinfo->dim_cnt; i++)
 			jobinfo->conn_type[i] = uint16[i];
-		}
-		break;
-	case SELECT_JOBDATA_BLOCK_PTR:
-		jobinfo->bg_record = bg_record;
 		break;
 	case SELECT_JOBDATA_BLOCK_ID:
 		/* we xfree() any preset value to avoid a memory leak */
 		xfree(jobinfo->bg_block_id);
 		jobinfo->bg_block_id = xstrdup(tmp_char);
 		break;
+	case SELECT_JOBDATA_BLOCK_NODE_CNT:
+		jobinfo->block_cnode_cnt = *uint32;
+		break;
+	case SELECT_JOBDATA_BLOCK_PTR:
+		jobinfo->bg_record = bg_record;
+		xfree(jobinfo->bg_block_id);
+		if (bg_record) {
+			jobinfo->bg_block_id = xstrdup(bg_record->bg_block_id);
+			jobinfo->block_cnode_cnt = bg_record->cnode_cnt;
+		} else {
+			jobinfo->bg_block_id = xstrdup("unassigned");
+			jobinfo->block_cnode_cnt = 0;
+		}
+		break;
 	case SELECT_JOBDATA_NODES:
 		xfree(jobinfo->mp_str);
 		jobinfo->mp_str = xstrdup(tmp_char);
@@ -289,6 +300,9 @@ extern int get_select_jobinfo(select_jobinfo_t *jobinfo,
 		else
 			*tmp_char = xstrdup(jobinfo->bg_block_id);
 		break;
+	case SELECT_JOBDATA_BLOCK_NODE_CNT:
+		*uint32 = jobinfo->block_cnode_cnt;
+		break;
 	case SELECT_JOBDATA_BLOCK_PTR:
 		*bg_record = jobinfo->bg_record;
 		break;
@@ -369,10 +383,12 @@ extern select_jobinfo_t *copy_select_jobinfo(select_jobinfo_t *jobinfo)
 		       sizeof(rc->conn_type));
 		rc->reboot = jobinfo->reboot;
 		rc->rotate = jobinfo->rotate;
+		rc->bg_record = jobinfo->bg_record;
 		rc->bg_block_id = xstrdup(jobinfo->bg_block_id);
 		rc->magic = JOBINFO_MAGIC;
 		rc->mp_str = xstrdup(jobinfo->mp_str);
 		rc->ionode_str = xstrdup(jobinfo->ionode_str);
+		rc->block_cnode_cnt = jobinfo->block_cnode_cnt;
 		rc->cnode_cnt = jobinfo->cnode_cnt;
 		rc->altered = jobinfo->altered;
 		rc->blrtsimage = xstrdup(jobinfo->blrtsimage);
@@ -417,6 +433,7 @@ extern int  pack_select_jobinfo(select_jobinfo_t *jobinfo, Buf buffer,
 			pack16(jobinfo->reboot, buffer);
 			pack16(jobinfo->rotate, buffer);
 
+			pack32(jobinfo->block_cnode_cnt, buffer);
 			pack32(jobinfo->cnode_cnt, buffer);
 
 			packstr(jobinfo->bg_block_id, buffer);
@@ -436,7 +453,8 @@ extern int  pack_select_jobinfo(select_jobinfo_t *jobinfo, Buf buffer,
 			for (i=0; i<((dims*2)+2); i++) {
 				pack16((uint16_t) 0, buffer);
 			}
-			pack32((uint32_t) 0, buffer); //node_cnt
+			pack32((uint32_t) 0, buffer); //block_cnode_cnt
+			pack32((uint32_t) 0, buffer); //cnode_cnt
 			packnull(buffer); //bg_block_id
 			packnull(buffer); //nodes
 			packnull(buffer); //ionodes
@@ -571,6 +589,7 @@ extern int unpack_select_jobinfo(select_jobinfo_t **jobinfo_pptr, Buf buffer,
 		safe_unpack16(&(jobinfo->reboot), buffer);
 		safe_unpack16(&(jobinfo->rotate), buffer);
 
+		safe_unpack32(&(jobinfo->block_cnode_cnt), buffer);
 		safe_unpack32(&(jobinfo->cnode_cnt), buffer);
 
 		safe_unpackstr_xmalloc(&(jobinfo->bg_block_id), &uint32_tmp,

src/plugins/select/bluegene/bg_job_info.h

@@ -50,7 +50,10 @@ struct select_jobinfo {
 	bg_record_t *bg_record; /* For internal use only DO NOT PACK */
 	char *bg_block_id;	/* Blue Gene block ID */
 	char *blrtsimage;       /* BlrtsImage for this block */
-	uint32_t cnode_cnt;     /* how many cnodes in block running job */
+	uint32_t block_cnode_cnt; /* how many cnodes in the block
+				   * This is used to say we are
+				   * running a sub-block job. */
+	uint32_t cnode_cnt;     /* how many cnodes in job running on block */
 	uint16_t conn_type[HIGHEST_DIMENSIONS];	/* see enum connection_type */
 	uint16_t dim_cnt;       /* how many dimensions this
 				 * represents in most cases this will

src/plugins/select/bluegene/bg_job_place.c

@@ -1595,8 +1595,8 @@ extern int submit_job(struct job_record *job_ptr, bitstr_t *slurm_block_bitmap,
 			      starttime, bg_record->mp_str);
 
 			set_select_jobinfo(job_ptr->select_jobinfo->data,
-					   SELECT_JOBDATA_BLOCK_ID,
-					   "unassigned");
+					   SELECT_JOBDATA_BLOCK_PTR,
+					   NULL);
 			set_select_jobinfo(job_ptr->select_jobinfo->data,
 					   SELECT_JOBDATA_NODE_CNT,
 					   &bg_record->cnode_cnt);
@@ -1614,10 +1614,6 @@ extern int submit_job(struct job_record *job_ptr, bitstr_t *slurm_block_bitmap,
 			      SELECT_IS_MODE_RUN_NOW(local_mode));
 
 			if (SELECT_IS_MODE_RUN_NOW(local_mode)) {
-				set_select_jobinfo(
-					job_ptr->select_jobinfo->data,
-					SELECT_JOBDATA_BLOCK_ID,
-					bg_record->bg_block_id);
 				/* Set this up to be the
 				   correct pointer since we
 				   probably are working off a
@@ -1638,10 +1634,6 @@ extern int submit_job(struct job_record *job_ptr, bitstr_t *slurm_block_bitmap,
 					last_bg_update = time(NULL);
 				}
 			} else {
-				set_select_jobinfo(
-					job_ptr->select_jobinfo->data,
-					SELECT_JOBDATA_BLOCK_ID,
-					"unassigned");
 				set_select_jobinfo(
 					job_ptr->select_jobinfo->data,
 					SELECT_JOBDATA_BLOCK_PTR,

src/plugins/select/bluegene/bg_job_run.c

@@ -72,8 +72,9 @@ enum update_op {START_OP, TERM_OP, SYNC_OP};
 typedef struct {
 	char *bg_block_id;
 	char *blrtsimage;       /* BlrtsImage for this block */
-	uint16_t conn_type[SYSTEM_DIMENSIONS]; /* needed to boot small
-				   blocks into HTC mode or not */
+	uint16_t conn_type[HIGHEST_DIMENSIONS]; /* needed to boot small
+						   blocks into HTC
+						   mode or not */
 	struct job_record *job_ptr;	/* pointer to job running on
 					 * block or NULL if no job */
 	char *linuximage;       /* LinuxImage for this block */