diff --git a/slurm/slurm.h.in b/slurm/slurm.h.in
index 1887b8980e928d5bf0ece1c1fc16308517ffca03..e34b4e586cb0f1532a9e431688e3791beab2c18b 100644
--- a/slurm/slurm.h.in
+++ b/slurm/slurm.h.in
@@ -1590,11 +1590,6 @@ typedef struct {
* start_range_1, end_range_1,
* start_range_2, .., -1 */
char *mp_str;
- int *mp_used_inx; /* list index pairs into node_table
- * for used *mp_str:
- * start_range_1, end_range_1,
- * start_range_2, .., -1 */
- char *mp_used_str;
uint16_t node_use;
char *ramdiskimage; /* RamDiskImage for this block */
char *reason;
diff --git a/src/common/slurm_protocol_defs.c b/src/common/slurm_protocol_defs.c
index 2ec178ba3aa905433f1b19ed3fc0abec70118bc4..b8f76b5ae443f4d04979632e4c21940983726d03 100644
--- a/src/common/slurm_protocol_defs.c
+++ b/src/common/slurm_protocol_defs.c
@@ -2228,8 +2228,6 @@ extern void slurm_free_block_info_members(block_info_t *block_info)
xfree(block_info->mloaderimage);
xfree(block_info->mp_inx);
xfree(block_info->mp_str);
- xfree(block_info->mp_used_inx);
- xfree(block_info->mp_used_str);
xfree(block_info->ramdiskimage);
xfree(block_info->reason);
}
diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index d55231c6d99b9ef34cde5541be4edd440ea34ae4..fe7a5869128fc154479afd7b2327d148563579eb 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -8726,18 +8726,11 @@ static void _pack_block_info_msg(block_info_t *block_info, Buf buffer,
packstr(block_info->linuximage, buffer);
packstr(block_info->mloaderimage, buffer);
packstr(block_info->mp_str, buffer);
- packstr(block_info->mp_used_str, buffer);
pack32(block_info->cnode_cnt, buffer);
pack16(block_info->node_use, buffer);
packstr(block_info->ramdiskimage, buffer);
packstr(block_info->reason, buffer);
pack16(block_info->state, buffer);
- if (block_info->mp_used_inx) {
- char *bitfmt = inx2bitfmt(block_info->mp_used_inx);
- packstr(bitfmt, buffer);
- xfree(bitfmt);
- } else
- packnull(buffer);
} else if (protocol_version >= SLURM_2_3_PROTOCOL_VERSION) {
if (!block_info) {
packnull(buffer);
@@ -8810,19 +8803,14 @@ static void _pack_block_info_msg(block_info_t *block_info, Buf buffer,
packstr(block_info->linuximage, buffer);
packstr(block_info->mloaderimage, buffer);
packstr(block_info->mp_str, buffer);
- packstr(block_info->mp_used_str, buffer);
+ packnull(buffer); /* for mp_used_str */
pack32(block_info->cnode_cnt, buffer);
pack16(block_info->node_use, buffer);
packnull(buffer); /* for user name */
packstr(block_info->ramdiskimage, buffer);
packstr(block_info->reason, buffer);
pack16(block_info->state, buffer);
- if (block_info->mp_used_inx) {
- char *bitfmt = inx2bitfmt(block_info->mp_used_inx);
- packstr(bitfmt, buffer);
- xfree(bitfmt);
- } else
- packnull(buffer);
+ packnull(buffer); /* for mp_used_inx */
} else if (protocol_version >= SLURM_2_2_PROTOCOL_VERSION) {
if (!block_info) {
packnull(buffer);
@@ -9033,8 +9021,6 @@ extern int slurm_unpack_block_info_members(block_info_t *block_info, Buf buffer,
&uint32_tmp, buffer);
safe_unpackstr_xmalloc(&(block_info->mp_str), &uint32_tmp,
buffer);
- safe_unpackstr_xmalloc(&(block_info->mp_used_str), &uint32_tmp,
- buffer);
safe_unpack32(&block_info->cnode_cnt, buffer);
safe_unpack16(&block_info->node_use, buffer);
safe_unpackstr_xmalloc(&block_info->ramdiskimage,
@@ -9042,13 +9028,6 @@ extern int slurm_unpack_block_info_members(block_info_t *block_info, Buf buffer,
safe_unpackstr_xmalloc(&block_info->reason,
&uint32_tmp, buffer);
safe_unpack16(&block_info->state, buffer);
- safe_unpackstr_xmalloc(&mp_inx_str, &uint32_tmp, buffer);
- if (mp_inx_str == NULL) {
- block_info->mp_used_inx = bitfmt2int("");
- } else {
- block_info->mp_used_inx = bitfmt2int(mp_inx_str);
- xfree(mp_inx_str);
- }
} else if (protocol_version >= SLURM_2_3_PROTOCOL_VERSION) {
safe_unpackstr_xmalloc(&block_info->bg_block_id,
&uint32_tmp, buffer);
@@ -9109,8 +9088,8 @@ extern int slurm_unpack_block_info_members(block_info_t *block_info, Buf buffer,
&uint32_tmp, buffer);
safe_unpackstr_xmalloc(&(block_info->mp_str), &uint32_tmp,
buffer);
- safe_unpackstr_xmalloc(&(block_info->mp_used_str), &uint32_tmp,
- buffer);
+ safe_unpackstr_xmalloc(&mp_inx_str, &uint32_tmp, buffer);
+ xfree(mp_inx_str);
safe_unpack32(&block_info->cnode_cnt, buffer);
safe_unpack16(&block_info->node_use, buffer);
safe_unpackstr_xmalloc(&mp_inx_str, &uint32_tmp, buffer);
@@ -9121,12 +9100,7 @@ extern int slurm_unpack_block_info_members(block_info_t *block_info, Buf buffer,
&uint32_tmp, buffer);
safe_unpack16(&block_info->state, buffer);
safe_unpackstr_xmalloc(&mp_inx_str, &uint32_tmp, buffer);
- if (mp_inx_str == NULL) {
- block_info->mp_used_inx = bitfmt2int("");
- } else {
- block_info->mp_used_inx = bitfmt2int(mp_inx_str);
- xfree(mp_inx_str);
- }
+ xfree(mp_inx_str);
} else if (protocol_version >= SLURM_2_2_PROTOCOL_VERSION) {
safe_unpackstr_xmalloc(&block_info->bg_block_id,
&uint32_tmp, buffer);
diff --git a/src/plugins/select/bluegene/ba_bgq/block_allocator.c b/src/plugins/select/bluegene/ba_bgq/block_allocator.c
index 610b2de099b49372c0e64291c04aa55c4389dda0..2afb99ae157edae48c67c99f46d18328eb91684a 100644
--- a/src/plugins/select/bluegene/ba_bgq/block_allocator.c
+++ b/src/plugins/select/bluegene/ba_bgq/block_allocator.c
@@ -1191,7 +1191,23 @@ extern int ba_sub_block_in_record_clear(
error("ba_sub_block_in_record_clear: "
"didn't have the units_used bitmap "
"for some reason?");
- continue;
+ break;
+ } else if (!ba_mp->cnode_bitmap) {
+ /* If the job allocation has already finished
+ before processing the job step completion
+ this could happen, but it should already be
+ checked before it gets here so this should
+ never happen, this is just for safely sake.
+ */
+ error("ba_sub_block_in_record_clear: no cnode_bitmap? "
+ "job %u(%p) is in state %s on block %s %u(%p). "
+ "This should never happen.",
+ step_ptr->job_ptr->job_id, step_ptr->job_ptr,
+ job_state_string(step_ptr->job_ptr->job_state
+ & (~JOB_CONFIGURING)),
+ bg_record->bg_block_id, bg_record->job_running,
+ bg_record->job_ptr);
+ break;
}
bit_not(jobinfo->units_used);
@@ -1222,6 +1238,7 @@ extern int ba_sub_block_in_record_clear(
xfree(tmp_char3);
FREE_NULL_BITMAP(total_bitmap);
}
+ break;
}
list_iterator_destroy(itr);
@@ -1244,6 +1261,17 @@ extern void ba_sync_job_to_block(bg_record_t *bg_record,
ba_mp = list_peek(bg_record->ba_mp_list);
list_append(bg_record->job_list, job_ptr);
jobinfo = job_ptr->select_jobinfo->data;
+ /* If you were switching from no sub-block
+ allocations to allowing it, the units_avail
+ wouldn't be around for any jobs, but no
+ problem since they were always the size of
+ the block.
+ */
+ if (!jobinfo->units_avail) {
+ jobinfo->units_avail =
+ bit_copy(ba_mp->cnode_bitmap);
+ bit_not(jobinfo->units_avail);
+ }
if (bit_overlap(ba_mp->cnode_bitmap,
jobinfo->units_avail)) {
error("we have an overlapping job allocation "
diff --git a/src/plugins/select/bluegene/bg_dynamic_block.c b/src/plugins/select/bluegene/bg_dynamic_block.c
index 2e688e821ada8cc1352090e8b14bd24baa74551c..192f7dcb8784fd13126a98d4e110fda49a0aa4ba 100644
--- a/src/plugins/select/bluegene/bg_dynamic_block.c
+++ b/src/plugins/select/bluegene/bg_dynamic_block.c
@@ -501,7 +501,6 @@ extern bg_record_t *create_small_record(bg_record_t *bg_record,
found_record->ionode_bitmap = bit_copy(ionodes);
ba_set_ionode_str(found_record);
- found_record->mp_used_bitmap = bit_alloc(node_record_count);
found_record->ba_mp_list = list_create(destroy_ba_mp);
diff --git a/src/plugins/select/bluegene/bg_record_functions.c b/src/plugins/select/bluegene/bg_record_functions.c
index 110bfb0220fa55a1fce2c49826ac5c21cc8f0f7f..f198e5d5432e016839234c1e0063734572786754 100644
--- a/src/plugins/select/bluegene/bg_record_functions.c
+++ b/src/plugins/select/bluegene/bg_record_functions.c
@@ -111,8 +111,6 @@ extern void destroy_bg_record(void *object)
xfree(bg_record->linuximage);
xfree(bg_record->mloaderimage);
FREE_NULL_BITMAP(bg_record->mp_bitmap);
- xfree(bg_record->mp_str);
- FREE_NULL_BITMAP(bg_record->mp_used_bitmap);
xfree(bg_record->ramdiskimage);
xfree(bg_record->reason);
@@ -361,14 +359,6 @@ extern void copy_bg_record(bg_record_t *fir_record, bg_record_t *sec_record)
sec_record->boot_state = fir_record->boot_state;
sec_record->boot_count = fir_record->boot_count;
- FREE_NULL_BITMAP(sec_record->mp_used_bitmap);
- if (fir_record->mp_used_bitmap
- && (sec_record->mp_used_bitmap
- = bit_copy(fir_record->mp_used_bitmap)) == NULL) {
- error("Unable to copy mp_used_bitmap for %s",
- fir_record->mp_str);
- sec_record->mp_used_bitmap = NULL;
- }
sec_record->cnode_cnt = fir_record->cnode_cnt;
sec_record->cnode_err_cnt = fir_record->cnode_err_cnt;
@@ -594,7 +584,6 @@ extern int add_bg_record(List records, List *used_nodes,
want anything set we also don't want the bg_record->ionode_str set.
*/
bg_record->ionode_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
- bg_record->mp_used_bitmap = bit_alloc(node_record_count);
len = strlen(blockreq->save_name);
i=0;
@@ -1401,7 +1390,8 @@ extern int bg_reset_block(bg_record_t *bg_record, struct job_record *job_ptr)
if (bg_record->job_list)
ba_remove_job_in_block_job_list(bg_record, job_ptr);
- if (bg_record->job_running > NO_JOB_RUNNING) {
+ if ((bg_record->job_running > NO_JOB_RUNNING)
+ && (!bg_record->job_list || !list_count(bg_record->job_list))) {
#ifndef HAVE_BG_L_P
/* Just in case the slurmctld wasn't up at the
time a step completion message came through
@@ -1420,10 +1410,11 @@ extern int bg_reset_block(bg_record_t *bg_record, struct job_record *job_ptr)
list_iterator_destroy(itr);
#endif
bg_record->job_running = NO_JOB_RUNNING;
- if (bg_record->job_ptr) {
- num_unused_cpus += bg_record->job_ptr->total_cpus;
- bg_record->job_ptr = NULL;
- }
+ }
+
+ if (bg_record->job_ptr) {
+ num_unused_cpus += bg_record->job_ptr->total_cpus;
+ bg_record->job_ptr = NULL;
}
/* remove user from list */
diff --git a/src/plugins/select/bluegene/bg_structs.h b/src/plugins/select/bluegene/bg_structs.h
index 926e7615c15d0c36fd22c60cfdf69ac55cc86c79..c9d593c30880c6b3b2e2f9619c38c0de00b86361 100644
--- a/src/plugins/select/bluegene/bg_structs.h
+++ b/src/plugins/select/bluegene/bg_structs.h
@@ -141,8 +141,6 @@ typedef struct bg_record {
of block */
int mp_count; /* size */
char *mp_str; /* String of midplanes in block */
- bitstr_t *mp_used_bitmap; /* midplanes used in this bitmap */
- char *mp_used_str; /* String of midplanes used in block */
uint16_t node_use; /* either COPROCESSOR or VIRTUAL */
struct bg_record *original; /* if this is a copy this is a
pointer to the original */
diff --git a/src/plugins/select/bluegene/bl_bgq/bridge_linker.cc b/src/plugins/select/bluegene/bl_bgq/bridge_linker.cc
index b79f4b55fd3451f9995717e0c838449e1f342475..f3b928dbecc349f432333983f804076bb8706deb 100644
--- a/src/plugins/select/bluegene/bl_bgq/bridge_linker.cc
+++ b/src/plugins/select/bluegene/bl_bgq/bridge_linker.cc
@@ -300,6 +300,8 @@ static void _remove_jobs_on_block_and_reset(char *block_id,
{
bg_record_t *bg_record = NULL;
int job_remove_failed = 0;
+ slurmctld_lock_t job_read_lock =
+ { NO_LOCK, READ_LOCK, NO_LOCK, NO_LOCK };
if (!block_id) {
error("_remove_jobs_on_block_and_reset: no block name given");
@@ -310,6 +312,13 @@ static void _remove_jobs_on_block_and_reset(char *block_id,
job_remove_failed = 1;
/* remove the block's users */
+
+ /* Lock job read before block to avoid
+ * issues where a step could complete after the job completion
+ * has taken place (since we are on a thread here).
+ */
+ if (job_ptr)
+ lock_slurmctld(job_read_lock);
slurm_mutex_lock(&block_state_mutex);
bg_record = find_bg_record_in_list(bg_lists->main, block_id);
if (bg_record) {
@@ -333,7 +342,8 @@ static void _remove_jobs_on_block_and_reset(char *block_id,
}
slurm_mutex_unlock(&block_state_mutex);
-
+ if (job_ptr)
+ unlock_slurmctld(job_read_lock);
}
extern int bridge_init(char *properties_file)
diff --git a/src/plugins/select/bluegene/select_bluegene.c b/src/plugins/select/bluegene/select_bluegene.c
index 6e97e5d5e5e4eb6cd66eb3eeb3851461e4db5854..269a30262abe7cbe2055aeb801d8f6c8630d8150 100644
--- a/src/plugins/select/bluegene/select_bluegene.c
+++ b/src/plugins/select/bluegene/select_bluegene.c
@@ -343,10 +343,9 @@ static void _set_bg_lists()
static bg_record_t *_translate_info_2_record(block_info_t *block_info)
{
bg_record_t *bg_record = NULL;
- bitstr_t *mp_bitmap = NULL, *ionode_bitmap = NULL, *used_bitmap = NULL;
+ bitstr_t *mp_bitmap = NULL, *ionode_bitmap = NULL;
mp_bitmap = bit_alloc(node_record_count);
- used_bitmap = bit_alloc(node_record_count);
ionode_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
if (block_info->mp_inx
@@ -354,11 +353,6 @@ static bg_record_t *_translate_info_2_record(block_info_t *block_info)
error("Job state recovered incompatible with "
"bluegene.conf. mp=%u",
node_record_count);
- if (block_info->mp_used_inx
- && inx2bitstr(used_bitmap, block_info->mp_used_inx) == -1)
- error("Job state recovered incompatible with "
- "bluegene.conf. used=%u",
- node_record_count);
if (block_info->ionode_inx
&& inx2bitstr(ionode_bitmap, block_info->ionode_inx) == -1)
error("Job state recovered incompatible with "
@@ -386,9 +380,6 @@ static bg_record_t *_translate_info_2_record(block_info_t *block_info)
}
}
- bg_record->mp_used_bitmap = used_bitmap;
- used_bitmap = NULL;
-
bg_record->mp_bitmap = mp_bitmap;
mp_bitmap = NULL;
@@ -441,8 +432,13 @@ static void _local_pack_block_job_info(struct job_record *job_ptr, Buf buffer,
memset(&block_job, 0, sizeof(block_job_info_t));
block_job.job_id = job_ptr->job_id;
block_job.user_id = job_ptr->user_id;
- block_job.user_name = jobinfo->user_name;
- block_job.cnodes = jobinfo->ionode_str;
+ if (jobinfo) {
+ block_job.user_name = jobinfo->user_name;
+ block_job.cnodes = jobinfo->ionode_str;
+ } else
+ error("NO JOBINFO for job %u magic %u!!!!!!!!!!!!!!",
+ job_ptr->job_id, job_ptr->magic);
+
/* block_job.cnode_inx -- try not to set */
slurm_pack_block_job_info(&block_job, buffer, protocol_version);
}
@@ -487,13 +483,17 @@ static void _pack_block(bg_record_t *bg_record, Buf buffer,
while ((job_ptr = list_next(itr))) {
if (job_ptr->magic != JOB_MAGIC) {
list_delete_item(itr);
+ slurm_pack_block_job_info(
+ NULL, buffer,
+ protocol_version);
continue;
}
_local_pack_block_job_info(
job_ptr, buffer, protocol_version);
}
list_iterator_destroy(itr);
- } else if (bg_record->job_ptr) {
+ } else if (bg_record->job_ptr
+ && (bg_record->job_ptr->magic == JOB_MAGIC)) {
pack32(1, buffer);
_local_pack_block_job_info(
bg_record->job_ptr, buffer, protocol_version);
@@ -505,13 +505,11 @@ static void _pack_block(bg_record_t *bg_record, Buf buffer,
packstr(bg_record->linuximage, buffer);
packstr(bg_record->mloaderimage, buffer);
packstr(bg_record->mp_str, buffer);
- packstr(bg_record->mp_used_str, buffer);
pack32((uint32_t)bg_record->cnode_cnt, buffer);
pack16((uint16_t)bg_record->node_use, buffer);
packstr(bg_record->ramdiskimage, buffer);
packstr(bg_record->reason, buffer);
pack16((uint16_t)bg_record->state, buffer);
- pack_bit_fmt(bg_record->mp_used_bitmap, buffer);
} else if (protocol_version >= SLURM_2_3_PROTOCOL_VERSION) {
packstr(bg_record->bg_block_id, buffer);
packstr(bg_record->blrtsimage, buffer);
@@ -551,14 +549,14 @@ static void _pack_block(bg_record_t *bg_record, Buf buffer,
packstr(bg_record->linuximage, buffer);
packstr(bg_record->mloaderimage, buffer);
packstr(bg_record->mp_str, buffer);
- packstr(bg_record->mp_used_str, buffer);
+ packnull(buffer); /* for mp_used_str */
pack32((uint32_t)bg_record->cnode_cnt, buffer);
pack16((uint16_t)bg_record->node_use, buffer);
packnull(buffer); /* for user_name */
packstr(bg_record->ramdiskimage, buffer);
packstr(bg_record->reason, buffer);
pack16((uint16_t)bg_record->state, buffer);
- pack_bit_fmt(bg_record->mp_used_bitmap, buffer);
+ packnull(buffer); /* for mp_used_inx */
} else if (protocol_version >= SLURM_2_2_PROTOCOL_VERSION) {
packstr(bg_record->bg_block_id, buffer);
#ifdef HAVE_BGL
@@ -1386,6 +1384,9 @@ extern int select_p_state_save(char *dir_name)
char *old_file, *new_file, *reg_file;
uint32_t blocks_packed = 0, tmp_offset, block_offset;
Buf buffer = init_buf(BUF_SIZE);
+ slurmctld_lock_t job_read_lock =
+ { NO_LOCK, READ_LOCK, NO_LOCK, NO_LOCK };
+
DEF_TIMERS;
debug("bluegene: select_p_state_save");
@@ -1395,6 +1396,9 @@ extern int select_p_state_save(char *dir_name)
block_offset = get_buf_offset(buffer);
pack32(blocks_packed, buffer);
+ /* Lock job read before block to avoid deadlock job lock is
+ * needed because we look at the job_ptr's to send job info. */
+ lock_slurmctld(job_read_lock);
/* write block records to buffer */
slurm_mutex_lock(&block_state_mutex);
itr = list_iterator_create(bg_lists->main);
@@ -1410,6 +1414,7 @@ extern int select_p_state_save(char *dir_name)
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
+ unlock_slurmctld(job_read_lock);
tmp_offset = get_buf_offset(buffer);
set_buf_offset(buffer, block_offset);
pack32(blocks_packed, buffer);
@@ -1812,7 +1817,7 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
{
bitstr_t *picked_mps = NULL;
bg_record_t *bg_record = NULL;
- char *tmp_char = NULL, *tmp_char2 = NULL;
+ char *tmp_char = NULL;
ba_mp_t *ba_mp = NULL;
select_jobinfo_t *jobinfo = NULL;
int dim;
@@ -1829,7 +1834,6 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
"trying to start a step on it?",
job_ptr->job_id);
- xassert(bg_record->mp_used_bitmap);
xassert(!step_jobinfo->units_used);
xfree(step_jobinfo->bg_block_id);
@@ -1872,7 +1876,6 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
bit_or(ba_mp->cnode_bitmap, step_jobinfo->units_used);
}
- bit_or(bg_record->mp_used_bitmap, picked_mps);
step_jobinfo->ionode_str = xstrdup(jobinfo->ionode_str);
} else if (jobinfo->units_avail) {
bitstr_t *total_bitmap = jobinfo->units_used;
@@ -1897,7 +1900,6 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
node_count = step_jobinfo->cnode_cnt;
if (!(picked_mps = bit_copy(job_ptr->node_bitmap)))
fatal("bit_copy malloc failure");
- bit_or(bg_record->mp_used_bitmap, picked_mps);
bit_or(jobinfo->units_used, step_jobinfo->units_used);
for (dim = 0; dim < step_jobinfo->dim_cnt; dim++) {
/* The IBM software works off a relative
@@ -1923,7 +1925,6 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
bg_record, &node_count, step_jobinfo))) {
if (!(picked_mps = bit_alloc(bit_size(job_ptr->node_bitmap))))
fatal("bit_copy malloc failure");
- bit_set(bg_record->mp_used_bitmap, ba_mp->index);
bit_set(picked_mps, ba_mp->index);
for (dim = 0; dim < step_jobinfo->dim_cnt; dim++) {
/* The IBM software works off a relative
@@ -1951,14 +1952,10 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
if (picked_mps) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK) {
tmp_char = bitmap2node_name(picked_mps);
- tmp_char2 = bitmap2node_name(
- bg_record->mp_used_bitmap);
- info("select_p_step_pick_nodes: picked %s mps on "
- "block %s used is now %s",
- tmp_char, bg_record->bg_block_id,
- tmp_char2);
+ info("select_p_step_pick_nodes: new step for job %u "
+ "will be running on %s(%s)",
+ job_ptr->job_id, bg_record->bg_block_id, tmp_char);
xfree(tmp_char);
- xfree(tmp_char2);
}
step_jobinfo->cnode_cnt = node_count;
}
@@ -1975,46 +1972,42 @@ extern int select_p_step_finish(struct step_record *step_ptr)
bg_record_t *bg_record = NULL;
select_jobinfo_t *jobinfo = NULL;
int rc = SLURM_SUCCESS;
- char *tmp_char = NULL, *tmp_char2 = NULL;
+ char *tmp_char = NULL;
xassert(step_ptr);
- slurm_mutex_lock(&block_state_mutex);
-
- jobinfo = step_ptr->job_ptr->select_jobinfo->data;
- bg_record = jobinfo->bg_record;
- if (!bg_record)
- fatal("This step %u.%u does not have a bg block "
- "assigned to it, but for some reason we are "
- "trying to end the step?",
+ if (IS_JOB_COMPLETING(step_ptr->job_ptr)) {
+ debug("step completion %u.%u was received after job "
+ "allocation is already completing, no cleanup needed",
step_ptr->job_ptr->job_id, step_ptr->step_id);
- /* At this moment the step_node_bitmap has already been
- cleared and the step_node_bitmap has been set so use it
- instead.
- */
- bit_not(step_ptr->step_node_bitmap);
- bit_and(bg_record->mp_used_bitmap, step_ptr->step_node_bitmap);
- bit_not(step_ptr->step_node_bitmap);
-
- if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK) {
- tmp_char = bitmap2node_name(bg_record->mp_used_bitmap);
- tmp_char2 = bitmap2node_name(step_ptr->step_node_bitmap);
- info("select_p_step_finish: cleared %s "
- "from job step %u.%u, now %s used",
- tmp_char2, step_ptr->job_ptr->job_id, step_ptr->step_id,
- tmp_char);
- xfree(tmp_char);
- xfree(tmp_char2);
+ return SLURM_SUCCESS;
}
+ jobinfo = step_ptr->job_ptr->select_jobinfo->data;
+
if (jobinfo->units_avail)
rc = ba_sub_block_in_bitmap_clear(
step_ptr->select_jobinfo->data, jobinfo->units_used);
- else
+ else {
+ slurm_mutex_lock(&block_state_mutex);
+ bg_record = jobinfo->bg_record;
+
+ if (!bg_record)
+ fatal("This step %u.%u does not have a bg block "
+ "assigned to it, but for some reason we are "
+ "trying to end the step?",
+ step_ptr->job_ptr->job_id, step_ptr->step_id);
rc = ba_sub_block_in_record_clear(bg_record, step_ptr);
+ slurm_mutex_unlock(&block_state_mutex);
+ }
- slurm_mutex_unlock(&block_state_mutex);
+ if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK) {
+ tmp_char = bitmap2node_name(step_ptr->step_node_bitmap);
+ info("select_p_step_finish: step %u.%u cleared from %s",
+ step_ptr->job_ptr->job_id, step_ptr->step_id, tmp_char);
+ xfree(tmp_char);
+ }
return rc;
}
@@ -2043,6 +2036,14 @@ extern int select_p_pack_select_info(time_t last_query_time,
if (protocol_version >= SLURM_2_1_PROTOCOL_VERSION) {
if (bg_lists->main) {
+ slurmctld_lock_t job_read_lock =
+ { NO_LOCK, READ_LOCK,
+ NO_LOCK, NO_LOCK };
+ /* Lock job read before block to avoid
+ * deadlock job lock is needed because
+ * we look at the job_ptr's to send
+ * job info. */
+ lock_slurmctld(job_read_lock);
slurm_mutex_lock(&block_state_mutex);
itr = list_iterator_create(bg_lists->main);
while ((bg_record = list_next(itr))) {
@@ -2054,6 +2055,7 @@ extern int select_p_pack_select_info(time_t last_query_time,
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
+ unlock_slurmctld(job_read_lock);
} else {
error("select_p_pack_select_info: "
"no bg_lists->main");