From bd2eb2ffb0e67aae0a1149f43dc8a6536f806c44 Mon Sep 17 00:00:00 2001
From: Danny Auble <da@llnl.gov>
Date: Tue, 24 Nov 2009 23:44:32 +0000
Subject: [PATCH] more modifications for a real bluegene system. 1. sfree and
scontrol now free blocks. 2. correct connection type is given for jobs now.
3. configuring jobs are set at the correct time now
---
src/common/slurm_protocol_defs.c | 10 ++--
.../select/bluegene/plugin/bg_block_info.c | 1 -
.../select/bluegene/plugin/bg_job_place.c | 10 +++-
.../select/bluegene/plugin/bg_job_run.c | 44 +++++++-------
.../bluegene/plugin/bg_record_functions.c | 9 +--
src/plugins/select/bluegene/plugin/bluegene.c | 27 ++++++---
src/plugins/select/bluegene/plugin/bluegene.h | 2 +-
src/plugins/select/bluegene/plugin/jobinfo.c | 57 +++++--------------
.../select/bluegene/plugin/select_bluegene.c | 1 +
src/slurmctld/node_scheduler.c | 11 +++-
src/slurmctld/proc_req.c | 4 +-
src/smap/job_functions.c | 6 +-
12 files changed, 92 insertions(+), 90 deletions(-)
diff --git a/src/common/slurm_protocol_defs.c b/src/common/slurm_protocol_defs.c
index 67471db6d2f..a99997ceff9 100644
--- a/src/common/slurm_protocol_defs.c
+++ b/src/common/slurm_protocol_defs.c
@@ -1254,11 +1254,11 @@ extern char *conn_type_string(enum connection_type conn_type)
{
switch (conn_type) {
case (SELECT_MESH):
- return "MESH";
+ return "Mesh";
case (SELECT_TORUS):
- return "TORUS";
+ return "Torus";
case (SELECT_SMALL):
- return "SMALL";
+ return "Small";
case (SELECT_NAV):
return "NAV";
#ifndef HAVE_BGL
@@ -1272,9 +1272,9 @@ extern char *conn_type_string(enum connection_type conn_type)
return "HTC_L";
#endif
default:
- return "";
+ return "n/a";
}
- return "";
+ return "n/a";
}
#ifdef HAVE_BGL
diff --git a/src/plugins/select/bluegene/plugin/bg_block_info.c b/src/plugins/select/bluegene/plugin/bg_block_info.c
index 9a819591202..892122017ea 100644
--- a/src/plugins/select/bluegene/plugin/bg_block_info.c
+++ b/src/plugins/select/bluegene/plugin/bg_block_info.c
@@ -470,7 +470,6 @@ extern int update_block_list()
bg_record->job_ptr->job_state |=
JOB_CONFIGURING;
last_job_update = time(NULL);
-
}
break;
case RM_PARTITION_ERROR:
diff --git a/src/plugins/select/bluegene/plugin/bg_job_place.c b/src/plugins/select/bluegene/plugin/bg_job_place.c
index 8313f3a29aa..a0cec8d9f40 100644
--- a/src/plugins/select/bluegene/plugin/bg_job_place.c
+++ b/src/plugins/select/bluegene/plugin/bg_job_place.c
@@ -1603,10 +1603,16 @@ preempt:
job_ptr->select_jobinfo,
SELECT_JOBDATA_BLOCK_ID,
bg_record->bg_block_id);
+ conn_type = bg_record->conn_type;
select_g_select_jobinfo_set(
job_ptr->select_jobinfo,
SELECT_JOBDATA_CONN_TYPE,
- &bg_record->conn_type);
+ &conn_type);
+ if(job_ptr) {
+ job_ptr->job_state
+ |= JOB_CONFIGURING;
+ last_job_update = time(NULL);
+ }
} else
select_g_select_jobinfo_set(
job_ptr->select_jobinfo,
@@ -1642,7 +1648,7 @@ preempt:
if(bg_conf->layout_mode == LAYOUT_DYNAMIC) {
slurm_mutex_lock(&block_state_mutex);
if(blocks_added)
- _sync_block_lists(block_list, bg_lists->main);
+ _sync_block_lists(block_list, bg_lists->main);
slurm_mutex_unlock(&block_state_mutex);
slurm_mutex_unlock(&create_dynamic_mutex);
}
diff --git a/src/plugins/select/bluegene/plugin/bg_job_run.c b/src/plugins/select/bluegene/plugin/bg_job_run.c
index c8dda9d9c9a..836dff60a52 100644
--- a/src/plugins/select/bluegene/plugin/bg_job_run.c
+++ b/src/plugins/select/bluegene/plugin/bg_job_run.c
@@ -594,15 +594,16 @@ static void _start_agent(bg_update_t *bg_update_ptr)
return;
}
if(bg_record->state == RM_PARTITION_DEALLOCATING) {
- slurm_mutex_unlock(&block_state_mutex);
debug("Block is in Deallocating state, waiting for free.");
- bg_free_block(bg_record);
+ bg_free_block(bg_record, 1, 1);
/* no reason to reboot here since we are already
deallocating */
bg_update_ptr->reboot = 0;
- slurm_mutex_lock(&block_state_mutex);
- if(!_make_sure_block_still_exists(bg_update_ptr, bg_record))
+ if(!_make_sure_block_still_exists(bg_update_ptr, bg_record)) {
+ slurm_mutex_unlock(&block_state_mutex);
+ slurm_mutex_unlock(&job_start_mutex);
return;
+ }
}
delete_list = list_create(NULL);
@@ -682,8 +683,11 @@ static void _start_agent(bg_update_t *bg_update_ptr)
num_block_to_free = num_block_freed = 0;
slurm_mutex_lock(&block_state_mutex);
- if(!_make_sure_block_still_exists(bg_update_ptr, bg_record))
+ if(!_make_sure_block_still_exists(bg_update_ptr, bg_record)) {
+ slurm_mutex_unlock(&block_state_mutex);
+ slurm_mutex_unlock(&job_start_mutex);
return;
+ }
if(bg_record->job_running <= NO_JOB_RUNNING) {
// _reset_block(bg_record); should already happened
@@ -751,13 +755,14 @@ static void _start_agent(bg_update_t *bg_update_ptr)
if(rc) {
bg_record->modifying = 1;
- slurm_mutex_unlock(&block_state_mutex);
- bg_free_block(bg_record);
+ bg_free_block(bg_record, 1, 1);
- slurm_mutex_lock(&block_state_mutex);
- if(!_make_sure_block_still_exists(bg_update_ptr, bg_record))
+ if(!_make_sure_block_still_exists(bg_update_ptr, bg_record)) {
+ slurm_mutex_unlock(&block_state_mutex);
+ slurm_mutex_unlock(&job_start_mutex);
return;
+ }
#ifdef HAVE_BG_FILES
#ifdef HAVE_BGL
if ((rc = bridge_modify_block(bg_record->bg_block_id,
@@ -835,13 +840,14 @@ static void _start_agent(bg_update_t *bg_update_ptr)
bg_record->modifying = 0;
} else if(bg_update_ptr->reboot) {
bg_record->modifying = 1;
- slurm_mutex_unlock(&block_state_mutex);
- bg_free_block(bg_record);
+ bg_free_block(bg_record, 1, 1);
- slurm_mutex_lock(&block_state_mutex);
- if(!_make_sure_block_still_exists(bg_update_ptr, bg_record))
+ if(!_make_sure_block_still_exists(bg_update_ptr, bg_record)) {
+ slurm_mutex_unlock(&block_state_mutex);
+ slurm_mutex_unlock(&job_start_mutex);
return;
+ }
bg_record->modifying = 0;
}
@@ -871,8 +877,11 @@ static void _start_agent(bg_update_t *bg_update_ptr)
return;
}
slurm_mutex_lock(&block_state_mutex);
- if(!_make_sure_block_still_exists(bg_update_ptr, bg_record))
+ if(!_make_sure_block_still_exists(bg_update_ptr, bg_record)) {
+ slurm_mutex_unlock(&block_state_mutex);
+ slurm_mutex_unlock(&job_start_mutex);
return;
+ }
} else if (bg_record->state == RM_PARTITION_CONFIGURING)
bg_record->boot_state = 1;
@@ -888,7 +897,8 @@ static void _start_agent(bg_update_t *bg_update_ptr)
bg_record->boot_count = 0;
xfree(bg_record->target_name);
- bg_record->target_name = uid_to_string(bg_update_ptr->job_ptr->user_id);
+ bg_record->target_name =
+ uid_to_string(bg_update_ptr->job_ptr->user_id);
debug("setting the target_name for Block %s to %s",
bg_record->bg_block_id, bg_record->target_name);
@@ -1450,10 +1460,6 @@ extern int boot_block(bg_record_t *bg_record)
if(bg_record->state != RM_PARTITION_CONFIGURING)
bg_record->state = RM_PARTITION_CONFIGURING;
debug("Setting bootflag for %s", bg_record->bg_block_id);
- if(bg_record->job_ptr) {
- bg_record->job_ptr->job_state |= JOB_CONFIGURING;
- last_job_update = time(NULL);
- }
bg_record->boot_state = 1;
//bg_record->boot_count = 0;
last_bg_update = time(NULL);
diff --git a/src/plugins/select/bluegene/plugin/bg_record_functions.c b/src/plugins/select/bluegene/plugin/bg_record_functions.c
index 1457cb4f8e4..d785675aea1 100644
--- a/src/plugins/select/bluegene/plugin/bg_record_functions.c
+++ b/src/plugins/select/bluegene/plugin/bg_record_functions.c
@@ -1461,12 +1461,13 @@ extern int resume_block(bg_record_t *bg_record)
{
xassert(bg_record);
- if(bg_record->job_running >= NO_JOB_RUNNING)
+ if(bg_record->job_running > NO_JOB_RUNNING)
return SLURM_SUCCESS;
- info("Block %s put back into service after "
- "being in an error state.",
- bg_record->bg_block_id);
+ if(bg_record->state == RM_PARTITION_ERROR)
+ info("Block %s put back into service after "
+ "being in an error state.",
+ bg_record->bg_block_id);
if(remove_from_bg_list(bg_lists->job_running, bg_record)
== SLURM_SUCCESS)
diff --git a/src/plugins/select/bluegene/plugin/bluegene.c b/src/plugins/select/bluegene/plugin/bluegene.c
index 3df42706ae2..b9859e1b5d5 100644
--- a/src/plugins/select/bluegene/plugin/bluegene.c
+++ b/src/plugins/select/bluegene/plugin/bluegene.c
@@ -448,23 +448,28 @@ extern bg_record_t *find_org_in_bg_list(List my_list, bg_record_t *bg_record)
return found_record;
}
-extern int bg_free_block(bg_record_t *bg_record)
+extern int bg_free_block(bg_record_t *bg_record, bool wait, bool locked)
{
#ifdef HAVE_BG_FILES
int rc;
+ int first=1;
#endif
if(!bg_record) {
error("bg_free_block: there was no bg_record");
return SLURM_ERROR;
}
-
+
+ if(!locked)
+ first = 0;
+
while (1) {
/* Here we don't need to check if the block is still
* in exsistance since this function can't be called on
* the same block twice. It may
* had already been removed at this point also.
*/
- slurm_mutex_lock(&block_state_mutex);
+ if(!first)
+ slurm_mutex_lock(&block_state_mutex);
if (bg_record->state != NO_VAL
&& bg_record->state != RM_PARTITION_FREE
&& bg_record->state != RM_PARTITION_DEALLOCATING) {
@@ -495,18 +500,24 @@ extern int bg_free_block(bg_record_t *bg_record)
#endif
}
- if ((bg_record->state == RM_PARTITION_FREE)
+ if (!wait || (bg_record->state == RM_PARTITION_FREE)
#ifdef HAVE_BGL
|| (bg_record->state == RM_PARTITION_ERROR)
#endif
) {
break;
}
- slurm_mutex_unlock(&block_state_mutex);
+ /* If we were locked outside of this we need to unlock
+ to not cause deadlock on this mutex until we are
+ done.
+ */
+ slurm_mutex_unlock(&block_state_mutex);
sleep(3);
+ first = 0;
}
remove_from_bg_list(bg_lists->booted, bg_record);
- slurm_mutex_unlock(&block_state_mutex);
+ if(!locked)
+ slurm_mutex_unlock(&block_state_mutex);
return SLURM_SUCCESS;
}
@@ -539,7 +550,7 @@ extern void *mult_free_block(void *args)
term_jobs_on_block(bg_record->bg_block_id);
}
debug("freeing the block %s.", bg_record->bg_block_id);
- bg_free_block(bg_record);
+ bg_free_block(bg_record, 1, 0);
debug("done\n");
slurm_mutex_lock(&freed_cnt_mutex);
num_block_freed++;
@@ -603,7 +614,7 @@ extern void *mult_destroy_block(void *args)
term_jobs_on_block(bg_record->bg_block_id);
debug2("destroying %s", (char *)bg_record->bg_block_id);
- if(bg_free_block(bg_record) == SLURM_ERROR) {
+ if(bg_free_block(bg_record, 1, 0) == SLURM_ERROR) {
debug("there was an error");
goto already_here;
}
diff --git a/src/plugins/select/bluegene/plugin/bluegene.h b/src/plugins/select/bluegene/plugin/bluegene.h
index 2a979f73394..4d7a616d829 100644
--- a/src/plugins/select/bluegene/plugin/bluegene.h
+++ b/src/plugins/select/bluegene/plugin/bluegene.h
@@ -154,7 +154,7 @@ extern void *block_agent(void *args);
* nodes, nodecards, and switches */
extern void *state_agent(void *args);
-extern int bg_free_block(bg_record_t *bg_record);
+extern int bg_free_block(bg_record_t *bg_record, bool wait, bool locked);
extern int remove_from_bg_list(List my_bg_list, bg_record_t *bg_record);
extern bg_record_t *find_and_remove_org_from_bg_list(List my_list,
diff --git a/src/plugins/select/bluegene/plugin/jobinfo.c b/src/plugins/select/bluegene/plugin/jobinfo.c
index 9b18113c071..39188762e16 100644
--- a/src/plugins/select/bluegene/plugin/jobinfo.c
+++ b/src/plugins/select/bluegene/plugin/jobinfo.c
@@ -41,37 +41,6 @@
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"
-static char *_job_conn_type_string(uint16_t inx)
-{
- switch(inx) {
- case SELECT_TORUS:
- return "torus";
- break;
- case SELECT_MESH:
- return "mesh";
- break;
- case SELECT_SMALL:
- return "small";
- break;
-#ifndef HAVE_BGL
- case SELECT_HTC_S:
- return "htc_s";
- break;
- case SELECT_HTC_D:
- return "htc_d";
- break;
- case SELECT_HTC_V:
- return "htc_v";
- break;
- case SELECT_HTC_L:
- return "htc_l";
- break;
-#endif
- default:
- return "n/a";
- }
-}
-
static char *_yes_no_string(uint16_t inx)
{
if (inx == (uint16_t) NO_VAL)
@@ -471,7 +440,7 @@ unpack_error:
* RET - the string, same as buf
*/
extern char *sprint_select_jobinfo(select_jobinfo_t *jobinfo,
- char *buf, size_t size, int mode)
+ char *buf, size_t size, int mode)
{
uint16_t geometry[SYSTEM_DIMENSIONS];
int i;
@@ -516,7 +485,7 @@ extern char *sprint_select_jobinfo(select_jobinfo_t *jobinfo,
UNIT_NONE);
snprintf(buf, size,
"%7.7s %6.6s %6.6s %8s %cx%cx%c %-16s",
- _job_conn_type_string(jobinfo->conn_type),
+ conn_type_string(jobinfo->conn_type),
_yes_no_string(jobinfo->reboot),
_yes_no_string(jobinfo->rotate),
max_cpus_char,
@@ -529,7 +498,7 @@ extern char *sprint_select_jobinfo(select_jobinfo_t *jobinfo,
snprintf(buf, size,
"Connection=%s Reboot=%s Rotate=%s "
"Geometry=%cx%cx%c",
- _job_conn_type_string(jobinfo->conn_type),
+ conn_type_string(jobinfo->conn_type),
_yes_no_string(jobinfo->reboot),
_yes_no_string(jobinfo->rotate),
alpha_num[geometry[0]],
@@ -540,7 +509,7 @@ extern char *sprint_select_jobinfo(select_jobinfo_t *jobinfo,
snprintf(buf, size,
"Connection=%s Reboot=%s Rotate=%s "
"Geometry=%cx%cx%c Block_ID=%s",
- _job_conn_type_string(jobinfo->conn_type),
+ conn_type_string(jobinfo->conn_type),
_yes_no_string(jobinfo->reboot),
_yes_no_string(jobinfo->rotate),
alpha_num[geometry[0]],
@@ -560,7 +529,7 @@ extern char *sprint_select_jobinfo(select_jobinfo_t *jobinfo,
break;
case SELECT_PRINT_CONNECTION:
snprintf(buf, size, "%s",
- _job_conn_type_string(jobinfo->conn_type));
+ conn_type_string(jobinfo->conn_type));
break;
case SELECT_PRINT_REBOOT:
snprintf(buf, size, "%s",
@@ -663,7 +632,7 @@ extern char *xstrdup_select_jobinfo(select_jobinfo_t *jobinfo, int mode)
UNIT_NONE);
xstrfmtcat(buf,
"%7.7s %6.6s %6.6s %8s %cx%cx%c %-16s",
- _job_conn_type_string(jobinfo->conn_type),
+ conn_type_string(jobinfo->conn_type),
_yes_no_string(jobinfo->reboot),
_yes_no_string(jobinfo->rotate),
max_cpus_char,
@@ -676,7 +645,7 @@ extern char *xstrdup_select_jobinfo(select_jobinfo_t *jobinfo, int mode)
xstrfmtcat(buf,
"Connection=%s Reboot=%s Rotate=%s "
"Geometry=%cx%cx%c Block_ID=%s",
- _job_conn_type_string(jobinfo->conn_type),
+ conn_type_string(jobinfo->conn_type),
_yes_no_string(jobinfo->reboot),
_yes_no_string(jobinfo->rotate),
alpha_num[geometry[0]],
@@ -696,21 +665,21 @@ extern char *xstrdup_select_jobinfo(select_jobinfo_t *jobinfo, int mode)
break;
case SELECT_PRINT_CONNECTION:
xstrfmtcat(buf, "%s",
- _job_conn_type_string(jobinfo->conn_type));
+ conn_type_string(jobinfo->conn_type));
break;
case SELECT_PRINT_REBOOT:
xstrfmtcat(buf, "%s",
- _yes_no_string(jobinfo->reboot));
+ _yes_no_string(jobinfo->reboot));
break;
case SELECT_PRINT_ROTATE:
xstrfmtcat(buf, "%s",
- _yes_no_string(jobinfo->rotate));
+ _yes_no_string(jobinfo->rotate));
break;
case SELECT_PRINT_GEOMETRY:
xstrfmtcat(buf, "%cx%cx%c",
- alpha_num[geometry[0]],
- alpha_num[geometry[1]],
- alpha_num[geometry[2]]);
+ alpha_num[geometry[0]],
+ alpha_num[geometry[1]],
+ alpha_num[geometry[2]]);
break;
case SELECT_PRINT_MAX_CPUS:
if (jobinfo->max_cpus == NO_VAL)
diff --git a/src/plugins/select/bluegene/plugin/select_bluegene.c b/src/plugins/select/bluegene/plugin/select_bluegene.c
index f1b49d578ae..97f1014293a 100644
--- a/src/plugins/select/bluegene/plugin/select_bluegene.c
+++ b/src/plugins/select/bluegene/plugin/select_bluegene.c
@@ -795,6 +795,7 @@ extern int select_p_update_block (update_block_msg_t *block_desc_ptr)
slurm_mutex_unlock(&block_state_mutex);
put_block_in_error_state(bg_record, BLOCK_ERROR_STATE, reason);
} else if(block_desc_ptr->state == RM_PARTITION_FREE) {
+ bg_free_block(bg_record, 0, 1);
resume_block(bg_record);
slurm_mutex_unlock(&block_state_mutex);
} else if (bg_conf->layout_mode == LAYOUT_DYNAMIC
diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index fbeeb57b2b4..dbde1f803a3 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -1041,6 +1041,7 @@ extern int select_nodes(struct job_record *job_ptr, bool test_only,
uint32_t min_nodes, max_nodes, req_nodes;
enum job_state_reason fail_reason;
time_t now = time(NULL);
+ bool configuring = false;
List preemptee_job_list = NULL;
xassert(job_ptr);
@@ -1062,7 +1063,7 @@ extern int select_nodes(struct job_record *job_ptr, bool test_only,
fail_reason = WAIT_NO_REASON;
if (part_ptr->state_up == 0)
fail_reason = WAIT_PART_STATE;
- else if (job_ptr->priority == 0) /* user or administrator hold */
+ else if (job_ptr->priority == 0) /* user or administrator hold */
fail_reason = WAIT_HELD;
else if ((job_ptr->time_limit != NO_VAL) &&
(job_ptr->time_limit > part_ptr->max_time))
@@ -1213,8 +1214,14 @@ extern int select_nodes(struct job_record *job_ptr, bool test_only,
select_bitmap = NULL; /* nothing left to free */
allocate_nodes(job_ptr);
build_node_details(job_ptr);
+
+ /* This could be set in the select plugin so we want to keep
+ the flag. */
+ configuring = IS_JOB_CONFIGURING(job_ptr);
+
job_ptr->job_state = JOB_RUNNING;
- if (bit_overlap(job_ptr->node_bitmap, power_node_bitmap))
+ if (configuring
+ || bit_overlap(job_ptr->node_bitmap, power_node_bitmap))
job_ptr->job_state |= JOB_CONFIGURING;
if (select_g_select_nodeinfo_set(job_ptr) != SLURM_SUCCESS) {
error("select_g_update_nodeinfo(%u): %m", job_ptr->job_id);
diff --git a/src/slurmctld/proc_req.c b/src/slurmctld/proc_req.c
index d3ab6df256f..d9da76dd3f9 100644
--- a/src/slurmctld/proc_req.c
+++ b/src/slurmctld/proc_req.c
@@ -2772,12 +2772,12 @@ static void _slurm_rpc_update_block(slurm_msg_t * msg)
name = block_desc_ptr->bg_block_id;
} else if(block_desc_ptr->nodes) {
error_code = select_g_update_sub_node(block_desc_ptr);
- END_TIMER2("_slurm_rpc_update_subbp");
+ END_TIMER2("_slurm_rpc_update_subbp");
name = block_desc_ptr->nodes;
} else {
error("Unknown update for blocks");
error_code = SLURM_ERROR;
- END_TIMER2("_slurm_rpc_update_block");
+ END_TIMER2("_slurm_rpc_update_block");
}
}
diff --git a/src/smap/job_functions.c b/src/smap/job_functions.c
index 1363a498b61..6b67eb43a66 100644
--- a/src/smap/job_functions.c
+++ b/src/smap/job_functions.c
@@ -165,7 +165,8 @@ extern void get_job(void)
if((count>=text_line_cnt)
&& (printed_jobs
< (text_win->_maxy-3))) {
- job_ptr->nodes = "waiting...";
+ xfree(job_ptr->nodes);
+ job_ptr->nodes = xstrdup("waiting...");
job_ptr->num_procs = (int) letters[count%62];
wattron(text_win,
COLOR_PAIR(colors[count%6]));
@@ -175,7 +176,8 @@ extern void get_job(void)
printed_jobs++;
}
} else {
- job_ptr->nodes = "waiting...";
+ xfree(job_ptr->nodes);
+ job_ptr->nodes = xstrdup("waiting...");
job_ptr->num_procs = (int) letters[count%62];
_print_text_job(job_ptr);
printed_jobs++;
--
GitLab