diff --git a/NEWS b/NEWS
index b2271f76c8e2d0edc796326a1c5a2f3f69a27cde..49fee380be548dc39c81899f40e8f8cd0686ac8b 100644
--- a/NEWS
+++ b/NEWS
@@ -53,6 +53,10 @@ documents those changes that are of interest to users and admins.
-- BLUEGENE - Handle job completion correctly if an admin removes a block
where other blocks on an overlapping midplane are running jobs.
-- BLUEGENE - correctly remove running jobs when freeing a block.
+ -- BGQ - correct logic to place multiple (< 1 midplane) steps inside a
+ multi midplane block allocation.
+ -- BGQ - Make it possible for a multi midplane allocation to run on more
+ than 1 midplane but not the entire allocation.
* Changes in SLURM 2.4.1
========================
diff --git a/doc/html/high_throughput.shtml b/doc/html/high_throughput.shtml
index 9dbb2e479edd5805aed84ba83931fca60b833a34..8a4606d8a280305430b6e7ed65222f9d925d2fa8 100644
--- a/doc/html/high_throughput.shtml
+++ b/doc/html/high_throughput.shtml
@@ -52,6 +52,9 @@ be set quite high for memory size, open file count and stack size.</p>
<h2>SLURM Configuration</h2>
+<p>NOTE: Substantial changes were made in SLURM version 2.4 to support higher
+throughput rates. Version 2.5 includes more enhancements.</p>
+
<p>Several SLURM configuration parameters should be adjusted to
reflect the needs of high throughput computing. The changes described below
will not be possible in all environments, but these are the configuration
@@ -83,7 +86,12 @@ Controls how soon the record of a completed job can be purged from the
<b>slurmctld</b> memory and thus not visible using the <b>squeue</b> command.
The record of jobs run will be preserved in accounting records and logs.
The default value is 300 seconds. The value should be reduced to a few
-seconds if possible.</li>
+seconds if possible. Use of accounting records for older jobs can increase
+the job throughput rate compared with retaining old jobs in the memory of
+the slurmctld daemon.</li>
+<li><b>PriorityType</b>:
+The <i>priority/builtin</i> is considerably faster than other options, but
+schedules jobs only on a First In First Out (FIFO) basis.</li>
<li><b>SchedulerParameters</b>:
Several scheduling parameters are available.
<ul>
@@ -126,10 +134,20 @@ messages on more than one port in order to avoid having incoming messages
discarded by the operating system due to exceeding the SOMAXCONN limit
described above. Using between two and ten ports is suggested when large
numbers of simultaneous requests are to be supported.</li>
+<li><b>SlurmctldDebug</b>:
+More detailed logging will decrease system throughput. Set to 2 (log errors
+only) or 3 (general information logging). Each increment in the logging level
+will increase the number of message by a factor of about 3.</li>
+<li><b>SlurmdDebug</b>:
+More detailed logging will decrease system throughput. Set to 2 (log errors
+only) or 3 (general information logging). Each increment in the logging level
+will increase the number of message by a factor of about 3.</li>
+<li><b>SlurmdLogFile</b>:
+Writing to local storage is recommended.</li>
<li>Other: Configure logging, accounting and other overhead to a minimum
appropriate for your environment.</li>
</ul>
-<p style="text-align:center;">Last modified 5 June 2012</p>
+<p style="text-align:center;">Last modified 12 July 2012</p>
<!--#include virtual="footer.txt"-->
diff --git a/src/plugins/select/bluegene/ba_bgq/block_allocator.c b/src/plugins/select/bluegene/ba_bgq/block_allocator.c
index 61a6a74b652cc10e0b45171086efb2f54ba98083..ee4c5712a7b0ffa67f33531503be694bbf0c5238 100644
--- a/src/plugins/select/bluegene/ba_bgq/block_allocator.c
+++ b/src/plugins/select/bluegene/ba_bgq/block_allocator.c
@@ -1154,17 +1154,8 @@ try_again:
break;
clear_cnt = bit_clear_count(total_bitmap);
+
FREE_NULL_BITMAP(total_bitmap);
- /* User asked for a bad CPU count or we can't place it
- here in this small allocation. */
- if (jobinfo->cnode_cnt < bg_conf->mp_cnode_cnt) {
- list_iterator_destroy(itr);
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("We couldn't place a sub block of %d",
- *node_count);
- (*node_count)++;
- goto try_again;
- }
/* Grab the most empty midplane to be used later if we
can't find a spot.
@@ -1721,6 +1712,7 @@ extern int ba_translate_coord2nc(uint16_t *cnode_coords)
return nc_loc;
}
+/* ba_system_mutex needs to be locked before calling this. */
extern ba_mp_t *ba_inx2ba_mp(int inx)
{
return ba_main_grid_array[inx];
diff --git a/src/plugins/select/bluegene/select_bluegene.c b/src/plugins/select/bluegene/select_bluegene.c
index 8945e0ce6a2a61de0a5290f88f34b33d0291495e..86b121fb5ac16172f7fa48a4e3dc8380c34193e7 100644
--- a/src/plugins/select/bluegene/select_bluegene.c
+++ b/src/plugins/select/bluegene/select_bluegene.c
@@ -1951,8 +1951,10 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
if (((cluster_flags & CLUSTER_FLAG_BGL)
|| (cluster_flags & CLUSTER_FLAG_BGP))
- || (node_count == bg_record->cnode_cnt)) {
- /* If we are using the whole block we need to verify
+ || ((node_count == bg_record->cnode_cnt)
+ || (node_count > bg_conf->mp_cnode_cnt))) {
+ /* If we are using the whole block (or more than 1
+ midplane of it) we need to verify
if anything else is used. If anything else is used
return NULL, else return that we can use the entire
thing.
@@ -1963,7 +1965,8 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
if (list_count(job_ptr->step_list)) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("select_p_step_pick_nodes: Looking "
- "for the entire block %s for job %u, "
+ "for more than one midplane of "
+ "block %s for job %u, "
"but some of it is used.",
bg_record->bg_block_id, job_ptr->job_id);
goto end_it;
@@ -1971,9 +1974,22 @@ extern bitstr_t *select_p_step_pick_nodes(struct job_record *job_ptr,
if (!(picked_mps = bit_copy(job_ptr->node_bitmap)))
fatal("bit_copy malloc failure");
- if (cluster_flags & CLUSTER_FLAG_BGQ
- && (bg_record->mp_count == 1)) {
+ if (cluster_flags & CLUSTER_FLAG_BGQ) {
bitstr_t *used_bitmap;
+ if (node_count > bg_conf->mp_cnode_cnt) {
+ /* Here we have to make sure nothing
+ else is able to run on this block
+ since we are using more than 1
+ midplane but potentially not the
+ entire allocation.
+ */
+ FREE_NULL_BITMAP(jobinfo->units_avail);
+ FREE_NULL_BITMAP(jobinfo->units_used);
+ jobinfo->units_avail =
+ ba_create_ba_mp_cnode_bitmap(bg_record);
+ jobinfo->units_used =
+ bit_copy(jobinfo->units_avail);
+ }
if (jobinfo->units_avail)
used_bitmap = jobinfo->units_used;
@@ -2091,7 +2107,7 @@ end_it:
extern int select_p_step_finish(struct step_record *step_ptr)
{
bg_record_t *bg_record = NULL;
- select_jobinfo_t *jobinfo = NULL;
+ select_jobinfo_t *jobinfo = NULL, *step_jobinfo = NULL;
int rc = SLURM_SUCCESS;
char *tmp_char = NULL;
@@ -2106,10 +2122,18 @@ extern int select_p_step_finish(struct step_record *step_ptr)
}
jobinfo = step_ptr->job_ptr->select_jobinfo->data;
+ step_jobinfo = step_ptr->select_jobinfo->data;
- if (jobinfo->units_avail)
+ if (step_jobinfo->cnode_cnt > bg_conf->mp_cnode_cnt) {
+ /* This means we were using units_avail and units_used
+ as midplanes not cnodes for either the whole job
+ allocation or a portion of it.
+ */
+ FREE_NULL_BITMAP(jobinfo->units_avail);
+ FREE_NULL_BITMAP(jobinfo->units_used);
+ } else if (jobinfo->units_avail)
rc = ba_sub_block_in_bitmap_clear(
- step_ptr->select_jobinfo->data, jobinfo->units_used);
+ step_jobinfo, jobinfo->units_used);
else {
slurm_mutex_lock(&block_state_mutex);
bg_record = jobinfo->bg_record;
@@ -2851,12 +2875,19 @@ extern int select_p_fail_cnode(struct step_record *step_ptr)
jobinfo = step_ptr->job_ptr->select_jobinfo->data;
step_jobinfo = step_ptr->select_jobinfo->data;
+ /* block_state must be locked before ba_system */
+ slurm_mutex_lock(&block_state_mutex);
+ slurm_mutex_lock(&ba_system_mutex);
for (i=0; i<bit_size(step_ptr->step_node_bitmap); i++) {
if (!bit_test(step_ptr->step_node_bitmap, i))
continue;
ba_mp = ba_inx2ba_mp(i);
xassert(ba_mp);
+ if (!ba_mp->cnode_err_bitmap)
+ ba_mp->cnode_err_bitmap =
+ bit_alloc(bg_conf->mp_cnode_cnt);
+
if (jobinfo->units_avail) {
bit_or(ba_mp->cnode_err_bitmap,
step_jobinfo->units_used);
@@ -2876,10 +2907,11 @@ extern int select_p_fail_cnode(struct step_record *step_ptr)
if (!ba_mp) {
error("select_p_fail_cnode: no ba_mp? "
"This should never happen");
+ slurm_mutex_unlock(&ba_system_mutex);
+ slurm_mutex_unlock(&block_state_mutex);
return SLURM_ERROR;
}
- slurm_mutex_lock(&block_state_mutex);
itr = list_iterator_create(bg_lists->main);
while ((bg_record = (bg_record_t *)list_next(itr))) {
if (!bit_overlap(step_ptr->step_node_bitmap,
@@ -2929,6 +2961,7 @@ extern int select_p_fail_cnode(struct step_record *step_ptr)
list_iterator_destroy(itr2);
}
list_iterator_destroy(itr);
+ slurm_mutex_unlock(&ba_system_mutex);
slurm_mutex_unlock(&block_state_mutex);
#endif
return SLURM_SUCCESS;
diff --git a/src/slurmctld/step_mgr.c b/src/slurmctld/step_mgr.c
index 686172d57a223cf1837211f3effed047b187a127..52d36418944f684d4f95a09b2316b53c5b93836a 100644
--- a/src/slurmctld/step_mgr.c
+++ b/src/slurmctld/step_mgr.c
@@ -1793,13 +1793,6 @@ step_create(job_step_create_request_msg_t *step_specs,
#if defined HAVE_BGQ
if (step_specs->min_nodes < node_count) {
- if (step_specs->min_nodes > 512) {
- error("step asked for more than 512 nodes but "
- "less than the allocation, on a "
- "bluegene/Q system that isn't allowed.");
- return ESLURM_INVALID_NODE_COUNT;
- }
- /* We are asking for less than we have. */
node_count = step_specs->min_nodes;
step_specs->min_nodes = 1;
diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index 32f02057075d190299e0bb729d9af3a8760e043f..3d2aa1ac68739ebe4656a4295da0d6d4e196c33e 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -4059,7 +4059,7 @@ run_spank_job_script (const char *mode, char **env)
fd_set_close_on_exec (pfds[1]);
- info ("Calling %s spank %s", conf->stepd_loc, mode);
+ debug ("Calling %s spank %s", conf->stepd_loc, mode);
if ((cpid = fork ()) < 0) {
error ("executing spank %s: %m", mode);
return (-1);