diff --git a/NEWS b/NEWS
index e940f5eab3dc62bf419a9062428c3d8ec03056a9..40f280f9c2779b841717d10207c9fe11304e5136 100644
--- a/NEWS
+++ b/NEWS
@@ -16,6 +16,16 @@ documents those changes that are of interest to users and administrators.
-- If AccountingEnforce=safe is set make sure a job can finish before going
over the limit with grpwall on a QOS or association.
-- burst_buffer/cray - Major updates based upon recent Cray changes.
+ -- Improve clean up logic of pmi2 plugin.
+ -- Improve job state reason string when required nodes not available.
+ -- Fix missing else when packing an update partition message
+ -- Fix srun from inheriting the SLURM_CPU_BIND and SLURM_MEM_BIND environment
+ variables when running in an existing srun (e.g. an srun within an salloc).
+ -- Fix missing else when packing an update partition message.
+ -- Use more flexible mechnanism to find json installation.
+ -- Make sure safe_limits was initialized before processing limits in the
+ slurmctld.
+ -- Fix for burst_buffer/cray to parse type option correctly.
* Changes in Slurm 15.08.0
==========================
@@ -370,6 +380,9 @@ documents those changes that are of interest to users and administrators.
-- Fix truncation of job reason in squeue.
-- If a node is in DOWN or DRAIN state, leave it unavailable for allocation
when powered down.
+ -- Update the slurm.conf man page documenting better nohold_on_prolog_fail
+ variable.
+ -- Don't trucate task ID information in "squeue --array/-r" or "sview".
* Changes in Slurm 14.11.9
==========================
diff --git a/auxdir/x_ac_json.m4 b/auxdir/x_ac_json.m4
index be865011ff0de88d3d7dc750740781f0d6987d53..b429bacda1a14e2ee9db06bbcc91adbcf48c2e80 100644
--- a/auxdir/x_ac_json.m4
+++ b/auxdir/x_ac_json.m4
@@ -30,7 +30,7 @@ AC_DEFUN([X_AC_JSON], [
for d in $x_ac_json_dirs; do
test -d "$d" || continue
test -d "$d/include" || continue
- test -f "$d/include/json-c/json_object.h" || continue
+ test -f "$d/include/json-c/json_object.h" || test -f "$d/include/json/json_object.h" || continue
for bit in $x_ac_json_libs; do
test -d "$d/$bit" || continue
_x_ac_json_libs_save="$LIBS"
@@ -48,6 +48,12 @@ AC_DEFUN([X_AC_JSON], [
if test -z "$x_ac_cv_json_dir"; then
AC_MSG_WARN([unable to locate json parser library])
else
+ if test -f "$d/include/json-c/json_object.h" ; then
+ AC_DEFINE([HAVE_JSON_C_INC], [1], [Define if headers in include/json-c.])
+ fi
+ if test -f "$d/include/json/json_object.h" ; then
+ AC_DEFINE([HAVE_JSON_INC], [1], [Define if headers in include/json.])
+ fi
AC_DEFINE([HAVE_JSON], [1], [Define if you are compiling with json.])
JSON_CPPFLAGS="-I$x_ac_cv_json_dir/include"
JSON_LDFLAGS="-L$x_ac_cv_json_dir/$bit -ljson-c"
diff --git a/config.h.in b/config.h.in
index 875c959315b385e78e08eff6bf88484d01ed1908..d7b34b5470b2525a77d61eee6f5d6d9b1dc1cd06 100644
--- a/config.h.in
+++ b/config.h.in
@@ -169,6 +169,12 @@
/* Define if you are compiling with json. */
#undef HAVE_JSON
+/* Define if headers in include/json-c. */
+#undef HAVE_JSON_C_INC
+
+/* Define if headers in include/json. */
+#undef HAVE_JSON_INC
+
/* Define to 1 if you have the <kstat.h> header file. */
#undef HAVE_KSTAT_H
diff --git a/configure b/configure
index bc86885bc047a1a51d075542ecd7ed735a8a50c0..e45b817c9956fe585bfa3bf46d58341ba0b53299 100755
--- a/configure
+++ b/configure
@@ -19430,7 +19430,7 @@ else
for d in $x_ac_json_dirs; do
test -d "$d" || continue
test -d "$d/include" || continue
- test -f "$d/include/json-c/json_object.h" || continue
+ test -f "$d/include/json-c/json_object.h" || test -f "$d/include/json/json_object.h" || continue
for bit in $x_ac_json_libs; do
test -d "$d/$bit" || continue
_x_ac_json_libs_save="$LIBS"
@@ -19472,6 +19472,16 @@ $as_echo "$x_ac_cv_json_dir" >&6; }
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unable to locate json parser library" >&5
$as_echo "$as_me: WARNING: unable to locate json parser library" >&2;}
else
+ if test -f "$d/include/json-c/json_object.h" ; then
+
+$as_echo "#define HAVE_JSON_C_INC 1" >>confdefs.h
+
+ fi
+ if test -f "$d/include/json/json_object.h" ; then
+
+$as_echo "#define HAVE_JSON_INC 1" >>confdefs.h
+
+ fi
$as_echo "#define HAVE_JSON 1" >>confdefs.h
diff --git a/contribs/pmi2/README b/contribs/pmi2/README
index 1032c6e8de4a526133ff696fd7b814afbcc4a886..d0919f5af41319154935278208c2cd37ae5e57f8 100644
--- a/contribs/pmi2/README
+++ b/contribs/pmi2/README
@@ -4,7 +4,7 @@
#export $SLURM_ROOT=slurm_install
# for example SLRUM_ROOT=/home/david/clusters/master/linux
#
-# gcc -g -O0 -o testpmi2 testpmi2.c -I$SLURM_ROOT/include /home/david/clusters/master/linux/lib/libpmi2.so
+# gcc -g -O0 -o testpmi2 testpmi2.c -I$SLURM_ROOT/include $SLURM_ROOT/lib/libpmi2.so
#
-# gcc -g -O0 -o testpmixring testpmixring.c -I$SLURM_ROOT/include /home/david/clusters/master/linux/lib/libpmi2.so
+# gcc -g -O0 -o testpmixring testpmixring.c -I$SLURM_ROOT/include $LSURM_ROOT/lib/libpmi2.so
#
diff --git a/contribs/pmi2/testpmi2.c b/contribs/pmi2/testpmi2.c
index 7042337ed3298c1e2aa1b3cca498903e8386c068..f8b75615122b04d7ad350c9de4839253804ba665 100644
--- a/contribs/pmi2/testpmi2.c
+++ b/contribs/pmi2/testpmi2.c
@@ -1,5 +1,39 @@
-/* Copyright (C) 2014 SchedMD
- */
+
+/*****************************************************************************\
+ * testpmi2.c - reservation creation function for scontrol.
+ *****************************************************************************
+ * Copyright (C) 2014 SchedMD LLC
+ * Written by David Bigagli <david@schedmd.com>
+ *
+ * This file is part of SLURM, a resource management program.
+ * For details, see <http://slurm.schedmd.com/>.
+ * Please also read the included file: DISCLAIMER.
+ *
+ * SLURM is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * In addition, as a special exception, the copyright holders give permission
+ * to link the code of portions of this program with the OpenSSL library under
+ * certain conditions as described in each individual source file, and
+ * distribute linked combinations including the two. You must obey the GNU
+ * General Public License in all respects for all of the code used other than
+ * OpenSSL. If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not obligated to do
+ * so. If you do not wish to do so, delete this exception statement from your
+ * version. If you delete this exception statement from all source files in
+ * the program, then also delete it here.
+ *
+ * SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with SLURM; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+\*****************************************************************************/
#include <stdio.h>
#include <time.h>
#include <stdlib.h>
@@ -12,100 +46,100 @@ static char *mrand(int, int);
int
main(int argc, char **argv)
{
- int rank;
- int size;
- int appnum;
- int spawned;
- int flag;
- int len;
- int i;
- struct timeval tv;
- struct timeval tv2;
- char jobid[128];
- char key[128];
- char val[128];
- char buf[128];
-
- {
- int x = 1;
- while (x == 0) {
- sleep(2);
- }
- }
-
- gettimeofday(&tv, NULL);
- srand(tv.tv_sec);
-
- PMI2_Init(&spawned, &size, &rank, &appnum);
-
- PMI2_Job_GetId(jobid, sizeof(buf));
-
- memset(val, 0, sizeof(val));
- PMI2_Info_GetJobAttr("mpi_reserved_ports",
- val,
- PMI2_MAX_ATTRVALUE,
- &flag);
-
- sprintf(key, "mpi_reserved_ports");
- PMI2_KVS_Put(key, val);
-
- memset(val, 0, sizeof(val));
- sprintf(buf, "PMI_netinfo_of_task");
- PMI2_Info_GetJobAttr(buf,
- val,
- PMI2_MAX_ATTRVALUE,
- &flag);
- sprintf(key, buf);
- PMI2_KVS_Put(key, val);
-
- memset(val, 0, sizeof(val));
- sprintf(key, "david@%d", rank);
- sprintf(val, "%s", mrand(97, 122));
- PMI2_KVS_Put(key, val);
-
- PMI2_KVS_Fence();
-
- for (i = 0; i < size; i++) {
-
- memset(val, 0, sizeof(val));
- sprintf(key, "PMI_netinfo_of_task");
- PMI2_KVS_Get(jobid,
- PMI2_ID_NULL,
- key,
- val,
- sizeof(val),
- &len);
- printf("rank: %d key:%s val:%s\n", rank, key, val);
-
- memset(val, 0, sizeof(val));
- sprintf(key, "david@%d", rank);
- PMI2_KVS_Get(jobid,
- PMI2_ID_NULL,
- key,
- val,
- sizeof(val),
- &len);
- printf("rank: %d key:%s val:%s\n", rank, key, val);
-
- memset(val, 0, sizeof(val));
- sprintf(key, "mpi_reserved_ports");
- PMI2_KVS_Get(jobid,
- PMI2_ID_NULL,
- key,
- val,
- sizeof(val),
- &len);
- printf("rank: %d key:%s val:%s\n", rank, key, val);
- }
-
- PMI2_Finalize();
-
- gettimeofday(&tv2, NULL);
- printf("%f\n",
- ((tv2.tv_sec - tv.tv_sec) * 1000.0
- + (tv2.tv_usec - tv.tv_usec) / 1000.0));
-
- return 0;
+ int rank;
+ int size;
+ int appnum;
+ int spawned;
+ int flag;
+ int len;
+ int i;
+ struct timeval tv;
+ struct timeval tv2;
+ char jobid[128];
+ char key[128];
+ char val[128];
+ char buf[128];
+
+ {
+ int x = 1;
+ while (x == 0) {
+ sleep(2);
+ }
+ }
+
+ gettimeofday(&tv, NULL);
+ srand(tv.tv_sec);
+
+ PMI2_Init(&spawned, &size, &rank, &appnum);
+
+ PMI2_Job_GetId(jobid, sizeof(buf));
+
+ memset(val, 0, sizeof(val));
+ PMI2_Info_GetJobAttr("mpi_reserved_ports",
+ val,
+ PMI2_MAX_ATTRVALUE,
+ &flag);
+
+ sprintf(key, "mpi_reserved_ports");
+ PMI2_KVS_Put(key, val);
+
+ memset(val, 0, sizeof(val));
+ sprintf(buf, "PMI_netinfo_of_task");
+ PMI2_Info_GetJobAttr(buf,
+ val,
+ PMI2_MAX_ATTRVALUE,
+ &flag);
+ sprintf(key, buf);
+ PMI2_KVS_Put(key, val);
+
+ memset(val, 0, sizeof(val));
+ sprintf(key, "david@%d", rank);
+ sprintf(val, "%s", mrand(97, 122));
+ PMI2_KVS_Put(key, val);
+
+ PMI2_KVS_Fence();
+
+ for (i = 0; i < size; i++) {
+
+ memset(val, 0, sizeof(val));
+ sprintf(key, "PMI_netinfo_of_task");
+ PMI2_KVS_Get(jobid,
+ PMI2_ID_NULL,
+ key,
+ val,
+ sizeof(val),
+ &len);
+ printf("rank: %d key:%s val:%s\n", rank, key, val);
+
+ memset(val, 0, sizeof(val));
+ sprintf(key, "david@%d", rank);
+ PMI2_KVS_Get(jobid,
+ PMI2_ID_NULL,
+ key,
+ val,
+ sizeof(val),
+ &len);
+ printf("rank: %d key:%s val:%s\n", rank, key, val);
+
+ memset(val, 0, sizeof(val));
+ sprintf(key, "mpi_reserved_ports");
+ PMI2_KVS_Get(jobid,
+ PMI2_ID_NULL,
+ key,
+ val,
+ sizeof(val),
+ &len);
+ printf("rank: %d key:%s val:%s\n", rank, key, val);
+ }
+
+ PMI2_Finalize();
+
+ gettimeofday(&tv2, NULL);
+ printf("%f\n",
+ ((tv2.tv_sec - tv.tv_sec) * 1000.0
+ + (tv2.tv_usec - tv.tv_usec) / 1000.0));
+
+ return 0;
}
/* Generate a random number between
diff --git a/doc/html/Makefile.am b/doc/html/Makefile.am
index 1be23af4bb92d0db48c9141daefc205abafc38a3..5729225fcac6f3524d916ae56a98702d911f900f 100644
--- a/doc/html/Makefile.am
+++ b/doc/html/Makefile.am
@@ -98,6 +98,7 @@ generated_html = \
testimonials.html \
topology.html \
topology_plugin.html \
+ tres.html \
troubleshoot.html \
tutorials.html \
wckey.html
diff --git a/doc/html/Makefile.in b/doc/html/Makefile.in
index c54e0b9f630f4f94511aeeab7fc70aa583af3ad8..2141d95dac98aec846ec8da7c00ffc3f836e80c7 100644
--- a/doc/html/Makefile.in
+++ b/doc/html/Makefile.in
@@ -528,6 +528,7 @@ generated_html = \
testimonials.html \
topology.html \
topology_plugin.html \
+ tres.html \
troubleshoot.html \
tutorials.html \
wckey.html
diff --git a/doc/html/documentation.shtml b/doc/html/documentation.shtml
index bc20b322ec463156e20d2cd55495f74ea4964714..1f29f77199bdd35d5a96194746c6305715f646af 100644
--- a/doc/html/documentation.shtml
+++ b/doc/html/documentation.shtml
@@ -71,6 +71,7 @@ Documentation for other versions of Slurm is distributed with the code</b></p>
<li><a href="reservations.html">Resource Reservation Guide</a></li>
<li><a href="cons_res_share.html">Sharing Consumable Resources</a></li>
<li><a href="topology.html">Topology</a></li>
+<li><a href="tres.html">Trackable Resources (TRES)</a></li>
</ul>
<li>External Schedulers</li>
<ul>
diff --git a/doc/html/news.shtml b/doc/html/news.shtml
index 512b9bf61f8ea2dbafeccdcd63707c07b2ed2714..c6ae18310009b2c31bfe07f23f92b846ee047745 100644
--- a/doc/html/news.shtml
+++ b/doc/html/news.shtml
@@ -73,7 +73,7 @@ to coordinate activities. Future development plans includes:
<a href="http://en.wikipedia.org/wiki/FlexNet_Publisher">FLEXlm
(Flexnet Publisher)</a> license management.</li>
<li>Distributed architecture to support the management of resources with Intel
- MIC processors.</li>
+ Knight's Landing processors.</li>
<li>IP communications over InfiniBand network for improved performance.</li>
<li>Fault-tolerance and jobs dynamic adaptation through communication protocol
between Slurm, MPI libraries and the application.</li>
diff --git a/doc/html/tres.shtml b/doc/html/tres.shtml
new file mode 100644
index 0000000000000000000000000000000000000000..b46dc4b744b146c29c0d319dce38e3de1a3f883d
--- /dev/null
+++ b/doc/html/tres.shtml
@@ -0,0 +1,113 @@
+<!--#include virtual="header.txt"-->
+
+<h1>Trackable RESources (TRES)</h1>
+
+<p>A TRES is a resource that can be tracked for usage or to enforce
+ limits against. A TRES is made of a combo of a Type and a Name.
+ Types are predefined.
+
+Current TRES Types are
+</p>
+<ul>
+<li>BB (burst buffers)</li>
+<li>CPU</li>
+<li>Energy</li>
+<li>GRES</li>
+<li>License</li>
+<li>Mem (Memory)</li>
+<li>Node</li>
+</ul>
+
+<h2>slurm.conf settings</h2>
+<ul>
+<li><b>AccountingStorageTRES</b>
+<p>Used to define which TRES are
+ to be track on the system. By default CPU, Energy, Memory, and Node
+ are tracked. This will be the case weither specified or not. The
+ following example
+ <pre>AccountingStorageTRES=gres/craynetwork,license/iop1,bb/cray</pre>
+ will track cpu, energy, memory, nodes along with a gres called craynetwork
+ as well as a license called iop1. It will also track usage on a
+ Cray burst buffer. Whenever these resources are used on the
+ cluster they are recorded. The TRES are automatically set up in the database
+ on the start of the slurmctld.
+</p>
+
+<p> The TRES that need/have associated names are BB, GRES, and
+ License. As seen in the above example GRES/License are typically
+ different on each system. The BB TRES is directly named the same as
+ the burst buffer plugin used in the above example we are using the
+ <i>Cray</i> burst buffer plugin.
+</p>
+</li>
+
+<li><b>PriorityWeightTRES</b>
+<p>A comma separated list of TRES Types and weights that sets the
+ degree that each TRES Type contributes to the job's priority.</p>
+<pre>PriorityWeightTRES=CPU=1000,Mem=2000,GRES/gpu=3000</pre>
+
+<p>Applicable only if PriorityType=priority/multifactor and if
+AccountingStorageTRES is configured with each TRES Type.
+The default values are 0.</p>
+</li>
+
+</li>
+<li><b>TRESBillingWeights</b>
+<p>For each partition this option is used to define the billing
+ weights of each TRES type that will be used in calcuating the usage
+ of a job.
+</p>
+<p>Billing weights are specified as a comma-separated list of
+<i>TRES=Weight</i> pairs.
+</p>
+<p>Any TRES Type is available for billing. Note that the memory is weighted per
+gigabyte.
+</p>
+<p>By default the billing of TRES is calculated as the sum of all TRES types
+multiplied by their corresponding billing weight. For example, when a job is
+allocated 1 CPU and 8 GB of memory on a partition configured with
+<pre>TRESBillingWeights="CPU=1.0,Mem=0.25,GRES/gpu=2.0"</pre>, the
+billable TRES will be: (1*1.0) + (8*0.25) + (0*2.0) = 3.0.
+</p>
+<p>If <i>PriorityFlags=MAX_TRES</i> is configured, the billable TRES is
+ calculated as the MAX of individual TRES' on a node (e.g. cpus, mem,
+ gres) plus the sum of all global TRES' (e.g. licenses). Using the
+ same example above the billable TRES will be:
+ MAX(1*1.0, 8*0.25) + (0*2.0) = 2.0.
+</p>
+<p>If TRESBillingWeights is not defined then the job is billed against the total
+number of allocated CPUs.
+</p>
+
+<p><b>NOTE:</b> TRESBillingWeights is only used when calcuating fairshare and
+doesn't affect job priority directly as it is currently not used for the size of
+the job. If you want TRES' to play a role in the job's priority then refer to
+the PriorityWeightTRES option.
+</p>
+<p><b>NOTE:</b> As with PriorityWeightTRES only TRES definied in
+ AccountingStorageTRES are available for TRESBillingWeights.
+</p>
+</li>
+
+</ul>
+
+<h2>sacct</h2>
+<p>Sacct can be used to view the TRES of each job by adding "tres" to the
+ --format option.
+</p>
+
+<h2>sacctmgr</h2>
+<p>Sacctmgr is used to view the various TRES available globally in the
+ system. <i>sacctmgr show tres</i> will do this.
+</p>
+
+<h2>sreport</h2>
+<p>Before 15.08 sreport would only report on CPU usage. It now will
+ work on different TRES. Simply using the comma separated input
+ option <i>--tres=</i> will have sreport generate reports available
+ for the requested TRES types. More information about these reports
+ can be found on the <a href="sreport.html">sreport manpage</a>.
+</p>
+<p style="text-align:center;">Last modified 8 September 2015</p>
+
+<!--#include virtual="footer.txt"-->
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index b4331aff13e861ea2018bf4b3dc7765fe075ca23..aa5cb866ff2eff46fb16a0efcbdc8ca28651fcd0 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -123,10 +123,10 @@ Also see \fBDefaultStoragePort\fR.
\fBAccountingStorageTRES\fR
Comma separated list of resources you wish to track on the cluster.
These are the resources requested by the sbatch/srun job when it
-is submitted.
-Currently this consists of any GRES, or license along with CPU, Memory,
-and Energy. By default CPU, Energy, Memory, and Node are tracked.
-AccountingStorageTRES=cpu,energy,mem,node,gres/craynetwork,license/iop1
+is submitted. Currently this consists of any GRES, BB (burst buffer) or
+license along with CPU, Memory, Node, and Energy.
+By default CPU, Energy, Memory, and Node are tracked.
+AccountingStorageTRES=gres/craynetwork,license/iop1
will track cpu, energy, memory, nodes along with a gres called craynetwork
as well as a license called iop1. Whenever these resources are used on the
cluster they are recorded. The TRES are automatically set up in the database
@@ -1965,7 +1965,9 @@ enable user login, etc. By default there is no prolog. Any configured script
is expected to complete execution quickly (in less time than
\fBMessageTimeout\fR).
If the prolog fails (returns a non\-zero exit code), this will result in the
-node being set to a DRAIN state and the job being requeued in a held state.
+node being set to a DRAIN state and the job being requeued in a held state,
+unless \fBnohold_on_prolog_fail\fR is configured in
+\fBSchedulerParameters\fR.
See \fBProlog and Epilog Scripts\fR for more information.
.TP
@@ -4460,10 +4462,13 @@ node being set to a DRAIN state.
If the EpilogSlurmctld fails (returns a non\-zero exit code), this will only
be logged.
If the Prolog fails (returns a non\-zero exit code), this will result in the
-node being set to a DRAIN state and the job being requeued in a held state.
+node being set to a DRAIN state and the job being requeued in a held state
+unless \fBnohold_on_prolog_fail\fR is configured in
+\fBSchedulerParameters\fR.
If the PrologSlurmctld fails (returns a non\-zero exit code), this will result
in the job requeued to executed on another node if possible. Only batch jobs
-can be requeued. Interactive jobs (salloc and srun) will be cancelled if the
+can be requeued.
+ Interactive jobs (salloc and srun) will be cancelled if the
PrologSlurmctld fails.
diff --git a/slurm/slurm.h.in b/slurm/slurm.h.in
index d6555228372be1ee44e3d43270ede2982bd3906d..6d0610a593565aa865085b4371466779f8e5bb07 100644
--- a/slurm/slurm.h.in
+++ b/slurm/slurm.h.in
@@ -703,6 +703,7 @@ enum acct_energy_type {
ENERGY_DATA_LAST_POLL,
ENERGY_DATA_SENSOR_CNT,
ENERGY_DATA_NODE_ENERGY,
+ ENERGY_DATA_NODE_ENERGY_UP,
};
/*
diff --git a/src/plugins/acct_gather_energy/ipmi/acct_gather_energy_ipmi.c b/src/plugins/acct_gather_energy/ipmi/acct_gather_energy_ipmi.c
index 406e5c22fc9e913ddcc52d3e9db2f9b01c9341fb..c0db34ae3da93c0fa64fb481c342639216e82d8c 100644
--- a/src/plugins/acct_gather_energy/ipmi/acct_gather_energy_ipmi.c
+++ b/src/plugins/acct_gather_energy/ipmi/acct_gather_energy_ipmi.c
@@ -628,6 +628,7 @@ static int _ipmi_send_profile(void)
uint16_t i, j;
uint64_t data[descriptions_len];
uint32_t id;
+ time_t last_time = last_update_time;
if (!_running_profile())
return SLURM_SUCCESS;
@@ -660,6 +661,8 @@ static int _ipmi_send_profile(void)
id = descriptions[i].sensor_idxs[j];
data[i] += sensors[id].energy.current_watts;
}
+ if (descriptions[i].sensor_cnt)
+ last_time = sensors[id].energy.poll_time;
}
if (debug_flags & DEBUG_FLAG_PROFILE) {
@@ -671,7 +674,7 @@ static int _ipmi_send_profile(void)
}
}
return acct_gather_profile_g_add_sample_data(dataset_id, (void *)data,
- last_update_time);
+ last_time);
}
@@ -950,6 +953,17 @@ extern int acct_gather_energy_p_get_data(enum acct_energy_type data_type,
xassert(_run_in_daemon());
switch (data_type) {
+ case ENERGY_DATA_NODE_ENERGY_UP:
+ slurm_mutex_lock(&ipmi_mutex);
+ if (_is_thread_launcher()) {
+ if (_thread_init() == SLURM_SUCCESS)
+ _thread_update_node_energy();
+ } else {
+ _get_joules_task(10);
+ }
+ _get_node_energy(energy);
+ slurm_mutex_unlock(&ipmi_mutex);
+ break;
case ENERGY_DATA_NODE_ENERGY:
slurm_mutex_lock(&ipmi_mutex);
_get_node_energy(energy);
diff --git a/src/plugins/acct_gather_energy/rapl/acct_gather_energy_rapl.c b/src/plugins/acct_gather_energy/rapl/acct_gather_energy_rapl.c
index d86111db2652522ad36db42485748ca34bb0b8a0..dab425782ce36681f5ed5e5685c4b037fb4cbe98 100644
--- a/src/plugins/acct_gather_energy/rapl/acct_gather_energy_rapl.c
+++ b/src/plugins/acct_gather_energy/rapl/acct_gather_energy_rapl.c
@@ -488,13 +488,14 @@ extern int acct_gather_energy_p_get_data(enum acct_energy_type data_type,
switch (data_type) {
case ENERGY_DATA_JOULES_TASK:
+ case ENERGY_DATA_NODE_ENERGY_UP:
if (local_energy->current_watts == NO_VAL)
energy->consumed_energy = NO_VAL;
else
_get_joules_task(energy);
break;
- case ENERGY_DATA_NODE_ENERGY:
case ENERGY_DATA_STRUCT:
+ case ENERGY_DATA_NODE_ENERGY:
memcpy(energy, local_energy, sizeof(acct_gather_energy_t));
break;
case ENERGY_DATA_LAST_POLL:
diff --git a/src/plugins/acct_gather_profile/hdf5/acct_gather_profile_hdf5.c b/src/plugins/acct_gather_profile/hdf5/acct_gather_profile_hdf5.c
index d68be3a5c39f2a71290fbae2f12ac9502b21e4d3..9fb076930c27d55ef606448a15b7238dd1d69b60 100644
--- a/src/plugins/acct_gather_profile/hdf5/acct_gather_profile_hdf5.c
+++ b/src/plugins/acct_gather_profile/hdf5/acct_gather_profile_hdf5.c
@@ -517,10 +517,11 @@ extern int acct_gather_profile_p_create_dataset(
}
/* insert fields */
- if (H5Tinsert(dtype_id, "ElapsedTime", sizeof(uint64_t),
+ if (H5Tinsert(dtype_id, "ElapsedTime", 0,
H5T_NATIVE_UINT64) < 0)
return SLURM_ERROR;
- if (H5Tinsert(dtype_id, "EpochTime", 0, H5T_NATIVE_UINT64) < 0)
+ if (H5Tinsert(dtype_id, "EpochTime", sizeof(uint64_t),
+ H5T_NATIVE_UINT64) < 0)
return SLURM_ERROR;
dataset_loc = dataset;
diff --git a/src/plugins/burst_buffer/cray/burst_buffer_cray.c b/src/plugins/burst_buffer/cray/burst_buffer_cray.c
index 4784341ac156158a4f38d53322368df8c1777053..a3b39872b1db1be557651b886c173bb94229f09b 100644
--- a/src/plugins/burst_buffer/cray/burst_buffer_cray.c
+++ b/src/plugins/burst_buffer/cray/burst_buffer_cray.c
@@ -46,8 +46,10 @@
#include <sys/stat.h>
#include <sys/types.h>
-#if HAVE_JSON
+#if HAVE_JSON_C_INC
# include <json-c/json.h>
+#elif HAVE_JSON_INC
+# include <json/json.h>
#endif
#include "slurm/slurm.h"
@@ -444,6 +446,9 @@ static bb_job_t *_get_bb_job(struct job_record *job_ptr)
tok++;
if (!strncmp(tok, "create_persistent", 17)) {
have_bb = true;
+ bb_access = NULL;
+ bb_name = NULL;
+ bb_type = NULL;
if ((sub_tok = strstr(tok, "access_mode="))) {
bb_access = xstrdup(sub_tok + 12);
sub_tok = strchr(bb_access, ' ');
@@ -485,24 +490,22 @@ static bb_job_t *_get_bb_job(struct job_record *job_ptr)
bb_job->buf_ptr[inx].size = tmp_cnt;
bb_job->buf_ptr[inx].state = BB_STATE_PENDING;
bb_job->buf_ptr[inx].type = bb_type;
- bb_access = NULL;
- bb_name = NULL;
- bb_type = NULL;
} else if (!strncmp(tok, "destroy_persistent", 17) ||
!strncmp(tok, "delete_persistent", 16)) {
have_bb = true;
+ bb_name = NULL;
if ((sub_tok = strstr(tok, "name="))) {
bb_name = xstrdup(sub_tok + 5);
sub_tok = strchr(bb_name, ' ');
if (sub_tok)
sub_tok[0] = '\0';
}
- if ((sub_tok = strstr(tok, "type="))) {
- bb_type = xstrdup(sub_tok + 5);
- sub_tok = strchr(bb_type, ' ');
- if (sub_tok)
- sub_tok[0] = '\0';
- }
+ /* if ((sub_tok = strstr(tok, "type="))) { */
+ /* bb_type = xstrdup(sub_tok + 5); */
+ /* sub_tok = strchr(bb_type, ' '); */
+ /* if (sub_tok) */
+ /* sub_tok[0] = '\0'; */
+ /* } */
bb_hurry = strstr(tok, "hurry");
inx = bb_job->buf_cnt++;
bb_job->buf_ptr = xrealloc(bb_job->buf_ptr,
@@ -535,6 +538,7 @@ static bb_job_t *_get_bb_job(struct job_record *job_ptr)
bb_job->total_size += tmp_cnt;
} else if (!strncmp(tok, "persistentdw", 12)) {
have_bb = true;
+ bb_name = NULL;
if ((sub_tok = strstr(tok, "name="))) {
bb_name = xstrdup(sub_tok + 5);
sub_tok = strchr(bb_name, ' ');
@@ -621,7 +625,7 @@ static void _save_bb_state(void)
{
static time_t last_save_time = 0;
static int high_buffer_size = 16 * 1024;
- time_t save_time;
+ time_t save_time = time(NULL);
bb_alloc_t *bb_alloc;
uint32_t rec_count = 0;
Buf buffer;
@@ -2055,7 +2059,7 @@ static int _parse_bb_opts(struct job_descriptor *job_desc, uint64_t *bb_size,
char *bb_name = NULL, *capacity;
char *end_ptr = NULL, *sub_tok, *tok;
uint64_t tmp_cnt;
- int rc = SLURM_SUCCESS, swap_cnt;
+ int rc = SLURM_SUCCESS, swap_cnt = 0;
bool enable_persist = false, have_bb = false;
xassert(bb_size);
@@ -2091,6 +2095,7 @@ static int _parse_bb_opts(struct job_descriptor *job_desc, uint64_t *bb_size,
break;
} else if (!strncmp(tok, "create_persistent", 17)) {
have_bb = true;
+ bb_name = NULL;
if ((sub_tok = strstr(tok, "capacity="))) {
tmp_cnt = bb_get_size_num(
sub_tok + 9,
@@ -2106,7 +2111,9 @@ static int _parse_bb_opts(struct job_descriptor *job_desc, uint64_t *bb_size,
} else {
rc =ESLURM_INVALID_BURST_BUFFER_REQUEST;
}
- if ((bb_name[0] >= '0') && (bb_name[0] <= '9'))
+ if (!bb_name ||
+ ((bb_name[0] >= '0') &&
+ (bb_name[0] <= '9')))
rc =ESLURM_INVALID_BURST_BUFFER_REQUEST;
xfree(bb_name);
if (rc != SLURM_SUCCESS)
@@ -2232,10 +2239,10 @@ static int _xlate_interactive(struct job_descriptor *job_desc)
if ((tok = strstr(job_desc->burst_buffer, "type="))) {
type = xstrdup(tok + 5);
- tok = strchr(access, ',');
+ tok = strchr(type, ',');
if (tok)
tok[0] = '\0';
- tok = strchr(access, ' ');
+ tok = strchr(type, ' ');
if (tok)
tok[0] = '\0';
}
diff --git a/src/plugins/jobacct_gather/common/common_jag.c b/src/plugins/jobacct_gather/common/common_jag.c
index 02bc90cdbdeb62488b3b1b2a872a158cbbbceff8..a774bdb22dece3fea3fa8e0bc0e5d12031a2ebab 100644
--- a/src/plugins/jobacct_gather/common/common_jag.c
+++ b/src/plugins/jobacct_gather/common/common_jag.c
@@ -59,7 +59,7 @@ static long hertz = 0;
static int my_pagesize = 0;
static DIR *slash_proc = NULL;
-static int energy_profile = ENERGY_DATA_JOULES_TASK;
+static int energy_profile = ENERGY_DATA_NODE_ENERGY_UP;
static uint64_t debug_flags = 0;
static int _find_prec(void *x, void *key)
@@ -743,6 +743,7 @@ extern void jag_common_init(long in_hertz)
acct_gather_profile_g_get(ACCT_GATHER_PROFILE_RUNNING,
&profile_opt);
+
/* If we are profiling energy it will be checked at a
different rate, so just grab the last one.
*/
diff --git a/src/plugins/mpi/pmi2/agent.c b/src/plugins/mpi/pmi2/agent.c
index 1929ca9f18bee9dc0be9c707f4ad9bff2ccf3867..32a85f59858575afff23063d4b27812319b91e8b 100644
--- a/src/plugins/mpi/pmi2/agent.c
+++ b/src/plugins/mpi/pmi2/agent.c
@@ -4,6 +4,9 @@
* Copyright (C) 2011-2012 National University of Defense Technology.
* Written by Hongjia Cao <hjcao@nudt.edu.cn>.
* All rights reserved.
+ * Portions copyright (C) 2015 Mellanox Technologies Inc.
+ * Written by Artem Y. Polyakov <artemp@mellanox.com>.
+ * All rights reserved.
*
* This file is part of SLURM, a resource management program.
* For details, see <http://slurm.schedmd.com/>.
@@ -66,7 +69,8 @@
static int *initialized = NULL;
static int *finalized = NULL;
-static pthread_t pmi2_agent_tid = 0;
+static eio_handle_t *pmi2_handle;
+static volatile int _agent_running;
static bool _tree_listen_readable(eio_obj_t *obj);
static int _tree_listen_read(eio_obj_t *obj, List objs);
@@ -297,10 +301,11 @@ send_response:
static void *
_agent(void * unused)
{
- eio_handle_t *pmi2_handle;
eio_obj_t *tree_listen_obj, *task_obj;
int i;
+ _agent_running = 1;
+
pmi2_handle = eio_handle_create(0);
//fd_set_nonblocking(tree_sock);
@@ -324,6 +329,8 @@ _agent(void * unused)
debug("mpi/pmi2: agent thread exit");
eio_handle_destroy(pmi2_handle);
+
+ _agent_running = 0;
return NULL;
}
@@ -335,6 +342,7 @@ pmi2_start_agent(void)
{
int retries = 0;
pthread_attr_t attr;
+ pthread_t pmi2_agent_tid = 0;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
@@ -351,6 +359,11 @@ pmi2_start_agent(void)
debug("mpi/pmi2: started agent thread (%lu)",
(unsigned long) pmi2_agent_tid);
+ /* wait for the agent to start */
+ while (!_agent_running) {
+ sched_yield();
+ }
+
return SLURM_SUCCESS;
}
@@ -360,8 +373,14 @@ pmi2_start_agent(void)
extern int
pmi2_stop_agent(void)
{
- if (pmi2_agent_tid)
- pthread_cancel(pmi2_agent_tid);
+ /* brake eio loop */
+ if (pmi2_handle != NULL) {
+ eio_signal_shutdown(pmi2_handle);
+ /* wait for the agent to finish */
+ while (_agent_running ) {
+ sched_yield();
+ }
+ }
return SLURM_SUCCESS;
}
diff --git a/src/plugins/mpi/pmi2/mpi_pmi2.c b/src/plugins/mpi/pmi2/mpi_pmi2.c
index 25f78407b625b6c67981e1660332b5502bdd4128..dce8824ff6a3c0f659ab49d1c4237054bc8addc7 100644
--- a/src/plugins/mpi/pmi2/mpi_pmi2.c
+++ b/src/plugins/mpi/pmi2/mpi_pmi2.c
@@ -4,6 +4,9 @@
* Copyright (C) 2011-2012 National University of Defense Technology.
* Written by Hongjia Cao <hjcao@nudt.edu.cn>.
* All rights reserved.
+ * Portions copyright (C) 2015 Mellanox Technologies Inc.
+ * Written by Artem Polyakov <artemp@mellanox.com>.
+ * All rights reserved.
*
* This file is part of SLURM, a resource management program.
* For details, see <http://slurm.schedmd.com/>.
@@ -176,3 +179,11 @@ int p_mpi_hook_client_fini(mpi_plugin_client_state_t *state)
return SLURM_SUCCESS;
}
+
+extern int fini()
+{
+ /* cleanup after ourself */
+ pmi2_stop_agent();
+ pmi2_cleanup_stepd();
+ return 0;
+}
diff --git a/src/plugins/mpi/pmi2/setup.c b/src/plugins/mpi/pmi2/setup.c
index 6829a404a3a4efd0721d92b36cf85a2bf71121f4..63573880d3a872665159f2d9eabfe3f1f9a1c1c9 100644
--- a/src/plugins/mpi/pmi2/setup.c
+++ b/src/plugins/mpi/pmi2/setup.c
@@ -4,6 +4,9 @@
* Copyright (C) 2011-2012 National University of Defense Technology.
* Written by Hongjia Cao <hjcao@nudt.edu.cn>.
* All rights reserved.
+ * Portions copyright (C) 2015 Mellanox Technologies Inc.
+ * Written by Artem Y. Polyakov <artemp@mellanox.com>.
+ * All rights reserved.
*
* This file is part of SLURM, a resource management program.
* For details, see <http://slurm.schedmd.com/>.
@@ -289,7 +292,6 @@ _setup_stepd_sockets(const stepd_step_rec_t *job, char ***env)
/* remove the tree socket file on exit */
strncpy(tree_sock_addr, sa.sun_path, 128);
- atexit(_remove_tree_sock);
task_socks = xmalloc(2 * job->node_tasks * sizeof(int));
for (i = 0; i < job->node_tasks; i ++) {
@@ -380,6 +382,12 @@ pmi2_setup_stepd(const stepd_step_rec_t *job, char ***env)
return SLURM_SUCCESS;
}
+extern void
+pmi2_cleanup_stepd()
+{
+ close(tree_sock);
+ _remove_tree_sock();
+}
/**************************************************************/
/* returned string should be xfree-ed by caller */
diff --git a/src/plugins/mpi/pmi2/setup.h b/src/plugins/mpi/pmi2/setup.h
index fdc8d7ecf23071eca202d9d70fc54ccc8a3ce2e7..408993f87a8ba6c98150a51b8d3ca99c51a33af9 100644
--- a/src/plugins/mpi/pmi2/setup.h
+++ b/src/plugins/mpi/pmi2/setup.h
@@ -4,6 +4,9 @@
* Copyright (C) 2011-2012 National University of Defense Technology.
* Written by Hongjia Cao <hjcao@nudt.edu.cn>.
* All rights reserved.
+ * Portions copyright (C) 2015 Mellanox Technologies Inc.
+ * Written by Artem Y. Polyakov <artemp@mellanox.com>.
+ * All rights reserved.
*
* This file is part of SLURM, a resource management program.
* For details, see <http://slurm.schedmd.com/>.
@@ -111,5 +114,6 @@ extern int *task_socks;
extern bool in_stepd(void);
extern int pmi2_setup_stepd(const stepd_step_rec_t *job, char ***env);
extern int pmi2_setup_srun(const mpi_plugin_client_info_t *job, char ***env);
+extern void pmi2_cleanup_stepd(void);
#endif /* _SETUP_H */
diff --git a/src/plugins/mpi/pmi2/tree.c b/src/plugins/mpi/pmi2/tree.c
index ecac5dc51a8e1f3aaecb2608b91a3482750990e3..bf6a1ac80b4054dd5d102f01947eead19285e0fc 100644
--- a/src/plugins/mpi/pmi2/tree.c
+++ b/src/plugins/mpi/pmi2/tree.c
@@ -6,7 +6,10 @@
* All rights reserved.
* Portions copyright (C) 2014 Institute of Semiconductor Physics
* Siberian Branch of Russian Academy of Science
- * Written by Artem Polyakov <artpol84@gmail.com>.
+ * Written by Artem Y. Polyakov <artpol84@gmail.com>.
+ * All rights reserved.
+ * Portions copyright (C) 2015 Mellanox Technologies Inc.
+ * Written by Artem Y. Polyakov <artemp@mellanox.com>.
* All rights reserved.
*
* This file is part of SLURM, a resource management program.
@@ -702,15 +705,18 @@ tree_msg_to_stepds(hostlist_t hl, uint32_t len, char *data)
nodelist = hostlist_ranged_string_xmalloc(hl);
+ debug("tree_msg_to_stepds: send to %s", nodelist);
+
if ((ret_list = slurm_send_recv_msgs(nodelist, msg, 0, false))) {
while ((ret_data_info = list_pop(ret_list))) {
temp_rc = slurm_get_return_code(ret_data_info->type,
ret_data_info->data);
if (temp_rc){
rc = temp_rc;
+ error("tree_msg_to_stepds: host=%s, rc = %d",
+ ret_data_info->node_name, rc);
} else {
- hostlist_delete_host(hl,
- ret_data_info->node_name);
+ hostlist_delete_host(hl, ret_data_info->node_name);
}
}
} else {
diff --git a/src/plugins/power/cray/power_cray.c b/src/plugins/power/cray/power_cray.c
index 974307324a11432529c85b5ddcbf70c3bbe4e50a..54f3a41fd0eb4234baa4ff12b6784f7e42cc5d25 100644
--- a/src/plugins/power/cray/power_cray.c
+++ b/src/plugins/power/cray/power_cray.c
@@ -46,8 +46,10 @@
#include <sys/stat.h>
#include <sys/types.h>
-#if HAVE_JSON
-#include <json-c/json.h>
+#if HAVE_JSON_C_INC
+# include <json-c/json.h>
+#elif HAVE_JSON_INC
+# include <json/json.h>
#endif
#include "slurm/slurm.h"
diff --git a/src/sacctmgr/event_functions.c b/src/sacctmgr/event_functions.c
index ada0863b1d0b037dff1063fec984d41f032cb202..f40c537d80399125fc1be017935964e420f10d77 100644
--- a/src/sacctmgr/event_functions.c
+++ b/src/sacctmgr/event_functions.c
@@ -501,8 +501,6 @@ extern int sacctmgr_list_event(int argc, char *argv[])
return SLURM_ERROR;
}
- print_fields_list = list_create(destroy_print_field);
-
if (!list_count(format_list)) {
if (event_cond->event_type == SLURMDB_EVENT_CLUSTER)
slurm_addto_char_list(format_list,
diff --git a/src/slurmctld/acct_policy.c b/src/slurmctld/acct_policy.c
index 777cfb462a0ab154f6b27f0126e461a50efbd621..0ee169812a000e16cdab0fbab5706dd21cc7b2a3 100644
--- a/src/slurmctld/acct_policy.c
+++ b/src/slurmctld/acct_policy.c
@@ -2304,7 +2304,7 @@ extern bool acct_policy_job_runnable_pre_select(struct job_record *job_ptr)
uint32_t time_limit = NO_VAL;
bool rc = true;
uint32_t wall_mins;
- bool safe_limits;
+ bool safe_limits = false;
int parent = 0; /* flag to tell us if we are looking at the
* parent or not
*/
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index 9d4283e83042eaa10d316c39f60508fd3e45c2a3..37b3f696ddcbe53af328639e16e50dba717bd829 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -7868,7 +7868,12 @@ void pack_job(struct job_record *dump_job_ptr, uint16_t show_flags, Buf buffer,
pack32(dump_job_ptr->job_state, buffer);
pack16(dump_job_ptr->batch_flag, buffer);
- pack16(dump_job_ptr->state_reason, buffer);
+ if ((dump_job_ptr->state_reason == WAIT_NO_REASON) &&
+ IS_JOB_PENDING(dump_job_ptr)) {
+ /* Scheduling cycle in progress, send latest reason */
+ pack16(dump_job_ptr->state_reason_prev, buffer);
+ } else
+ pack16(dump_job_ptr->state_reason, buffer);
pack8(dump_job_ptr->power_flags, buffer);
pack8(dump_job_ptr->reboot, buffer);
pack8(dump_job_ptr->sicp_mode, buffer);
diff --git a/src/slurmctld/job_scheduler.c b/src/slurmctld/job_scheduler.c
index f7c6e14380500e6e9f7b27112d892fa2c458b478..620b86f82a2d6c5d79cc8a1a324c59a33bd25354 100644
--- a/src/slurmctld/job_scheduler.c
+++ b/src/slurmctld/job_scheduler.c
@@ -373,6 +373,8 @@ extern List build_job_queue(bool clear_start, bool backfill)
}
tested_jobs++;
job_ptr->preempt_in_progress = false; /* initialize */
+ if (job_ptr->state_reason != WAIT_NO_REASON)
+ job_ptr->state_reason_prev = job_ptr->state_reason;
if (!_job_runnable_test1(job_ptr, clear_start))
continue;
diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index 3ea00e6e94a34d4cfbe8f8600f6c632f1614ee26..9ca2ddc0d46d841a216be098bee0bc56328e6b8c 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -2027,17 +2027,43 @@ extern int select_nodes(struct job_record *job_ptr, bool test_only,
/* Non-fatal errors for job below */
} else if (error_code == ESLURM_NODE_NOT_AVAIL) {
/* Required nodes are down or drained */
+ char *node_str = NULL, *unavail_node = NULL;
debug3("JobId=%u required nodes not avail",
job_ptr->job_id);
job_ptr->state_reason = WAIT_NODE_NOT_AVAIL;
xfree(job_ptr->state_desc);
- xstrfmtcat(job_ptr->state_desc,
- "ReqNodeNotAvail, May be reserved for other job");
- if (unavail_node_str) {
+ if (unavail_node_str) { /* Set in few cases */
+ node_str = unavail_node_str;
+ } else {
+ bitstr_t *unavail_bitmap;
+ unavail_bitmap = bit_copy(avail_node_bitmap);
+ bit_not(unavail_bitmap);
+ if (job_ptr->details &&
+ job_ptr->details->req_node_bitmap &&
+ bit_overlap(unavail_bitmap,
+ job_ptr->details->req_node_bitmap)) {
+ bit_and(unavail_bitmap,
+ job_ptr->details->
+ req_node_bitmap);
+ }
+ if (bit_ffs(unavail_bitmap) != -1) {
+ unavail_node = bitmap2node_name(
+ unavail_bitmap);
+ node_str = unavail_node;
+ }
+ FREE_NULL_BITMAP(unavail_bitmap);
+ }
+ if (node_str) {
+ xstrfmtcat(job_ptr->state_desc,
+ "ReqNodeNotAvail, "
+ "UnavailableNodes:%s",
+ node_str);
+ } else {
xstrfmtcat(job_ptr->state_desc,
- ", UnavailableNodes:%s",
- unavail_node_str);
+ "ReqNodeNotAvail, May be reserved "
+ "for other job");
}
+ xfree(unavail_node);
last_job_update = now;
} else if ((error_code == ESLURM_RESERVATION_NOT_USABLE) ||
(error_code == ESLURM_RESERVATION_BUSY)) {
diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h
index ba8f41b2f1ea571d2dbb0ab19fe0276b4b44d6f6..81a7830ce6d3d3f50b7f796734e854b0d5e51756 100644
--- a/src/slurmctld/slurmctld.h
+++ b/src/slurmctld/slurmctld.h
@@ -726,6 +726,10 @@ struct job_record {
char *state_desc; /* optional details for state_reason */
uint32_t state_reason; /* reason job still pending or failed
* see slurm.h:enum job_wait_reason */
+ uint32_t state_reason_prev; /* Previous state_reason, needed to
+ * return valid job information during
+ * scheduling cycle (state_reason is
+ * cleared at start of cycle) */
List step_list; /* list of job's steps */
time_t suspend_time; /* time job last suspended or resumed */
time_t time_last_active; /* time of last job activity */
diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index 4cb9e0c8f12e6dfa18ee4e11897a88ed01ea28f1..2c0f823a2388eef77f7e4c93ad0476265fef04a9 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -4,6 +4,7 @@
* Copyright (C) 2002-2007 The Regents of the University of California.
* Copyright (C) 2008-2010 Lawrence Livermore National Security.
* Portions Copyright (C) 2010-2013 SchedMD LLC.
+ * Portions copyright (C) 2015 Mellanox Technologies Inc.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
* Written by Mark Grondona <mgrondona@llnl.gov>.
* CODE-OCEC-09-009. All rights reserved.
@@ -5908,7 +5909,7 @@ _rpc_forward_data(slurm_msg_t *msg)
forward_data_msg_t *req = (forward_data_msg_t *)msg->data;
uint32_t req_uid;
struct sockaddr_un sa;
- int fd = -1, rc;
+ int fd = -1, rc = 0;
debug3("Entering _rpc_forward_data, address: %s, len: %u",
req->address, req->len);
@@ -5916,12 +5917,14 @@ _rpc_forward_data(slurm_msg_t *msg)
/* sanity check */
if (strlen(req->address) > sizeof(sa.sun_path) - 1) {
slurm_seterrno(EINVAL);
+ rc = errno;
goto done;
}
/* connect to specified address */
fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (fd < 0) {
+ rc = errno;
error("failed creating UNIX domain socket: %m");
goto done;
}
@@ -5931,6 +5934,7 @@ _rpc_forward_data(slurm_msg_t *msg)
while ((rc = connect(fd, (struct sockaddr *)&sa, SUN_LEN(&sa)) < 0) &&
(errno == EINTR));
if (rc < 0) {
+ rc = errno;
debug2("failed connecting to specified socket '%s': %m",
req->address);
goto done;
@@ -5950,9 +5954,9 @@ _rpc_forward_data(slurm_msg_t *msg)
rwfail:
done:
- if (fd >= 0)
+ if (fd >= 0){
close(fd);
- rc = errno;
+ }
slurm_send_rc_msg(msg, rc);
}
diff --git a/src/slurmd/slurmstepd/req.c b/src/slurmd/slurmstepd/req.c
index 31945cbd60f8ef60da68860fe6738cdc57fd8233..92d7c95c64bd66b8680efaf55090876c3b8251eb 100644
--- a/src/slurmd/slurmstepd/req.c
+++ b/src/slurmd/slurmstepd/req.c
@@ -786,29 +786,30 @@ _handle_signal_container(int fd, stepd_step_rec_t *job, uid_t uid)
/* Not really errors,
* but we want messages displayed by default */
if (sig == SIG_TIME_LIMIT) {
- error("*** %s CANCELLED AT %s DUE TO TIME LIMIT on %s ***",
- entity, time_str, job->node_name);
+ error("*** %s ON %s CANCELLED AT %s DUE TO TIME LIMIT ***",
+ entity, job->node_name, time_str);
msg_sent = 1;
} else if (sig == SIG_PREEMPTED) {
- error("*** %s CANCELLED AT %s DUE TO PREEMPTION on %s ***",
- entity, time_str, job->node_name);
+ error("*** %s ON %s CANCELLED AT %s DUE TO PREEMPTION ***",
+ entity, job->node_name, time_str);
msg_sent = 1;
} else if (sig == SIG_NODE_FAIL) {
- error("*** %s CANCELLED AT %s DUE TO NODE %s FAILURE ***",
- entity, time_str, job->node_name);
+ error("*** %s ON %s CANCELLED AT %s DUE TO NODE "
+ "FAILURE, SEE SLURMCTLD LOG FOR DETAILS ***",
+ entity, job->node_name, time_str);
msg_sent = 1;
} else if (sig == SIG_REQUEUED) {
- error("*** %s CANCELLED AT %s DUE TO JOB REQUEUE ***",
- entity, time_str);
+ error("*** %s ON %s CANCELLED AT %s DUE TO JOB REQUEUE ***",
+ entity, job->node_name, time_str);
msg_sent = 1;
} else if (sig == SIG_FAILURE) {
- error("*** %s FAILED (non-zero exit code or other "
- "failure mode) on %s ***",
+ error("*** %s ON %s FAILED (non-zero exit code or other "
+ "failure mode) ***",
entity, job->node_name);
msg_sent = 1;
} else if ((sig == SIGTERM) || (sig == SIGKILL)) {
- error("*** %s CANCELLED AT %s *** on %s",
- entity, time_str, job->node_name);
+ error("*** %s ON %s CANCELLED AT %s ***",
+ entity, job->node_name, time_str);
msg_sent = 1;
}
}
diff --git a/src/squeue/opts.c b/src/squeue/opts.c
index f9e6d2622d6645f89e5938d670049d5a10c05a58..3e45216de4223e4a84f280b88cd840cfcbd0ac5b 100644
--- a/src/squeue/opts.c
+++ b/src/squeue/opts.c
@@ -259,6 +259,7 @@ parse_command_line( int argc, char* argv[] )
break;
case (int)'r':
params.array_flag = true;
+ setenv("SLURM_BITSTR_LEN", "0", 1);
break;
case (int) 's':
if (optarg) {
diff --git a/src/srun/libsrun/opt.c b/src/srun/libsrun/opt.c
index 5beb47390874c97da10d94ac4ba7b4b457f935c1..3cd151a3b584f98f84c5088dea0d972b29ca5344 100644
--- a/src/srun/libsrun/opt.c
+++ b/src/srun/libsrun/opt.c
@@ -661,6 +661,14 @@ static void _opt_env(void)
_process_env_var(e, val);
e++;
}
+
+ /* Running srun within an existing srun. Don't inherit values. */
+ if (getenv("SLURM_STEP_ID")) {
+ xfree(opt.cpu_bind);
+ opt.cpu_bind_type = 0;
+ xfree(opt.mem_bind);
+ opt.mem_bind_type = 0;
+ }
}
diff --git a/src/sview/job_info.c b/src/sview/job_info.c
index 39bd5670c4b29a470c03b086c8d586558ac5ef46..4d964127426cd059f9cca9d073ad407041330446 100644
--- a/src/sview/job_info.c
+++ b/src/sview/job_info.c
@@ -1328,7 +1328,7 @@ static void _layout_job_record(GtkTreeView *treeview,
int update)
{
char *nodes = NULL, *reason = NULL, *uname = NULL;
- char tmp_char[50];
+ char tmp_char[64];
char time_buf[32];
char tmp1[128];
char running_char[50];
@@ -3190,7 +3190,7 @@ static List _create_job_info_list(job_info_msg_t *job_info_ptr,
if (job_ptr->array_task_str ||
(job_ptr->array_task_id != NO_VAL)) {
- char task_str[16];
+ char task_str[64];
sview_job_info_t *first_job_info_ptr =
list_find_first(info_list,
_task_array_match, job_ptr);
diff --git a/src/sview/sview.c b/src/sview/sview.c
index d9805ea13e297218f81d1034b68b2477b4602819..0eea4a0c94d71cb3b4d6dab316c53f8e74aae899 100644
--- a/src/sview/sview.c
+++ b/src/sview/sview.c
@@ -1454,6 +1454,8 @@ int main(int argc, char *argv[])
int i=0;
log_options_t lopts = LOG_OPTS_STDERR_ONLY;
+ if (!getenv("SLURM_BITSTR_LEN"))
+ setenv("SLURM_BITSTR_LEN", "128", 1); /* More array info */
slurm_conf_init(NULL);
log_init(argv[0], lopts, SYSLOG_FACILITY_USER, NULL);
load_defaults();
diff --git a/testsuite/expect/test1.61 b/testsuite/expect/test1.61
index 2b1f74d44f01d93abb5399050e451e7fc9a17e7c..bcd0f75b9082c4b92d6ae1f16b6cad9454311119 100755
--- a/testsuite/expect/test1.61
+++ b/testsuite/expect/test1.61
@@ -115,7 +115,7 @@ if {[wait_for_file $file_err] == 0} {
set timed_out 0
spawn cat $file_err
expect {
- -re " STEP $job_id.$step_id CANCELLED AT.*DUE TO TIME LIMIT " {
+ -re " STEP $job_id.$step_id ON .* CANCELLED AT.*DUE TO TIME LIMIT " {
set timed_out 1
exp_continue
}