diff --git a/NEWS b/NEWS
index 8ef496dbccb1a6d622081c1d2ec209a2cf9e8cd3..c6d4d9e73b68e058501c392e9e23089456fd0a3d 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,9 @@ documents those changes that are of interest to users and admins.
-- Changed DbdBackup to DbdBackupHost for slurmdbd.conf file
-- Add support for spank_strerror() function and improve error handling in
general for SPANK plugins.
+ -- Added configuration parameter SrunIOTimeout to optionally ping srun's tasks
+ for better fault tolerance (e.g. killed and restarteed SLURM daemons on
+ compute node).
* Changes in SLURM 1.4.0-pre13
==============================
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 1bff4ea96a61970b15717dc2b7f860035bca95f0..f295304864434c8a3fdde6339cc6f2d92eab07d8 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -106,6 +106,8 @@ CONFIGURATION FILE CHANGES (see "man slurm.conf" for details)
TopologyPlugin and in a new topology.conf file: SwitchName, Nodes,
Switches. More information is available in man pages for slurm.conf,
topology.conf, and https://computing.llnl.gov/linux/slurm/topology.html
+* SrunIOTimeout has been added to optionally ping srun's tasks for better
+ fault tolerance (e.g. killed and restarteed SLURM daemons on compute node).
* BLUEGENE - Added option DenyPassthrough in the bluegene.conf. Can be set
to any combination of X,Y,Z to not allow passthroughs when running in
dynamic layout mode. (see "man bluegene.conf" for details)
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index 098b31b429888352ac9d840435a9e4eb35004773..351d8f5ebdb05d01da79bbab1f5039b995c48938 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -1,4 +1,4 @@
-.TH "slurm.conf" "5" "April 2009" "slurm.conf 2.0" "Slurm configuration file"
+.TH "slurm.conf" "5" "May 2009" "slurm.conf 2.0" "Slurm configuration file"
.SH "NAME"
slurm.conf \- Slurm configuration file
@@ -1349,6 +1349,21 @@ completion of a job step. The command line arguments for the executable will
be the command and arguments of the job step. This configuration parameter
may be overridden by srun's \fB\-\-epilog\fR parameter.
+.TP
+\fBSrunIOTimeout\fR
+While the \fBsrun\fR detects the termination of tasks under almost all
+circumstances, there are abnormal deamon failures which may not be
+detected immediately.
+Such abnormal failures can be detected by \fBsrun\fR using a more active
+polling mechanism.
+Note that polling does have an impact upon application performance.
+The interval of polling is specified by the \fBSrunIOTimeout\fR
+parameter and its units are seconds.
+\fBsrun\fR's \fB\-\-io\-timeout\fR option takes precedence over
+this configuration parameter.
+The default value is 0 (no polling).
+The value may not exceed 65533 seconds.
+
.TP
\fBSrunProlog\fR
Fully qualified pathname of an executable to be run by srun prior to the
diff --git a/slurm/slurm.h.in b/slurm/slurm.h.in
index 663a4ff35f6b6c50d2ecbd1ed5be0cbf57370394..1bf4da80db7427456d097c5ff5fd43860f32f7c3 100644
--- a/slurm/slurm.h.in
+++ b/slurm/slurm.h.in
@@ -1200,6 +1200,7 @@ typedef struct slurm_ctl_conf {
* considering node DOWN */
char *slurm_conf; /* pathname of slurm config file */
char *srun_epilog; /* srun epilog program */
+ uint16_t srun_io_timeout; /* timeout for non-responsive tasks */
char *srun_prolog; /* srun prolog program */
char *state_save_location;/* pathname of slurmctld state save
* directory */
diff --git a/src/api/config_info.c b/src/api/config_info.c
index 87869598271df747a5229e1baf61d65bc5948c05..50deedcd9d2dce8cac1cba87493c5397c68a5ea9 100644
--- a/src/api/config_info.c
+++ b/src/api/config_info.c
@@ -406,6 +406,8 @@ void slurm_print_ctl_conf ( FILE* out,
fprintf(out, "SLURM_VERSION = %s\n", SLURM_VERSION);
fprintf(out, "SrunEpilog = %s\n",
slurm_ctl_conf_ptr->srun_epilog);
+ fprintf(out, "SrunIOTimeout = %u sec\n",
+ slurm_ctl_conf_ptr->srun_io_timeout);
fprintf(out, "SrunProlog = %s\n",
slurm_ctl_conf_ptr->srun_prolog);
fprintf(out, "StateSaveLocation = %s\n",
diff --git a/src/common/read_config.c b/src/common/read_config.c
index 1b0f33b92a85103f3cebdf3435015d48a003bae4..e05787224e45d92dec3fcfb3539539951f21bcb4 100644
--- a/src/common/read_config.c
+++ b/src/common/read_config.c
@@ -244,6 +244,7 @@ s_p_options_t slurm_conf_options[] = {
{"SlurmdSpoolDir", S_P_STRING},
{"SlurmdTimeout", S_P_UINT16},
{"SrunEpilog", S_P_STRING},
+ {"SrunIOTimeout", S_P_UINT16},
{"SrunProlog", S_P_STRING},
{"StateSaveLocation", S_P_STRING},
{"SuspendExcNodes", S_P_STRING},
@@ -1417,6 +1418,7 @@ init_slurm_conf (slurm_ctl_conf_t *ctl_conf_ptr)
xfree (ctl_conf_ptr->slurmd_spooldir);
ctl_conf_ptr->slurmd_timeout = (uint16_t) NO_VAL;
xfree (ctl_conf_ptr->srun_prolog);
+ ctl_conf_ptr->srun_io_timeout = 0;
xfree (ctl_conf_ptr->srun_epilog);
xfree (ctl_conf_ptr->state_save_location);
xfree (ctl_conf_ptr->suspend_exc_nodes);
@@ -2283,6 +2285,7 @@ validate_and_set_defaults(slurm_ctl_conf_t *conf, s_p_hashtbl_t *hashtbl)
conf->slurmd_timeout = DEFAULT_SLURMD_TIMEOUT;
s_p_get_string(&conf->srun_prolog, "SrunProlog", hashtbl);
+ s_p_get_uint16(&conf->srun_io_timeout, "SrunIOTimeout", hashtbl);
s_p_get_string(&conf->srun_epilog, "SrunEpilog", hashtbl);
if (!s_p_get_string(&conf->state_save_location,
diff --git a/src/common/slurm_protocol_api.c b/src/common/slurm_protocol_api.c
index 5ab6ba4b2a5134fec4fcc731ba48e24da14f6277..eaf71ced23aae58aae7e2bbea407c312607033bb 100644
--- a/src/common/slurm_protocol_api.c
+++ b/src/common/slurm_protocol_api.c
@@ -1439,6 +1439,23 @@ char *slurm_get_select_type(void)
return select_type;
}
+/* slurm_get_srun_io_timeout
+ * get default srun I/O task timeout value from slurmctld_conf object
+ */
+uint16_t slurm_get_srun_io_timeout(void)
+{
+ uint16_t srun_io_timeout = 0;
+ slurm_ctl_conf_t *conf;
+
+ if(slurmdbd_conf) {
+ } else {
+ conf = slurm_conf_lock();
+ srun_io_timeout = conf->srun_io_timeout;
+ slurm_conf_unlock();
+ }
+ return srun_io_timeout;
+}
+
/* slurm_get_switch_type
* get switch type from slurmctld_conf object
* RET char * - switch type, MUST be xfreed by caller
diff --git a/src/common/slurm_protocol_api.h b/src/common/slurm_protocol_api.h
index 29ad64c7e080c43eb2b33b99a6b0ecd31857d83e..b4a5137708ab0162a1beddca8e3fdc1e051a47ca 100644
--- a/src/common/slurm_protocol_api.h
+++ b/src/common/slurm_protocol_api.h
@@ -486,6 +486,11 @@ char *slurm_get_sched_type(void);
*/
char *slurm_get_select_type(void);
+/* slurm_get_srun_io_timeout
+ * get default srun I/O task timeout value from slurmctld_conf object
+ */
+uint16_t slurm_get_srun_io_timeout(void);
+
/* slurm_get_switch_type
* get switch type from slurmctld_conf object
* RET char * - switch type, MUST be xfreed by caller
diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index d1650c01b2153107ce1b10603f505c43ceea5f53..d004ae76692ff681c86aec505e2f849c105907a8 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -2968,6 +2968,7 @@ _pack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t * build_ptr, Buf buffer)
pack16(build_ptr->slurmd_timeout, buffer);
packstr(build_ptr->srun_epilog, buffer);
+ pack16(build_ptr->srun_io_timeout, buffer);
packstr(build_ptr->srun_prolog, buffer);
packstr(build_ptr->state_save_location, buffer);
packstr(build_ptr->suspend_exc_nodes, buffer);
@@ -3190,6 +3191,7 @@ _unpack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t **
safe_unpack16(&build_ptr->slurmd_timeout, buffer);
safe_unpackstr_xmalloc(&build_ptr->srun_epilog, &uint32_tmp, buffer);
+ safe_unpack16(&build_ptr->srun_io_timeout, buffer);
safe_unpackstr_xmalloc(&build_ptr->srun_prolog, &uint32_tmp, buffer);
safe_unpackstr_xmalloc(&build_ptr->state_save_location,
&uint32_tmp, buffer);
diff --git a/src/slurmctld/proc_req.c b/src/slurmctld/proc_req.c
index 18d7b4431120a3208544034fac4402d8aac2a7fc..723702b7b78be29f0b0fab4d9ab39d5a0d4e5d2c 100644
--- a/src/slurmctld/proc_req.c
+++ b/src/slurmctld/proc_req.c
@@ -525,6 +525,7 @@ void _fill_ctld_conf(slurm_ctl_conf_t * conf_ptr)
conf_ptr->slurmd_user_name = xstrdup(conf->slurmd_user_name);
conf_ptr->slurm_conf = xstrdup(conf->slurm_conf);
conf_ptr->srun_prolog = xstrdup(conf->srun_prolog);
+ conf_ptr->srun_io_timeout = conf->srun_io_timeout;
conf_ptr->srun_epilog = xstrdup(conf->srun_epilog);
conf_ptr->state_save_location = xstrdup(conf->state_save_location);
conf_ptr->suspend_exc_nodes = xstrdup(conf->suspend_exc_nodes);