diff --git a/NEWS b/NEWS
index 97e575996af17722caca0039c5dc988635ebdb9b..687ae9a190acab040fee1e0fc9eda2ff799849fc 100644
--- a/NEWS
+++ b/NEWS
@@ -216,6 +216,12 @@ documents those changes that are of interest to users and admins.
srun under a bash shell in an emacs *shell* buffer.
-- Fix srun race condition that occasionally causes segfaults at shutdown
-- Fix obscure locking issues in log.c code.
+ -- Explicitly close IO related sockets. If an srun gets "stuck", possibly
+ because of unkillable tasks in its job step, it will not hold many TCP
+ sockets in the CLOSE_WAIT state.
+ -- Increase the SLURM protocol timeout from 5 seconds to 10 seconds.
+ (In 1.2 there will be a slurm.conf parameter for this, rather than having
+ it hardcoded.)
* Changes in SLURM 1.0.14
=========================
diff --git a/src/api/step_client_io.c b/src/api/step_client_io.c
index 7dfb750b860dc4917c23c2fb51a2a63f6cd44275..8096c3e429b957a628756a5ab5c8f7ed5f420d4d 100644
--- a/src/api/step_client_io.c
+++ b/src/api/step_client_io.c
@@ -182,6 +182,10 @@ _listening_socket_readable(eio_obj_t *obj)
{
debug3("Called _listening_socket_readable");
if (obj->shutdown == true) {
+ if (obj->fd != -1) {
+ close(obj->fd);
+ obj->fd = -1;
+ }
debug2(" false, shutdown");
return false;
}
@@ -257,7 +261,18 @@ _server_readable(eio_obj_t *obj)
return true;
}
- debug4(" false");
+ if (obj->shutdown) {
+ if (obj->fd != -1) {
+ close(obj->fd);
+ obj->fd = -1;
+ s->in_eof = true;
+ s->out_eof = true;
+ }
+ debug3(" false, shutdown");
+ return false;
+ }
+
+ debug3(" false");
return false;
}
@@ -280,7 +295,10 @@ _server_read(eio_obj_t *obj, List objs)
n = io_hdr_read_fd(obj->fd, &s->header);
if (n <= 0) { /* got eof or error on socket read */
debug3( "got eof or error on _server_read header");
+ close(obj->fd);
+ obj->fd = -1;
s->in_eof = true;
+ s->out_eof = true;
list_enqueue(s->cio->free_outgoing, s->in_msg);
s->in_msg = NULL;
return SLURM_SUCCESS;
@@ -316,7 +334,10 @@ _server_read(eio_obj_t *obj, List objs)
}
if (n <= 0) { /* got eof or unhandled error */
debug3( "got eof on _server_read body");
+ close(obj->fd);
+ obj->fd = -1;
s->in_eof = true;
+ s->out_eof = true;
list_enqueue(s->cio->free_outgoing, s->in_msg);
s->in_msg = NULL;
return SLURM_SUCCESS;
diff --git a/src/common/env.c b/src/common/env.c
index 8636912ce36043a56a416318c75086dd2199cbe5..71ed7f4d85fbab9af244f6952bd210108052a37a 100644
--- a/src/common/env.c
+++ b/src/common/env.c
@@ -254,9 +254,8 @@ int setup_env(env_t *env)
error("Unable to set SLURM_CPUS_PER_TASK");
rc = SLURM_FAILURE;
}
-
- if (env->distribution
- && env->distribution != SLURM_DIST_UNKNOWN) {
+
+ if (env->distribution != SLURM_DIST_UNKNOWN) {
switch(env->distribution) {
case SLURM_DIST_CYCLIC:
dist = "cyclic";
@@ -268,14 +267,14 @@ int setup_env(env_t *env)
dist = "arbitrary";
break;
default:
- dist = "unknown";
+ dist = "cyclic";
}
if (setenvf(&env->env, "SLURM_DISTRIBUTION", "%s", dist)) {
error("Can't set SLURM_DISTRIBUTION env variable");
rc = SLURM_FAILURE;
}
- }
+ }
if (env->cpu_bind_type) {
int setstat = 0;
diff --git a/src/slurmd/slurmstepd/mgr.c b/src/slurmd/slurmstepd/mgr.c
index 48a8002bea67e6a3cf0e261dfef8244335bcb887..e1d4e365d77e0b7bb60aa4856624377671af6bb8 100644
--- a/src/slurmd/slurmstepd/mgr.c
+++ b/src/slurmd/slurmstepd/mgr.c
@@ -1095,12 +1095,16 @@ _wait_for_any_task(slurmd_job_t *job, bool waitflag)
job->envtp->env = job->env;
job->envtp->procid = job->task[i]->gtid;
job->envtp->localid = job->task[i]->id;
+
+ /* need to take this out in 1.2 */
+ job->envtp->distribution = SLURM_DIST_UNKNOWN;
setup_env(job->envtp);
job->env = job->envtp->env;
if (job->task_epilog) {
run_script("user task_epilog",
job->task_epilog,
- job->jobid, job->uid, 2, job->env);
+ job->jobid, job->uid,
+ 2, job->env);
}
if (conf->task_epilog) {
char *my_epilog;
diff --git a/src/slurmd/slurmstepd/task.c b/src/slurmd/slurmstepd/task.c
index 6a3dbb97cc888150c7e1cbda7d1843c74229a4d3..aa1d3198e3c45669d83d1276ad27954855205403 100644
--- a/src/slurmd/slurmstepd/task.c
+++ b/src/slurmd/slurmstepd/task.c
@@ -272,7 +272,10 @@ exec_task(slurmd_job_t *job, int i, int waitfd)
job->envtp->mem_bind = xstrdup(job->mem_bind);
job->envtp->mem_bind_type = job->mem_bind_type;
+ /* need to take this out in 1.2 */
+ job->envtp->distribution = SLURM_DIST_UNKNOWN;
setup_env(job->envtp);
+
job->env = job->envtp->env;
job->envtp->env = NULL;
xfree(job->envtp->task_count);
diff --git a/src/srun/allocate.c b/src/srun/allocate.c
index 1c6f75b31c4a6cf0bac5d6f76d47b9c8978434cd..57dddc24bda2adf4fcd56fc558a3016add895a14 100644
--- a/src/srun/allocate.c
+++ b/src/srun/allocate.c
@@ -74,7 +74,9 @@ allocate_test(void)
{
int rc;
job_desc_msg_t *j = job_desc_msg_create_from_opts (NULL);
-
+ if(!j)
+ return SLURM_ERROR;
+
rc = slurm_job_will_run(j);
job_desc_msg_destroy(j);
return rc;
@@ -89,7 +91,9 @@ allocate_nodes(void)
sigset_t oset;
resource_allocation_response_msg_t *resp = NULL;
job_desc_msg_t *j = job_desc_msg_create_from_opts (NULL);
-
+ if(!j)
+ return NULL;
+
oquitf = xsignal(SIGQUIT, _intr_handler);
ointf = xsignal(SIGINT, _intr_handler);
otermf = xsignal(SIGTERM, _intr_handler);
@@ -424,6 +428,12 @@ job_desc_msg_create_from_opts (char *script)
}
}
}
+ if(opt.distribution == SLURM_DIST_ARBITRARY
+ && !j->req_nodes) {
+ error("With Arbitrary distribution you need to "
+ "specify a nodelist or hostfile with the -w option");
+ return NULL;
+ }
j->exc_nodes = opt.exc_nodes;
j->partition = opt.partition;
j->min_nodes = opt.min_nodes;
@@ -541,25 +551,25 @@ _step_req_create(srun_job_t *j)
r->name = xstrdup(opt.job_name);
r->relative = false; /* XXX fix this oneday */
- /* CJM - why are "UNKNOWN" and "default" behaviours different? */
- /* why do we even HAVE the SLURM_DIST_UNKNOWN state?? */
switch (opt.distribution) {
- case SLURM_DIST_UNKNOWN:
- r->task_dist = (opt.nprocs <= j->nhosts) ? SLURM_DIST_CYCLIC
- : SLURM_DIST_BLOCK;
- break;
case SLURM_DIST_CYCLIC:
r->task_dist = SLURM_DIST_CYCLIC;
break;
+ case SLURM_DIST_BLOCK:
+ r->task_dist = SLURM_DIST_BLOCK;
+ break;
case SLURM_DIST_ARBITRARY:
r->task_dist = SLURM_DIST_ARBITRARY;
break;
- case SLURM_DIST_BLOCK:
+ case SLURM_DIST_UNKNOWN:
default:
- r->task_dist = SLURM_DIST_BLOCK;
+ r->task_dist = (opt.nprocs <= j->nhosts) ? SLURM_DIST_CYCLIC
+ : SLURM_DIST_BLOCK;
break;
}
-
+ /* make sure we set the env correctly */
+ opt.distribution = r->task_dist;
+
if (slurmctld_comm_addr.port) {
r->host = xstrdup(slurmctld_comm_addr.hostname);
r->port = slurmctld_comm_addr.port;
@@ -591,11 +601,11 @@ create_job_step(srun_job_t *job,
job->step_layout = step_layout_create(alloc_resp, resp, req);
if(!job->step_layout) {
error("step_layout not created correctly");
- return 1;
+ return -1;
}
if(task_layout(job->step_layout) != SLURM_SUCCESS) {
error("problem with task layout");
- return 1;
+ return -1;
}
/*
diff --git a/src/srun/srun.c b/src/srun/srun.c
index 39609a084201346e29f806334feafe75b2b999e5..2718aa0c5b9d07bdafeaf592c5f5a2e9940d849b 100644
--- a/src/srun/srun.c
+++ b/src/srun/srun.c
@@ -392,7 +392,7 @@ int srun(int ac, char **av)
if (slurm_mpi_exit () < 0)
; /* eh, ignore errors here */
-
+
_run_srun_epilog(job);
/*