diff --git a/NEWS b/NEWS
index cbe0fb3d44e3fafcb66aff9a5a7e34eb9ef6fc47..123d2eaabf46e1ace80bc902aa90adfba5ba871b 100644
--- a/NEWS
+++ b/NEWS
@@ -410,7 +410,10 @@ documents those changes that are of interest to users and admins.
-- Cancelled or Failed jobs will now report there job and step id on exit
-- Add SPANK items available to get: SLURM_VERSION, SLURM_VERSION_MAJOR,
SLURM_VERISON_MINOR and SLURM_VERSION_MICRO.
- -- Fixed handling of SIGPIPE in srun
+ -- Fixed handling of SIGPIPE in srun. Abort job.
+ -- Fix bug introduced to MVAPICH plugin preventing use of TotalView debugger.
+ -- Modify slurmctld to get srun/salloc network address based upon the incoming
+ message rather than hostname set by the user command (more reliable).
* Changes in SLURM 1.2.32
=========================
diff --git a/doc/man/man8/spank.8 b/doc/man/man8/spank.8
index 7813a6d646df0c26124810c0c6e020bd23b66542..2ee7e5886d274048c78f9c153b69160b00340476 100644
--- a/doc/man/man8/spank.8
+++ b/doc/man/man8/spank.8
@@ -1,4 +1,4 @@
-.TH "SPANK" "8" "May 2006" "SPANK" "SLURM plug\-in architecture for Node and job (K)control"
+.TH "SPANK" "8" "Jul 2008" "SPANK" "SLURM plug\-in architecture for Node and job (K)control"
.SH "NAME"
\fBSPANK\fR \- SLURM Plug\-in Architecture for Node and job (K)control
@@ -34,9 +34,17 @@ Plugins may query the context in which they are running with the
launch. A plugin may define the following functions:
.TP 2
\fBslurm_spank_init\fR
-Called just after plugins are loaded. In remote context, this is
-just after job step is initialized. For local context, this is before
-user options are processed.
+Called just after plugins are loaded. In remote context, this is just
+after job step is initialized. This function is called before any plugin
+option processing.
+.TP
+\fBslurm_spank_init_post_opt\fR
+Called at the same point as \fBslurm_spank_init\fR, but after all
+user options to the plugin have been processed. The reason that the
+\fBinit\fR and \fBinit_post_opt\fR callbacks are separated is so that
+plugins can process system-wide options specified in plugstack.conf in
+the \fBinit\fR callback, then process user options, and finaly take some
+action in \fBslurm_spank_init_post_opt\fR if necessary.
.TP
\fBslurm_spank_local_user_init\fR
Called in local (\fBsrun\fR or \fBsbatch\fR) context only after all
diff --git a/slurm/spank.h b/slurm/spank.h
index 6ac02321be5e78419463900bc131eee81da657db..5ef0a5eb10a59a783812b57e59c10369f859d282 100644
--- a/slurm/spank.h
+++ b/slurm/spank.h
@@ -62,6 +62,8 @@ typedef int (spank_f) (spank_t spank, int ac, char *argv[]);
*
* slurmd -> slurmstepd
* `-> init ()
+ * -> process spank options
+ * -> init_post_opt ()
* + drop privileges (initgroups(), seteuid(), chdir())
* `-> user_init ()
* + for each task
@@ -83,6 +85,7 @@ typedef int (spank_f) (spank_t spank, int ac, char *argv[]);
*/
extern spank_f slurm_spank_init;
+extern spank_f slurm_spank_init_post_opt;
extern spank_f slurm_spank_local_user_init;
extern spank_f slurm_spank_user_init;
extern spank_f slurm_spank_task_init;
diff --git a/src/api/allocate.c b/src/api/allocate.c
index 7dda2e7da291ecf2287605b6b468b0cdd71985b0..fd2bc7257c7e363468a7ef504913d82e4fe45593 100644
--- a/src/api/allocate.c
+++ b/src/api/allocate.c
@@ -633,6 +633,9 @@ _handle_msg(slurm_msg_t *msg, resource_allocation_response_msg_t **resp)
*resp = msg->data;
rc = 1;
break;
+ case SRUN_JOB_COMPLETE:
+ info("Job has been cancelled");
+ break;
default:
error("received spurious message type: %d\n",
msg->msg_type);
diff --git a/src/common/plugstack.c b/src/common/plugstack.c
index 65d6235105ff53bd1b8e6e9449e53370777338f6..831c5344a5466c74c90eac02e1c8bebc55ae71e5 100644
--- a/src/common/plugstack.c
+++ b/src/common/plugstack.c
@@ -63,6 +63,7 @@
struct spank_plugin_operations {
spank_f *init;
+ spank_f *init_post_opt;
spank_f *local_user_init;
spank_f *user_init;
spank_f *user_task_init;
@@ -71,9 +72,10 @@ struct spank_plugin_operations {
spank_f *exit;
};
-const int n_spank_syms = 7;
+const int n_spank_syms = 8;
const char *spank_syms[] = {
"slurm_spank_init",
+ "slurm_spank_init_post_opt",
"slurm_spank_local_user_init",
"slurm_spank_user_init",
"slurm_spank_task_init",
@@ -131,6 +133,7 @@ typedef enum spank_handle_type {
*/
typedef enum step_fn {
SPANK_INIT = 0,
+ SPANK_INIT_POST_OPT,
LOCAL_USER_INIT,
STEP_USER_INIT,
STEP_USER_TASK_INIT,
@@ -449,6 +452,8 @@ static const char *_step_fn_name(step_fn_t type)
switch (type) {
case SPANK_INIT:
return ("init");
+ case SPANK_INIT_POST_OPT:
+ return ("init_post_opt");
case LOCAL_USER_INIT:
return ("local_user_init");
case STEP_USER_INIT:
@@ -498,6 +503,14 @@ static int _do_call_stack(step_fn_t type, void * job, int taskid)
fn_name, rc);
}
break;
+ case SPANK_INIT_POST_OPT:
+ if (sp->ops.init_post_opt) {
+ rc = (*sp->ops.init_post_opt) (spank, sp->ac,
+ sp->argv);
+ debug2("spank: %s: %s = %d\n", name,
+ fn_name, rc);
+ }
+ break;
case LOCAL_USER_INIT:
if (sp->ops.local_user_init) {
rc = (*sp->ops.local_user_init) (spank, sp->ac,
@@ -586,6 +599,9 @@ int spank_init(slurmd_job_t * job)
return (-1);
}
+ if (_do_call_stack(SPANK_INIT_POST_OPT, job, -1) < 0)
+ return (-1);
+
return (0);
}
diff --git a/src/plugins/mpi/mvapich/mvapich.c b/src/plugins/mpi/mvapich/mvapich.c
index f817698b7e73b3f74833c13067e1b20c7a571d1b..b361707353d4fe9168bc9f7945a199e2fc64e109 100644
--- a/src/plugins/mpi/mvapich/mvapich.c
+++ b/src/plugins/mpi/mvapich/mvapich.c
@@ -408,6 +408,20 @@ static void mvapich_poll_destroy (struct mvapich_poll *mp)
xfree (mp);
}
+
+/*
+ * Call poll(2) on mvapich_poll object, handling EAGAIN and EINTR errors.
+ */
+static int mvapich_poll_internal (struct mvapich_poll *mp)
+{
+ int n;
+ while ((n = poll (mp->fds, mp->nfds, startup_timeout (mp->st))) < 0) {
+ if (errno != EINTR && errno != EAGAIN)
+ return (-1);
+ }
+ return (n);
+}
+
/*
* Poll for next available mvapich_info object with read/write activity
*
@@ -457,7 +471,7 @@ again:
mvapich_debug3 ("mvapich_poll_next (nfds=%d, timeout=%d)\n",
mp->nfds, startup_timeout (st));
- if ((rc = poll (mp->fds, mp->nfds, startup_timeout (st))) < 0)
+ if ((rc = mvapich_poll_internal (mp)) < 0)
mvapich_terminate_job (st, "mvapich_poll_next: %m");
else if (rc == 0) {
/*
@@ -1284,8 +1298,9 @@ static int mvapich_abort_accept (mvapich_state_t *st)
mvapich_abort_timeout ());
while ((rc = poll (pfds, 1, mvapich_abort_timeout ())) < 0) {
- if (errno != EINTR)
- return (-1);
+ if (errno == EINTR || errno == EAGAIN)
+ continue;
+ return (-1);
}
/*
@@ -1689,11 +1704,13 @@ mvapich_initialize_connections (mvapich_state_t *st,
mvapich_debug3 ("do_poll (nfds=%d)\n", nfds);
- if ((rc = poll (fds, nfds, startup_timeout (st))) < 0) {
+ while ((rc = poll (fds, nfds, startup_timeout (st))) < 0) {
+ if (errno == EINTR || errno == EAGAIN)
+ continue;
error ("mvapich: poll: %m");
break;
}
- else if (rc == 0) {
+ if (rc == 0) {
report_absent_tasks (st, 1);
mvapich_terminate_job (st, NULL);
}
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index 6239b2a6374f460f70c82914f94ce334a4f4cba0..82576a90bb2d244b261726a38a8f221b7d96975d 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -1684,6 +1684,7 @@ extern int job_signal(uint32_t job_id, uint16_t signal, uint16_t batch_flag,
job_ptr->job_state = JOB_CANCELLED;
job_ptr->start_time = now;
job_ptr->end_time = now;
+ srun_allocate_abort(job_ptr);
job_completion_logger(job_ptr);
delete_job_details(job_ptr);
verbose("job_signal of pending job %u successful", job_id);
diff --git a/src/slurmctld/srun_comm.c b/src/slurmctld/srun_comm.c
index 322c6dbb9e48aae7a0c703459d509e5884c84920..8bcfb519ba77fc11742f9f4a23aa096060d2820f 100644
--- a/src/slurmctld/srun_comm.c
+++ b/src/slurmctld/srun_comm.c
@@ -107,6 +107,28 @@ extern void srun_allocate (uint32_t job_id)
}
}
+/*
+ * srun_allocate_abort - notify srun of a resource allocation failure
+ * IN job_id - id of the job allocated resource
+ */
+extern void srun_allocate_abort(struct job_record *job_ptr)
+{
+ if (job_ptr && job_ptr->alloc_resp_port && job_ptr->alloc_node
+ && job_ptr->resp_host) {
+ slurm_addr * addr;
+ srun_job_complete_msg_t *msg_arg;
+ addr = xmalloc(sizeof(struct sockaddr_in));
+ slurm_set_addr(addr, job_ptr->alloc_resp_port,
+ job_ptr->resp_host);
+ msg_arg = xmalloc(sizeof(srun_timeout_msg_t));
+ msg_arg->job_id = job_ptr->job_id;
+ msg_arg->step_id = NO_VAL;
+ _srun_agent_launch(addr, job_ptr->alloc_node,
+ SRUN_JOB_COMPLETE,
+ msg_arg);
+ }
+}
+
/*
* srun_node_fail - notify srun of a node's failure
* IN job_id - id of job to notify
diff --git a/src/slurmctld/srun_comm.h b/src/slurmctld/srun_comm.h
index 6e796100beaef65880d401b17d008daed19f9939..7858e5a2ba7f45a845f4e6948fcf7d45d36e30f0 100644
--- a/src/slurmctld/srun_comm.h
+++ b/src/slurmctld/srun_comm.h
@@ -49,6 +49,12 @@
*/
extern void srun_allocate (uint32_t job_id);
+/*
+ * srun_allocate_abort - notify srun of a resource allocation failure
+ * IN job_id - id of the job allocated resource
+ */
+extern void srun_allocate_abort(struct job_record *job_ptr);
+
/*
* srun_exec - request that srun execute a specific command
* and route it's output to stdout