diff --git a/doc/html/mpiplugins.shtml b/doc/html/mpiplugins.shtml
index afffe045d6b5d13694b13dbdeb5018120cbafaf6..4082ef9f8d22d2ab4ba0857a6752f9c9c3396ffd 100644
--- a/doc/html/mpiplugins.shtml
+++ b/doc/html/mpiplugins.shtml
@@ -42,6 +42,12 @@ slurmd daemon runs
<br>
which will set configure the slurmd to use the correct mpi as well to interact with the srun.
<br>
+slurmstepd process runs
+<br>
+<i>p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env);</i>
+<br>
+which executes immediately before fork/exec of tasks.
+<br>
<p class="footer"><a href="#top">top</a></p>
@@ -84,6 +90,17 @@ can run at the same time </p>
<p style="margin-left:.2in"><b>Returns</b>: false if multiple tasks can run and true if only
a single task can run at one time.</p>
+<p class="commandline">int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env);</p>
+<p style="margin-left:.2in"><b>Description</b>: Used by slurmstepd process immediately prior
+to fork and exec of user tasks.</p>
+<p style="margin-left:.2in"><b>Arguments</b>:<br>
+<span class="commandline"> job</span> (input)
+Pointer to the slurmd sturcture for the job that is running.<br>
+<span class="commandline"> env</span> (input)
+Environment variables for tasks to be spawned.</p>
+<p style="margin-left:.2in"><b>Returns</b>: SLURM_SUCCESS if successful. On failure,
+the plugin should return -1.</p>
+
<p class="commandline">int mpi_p_exit();</p>
<p style="margin-left:.2in"><b>Description</b>: Cleans up anything that needs cleaning up after
execution.</p>
@@ -104,6 +121,6 @@ to maintain data format compatibility across different versions of the plugin.</
<p class="footer"><a href="#top">top</a></p>
-<p style="text-align:center;">Last modified 11 April 2006</p>
+<p style="text-align:center;">Last modified 3 February 2012</p>
<!--#include virtual="footer.txt"-->
diff --git a/src/common/mpi.c b/src/common/mpi.c
index 8b3a2952edccf43c48f1b0a2271796600ee933d1..9ecdf1da837a8e9ef7ab4d88819e03d73964fc8a 100644
--- a/src/common/mpi.c
+++ b/src/common/mpi.c
@@ -51,7 +51,6 @@
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"
-
/*
* WARNING: Do not change the order of these fields or add additional
* fields at the beginning of the structure. If you do, MPI plugins
@@ -59,6 +58,8 @@
* at the end of the structure.
*/
typedef struct slurm_mpi_ops {
+ int (*slurmstepd_prefork)(const slurmd_job_t *job,
+ char ***env);
int (*slurmstepd_init) (const mpi_plugin_task_info_t *job,
char ***env);
mpi_plugin_client_state_t *
@@ -142,6 +143,7 @@ _slurm_mpi_get_ops( slurm_mpi_context_t c )
* declared for slurm_mpi_ops_t.
*/
static const char *syms[] = {
+ "p_mpi_hook_slurmstepd_prefork",
"p_mpi_hook_slurmstepd_task",
"p_mpi_hook_client_prelaunch",
"p_mpi_hook_client_single_task_per_node",
@@ -265,6 +267,14 @@ int mpi_hook_slurmstepd_init (char ***env)
return SLURM_SUCCESS;
}
+int mpi_hook_slurmstepd_prefork (const slurmd_job_t *job, char ***env)
+{
+ if (mpi_hook_slurmstepd_init(env) == SLURM_ERROR)
+ return SLURM_ERROR;
+
+ return (*(g_context->ops.slurmstepd_prefork))(job, env);
+}
+
int mpi_hook_slurmstepd_task (const mpi_plugin_task_info_t *job, char ***env)
{
if (mpi_hook_slurmstepd_init(env) == SLURM_ERROR)
diff --git a/src/common/mpi.h b/src/common/mpi.h
index 37d1b08ed97022d70d3161342fce8376d2dab7e5..0be96edbeb6c15837f48d3250017d272088eae94 100644
--- a/src/common/mpi.h
+++ b/src/common/mpi.h
@@ -46,6 +46,8 @@
#include <stdbool.h>
#include "slurm/slurm.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
+
typedef struct slurm_mpi_context *slurm_mpi_context_t;
typedef void mpi_plugin_client_state_t;
@@ -103,6 +105,9 @@ int mpi_hook_slurmstepd_init (char ***env);
*/
int mpi_hook_slurmstepd_task (const mpi_plugin_task_info_t *job, char ***env);
+
+int mpi_hook_slurmstepd_prefork (const slurmd_job_t *job, char ***env);
+
/**********************************************************************
* Hooks called by client applications.
* For instance: srun, slaunch, slurm_step_launch().
diff --git a/src/plugins/mpi/lam/mpi_lam.c b/src/plugins/mpi/lam/mpi_lam.c
index 835377d2b0e44cd7d59189a927ae94e455897a1c..e02911fe84eecd81ffe8be338c7cd39a6d311d25 100644
--- a/src/plugins/mpi/lam/mpi_lam.c
+++ b/src/plugins/mpi/lam/mpi_lam.c
@@ -48,6 +48,7 @@
#include "slurm/slurm_errno.h"
#include "src/common/slurm_xlator.h"
#include "src/plugins/mpi/lam/lam.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -80,6 +81,12 @@ const char plugin_name[] = "mpi LAM plugin";
const char plugin_type[] = "mpi/lam";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/lam: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task(const mpi_plugin_task_info_t *job,
char ***env)
{
diff --git a/src/plugins/mpi/mpich1_p4/mpich1_p4.c b/src/plugins/mpi/mpich1_p4/mpich1_p4.c
index 201960f730ad64f811309afa2a51f38ccf84edda..d4cad0f9725d7f2ebceb570cf26ecfe523e41e99 100644
--- a/src/plugins/mpi/mpich1_p4/mpich1_p4.c
+++ b/src/plugins/mpi/mpich1_p4/mpich1_p4.c
@@ -55,6 +55,7 @@
#include "src/common/net.h"
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -103,7 +104,13 @@ static pthread_mutex_t shutdown_lock;
static pthread_cond_t shutdown_cond;
-int p_mpi_hook_slurmstepd_task (const mpi_plugin_client_info_t *job,
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/mpich1_p4: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
+int p_mpi_hook_slurmstepd_task (const mpi_plugin_task_info_t *job,
char ***env)
{
char *nodelist, *task_cnt;
diff --git a/src/plugins/mpi/mpich1_shmem/mpich1_shmem.c b/src/plugins/mpi/mpich1_shmem/mpich1_shmem.c
index bc1e2f89c06bc0983c72fe55ceb85dbbe1fe5b9d..212c20cb800ef18c8295702cfe7e91580e9af526 100644
--- a/src/plugins/mpi/mpich1_shmem/mpich1_shmem.c
+++ b/src/plugins/mpi/mpich1_shmem/mpich1_shmem.c
@@ -48,6 +48,7 @@
#include "slurm/slurm_errno.h"
#include "src/common/slurm_xlator.h"
#include "src/plugins/mpi/lam/lam.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -80,6 +81,12 @@ const char plugin_name[] = "mpich1_shmem plugin";
const char plugin_type[] = "mpi/mpich1_shmem";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/mpich1_shmem: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task(const mpi_plugin_task_info_t *job,
char ***env)
{
diff --git a/src/plugins/mpi/mpichgm/mpi_mpichgm.c b/src/plugins/mpi/mpichgm/mpi_mpichgm.c
index 422718a36da733ed3421c1fabfbb741ab9372683..e62a929f210e1af34c836cda0b0386530680a4b9 100644
--- a/src/plugins/mpi/mpichgm/mpi_mpichgm.c
+++ b/src/plugins/mpi/mpichgm/mpi_mpichgm.c
@@ -49,6 +49,7 @@
#include "slurm/slurm_errno.h"
#include "src/common/slurm_xlator.h"
#include "src/plugins/mpi/mpichgm/mpichgm.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -81,6 +82,12 @@ const char plugin_name[] = "mpi MPICH-GM plugin";
const char plugin_type[] = "mpi/mpichgm";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/mpichgm: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task(const mpi_plugin_task_info_t *job,
char ***env)
{
diff --git a/src/plugins/mpi/mpichmx/mpi_mpichmx.c b/src/plugins/mpi/mpichmx/mpi_mpichmx.c
index f13b69816ce73d7eef4fa984a93ff720a22c7b18..cb89201af1093009bca5ca19f9455589e643eb46 100644
--- a/src/plugins/mpi/mpichmx/mpi_mpichmx.c
+++ b/src/plugins/mpi/mpichmx/mpi_mpichmx.c
@@ -48,6 +48,7 @@
#include "slurm/slurm_errno.h"
#include "src/common/slurm_xlator.h"
#include "src/plugins/mpi/mpichmx/mpichmx.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -80,6 +81,12 @@ const char plugin_name[] = "mpi MPICH-MX plugin";
const char plugin_type[] = "mpi/mpichmx";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/mpichmx: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task(const mpi_plugin_task_info_t *job,
char ***env)
{
diff --git a/src/plugins/mpi/mvapich/mpi_mvapich.c b/src/plugins/mpi/mvapich/mpi_mvapich.c
index 0b33ead35421b54caf3e996d22e253ce4faa1b75..c80dc18b24a0092722e1281aec05a624ee4a40e3 100644
--- a/src/plugins/mpi/mvapich/mpi_mvapich.c
+++ b/src/plugins/mpi/mvapich/mpi_mvapich.c
@@ -49,6 +49,7 @@
#include "slurm/slurm_errno.h"
#include "src/common/slurm_xlator.h"
#include "src/plugins/mpi/mvapich/mvapich.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -81,6 +82,12 @@ const char plugin_name[] = "mpi MVAPICH plugin";
const char plugin_type[] = "mpi/mvapich";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/mvapich: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task (const mpi_plugin_task_info_t *job,
char ***env)
{
diff --git a/src/plugins/mpi/none/mpi_none.c b/src/plugins/mpi/none/mpi_none.c
index 514d2376bda4ceb29929b24eaf88f6edc2259ccc..a4d42ca12866e1ae9c615e2dc14635b00297201c 100644
--- a/src/plugins/mpi/none/mpi_none.c
+++ b/src/plugins/mpi/none/mpi_none.c
@@ -50,6 +50,7 @@
#include "src/common/slurm_xlator.h"
#include "src/common/mpi.h"
#include "src/common/env.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -82,6 +83,12 @@ const char plugin_name[] = "mpi none plugin";
const char plugin_type[] = "mpi/none";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/none: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task(const mpi_plugin_task_info_t*job,
char ***env)
{
diff --git a/src/plugins/mpi/openmpi/mpi_openmpi.c b/src/plugins/mpi/openmpi/mpi_openmpi.c
index c7e9ea55f9e2ef16323159017b2f3b3866363db7..556b0f79e93ae1e6a8bf2cd8ddaf7d521aaeb958 100644
--- a/src/plugins/mpi/openmpi/mpi_openmpi.c
+++ b/src/plugins/mpi/openmpi/mpi_openmpi.c
@@ -50,6 +50,7 @@
#include "src/common/slurm_xlator.h"
#include "src/common/mpi.h"
#include "src/common/env.h"
+#include "src/slurmd/slurmstepd/slurmstepd_job.h"
/*
* These variables are required by the generic plugin interface. If they
@@ -82,6 +83,12 @@ const char plugin_name[] = "OpenMPI plugin";
const char plugin_type[] = "mpi/openmpi";
const uint32_t plugin_version = 100;
+int p_mpi_hook_slurmstepd_prefork(const slurmd_job_t *job, char ***env)
+{
+ debug("mpi/openmpi: slurmstepd prefork");
+ return SLURM_SUCCESS;
+}
+
int p_mpi_hook_slurmstepd_task(const mpi_plugin_task_info_t *job,
char ***env)
{
diff --git a/src/slurmd/slurmstepd/mgr.c b/src/slurmd/slurmstepd/mgr.c
index 8afd6e058dc86dc1d24b181e06e3b3274ab322df..5da4a9dc54ba053d43cab2a9fc7cb282345039a6 100644
--- a/src/slurmd/slurmstepd/mgr.c
+++ b/src/slurmd/slurmstepd/mgr.c
@@ -989,6 +989,14 @@ job_manager(slurmd_job_t *job)
goto fail2;
}
+ /* fork necessary threads for MPI */
+ if (mpi_hook_slurmstepd_prefork(job, &job->env) != SLURM_SUCCESS) {
+ error("Failed mpi_hook_slurmstepd_prefork");
+ rc = SLURM_FAILURE;
+ io_close_task_fds(job);
+ goto fail2;
+ }
+
/* calls pam_setup() and requires pam_finish() if successful */
if (_fork_all_tasks(job) < 0) {
debug("_fork_all_tasks failed");