From 7db90c8a28da736d0ed6ec64d70ea9d28f949ec8 Mon Sep 17 00:00:00 2001
From: Danny Auble <da@schedmd.com>
Date: Wed, 8 Feb 2017 17:52:41 -0700
Subject: [PATCH] PMIX - Make it possible to use %n or %h in a spool dir.
---
src/common/slurm_protocol_api.c | 8 ++++++--
src/common/slurm_protocol_api.h | 2 +-
src/plugins/mpi/pmi2/setup.c | 9 +++++++--
src/plugins/mpi/pmix/pmixp_info.c | 23 ++++++++++++-----------
src/plugins/mpi/pmix/pmixp_info.h | 9 ++++-----
5 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/src/common/slurm_protocol_api.c b/src/common/slurm_protocol_api.c
index fc03e6fd201..9312b173079 100644
--- a/src/common/slurm_protocol_api.c
+++ b/src/common/slurm_protocol_api.c
@@ -2751,7 +2751,7 @@ char *slurm_get_job_container_plugin(void)
/* slurm_get_slurmd_spooldir
* RET slurmd_spooldir name, must be xfreed by caller */
-char *slurm_get_slurmd_spooldir(void)
+char *slurm_get_slurmd_spooldir(char *node_name)
{
char *slurmd_spooldir = NULL;
slurm_ctl_conf_t *conf;
@@ -2759,7 +2759,11 @@ char *slurm_get_slurmd_spooldir(void)
if (slurmdbd_conf) {
} else {
conf = slurm_conf_lock();
- slurmd_spooldir = xstrdup(conf->slurmd_spooldir);
+ if (!node_name)
+ slurmd_spooldir = xstrdup(conf->slurmd_spooldir);
+ else
+ slurmd_spooldir = slurm_conf_expand_slurmd_path(
+ conf->slurmd_spooldir, node_name);
slurm_conf_unlock();
}
return slurmd_spooldir;
diff --git a/src/common/slurm_protocol_api.h b/src/common/slurm_protocol_api.h
index 21419409d07..cafa4775554 100644
--- a/src/common/slurm_protocol_api.h
+++ b/src/common/slurm_protocol_api.h
@@ -895,7 +895,7 @@ char *slurm_get_job_container_plugin(void);
/* slurm_get_slurmd_spooldir
* RET slurmd_spooldir name, must be xfreed by caller */
-char *slurm_get_slurmd_spooldir(void);
+char *slurm_get_slurmd_spooldir(char *node_name);
/* slurm_get_layouts
* RET comma seperated list of layouts in a string, must be xfreed by caller
diff --git a/src/plugins/mpi/pmi2/setup.c b/src/plugins/mpi/pmi2/setup.c
index ab9f6f5b3ac..56dfaf6fbee 100644
--- a/src/plugins/mpi/pmi2/setup.c
+++ b/src/plugins/mpi/pmi2/setup.c
@@ -276,7 +276,10 @@ _setup_stepd_sockets(const stepd_step_rec_t *job, char ***env)
}
sa.sun_family = PF_UNIX;
- spool = slurm_get_slurmd_spooldir();
+ /* FIXME: We need to handle %n and %h in the spool dir, we have
+ * job->node_name, but the other place we call get_slurmd_spooldir
+ * we don't so we aren't using it here */
+ spool = slurm_get_slurmd_spooldir(NULL);
snprintf(sa.sun_path, sizeof(sa.sun_path), PMI2_SOCK_ADDR_FMT,
spool, job->jobid, job->stepid);
unlink(sa.sun_path); /* remove possible old socket */
@@ -632,7 +635,9 @@ _setup_srun_tree_info(const mpi_plugin_client_info_t *job)
} else
tree_info.srun_addr = NULL;
- spool = slurm_get_slurmd_spooldir();
+ /* FIXME: We need to handle %n and %h in the spool dir, but don't have
+ * the node name here */
+ spool = slurm_get_slurmd_spooldir(NULL);
snprintf(tree_sock_addr, 128, PMI2_SOCK_ADDR_FMT,
spool, job->jobid, job->stepid);
xfree(spool);
diff --git a/src/plugins/mpi/pmix/pmixp_info.c b/src/plugins/mpi/pmix/pmixp_info.c
index ed81578ef99..a23a2b183cc 100644
--- a/src/plugins/mpi/pmix/pmixp_info.c
+++ b/src/plugins/mpi/pmix/pmixp_info.c
@@ -286,26 +286,27 @@ err_exit:
static int _env_set(char ***env)
{
char *p = NULL;
- char *spool = slurm_get_slurmd_spooldir();
+
+ xassert(_pmixp_job_info.hostname);
+
+ _pmixp_job_info.lib_tmpdir = slurm_get_slurmd_spooldir(
+ _pmixp_job_info.hostname);
/* ----------- Temp directories settings ------------- */
- _pmixp_job_info.lib_tmpdir = xstrdup_printf("%s/pmix.%d.%d/", spool,
- pmixp_info_jobid(), pmixp_info_stepid());
- xfree(spool);
+ xstrfmtcat(_pmixp_job_info.lib_tmpdir, "/pmix.%d.%d/",
+ pmixp_info_jobid(), pmixp_info_stepid());
/* save client temp directory if requested
* TODO: We want to get TmpFS value as well if exists.
* Need to sync with SLURM developers.
*/
p = getenvp(*env, PMIXP_TMPDIR_CLI);
- if (NULL != p) {
+
+ if (p)
_pmixp_job_info.cli_tmpdir_base = xstrdup(p);
- } else {
- p = slurm_get_tmp_fs();
- if (NULL != p) {
- _pmixp_job_info.cli_tmpdir_base = p;
- }
- }
+ else
+ _pmixp_job_info.cli_tmpdir_base = slurm_get_tmp_fs();
+
_pmixp_job_info.cli_tmpdir =
xstrdup_printf("%s/spmix_appdir_%d.%d",
_pmixp_job_info.cli_tmpdir_base,
diff --git a/src/plugins/mpi/pmix/pmixp_info.h b/src/plugins/mpi/pmix/pmixp_info.h
index 5fe7620adda..280a0ec11c9 100644
--- a/src/plugins/mpi/pmix/pmixp_info.h
+++ b/src/plugins/mpi/pmix/pmixp_info.h
@@ -272,12 +272,11 @@ static inline char *pmixp_info_job_host(int nodeid)
/* namespaces list operations */
static inline char *pmixp_info_nspace_usock(const char *nspace)
{
- char *spool, *usock = NULL;
+ char *spool;
debug("mpi/pmix: setup sockets");
- spool = slurm_get_slurmd_spooldir();
- xstrfmtcat(usock, "%s/stepd.%s", spool, nspace);
- xfree(spool);
- return usock;
+ spool = slurm_get_slurmd_spooldir(_pmixp_job_info.hostname);
+ xstrfmtcat(spool, "/stepd.%s", nspace);
+ return spool;
}
#endif /* PMIXP_INFO_H */
--
GitLab