diff --git a/doc/html/mpi_guide.shtml b/doc/html/mpi_guide.shtml
index ef3632f4c9fd72ba799b505234ab76b7424d3557..ac15ac4509f87086b21947cec13678ac0bc41d58 100644
--- a/doc/html/mpi_guide.shtml
+++ b/doc/html/mpi_guide.shtml
@@ -68,15 +68,17 @@ options '--mpi=pmi2' has to be specified on the srun command line.
Open MPI application using PMIx the '--mpi=pmix' option has to be
specified on the srun command line.
-<p>A set of environment variables are available to control the behavior of SLURM
+<p>A set of environment variables are available to control the behavior of Slurm
PMIx plugin:
<ul>
<li><i>SLURM_PMIX_SRV_TMPDIR</i> base directory for PMIx/server service files.
-<li><i>SLURM_PMIX_TMPDIR</i> base directory for applications session directories.
-<li><i>SLURM_PMIX_DIRECT_CONN</i> (default - yes) enables (1/yes/true) or disables (0/no/false)
-controls wheter direct connections between slurmstepd's are astablished or SLURM RPCs are used
-for data exchange. Direct connection shows better performanse for fully-packed nodes when PMIx
-is running in the direct-modex mode.
+<li><i>SLURM_PMIX_TMPDIR</i> base directory for applications session
+directories.
+<li><i>SLURM_PMIX_DIRECT_CONN</i> (default - yes) enables (1/yes/true) or
+disables (0/no/false) controls wheter direct connections between slurmstepd's
+are astablished or SLURM RPCs are used for data exchange. Direct connection
+shows better performanse for fully-packed nodes when PMIx is running in the
+direct-modex mode.
</ul>
<p>For older versions of OMPI not compiled with the pmi support
diff --git a/src/plugins/mpi/pmix/Makefile.am b/src/plugins/mpi/pmix/Makefile.am
index 3e9c277f39ce145bb5dcec647f9e8843ce7b5244..6b38414e1da0e040b02a890589030d4617ad95ea 100644
--- a/src/plugins/mpi/pmix/Makefile.am
+++ b/src/plugins/mpi/pmix/Makefile.am
@@ -4,24 +4,24 @@ AUTOMAKE_OPTIONS = foreign
PLUGIN_FLAGS = -module -avoid-version --export-dynamic
-AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/src/common $(HWLOC_CPPFLAGS) $(UCX_CPPFLAGS)
-
-
+AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/src/common $(HWLOC_CPPFLAGS) \
+ $(UCX_CPPFLAGS)
pmix_src = mpi_pmix.c \
- pmixp_common.h \
- pmixp_agent.c pmixp_client.c pmixp_coll.c pmixp_nspaces.c pmixp_info.c \
- pmixp_agent.h pmixp_client.h pmixp_coll.h pmixp_nspaces.h pmixp_info.h \
- pmixp_server.c pmixp_state.c pmixp_io.c pmixp_utils.c pmixp_dmdx.c \
- pmixp_server.h pmixp_state.h pmixp_io.h pmixp_utils.h pmixp_dmdx.h \
- pmixp_conn.c pmixp_dconn.c pmixp_dconn_tcp.c \
- pmixp_conn.h pmixp_dconn.h pmixp_dconn_tcp.h
+ pmixp_common.h \
+ pmixp_agent.c pmixp_client.c pmixp_coll.c pmixp_nspaces.c pmixp_info.c \
+ pmixp_agent.h pmixp_client.h pmixp_coll.h pmixp_nspaces.h pmixp_info.h \
+ pmixp_server.c pmixp_state.c pmixp_io.c pmixp_utils.c pmixp_dmdx.c \
+ pmixp_server.h pmixp_state.h pmixp_io.h pmixp_utils.h pmixp_dmdx.h \
+ pmixp_conn.c pmixp_dconn.c pmixp_dconn_tcp.c \
+ pmixp_conn.h pmixp_dconn.h pmixp_dconn_tcp.h
if HAVE_UCX
pmix_src += pmixp_dconn_ucx.c pmixp_dconn_ucx.h
endif
-pmix_internal_libs = $(top_builddir)/src/slurmd/common/libslurmd_reverse_tree_math.la
+pmix_internal_libs = \
+ $(top_builddir)/src/slurmd/common/libslurmd_reverse_tree_math.la
pmix_ldflags = $(SO_LDFLAGS) $(PLUGIN_FLAGS) $(HWLOC_LDFLAGS) $(UCX_LDFLAGS)
pmix_libadd = $(pmix_internal_libs) $(PMIX_LIBS) $(HWLOC_LIBS) $(UCX_LIBS)
diff --git a/src/plugins/mpi/pmix/mpi_pmix.c b/src/plugins/mpi/pmix/mpi_pmix.c
index ca4bd7dcdad6313c625ab738b3956dfc0a557b85..1ce740cec7f977b95060cfdbf398db438e5345af 100644
--- a/src/plugins/mpi/pmix/mpi_pmix.c
+++ b/src/plugins/mpi/pmix/mpi_pmix.c
@@ -97,9 +97,7 @@ extern int init(void)
* found in the pmix source we are dynamically linking against.
*/
if (HAVE_PMIX_VER != PMIX_VERSION_MAJOR)
- fatal("pmix_init: Slurm was compiled against PMIx v%d but "
- "we are now linking against v%ld."
- " Please check your install.",
+ fatal("pmix_init: Slurm was compiled against PMIx v%d but we are now linking against v%ld. Please check your install.",
HAVE_PMIX_VER, PMIX_VERSION_MAJOR);
return SLURM_SUCCESS;
diff --git a/src/plugins/mpi/pmix/pmixp_agent.c b/src/plugins/mpi/pmix/pmixp_agent.c
index 96cbb0eb4a5ed256953940077eac071214bb3f61..3a7bc99c93ae0487c5da8cea56036b1d39f3958a 100644
--- a/src/plugins/mpi/pmix/pmixp_agent.c
+++ b/src/plugins/mpi/pmix/pmixp_agent.c
@@ -361,7 +361,7 @@ int pmixp_agent_start(void)
if (pmixp_server_want_pp()) {
pmixp_server_run_pp();
}
-
+
PMIXP_DEBUG("agent thread started: tid = %lu",
(unsigned long) _agent_tid);
diff --git a/src/plugins/mpi/pmix/pmixp_client.c b/src/plugins/mpi/pmix/pmixp_client.c
index 01e689486c9f4516ec782a942dab121956a3048d..aeb2e6f9f61ed13092f3b5d9f13c34a4dc531fce 100644
--- a/src/plugins/mpi/pmix/pmixp_client.c
+++ b/src/plugins/mpi/pmix/pmixp_client.c
@@ -56,10 +56,7 @@
#if (HAVE_PMIX_VER != PMIX_VERSION_MAJOR)
#define VALUE_TO_STRING(x) #x
#define VALUE(x) VALUE_TO_STRING(x)
-#pragma message "PMIx version mismatch: the major version seen during " \
- "configuration was " VALUE(HAVE_PMIX_VER) "L but found " \
- VALUE(PMIX_VERSION_MAJOR) " compilation will most likely fail." \
- " Please reconfigure against the new version."
+#pragma message "PMIx version mismatch: the major version seen during configuration was " VALUE(HAVE_PMIX_VER) "L but found " VALUE(PMIX_VERSION_MAJOR) " compilation will most likely fail. Please reconfigure against the new version."
#endif
// define some additional keys
@@ -735,8 +732,7 @@ int pmixp_libpmix_job_set(void)
_release_cb,
®ister_caddy[i + 1]);
if (PMIX_SUCCESS != rc) {
- PMIXP_ERROR("Cannot register client %d(%d) in "
- "namespace %s",
+ PMIXP_ERROR("Cannot register client %d(%d) in namespace %s",
pmixp_info_taskid(i), i,
pmixp_info_namespace());
return SLURM_ERROR;
@@ -750,7 +746,7 @@ int pmixp_libpmix_job_set(void)
ts.tv_sec = 0;
ts.tv_nsec = 100;
- for (i=0; i < pmixp_info_tasks_loc() + 1; i++) {
+ for (i=0; i < pmixp_info_tasks_loc() + 1; i++) {
if (register_caddy[i].active) {
exit_flag = 0;
}
diff --git a/src/plugins/mpi/pmix/pmixp_coll.c b/src/plugins/mpi/pmix/pmixp_coll.c
index a000299fa1ab0e03937d044edfca20a1a544d0df..e402fb12bc5f0b6336921e00ba7a58fa47ff4c7d 100644
--- a/src/plugins/mpi/pmix/pmixp_coll.c
+++ b/src/plugins/mpi/pmix/pmixp_coll.c
@@ -193,8 +193,7 @@ int pmixp_coll_unpack_ranges(Buf buf, pmixp_coll_type_t *type,
rc = unpack32(&tmp, buf);
procs[i].rank = tmp;
if (SLURM_SUCCESS != rc) {
- PMIXP_ERROR("Cannot unpack ranks for process #%d,"
- " nsp=%s",
+ PMIXP_ERROR("Cannot unpack ranks for process #%d, nsp=%s",
i, procs[i].nspace);
return rc;
}
@@ -219,8 +218,7 @@ int pmixp_coll_belong_chk(pmixp_coll_type_t type,
}
}
/* we don't participate in this collective! */
- PMIXP_ERROR("Have collective that doesn't include this job's "
- "namespace");
+ PMIXP_ERROR("Have collective that doesn't include this job's namespace");
return -1;
}
@@ -354,8 +352,7 @@ int pmixp_coll_contrib_local(pmixp_coll_t *coll, char *data, size_t size)
/* change the collective state if need */
if (PMIXP_COLL_SYNC == coll->state) {
- PMIXP_DEBUG("%s:%d: get local contribution: "
- "switch to PMIXP_COLL_FAN_IN",
+ PMIXP_DEBUG("%s:%d: get local contribution: switch to PMIXP_COLL_FAN_IN",
pmixp_info_namespace(), pmixp_info_nodeid());
coll->state = PMIXP_COLL_FAN_IN;
coll->ts = time(NULL);
@@ -395,8 +392,7 @@ int pmixp_coll_contrib_node(pmixp_coll_t *coll, uint32_t glob_nodeid, Buf buf)
xassert(0 <= nodeid);
if (0 > nodeid) {
/* protect ourselfs if we are running with no asserts */
- PMIXP_ERROR("Contribution from the node not participating in "
- "this collective");
+ PMIXP_ERROR("Contribution from the node not participating in this collective");
goto proceed;
}
PMIXP_DEBUG("%s:%d: get contribution from node %s",
@@ -409,15 +405,13 @@ int pmixp_coll_contrib_node(pmixp_coll_t *coll, uint32_t glob_nodeid, Buf buf)
/* fix the collective status if need */
if (PMIXP_COLL_SYNC == coll->state) {
- PMIXP_DEBUG("%s:%d: get contribution from node %s: switch"
- " to PMIXP_COLL_FAN_IN",
+ PMIXP_DEBUG("%s:%d: get contribution from node %s: switch to PMIXP_COLL_FAN_IN",
pmixp_info_namespace(), pmixp_info_nodeid(),
nodename);
coll->state = PMIXP_COLL_FAN_IN;
coll->ts = time(NULL);
- } else if( PMIXP_COLL_FAN_OUT == coll->state) {
- PMIXP_DEBUG("%s:%d: get contribution from node %s: switch"
- " to PMIXP_COLL_FAN_OUT_IN (next collective!)",
+ } else if (PMIXP_COLL_FAN_OUT == coll->state) {
+ PMIXP_DEBUG("%s:%d: get contribution from node %s: switch to PMIXP_COLL_FAN_OUT_IN (next collective!)",
pmixp_info_namespace(), pmixp_info_nodeid(),
nodename);
coll->state = PMIXP_COLL_FAN_OUT_IN;
@@ -431,8 +425,7 @@ int pmixp_coll_contrib_node(pmixp_coll_t *coll, uint32_t glob_nodeid, Buf buf)
* by checking our records. */
if (0 < coll->ch_contribs[nodeid]) {
/* May be 0 or 1. If grater - transmission skew, ignore. */
- PMIXP_DEBUG("Multiple contributions from child_id=%d,"
- " hostname=%s",
+ PMIXP_DEBUG("Multiple contributions from child_id=%d, hostname=%s",
nodeid, nodename);
/* this is duplication, skip. */
goto proceed;
@@ -453,12 +446,12 @@ int pmixp_coll_contrib_node(pmixp_coll_t *coll, uint32_t glob_nodeid, Buf buf)
proceed:
- if( PMIXP_COLL_FAN_IN == coll->state ){
+ if (PMIXP_COLL_FAN_IN == coll->state) {
/* make a progress if we are in fan-in state */
_progress_fan_in(coll);
}
- switch( coll->state ){
+ switch (coll->state) {
case PMIXP_COLL_SYNC:
state = "sync";
break;
@@ -527,13 +520,13 @@ static void _sent_complete_cb(int rc, pmixp_p2p_ctx_t ctx, void *cb_data)
{
pmixp_coll_t *coll = (pmixp_coll_t *)cb_data;
- if( PMIXP_P2P_REGULAR == ctx ){
+ if (PMIXP_P2P_REGULAR == ctx) {
/* lock the collective */
slurm_mutex_lock(&coll->lock);
}
/* We don't want to release buffer */
- if( SLURM_SUCCESS == rc ){
+ if (SLURM_SUCCESS == rc) {
_fan_in_finished(coll);
/* if we are root - push data to PMIx here.
@@ -554,7 +547,7 @@ static void _sent_complete_cb(int rc, pmixp_p2p_ctx_t ctx, void *cb_data)
coll->cbfunc(PMIX_ERROR, NULL, 0, coll->cbdata, NULL, NULL);
}
- if( PMIXP_P2P_REGULAR == ctx ){
+ if (PMIXP_P2P_REGULAR == ctx) {
/* unlock the collective */
slurm_mutex_unlock(&coll->lock);
}
@@ -606,8 +599,7 @@ static void _progress_fan_in(pmixp_coll_t *coll)
rc = _copy_payload(coll->buf, coll->serv_offs,
&coll->root_buf);
xassert(0 == rc);
- PMIXP_DEBUG("%s:%d: finish with this collective "
- "(I am the root)",
+ PMIXP_DEBUG("%s:%d: finish with this collective (I am the root)",
pmixp_info_namespace(), pmixp_info_nodeid());
}
@@ -620,8 +612,7 @@ static void _progress_fan_in(pmixp_coll_t *coll)
_sent_complete_cb, coll);
if (SLURM_SUCCESS != rc) {
- PMIXP_ERROR("Cannot send data (size = %lu), "
- "to hostlist:\n%s",
+ PMIXP_ERROR("Cannot send data (size = %lu), to hostlist:\n%s",
(uint64_t) get_buf_offset(coll->buf),
hostlist);
/* Notify local PMIx server about this failure
@@ -633,7 +624,7 @@ static void _progress_fan_in(pmixp_coll_t *coll)
}
/* release the endpoint */
- switch( ep.type ){
+ switch (ep.type) {
case PMIXP_EP_HLIST:
xfree(ep.ep.hostlist);
break;
diff --git a/src/plugins/mpi/pmix/pmixp_conn.c b/src/plugins/mpi/pmix/pmixp_conn.c
index 3b8fffd84fea6854d8f630b4b1e806a3558f590f..20188410a5c7ae7793563cf172e91dc2fef4b6aa 100644
--- a/src/plugins/mpi/pmix/pmixp_conn.c
+++ b/src/plugins/mpi/pmix/pmixp_conn.c
@@ -63,7 +63,6 @@ void pmixp_conn_fini(void)
list_destroy(_conn_list);
list_destroy(_empty_hndl_list);
_tmp_engines_fini();
-
}
static void _msg_handler_destruct(void *obj)
@@ -105,11 +104,10 @@ void pmixp_conn_cleanup(void)
}
}
-pmixp_conn_t *
-pmixp_conn_new_temp(pmixp_conn_proto_t proto, int fd,
- pmixp_conn_new_msg_cb_t nmsg_cb)
+pmixp_conn_t *pmixp_conn_new_temp(pmixp_conn_proto_t proto, int fd,
+ pmixp_conn_new_msg_cb_t nmsg_cb)
{
- xassert( proto == PMIXP_PROTO_SLURM || proto == PMIXP_PROTO_DIRECT);
+ xassert(proto == PMIXP_PROTO_SLURM || proto == PMIXP_PROTO_DIRECT);
pmixp_conn_t *conn = list_pop(_empty_hndl_list);
if (NULL == conn) {
@@ -143,13 +141,13 @@ pmixp_conn_new_temp(pmixp_conn_proto_t proto, int fd,
return conn;
}
-pmixp_conn_t *
-pmixp_conn_new_persist(pmixp_conn_proto_t proto,
- pmixp_io_engine_t *eng, pmixp_conn_new_msg_cb_t nmsg_cb,
- pmixp_conn_ret_cb_t ret_cb, void *ret_data)
+pmixp_conn_t *pmixp_conn_new_persist(pmixp_conn_proto_t proto,
+ pmixp_io_engine_t *eng,
+ pmixp_conn_new_msg_cb_t nmsg_cb,
+ pmixp_conn_ret_cb_t ret_cb, void *ret_data)
{
- xassert( proto == PMIXP_PROTO_SLURM || proto == PMIXP_PROTO_DIRECT);
- xassert( NULL != eng );
+ xassert(proto == PMIXP_PROTO_SLURM || proto == PMIXP_PROTO_DIRECT);
+ xassert(NULL != eng);
pmixp_conn_t *conn = list_pop(_empty_hndl_list);
if (NULL == conn) {
@@ -170,8 +168,7 @@ pmixp_conn_new_persist(pmixp_conn_proto_t proto,
return conn;
}
-void
-pmixp_conn_return(pmixp_conn_t *conn)
+void pmixp_conn_return(pmixp_conn_t *conn)
{
/* if this is a temp connection - return I/O engine */
if (NULL != conn->ret_cb) {
@@ -224,15 +221,13 @@ pmixp_conn_return(pmixp_conn_t *conn)
static void _temp_engine_destruct(void *obj);
static List _slurm_engines, _direct_engines;
-static void
-_tmp_engines_init()
+static void _tmp_engines_init()
{
_slurm_engines = list_create(_temp_engine_destruct);
_direct_engines = list_create(_temp_engine_destruct);
}
-static void
-_tmp_engines_fini()
+static void _tmp_engines_fini()
{
list_destroy(_slurm_engines);
list_destroy(_direct_engines);
@@ -245,8 +240,7 @@ static void _temp_engine_destruct(void *obj)
xfree(eng);
}
-static inline pmixp_io_engine_t*
-_tmp_engines_get_slurm()
+static inline pmixp_io_engine_t *_tmp_engines_get_slurm()
{
pmixp_io_engine_t *eng = list_pop(_slurm_engines);
if (NULL == eng){
@@ -256,15 +250,13 @@ _tmp_engines_get_slurm()
return eng;
}
-static inline void
-_tmp_engines_return_slurm(pmixp_io_engine_t *eng)
+static inline void _tmp_engines_return_slurm(pmixp_io_engine_t *eng)
{
xassert(NULL != eng);
list_push(_slurm_engines, eng);
}
-static inline pmixp_io_engine_t*
-_tmp_engines_get_direct()
+static inline pmixp_io_engine_t *_tmp_engines_get_direct()
{
pmixp_io_engine_t *eng = list_pop(_direct_engines);
if (NULL == eng){
@@ -274,8 +266,7 @@ _tmp_engines_get_direct()
return eng;
}
-static inline void
-_tmp_engines_return_direct(pmixp_io_engine_t *eng)
+static inline void _tmp_engines_return_direct(pmixp_io_engine_t *eng)
{
xassert(NULL != eng);
list_push(_direct_engines, eng);
diff --git a/src/plugins/mpi/pmix/pmixp_conn.h b/src/plugins/mpi/pmix/pmixp_conn.h
index 5fe9d6b73cc1d478cb18da10c0dadf469663f97d..9fa6c298f868ef5365a7f9e7579b4fbef1174273 100644
--- a/src/plugins/mpi/pmix/pmixp_conn.h
+++ b/src/plugins/mpi/pmix/pmixp_conn.h
@@ -81,22 +81,21 @@ void pmixp_conn_init(pmixp_p2p_data_t slurm_hdr,
void pmixp_conn_fini(void);
void pmixp_conn_cleanup(void);
-pmixp_conn_t *
-pmixp_conn_new_temp(pmixp_conn_proto_t proto, int fd,
- pmixp_conn_new_msg_cb_t msg_cb);
-pmixp_conn_t *
-pmixp_conn_new_persist(pmixp_conn_proto_t proto,
- pmixp_io_engine_t *eng, pmixp_conn_new_msg_cb_t msg_cb,
- pmixp_conn_ret_cb_t ret_cb, void *conn_data);
+pmixp_conn_t *pmixp_conn_new_temp(pmixp_conn_proto_t proto, int fd,
+ pmixp_conn_new_msg_cb_t msg_cb);
+pmixp_conn_t *pmixp_conn_new_persist(pmixp_conn_proto_t proto,
+ pmixp_io_engine_t *eng,
+ pmixp_conn_new_msg_cb_t msg_cb,
+ pmixp_conn_ret_cb_t ret_cb,
+ void *conn_data);
void pmixp_conn_return(pmixp_conn_t *hndl);
-static inline bool
-pmixp_conn_is_alive(pmixp_conn_t *conn)
+
+static inline bool pmixp_conn_is_alive(pmixp_conn_t *conn)
{
return pmixp_io_operating(conn->eng);
}
-static inline bool
-pmixp_conn_progress_rcv(pmixp_conn_t *conn)
+static inline bool pmixp_conn_progress_rcv(pmixp_conn_t *conn)
{
bool ret = false;
if (NULL == conn->hdr) {
@@ -114,23 +113,19 @@ pmixp_conn_progress_rcv(pmixp_conn_t *conn)
return ret;
}
-static inline void
-pmixp_conn_progress_snd(pmixp_conn_t *conn)
+static inline void pmixp_conn_progress_snd(pmixp_conn_t *conn)
{
pmixp_io_send_progress(conn->eng);
}
-static inline pmixp_io_engine_t *
-pmixp_conn_get_eng(pmixp_conn_t *conn)
+static inline pmixp_io_engine_t *pmixp_conn_get_eng(pmixp_conn_t *conn)
{
return conn->eng;
}
-static inline void *
-pmixp_conn_get_data(pmixp_conn_t *conn)
+static inline void *pmixp_conn_get_data(pmixp_conn_t *conn)
{
return conn->ret_data;
}
-
#endif // PMIXP_DIRECT_CONN_H
diff --git a/src/plugins/mpi/pmix/pmixp_dconn.h b/src/plugins/mpi/pmix/pmixp_dconn.h
index d22620820d174d32ac33ddb83e1d4de61dad368d..624634556e9a7b7dba948dcdd72fdc8a8f70fcb8 100644
--- a/src/plugins/mpi/pmix/pmixp_dconn.h
+++ b/src/plugins/mpi/pmix/pmixp_dconn.h
@@ -108,9 +108,15 @@ extern uint32_t _pmixp_dconn_conn_cnt;
extern pmixp_dconn_t *_pmixp_dconn_conns;
extern pmixp_dconn_handlers_t _pmixp_dconn_h;
-int pmixp_dconn_init(int node_cnt,
- pmixp_p2p_data_t direct_hdr);
+int pmixp_dconn_init(int node_cnt, pmixp_p2p_data_t direct_hdr);
void pmixp_dconn_fini();
+int pmixp_dconn_connect_do(pmixp_dconn_t *dconn, void *ep_data,
+ size_t ep_len, void *init_msg);
+pmixp_dconn_progress_type_t pmixp_dconn_progress_type();
+pmixp_dconn_conn_type_t pmixp_dconn_connect_type();
+int pmixp_dconn_poll_fd();
+size_t pmixp_dconn_ep_len();
+char *pmixp_dconn_ep_data();
#ifndef NDEBUG
static void pmixp_dconn_verify(pmixp_dconn_t *dconn)
@@ -122,36 +128,26 @@ static void pmixp_dconn_verify(pmixp_dconn_t *dconn)
#define pmixp_dconn_verify(dconn)
#endif
-static inline pmixp_dconn_t *
-pmixp_dconn_lock(int nodeid)
+static inline pmixp_dconn_t *pmixp_dconn_lock(int nodeid)
{
xassert(nodeid < _pmixp_dconn_conn_cnt);
slurm_mutex_lock(&_pmixp_dconn_conns[nodeid].lock);
return &_pmixp_dconn_conns[nodeid];
}
-static inline void
-pmixp_dconn_unlock(pmixp_dconn_t *dconn)
+static inline void pmixp_dconn_unlock(pmixp_dconn_t *dconn)
{
pmixp_dconn_verify(dconn);
slurm_mutex_unlock(&dconn->lock);
}
-pmixp_dconn_progress_type_t pmixp_dconn_progress_type();
-pmixp_dconn_conn_type_t pmixp_dconn_connect_type();
-int pmixp_dconn_poll_fd();
-size_t pmixp_dconn_ep_len();
-char *pmixp_dconn_ep_data();
-
-static inline pmixp_dconn_state_t
-pmixp_dconn_state(pmixp_dconn_t *dconn)
+static inline pmixp_dconn_state_t pmixp_dconn_state(pmixp_dconn_t *dconn)
{
pmixp_dconn_verify(dconn);
return dconn->state;
}
-static inline void
-pmixp_dconn_req_sent(pmixp_dconn_t *dconn)
+static inline void pmixp_dconn_req_sent(pmixp_dconn_t *dconn)
{
if (PMIXP_DIRECT_INIT != dconn->state) {
PMIXP_ERROR("State machine violation, when transition "
@@ -163,8 +159,7 @@ pmixp_dconn_req_sent(pmixp_dconn_t *dconn)
dconn->state = PMIXP_DIRECT_EP_SENT;
}
-static inline int
-pmixp_dconn_send(pmixp_dconn_t *dconn, void *msg)
+static inline int pmixp_dconn_send(pmixp_dconn_t *dconn, void *msg)
{
return _pmixp_dconn_h.send(dconn->priv, msg);
}
@@ -174,13 +169,10 @@ static inline void pmixp_dconn_regio(eio_handle_t *h)
return _pmixp_dconn_h.regio(h);
}
-int pmixp_dconn_connect_do(pmixp_dconn_t *dconn, void *ep_data,
- size_t ep_len, void *init_msg);
-
/* Returns locked direct connection descriptor */
-static inline bool
-pmixp_dconn_require_connect(pmixp_dconn_t *dconn, bool *send_init)
+static inline bool pmixp_dconn_require_connect(
+ pmixp_dconn_t *dconn, bool *send_init)
{
*send_init = false;
switch( pmixp_dconn_state(dconn) ){
@@ -227,9 +219,8 @@ pmixp_dconn_require_connect(pmixp_dconn_t *dconn, bool *send_init)
return false;
}
-static inline int
-pmixp_dconn_connect(pmixp_dconn_t *dconn, void *ep_data, int ep_len,
- void *init_msg)
+static inline int pmixp_dconn_connect(
+ pmixp_dconn_t *dconn, void *ep_data, int ep_len, void *init_msg)
{
int rc;
/* establish the connection */
@@ -257,8 +248,7 @@ pmixp_dconn_connect(pmixp_dconn_t *dconn, void *ep_data, int ep_len,
}
/* POLL-based specific API */
-static inline pmixp_io_engine_t*
-pmixp_dconn_engine(pmixp_dconn_t *dconn)
+static inline pmixp_io_engine_t *pmixp_dconn_engine(pmixp_dconn_t *dconn)
{
pmixp_dconn_verify(dconn);
xassert( PMIXP_DCONN_PROGRESS_SW == pmixp_dconn_progress_type(dconn));
@@ -269,8 +259,7 @@ pmixp_dconn_engine(pmixp_dconn_t *dconn)
}
/* Returns locked direct connection descriptor */
-static inline pmixp_dconn_t *
-pmixp_dconn_accept(int nodeid, int fd)
+static inline pmixp_dconn_t *pmixp_dconn_accept(int nodeid, int fd)
{
if( PMIXP_DCONN_PROGRESS_SW != pmixp_dconn_progress_type() ){
PMIXP_ERROR("Accept is not supported by direct connection "
@@ -304,8 +293,7 @@ pmixp_dconn_accept(int nodeid, int fd)
}
/* Returns locked direct connection descriptor */
-static inline void
-pmixp_dconn_disconnect(pmixp_dconn_t *dconn)
+static inline void pmixp_dconn_disconnect(pmixp_dconn_t *dconn)
{
switch( pmixp_dconn_state(dconn) ){
case PMIXP_DIRECT_INIT:
diff --git a/src/plugins/mpi/pmix/pmixp_debug.h b/src/plugins/mpi/pmix/pmixp_debug.h
index 345ffaa8889a9b276f4cb4196e87a19f446e8e76..27beac50101f79b0cc45b46fac986f5f7d86b66d 100644
--- a/src/plugins/mpi/pmix/pmixp_debug.h
+++ b/src/plugins/mpi/pmix/pmixp_debug.h
@@ -49,20 +49,20 @@
file_base = file; \
} \
debug("%s [%d] %s:%d [%s] mpi/pmix: " format "", \
- pmixp_info_hostname(), pmixp_info_nodeid(), \
- file_base, __LINE__, __func__, ## args); \
+ pmixp_info_hostname(), pmixp_info_nodeid(), \
+ file_base, __LINE__, __func__, ## args); \
}
#define PMIXP_ERROR_STD(format, args...) { \
char file[] = __FILE__; \
char *file_base = strrchr(file, '/'); \
if (file_base == NULL) { \
- file_base = file; \
+ file_base = file; \
} \
- error("%s [%d] %s:%d [%s] mpi/pmix: ERROR: " format ": %s (%d)", \
- pmixp_info_hostname(), pmixp_info_nodeid(), \
- file_base, __LINE__, __func__, \
- ## args, strerror(errno), errno); \
+ error("%s [%d] %s:%d [%s] mpi/pmix: ERROR: " format ": %s (%d)", \
+ pmixp_info_hostname(), pmixp_info_nodeid(), \
+ file_base, __LINE__, __func__, \
+ ## args, strerror(errno), errno); \
}
#define PMIXP_ERROR(format, args...) { \
@@ -72,29 +72,29 @@
file_base = file; \
} \
error("%s [%d] %s:%d [%s] mpi/pmix: ERROR: " format, \
- pmixp_info_hostname(), pmixp_info_nodeid(), \
- file_base, __LINE__, __func__, ## args); \
+ pmixp_info_hostname(), pmixp_info_nodeid(), \
+ file_base, __LINE__, __func__, ## args); \
}
#define PMIXP_ABORT(format, args...) { \
PMIXP_ERROR(format, ##args); \
error("%s [%d] %s:%d [%s] mpi/pmix: ERROR: " format, \
- pmixp_info_hostname(), pmixp_info_nodeid(), \
- file_base, __LINE__, __func__, ## args); \
- slurm_kill_job_step(pmixp_info_jobid(), \
- pmixp_info_stepid(), SIGKILL); \
+ pmixp_info_hostname(), pmixp_info_nodeid(), \
+ file_base, __LINE__, __func__, ## args); \
+ slurm_kill_job_step(pmixp_info_jobid(), \
+ pmixp_info_stepid(), SIGKILL); \
}
-#define PMIXP_ERROR_NO(err, format, args...) { \
- char file[] = __FILE__; \
- char *file_base = strrchr(file, '/'); \
- if (file_base == NULL) { \
- file_base = file; \
- } \
- error("%s [%d] %s:%d [%s] mpi/pmix: ERROR: " format ": %s (%d)",\
- pmixp_info_hostname(), pmixp_info_nodeid(), \
- file_base, __LINE__, __func__, \
- ## args, strerror(err), err); \
+#define PMIXP_ERROR_NO(err, format, args...) { \
+ char file[] = __FILE__; \
+ char *file_base = strrchr(file, '/'); \
+ if (file_base == NULL) { \
+ file_base = file; \
+ } \
+ error("%s [%d] %s:%d [%s] mpi/pmix: ERROR: " format ": %s (%d)", \
+ pmixp_info_hostname(), pmixp_info_nodeid(), \
+ file_base, __LINE__, __func__, \
+ ## args, strerror(err), err); \
}
#ifdef NDEBUG
diff --git a/src/plugins/mpi/pmix/pmixp_dmdx.c b/src/plugins/mpi/pmix/pmixp_dmdx.c
index 512aa3d5a081d043971dd967519e08adcddc1bff..abfc4cef92f0334d787817f5952977e6d4625622 100644
--- a/src/plugins/mpi/pmix/pmixp_dmdx.c
+++ b/src/plugins/mpi/pmix/pmixp_dmdx.c
@@ -196,7 +196,7 @@ static void _respond_with_error(int seq_num, int nodeid,
pmixp_server_sent_buf_cb, buf);
if (SLURM_SUCCESS != rc) {
char *nodename = pmixp_info_job_host(nodeid);
- PMIXP_ERROR("Cannot send direct modex error" " response to %s",
+ PMIXP_ERROR("Cannot send direct modex error response to %s",
nodename);
xfree(nodename);
}
@@ -293,8 +293,7 @@ static void _dmdx_req(Buf buf, int nodeid, uint32_t seq_num)
char *nodename = pmixp_info_job_host(nodeid);
/* there is not much we can do here, but data corruption
* shouldn't happen */
- PMIXP_ERROR("Fail to unpack header data in" " request from %s,"
- " rc = %d",
+ PMIXP_ERROR("Fail to unpack header data in request from %s, rc = %d",
nodename, rc);
xfree(nodename);
goto exit;
@@ -304,8 +303,7 @@ static void _dmdx_req(Buf buf, int nodeid, uint32_t seq_num)
/* request for namespase that is not controlled by this daemon
* considered as error. This may change in future. */
char *nodename = pmixp_info_job_host(nodeid);
- PMIXP_ERROR("Bad request from %s: asked for" " nspace = %s, "
- "mine is %s",
+ PMIXP_ERROR("Bad request from %s: asked for nspace = %s, mine is %s",
nodename, ns, pmixp_info_namespace());
_respond_with_error(seq_num, nodeid, sender_ns,
PMIX_ERR_INVALID_NAMESPACE);
@@ -316,8 +314,7 @@ static void _dmdx_req(Buf buf, int nodeid, uint32_t seq_num)
nsptr = pmixp_nspaces_local();
if (nsptr->ntasks <= rank) {
char *nodename = pmixp_info_job_host(nodeid);
- PMIXP_ERROR("Bad request from %s: nspace \"%s\"" " has only %d "
- "ranks, asked for %d",
+ PMIXP_ERROR("Bad request from %s: nspace \"%s\" has only %d ranks, asked for %d",
nodename, ns, nsptr->ntasks, rank);
_respond_with_error(seq_num, nodeid, sender_ns,
PMIX_ERR_BAD_PARAM);
@@ -345,9 +342,7 @@ static void _dmdx_req(Buf buf, int nodeid, uint32_t seq_num)
(void *)caddy);
if (PMIX_SUCCESS != rc) {
char *nodename = pmixp_info_job_host(nodeid);
- PMIXP_ERROR("Can't request modex data from libpmix-server,"
- "requesting host = %s, nspace = %s, rank = %d, "
- "rc = %d",
+ PMIXP_ERROR("Can't request modex data from libpmix-server, requesting host = %s, nspace = %s, rank = %d, rc = %d",
nodename, caddy->proc.nspace,
caddy->proc.rank, rc);
_respond_with_error(seq_num, nodeid, caddy->sender_ns, rc);
@@ -384,8 +379,7 @@ static void _dmdx_resp(Buf buf, int nodeid, uint32_t seq_num)
if (NULL == req) {
char *nodename = pmixp_info_job_host(nodeid);
/* We haven't sent this request! */
- PMIXP_ERROR("Received DMDX response with bad " "seq_num=%d "
- "from %s!",
+ PMIXP_ERROR("Received DMDX response with bad seq_num=%d from %s!",
seq_num, nodename);
list_iterator_destroy(it);
rc = SLURM_ERROR;
@@ -463,8 +457,7 @@ void pmixp_dmdx_timeout_cleanup(void)
req->rank);
char *nodename = pmixp_info_job_host(nodeid);
xassert(NULL != nodename);
- PMIXP_ERROR("timeout: ns=%s, rank=%d," " host=%s, "
- "ts=%lu",
+ PMIXP_ERROR("timeout: ns=%s, rank=%d, host=%s, ts=%lu",
req->nspace, req->rank,
(NULL != nodename) ? nodename : "unknown",
ts);
diff --git a/src/plugins/mpi/pmix/pmixp_info.c b/src/plugins/mpi/pmix/pmixp_info.c
index 413abf19017672c08f93fa008baa913105413fe5..abea8b0278e642b074daa7e56bf0fe382292e14a 100644
--- a/src/plugins/mpi/pmix/pmixp_info.c
+++ b/src/plugins/mpi/pmix/pmixp_info.c
@@ -194,7 +194,7 @@ eio_handle_t *pmixp_info_io()
cpus_per_node = getenvp(*env, PMIX_CPUS_PER_NODE_ENV);
if (cpus_per_node == NULL) {
PMIXP_ERROR_NO(0,"Cannot find %s environment variable",
- PMIX_CPUS_PER_NODE_ENV);
+ PMIX_CPUS_PER_NODE_ENV);
return SLURM_ERROR;
}
cpus_per_task_env = getenvp(*env, PMIX_CPUS_PER_TASK);
@@ -263,8 +263,8 @@ static int _resources_set(char ***env)
p = getenvp(*env, PMIXP_JOB_NODES_ENV_DEP);
if (!p) {
/* shouldn't happen if we are under SLURM! */
- PMIXP_ERROR_NO(ENOENT, "Neither of nodelist environment"
- " variables: %s OR %s was found!",
+ PMIXP_ERROR_NO(ENOENT,
+ "Neither of nodelist environment variables: %s OR %s was found!",
PMIXP_JOB_NODES_ENV,
PMIXP_JOB_NODES_ENV_DEP);
goto err_exit;
diff --git a/src/plugins/mpi/pmix/pmixp_io.c b/src/plugins/mpi/pmix/pmixp_io.c
index 76aa2545e7bc25ff02a9aab32e369851a4a215f5..95c0518485c24c01a373c6c35551f8ed30190d9a 100644
--- a/src/plugins/mpi/pmix/pmixp_io.c
+++ b/src/plugins/mpi/pmix/pmixp_io.c
@@ -187,8 +187,7 @@ void pmixp_io_finalize(pmixp_io_engine_t *eng, int err)
{
case PMIXP_IO_FINALIZED:
/* avoid double finalization */
- PMIXP_ERROR("Attempt to finalize already finalized "
- "I/O engine");
+ PMIXP_ERROR("Attempt to finalize already finalized I/O engine");
return;
case PMIXP_IO_OPERATING:
close(eng->sd);
@@ -486,8 +485,7 @@ static bool _send_pending(pmixp_io_engine_t *eng)
void *msg = list_dequeue(eng->send_queue);
xassert(msg != NULL);
if ((rc = _send_set_current(eng, msg))) {
- PMIXP_ERROR_NO(rc, "Cannot switch to the "
- "next message");
+ PMIXP_ERROR_NO(rc, "Cannot switch to the next message");
pmixp_io_finalize(eng, rc);
}
}
@@ -528,7 +526,7 @@ static void _send_progress(pmixp_io_engine_t *eng)
pmixp_io_finalize(eng, shutdown);
break;
}
-
+
eng->send_offs += written;
if (!written) {
break;
@@ -553,7 +551,7 @@ int pmixp_io_send_enqueue(pmixp_io_engine_t *eng, void *msg)
return SLURM_ERROR;
}
list_enqueue(eng->send_queue, msg);
-
+
/* if we don't send anything now - try
* to progress immediately
*/
diff --git a/src/plugins/mpi/pmix/pmixp_server.c b/src/plugins/mpi/pmix/pmixp_server.c
index d9d5b659fa6a01702715c7406fd5fc9ebed53fdf..cbff3ffb70b9cde18d45751bde4a3ad0e427b5d5 100644
--- a/src/plugins/mpi/pmix/pmixp_server.c
+++ b/src/plugins/mpi/pmix/pmixp_server.c
@@ -1098,8 +1098,8 @@ _direct_send(pmixp_dconn_t *dconn, pmixp_ep_t *ep,
msg->hdr = bhdr;
msg->buffer = _buf_finalize(buf, nhdr, hsize, &dsize);
msg->buf_ptr = buf;
-
-
+
+
rc = pmixp_dconn_send(dconn, msg);
if (SLURM_SUCCESS != rc) {
msg->sent_cb(rc, PMIXP_P2P_INLINE, msg->cbdata);
@@ -1456,7 +1456,7 @@ void pmixp_server_run_pp()
PMIXP_ERROR("latency: %d - %.9lf", i, time / iters );
} else {
int count = iters + iters/10;
-
+
slurm_mutex_lock(&_pmixp_pp_lock);
_pmixp_pp_warmup = iters/10;
_pmixp_pp_iters = iters;
diff --git a/src/plugins/mpi/pmix/pmixp_server.h b/src/plugins/mpi/pmix/pmixp_server.h
index e6b8ff1e6507f533599eb29ccb64c3385d71dd32..c067e191ecb21118f82d77e33e04d74f4a9c3fd9 100644
--- a/src/plugins/mpi/pmix/pmixp_server.h
+++ b/src/plugins/mpi/pmix/pmixp_server.h
@@ -84,15 +84,6 @@ int pmixp_server_send_nb(pmixp_ep_t *ep, pmixp_srv_cmd_t type,
void *cb_data);
Buf pmixp_server_buf_new(void);
size_t pmixp_server_buf_reset(Buf buf);
-static inline void
-pmixp_server_buf_reserve(Buf buf, uint32_t size)
-{
- if (remaining_buf(buf) < size) {
- uint32_t to_reserve = size - remaining_buf(buf);
- grow_buf(buf, to_reserve);
- }
-}
-
#ifndef NDEBUG
/* Debug tools used only if debug was enabled */
@@ -114,4 +105,12 @@ bool pmixp_server_pp_check_fini();
#define pmixp_server_init_pp(env)
#endif
+static inline void pmixp_server_buf_reserve(Buf buf, uint32_t size)
+{
+ if (remaining_buf(buf) < size) {
+ uint32_t to_reserve = size - remaining_buf(buf);
+ grow_buf(buf, to_reserve);
+ }
+}
+
#endif /* PMIXP_SERVER_H */
diff --git a/src/plugins/mpi/pmix/pmixp_utils.c b/src/plugins/mpi/pmix/pmixp_utils.c
index 988cb0c03c2ecca45861694b174f0dfc3c208b3a..1a4cc07aca25561f4766ad25492fec973eb77f8d 100644
--- a/src/plugins/mpi/pmix/pmixp_utils.c
+++ b/src/plugins/mpi/pmix/pmixp_utils.c
@@ -226,13 +226,13 @@ static int _iov_shift(struct iovec *iov, size_t iovcnt, int offset)
return iovcnt - skip;
}
-size_t pmixp_writev_buf(int sd, struct iovec *iov, size_t iovcnt,
+size_t pmixp_writev_buf(int sd, struct iovec *iov, size_t iovcnt,
size_t offset, int *shutdown)
{
ssize_t ret;
size_t size = 0, written = 0;
int i;
-
+
for (i=0; i < iovcnt; i++) {
size += iov[i].iov_len;
}
@@ -306,7 +306,7 @@ bool pmixp_fd_write_ready(int fd, int *shutdown)
cur = start;
while ((cur - start) < 0.01) {
rc = poll(pfd, 1, 10);
-
+
/* update current timestamp */
gettimeofday(&tv,NULL);
cur = tv.tv_sec + 1E-6*tv.tv_usec;
diff --git a/src/plugins/mpi/pmix/pmixp_utils.h b/src/plugins/mpi/pmix/pmixp_utils.h
index 31d75726128d292e57d844496391992438712733..639d6ed50084b014f5ee31f26b41e23d33dcbf62 100644
--- a/src/plugins/mpi/pmix/pmixp_utils.h
+++ b/src/plugins/mpi/pmix/pmixp_utils.h
@@ -1,5 +1,5 @@
/*****************************************************************************\
- ** pmix_utils.c - Various PMIx utility functions
+ ** pmix_utils.h - Various PMIx utility functions
*****************************************************************************
* Copyright (C) 2014-2015 Artem Polyakov. All rights reserved.
* Copyright (C) 2015-2017 Mellanox Technologies. All rights reserved.
@@ -47,7 +47,7 @@ size_t pmixp_read_buf(int fd, void *buf, size_t count, int *shutdown,
bool blocking);
size_t pmixp_write_buf(int fd, void *buf, size_t count, int *shutdown,
bool blocking);
-size_t pmixp_writev_buf(int sd, struct iovec *iov, size_t iovcnt,
+size_t pmixp_writev_buf(int sd, struct iovec *iov, size_t iovcnt,
size_t offset, int *shutdown);
int pmixp_fd_set_nodelay(int fd);
@@ -66,6 +66,7 @@ int pmixp_mkdir(char *path, mode_t rights);
/* lightweight pmix list of pointers */
#define PMIXP_LIST_DEBUG 0
+#define PMIXP_LIST_VAL(elem) (elem->data)
typedef struct pmixp_list_elem_s {
#ifndef NDEBUG
@@ -75,45 +76,47 @@ typedef struct pmixp_list_elem_s {
struct pmixp_list_elem_s *next, *prev;
} pmixp_list_elem_t;
-#define PMIXP_LIST_VAL(elem) (elem->data)
+typedef struct pmixp_list_s {
+ pmixp_list_elem_t *head, *tail;
+ size_t count;
+} pmixp_list_t;
+
+/* PMIx list of pointers with element reuse */
+typedef struct pmixp_rlist_s {
+ pmixp_list_t list;
+ pmixp_list_t *src_list;
+ size_t pre_alloc;
+} pmixp_rlist_t;
-static inline pmixp_list_elem_t*
-pmixp_list_elem_new()
+static inline pmixp_list_elem_t *pmixp_list_elem_new()
{
return xmalloc(sizeof(pmixp_list_elem_t));
}
-static inline void
-pmixp_list_elem_free(pmixp_list_elem_t *elem)
+static inline void pmixp_list_elem_free(pmixp_list_elem_t *elem)
{
xfree(elem);
}
-typedef struct pmixp_list_s {
- pmixp_list_elem_t *head, *tail;
- size_t count;
-} pmixp_list_t;
-
-static inline bool
-pmixp_list_empty(pmixp_list_t *l) {
+static inline bool pmixp_list_empty(pmixp_list_t *l)
+{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
#endif
return !(l->count);
}
-static inline size_t
-pmixp_list_count(pmixp_list_t *l) {
+static inline size_t pmixp_list_count(pmixp_list_t *l)
+{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
#endif
return l->count;
}
-static inline void
-pmixp_list_init_pre(pmixp_list_t *l,
- pmixp_list_elem_t *h,
- pmixp_list_elem_t *t)
+static inline void pmixp_list_init_pre(pmixp_list_t *l,
+ pmixp_list_elem_t *h,
+ pmixp_list_elem_t *t)
{
l->head = h;
l->tail = t;
@@ -129,10 +132,9 @@ pmixp_list_init_pre(pmixp_list_t *l,
l->count = 0;
}
-static inline void
-pmixp_list_fini_pre(pmixp_list_t *l,
- pmixp_list_elem_t **h,
- pmixp_list_elem_t **t)
+static inline void pmixp_list_fini_pre(pmixp_list_t *l,
+ pmixp_list_elem_t **h,
+ pmixp_list_elem_t **t)
{
#if PMIXP_LIST_DEBUG
/* list supposed to be empty */
@@ -151,15 +153,12 @@ pmixp_list_fini_pre(pmixp_list_t *l,
}
-static inline void
-pmixp_list_init(pmixp_list_t *l)
+static inline void pmixp_list_init(pmixp_list_t *l)
{
- pmixp_list_init_pre(l, pmixp_list_elem_new(),
- pmixp_list_elem_new());
+ pmixp_list_init_pre(l, pmixp_list_elem_new(), pmixp_list_elem_new());
}
-static inline void
-pmixp_list_fini(pmixp_list_t *l)
+static inline void pmixp_list_fini(pmixp_list_t *l)
{
pmixp_list_elem_t *elem1, *elem2;
pmixp_list_fini_pre(l, &elem1, &elem2);
@@ -167,8 +166,7 @@ pmixp_list_fini(pmixp_list_t *l)
pmixp_list_elem_free(elem2);
}
-static inline void
-pmixp_list_enq(pmixp_list_t *l, pmixp_list_elem_t *elem)
+static inline void pmixp_list_enq(pmixp_list_t *l, pmixp_list_elem_t *elem)
{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
@@ -191,8 +189,7 @@ pmixp_list_enq(pmixp_list_t *l, pmixp_list_elem_t *elem)
l->count++;
}
-static inline pmixp_list_elem_t *
-pmixp_list_deq(pmixp_list_t *l)
+static inline pmixp_list_elem_t *pmixp_list_deq(pmixp_list_t *l)
{
pmixp_list_elem_t *ret;
#if PMIXP_LIST_DEBUG
@@ -219,8 +216,7 @@ pmixp_list_deq(pmixp_list_t *l)
return ret;
}
-static inline void
-pmixp_list_push(pmixp_list_t *l, pmixp_list_elem_t *elem)
+static inline void pmixp_list_push(pmixp_list_t *l, pmixp_list_elem_t *elem)
{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
@@ -243,8 +239,7 @@ pmixp_list_push(pmixp_list_t *l, pmixp_list_elem_t *elem)
l->count++;
}
-static inline pmixp_list_elem_t *
-pmixp_list_pop(pmixp_list_t *l)
+static inline pmixp_list_elem_t *pmixp_list_pop(pmixp_list_t *l)
{
pmixp_list_elem_t *ret = NULL;
@@ -270,8 +265,8 @@ pmixp_list_pop(pmixp_list_t *l)
return ret;
}
-static inline pmixp_list_elem_t *
-pmixp_list_rem(pmixp_list_t *l, pmixp_list_elem_t *elem)
+static inline pmixp_list_elem_t *pmixp_list_rem(
+ pmixp_list_t *l, pmixp_list_elem_t *elem)
{
pmixp_list_elem_t *next;
@@ -295,8 +290,7 @@ pmixp_list_rem(pmixp_list_t *l, pmixp_list_elem_t *elem)
return next;
}
-static inline pmixp_list_elem_t*
-pmixp_list_begin(pmixp_list_t *l)
+static inline pmixp_list_elem_t *pmixp_list_begin(pmixp_list_t *l)
{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
@@ -306,8 +300,7 @@ pmixp_list_begin(pmixp_list_t *l)
return l->head->next;
}
-static inline pmixp_list_elem_t*
-pmixp_list_end(pmixp_list_t *l)
+static inline pmixp_list_elem_t *pmixp_list_end(pmixp_list_t *l)
{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
@@ -317,8 +310,8 @@ pmixp_list_end(pmixp_list_t *l)
return l->tail;
}
-static inline pmixp_list_elem_t *
-pmixp_list_next(pmixp_list_t *l, pmixp_list_elem_t *cur)
+static inline pmixp_list_elem_t *pmixp_list_next(
+ pmixp_list_t *l, pmixp_list_elem_t *cur)
{
#if PMIXP_LIST_DEBUG
xassert(l->head && l->tail);
@@ -330,15 +323,8 @@ pmixp_list_next(pmixp_list_t *l, pmixp_list_elem_t *cur)
return cur->next;
}
-/* PMIx list of pointers with element reuse */
-typedef struct pmixp_rlist_s {
- pmixp_list_t list;
- pmixp_list_t *src_list;
- size_t pre_alloc;
-} pmixp_rlist_t;
-
-static inline pmixp_list_elem_t *
-__pmixp_rlist_get_free(pmixp_list_t *l, size_t pre_alloc)
+static inline pmixp_list_elem_t *__pmixp_rlist_get_free(
+ pmixp_list_t *l, size_t pre_alloc)
{
if (pmixp_list_empty(l)) {
/* add l->pre_alloc elements to the source list */
@@ -350,8 +336,8 @@ __pmixp_rlist_get_free(pmixp_list_t *l, size_t pre_alloc)
return pmixp_list_deq(l);
}
-static inline void
-pmixp_rlist_init(pmixp_rlist_t *l, pmixp_list_t *elem_src, size_t pre_alloc)
+static inline void pmixp_rlist_init(
+ pmixp_rlist_t *l, pmixp_list_t *elem_src, size_t pre_alloc)
{
pmixp_list_elem_t *h, *t;
@@ -364,8 +350,7 @@ pmixp_rlist_init(pmixp_rlist_t *l, pmixp_list_t *elem_src, size_t pre_alloc)
pmixp_list_init_pre(&l->list,h, t);
}
-static inline void
-pmixp_rlist_fini(pmixp_rlist_t *l)
+static inline void pmixp_rlist_fini(pmixp_rlist_t *l)
{
pmixp_list_elem_t *h, *t;
@@ -374,20 +359,17 @@ pmixp_rlist_fini(pmixp_rlist_t *l)
pmixp_list_enq(l->src_list, t);
}
-static inline bool
-pmixp_rlist_empty(pmixp_rlist_t *l)
+static inline bool pmixp_rlist_empty(pmixp_rlist_t *l)
{
return pmixp_list_empty(&l->list);
}
-static inline size_t
-pmixp_rlist_count(pmixp_rlist_t *l)
+static inline size_t pmixp_rlist_count(pmixp_rlist_t *l)
{
return pmixp_list_count(&l->list);
}
-static inline void
-pmixp_rlist_enq(pmixp_rlist_t *l, void *ptr)
+static inline void pmixp_rlist_enq(pmixp_rlist_t *l, void *ptr)
{
pmixp_list_elem_t *elem = NULL;
elem = __pmixp_rlist_get_free(l->src_list, l->pre_alloc);
@@ -395,8 +377,7 @@ pmixp_rlist_enq(pmixp_rlist_t *l, void *ptr)
pmixp_list_enq(&l->list, elem);
}
-static inline void *
-pmixp_rlist_deq(pmixp_rlist_t *l)
+static inline void *pmixp_rlist_deq(pmixp_rlist_t *l)
{
pmixp_list_elem_t *elem = NULL;
void *val = NULL;
@@ -410,8 +391,7 @@ pmixp_rlist_deq(pmixp_rlist_t *l)
return val;
}
-static inline void
-pmixp_rlist_push(pmixp_rlist_t *l, void *ptr)
+static inline void pmixp_rlist_push(pmixp_rlist_t *l, void *ptr)
{
pmixp_list_elem_t *elem = NULL;
elem = __pmixp_rlist_get_free(l->src_list, l->pre_alloc);
@@ -419,8 +399,7 @@ pmixp_rlist_push(pmixp_rlist_t *l, void *ptr)
pmixp_list_push(&l->list, elem);
}
-static inline void *
-pmixp_rlist_pop(pmixp_rlist_t *l)
+static inline void *pmixp_rlist_pop(pmixp_rlist_t *l)
{
pmixp_list_elem_t *elem = NULL;
void *val = NULL;
@@ -434,26 +413,24 @@ pmixp_rlist_pop(pmixp_rlist_t *l)
return val;
}
-static inline pmixp_list_elem_t*
-pmixp_rlist_begin(pmixp_rlist_t *l)
+static inline pmixp_list_elem_t *pmixp_rlist_begin(pmixp_rlist_t *l)
{
return pmixp_list_begin(&l->list);
}
-static inline pmixp_list_elem_t*
-pmixp_rlist_end(pmixp_rlist_t *l)
+static inline pmixp_list_elem_t *pmixp_rlist_end(pmixp_rlist_t *l)
{
return pmixp_list_end(&l->list);
}
-static inline pmixp_list_elem_t *
-pmixp_rlist_next(pmixp_rlist_t *l, pmixp_list_elem_t *cur)
+static inline pmixp_list_elem_t *pmixp_rlist_next(
+ pmixp_rlist_t *l, pmixp_list_elem_t *cur)
{
return pmixp_list_next(&l->list, cur);
}
-static inline pmixp_list_elem_t *
-pmixp_rlist_rem(pmixp_rlist_t *l, pmixp_list_elem_t *elem)
+static inline pmixp_list_elem_t *pmixp_rlist_rem(
+ pmixp_rlist_t *l, pmixp_list_elem_t *elem)
{
pmixp_list_elem_t *ret = NULL;
ret = pmixp_list_rem(&l->list, elem);