Switch plugin module added. While many files were modified, these

modifications were relatively minor - mostly changes in function
names or arguments.

NEWS

 This file describes changes in recent versions of SLURM. It primarily
 documents those changes that are of interest to users and admins. 
 
+* Changes in SLURM 0.3.0.0-pre6 (NOT TAGGED YET)
+===============================
+ -- Switch plugin added. Add "SwitchType=switch/elan" to slurm.conf for 
+    systems with Quadrics Elan3 or Elan4 switches.
+
 * Changes in SLURM 0.3.0.0-pre5
 ===============================
  -- Fixes for reported problems:

configure.ac

@@ -160,54 +160,32 @@ AC_SUBST(SLURMD_PORT)
 dnl check for whether to include Elan support
 dnl
 AC_MSG_CHECKING(whether to include Elan support)
-AC_ARG_WITH(elan,
-  AC_HELP_STRING([--with-elan],[compile with Elan support]),
-  [ case "${withval}" in
-      yes) elan=yes ;;
-      no)  elan=no ;;
-      *) AC_MSG_ERROR([bad value ${enableval} for --with-elan]) ;;
-    esac
-  ]
+savedLIBS="$LIBS"
+AC_CHECK_LIB([elanctrl], [elanctrl_open], 
+             [ have_elanctrl=yes
+               have_elan=yes
+               ELAN_LIBS="-lelanctrl"
+               AC_DEFINE(HAVE_LIBELANCTRL, 1, 
+                         [define if you have libelanctrl.]) ], 
+             [ have_elanctrl=no ]
 )
-AC_MSG_RESULT(${elan=no})
-AM_CONDITIONAL(WITH_ELAN, test "x$with_elan" = "xyes")
-HAVE_ELAN=0
-if test "$with_elan" = "yes"; then
-  savedLIBS="$LIBS"
-  AC_CHECK_LIB([elanctrl], [elanctrl_open], 
-               [ have_elanctrl=yes
-                 ELAN_LIBS="-lelanctrl"
-                 AC_DEFINE(HAVE_LIBELANCTRL, 1, 
-                           [define if you have libelanctrl.])
-               ], 
-               [ have_elanctrl=no
-               ]
-  )
-
-  AC_CHECK_LIB([elan3], [elan3_create], 
-               [ have_elan3=yes
-                 ELAN_LIBS="-lelan3"
-                 AC_DEFINE(HAVE_LIBELAN3, 1, 
-                           [define if you have libelan3.])
-               ], 
-               [ have_elan3=no
-               ]
+AC_CHECK_LIB([elan3], [elan3_create], 
+             [ have_elan3=yes
+               have_elan=yes
+               ELAN_LIBS="-lelan3"
+               AC_DEFINE(HAVE_LIBELAN3, 1, 
+                         [define if you have libelan3.]) ], 
+             [ have_elan3=no ]
   )
-
-  if test "$have_elanctrl" = "no" -a "$have_elan3" = "no"; then
-      AC_MSG_ERROR([Unable to find libelan3 or libelanctrl for Elan support!])
-  fi
-
+AM_CONDITIONAL(HAVE_ELAN, test "x$have_elan" = "xyes")
+if test "x$have_elan" = "xyes"; then
+  AC_DEFINE(HAVE_ELAN, 1, [Define to enable Elan support.])
   AC_CHECK_LIB(rmscall, rms_prgcreate, 
-    [],
+    [ELAN_LIBS="$ELAN_LIBS -lrmscall"],
     [AC_MSG_ERROR([unable to find the RMS library needed for Elan support])],
   )
-  HAVE_ELAN=1
-  AC_DEFINE(HAVE_ELAN, 1, [Define to enable Elan support.])
-  ELAN_LIBS="$ELAN_LIBS -lrmscall"
-  LIBS="$savedLIBS" 
 fi
-
+LIBS="$savedLIBS" 
 AC_SUBST(HAVE_ELAN)
 AC_SUBST(ELAN_LIBS)
 
@@ -316,6 +294,7 @@ AC_CONFIG_FILES([Makefile
 		 src/plugins/sched/backfill/Makefile
 		 src/plugins/sched/builtin/Makefile
 		 src/plugins/sched/wiki/Makefile
+		 src/plugins/switch/Makefile
 		 doc/Makefile
 		 doc/man/Makefile
 		 testsuite/Makefile

doc/html/quickstart.html

@@ -248,9 +248,6 @@ The most commonly used arguments to the <i>configure</i> command include:
 value is <i>/usr/local</i>
 <dt>--sysconfdir=<i>DIR</i>
 <dd>Specify location of SLURM configuration file
-<dt>--with-elan
-<dd>Ccompile with support for the Quadrics Elan switch (see
-<a href="http://www.quadrics.com">http://www.quadrics.com</a>)
 <dt>--with-totalview
 <dd>compile with support for the TotalView debugger (see 
 <a href="http://www.etnus.com/">http://www.etnus.com</a>)
@@ -304,6 +301,7 @@ SlurmdPort=7003
 SlurmdSpoolDir=/var/tmp/slurmd.spool
 SlurmdTimeout=300
 StateSaveLocation=/tmp/slurm.state
+SwitchType=switch/elan
 #
 # Node Configurations
 #

doc/man/man5/slurm.conf.5

@@ -267,6 +267,14 @@ The default value is "/tmp".
 If any slurm daemons terminate abnormally, their core files will also be written 
 into this directory.
 .TP
+\fBSwitchType\fR
+Identifies the type of switch or interconnect used for application communications. 
+Acceptable values include
+"switch/none" for switches not requiring special processing for job launch 
+or termination (Myrinet, Ethernet, and InfiniBand),
+"switch/elan" for Quadrics Elan 3 or Elan 4 interconnect.
+The default value is "switch/none".
+.TP
 \fBTmpFS\fR
 Fully qualified pathname of the file system available to user jobs for 
 temporary storage. This parameter is used in establishing a node's \fBTmpDisk\fR space. 
@@ -547,6 +555,8 @@ SlurmdSpoolDir=/usr/local/slurm/slurmd.spool
 .br
 StateSaveLocation=/usr/local/slurm/slurm.state
 .br
+SwitchType=switch/elan
+.br
 TmpFS=/tmp
 .br
 WaitTime=30

etc/slurm.conf.example

@@ -157,6 +157,7 @@
 # SchedulerAuth=42
 # SchedulerPort=7321
 
+
 #
 # o Define the job completion logging mechanism to be used
 #
@@ -166,6 +167,18 @@
 # JobCompType=jobcomp/filetxt
 
 
+#
+# o Define the switch or interconnect in use.
+#
+# "SwitchType"        : the type of switch or interconnect.
+#     "switch/none"   : the default, supports all switches not requiring
+#                       special set-up for job launch including Myrinet, 
+#                       Ethernet, and InfiniBand.
+#     "switch/elan"   : Quadrics Elan 3 or Elan 4 interconnect.
+#
+# SwitchType=switch/none
+
+
 #
 # o Define location where job completion logs are to be written
 #   Interpretation of the parameter is dependent upon the logging 

slurm/slurm.h.in

@@ -81,12 +81,10 @@ BEGIN_C_DECLS
    typedef struct slurm_job_credential * slurm_cred_t;
 #endif
 
-/* Define qsw_jobinfo_t below to avoid including extraneous slurm headers */
-#ifdef	HAVE_ELAN
-#  ifndef __qsw_jobinfo_t_defined
-#    define  __qsw_jobinfo_t_defined
-     typedef struct qsw_jobinfo *qsw_jobinfo_t;	/* opaque data type */
-#  endif
+/* Define switch_jobinfo_t below to avoid including extraneous slurm headers */
+#ifndef __switch_jobinfo_t_defined
+#  define  __switch_jobinfo_t_defined
+   typedef struct switch_jobinfo *switch_jobinfo_t;	/* opaque data type */
 #endif
 
 /*****************************************************************************\
@@ -265,9 +263,7 @@ typedef struct job_step_create_response_msg {
 	uint32_t job_step_id;	/* assigned job step id */
 	char *node_list;	/* list of allocated nodes */
 	slurm_cred_t cred;      /* slurm job credential */
-#ifdef	HAVE_ELAN
-	qsw_jobinfo_t qsw_job;	/* Elan3 switch context, opaque data structure */
-#endif
+	switch_jobinfo_t switch_job;	/* switch context, opaque data structure */
 } job_step_create_response_msg_t;
 
 typedef struct {
@@ -356,9 +352,7 @@ typedef struct resource_allocation_and_run_response_msg {
 
 	uint32_t job_step_id;	/* assigned step id */
 	slurm_cred_t cred;      /* slurm job credential */
-#ifdef HAVE_ELAN
-	qsw_jobinfo_t qsw_job;	/* Elan3 switch context, opaque data type */
-#endif
+	switch_jobinfo_t switch_job;	/* switch context, opaque data type */
 } resource_allocation_and_run_response_msg_t;
 
 typedef struct partition_info_msg {
@@ -409,12 +403,12 @@ typedef struct slurm_ctl_conf {
 	uint32_t slurmd_port;	/* default communications port to slurmd */
 	char *slurmd_spooldir;	/* where slurmd put temporary state info */
 	char *slurmd_pidfile;   /* where to put slurmd pidfile           */
-	char *switch_type;	/* switch or interconnect type */
 	uint16_t slurmd_timeout;/* how long slurmctld waits for slurmd before 
 				 * considering node DOWN */
 	char *slurm_conf;	/* pathname of slurm config file */
 	char *state_save_location;/* pathname of slurmctld state save
 				 * directory */
+	char *switch_type;	/* switch or interconnect type */
 	char *tmp_fs;		/* pathname of temporary file system */
 	uint16_t wait_time;	/* default job --wait time */
 	char *job_credential_private_key;	/* path to private key */

src/api/config_info.c

@@ -129,11 +129,8 @@ void slurm_print_ctl_conf ( FILE* out,
 		slurm_ctl_conf_ptr->slurm_conf);
 	fprintf(out, "StateSaveLocation = %s\n", 
 		slurm_ctl_conf_ptr->state_save_location);
-#if 0
-Not quite ready to check in
 	fprintf(out, "SwitchType        = %s\n",
 		slurm_ctl_conf_ptr->switch_type);
-#endif
 	fprintf(out, "TmpFS             = %s\n", 
 		slurm_ctl_conf_ptr->tmp_fs);
 	fprintf(out, "WaitTime          = %u\n", 

src/common/Makefile.am

@@ -5,12 +5,6 @@ AUTOMAKE_OPTIONS = foreign
 
 INCLUDES     = -I$(top_srcdir) $(SSL_CPPFLAGS) 
 
-if WITH_ELAN
-elan_sources = qsw.c qsw.h elanhosts.c elanhosts.h
-else
-elan_sources = 
-endif
-
 noinst_LTLIBRARIES = 			\
 	libcommon.la 			\
 	libdaemonize.la 		\
@@ -34,6 +28,7 @@ libcommon_la_SOURCES = 			\
 	plugin.c plugin.h		\
 	plugrack.c plugrack.h		\
 	read_config.c read_config.h	\
+	setenvpf.c setenvpf.h		\
 	slurm_cred.h       		\
 	slurm_cred.c			\
 	slurm_errno.c			\
@@ -53,10 +48,10 @@ libcommon_la_SOURCES = 			\
 	util-net.c util-net.h		\
 	slurm_auth.c slurm_auth.h	\
 	slurm_jobcomp.c slurm_jobcomp.h	\
+	switch.c switch.h		\
 	arg_desc.c arg_desc.h		\
 	macros.h			\
-	hostlist.c hostlist.h		\
-	$(elan_sources) 
+	hostlist.c hostlist.h
 
 libdaemonize_la_SOURCES =  		\
 	daemonize.c       	 	\
@@ -67,9 +62,5 @@ libeio_la_SOURCES = 	   		\
 	eio.c eio.h	   		\
 	io_hdr.c io_hdr.h
 
-EXTRA_libcommon_la_SOURCES =    	\
-	qsw.c qsw.h          		\
-	elanhosts.c elanhosts.h
-
-libcommon_la_LIBADD   = $(SSL_LIBS) $(ELAN_LIBS) -ldl
+libcommon_la_LIBADD   = $(SSL_LIBS) -ldl
 libcommon_la_LDFLAGS  = $(SSL_LDFLAGS)	

src/common/elanhosts.c

-/*****************************************************************************\
- *  $Id$
- *****************************************************************************
- *  Copyright (C) 2001-2002 The Regents of the University of California.
- *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- *  Written by Mark Grondona <mgrondona@llnl.gov>.
- *  UCRL-CODE-2003-005.
- *  
- *  This file is part of Pdsh, a parallel remote shell program.
- *  For details, see <http://www.llnl.gov/linux/pdsh/>.
- *  
- *  Pdsh is free software; you can redistribute it and/or modify it under
- *  the terms of the GNU General Public License as published by the Free
- *  Software Foundation; either version 2 of the License, or (at your option)
- *  any later version.
- *  
- *  Pdsh is distributed in the hope that it will be useful, but WITHOUT ANY
- *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
- *  details.
- *  
- *  You should have received a copy of the GNU General Public License along
- *  with Pdsh; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
-\*****************************************************************************/
-
-#if     HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include <assert.h>
-#include <stdarg.h>
-#include <stdlib.h>
-
-#include "src/common/list.h"
-#include "src/common/hostlist.h"
-#include "elanhosts.h"
-
-/* Default ElanId config file */
-#define ELANID_CONFIG_FILE "/etc/elanhosts"
-
-/* 
- * Error strings for error codes returned by parse_elanid_config()
- */
-static char *errstr[] = 
-{ "No error", 
-  "Out of memory!",
-  "Parse error", 
-  "Number of ElanIds specified != number of hosts",
-  "Type must be \"eip\" \"eth\" or \"other\"",
-  NULL
-};
-
-/*
- *  Container for converting hostnames to ElanIDs
- */
-struct elan_info {
-    elanhost_type_t type;     /* type of entry                              */
-    int             elanid;   /* ElanID corresponding to this hostname      */
-    char           *hostname; /* Resolveable hostname                       */
-};
-
-struct elanhost_config {
-#ifndef NDEBUG
-    int magic;
-#   define ELANHOST_CONFIG_MAGIC 0xe100e100
-#endif
-    int maxid;         /* Storage for max ElanID in config                   */
-	List elanid_list;  /* List of elan_info objects describing configuration */
-    char errstr[1024]; /* String describing last error from this object      */
-};
-
-
-/* 
- * Static Prototypes:
- */
-static elanhost_config_t _elanhost_config_alloc(void);
-static void _elanhost_err(elanhost_config_t ec, const char *fmt, ...);
-static int _find_host(struct elan_info *ei, char *key);
-static int _parse_elanid_config(elanhost_config_t ec, const char *path);
-static int _parse_elanid_line(elanhost_config_t ec, char *buf); 
-static struct elan_info * _elan_info_create(elanhost_type_t type, 
-                                            int elanid, char *hostname);
-static void _elan_info_destroy(struct elan_info *ei);
-
-
-elanhost_config_t elanhost_config_create()
-{
-    return _elanhost_config_alloc();
-}
-
-
-int elanhost_config_read(elanhost_config_t ec, const char *filename)
-{
-    assert(ec != NULL);
-    assert(ec->magic == ELANHOST_CONFIG_MAGIC);
-    assert(ec->elanid_list != NULL);
-
-    if (filename == NULL)
-        filename = ELANID_CONFIG_FILE;
-
-    if (_parse_elanid_config(ec, filename) < 0) 
-        return(-1);
-
-    return(0);
-}
-
-void elanhost_config_destroy(elanhost_config_t ec)
-{
-    assert(ec != NULL);
-    assert(ec->magic == ELANHOST_CONFIG_MAGIC);
-    list_destroy(ec->elanid_list);
-    assert(ec->magic = ~ELANHOST_CONFIG_MAGIC);
-    free(ec);
-}
-
-int elanhost_config_maxid(elanhost_config_t ec)
-{
-    assert(ec != NULL);
-    assert(ec->magic == ELANHOST_CONFIG_MAGIC);
-
-    return ec->maxid;
-}
-
-int elanhost_host2elanid(elanhost_config_t ec, char *host)
-{
-    struct elan_info *ei;
-
-    assert(ec != NULL);
-    assert(host != NULL);
-    assert(ec->magic == ELANHOST_CONFIG_MAGIC);
-
-    ei = list_find_first(ec->elanid_list, (ListFindF) _find_host, host);
-
-    if (!ei) {
-        _elanhost_err(ec, "Unable to find host \"%s\" in configuration", host);
-        return -1;
-    }
-
-    return ei->elanid;
-}
-
-const char *elanhost_config_err(elanhost_config_t ec)
-{
-    return ec->errstr;
-}
-
-
-struct elanid_find_arg {
-    elanhost_type_t type;
-    int elanid;
-};
-
-static int _find_elanid(struct elan_info *ei, struct elanid_find_arg *arg)
-{ 
-    if (ei->type != arg->type)
-        return 0;
-
-    if (ei->elanid != arg->elanid)
-        return 0;
-
-    return 1;
-}
-
-char *elanhost_elanid2host(elanhost_config_t ec, elanhost_type_t type, int eid)
-{
-    struct elan_info *ei;
-    struct elanid_find_arg arg;
-
-    assert(ec != NULL);
-    assert(eid >= 0);
-    assert(ec->magic == ELANHOST_CONFIG_MAGIC);
-
-    arg.type = type;
-    arg.elanid = eid;
-
-    ei = list_find_first(ec->elanid_list, (ListFindF) _find_elanid, &arg);
-
-    if (!ei) {
-        _elanhost_err(ec, "Unable to find host with type=%d elanid=%d", 
-                         type, eid);
-        return(NULL);
-    }
-
-    return ei->hostname;
-}
-
-static elanhost_config_t _elanhost_config_alloc(void)
-{
-    elanhost_config_t new = malloc(sizeof(*new));
-
-    new->maxid = -1;
-    new->elanid_list = list_create((ListDelF) _elan_info_destroy);
-
-    assert(new->magic = ELANHOST_CONFIG_MAGIC);
-
-    return new;
-}
-
-static void _elanhost_err(elanhost_config_t ec, const char *fmt, ...)
-{
-    va_list ap;
-
-    assert(ec != NULL);
-    assert(fmt != NULL);
-
-    va_start(ap, fmt);
-    vsnprintf(ec->errstr, 1024, fmt, ap);
-    va_end(ap);
-
-    return;
-}
-
-/*
- * Parse the "elanhosts" config file which has the form
- * 
- *   ElanIds  Hostnames
- *   [n-m]    host_n,...,host_m
- *   [n-m]    host[n-m]
- *   etc.
- *
- * and which maps ElanIds to hostnames on the cluster.
- * The results are stored in the config object's elanid_list member. 
- *
- * Returns 0 on Success, and an error code < 0 on failure.
- */
-static int _parse_elanid_config(elanhost_config_t ec, const char *path)
-{
-	char  buf[4096];
-	int   line;
-	FILE *fp;
-
-	if (!(fp = fopen(path, "r"))) {
-		_elanhost_err(ec, "failed to open %s\n",  path);
-		return -1;
-	}
-
-	line = 1;
-	while (fgets(buf, 4096, fp)) {
-		int rc;
-		if ((rc = _parse_elanid_line(ec, buf)) < 0) {
-			_elanhost_err(ec, "%s: line %d: %s", path, line, errstr[-rc]);
-			return -1;
-		}
-		line++;
-	}
-
-	if (fclose(fp) < 0)
-		_elanhost_err(ec, "close(%s): %m", path);
-
-	return 0;
-}
-
-
-/*
- *  Translate type strings "eip," "eth," or "other" into their
- *   corresponding elanhost_type_t number
- */
-static elanhost_type_t _get_type_num(char *type)
-{
-    if (strcasecmp(type, "eip") == 0)
-        return ELANHOST_EIP;
-    else if (strcasecmp(type, "eth") == 0)
-        return ELANHOST_ETH;
-    else if (strcasecmp(type, "other") == 0)
-        return ELANHOST_OTHER;
-    else
-        return -1;
-}
-
-/*
- *  Parse one line of elanId list appending results to list "eil"
- *
- *  Returns -1 for parse error, -2 if the number of elanids specified
- *  doesn't equal the number of hosts.
- *
- *  Returns 0 on success
- */
-static int 
-_parse_elanid_line(elanhost_config_t ec, char *buf)
-{
-	hostlist_t  el, hl;
-	const char *separators = " \t\n";
-    char       *type;
-	char       *elanids;
-	char       *hosts;
-	char       *sp, *s;
-	int         rc = 0;
-    int         typenum;
-
-	/* 
-	 *  Nullify any comments
-	 */
-	if ((s = strchr(buf, '#')))
-		*s = '\0';
-
-    if (!(type = strtok_r(buf, separators, &sp)))
-        return 0;
-
-	if (!(elanids = strtok_r(NULL, separators, &sp)))
-		return -1;
-
-	if (!(hosts = strtok_r(NULL, separators, &sp)))
-		return -2;
-
-	el = hostlist_create(NULL);
-	hl = hostlist_create(NULL);
-
-	if (!el || !hl) {
-		rc = -1;
-		goto done;
-	}
-
-	if (hostlist_push(el, elanids) != hostlist_push(hl, hosts)) {
-		rc = -3; 
-		goto done;
-	}
-
-    if ((typenum = _get_type_num(type)) < 0)
-        return -4;
-
-	while ((s = hostlist_shift(el))) {
-		char *eptr;
-		int   elanid = (int) strtoul(s, &eptr, 10);
-
-		if (*eptr != '\0') {
-			rc = -2;
-			goto done;
-		}
-
-		free(s);
-		if (!(s = hostlist_shift(hl))) {
-			rc = -1;
-			goto done;
-		}
-
-        if (elanid > ec->maxid)
-            ec->maxid = elanid;
-
-		list_append(ec->elanid_list, _elan_info_create(typenum, elanid, s));
-	}
-
-    done:
-	hostlist_destroy(el);
-	hostlist_destroy(hl);
-
-	return rc;
-}
-
-static struct elan_info *
-_elan_info_create(elanhost_type_t type, int elanid, char *hostname)
-{
-	struct elan_info *ei = (struct elan_info *) malloc(sizeof(*ei));
-    ei->type     = type;
-	ei->elanid   = elanid;
-	ei->hostname = hostname;
-	return ei;
-}
-
-static void
-_elan_info_destroy(struct elan_info *ei)
-{
-	if (ei->hostname)
-		free(ei->hostname);
-	free(ei);
-}
-
-
-/*
- *  List Find function for mapping hostname to an ElanId
- */
-static int _find_host(struct elan_info *ei, char *key)
-{
-    if (strcmp(ei->hostname, key) != 0)
-        return 0;
-    else
-        return 1;
-}
-
-
-/*
- * vi:tabstop=4 shiftwidth=4 expandtab
- */
-

src/common/elanhosts.h

-/*****************************************************************************\
- *  $Id$
- *****************************************************************************
- *  Copyright (C) 2001-2002 The Regents of the University of California.
- *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- *  Written by Mark Grondona <mgrondona@llnl.gov>.
- *  UCRL-CODE-2003-005.
- *  
- *  This file is part of Pdsh, a parallel remote shell program.
- *  For details, see <http://www.llnl.gov/linux/pdsh/>.
- *  
- *  Pdsh is free software; you can redistribute it and/or modify it under
- *  the terms of the GNU General Public License as published by the Free
- *  Software Foundation; either version 2 of the License, or (at your option)
- *  any later version.
- *  
- *  Pdsh is distributed in the hope that it will be useful, but WITHOUT ANY
- *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
- *  details.
- *  
- *  You should have received a copy of the GNU General Public License along
- *  with Pdsh; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
-\*****************************************************************************/
-
-#ifndef _ELANHOSTS_H
-#define _ELANHOSTS_H
-
-/*
- * Type of Elan "hostname" 
- *   Hostname corresponds to the eip adapter, an ethernet adapter, or "other"
- */
-typedef enum {
-	ELANHOST_EIP,
-	ELANHOST_ETH,
-	ELANHOST_OTHER
-} elanhost_type_t;
-
-/*  Opaque type which holds the elanhost configuration
- */
-typedef struct elanhost_config * elanhost_config_t;
-
-
-/*
- *  Functions
- */
-
-/*
- * Create an empty Elanhost config object
- */
-elanhost_config_t elanhost_config_create(void);
-
-/*
- *  Read elanhosts configuration from `file'  
- *    (Default /etc/elanhosts)
- *
- *  Config file format is as follows:
- *
- *    Type  ElanIDs  Hostnames
- *
- *  The "type" field may be "eip" for eip interface,  "eth" for an
- *    ethernet interface, or "other" for anything else. ("eth" and
- *    "other" are equivalent at this time)
- *
- *  The "ElanIDs" field consists of a list of one or more ElanIDs in
- *    the form "[i-j,n-m,..]" or just "N" for a single ElanID.
- *
- *  The "Hostname" field consists of the hostnames which correspond
- *    to the ElanIDs. If the hostnames have a numeric suffix a bracketed
- *    hostlist is allowed (see hostlist.[ch]) 
- *
- *  For Example:
- *
- *    Type  ElanIDs  Hostnames
- *    eip   [0-10]   host[0-10]
- *    eth   [0-10]   ehost[0-10]
- *    eth   [0,1]    host0-eth1,host1-eth1
- *
- *  Returns 0 on succes, -1 for failure.
- *
- */
-int elanhost_config_read(elanhost_config_t ec, const char *filename);
-
-
-/*
- *  Destroy an elanhost configuration object.
- */
-void elanhost_config_destroy(elanhost_config_t conf);
-
-
-/*
- *  Given a hostname, return the corresponding ElanID
- *
- *  Returns the ElanId on success, -1 if no host matching "hostname"
- *    was found in the configuration.
- *
- */
-int elanhost_host2elanid(elanhost_config_t ec, char *host);
-
-
-/*
- *  Given an ElanId and adapter type, return the first matching hostname
- *    from the configuration.
- */
-char *elanhost_elanid2host(elanhost_config_t ec, 
-		           elanhost_type_t type, int elanid);
-
-
-/* 
- *  Returns the max ElanID from the configuration
- */
-int elanhost_config_maxid(elanhost_config_t ec);
-
-
-/*
- *  Returns the last error string generated for the elan config obj `ec'
- */
-const char *elanhost_config_err(elanhost_config_t ec);
-
-#endif

src/common/plugin.c

@@ -60,7 +60,8 @@ plugin_peek( const char *fq_path,
 		}
 	} else {
 		dlclose( plug );
-		error( "%s: not a SLURM plugin", fq_path );
+		/* could be vestigial library, don't treat as an error */
+		verbose( "%s: not a SLURM plugin", fq_path );
 		return SLURM_ERROR;
 	}
 	if ( ( version = (uint32_t *) dlsym( plug, PLUGIN_VERSION ) ) != NULL ) {
@@ -69,7 +70,8 @@ plugin_peek( const char *fq_path,
 		}
 	} else {
 		dlclose( plug );
-		error( "%s: not a SLURM plugin", fq_path );
+		/* could be vestigial library, don't treat as an error */
+		verbose( "%s: not a SLURM plugin", fq_path );
 		return SLURM_ERROR;
 	}
 
@@ -93,7 +95,7 @@ plugin_load_from_file( const char *fq_path )
          */
         plug = dlopen( fq_path, RTLD_NOW );
         if ( plug == NULL ) {
-		debug2( "plugin_load_from_file: dlopen(%s): %s",
+		debug( "plugin_load_from_file: dlopen(%s): %s",
 			fq_path,
 			dlerror() );
                 return PLUGIN_INVALID_HANDLE;
@@ -103,6 +105,7 @@ plugin_load_from_file( const char *fq_path )
         if ( ( dlsym( plug, PLUGIN_NAME ) == NULL ) ||
              ( dlsym( plug, PLUGIN_TYPE ) == NULL ) ||
              ( dlsym( plug, PLUGIN_VERSION ) == NULL ) ) {
+		debug( "plugin_load_from_file: invalid symbol");
                 /* slurm_seterrno( SLURM_PLUGIN_SYMBOLS ); */
                 return PLUGIN_INVALID_HANDLE;
         }
@@ -113,7 +116,8 @@ plugin_load_from_file( const char *fq_path )
          */
         if ( ( init = dlsym( plug, "init" ) ) != NULL ) {
                 if ( (*init)() != 0 ) {
-			debug( "plugin_load_from_file(%s): init() returned SLURM_ERROR", fq_path );
+			debug( "plugin_load_from_file(%s): init() returned SLURM_ERROR", 
+				fq_path );
                         (void) dlclose( plug );
                         return PLUGIN_INVALID_HANDLE;
                 }

src/common/qsw.c

-/*****************************************************************************\
- *  qsw.c - Library routines for initiating jobs on QsNet. 
- *****************************************************************************
- *  Copyright (C) 2002 The Regents of the University of California.
- *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- *  Written by Jim Garlick <garlick@llnl.gov>
- *  UCRL-CODE-2002-040.
- *  
- *  This file is part of SLURM, a resource management program.
- *  For details, see <http://www.llnl.gov/linux/slurm/>.
- *  
- *  SLURM is free software; you can redistribute it and/or modify it under
- *  the terms of the GNU General Public License as published by the Free
- *  Software Foundation; either version 2 of the License, or (at your option)
- *  any later version.
- *  
- *  SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
- *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
- *  details.
- *  
- *  You should have received a copy of the GNU General Public License along
- *  with SLURM; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
-\*****************************************************************************/
-
-#if     HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#ifdef WITH_PTHREADS
-#  include <pthread.h>
-#endif /* WITH_PTHREADS */
-
-#include <sys/param.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <syslog.h>
-#include <errno.h>
-#include <string.h>
-#include <paths.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <limits.h>	/* INT_MAX */
-#include <stdio.h>
-
-
-#if HAVE_LIBELANCTRL
-# include <elan/elanctrl.h>
-# include <elan/capability.h>
-
-/* These are taken from elan3/elanvp.h, which we don't
- *  want to include here since we are using the new
- *  version-nonspecific libelanctrl.
- *  (XXX: What is the equivalent in libelanctrl?)
- */
-# define ELAN_USER_BASE_CONTEXT_NUM    0x020
-# define ELAN_USER_TOP_CONTEXT_NUM     0x7ff
-
-# define Version      cap_version
-# define HighNode     cap_highnode
-# define LowNode      cap_lownode
-# define HighContext  cap_highcontext
-# define LowContext   cap_lowcontext
-# define MyContext    cap_mycontext
-# define Bitmap       cap_bitmap
-# define Type         cap_type
-# define UserKey      cap_userkey
-# define RailMask     cap_railmask
-# define Values       key_values
-#elif HAVE_LIBELAN3 
-# include <elan3/elan3.h>
-# include <elan3/elanvp.h>
-#else
-# error "Must have either libelan3 or libelanctrl to compile this module!"
-#endif /* HAVE_LIBELANCTRL */
-
-#include <rms/rmscall.h>
-
-#include <slurm/slurm_errno.h>
-
-#include "src/common/elanhosts.h"
-#include "src/common/xassert.h"
-#include "src/common/strlcpy.h"
-#include "src/common/bitstring.h"
-#include "src/common/log.h"
-#include "src/common/pack.h"
-#include "src/common/qsw.h"
-
-/*
- * Definitions local to this module.
- */
-#define QSW_JOBINFO_MAGIC 	0xf00ff00e
-#define QSW_LIBSTATE_MAGIC 	0xf00ff00f
-
-/* we will allocate program descriptions in this range */
-/* XXX note: do not start at zero as libelan shifts to get unique shm id */
-#define QSW_PRG_START  		1
-#define QSW_PRG_END    		INT_MAX
-#define QSW_PRG_INVAL		(-1)
-
-/* we allocate elan hardware context numbers in this range */
-#define QSW_CTX_START		ELAN_USER_BASE_CONTEXT_NUM
-
-/* XXX: Temporary workaround for slurm/222 (qws sw-kernel/5478) 
- *      (sys_validate_cap does not allow ELAN_USER_TOP_CONTEXT_NUM)
- */
-#define QSW_CTX_END		ELAN_USER_TOP_CONTEXT_NUM - 1 
-#define QSW_CTX_INVAL		(-1)
-
-/* 
- * We are going to some trouble to keep these defs private so slurm
- * hackers not interested in the interconnect details can just pass around
- * the opaque types.  All use of the data structure internals is local to this
- * module.
- */
-struct qsw_libstate {
-	int ls_magic;
-	int ls_prognum;
-	int ls_hwcontext;
-};
-
-struct qsw_jobinfo {
-	int             j_magic;
-	int             j_prognum;
-	ELAN_CAPABILITY j_cap;
-};
-
-/* Copy library state */
-#define _copy_libstate(dest, src) do { 			\
-	assert((src)->ls_magic == QSW_LIBSTATE_MAGIC); 	\
-	assert((dest)->ls_magic == QSW_LIBSTATE_MAGIC); \
-	memcpy(dest, src, sizeof(struct qsw_libstate));	\
-} while (0)
-
-/* Lock on library state */
-#define _lock_qsw() do {				\
-	int err;					\
-	err = pthread_mutex_lock(&qsw_lock);		\
-	assert(err == 0);				\
-} while (0)
-#define _unlock_qsw() do {				\
-	int err;					\
-	err = pthread_mutex_unlock(&qsw_lock);		\
-	assert(err == 0);				\
-} while (0)
-
-/*
- * Globals
- */
-static qsw_libstate_t qsw_internal_state = NULL;
-static pthread_mutex_t qsw_lock = PTHREAD_MUTEX_INITIALIZER;
-static elanhost_config_t elanconf = NULL;
-
-
-/*
- * Allocate a qsw_libstate_t.
- *   lsp (IN)		store pointer to new instantiation here
- *   RETURN		0 on success, -1 on failure (sets errno)
- */
-int
-qsw_alloc_libstate(qsw_libstate_t *lsp)
-{
-	qsw_libstate_t new;
-
-	assert(lsp != NULL);
-	new = (qsw_libstate_t)malloc(sizeof(struct qsw_libstate));
-	if (!new)
-		slurm_seterrno_ret(ENOMEM);
-	new->ls_magic = QSW_LIBSTATE_MAGIC;
-	*lsp = new;
-	return 0;
-}
-
-/*
- * Free a qsw_libstate_t.
- *   ls (IN)		qsw_libstate_t to free
- */
-void
-qsw_free_libstate(qsw_libstate_t ls)
-{
-	assert(ls->ls_magic == QSW_LIBSTATE_MAGIC);
-	ls->ls_magic = 0;
-	free(ls);
-}
-
-/*
- * Pack libstate structure in a format that can be shipped over the
- * network and unpacked on a different architecture.
- *   ls (IN)		libstate structure to be packed
- *   buffer (IN/OUT)	where to store packed data
- *   RETURN		#bytes unused in 'data'
- */
-int
-qsw_pack_libstate(qsw_libstate_t ls, Buf buffer)
-{
-	int offset;
-
-	assert(ls->ls_magic == QSW_LIBSTATE_MAGIC);
-	offset = get_buf_offset(buffer);
-
-	pack32(ls->ls_magic, buffer);
-	pack32(ls->ls_prognum, buffer);
-	pack32(ls->ls_hwcontext, buffer);
-
-	return (get_buf_offset(buffer) - offset);
-}
-
-/*
- * Unpack libstate packed by qsw_pack_libstate.
- *   ls (IN/OUT)	where to put libstate structure
- *   buffer (IN/OUT)	where to get packed data
- *   RETURN		#bytes unused or -1 on error (sets errno)
- */
-int
-qsw_unpack_libstate(qsw_libstate_t ls, Buf buffer)
-{
-	int offset;
-
-	assert(ls->ls_magic == QSW_LIBSTATE_MAGIC);
-	offset = get_buf_offset(buffer);
-
-	safe_unpack32(&ls->ls_magic, buffer);
-	safe_unpack32(&ls->ls_prognum, buffer);
-	safe_unpack32(&ls->ls_hwcontext, buffer);
-
-	if (ls->ls_magic != QSW_LIBSTATE_MAGIC)
-		goto unpack_error;
-
-	return SLURM_SUCCESS; 
-
-    unpack_error:
-	slurm_seterrno_ret(EBADMAGIC_QSWLIBSTATE); /* corrupted libstate */
-	return SLURM_ERROR;
-}
-
-/*
- * Seed the random number generator.  This can be called multiple times,
- * but srand48 will only be called once per program invocation.
- */
-static void
-_srand_if_needed(void)
-{
-	static int done = 0;
-
-	if (!done) {
-		srand48(getpid());
-		done = 1;
-	}
-}
-
-/*
- * Initialize this library, optionally restoring a previously saved state.
- *   oldstate (IN)	old state retrieved from qsw_fini() or NULL
- *   RETURN		0 on success, -1 on failure (sets errno)
- */
-int
-qsw_init(qsw_libstate_t oldstate)
-{
-	qsw_libstate_t new;
-
-	assert(qsw_internal_state == NULL);
-	_srand_if_needed();
-	if (qsw_alloc_libstate(&new) < 0)
-		return -1; /* errno set by qsw_alloc_libstate */
-	if (oldstate)
-		_copy_libstate(new, oldstate);
-	else {
-		new->ls_prognum = QSW_PRG_START;
-		new->ls_hwcontext = QSW_CTX_START;
-	}
-	qsw_internal_state = new;
-	return 0;
-}
-
-/*
- * Finalize use of this library.  If 'savestate' is non-NULL, final
- * state is copied there before it is destroyed.
- *   savestate (OUT)	place to put state
- */
-void
-qsw_fini(qsw_libstate_t savestate)
-{
-	assert(qsw_internal_state != NULL);
-	_lock_qsw();
-	if (savestate)
-		_copy_libstate(savestate, qsw_internal_state);
-	qsw_free_libstate(qsw_internal_state);
-	qsw_internal_state = NULL;
-	_unlock_qsw();
-}
-
-/*
- * Allocate a qsw_jobinfo_t.
- *   jp (IN)		store pointer to new instantiation here
- *   RETURN		0 on success, -1 on failure (sets errno)
- */
-int
-qsw_alloc_jobinfo(qsw_jobinfo_t *jp)
-{
-	qsw_jobinfo_t new; 
-
-	assert(jp != NULL);
-	new = (qsw_jobinfo_t)malloc(sizeof(struct qsw_jobinfo));
-	if (!new)
-		slurm_seterrno_ret(ENOMEM);
-	new->j_magic = QSW_JOBINFO_MAGIC;
-	
-	*jp = new;
-	return 0;
-}
-
-/*
- * Make a copy of a qsw_jobinfo_t.
- *   j (IN)		qsw_jobinfo_t to be copied
- *   RETURN		qsw_jobinfo_t on success, NULL on failure
- */
-qsw_jobinfo_t
-qsw_copy_jobinfo(qsw_jobinfo_t j)
-{
-	qsw_jobinfo_t new; 
-	if (qsw_alloc_jobinfo(&new))
-		return NULL;
-	memcpy(new, j, sizeof(struct qsw_jobinfo));
-
-	return new;
-}
-
-/*
- * Free a qsw_jobinfo_t.
- *   ls (IN)		qsw_jobinfo_t to free
- */
-void
-qsw_free_jobinfo(qsw_jobinfo_t j)
-{
-	if (j == NULL)
-		return;
-	assert(j->j_magic == QSW_JOBINFO_MAGIC);
-	j->j_magic = 0;
-	free(j);
-}
-
-/*
- * Pack jobinfo structure in a format that can be shipped over the
- * network and unpacked on a different architecture.
- *   j (IN)		jobinfo structure to be packed
- *   buffer (OUT)		where to store packed data
- *   RETURN		#bytes unused in 'data' or -1 on error (sets errno)
- * NOTE: Keep in sync with QSW_PACK_SIZE above
- */
-int
-qsw_pack_jobinfo(qsw_jobinfo_t j, Buf buffer)
-{
-	int i, offset;
-
-	assert(j->j_magic == QSW_JOBINFO_MAGIC);
-	offset = get_buf_offset(buffer);
-
-	pack32(j->j_magic, 		buffer);
-	pack32(j->j_prognum, 		buffer);
-	for (i = 0; i < 4; i++)
-		pack32(j->j_cap.UserKey.Values[i], buffer);
-	pack16(j->j_cap.Type, 		buffer);
-#if HAVE_LIBELANCTRL
-#  ifdef ELAN_CAP_ELAN3
-	pack16(j->j_cap.cap_elan_type,  buffer);
-#  else
-	j->j_cap.cap_spare = ELAN_CAP_UNINITIALISED;
-	pack16(j->j_cap.cap_spare,      buffer);
-#  endif 
-#endif
-#if HAVE_LIBELAN3
-	pack16(j->j_cap.padding, 	buffer);
-#endif 
-	pack32(j->j_cap.Version,	buffer);
-	pack32(j->j_cap.LowContext, 	buffer);
-	pack32(j->j_cap.HighContext, 	buffer);
-	pack32(j->j_cap.MyContext, 	buffer);
-	pack32(j->j_cap.LowNode, 	buffer);
-	pack32(j->j_cap.HighNode, 	buffer);
-#if HAVE_LIBELAN3
-	pack32(j->j_cap.Entries, 	buffer);
-#endif 
-	pack32(j->j_cap.RailMask, 	buffer);
-	for (i = 0; i < ELAN_BITMAPSIZE; i++)
-		pack32(j->j_cap.Bitmap[i], buffer);
-
-	return (get_buf_offset(buffer) - offset);
-}
-
-/*
- * Unpack jobinfo structure packed by qsw_pack_jobinfo.
- *   j (IN/OUT)		where to store libstate structure
- *   buffer (OUT)		where to load packed data
- *   RETURN		#bytes unused in 'data' or -1 on error (sets errno)
- */
-int
-qsw_unpack_jobinfo(qsw_jobinfo_t j, Buf buffer)
-{
-	int i, offset;
-
-	assert(j->j_magic == QSW_JOBINFO_MAGIC);
-	offset = get_buf_offset(buffer);
- 
-	safe_unpack32(&j->j_magic, 		buffer);
-	safe_unpack32(&j->j_prognum, 		buffer);
-	for (i = 0; i < 4; i++)
-		safe_unpack32(&j->j_cap.UserKey.Values[i], buffer);
-	safe_unpack16(&j->j_cap.Type,		buffer);
-#if HAVE_LIBELANCTRL
-#  ifdef ELAN_CAP_ELAN3
-	safe_unpack16(&j->j_cap.cap_elan_type,  buffer);
-#  else
-	safe_unpack16(&j->j_cap.cap_spare,      buffer);
-#  endif
-#endif
-#if HAVE_LIBELAN3  
-	safe_unpack16(&j->j_cap.padding, 	buffer);	    
-#endif
-	safe_unpack32(&j->j_cap.Version,	buffer); 	    
-	safe_unpack32(&j->j_cap.LowContext, 	buffer);
-	safe_unpack32(&j->j_cap.HighContext,	buffer);
-	safe_unpack32(&j->j_cap.MyContext,	buffer);
-	safe_unpack32(&j->j_cap.LowNode, 	buffer);
-	safe_unpack32(&j->j_cap.HighNode,	buffer);
-#if HAVE_LIBELAN3
-	safe_unpack32(&j->j_cap.Entries, 	buffer);
-#endif
-	safe_unpack32(&j->j_cap.RailMask, 	buffer);
-	for (i = 0; i < ELAN_BITMAPSIZE; i++)
-		safe_unpack32(&j->j_cap.Bitmap[i], buffer);
-	
-	if (j->j_magic != QSW_JOBINFO_MAGIC)
-		goto unpack_error;
-
-	return SLURM_SUCCESS;
-
-    unpack_error:
-	slurm_seterrno_ret(EBADMAGIC_QSWJOBINFO);
-	return SLURM_ERROR;
-}
-
-/*
- * Allocate a program description number.  Program descriptions, which are the
- * key abstraction maintained by the rms.o kernel module, must not be used
- * more than once simultaneously on a single node.  We allocate one to each
- * parallel job which more than meets this requirement.  A program description
- * can be compared to a process group, except there is no way for a process to
- * disassociate itself or its children from the program description.  
- * If the library is initialized, we allocate these consecutively, otherwise 
- * we generate a random one, assuming we are being called by a transient 
- * program like pdsh.  Ref: rms_prgcreate(3).
- */
-static int
-_generate_prognum(void)
-{
-	int new;
-
-	if (qsw_internal_state) {
-		_lock_qsw();
-		new = qsw_internal_state->ls_prognum;
-		if (new == QSW_PRG_END)
-			qsw_internal_state->ls_prognum = QSW_PRG_START;
-		else
-			qsw_internal_state->ls_prognum++;
-		_unlock_qsw();
-	} else {
-		_srand_if_needed();
-		new = lrand48() % (QSW_PRG_END - QSW_PRG_START + 1);
-		new += QSW_PRG_START;
-	}
-	return new;
-}
-
-/*
- * Elan hardware context numbers are an adapter resource that must not be used
- * more than once on a single node.  One is allocated to each process on the
- * node that will be communication over Elan.  In order for processes on the 
- * same node to communicate with one another and with other nodes across QsNet,
- * they must use contexts in the hi-lo range of a common capability.
- * If the library is initialized, we allocate these consecutively, otherwise 
- * we generate a random one, assuming we are being called by a transient 
- * program like pdsh.  Ref: rms_setcap(3).
- */
-static int
-_generate_hwcontext(int num)
-{
-	int new;
-
-	if (qsw_internal_state) {
-		_lock_qsw();
-		if (qsw_internal_state->ls_hwcontext + num - 1 > QSW_CTX_END)
-			qsw_internal_state->ls_hwcontext = QSW_CTX_START;
-		new = qsw_internal_state->ls_hwcontext;
-		qsw_internal_state->ls_hwcontext += num;
-		_unlock_qsw();
-	} else {
-		_srand_if_needed();
-		new = lrand48() % 
-		      (QSW_CTX_END - (QSW_CTX_START + num - 1) - 1);
-		new +=  QSW_CTX_START;
-	}
-	return new;
-}
-
-/*
- * Initialize the elan capability for this job.
- */
-static void
-_init_elan_capability(ELAN_CAPABILITY *cap, int nprocs, int nnodes,
-		bitstr_t *nodeset, int cyclic_alloc)
-{
-	int i, node_num, full_node_cnt, min_procs_per_node, max_procs_per_node;
-
-	/* Task count may not be identical for all nodes */
-	full_node_cnt = nprocs % nnodes;
-	min_procs_per_node = nprocs / nnodes;
-	max_procs_per_node = (nprocs + nnodes - 1) / nnodes;
-
-	_srand_if_needed();
-
-	/* start with a clean slate */
-#if HAVE_LIBELANCTRL
-	elan_nullcap(cap);
-#else
-	elan3_nullcap(cap);
-#endif
-
-	/* initialize for single rail and either block or cyclic allocation */
-	if (cyclic_alloc)
-		cap->Type = ELAN_CAP_TYPE_CYCLIC;
-	else
-		cap->Type = ELAN_CAP_TYPE_BLOCK;
-	cap->Type |= ELAN_CAP_TYPE_MULTI_RAIL;
-	cap->RailMask = 1;
-
-#if HAVE_LIBELANCTRL
-#  ifdef ELAN_CAP_ELAN3
-	cap->cap_elan_type = ELAN_CAP_ELAN3;
-#  else
-	cap->cap_spare = ELAN_CAP_UNINITIALISED;
-#  endif
-#endif 
-
-	/* UserKey is 128 bits of randomness which should be kept private */
-        for (i = 0; i < 4; i++)
-		cap->UserKey.Values[i] = lrand48();
-
-	/* set up hardware context range */
-	cap->LowContext = _generate_hwcontext(max_procs_per_node);
-	cap->HighContext = cap->LowContext + max_procs_per_node - 1;
-	/* Note: not necessary to initialize cap->MyContext */
-
-	/* set the range of nodes to be used and number of processes */
-	cap->LowNode = bit_ffs(nodeset);
-	assert(cap->LowNode != -1);
-	cap->HighNode = bit_fls(nodeset);
-	assert(cap->HighNode != -1);
-
-#if HAVE_LIBELAN3
-	cap->Entries = nprocs;
-#endif
-
-#if USE_OLD_LIBELAN
-	/* set the hw broadcast bit if consecutive nodes */
-	if (abs(cap->HighNode - cap->LowNode) == nnodes - 1)
-		cap->Type |= ELAN_CAP_TYPE_BROADCASTABLE;
-#else
-	/* set unconditionally per qsw gnat sw-elan/4334 */
-	/* only time we don't want this is unsupported rev A hardware */
-	cap->Type |= ELAN_CAP_TYPE_BROADCASTABLE;
-#endif
-	/*
-	 * Set up cap->Bitmap, which describes the mapping of processes to 
-	 * the nodes in the range of cap->LowNode - cap->Highnode.
-	 * There are (nprocs * nnodes) significant bits in the mask, each 
- 	 * representing a process slot.  Bits are off for process slots 
-	 * corresponding to unallocated nodes.  For example, if nodes 4 and 6 
-	 * are running two processes per node, bits 0,1 (corresponding to the 
-	 * two processes on node 4) and bits 4,5 (corresponding to the two 
-	 * processes running on node 6) are set.  
-	 */
-	node_num = 0;
-	for (i = cap->LowNode; i <= cap->HighNode; i++) {
-		if (bit_test(nodeset, i)) {
-			int j, bit, task_cnt;
-
-			if (node_num++ < full_node_cnt)
-				task_cnt = max_procs_per_node;
-			else
-				task_cnt = min_procs_per_node;
-
-			for (j = 0; j < task_cnt; j++) {
-				if (cyclic_alloc)
-					bit = (i-cap->LowNode) + ( j * 
-					 (cap->HighNode - cap->LowNode + 1));
-				else
-					bit = ((i-cap->LowNode)
-					       * max_procs_per_node) + j;
-
-				assert(bit < (sizeof(cap->Bitmap) * 8));
-				BT_SET(cap->Bitmap, bit);
-			}
-		}
-	}
-}
-
-/*
- * Create all the QsNet related information needed to set up a QsNet parallel
- * program and store it in the qsw_jobinfo struct.  
- * Call this on the "client" process, e.g. pdsh, srun, slurmctld, etc..
- */
-int
-qsw_setup_jobinfo(qsw_jobinfo_t j, int nprocs, bitstr_t *nodeset, 
-		int cyclic_alloc)
-{
-	int nnodes = bit_set_count(nodeset);
-
-	assert(j != NULL);
-	assert(j->j_magic == QSW_JOBINFO_MAGIC);
-
-	/* sanity check on args */
-	/* Note: ELAN_MAX_VPS is 512 on "old" Elan driver, 16384 on new. */
-	if ((nprocs <= 0) || (nprocs > ELAN_MAX_VPS) || (nnodes <= 0)) {
-		slurm_seterrno_ret(EINVAL);
-	}
-      
-	/* initialize jobinfo */
-	j->j_prognum = _generate_prognum();
-	_init_elan_capability(&j->j_cap, nprocs, nnodes, nodeset, 
-	                      cyclic_alloc);
-
-	return 0;
-}
-
-/*
- * Here are the necessary steps to set up to run an Elan MPI parallel program
- * (set of processes) on a node (possibly one of many allocated to the prog):
- *
- * Process 1	Process 2	|	Process 3
- * read args			|
- * fork	-------	rms_prgcreate	|
- * waitpid 	elan3_create	|
- * 		rms_prgaddcap	|
- *		fork N procs ---+------	rms_setcap
- *		wait all	|	setup RMS_ env	
- *				|	setuid, etc.
- *				|	exec mpi process
- *				|	
- *		exit		|
- * rms_prgdestroy		|
- * exit				|     (one pair of processes per mpi proc!)
- *
- * - The first fork is required because rms_prgdestroy can't occur in the 
- *   process that calls rms_prgcreate (since it is a member, ECHILD).
- * - The second fork is required when running multiple processes per node 
- *   because each process must announce its use of one of the hw contexts 
- *   in the range allocated in the capability.
- */
-
-/*
- * Process 1: issue the rms_prgdestroy for the job.
- */
-int
-qsw_prgdestroy(qsw_jobinfo_t jobinfo)
-{
-	if (rms_prgdestroy(jobinfo->j_prognum) < 0) {
-		/* translate errno values to more descriptive ones */
-		switch (errno) {
-			case ECHILD:
-				slurm_seterrno(ECHILD_PRGDESTROY);
-				break;
-			case EEXIST:
-				slurm_seterrno(EEXIST_PRGDESTROY);
-				break;
-			default:
-				break;
-		}
-		return -1;
-	}
-	return 0;
-}
-
-/*
- * Process 2: Destroy the context after children are dead.
- */
-void
-qsw_prog_fini(qsw_jobinfo_t jobinfo)
-{
-	/* Do nothing... apparently this will be handled by
-	 *  callbacks in the kernel exit handlers ... 
-	 */
-#if 0
-	if (jobinfo->j_ctx) {
-		elan3_control_close(jobinfo->j_ctx);
-		jobinfo->j_ctx = NULL;
-	}
-#endif
-}
-
-/*
- * Process 2: Create the context and make capability available to children.
- */
-int
-qsw_prog_init(qsw_jobinfo_t jobinfo, uid_t uid)
-{
-	int err;
-	int i, nrails;
-#if HAVE_LIBELANCTRL
-	nrails = elan_nrails(&jobinfo->j_cap);
-
-	for (i = 0; i < nrails; i++) {
-		ELANCTRL_HANDLE handle;
-		/*
-		 *  Open up the Elan control device so we can create
-		 *   a new capability.
-		 */
-		if (elanctrl_open(&handle) != 0) {
-			slurm_seterrno(EELAN3CONTROL);
-			goto fail;
-		}
-
-		/*  Push capability into device driver */
-		if (elanctrl_create_cap(handle, &jobinfo->j_cap) < 0) {
-			error("elanctrl_create_cap: %m");
-			slurm_seterrno(EELAN3CREATE);
-			goto fail;
-		}
-	}
-
-#else /* !HAVE_LIBELANCTRL */
-	nrails = elan3_nrails(&jobinfo->j_cap);
-
-	for (i = 0; i < nrails; i++) {
-
-		ELAN3_CTX *ctx;
-
-		/* see qsw gnat sw-elan/4334: elan3_control_open can ret -1 */
-		if ((ctx = elan3_control_open(i)) == NULL 
-				|| ctx == (void *)-1) {
-			slurm_seterrno(EELAN3CONTROL);
-			goto fail;
-		}
-		
-	
-		/* make cap known via rms_getcap/rms_ncaps to members 
-		 * of this prgnum */
-		if (elan3_create(ctx, &jobinfo->j_cap) < 0) {
-			/* XXX masking errno value better than not knowing 
-			 * which function failed? */
-		        error("elan3_create(%d): %m", i);
-			slurm_seterrno(EELAN3CREATE); 
-			goto fail;
-		}
-	}
-#endif
-	/* associate this process and its children with prgnum */
-	if (rms_prgcreate(jobinfo->j_prognum, uid, 1) < 0) {
-		/* translate errno values to more descriptive ones */
-		switch (errno) {
-			case EINVAL:
-				slurm_seterrno(EINVAL_PRGCREATE);
-				break;
-			default:
-				break;
-		}
-		goto fail;
-	}
-
-	if (rms_prgaddcap(jobinfo->j_prognum, 0, &jobinfo->j_cap) < 0) {
-		/* translate errno values to more descriptive ones */
-		switch (errno) {
-			case ESRCH:
-				slurm_seterrno(ESRCH_PRGADDCAP);
-				break;
-			case EFAULT:
-				slurm_seterrno(EFAULT_PRGADDCAP);
-				break;
-			default:
-				break;
-		}
-		goto fail;
-	}
-
-	/* note: _elan3_fini() destroys context and makes capability unavail */
-	/* do it in qsw_prog_fini() after app terminates */
-	return 0;
-fail:
-	err = errno; /* presrve errno in case _elan3_fini touches it */
-	qsw_prog_fini(jobinfo); 
-	slurm_seterrno(err);
-	return -1;
-}
-
-/*
- * Process 3: Do the rms_setcap.
- */
-int
-qsw_setcap(qsw_jobinfo_t jobinfo, int procnum)
-{
-	/*
-	 * Assign elan hardware context to current process.
-	 * - arg1 (0 below) is an index into the kernel's list of caps for this 
-	 *   program desc (added by rms_prgaddcap).  There will be
-	 *   one per rail.
-	 * - arg2 indexes the hw ctxt range in the capability
-	 *   [cap->LowContext, cap->HighContext]
-	 */
-	if (rms_setcap(0, procnum) < 0) {
-		/* translate errno values to more descriptive ones */
-		switch (errno) {
-			case EINVAL:
-				slurm_seterrno(EINVAL_SETCAP);
-				break;
-			case EFAULT:
-				slurm_seterrno(EFAULT_SETCAP);
-				break;
-			default:
-				break;
-		}
-		return -1;
-	}
-	return 0;
-}
-
-
-/*
- * Return the local elan address (for rail 0) or -1 on failure.
- */
-int
-qsw_getnodeid(void)
-{
-	int nodeid = -1;
-#if HAVE_LIBELANCTRL
-	ELAN_DEV_IDX    devidx = 0;
-	ELANCTRL_HANDLE handle;
-	ELAN_POSITION   position;
-
-	if (elanctrl_open(&handle) != 0) 
-		slurm_seterrno_ret(EGETNODEID);
-
-	if (elanctrl_get_position(handle, devidx, &position) != 0)
-		slurm_seterrno_ret(EGETNODEID);
-
-	nodeid = position.pos_nodeid;
-
-#else
-	ELAN3_CTX *ctx = _elan3_init(0); /* rail 0 */
-	if (ctx) {
-		nodeid = ctx->devinfo.Position.NodeId;
-		elan3_control_close(ctx);
-	}
-#endif
-	if (nodeid == -1)
-		slurm_seterrno(EGETNODEID);
-	return nodeid;
-
-}
-
-static int 
-_read_elanhost_config (void)
-{
-	int rc;
-
-	if (!(elanconf = elanhost_config_create ()))
-		return (-1);
-
-	if ((rc = elanhost_config_read (elanconf, NULL)) < 0) {
-		error ("Unable to read Elan config: %s", 
-		       elanhost_config_err (elanconf));
-		elanhost_config_destroy (elanconf);
-		elanconf = NULL;
-		return (-1);
-	}
-
-	return (0);
-}
-
-int
-qsw_maxnodeid(void)
-{
-	int maxid = -1;
-
-	_lock_qsw();
-	if (!elanconf && (_read_elanhost_config() < 0))
-		goto done;
-
-	maxid = elanhost_config_maxid (elanconf);
-
-    done:
-	_unlock_qsw();
-	return maxid;
-}
-
-/*
- * Given a hostname, return the elanid or -1 on error.  
- *  Initializes the elanconfig from the default /etc/elanhosts
- *  config file.
- */
-int
-qsw_getnodeid_byhost(char *host)
-{
-	int id = -1;
-
-	if (host == NULL)
-		return (-1);
-
-	_lock_qsw();
-	if (!elanconf && (_read_elanhost_config() < 0))
-		goto done;
-
-	xassert (elanconf != NULL);
-
-	id = elanhost_host2elanid (elanconf, host);
-
-    done:
-	_unlock_qsw();
-	return id;
-}
-
-/*
- * Given an elanid, determine the hostname.  Returns -1 on error or the number
- * of characters copied on success.  
- * XXX - assumes RMS style hostnames (see above)
- */
-int
-qsw_gethost_bynodeid(char *buf, int len, int id)
-{
-	int rc = -1;
-	char *hostp;
-
-	if (id < 0) slurm_seterrno_ret(EGETHOST_BYNODEID);
-
-	_lock_qsw();
-	if (!elanconf && (_read_elanhost_config() < 0))
-		goto done;
-
-	if (!(hostp = elanhost_elanid2host (elanconf, ELANHOST_EIP, id))) {
-		slurm_seterrno (EGETHOST_BYNODEID);
-		goto done;
-	}
-	
-	rc = strlcpy (buf, hostp, len);
-
-    done:
-	_unlock_qsw();
-	return (rc);
-}
-
-/*
- * Send the specified signal to all members of a program description.
- * Returns -1 on failure and sets errno.  Ref: rms_prgsignal(3).
- */
-int
-qsw_prgsignal(qsw_jobinfo_t jobinfo, int signum)
-{
-	if (rms_prgsignal(jobinfo->j_prognum, signum) < 0) {
-		/* translate errno values to more descriptive ones */
-		switch (errno) {
-			case EINVAL:
-				slurm_seterrno(EINVAL_PRGSIGNAL);
-				break;
-			case ESRCH:
-				slurm_seterrno(ESRCH_PRGSIGNAL);
-				break;
-			default:
-				break;
-		}
-		return -1;
-	}
-	return 0;
-}
-
-#define _USE_ELAN3_CAPABILITY_STRING 1
-
-#ifndef _USE_ELAN3_CAPABILITY_STRING
-#define TRUNC_BITMAP 1
-static void
-_print_capbitmap(FILE *fp, ELAN_CAPABILITY *cap)
-{
-	int bit_max = sizeof(cap->Bitmap)*8 - 1;
-	int bit;
-#if TRUNC_BITMAP
-	bit_max = bit_max >= 64 ? 64 : bit_max;
-#endif
-	for (bit = bit_max; bit >= 0; bit--)
-		fprintf(fp, "%c", BT_TEST(cap->Bitmap, bit) ? '1' : '0');
-	fprintf(fp, "\n");
-}
-#endif /* !_USE_ELAN3_CAPABILITY_STRING */
-
-char *
-qsw_capability_string(struct qsw_jobinfo *j, char *buf, size_t size)
-{
-	ELAN_CAPABILITY *cap;
-
-	assert(buf != NULL);
-	assert(j->j_magic == QSW_JOBINFO_MAGIC);
-
-	cap = &j->j_cap;
-
-#if HAVE_LIBELANCTRL
-	snprintf(buf, size, "prg=%d ctx=%x.%x nodes=%d.%d",
-	         j->j_prognum, cap->LowContext, cap->HighContext, 
-		 cap->LowNode, cap->HighNode);
-#else 
-	snprintf(buf, size, "prg=%d ctx=%x.%x nodes=%d.%d entries=%d",
-	         j->j_prognum, cap->LowContext, cap->HighContext, 
-		 cap->LowNode, cap->HighNode, 
-	         cap->Entries);
-#endif
-         
-	return buf;
-}
-
-void
-qsw_print_jobinfo(FILE *fp, struct qsw_jobinfo *jobinfo)
-{
-	ELAN_CAPABILITY *cap;
-	char str[8192];
-
-	assert(jobinfo->j_magic == QSW_JOBINFO_MAGIC);
-
-	fprintf(fp, "__________________\n");
-	fprintf(fp, "prognum=%d\n", jobinfo->j_prognum);
-
-	cap = &jobinfo->j_cap;
-	/* use elan3_capability_string as a shorter alternative for now */
-#if _USE_ELAN3_CAPABILITY_STRING
-#  if HAVE_LIBELANCTRL
-	fprintf(fp, "%s\n", elan_capability_string(cap, str));
-#  else
-	fprintf(fp, "%s\n", elan3_capability_string(cap, str));
-#  endif
-#else 
-	fprintf(fp, "cap.UserKey=%8.8x.%8.8x.%8.8x.%8.8x\n",
-			cap->UserKey.Values[0], cap->UserKey.Values[1],
-			cap->UserKey.Values[2], cap->UserKey.Values[3]);
-	/*fprintf(fp, "cap.Version=%d\n", cap->Version);*/
-	fprintf(fp, "cap.Type=0x%hx\n", cap->Type);
-	fprintf(fp, "cap.LowContext=%d\n", cap->LowContext);
-	fprintf(fp, "cap.HighContext=%d\n", cap->HighContext);
-	fprintf(fp, "cap.MyContext=%d\n", cap->MyContext);
-	fprintf(fp, "cap.LowNode=%d\n", cap->LowNode);
-	fprintf(fp, "cap.HighNode=%d\n", cap->HighNode);
-#if HAVE_LIBELAN3
-	fprintf(fp, "cap.padding=%hd\n", cap->padding);
-	fprintf(fp, "cap.Entries=%d\n", cap->Entries);
-#endif
-	fprintf(fp, "cap.Railmask=0x%x\n", cap->RailMask);
-	fprintf(fp, "cap.Bitmap=");
-	_print_capbitmap(fp, cap);
-#endif
-	fprintf(fp, "\n------------------\n");
-}

src/common/qsw.h

-/*****************************************************************************\
- *  qsw.h - Library routines for initiating jobs on QsNet. 
- *****************************************************************************
- *  Copyright (C) 2002 The Regents of the University of California.
- *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- *  Written by Jim Garlick <garlick@llnl.gov>
- *  UCRL-CODE-2002-040.
- *  
- *  This file is part of SLURM, a resource management program.
- *  For details, see <http://www.llnl.gov/linux/slurm/>.
- *  
- *  SLURM is free software; you can redistribute it and/or modify it under
- *  the terms of the GNU General Public License as published by the Free
- *  Software Foundation; either version 2 of the License, or (at your option)
- *  any later version.
- *  
- *  SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
- *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
- *  details.
- *  
- *  You should have received a copy of the GNU General Public License along
- *  with SLURM; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
-\*****************************************************************************/
-
-#include <stdio.h>
-#include <sys/types.h>
-
-#if HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "src/common/bitstring.h"
-#include "src/common/pack.h"
-
-#ifndef _QSW_INCLUDED
-#define _QSW_INCLUDED
-
-#if HAVE_LIBELANCTRL
-# include <elan/capability.h>
-#elif HAVE_LIBELAN3
-# include <elan3/elanvp.h>
-#else
-# error "Don't have either libelanctrl or libelan3!"
-#endif
-
-/* opaque data structures - no peeking! */
-typedef struct qsw_libstate 	*qsw_libstate_t;
-#ifndef __qsw_jobinfo_t_defined
-#  define  __qsw_jobinfo_t_defined
-   typedef struct qsw_jobinfo *qsw_jobinfo_t;	/* opaque data type */
-#endif
-
-#define QSW_LIBSTATE_PACK_MAX	12
-#define QSW_JOBINFO_PACK_MAX	120
-#define QSW_MAX_TASKS		ELAN_MAX_VPS
-#define QSW_PACK_SIZE		(4 * (2+4+1+8+ELAN_BITMAPSIZE))
-
-int		qsw_alloc_libstate(qsw_libstate_t *lsp);
-void		qsw_free_libstate(qsw_libstate_t ls);
-
-int		qsw_pack_libstate(qsw_libstate_t ls, Buf buffer);
-int		qsw_unpack_libstate(qsw_libstate_t ls, Buf buffer);
-
-int 		qsw_init(qsw_libstate_t restorestate);
-void 		qsw_fini(qsw_libstate_t savestate);
-
-int		qsw_alloc_jobinfo(qsw_jobinfo_t *jp);
-qsw_jobinfo_t	qsw_copy_jobinfo(qsw_jobinfo_t j);
-void		qsw_free_jobinfo(qsw_jobinfo_t j);
-
-int		qsw_pack_jobinfo(qsw_jobinfo_t j, Buf buffer);
-int		qsw_unpack_jobinfo(qsw_jobinfo_t j, Buf buffer);
-
-int 		qsw_setup_jobinfo(qsw_jobinfo_t j, int nprocs, 
-			bitstr_t *nodeset, int cyclic_alloc);
-
-int 		qsw_prog_init(qsw_jobinfo_t jobinfo, uid_t uid);
-void 		qsw_prog_fini(qsw_jobinfo_t jobinfo);
-
-int 		qsw_prgdestroy(qsw_jobinfo_t jobinfo); /* was qsw_prog_reap */
-
-int 		qsw_setcap(qsw_jobinfo_t jobinfo, int procnum); 
-		/* was qsw_attach */
-
-int		qsw_prgsignal(qsw_jobinfo_t jobinfo, int signum); 
-		/* was qsw_signal_job */
-
-		/* return max ElanID in configuration */
-int             qsw_maxnodeid(void);
-
-int		qsw_getnodeid(void);
-int		qsw_getnodeid_byhost(char *host);
-int		qsw_gethost_bynodeid(char *host, int len, int elanid);
-
-char *		qsw_capability_string(qsw_jobinfo_t j, char *buf, size_t len);
-void		qsw_print_jobinfo(FILE *fp, struct qsw_jobinfo *jobinfo);
-
-#endif /* _QSW_INCLUDED */

src/slurmd/setenvpf.c → src/common/setenvpf.c

 /*****************************************************************************\
- * src/slurmd/setenvpf.c - add an environment variable to environment vector
+ * src/common/setenvpf.c - add an environment variable to environment vector
  * $Id$
  *****************************************************************************
  *  Copyright (C) 2002 The Regents of the University of California.

src/slurmd/setenvpf.h → src/common/setenvpf.h

 /*****************************************************************************\
- * src/slurmd/setenvpf.h - environment vector manipulation
+ * src/common/setenvpf.h - environment vector manipulation
  *****************************************************************************
  *  Copyright (C) 2002 The Regents of the University of California.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).

src/common/slurm_protocol_defs.c

@@ -39,6 +39,7 @@
 #include "src/common/log.h"
 #include "src/common/slurm_cred.h"
 #include "src/common/slurm_protocol_defs.h"
+#include "src/common/switch.h"
 #include "src/common/xmalloc.h"
 
 static void _free_all_job_info (job_info_msg_t *msg);
@@ -282,9 +283,8 @@ void slurm_free_launch_tasks_request_msg(launch_tasks_request_msg_t * msg)
 	xfree(msg->ofname);
 	xfree(msg->ofname);
 
-#	ifdef HAVE_ELAN
-	qsw_free_jobinfo(msg->qsw_job);
-#	endif
+	if (msg->switch_job)
+		switch_free_jobinfo(msg->switch_job);
 
 	xfree(msg);
 }
@@ -473,10 +473,8 @@ void slurm_free_resource_allocation_and_run_response_msg (
 		xfree(msg->cpu_count_reps);
 		xfree(msg->node_addr);
 		slurm_cred_destroy(msg->cred);
-#		ifdef HAVE_LIBELAN3
-		if (msg->qsw_job)
-			qsw_free_jobinfo(msg->qsw_job);
-#		endif
+		if (msg->switch_job)
+			switch_free_jobinfo(msg->switch_job);
 		xfree(msg);
 	}
 }
@@ -494,10 +492,8 @@ void slurm_free_job_step_create_response_msg(
 	if (msg) {
 		slurm_cred_destroy(msg->cred);
 
-#		ifdef HAVE_LIBELAN3
-		if (msg->qsw_job)
-			qsw_free_jobinfo(msg->qsw_job);
-#		endif
+		if (msg->switch_job)
+			switch_free_jobinfo(msg->switch_job);
 
 		xfree(msg);
 	}
@@ -546,7 +542,6 @@ void slurm_free_ctl_conf(slurm_ctl_conf_info_msg_t * config_ptr)
 		xfree(config_ptr->slurmd_spooldir);
 		xfree(config_ptr->slurm_conf);
 		xfree(config_ptr->state_save_location);
-		xfree(config_ptr->switch_type);
 		xfree(config_ptr->tmp_fs);
 		xfree(config_ptr);
 	}

src/common/slurm_protocol_defs.h

@@ -36,9 +36,6 @@
 #      include <stdint.h>
 #    endif
 #  endif			/* HAVE_INTTYPES_H */
-#  if HAVE_ELAN
-#    include <src/common/qsw.h>
-#  endif
 #else				/* !HAVE_CONFIG_H */
 #  include <inttypes.h>
 #endif				/*  HAVE_CONFIG_H */
@@ -47,6 +44,7 @@
 
 #include "src/common/macros.h"
 #include "src/common/slurm_protocol_common.h"
+#include "src/common/switch.h"
 #include "src/common/xassert.h"
 
 
@@ -258,10 +256,7 @@ typedef struct launch_tasks_request_msg {
 	int32_t   slurmd_debug; /* remote slurmd debug level */
 
 	slurm_cred_t cred;	/* job credential            */
-
-#ifdef HAVE_ELAN
-	qsw_jobinfo_t qsw_job;	/* Elan3 switch context */
-#endif
+	switch_jobinfo_t switch_job;	/* switch credential for the job */
 } launch_tasks_request_msg_t;
 
 typedef struct launch_tasks_response_msg {

src/common/slurm_protocol_pack.c

@@ -28,9 +28,9 @@
 #  include "config.h"
 #endif
 
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <errno.h>
 #include <string.h>
 
 #include "src/common/bitstring.h"
@@ -41,12 +41,9 @@
 #include "src/common/slurm_protocol_api.h"
 #include "src/common/slurm_protocol_defs.h"
 #include "src/common/slurm_protocol_pack.h"
+#include "src/common/switch.h"
 #include "src/common/xmalloc.h"
 
-#if HAVE_ELAN
-#  include "src/common/qsw.h"
-#endif
-
 #define _pack_job_info_msg(msg,buf)		_pack_buffer_msg(msg,buf)
 #define _pack_job_step_info_msg(msg,buf)	_pack_buffer_msg(msg,buf)
 
@@ -890,9 +887,7 @@ static void
 	_pack_slurm_addr_array(msg->node_addr, msg->node_cnt, buffer);
 
 	slurm_cred_pack(msg->cred, buffer);
-#ifdef HAVE_ELAN
-	qsw_pack_jobinfo(msg->qsw_job, buffer);
-#endif
+	switch_pack_jobinfo(msg->switch_job, buffer);
 }
 
 static int
@@ -941,14 +936,12 @@ static int
 	if (!(tmp_ptr->cred = slurm_cred_unpack(buffer)))
 		goto unpack_error;
 
-#ifdef HAVE_ELAN
-	qsw_alloc_jobinfo(&tmp_ptr->qsw_job);
-	if (qsw_unpack_jobinfo(tmp_ptr->qsw_job, buffer) < 0) {
-		error("qsw_unpack_jobinfo: %m");
-		qsw_free_jobinfo(tmp_ptr->qsw_job);
+	switch_alloc_jobinfo(&tmp_ptr->switch_job);
+	if (switch_unpack_jobinfo(tmp_ptr->switch_job, buffer) < 0) {
+		error("switch_unpack_jobinfo: %m");
+		switch_free_jobinfo(tmp_ptr->switch_job);
 		goto unpack_error;
 	}
-#endif
 	return SLURM_SUCCESS;
 
       unpack_error:
@@ -1286,9 +1279,7 @@ _pack_job_step_create_response_msg(job_step_create_response_msg_t * msg,
 	pack32(msg->job_step_id, buffer);
 	packstr(msg->node_list, buffer);
 	slurm_cred_pack(msg->cred, buffer);
-#ifdef HAVE_ELAN
-	qsw_pack_jobinfo(msg->qsw_job, buffer);
-#endif
+	switch_pack_jobinfo(msg->switch_job, buffer);
 
 }
 
@@ -1309,14 +1300,12 @@ _unpack_job_step_create_response_msg(job_step_create_response_msg_t ** msg,
 	if (!(tmp_ptr->cred = slurm_cred_unpack(buffer)))
 		goto unpack_error;
 
-#ifdef HAVE_ELAN
-	qsw_alloc_jobinfo(&tmp_ptr->qsw_job);
-	if (qsw_unpack_jobinfo(tmp_ptr->qsw_job, buffer)) {
-		error("qsw_unpack_jobinfo: %m");
-		qsw_free_jobinfo(tmp_ptr->qsw_job);
+	switch_alloc_jobinfo(&tmp_ptr->switch_job);
+	if (switch_unpack_jobinfo(tmp_ptr->switch_job, buffer)) {
+		error("switch_unpack_jobinfo: %m");
+		switch_free_jobinfo(tmp_ptr->switch_job);
 		goto unpack_error;
 	}
-#endif
 	return SLURM_SUCCESS;
 
       unpack_error:
@@ -2139,9 +2128,7 @@ _pack_launch_tasks_request_msg(launch_tasks_request_msg_t * msg, Buf buffer)
 	pack32(msg->slurmd_debug, buffer);
 	pack32_array(msg->global_task_ids,
 		     msg->tasks_to_launch, buffer);
-#ifdef HAVE_ELAN
-	qsw_pack_jobinfo(msg->qsw_job, buffer);
-#endif
+	switch_pack_jobinfo(msg->switch_job, buffer);
 }
 
 static int
@@ -2179,13 +2166,13 @@ _unpack_launch_tasks_request_msg(launch_tasks_request_msg_t **
 	if (msg->tasks_to_launch != uint32_tmp)
 		goto unpack_error;
 
-#ifdef HAVE_ELAN
-	qsw_alloc_jobinfo(&msg->qsw_job);
-	if (qsw_unpack_jobinfo(msg->qsw_job, buffer) < 0) {
-		error("qsw_unpack_jobinfo: %m");
+	switch_alloc_jobinfo(&msg->switch_job);
+	if (switch_unpack_jobinfo(msg->switch_job, buffer) < 0) {
+		error("switch_unpack_jobinfo: %m");
+		switch_free_jobinfo(msg->switch_job);
 		goto unpack_error;
 	}
-#endif
+
 	return SLURM_SUCCESS;
 
       unpack_error:

src/common/switch.c

+/*****************************************************************************\
+ * src/common/switch.c - Generic switch (interconnect) for slurm
+ *****************************************************************************
+ *  Copyright (C) 2002 The Regents of the University of California.
+ *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ *  Written by Moe Jette <jette@llnl.gov>.
+ *  UCRL-CODE-2002-040.
+ *  
+ *  This file is part of SLURM, a resource management program.
+ *  For details, see <http://www.llnl.gov/linux/slurm/>.
+ *  
+ *  SLURM is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU General Public License as published by the Free
+ *  Software Foundation; either version 2 of the License, or (at your option)
+ *  any later version.
+ *  
+ *  SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
+ *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ *  details.
+ *  
+ *  You should have received a copy of the GNU General Public License along
+ *  with SLURM; if not, write to the Free Software Foundation, Inc.,
+ *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+\*****************************************************************************/
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/common/macros.h"
+#include "src/common/plugin.h"
+#include "src/common/plugrack.h"
+#include "src/common/slurm_protocol_api.h"
+#include "src/common/switch.h"
+#include "src/common/xmalloc.h"
+#include "src/common/xstring.h"
+
+/*
+ * WARNING:  Do not change the order of these fields or add additional
+ * fields at the beginning of the structure.  If you do, job completion
+ * logging plugins will stop working.  If you need to add fields, add them 
+ * at the end of the structure.
+ */
+typedef struct slurm_switch_ops {
+	int          (*state_save)        ( char *dir_name );
+	int          (*state_restore)     ( char *dir_name );
+	bool         (*no_frag)           ( void );
+	int          (*alloc_jobinfo)     ( switch_jobinfo_t *jobinfo );
+	int          (*build_jobinfo)     ( switch_jobinfo_t jobinfo,
+						char *nodelist, int nprocs, 
+						int cyclic_alloc);
+	switch_jobinfo_t (*copy_jobinfo)  ( switch_jobinfo_t jobinfo );
+	void         (*free_jobinfo)      ( switch_jobinfo_t jobinfo );
+	int          (*pack_jobinfo)      ( switch_jobinfo_t jobinfo, 
+						Buf buffer );
+	int          (*unpack_jobinfo)    ( switch_jobinfo_t jobinfo, 
+						Buf buffer );
+	void         (*print_jobinfo)     ( FILE *fp, 
+						switch_jobinfo_t jobinfo );
+	char *       (*string_jobinfo)    ( switch_jobinfo_t jobinfo, 
+						char *buf, size_t size);
+	int          (*node_init)         ( void );
+	int          (*node_fini)         ( void );
+	int          (*job_preinit)       ( switch_jobinfo_t jobinfo );
+	int          (*job_init)          ( switch_jobinfo_t jobinfo,
+						uid_t uid );
+	int          (*job_fini)          ( switch_jobinfo_t jobinfo );
+	int          (*job_postfini)      ( switch_jobinfo_t jobinfo,
+						uid_t pgid, 
+						uint32_t job_id, 
+						uint32_t step_id );
+	int          (*job_attach)        ( switch_jobinfo_t jobinfo, 
+						char ***env, int nodeid, 
+						int procid, int nnodes, 
+						int nprocs, gid_t gid);
+} slurm_switch_ops_t;
+
+struct slurm_switch_context {
+	char *			switch_type;
+	plugrack_t              plugin_list;
+	plugin_handle_t         cur_plugin;
+	int                     switch_errno;
+	slurm_switch_ops_t	ops;
+};
+
+static slurm_switch_context_t g_context = NULL;
+static pthread_mutex_t      context_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static slurm_switch_context_t
+_slurm_switch_context_create( const char *switch_type)
+{
+	slurm_switch_context_t c;
+
+	if ( switch_type == NULL ) {
+		debug3( "_slurm_switch_context_create: no switch type" );
+		return NULL;
+	}
+
+	c = xmalloc( sizeof( struct slurm_switch_context ) );
+
+	c->switch_errno = SLURM_SUCCESS;
+
+	/* Copy the job completion authentication type. */
+	c->switch_type = xstrdup( switch_type );
+	if (c->switch_type == NULL ) {
+		debug3( "can't make local copy of switch type" );
+		xfree( c );
+		return NULL;
+	}
+
+	/* Plugin rack is demand-loaded on first reference. */
+	c->plugin_list = NULL;
+	c->cur_plugin = PLUGIN_INVALID_HANDLE;
+
+	return c;
+}
+
+static int
+_slurm_switch_context_destroy( slurm_switch_context_t c )
+{
+	/*
+	 * Must check return code here because plugins might still
+	 * be loaded and active.
+	 */
+	if ( c->plugin_list ) {
+		if ( plugrack_destroy( c->plugin_list ) != SLURM_SUCCESS ) {
+			return SLURM_ERROR;
+		}
+	}
+
+	xfree( c->switch_type );
+	xfree( c );
+
+	return SLURM_SUCCESS;
+}
+
+/*
+ * Resolve the operations from the plugin.
+ */
+static slurm_switch_ops_t * 
+_slurm_switch_get_ops( slurm_switch_context_t c )
+{
+	/*
+	 * These strings must be kept in the same order as the fields
+	 * declared for slurm_switch_ops_t.
+	 */
+	static const char *syms[] = {
+		"switch_p_libstate_save",
+		"switch_p_libstate_restore",
+		"switch_p_no_frag",
+		"switch_p_alloc_jobinfo",
+		"switch_p_build_jobinfo",
+		"switch_p_copy_jobinfo",
+		"switch_p_free_jobinfo",
+		"switch_p_pack_jobinfo",
+		"switch_p_unpack_jobinfo",
+		"switch_p_print_jobinfo",
+		"switch_p_sprint_jobinfo",
+		"switch_p_node_init",
+		"switch_p_node_fini",
+		"switch_p_job_preinit",
+		"switch_p_job_init",
+		"switch_p_job_fini",
+		"switch_p_job_postfini",
+		"switch_p_job_attach"
+	};
+	int n_syms = sizeof( syms ) / sizeof( char * );
+
+	/* Get the plugin list, if needed. */
+	if ( c->plugin_list == NULL ) {
+		char *plugin_dir;
+		c->plugin_list = plugrack_create();
+		if ( c->plugin_list == NULL ) {
+			verbose( "Unable to create a plugin manager" );
+			return NULL; 
+		}
+
+		plugrack_set_major_type( c->plugin_list, "switch" );
+		plugrack_set_paranoia( c->plugin_list,
+				PLUGRACK_PARANOIA_NONE,
+				 0 );
+		plugin_dir = slurm_get_plugin_dir();
+		plugrack_read_dir( c->plugin_list, plugin_dir );
+		xfree(plugin_dir);
+	}
+
+	/* Find the correct plugin. */
+	c->cur_plugin =
+		plugrack_use_by_type( c->plugin_list, c->switch_type );
+	if ( c->cur_plugin == PLUGIN_INVALID_HANDLE ) {
+		verbose( "can't find a plugin for type %s", c->switch_type );
+		return NULL;
+	}
+
+	/* Dereference the API. */
+	if ( plugin_get_syms( c->cur_plugin,
+				n_syms,
+				syms,
+				(void **) &c->ops ) < n_syms ) {
+		verbose( "incomplete switch plugin detected" );
+		return NULL;
+	}
+
+	return &c->ops;
+}
+
+extern int switch_init( void )
+{
+	int retval = SLURM_SUCCESS;
+	char *switch_type = NULL;
+
+	slurm_mutex_lock( &context_lock );
+
+	if ( g_context )
+		goto done;
+
+	switch_type = slurm_get_switch_type();
+	g_context = _slurm_switch_context_create( switch_type );
+	if ( g_context == NULL ) {
+		error( "cannot create a context for %s", switch_type );
+		retval = SLURM_ERROR;
+		goto done;
+	}
+
+	if ( _slurm_switch_get_ops( g_context ) == NULL ) {
+		error( "cannot resolve plugin operations for %s", switch_type );
+		_slurm_switch_context_destroy( g_context );
+		g_context = NULL;
+		retval = SLURM_ERROR;
+	}
+
+      done:
+	slurm_mutex_unlock( &context_lock );
+	xfree(switch_type);
+	return retval;
+}
+
+extern int  switch_save(char *dir_name)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.state_save))( dir_name );
+}
+
+extern int  switch_restore(char *dir_name)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.state_restore))( dir_name );
+}
+
+extern bool switch_no_frag(void)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.no_frag))( );
+}
+
+extern int  switch_alloc_jobinfo(switch_jobinfo_t *jobinfo)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.alloc_jobinfo))( jobinfo );
+}
+
+extern int  switch_build_jobinfo(switch_jobinfo_t jobinfo, 
+		char *nodelist, int nprocs, int cyclic_alloc)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.build_jobinfo))( jobinfo, nodelist, 
+			nprocs, cyclic_alloc );
+}
+
+extern switch_jobinfo_t switch_copy_jobinfo(switch_jobinfo_t jobinfo)
+{
+	if ( switch_init() < 0 )
+		return NULL;
+
+	return (*(g_context->ops.copy_jobinfo))( jobinfo );
+}
+
+extern void switch_free_jobinfo(switch_jobinfo_t jobinfo)
+{
+	if ( switch_init() < 0 )
+		return;
+
+	(*(g_context->ops.free_jobinfo))( jobinfo );
+}
+
+extern int switch_pack_jobinfo(switch_jobinfo_t jobinfo, Buf buffer)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.pack_jobinfo))( jobinfo, buffer );
+}
+
+extern int switch_unpack_jobinfo(switch_jobinfo_t jobinfo, Buf buffer)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.unpack_jobinfo))( jobinfo, buffer );
+}
+
+extern void switch_print_jobinfo(FILE *fp, switch_jobinfo_t jobinfo)
+{
+	if ( switch_init() < 0 )
+		return;
+
+	(*(g_context->ops.print_jobinfo)) (fp, jobinfo);
+}
+
+extern char *switch_sprint_jobinfo( switch_jobinfo_t jobinfo,
+	       char *buf, size_t size)
+{
+	if ( switch_init() < 0 )
+		return NULL;
+
+	return (*(g_context->ops.string_jobinfo)) (jobinfo, buf, size);
+}
+
+extern int interconnect_node_init(void)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.node_init)) ();
+}
+
+extern int interconnect_node_fini(void)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.node_fini)) ();
+}
+
+extern int interconnect_preinit(switch_jobinfo_t jobinfo)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.job_preinit)) (jobinfo);
+}
+
+extern int interconnect_init(switch_jobinfo_t jobinfo, uid_t uid)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.job_init)) (jobinfo, uid);
+}
+
+extern int interconnect_fini(switch_jobinfo_t jobinfo)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.job_fini)) (jobinfo);
+}
+
+extern int interconnect_postfini(switch_jobinfo_t jobinfo, uid_t pgid, 
+				uint32_t job_id, uint32_t step_id )
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.job_postfini)) (jobinfo, pgid, 
+		job_id, step_id);
+}
+
+extern int interconnect_attach(switch_jobinfo_t jobinfo, char ***env,
+		int nodeid, int procid, int nnodes, int nprocs,
+		gid_t gid)
+{
+	if ( switch_init() < 0 )
+		return SLURM_ERROR;
+
+	return (*(g_context->ops.job_attach)) (jobinfo, env,
+		nodeid, procid, nnodes, nprocs, gid);
+}

src/slurmd/interconnect.h → src/common/switch.h

 /*****************************************************************************\
- *  src/slurmd/interconnect.h - general interconnect routines for slurmd
- *  $Id$
+ * src/common/switch.h - Generic switch (interconnect) info for slurm
  *****************************************************************************
  *  Copyright (C) 2002 The Regents of the University of California.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- *  Written by Kevin Tew <tew1@llnl.gov> 
- *          modified by Mark Grondona <mgrondona@llnl.gov>
+ *  Written by Moe Jette <jette@llnl.gov>.
  *  UCRL-CODE-2002-040.
  *  
  *  This file is part of SLURM, a resource management program.
@@ -26,10 +24,120 @@
  *  59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
 \*****************************************************************************/
 
-#ifndef _INTERCONNECT_H_
-#define _INTERCONNECT_H_
+#ifndef _SWITCH_H
+#define _SWITCH_H 	1
 
-#include "src/slurmd/job.h"
+#if HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "src/common/macros.h"
+#include "src/common/pack.h"
+
+/* opaque data structures - no peeking! */
+#ifndef __switch_jobinfo_t_defined
+#  define __switch_jobinfo_t_defined
+   typedef struct switch_jobinfo *switch_jobinfo_t;
+#endif
+typedef struct slurm_switch_context * slurm_switch_context_t;
+
+/*****************************************\
+ * GLOBAL SWITCH STATE MANGEMENT FUNCIONS*
+\ *****************************************/
+
+/* initialize the switch plugin */
+extern int  switch_init   (void);
+
+/* save any global switch state to a file within the specified directory
+ * the actual file name used in plugin specific
+ * IN dir_name - directory into which switch state is saved
+ * RET         - slurm error code
+ */
+extern int  switch_save   (char *dir_name);
+
+/* restore any global switch state from a file within the specified directory
+ * the actual file name used in plugin specific
+ * IN dir_name - directory from hich switch state is restored or NULL for 
+ *               switch restart with no state restored
+ * RET         - slurm error code
+ */
+extern int  switch_restore(char *dir_name);
+
+/* report if resource fragmentation is important. if so, delay scheduling a 
+ * new job while another is in the process of terminating.
+ * RET          - true if fragmentation is important
+ */
+extern bool switch_no_frag(void);
+
+/******************************************************\
+ * JOB-SPECIFIC SWITCH CREDENTIAL MANAGEMENT FUNCIONS *
+\******************************************************/
+
+/* allocate storage for a switch job credential
+ * OUT jobinfo - storage for a switch job credential
+ * RET         - slurm error code
+ * NOTE: storage must be freed using g_switch_free_jobinfo
+ */
+extern int  switch_alloc_jobinfo (switch_jobinfo_t *jobinfo);
+
+/* fill a job's switch credential
+ * OUT jobinfo  - storage for a switch job credential
+ * IN  nodelist - list of nodes to be used by the job
+ * IN  nprocs   - count of tasks in the job
+ * IN  cyclic_alloc - task distribution pattern, 1=cyclic, 0=block
+ * NOTE: storage must be freed using g_switch_free_jobinfo
+ */
+extern int  switch_build_jobinfo (switch_jobinfo_t jobinfo, 
+		char *nodelist, int nprocs, int cyclic_alloc);
+
+/* copy a switch job credential
+ * IN jobinfo - the switch job credential to be copied
+ * RET        - the copy
+ * NOTE: returned value must be freed using g_switch_free_jobinfo
+ */
+extern switch_jobinfo_t switch_copy_jobinfo(switch_jobinfo_t jobinfo);
+
+/* free storage previously allocated for a switch job credential
+ * IN jobinfo  - the switch job credential to be freed
+ */
+extern void switch_free_jobinfo  (switch_jobinfo_t jobinfo);
+
+/* pack a switch job credential into a buffer in machine independent form
+ * IN jobinfo  - the switch job credential to be saved
+ * OUT buffer  - buffer with switch credential appended
+ * RET         - slurm error code
+ */
+extern int  switch_pack_jobinfo  (switch_jobinfo_t jobinfo, Buf buffer);
+
+/* unpack a switch job credential from a buffer
+ * OUT jobinfo - the switch job credential read
+ * IN  buffer  - buffer with switch credential read from current pointer loc
+ * RET         - slurm error code
+ * NOTE: returned value must be freed using g_switch_free_jobinfo
+ */
+extern int  switch_unpack_jobinfo(switch_jobinfo_t jobinfo, Buf buffer);
+
+/* write job credential string representation to a file
+ * IN fp      - an open file pointer
+ * IN jobinfo - a switch job credential
+ */
+extern void switch_print_jobinfo(FILE *fp, switch_jobinfo_t jobinfo);
+
+/* write job credential to a string
+ * IN jobinfo - a switch job credential
+ * OUT buf    - location to write job credential contents
+ * IN size    - byte size of buf
+ * RET        - the string, same as buf
+ */
+extern char *switch_sprint_jobinfo( switch_jobinfo_t jobinfo,
+			char *buf, size_t size);
+/********************************************************************\
+ * JOB LAUNCH AND MANAGEMENT FUNCTIONS RELATED TO SWITCH CREDENTIAL *
+\********************************************************************/
 
 /*
  * Setup node for interconnect use.
@@ -39,7 +147,7 @@
  * interconnect setup or spawn an error handling thread.
  *
  */
-int interconnect_node_init(void);
+extern int interconnect_node_init(void);
 
 /*
  * Finalize interconnect on node. 
@@ -47,7 +155,7 @@ int interconnect_node_init(void);
  * This function is called once as slurmd exits (slurmd will wait for
  * this function to return before continuing the exit process)
  */
-int interconnect_node_fini(void);
+extern int interconnect_node_fini(void);
 
 
 /*
@@ -69,6 +177,7 @@ int interconnect_node_fini(void);
  *
  * [ *Note: interconnect_fini() is run as the uid of the job owner, not root ]
  */
+
 /*
  * Prepare node for job. 
  *
@@ -77,16 +186,16 @@ int interconnect_node_fini(void);
  * that needs to be performed in the same process as interconnect_fini()
  * 
  */
-int interconnect_preinit(slurmd_job_t *job);
+extern int interconnect_preinit(switch_jobinfo_t jobinfo);
 
 /* 
  * initialize interconnect on node for job. This function is run from the 
  * 2nd slurmd process (some interconnect implementations may require
  * interconnect init functions to be executed from a separate process
- * than the process executing initerconnect_fini() [e.g. QsNet])
+ * than the process executing interconnect_fini() [e.g. QsNet])
  *
  */
-int interconnect_init(slurmd_job_t *job);
+extern int interconnect_init(switch_jobinfo_t jobinfo, uid_t uid);
 
 /*
  * This function is run from the same process as interconnect_init()
@@ -94,7 +203,7 @@ int interconnect_init(slurmd_job_t *job);
  * the process in question has already setuid to the job owner.
  *
  */
-int interconnect_fini(slurmd_job_t *job);
+extern int interconnect_fini(switch_jobinfo_t jobinfo);
 
 /*
  * Finalize interconnect on node.
@@ -104,13 +213,16 @@ int interconnect_fini(slurmd_job_t *job);
  * that need to be run with root privileges should be run from this
  * function.
  */
-int interconnect_postfini(slurmd_job_t *job);
+extern int interconnect_postfini(switch_jobinfo_t jobinfo, uid_t pgid, 
+				uint32_t job_id, uint32_t step_id );
 
 /* 
  * attach process to interconnect
  * (Called from within the process, so it is appropriate to set 
  * interconnect specific environment variables here)
  */
-int interconnect_attach(slurmd_job_t *job, int taskid);
+extern int interconnect_attach(switch_jobinfo_t jobinfo, char ***env,
+		int nodeid, int procid, int nnodes, int nprocs,
+		gid_t gid);
 
-#endif /* _INTERCONNECT_H */
+#endif /* _SWITCH_H */