From fd30bb1b95621114b7e7cb35ea27efd1126b50dd Mon Sep 17 00:00:00 2001
From: "Christopher J. Morrone" <morrone2@llnl.gov>
Date: Fri, 8 Sep 2006 01:41:04 +0000
Subject: [PATCH] sattach is working pretty well now.
MPIR_proctable is now supported.
---
src/common/slurm_protocol_defs.h | 2 +-
src/sattach/attach.c | 2 +-
src/sattach/attach.h | 2 -
src/sattach/opt.c | 39 +++---
src/sattach/opt.h | 3 +-
src/sattach/sattach.c | 198 +++++++++++++++++++++++--------
src/slaunch/opt.c | 16 +--
src/slurmd/slurmd/req.c | 1 -
8 files changed, 177 insertions(+), 86 deletions(-)
diff --git a/src/common/slurm_protocol_defs.h b/src/common/slurm_protocol_defs.h
index cbb08922412..ec3bd56727c 100644
--- a/src/common/slurm_protocol_defs.h
+++ b/src/common/slurm_protocol_defs.h
@@ -505,7 +505,7 @@ typedef struct reattach_tasks_response_msg {
uint32_t return_code;
uint32_t srun_node_id;
uint32_t ntasks; /* number of tasks on this node */
- uint32_t *gtids; /* Global task id assignments */
+ uint32_t *gtids; /* Global task id assignments */
uint32_t *local_pids; /* list of process ids on this node */
} reattach_tasks_response_msg_t;
diff --git a/src/sattach/attach.c b/src/sattach/attach.c
index 0e4910368dc..85e619a148e 100644
--- a/src/sattach/attach.c
+++ b/src/sattach/attach.c
@@ -42,7 +42,7 @@
#include "src/common/log.h"
-#include "src/srun/attach.h"
+#include "src/sattach/attach.h"
/*
* Instantiate extern variables from attach.h
diff --git a/src/sattach/attach.h b/src/sattach/attach.h
index 965f264b39b..7d3c396a90f 100644
--- a/src/sattach/attach.h
+++ b/src/sattach/attach.h
@@ -29,7 +29,6 @@
#define VOLATILE
#endif
#endif
-#include "src/srun/srun_job.h"
/*****************************************************************************
* DEBUGGING SUPPORT *
*****************************************************************************/
@@ -75,7 +74,6 @@ extern int MPIR_i_am_starter;
extern int MPIR_acquired_pre_main;
extern void MPIR_Breakpoint(void);
-extern void debugger_launch_failure(srun_job_t *job);
/* Value for totalview %J expansion in bulk launch string
*/
diff --git a/src/sattach/opt.c b/src/sattach/opt.c
index e503c99c6ca..92962ee05d3 100644
--- a/src/sattach/opt.c
+++ b/src/sattach/opt.c
@@ -73,7 +73,8 @@
#include "src/common/mpi.h"
/* generic getopt_long flags, integers and *not* valid characters */
-#define LONG_OPT_INFO_ONLY 0x100
+#define LONG_OPT_LAYOUT_ONLY 0x100
+#define LONG_OPT_DEBUGGER_TEST 0x101
/*---- global variables, defined in opt.h ----*/
opt_t opt;
@@ -180,7 +181,8 @@ static void _opt_default()
opt.labelio = false;
opt.ctrl_comm_ifhn = xshort_hostname();
memcpy(&opt.fds, &fds, sizeof(fds));
- opt.info_only = false;
+ opt.layout_only = false;
+ opt.debugger_test = false;
}
/*---[ env var processing ]-----------------------------------------------*/
@@ -250,7 +252,8 @@ void set_options(const int argc, char **argv)
{"usage", no_argument, 0, 'u'},
{"verbose", no_argument, 0, 'v'},
{"version", no_argument, 0, 'V'},
- {"info", no_argument, 0, LONG_OPT_INFO_ONLY},
+ {"layout", no_argument, 0, LONG_OPT_LAYOUT_ONLY},
+ {"debugger-test",no_argument, 0, LONG_OPT_DEBUGGER_TEST},
{NULL}
};
char *opt_string = "+hlquvV";
@@ -285,8 +288,11 @@ void set_options(const int argc, char **argv)
_print_version();
exit(0);
break;
- case LONG_OPT_INFO_ONLY:
- opt.info_only = true;
+ case LONG_OPT_LAYOUT_ONLY:
+ opt.layout_only = true;
+ break;
+ case LONG_OPT_DEBUGGER_TEST:
+ opt.debugger_test = true;
break;
default:
fatal("Unrecognized command line parameter %c",
@@ -400,21 +406,14 @@ static void _usage(void)
static void _help(void)
{
- printf("Usage: sattach <jobid.stepid>\n");
+ printf("Usage: sattach [options] <jobid.stepid>\n");
printf(
-" -v, --verbose verbose mode (multiple -v's increase verbosity)\n"
-" -Q, --quiet quiet mode (suppress informational messages)\n"
+" -h, --help print this help message\n"
+" -l, --label prepend task number to lines of stdout & stderr\n"
+" --layout print task layout info and exit (does not attach to tasks)\n"
+" -q, --quiet quiet mode (suppress informational messages)\n"
+" -u, --usage print a brief usage message\n"
+" -v, --verbose verbose mode (multiple -v's increase verbosity)\n"
+" -V, --version print the SLURM version and exit\n"
);
- printf("\n");
-
- printf(
-"Help options:\n"
-" -h, --help show this help message\n"
-" -u, --usage display brief usage message\n"
-"\n"
-"Other options:\n"
-" -V, --version output version information and exit\n"
-"\n"
-);
-
}
diff --git a/src/sattach/opt.h b/src/sattach/opt.h
index c6e97d56ed5..5548e4fe775 100644
--- a/src/sattach/opt.h
+++ b/src/sattach/opt.h
@@ -65,7 +65,8 @@ typedef struct sbatch_options {
char *ctrl_comm_ifhn;
bool labelio;
slurm_step_io_fds_t fds;
- bool info_only;
+ bool layout_only;
+ bool debugger_test;
} opt_t;
extern opt_t opt;
diff --git a/src/sattach/sattach.c b/src/sattach/sattach.c
index da23c60851f..147efb22514 100644
--- a/src/sattach/sattach.c
+++ b/src/sattach/sattach.c
@@ -60,7 +60,11 @@
#include "src/api/step_io.h"
#include "src/sattach/opt.h"
+#include "src/sattach/attach.h"
+static void _mpir_init(int num_tasks);
+static void _mpir_cleanup(void);
+static void _mpir_dump_proctable(void);
static void print_layout_info(slurm_step_layout_t *layout);
static slurm_cred_t _generate_fake_cred(uint32_t jobid, uint32_t stepid,
uid_t uid, char *nodelist);
@@ -93,7 +97,7 @@ typedef struct message_thread_state {
uint16_t *resp_port; /* array of message response ports */
} message_thread_state_t;
static message_thread_state_t *_msg_thr_create(int num_nodes, int num_tasks);
-static void _msg_thr_destroy(message_thread_state_t *mts);
+static void _msg_thr_wait_and_destroy(message_thread_state_t *mts);
static void _handle_msg(message_thread_state_t *mts, slurm_msg_t *msg);
static bool _message_socket_readable(eio_obj_t *obj);
static int _message_socket_accept(eio_obj_t *obj, List objs);
@@ -115,7 +119,7 @@ int main(int argc, char *argv[])
log_init(xbasename(argv[0]), logopt, 0, NULL);
if (initialize_and_process_args(argc, argv) < 0) {
- fatal("salloc parameter parsing");
+ fatal("sattach parameter parsing");
}
/* reinit log with new verbosity (if changed by command line) */
if (opt.verbose || opt.quiet) {
@@ -130,11 +134,15 @@ int main(int argc, char *argv[])
error("Could not get job step info: %m");
return 1;
}
- if (opt.info_only) {
+ if (opt.layout_only) {
print_layout_info(layout);
exit(0);
}
+ totalview_jobid = NULL;
+ xstrfmtcat(totalview_jobid, "%u", opt.jobid);
+ _mpir_init(layout->task_cnt);
+
fake_cred = _generate_fake_cred(opt.jobid, opt.stepid,
opt.uid, layout->node_list);
@@ -149,11 +157,17 @@ int main(int argc, char *argv[])
mts->num_resp_port, mts->resp_port,
io->num_listen, io->listenport);
- _msg_thr_destroy(mts);
+ MPIR_debug_state = MPIR_DEBUG_SPAWNED;
+ MPIR_Breakpoint();
+ if (opt.debugger_test)
+ _mpir_dump_proctable();
+ _msg_thr_wait_and_destroy(mts);
slurm_job_step_layout_free(layout);
client_io_handler_finish(io);
client_io_handler_destroy(io);
+ _mpir_cleanup();
+
return 0;
}
@@ -278,8 +292,8 @@ static uint32_t *_create_range_array(uint32_t first, uint32_t last)
uint32_t i, current, len, *array;
if (first > last) {
- error("_create_range_array, \"first\""
- " cannot be greater than \"last\"");
+ error("_create_range_array, \"first\" (%d)"
+ " cannot be greater than \"last\" (%d)", first, last);
return NULL;
}
@@ -292,6 +306,71 @@ static uint32_t *_create_range_array(uint32_t first, uint32_t last)
return array;
}
+void _handle_response_msg(slurm_msg_type_t msg_type, void *msg)
+{
+ reattach_tasks_response_msg_t *resp;
+ MPIR_PROCDESC *table;
+ int i;
+
+ switch(msg_type) {
+ case RESPONSE_REATTACH_TASKS:
+ resp = (reattach_tasks_response_msg_t *)msg;
+ if (resp->return_code != SLURM_SUCCESS) {
+ info("Node %s: no tasks running", resp->node_name);
+ break;
+ }
+
+ debug("Node %s (%u), executable %s, %d tasks",
+ resp->node_name, resp->srun_node_id,
+ resp->executable_name, resp->ntasks);
+ for (i = 0; i < resp->ntasks; i++) {
+ table = &MPIR_proctable[resp->gtids[i]];
+ /* FIXME - node_name is not necessarily
+ a valid hostname */
+ table->host_name = xstrdup(resp->node_name);
+ table->executable_name = xstrdup(resp->executable_name);
+ table->pid = (int)resp->local_pids[i];
+ debug("\tTask id %u has pid %u",
+ resp->gtids[i], resp->local_pids[i]);
+ }
+ break;
+ default:
+ error("Unrecognized response to REQUEST_REATTACH_TASKS: %d",
+ msg_type);
+ break;
+ }
+}
+
+void _handle_response_msg_list(List other_nodes_resp)
+{
+ ListIterator itr1, itr2;
+ ret_types_t *ret;
+ ret_data_info_t *ret_msg_wrapper;
+
+ itr1 = list_iterator_create(other_nodes_resp);
+ while ((ret = list_next(itr1)) != NULL) {
+ debug("Attach returned msg_rc=%d err=%d type=%d",
+ ret->msg_rc, ret->err, ret->type);
+ if (ret->msg_rc != SLURM_SUCCESS) {
+ itr2 = list_iterator_create(ret->ret_data_list);
+ while ((ret_msg_wrapper = list_next(itr2)) != NULL) {
+ errno = ret->err;
+ _handle_response_msg(ret->type,
+ ret_msg_wrapper->data);
+ }
+ list_iterator_destroy(itr2);
+ } else {
+ itr2 = list_iterator_create(ret->ret_data_list);
+ while ((ret_msg_wrapper = list_next(itr2)) != NULL) {
+ _handle_response_msg(ret->type,
+ ret_msg_wrapper->data);
+ }
+ list_iterator_destroy(itr2);
+ }
+ }
+
+ list_iterator_destroy(itr1);
+}
static int _attach_to_tasks(uint32_t jobid,
uint32_t stepid,
@@ -303,11 +382,7 @@ static int _attach_to_tasks(uint32_t jobid,
uint16_t *io_ports)
{
slurm_msg_t msg, first_node_resp;
- List ret_list = NULL;
- ListIterator ret_itr;
- ListIterator ret_data_itr;
- ret_types_t *ret;
- ret_data_info_t *ret_data;
+ List other_nodes_resp = NULL;
int timeout;
reattach_tasks_request_msg_t reattach_msg;
@@ -327,55 +402,39 @@ static int _attach_to_tasks(uint32_t jobid,
msg.msg_type = REQUEST_REATTACH_TASKS;
msg.data = &reattach_msg;
msg.srun_node_id = 0;
-
+
msg.forward.cnt = layout->node_cnt - 1;
- msg.forward.node_id = _create_range_array(1, layout->node_cnt-1);
- msg.forward.name = _create_ugly_nodename_string(layout->node_list,
- layout->node_cnt-1);
- msg.forward.addr = layout->node_addr + 1;
- msg.forward.timeout = timeout;
- memcpy(&msg.address, layout->node_addr + 0, sizeof(slurm_addr));
-
- ret_list = slurm_send_recv_node_msg(&msg, &first_node_resp, timeout);
- if (ret_list == NULL) {
+ if (layout->node_cnt > 1) {
+ msg.forward.node_id =
+ _create_range_array(1, layout->node_cnt-1);
+ msg.forward.name = _create_ugly_nodename_string(
+ layout->node_list, layout->node_cnt-1);
+ msg.forward.addr = layout->node_addr + 1;
+ msg.forward.timeout = timeout;
+ }
+
+ memcpy(&msg.address, layout->node_addr + 0,
+ sizeof(slurm_addr));
+
+ other_nodes_resp = slurm_send_recv_node_msg(&msg, &first_node_resp,
+ timeout);
+ if (other_nodes_resp == NULL) {
error("slurm_send_recv_node_msg failed: %m");
xfree(msg.forward.node_id);
xfree(msg.forward.name);
return SLURM_ERROR;
}
- ret_itr = list_iterator_create(ret_list);
- while ((ret = list_next(ret_itr)) != NULL) {
- debug("Attach returned msg_rc=%d err=%d type=%d",
- ret->msg_rc, ret->err, ret->type);
- if (ret->msg_rc != SLURM_SUCCESS) {
- ret_data_itr =
- list_iterator_create(ret->ret_data_list);
- while ((ret_data = list_next(ret_data_itr)) != NULL) {
- errno = ret->err;
- error("Attach failed on node %s(%d): %m",
- ret_data->node_name, ret_data->nodeid);
- }
- list_iterator_destroy(ret_data_itr);
- } else {
- ret_data_itr =
- list_iterator_create(ret->ret_data_list);
- while ((ret_data = list_next(ret_data_itr)) != NULL) {
- errno = ret->err;
- info("Attach success on node %s(%d)",
- ret_data->node_name, ret_data->nodeid);
- }
- list_iterator_destroy(ret_data_itr);
- }
- }
+ _handle_response_msg(first_node_resp.msg_type, first_node_resp.data);
+ _handle_response_msg_list(other_nodes_resp);
+
+ list_destroy(other_nodes_resp);
xfree(msg.forward.node_id);
xfree(msg.forward.name);
- list_iterator_destroy(ret_itr);
- list_destroy(ret_list);
+
return SLURM_SUCCESS;
}
-
/**********************************************************************
* Message handler functions
**********************************************************************/
@@ -438,7 +497,7 @@ fail:
return NULL;
}
-static void _msg_thr_destroy(message_thread_state_t *mts)
+static void _msg_thr_wait_and_destroy(message_thread_state_t *mts)
{
eio_signal_shutdown(mts->msg_handle);
pthread_join(mts->msg_thread, NULL);
@@ -636,3 +695,44 @@ _handle_msg(message_thread_state_t *mts, slurm_msg_t *msg)
}
return;
}
+
+/**********************************************************************
+ * Functions for manipulating the MPIR_* global variables which
+ * are accessed by parallel debuggers which trace slaunch.
+ **********************************************************************/
+static void
+_mpir_init(int num_tasks)
+{
+ MPIR_proctable_size = num_tasks;
+ MPIR_proctable = xmalloc(sizeof(MPIR_PROCDESC) * num_tasks);
+ if (MPIR_proctable == NULL)
+ fatal("Unable to initialize MPIR_proctable: %m");
+}
+
+static void
+_mpir_cleanup()
+{
+ int i;
+
+ for (i = 0; i < MPIR_proctable_size; i++) {
+ xfree(MPIR_proctable[i].host_name);
+ xfree(MPIR_proctable[i].executable_name);
+ }
+ xfree(MPIR_proctable);
+}
+
+static void
+_mpir_dump_proctable()
+{
+ MPIR_PROCDESC *tv;
+ int i;
+
+ for (i = 0; i < MPIR_proctable_size; i++) {
+ tv = &MPIR_proctable[i];
+ if (!tv)
+ break;
+ info("task:%d, host:%s, pid:%d, executable:%s",
+ i, tv->host_name, tv->pid, tv->executable_name);
+ }
+}
+
diff --git a/src/slaunch/opt.c b/src/slaunch/opt.c
index 2d478e7b1aa..fce886844bd 100644
--- a/src/slaunch/opt.c
+++ b/src/slaunch/opt.c
@@ -159,7 +159,7 @@ static void _print_version(void);
static void _process_env_var(env_vars_t *e, const char *val);
/* search PATH for command returns full path */
-static char *_search_path(char *, bool, int);
+static char *_search_path(char *, int);
static bool _under_parallel_debugger(void);
@@ -1454,11 +1454,8 @@ static void _opt_args(int argc, char **argv)
}
else if (opt.argc > 0) {
char *fullpath;
- char *cmd = opt.argv[0];
- bool search_cwd = false; /* was: (opt.batch || opt.allocate); */
- int mode = (search_cwd) ? R_OK : R_OK | X_OK;
- if ((fullpath = _search_path(cmd, search_cwd, mode))) {
+ if ((fullpath = _search_path(opt.argv[0], R_OK|X_OK))) {
xfree(opt.argv[0]);
opt.argv[0] = fullpath;
}
@@ -1850,7 +1847,7 @@ _create_path_list(void)
}
static char *
-_search_path(char *cmd, bool check_current_dir, int access_mode)
+_search_path(char *cmd, int access_mode)
{
List l = _create_path_list();
ListIterator i = NULL;
@@ -1859,17 +1856,14 @@ _search_path(char *cmd, bool check_current_dir, int access_mode)
if (l == NULL)
return NULL;
- if ( (cmd[0] == '.' || cmd[0] == '/')
- && (access(cmd, access_mode) == 0 ) ) {
+ if ((cmd[0] == '.' || cmd[0] == '/')
+ && (access(cmd, access_mode) == 0 ) ) {
if (cmd[0] == '.')
xstrfmtcat(fullpath, "%s/", opt.cwd);
xstrcat(fullpath, cmd);
goto done;
}
- if (check_current_dir)
- list_prepend(l, xstrdup(opt.cwd));
-
i = list_iterator_create(l);
while ((path = list_next(i))) {
xstrfmtcat(fullpath, "%s/%s", path, cmd);
diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index c14abdc2903..bea5ab2619b 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -1501,7 +1501,6 @@ done:
debug2("node %s (id %u) sending rc = %d", conf->node_name, nodeid, rc);
slurm_send_node_msg(msg->conn_fd, &resp_msg);
- //slurm_send_only_node_msg(&resp_msg);
slurm_free_reattach_tasks_response_msg(resp);
}
--
GitLab