From 5d663b8775471193a253eb83f30c4bcbe62271b1 Mon Sep 17 00:00:00 2001
From: David Bigagli <david@schedmd.com>
Date: Fri, 2 Aug 2013 13:06:42 -0700
Subject: [PATCH] Fix for bug377. Restructure sh5util a bit try to follow SLURM
coding standard.
---
doc/man/man1/sh5util.1 | 6 +-
.../acct_gather_profile/hdf5/hdf5_api.c | 2 +-
.../hdf5/sh5util/sh5util.c | 880 ++++++++++--------
3 files changed, 488 insertions(+), 400 deletions(-)
diff --git a/doc/man/man1/sh5util.1 b/doc/man/man1/sh5util.1
index a17b8dd58c6..05247c48ff4 100644
--- a/doc/man/man1/sh5util.1
+++ b/doc/man/man1/sh5util.1
@@ -34,8 +34,7 @@ sample
.TP
\fB\-E\fR, \fB\-\-extract\fR
-Instead of merging node-step files into a job file (default operation)
-extract data series from a job file.
+Extract data series from a merged job file.
.RS
.TP 10
@@ -62,8 +61,7 @@ Level to which series is attached. (default Node:Totals)
.TP
\fB\-I\fR, \fB\-\-item\-extract\fR
-Instead of merging node-step files into a job file extract one data item from
-all samples of one data series from all nodes in a job file.
+Extract one data item from all samples of one data series from all nodes in a merged job file.
.RS
.TP 10
diff --git a/src/plugins/acct_gather_profile/hdf5/hdf5_api.c b/src/plugins/acct_gather_profile/hdf5/hdf5_api.c
index 013d6dd8030..25f0580fa59 100644
--- a/src/plugins/acct_gather_profile/hdf5/hdf5_api.c
+++ b/src/plugins/acct_gather_profile/hdf5/hdf5_api.c
@@ -1530,7 +1530,7 @@ extern void profile_init(void)
extern void profile_fini(void)
{
H5Tclose(typTOD);
- H5close(); // make sure all H5 Objects are closed
+ H5close(); /* make sure all H5 Objects are closed */
return;
}
diff --git a/src/plugins/acct_gather_profile/hdf5/sh5util/sh5util.c b/src/plugins/acct_gather_profile/hdf5/sh5util/sh5util.c
index a2bb167ddfc..7e69a2b6a5c 100644
--- a/src/plugins/acct_gather_profile/hdf5/sh5util/sh5util.c
+++ b/src/plugins/acct_gather_profile/hdf5/sh5util/sh5util.c
@@ -93,62 +93,132 @@ typedef struct {
char *user;
int verbose;
} sh5util_opts_t;
-// ============================================================================
-// Options
+
static sh5util_opts_t params;
-static char** series_names = NULL;
-static int num_series = 0;
+static char **series_names;
+static int num_series;
+
+static int _set_options(const int argc, char **argv);
+static int _merge_step_files(void);
+static int _extract_data(void);
+static int _series_data(void);
+static int _check_params(void);
+static void _free_options(void);
+static void _remove_empty_output(void);
static void _help_msg(void)
{
printf("\
-sh5util [<OPTION>] -j <job[.stepid]> \n\
- Valid <OPTION> values are: \n\
- -E, --extract: \n\
- Instead of merge node-step files (default) Extract data \n\
- series from job file. \n\
- Extract mode options \n\
- -i, --input: merged file to extract from \n\
- (default ./job_$jobid.h5) \n\
- -N --node: Node name to extract (default is all) \n\
- -l, --level: Level to which series is attached \n\
- [Node:Totals|Node:TimeSeries] \n\
- (default Node:Totals) \n\
- -s, --series: Name of series \n\
- Energy | Lustre | Network | Tasks | Task_# \n\
- 'Tasks' is all tasks, Task_# is task_id \n\
- (default is everything) \n\
- -I, --item-extract: \n\
- Instead of merge node-step files (default) Extract data \n\
- item from one series from all samples on all nodes \n\
- from the job file. \n\
- Item-Extract mode options \n\
- -i, --input: merged file to extract from \n\
- (default ./job_$jobid.h5) \n\
- -s, --series: Name of series \n\
- Energy | Lustre | Network | Task \n\
- -d, --data: Name of data item in series (see man page) \n\
- -j, --jobs: Format is <job(.step)>. Merge this job/step. \n\
- or comma-separated list of job steps. This option is \n\
- required. Not specifying a step will result in all \n\
- steps found to be processed. \n\
- -h, --help: Print this description of use. \n\
- -o, --output: Path to a file into which to write. \n\
- Default for merge is ./job_$jobid.h5 \n\
- Default for extract is ./extract_$jobid.csv \n\
- -p, --profiledir: \n\
- Profile directory location where node-step files exist \n\
- default is what is set in acct_gather.conf \n\
- -S, --savefiles: \n\
- Instead of remove node-step files after merge keep them \n\
- around. \n\
- --user: User who profiled job. (Handy for root user, defaults to \n\
- user running this command.) \n\
- --usage: Display brief usage message. \n\
-\n");
+Usage sh5util [<OPTION>] -j <job[.stepid]>\n"
+"\n"
+"Valid <OPTION> values are:\n"
+" -E, --extract Extract data series from job file.\n"
+" -i, --input merged file to extract from (default ./job_$jobid.h5)\n"
+" -N, --node Node name to extract (default is all)\n"
+" -l, --level Level to which series is attached\n"
+" [Node:Totals|Node:TimeSeries] (default Node:Totals)\n"
+" -s, --series Name of series:\n"
+" Energy | Lustre | Network | Tasks | Task_#\n"
+" 'Tasks' is all tasks, Task_# is task_id (default is all)\n"
+" -I, --item-extract Extract data item from one series from \n"
+" all samples on all nodes from thejob file.\n"
+" -i, --input merged file to extract from (default ./job_$jobid.h5)\n"
+" -s, --series Name of series:\n"
+" Energy | Lustre | Network | Task\n"
+" -d, --data Name of data item in series (see man page) \n"
+" -j, --jobs Format is <job(.step)>. Merge this job/step.\n"
+" or comma-separated list of job steps. This option is\n"
+" required. Not specifying a step will result in all\n"
+" steps found to be processed.\n"
+" -h, --help Print this description of use.\n"
+" -o, --output Path to a file into which to write.\n"
+" Default for merge is ./job_$jobid.h5\n"
+" Default for extract is ./extract_$jobid.csv\n"
+" -p, --profiledir Profile directory location where node-step files exist\n"
+" default is what is set in acct_gather.conf\n"
+" -S, --savefiles Don't remove node-step files after merging them \n"
+" --user User who profiled job. (Handy for root user, defaults to \n"
+" user running this command.)\n"
+" --usage Display brief usage message\n");
}
+int
+main(int argc, char **argv)
+{
+ int cc;
+
+ cc = _set_options(argc, argv);
+ if (cc < 0)
+ goto ouch;
+
+ cc = _check_params();
+ if (cc < 0)
+ goto ouch;
+
+ profile_init();
+
+ switch (params.mode) {
+
+ case SH5UTIL_MODE_MERGE:
+
+ info("Merging node-step files into %s",
+ params.output);
+ cc = _merge_step_files();
+ if (cc < 0)
+ goto ouch;
+ break;
+
+ case SH5UTIL_MODE_EXTRACT:
+
+ info("Extracting job data from %s into %s",
+ params.input, params.output);
+ cc = _extract_data();
+ if (cc < 0)
+ goto ouch;
+ break;
+
+ case SH5UTIL_MODE_ITEM_EXTRACT:
+
+ info("Extracting '%s' from '%s' data from %s into %s",
+ params.data_item, params.series,
+ params.input, params.output);
+ cc = _series_data();
+ if (cc < 0)
+ goto ouch;
+ break;
+
+ default:
+ error("Unknown type %d", params.mode);
+ break;
+ }
+
+ _remove_empty_output();
+ profile_fini();
+ _free_options();
+
+ return 0;
+
+ouch:
+ _remove_empty_output();
+ _free_options();
+
+ return -1;
+}
+
+/* _free_options()
+ */
+static void
+_free_options(void)
+{
+ xfree(params.dir);
+ xfree(params.input);
+ xfree(params.node);
+ xfree(params.output);
+ xfree(params.series);
+ xfree(params.data_item);
+ xfree(params.user);
+}
/*
* delete list of strings
*
@@ -156,34 +226,41 @@ sh5util [<OPTION>] -j <job[.stepid]> \n\
* list - xmalloc'd list of pointers of xmalloc'd strings.
* listlen - number of strings in the list
*/
-static void _delete_string_list(char** list, int listLen)
+static void _delete_string_list(char **list, int listLen)
{
int ix;
+
if (list == NULL)
return;
- for (ix=0; ix<listLen; ix++) {
+
+ for (ix = 0; ix < listLen; ix++) {
xfree(list[ix]);
}
+
xfree(list);
- return;
+
}
-static void _remove_empty_outout()
+static void _remove_empty_output(void)
{
struct stat sb;
if (stat(params.output, &sb) == -1) {
- error("Failed to stat %s: %m",params.output);
- exit(1);
- }
- if (sb.st_size == 0) {
- error("No data in %s (it has been removed)", params.output);
- remove(params.output);
+ /* Ignore the error as the file may have
+ * not been created yet.
+ */
+ return;
}
+ /* Remove the file if 0 size which means
+ * the program failed somewhere along the
+ * way and the file is just left hanging...
+ */
+ if (sb.st_size == 0)
+ remove(params.output);
}
-static void _init_opts()
+static void _init_opts(void)
{
memset(¶ms, 0, sizeof(sh5util_opts_t));
params.job_id = -1;
@@ -191,12 +268,13 @@ static void _init_opts()
params.step_id = -1;
}
-static void _set_options(const int argc, char **argv)
+static int _set_options(const int argc, char **argv)
{
- int option_index = 0, c;
+ int option_index = 0;
+ int cc;
log_options_t logopt = LOG_OPTS_STDERR_ONLY;
char *next_str = NULL;
- uid_t uid = -1;
+ uid_t u;
static struct option long_options[] = {
{"extract", no_argument, 0, 'E'},
@@ -221,119 +299,106 @@ static void _set_options(const int argc, char **argv)
_init_opts();
- while (1) { /* now cycle through the command line */
- c = getopt_long(argc, argv, "d:Ehi:Ij:l:N:o:p:s:Su:vV",
- long_options, &option_index);
- if (c == -1)
- break;
- switch (c) {
- case 'd':
- params.data_item = xstrdup(optarg);
- params.data_item = xstrtolower(params.data_item);
- break;
- case 'E':
- params.mode = SH5UTIL_MODE_EXTRACT;
- break;
- case 'I':
- params.mode = SH5UTIL_MODE_ITEM_EXTRACT;
- break;
- case 'h':
- params.help = 1;
- break;
- case 'i':
- params.input = xstrdup(optarg);
- break;
- case 'j':
- params.job_id = strtol(optarg, &next_str, 10);
- if (next_str[0] == '.')
- params.step_id =
- strtol(next_str+1, NULL, 10);
- break;
- case 'l':
- params.level = xstrdup(optarg);
- break;
- case 'N':
- params.node = xstrdup(optarg);
- break;
- case 'o':
- params.output = xstrdup(optarg);
- break;
- case 'p':
- params.dir = xstrdup(optarg);
- break;
- case 's':
- if (strcmp(optarg,GRP_ENERGY)
- && strcmp(optarg,GRP_LUSTRE)
- && strcmp(optarg,GRP_NETWORK)
- && strncmp(optarg,GRP_TASK,strlen(GRP_TASK))) {
- error("--series=\"%s\" invalid", optarg);
- exit(1);
- }
- params.series = xstrdup(optarg);
- break;
- case 'S':
- params.keepfiles = 1;
- break;
- case 'u':
- if (uid_from_string (optarg, &uid) < 0) {
- error("--uid=\"%s\" invalid", optarg);
- exit(1);
- }
- break;
- case (int)'v':
- params.verbose++;
- break;
- case (int)'V':
- print_slurm_version();
- exit(0);
- break;
- case ':':
- case '?': /* getopt() has explained it */
- exit(1);
+ while ((cc = getopt_long(argc, argv, "d:Ehi:Ij:l:N:o:p:s:Su:vV",
+ long_options, &option_index)) != EOF) {
+ switch (cc) {
+ case 'd':
+ params.data_item = xstrdup(optarg);
+ params.data_item = xstrtolower(params.data_item);
+ break;
+ case 'E':
+ params.mode = SH5UTIL_MODE_EXTRACT;
+ break;
+ case 'I':
+ params.mode = SH5UTIL_MODE_ITEM_EXTRACT;
+ break;
+ case 'h':
+ _help_msg();
+ return -1;
+ break;
+ case 'i':
+ params.input = xstrdup(optarg);
+ break;
+ case 'j':
+ params.job_id = strtol(optarg, &next_str, 10);
+ if (next_str[0] == '.')
+ params.step_id =
+ strtol(next_str + 1, NULL, 10);
+ break;
+ case 'l':
+ params.level = xstrdup(optarg);
+ break;
+ case 'N':
+ params.node = xstrdup(optarg);
+ break;
+ case 'o':
+ params.output = xstrdup(optarg);
+ break;
+ case 'p':
+ params.dir = xstrdup(optarg);
+ break;
+ case 's':
+ if (strcmp(optarg, GRP_ENERGY)
+ && strcmp(optarg, GRP_LUSTRE)
+ && strcmp(optarg, GRP_NETWORK)
+ && strncmp(optarg,GRP_TASK,strlen(GRP_TASK))) {
+ error("Bad value for --series=\"%s\"", optarg);
+ return -1;
+ }
+ params.series = xstrdup(optarg);
+ break;
+ case 'S':
+ params.keepfiles = 1;
+ break;
+ case 'u':
+ u = atoi(optarg);
+ if (uid_from_string(optarg, &u) < 0) {
+ error("No such user --uid=\"%s\"", optarg);
+ return -1;
+ }
+ break;
+ case 'v':
+ params.verbose++;
+ break;
+ case 'V':
+ print_slurm_version();
+ return -1;
+ break;
+ case ':':
+ case '?': /* getopt() has explained it */
+ return -1;
}
}
- if (params.help) {
- switch (params.help) {
- case 1:
- case 3:
- _help_msg();
- break;
- default:
- fprintf(stderr, "bug: --help=%d\n",
- params.help);
- }
- exit(0);
+ if (params.verbose) {
+ logopt.stderr_level += params.verbose;
+ log_alter(logopt, SYSLOG_FACILITY_USER, NULL);
}
- if (params.job_id == -1)
- fatal("You need to supply a --jobs value.");
+ return 0;
+}
- if (uid == -1)
- uid = getuid();
+/* _check_params()
+ */
+static int
+_check_params(void)
+{
+ if (params.job_id == -1) {
+ error("JobID must be specified.");
+ return -1;
+ }
- params.user = uid_to_string(uid);
+ if (params.user == NULL)
+ params.user = uid_to_string(getuid());
if (!params.dir)
acct_gather_profile_g_get(ACCT_GATHER_PROFILE_DIR, ¶ms.dir);
- if (!params.dir)
- fatal("You need to supply a --profiledir or be on a "
- "node with a valid acct_gather.conf");
-
- if (params.verbose) {
- logopt.stderr_level += params.verbose;
- log_alter(logopt, SYSLOG_FACILITY_USER, NULL);
+ if (!params.dir) {
+ error("Cannot read/parse acct_gather.conf");
+ return -1;
}
- /* FIXME : For now all these only work for extract. Seems
- * like it would be easy to add the logic to "merge" as well.
- */
- if (params.level || params.node || params.series)
- params.mode = SH5UTIL_MODE_EXTRACT;
- if (params.data_item)
- params.mode = SH5UTIL_MODE_ITEM_EXTRACT;
-
if (params.mode == SH5UTIL_MODE_EXTRACT) {
if (!params.level)
params.level = xstrdup("Node:Totals");
@@ -347,25 +412,25 @@ static void _set_options(const int argc, char **argv)
}
if (params.mode == SH5UTIL_MODE_ITEM_EXTRACT) {
if (!params.data_item)
- fatal("You need to supply a --data name ");
+ fatal("Must specify data option --data ");
if (!params.series)
- fatal("You need to supply a --series name ");
+ fatal("Must specify series option --series");
if (!params.input)
- params.input = xstrdup_printf(
- "./job_%d.h5", params.job_id);
+ params.input = xstrdup_printf("./job_%d.h5", params.job_id);
if (!params.output)
params.output = xstrdup_printf("./%s_%s_%d.csv",
- params.series,
- params.data_item,
- params.job_id);
+ params.series,
+ params.data_item,
+ params.job_id);
}
if (!params.output)
params.output = xstrdup_printf("./job_%d.h5", params.job_id);
+ return 0;
}
/* ============================================================================
@@ -375,7 +440,7 @@ static void _set_options(const int argc, char **argv)
* ========================================================================= */
static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
- int nsamples)
+ int nsamples)
{
void* data = NULL;
@@ -403,13 +468,13 @@ static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
xfree(ops);
xfree(data);
error("Failed to create %s memory datatype",
- acct_gather_profile_type_to_string(type));
+ acct_gather_profile_type_to_string(type));
return NULL;
}
for (smpx=0; smpx<nsamples; smpx++) {
len = H5Lget_name_by_idx(gid_series, ".", H5_INDEX_NAME,
- H5_ITER_INC, smpx, name_sample,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, smpx, name_sample,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if (len<1 || len>MAX_GROUP_NAME) {
error("Invalid group name %s", name_sample);
continue;
@@ -419,11 +484,11 @@ static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
info("Failed to open %s", name_sample);
}
id_data_set = H5Dopen(g_sample, get_data_set_name(name_sample),
- H5P_DEFAULT);
+ H5P_DEFAULT);
if (id_data_set < 0) {
H5Gclose(g_sample);
error("Failed to open %s dataset",
- acct_gather_profile_type_to_string(type));
+ acct_gather_profile_type_to_string(type));
continue;
}
sz_dest = (*(ops->dataset_size))();
@@ -435,7 +500,7 @@ static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
continue;
}
ec = H5Dread(id_data_set, dtyp_memory, H5S_ALL, H5S_ALL,
- H5P_DEFAULT, data_cur);
+ H5P_DEFAULT, data_cur);
if (ec < 0) {
xfree(data_cur);
H5Dclose(id_data_set);
@@ -445,7 +510,7 @@ static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
continue;
}
(*(ops->merge_step_series))(g_sample, data_prior, data_cur,
- data+(smpx)*sz_dest);
+ data+(smpx)*sz_dest);
xfree(data_prior);
data_prior = data_cur;
@@ -461,9 +526,9 @@ static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
static void _merge_series_data(hid_t jgid_tasks, hid_t jg_node, hid_t nsg_node)
{
- hid_t jg_samples, nsg_samples;
+ hid_t jg_samples, nsg_samples;
hid_t g_series, g_series_total = -1;
- hsize_t num_samples, n_series;
+ hsize_t num_samples, n_series;
int idsx, len;
void *data = NULL, *series_total = NULL;
uint32_t type;
@@ -483,7 +548,7 @@ static void _merge_series_data(hid_t jgid_tasks, hid_t jg_node, hid_t nsg_node)
}
jg_samples = H5Gcreate(jg_node, GRP_SAMPLES,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (jg_samples < 0) {
info("Failed to create job node Samples");
return;
@@ -505,13 +570,13 @@ static void _merge_series_data(hid_t jgid_tasks, hid_t jg_node, hid_t nsg_node)
}
for (idsx = 0; idsx < n_series; idsx++) {
H5Oget_info_by_idx(nsg_samples, ".", H5_INDEX_NAME, H5_ITER_INC,
- idsx, &object_info, H5P_DEFAULT);
+ idsx, &object_info, H5P_DEFAULT);
if (object_info.type != H5O_TYPE_GROUP)
continue;
len = H5Lget_name_by_idx(nsg_samples, ".", H5_INDEX_NAME,
- H5_ITER_INC, idsx, nam_series,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, idsx, nam_series,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if (len<1 || len>MAX_GROUP_NAME) {
info("Invalid group name %s", nam_series);
continue;
@@ -538,14 +603,14 @@ static void _merge_series_data(hid_t jgid_tasks, hid_t jg_node, hid_t nsg_node)
type = acct_gather_profile_type_from_string(data_type);
xfree(data_type);
data = _get_all_samples(g_series, nam_series, type,
- num_samples);
+ num_samples);
if (data == NULL) {
H5Gclose(g_series);
info("Failed to get memory for Time Series Dataset");
continue;
}
put_hdf5_data(jg_samples, type, SUBDATA_SERIES, nam_series,
- data, num_samples);
+ data, num_samples);
ops = profile_factory(type);
if (ops == NULL) {
xfree(data);
@@ -567,8 +632,8 @@ static void _merge_series_data(hid_t jgid_tasks, hid_t jg_node, hid_t nsg_node)
continue;
}
put_hdf5_data(g_series_total, type,
- SUBDATA_SUMMARY,
- nam_series, series_total, 1);
+ SUBDATA_SUMMARY,
+ nam_series, series_total, 1);
H5Gclose(g_series_total);
}
xfree(series_total);
@@ -616,8 +681,8 @@ static void _merge_task_totals(hid_t jg_tasks, hid_t nsg_node, char* node_name)
for (taskx = 0; ((int)ntasks>0) && (taskx<((int)ntasks)); taskx++) {
// Get the name of the group.
len = H5Lget_name_by_idx(nsg_tasks, ".", H5_INDEX_NAME,
- H5_ITER_INC, taskx, buf,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, taskx, buf,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if (len<1 || len>MAX_GROUP_NAME) {
info("Invalid group name %s", buf);
continue;
@@ -630,7 +695,7 @@ static void _merge_task_totals(hid_t jg_tasks, hid_t nsg_node, char* node_name)
taskid = get_int_attribute(nsg_task, ATTR_TASKID);
sprintf(group_name, "%s_%d", GRP_TASK, taskid);
jg_task = H5Gcreate(jg_tasks, group_name,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (jg_task < 0) {
H5Gclose(nsg_task);
info("Failed to create job task group");
@@ -647,7 +712,7 @@ static void _merge_task_totals(hid_t jg_tasks, hid_t nsg_node, char* node_name)
continue;
}
jg_totals = H5Gcreate(jg_task, GRP_TOTALS,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (jg_totals < 0) {
H5Gclose(jg_task);
H5Gclose(nsg_task);
@@ -659,8 +724,8 @@ static void _merge_task_totals(hid_t jg_tasks, hid_t nsg_node, char* node_name)
for (i = 0; (nobj>0) && (i<nobj); i++) {
// Get the name of the group.
len = H5Lget_name_by_idx(nsg_totals, ".", H5_INDEX_NAME,
- H5_ITER_INC, i, buf,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, i, buf,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if (len<1 || len>MAX_GROUP_NAME) {
info("Invalid group name %s", buf);
@@ -685,7 +750,7 @@ static void _merge_task_totals(hid_t jg_tasks, hid_t nsg_node, char* node_name)
continue;
}
put_hdf5_data(jg_totals, type, SUBDATA_DATA,
- buf, data, 1);
+ buf, data, 1);
xfree(data);
H5Gclose(g_total);
}
@@ -720,7 +785,7 @@ static void _merge_node_totals(hid_t jg_node, hid_t nsg_node)
return;
}
jg_totals = H5Gcreate(jg_node, GRP_TOTALS,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (jg_totals < 0) {
info("Failed to create job node totals");
return;
@@ -736,8 +801,8 @@ static void _merge_node_totals(hid_t jg_node, hid_t nsg_node)
for (i = 0; (nobj>0) && (i<nobj); i++) {
// Get the name of the group.
len = H5Lget_name_by_idx(nsg_totals, ".", H5_INDEX_NAME,
- H5_ITER_INC, i, buf,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, i, buf,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if (len<1 || len>MAX_GROUP_NAME) {
info("invalid group name %s", buf);
continue;
@@ -774,15 +839,15 @@ static void _merge_node_totals(hid_t jg_node, hid_t nsg_node)
==========================================================================*/
static void _merge_node_step_data(hid_t fid_job, char* file_name, int nodeIndex,
- char* node_name, hid_t jgid_nodes,
- hid_t jgid_tasks)
+ char* node_name, hid_t jgid_nodes,
+ hid_t jgid_tasks)
{
hid_t fid_nodestep, jgid_node, nsgid_root, nsgid_node;
char *start_time;
char group_name[MAX_GROUP_NAME+1];
jgid_node = H5Gcreate(jgid_nodes, node_name,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (jgid_node < 0) {
error("Failed to create group %s",node_name);
return;
@@ -825,112 +890,130 @@ static void _merge_node_step_data(hid_t fid_job, char* file_name, int nodeIndex,
return;
}
-static void _merge_step_files(void)
+static int _merge_step_files(void)
{
- hid_t fid_job = -1, jgid_step = -1, jgid_nodes = -1, jgid_tasks = -1;
- DIR *dir;
+ hid_t fid_job;
+ hid_t jgid_step;
+ hid_t jgid_nodes;
+ hid_t jgid_tasks;
+ DIR *dir;
struct dirent *de;
- char file_name[MAX_PROFILE_PATH+1];
- char step_dir[MAX_PROFILE_PATH+1];
- char step_path[MAX_PROFILE_PATH+1];
- char jgrp_step_name[MAX_GROUP_NAME+1];
- char jgrp_nodes_name[MAX_GROUP_NAME+1];
- char jgrp_tasks_name[MAX_GROUP_NAME+1];
- char *step_node, *pos_char, *stepno;
- int stepx = 0, num_steps = 0, nodex = -1, max_step = -1;
+ char file_name[MAX_PROFILE_PATH+1];
+ char step_dir[MAX_PROFILE_PATH+1];
+ char step_path[MAX_PROFILE_PATH+1];
+ char jgrp_step_name[MAX_GROUP_NAME+1];
+ char jgrp_nodes_name[MAX_GROUP_NAME+1];
+ char jgrp_tasks_name[MAX_GROUP_NAME+1];
+ char *step_node;
+ char *pos_char;
+ char *stepno;
+ int stepx = 0;
+ int num_steps = 0;
+ int nodex = -1;
+ int max_step = -1;
int jobid, stepid;
- bool found_files = false;
+ bool found_files = false;
sprintf(step_dir, "%s/%s", params.dir, params.user);
+
while (max_step == -1 || stepx <= max_step) {
+
if (!(dir = opendir(step_dir))) {
- error("opendir for job profile directory: %m");
- exit(1);
+ error("Cannot open %s job profile directory: %m", step_dir);
+ return -1;
}
+
nodex = 0;
while ((de = readdir(dir))) {
+
strcpy(file_name, de->d_name);
if (file_name[0] == '.')
- continue; // Not HDF5 file
+ continue;
+
pos_char = strstr(file_name,".h5");
- if (!pos_char) {
- error("error processing this file, %s, "
- "(not .h5)", de->d_name);
- continue; // Not HDF5 file
- }
- *pos_char = 0; // truncate .hf
+ if (!pos_char)
+ continue;
+ *pos_char = 0;
+
pos_char = strchr(file_name,'_');
if (!pos_char)
- continue; // not right format
- *pos_char = 0; // make jobid string
+ continue;
+ *pos_char = 0;
+
jobid = strtol(file_name, NULL, 10);
if (jobid != params.job_id)
- continue; // not desired job
+ continue;
+
stepno = pos_char + 1;
pos_char = strchr(stepno,'_');
if (!pos_char) {
- continue; // not right format
+ continue;
}
- *pos_char = 0; // make stepid string
+ *pos_char = 0;
+
stepid = strtol(stepno, NULL, 10);
if (stepid > max_step)
max_step = stepid;
if (stepid != stepx)
- continue; // Not step we are merging
+ continue;
+
step_node = pos_char + 1;
- // Found a node step file for this job
+
if (!found_files) {
- // Need to create the job file
fid_job = H5Fcreate(params.output,
- H5F_ACC_TRUNC,
- H5P_DEFAULT,
- H5P_DEFAULT);
+ H5F_ACC_TRUNC,
+ H5P_DEFAULT,
+ H5P_DEFAULT);
if (fid_job < 0) {
- fatal("Failed to %s %s",
- "create HDF5 file:",
- params.output);
+ error("Failed create HDF5 file %s", params.output);
+ return -1;
}
found_files = true;
}
+
if (nodex == 0) {
+
num_steps++;
sprintf(jgrp_step_name, "/%s_%d", GRP_STEP,
- stepx);
+ stepx);
+
jgid_step = make_group(fid_job, jgrp_step_name);
if (jgid_step < 0) {
- error("Failed to create %s",
- jgrp_step_name);
+ error("Failed to create %s", jgrp_step_name);
continue;
}
+
sprintf(jgrp_nodes_name,"%s/%s",
- jgrp_step_name,
- GRP_NODES);
+ jgrp_step_name,
+ GRP_NODES);
jgid_nodes = make_group(jgid_step,
- jgrp_nodes_name);
+ jgrp_nodes_name);
if (jgid_nodes < 0) {
- error("Failed to create %s",
- jgrp_nodes_name);
+ error("Failed to create %s", jgrp_nodes_name);
continue;
}
+
sprintf(jgrp_tasks_name,"%s/%s",
- jgrp_step_name,
- GRP_TASKS);
+ jgrp_step_name,
+ GRP_TASKS);
jgid_tasks = make_group(jgid_step,
- jgrp_tasks_name);
+ jgrp_tasks_name);
if (jgid_tasks < 0) {
- error("Failed to create %s",
- jgrp_tasks_name);
+ error("Failed to create %s", jgrp_tasks_name);
continue;
}
}
+
sprintf(step_path, "%s/%s", step_dir, de->d_name);
debug("Adding %s to the job file", step_path);
_merge_node_step_data(fid_job, step_path,
- nodex, step_node,
- jgid_nodes, jgid_tasks);
+ nodex, step_node,
+ jgid_nodes, jgid_tasks);
nodex++;
}
+
closedir(dir);
+
if (nodex > 0) {
put_int_attribute(jgid_step, ATTR_NNODES, nodex);
H5Gclose(jgid_tasks);
@@ -939,12 +1022,16 @@ static void _merge_step_files(void)
}
stepx++;
}
+
if (!found_files)
- info("No node-step files found for jobid=%d", params.job_id);
+ info("No node-step files found for jobid %d", params.job_id);
else
put_int_attribute(fid_job, ATTR_NSTEPS, num_steps);
+
if (fid_job != -1)
H5Fclose(fid_job);
+
+ return 0;
}
/* ============================================================================
@@ -992,8 +1079,8 @@ static void _get_series_names(hid_t group)
series_names = xmalloc(sizeof(char*)*num_series);
for (i = 0; (num_series>0) && (i<num_series); i++) {
len = H5Lget_name_by_idx(group, ".", H5_INDEX_NAME,
- H5_ITER_INC, i, buf,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, i, buf,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if ((len < 0) || (len > MAX_GROUP_NAME)) {
info("Invalid series name=%s", buf);
// put into list anyway so list doesn't have a null.
@@ -1005,7 +1092,7 @@ static void _get_series_names(hid_t group)
static void _extract_node_level(FILE* fp, int stepx, hid_t jgid_nodes,
- int nnodes, bool header, char* data_set_name)
+ int nnodes, bool header, char* data_set_name)
{
hid_t jgid_node, gid_level, gid_series;
@@ -1018,8 +1105,8 @@ static void _extract_node_level(FILE* fp, int stepx, hid_t jgid_nodes,
for (nodex=0; nodex<nnodes; nodex++) {
len = H5Lget_name_by_idx(jgid_nodes, ".", H5_INDEX_NAME,
- H5_ITER_INC, nodex, jgrp_node_name,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, nodex, jgrp_node_name,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if ((len < 0) || (len > MAX_GROUP_NAME)) {
info("Invalid node name=%s", jgrp_node_name);
continue;
@@ -1092,8 +1179,8 @@ static void _extract_node_level(FILE* fp, int stepx, hid_t jgid_nodes,
xfree(data);
} else {
fprintf(fp, "%d,%d,%s,No %s Data\n",
- params.job_id, stepx, jgrp_node_name,
- data_set_name);
+ params.job_id, stepx, jgrp_node_name,
+ data_set_name);
}
xfree(ops);
H5Gclose(gid_series);
@@ -1104,61 +1191,80 @@ static void _extract_node_level(FILE* fp, int stepx, hid_t jgid_nodes,
}
-static void _extract_data()
+/* _extract_data()
+ */
+static int _extract_data(void)
{
+ hid_t fid_job;
+ hid_t jgid_root;
+ hid_t jgid_step;
+ hid_t jgid_nodes;
+ hid_t jgid_node;
+ hid_t jgid_level;
+ int nsteps;
+ int nnodes;
+ int stepx;
+ int isx;
+ int len;
+ char jgrp_step_name[MAX_GROUP_NAME+1];
+ char jgrp_node_name[MAX_GROUP_NAME+1];
+ bool header;
+ FILE *fp;
- hid_t fid_job, jgid_root, jgid_step, jgid_nodes,
- jgid_node, jgid_level;
- int nsteps, nnodes, stepx, isx, len;
- char jgrp_step_name[MAX_GROUP_NAME+1];
- char jgrp_node_name[MAX_GROUP_NAME+1];
- bool header;
-
- FILE* fp = fopen(params.output, "w");
-
+ fp = fopen(params.output, "w");
if (fp == NULL) {
error("Failed to create output file %s -- %m",
params.output);
}
+
fid_job = H5Fopen(params.input, H5F_ACC_RDONLY, H5P_DEFAULT);
if (fid_job < 0) {
error("Failed to open %s", params.input);
- return;
+ return -1;
}
+
jgid_root = H5Gopen(fid_job, "/", H5P_DEFAULT);
if (jgid_root < 0) {
H5Fclose(fid_job);
error("Failed to open root");
- return;
+ return -1;
}
+
nsteps = get_int_attribute(jgid_root, ATTR_NSTEPS);
header = true;
- for (stepx=0; stepx<nsteps; stepx++) {
+ for (stepx = 0; stepx < nsteps; stepx++) {
+
if ((params.step_id != -1) && (stepx != params.step_id))
continue;
+
sprintf(jgrp_step_name, "%s_%d", GRP_STEP, stepx);
jgid_step = get_group(jgid_root, jgrp_step_name);
if (jgid_step < 0) {
- error("Failed to open group %s", jgrp_step_name);
+ error("Failed to open group %s", jgrp_step_name);
continue;
}
+
if (params.level && !strncasecmp(params.level, "Node:", 5)) {
+
nnodes = get_int_attribute(jgid_step, ATTR_NNODES);
+
jgid_nodes = get_group(jgid_step, GRP_NODES);
if (jgid_nodes < 0) {
H5Gclose(jgid_step);
error("Failed to open group %s", GRP_NODES);
continue;
}
+
len = H5Lget_name_by_idx(jgid_nodes, ".", H5_INDEX_NAME,
- H5_ITER_INC, 0, jgrp_node_name,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, 0, jgrp_node_name,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if ((len < 0) || (len > MAX_GROUP_NAME)) {
H5Gclose(jgid_nodes);
H5Gclose(jgid_step);
error("Invalid node name %s", jgrp_node_name);
continue;
}
+
jgid_node = get_group(jgid_nodes, jgrp_node_name);
if (jgid_node < 0) {
H5Gclose(jgid_nodes);
@@ -1166,6 +1272,7 @@ static void _extract_data()
info("Failed to open group %s", jgrp_node_name);
continue;
}
+
jgid_level = _get_series_parent(jgid_node);
if (jgid_level == -1) {
H5Gclose(jgid_node);
@@ -1173,11 +1280,14 @@ static void _extract_data()
H5Gclose(jgid_step);
continue;
}
+
_get_series_names(jgid_level);
H5Gclose(jgid_level);
H5Gclose(jgid_node);
+
if (!params.series || !strcmp(params.series, "*")) {
- for (isx=0; isx<num_series; isx++) {
+
+ for (isx = 0; isx < num_series; isx++) {
_extract_node_level(fp, stepx, jgid_nodes,
nnodes, header,
series_names[isx]);
@@ -1186,13 +1296,12 @@ static void _extract_data()
}
} else if (strcasecmp(params.series, GRP_TASKS) == 0
|| strcasecmp(params.series, GRP_TASK) == 0) {
- for (isx=0; isx<num_series; isx++) {
+ for (isx = 0; isx < num_series; isx++) {
if (strstr(series_names[isx],
- GRP_TASK)) {
- _extract_node_level(
- fp, stepx, jgid_nodes,
- nnodes, header,
- series_names[isx]);
+ GRP_TASK)) {
+ _extract_node_level(fp, stepx, jgid_nodes,
+ nnodes, header,
+ series_names[isx]);
header = false;
}
}
@@ -1202,6 +1311,7 @@ static void _extract_data()
params.series);
header = false;
}
+
_delete_string_list(series_names, num_series);
series_names = NULL;
num_series = 0;
@@ -1211,10 +1321,12 @@ static void _extract_data()
}
H5Gclose(jgid_step);
}
+
H5Gclose(jgid_root);
H5Fclose(fid_job);
fclose(fp);
+ return 0;
}
/* ============================================================================
@@ -1224,8 +1336,8 @@ static void _extract_data()
* ========================================================================= */
// Get the data_set for a node
-static void* _get_series_data(hid_t jgid_node, char* series,
- hdf5_api_ops_t **ops_p, int *nsmp)
+static void *_get_series_data(hid_t jgid_node, char* series,
+ hdf5_api_ops_t **ops_p, int *nsmp)
{
hid_t gid_level, gid_series;
@@ -1265,7 +1377,7 @@ static void* _get_series_data(hid_t jgid_node, char* series,
H5Gclose(gid_series);
H5Gclose(gid_level);
debug("Failed to create operations for %s",
- acct_gather_profile_type_to_string(type));
+ acct_gather_profile_type_to_string(type));
return NULL;
}
data = get_hdf5_data(gid_series, type, series, &size_data);
@@ -1281,8 +1393,8 @@ static void* _get_series_data(hid_t jgid_node, char* series,
}
static void _series_analysis(FILE *fp, bool hd, int stepx, int nseries,
- int nsmp, char **series_name, char **tod, double *et,
- double **all_series, uint64_t *series_smp)
+ int nsmp, char **series_name, char **tod, double *et,
+ double **all_series, uint64_t *series_smp)
{
double *mn_series; // Min Value, each sample
double *mx_series; // Max value, each sample
@@ -1291,18 +1403,19 @@ static void _series_analysis(FILE *fp, bool hd, int stepx, int nseries,
uint64_t *mn_sx; // Index of series with minimum value
uint64_t *mx_sx; // Index of series with maximum value
uint64_t *series_in_smp; // Number of series in the sample
-
- int max_smpx = 0;
- double max_smp_series = 0;
- double ave_series;
+ int max_smpx = 0;
+ double max_smp_series = 0;
+ double ave_series;
int ix, isx;
- mn_series = (double*) xmalloc(nsmp*sizeof(double));
- mx_series = (double*) xmalloc(nsmp*sizeof(double));
- sum_series = (double*) xmalloc(nsmp*sizeof(double));
- mn_sx = (uint64_t*) xmalloc(nsmp*sizeof(uint64_t));
- mx_sx = (uint64_t*) xmalloc(nsmp*sizeof(uint64_t));
- series_in_smp = (uint64_t*) xmalloc(nsmp*sizeof(uint64_t));
- for (ix=0; ix<nsmp; ix++) {
+
+ mn_series = xmalloc(nsmp * sizeof(double));
+ mx_series = xmalloc(nsmp * sizeof(double));
+ sum_series =xmalloc(nsmp * sizeof(double));
+ mn_sx = xmalloc(nsmp * sizeof(uint64_t));
+ mx_sx = xmalloc(nsmp * sizeof(uint64_t));
+ series_in_smp = xmalloc(nsmp * sizeof(uint64_t));
+
+ for (ix = 0; ix < nsmp; ix++) {
for (isx=0; isx<nseries; isx++) {
if (series_smp[isx]<nsmp && ix>=series_smp[isx])
continue;
@@ -1324,7 +1437,7 @@ static void _series_analysis(FILE *fp, bool hd, int stepx, int nseries,
}
}
- for (ix=0; ix<nsmp; ix++) {
+ for (ix = 0; ix < nsmp; ix++) {
if (sum_series[ix] > max_smp_series) {
max_smpx = ix;
max_smp_series = sum_series[ix];
@@ -1332,34 +1445,35 @@ static void _series_analysis(FILE *fp, bool hd, int stepx, int nseries,
}
ave_series = sum_series[max_smpx] / series_in_smp[max_smpx];
- printf(" Step %d Maximum accumlated %s Value (%f) occurred "
- "at %s (Elapsed Time=%d) Ave Node %f\n\n",
- stepx, params.data_item, max_smp_series,
- tod[max_smpx], (int) et[max_smpx], ave_series);
+ printf(" Step %d Maximum accumulated %s Value (%f) occurred "
+ "at %s (Elapsed Time=%d) Ave Node %f\n",
+ stepx, params.data_item, max_smp_series,
+ tod[max_smpx], (int) et[max_smpx], ave_series);
// Put data for step
if (!hd) {
fprintf(fp,"TOD,Et,JobId,StepId,Min Node,Min %s,"
- "Ave %s,Max Node,Max %s,Total %s,"
- "Num Nodes",params.data_item,params.data_item,
- params.data_item,params.data_item);
- for (isx=0; isx<nseries; isx++) {
+ "Ave %s,Max Node,Max %s,Total %s,"
+ "Num Nodes",params.data_item,params.data_item,
+ params.data_item,params.data_item);
+ for (isx = 0; isx < nseries; isx++) {
fprintf(fp,",%s",series_name[isx]);
}
fprintf(fp,"\n");
}
- for (ix=0; ix<nsmp; ix++) {
+
+ for (ix = 0; ix < nsmp; ix++) {
fprintf(fp,"%s, %d",tod[ix], (int) et[ix]);
fprintf(fp,",%d,%d",params.job_id,stepx);
fprintf(fp,",%s,%f",series_name[mn_sx[ix]],
- mn_series[ix]);
+ mn_series[ix]);
ave_series = sum_series[ix] / series_in_smp[ix];
fprintf(fp,",%f",ave_series);
fprintf(fp,",%s,%f",series_name[mx_sx[ix]],
- mx_series[ix]);
+ mx_series[ix]);
fprintf(fp,",%f",sum_series[ix]);
fprintf(fp,",%"PRIu64"",series_in_smp[ix]);
- for (isx=0; isx<nseries; isx++) {
+ for (isx = 0; isx < nseries; isx++) {
if (series_smp[isx]<nsmp && ix>=series_smp[isx]) {
fprintf(fp,",0.0");
} else {
@@ -1375,7 +1489,6 @@ static void _series_analysis(FILE *fp, bool hd, int stepx, int nseries,
xfree(sum_series);
xfree(mn_sx);
xfree(mx_sx);
-
}
static void _get_all_node_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
@@ -1394,23 +1507,28 @@ static void _get_all_node_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
nnodes = get_int_attribute(jgid_step, ATTR_NNODES);
// allocate node arrays
- series_smp = (uint64_t*) xmalloc(nnodes*(sizeof(uint64_t)));
+
+ series_smp = xmalloc(nnodes * (sizeof(uint64_t)));
if (series_smp == NULL)
fatal("Failed to get memory for node_samples");
- node_name = (char**) xmalloc(nnodes*(sizeof(char*)));
+
+ node_name = xmalloc(nnodes * (sizeof(char*)));
if (node_name == NULL)
fatal("Failed to get memory for node_name");
- all_series = (double**) xmalloc(nnodes*(sizeof(double*)));
+
+ all_series = xmalloc(nnodes * (sizeof(double*)));
if (all_series == NULL)
fatal("Failed to get memory for all_series");
+
jgid_nodes = get_group(jgid_step, GRP_NODES);
if (jgid_nodes < 0)
fatal("Failed to open group %s", GRP_NODES);
+
for (ndx=0; ndx<nnodes; ndx++) {
len = H5Lget_name_by_idx(jgid_nodes, ".", H5_INDEX_NAME,
- H5_ITER_INC, ndx, jgrp_node_name,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, ndx, jgrp_node_name,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if ((len < 0) || (len > MAX_GROUP_NAME)) {
debug("Invalid node name=%s", jgrp_node_name);
continue;
@@ -1424,14 +1542,14 @@ static void _get_all_node_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
ops = NULL;
nitem = 0;
series_data = _get_series_data(jgid_node, params.series,
- &ops, &nitem);
+ &ops, &nitem);
if (series_data==NULL || nitem==0 || ops==NULL) {
if (ops != NULL)
xfree(ops);
continue;
}
all_series[ndx] = ops->get_series_values(
- params.data_item, series_data, nitem);
+ params.data_item, series_data, nitem);
if (!all_series[ndx])
fatal("No data item %s",params.data_item);
series_smp[ndx] = nitem;
@@ -1439,7 +1557,7 @@ static void _get_all_node_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
nsmp = nitem;
tod = ops->get_series_tod(series_data, nitem);
et = ops->get_series_values("time",
- series_data, nitem);
+ series_data, nitem);
} else {
if (nitem > nsmp) {
// new largest number of samples
@@ -1447,9 +1565,9 @@ static void _get_all_node_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
xfree(et);
nsmp = nitem;
tod = ops->get_series_tod(series_data,
- nitem);
+ nitem);
et = ops->get_series_values("time",
- series_data, nitem);
+ series_data, nitem);
}
}
xfree(ops);
@@ -1459,11 +1577,11 @@ static void _get_all_node_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
if (nsmp == 0) {
// May be bad series name
info("No values %s for series %s found in step %d",
- params.data_item,params.series,
- stepx);
+ params.data_item,params.series,
+ stepx);
} else {
_series_analysis(fp, hd, stepx, nnodes, nsmp,
- node_name, tod, et, all_series, series_smp);
+ node_name, tod, et, all_series, series_smp);
}
for (ndx=0; ndx<nnodes; ndx++) {
xfree(node_name[ndx]);
@@ -1486,14 +1604,12 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
H5G_info_t group_info;
int ntasks,itx, tid;
uint64_t *task_id;
- char **task_node_name; // Node Name for each task
-
- char **tod = NULL; // Date time at each sample
- char **series_name; // Node Names
- double **all_series; // Pointers to all sampled for each node
- double *et = NULL; // Elapsed time at each sample
- uint64_t *series_smp; // Number of samples in this series
-
+ char **task_node_name; /* Node Name for each task */
+ char **tod = NULL; /* Date time at each sample */
+ char **series_name; /* Node Names */
+ double **all_series; /* Pointers to all sampled for each node */
+ double *et = NULL; /* Elapsed time at each sample */
+ uint64_t *series_smp; /* Number of samples in this series */
int nnodes, ndx, len, nsmp = 0, nitem = -1;
char jgrp_node_name[MAX_GROUP_NAME+1];
char jgrp_task_name[MAX_GROUP_NAME+1];
@@ -1521,8 +1637,8 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
for (itx = 0; itx<ntasks; itx++) {
// Get the name of the group.
len = H5Lget_name_by_idx(jgid_tasks, ".", H5_INDEX_NAME,
- H5_ITER_INC, itx, buf, MAX_GROUP_NAME,
- H5P_DEFAULT);
+ H5_ITER_INC, itx, buf, MAX_GROUP_NAME,
+ H5P_DEFAULT);
if ((len > 0) && (len < MAX_GROUP_NAME)) {
jgid_task = H5Gopen(jgid_tasks, buf, H5P_DEFAULT);
if (jgid_task < 0)
@@ -1531,7 +1647,7 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
fatal("Illegal task name %s",buf);
task_id[itx] = get_int_attribute(jgid_task, ATTR_TASKID);
task_node_name[itx] = get_string_attribute(jgid_task,
- ATTR_NODENAME);
+ ATTR_NODENAME);
H5Gclose(jgid_task);
}
H5Gclose(jgid_tasks);
@@ -1551,8 +1667,8 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
for (ndx=0; ndx<nnodes; ndx++) {
len = H5Lget_name_by_idx(jgid_nodes, ".", H5_INDEX_NAME,
- H5_ITER_INC, ndx, jgrp_node_name,
- MAX_GROUP_NAME, H5P_DEFAULT);
+ H5_ITER_INC, ndx, jgrp_node_name,
+ MAX_GROUP_NAME, H5P_DEFAULT);
if ((len < 0) || (len > MAX_GROUP_NAME))
fatal("Invalid node name=%s", jgrp_node_name);
jgid_node = get_group(jgid_nodes, jgrp_node_name);
@@ -1564,20 +1680,20 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
continue;
tid = task_id[itx];
series_name[itx] = xstrdup_printf("%s_%d %s",
- GRP_TASK,tid,jgrp_node_name);
+ GRP_TASK,tid,jgrp_node_name);
sprintf(jgrp_task_name,"%s_%d",GRP_TASK, tid);
ops = NULL;
nitem = 0;
series_data = _get_series_data(jgid_node,
- jgrp_task_name, &ops, &nitem);
+ jgrp_task_name, &ops, &nitem);
if (series_data==NULL || nitem==0 || ops==NULL) {
if (ops != NULL)
xfree(ops);
continue;
}
all_series[itx] = ops->get_series_values(
- params.data_item, series_data, nitem);
+ params.data_item, series_data, nitem);
if (!all_series[ndx])
fatal("No data item %s",params.data_item);
series_smp[itx] = nitem;
@@ -1585,7 +1701,7 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
nsmp = nitem;
tod = ops->get_series_tod(series_data, nitem);
et = ops->get_series_values("time",
- series_data, nitem);
+ series_data, nitem);
} else {
if (nitem > nsmp) {
// new largest number of samples
@@ -1593,9 +1709,9 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
xfree(et);
nsmp = nitem;
tod = ops->get_series_tod(series_data,
- nitem);
+ nitem);
et = ops->get_series_values("time",
- series_data, nitem);
+ series_data, nitem);
}
}
xfree(ops);
@@ -1606,11 +1722,11 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
if (nsmp == 0) {
// May be bad series name
info("No values %s for series %s found in step %d",
- params.data_item,params.series,
- stepx);
+ params.data_item,params.series,
+ stepx);
} else {
_series_analysis(fp, hd, stepx, ntasks, nsmp,
- series_name, tod, et, all_series, series_smp);
+ series_name, tod, et, all_series, series_smp);
}
for (itx=0; itx<ntasks; itx++) {
xfree(all_series[itx]);
@@ -1625,89 +1741,63 @@ static void _get_all_task_series(FILE *fp, bool hd, hid_t jgid_step, int stepx)
H5Gclose(jgid_nodes);
}
-static void _series_data()
+static int _series_data(void)
{
- FILE *fp = NULL;
- bool hd = false;
- hid_t fid_job, jgid_root, jgid_step;
- int nsteps, stepx;
- char jgrp_step_name[MAX_GROUP_NAME+1];
+ FILE *fp;
+ bool hd = false;
+ hid_t fid_job;
+ hid_t jgid_root;
+ hid_t jgid_step;
+ int nsteps;
+ int stepx;
+ char jgrp_step_name[MAX_GROUP_NAME + 1];
fp = fopen(params.output, "w");
if (fp == NULL) {
- error("Failed to create output file %s -- %m", params.output);
- return;
+ error("Failed open file %s -- %m", params.output);
+ return -1;
}
fid_job = H5Fopen(params.input, H5F_ACC_RDONLY, H5P_DEFAULT);
if (fid_job < 0) {
fclose(fp);
error("Failed to open %s", params.input);
- return;
+ return -1;
}
+
jgid_root = H5Gopen(fid_job, "/", H5P_DEFAULT);
if (jgid_root < 0) {
fclose(fp);
H5Fclose(fid_job);
- error("Failed to open root");
- return;
+ error("Failed to open root");
+ return -1;
}
+
nsteps = get_int_attribute(jgid_root, ATTR_NSTEPS);
- for (stepx=0; stepx<nsteps; stepx++) {
+ for (stepx = 0; stepx < nsteps; stepx++) {
+
if ((params.step_id != -1) && (stepx != params.step_id))
continue;
+
sprintf(jgrp_step_name, "%s_%d", GRP_STEP, stepx);
jgid_step = get_group(jgid_root, jgrp_step_name);
if (jgid_step < 0) {
- fatal("Failed to open group %s", jgrp_step_name);
+ error("Failed to open group %s", jgrp_step_name);
+ return -1;
}
+
if (strncmp(params.series,GRP_TASK,strlen(GRP_TASK)) == 0)
_get_all_task_series(fp,hd,jgid_step, stepx);
else
_get_all_node_series(fp,hd,jgid_step, stepx);
+
hd = true;
H5Gclose(jgid_step);
}
-
H5Gclose(jgid_root);
H5Fclose(fid_job);
fclose(fp);
-}
-
-int main (int argc, char **argv)
-{
- if (argc <= 1) {
- _help_msg();
- exit(0);
- }
- _set_options(argc, argv);
- profile_init();
- switch (params.mode) {
- case SH5UTIL_MODE_MERGE:
- info("Merging node-step files into %s",
- params.output);
- _merge_step_files();
- break;
- case SH5UTIL_MODE_EXTRACT:
- info("Extracting job data from %s into %s",
- params.input, params.output);
- _extract_data();
- break;
- case SH5UTIL_MODE_ITEM_EXTRACT:
- info("Extracting '%s' from '%s' data from %s into %s\n",
- params.data_item, params.series,
- params.input, params.output);
- _series_data();
- break;
- default:
- error("Unknown type %d", params.mode);
- break;
- }
- _remove_empty_outout();
- profile_fini();
- xfree(params.dir);
- xfree(params.node);
return 0;
}
--
GitLab