From 6cd8a6b463afebdbb6e46d75258c9d18ecf849e6 Mon Sep 17 00:00:00 2001 From: Danny Auble <da@llnl.gov> Date: Thu, 12 Jun 2008 23:48:38 +0000 Subject: [PATCH] switch over to 64 bit stuff for accounting --- src/common/pack.c | 36 +++++++++- src/common/pack.h | 17 +++++ src/common/print_fields.c | 42 ++++++++++- src/common/print_fields.h | 7 +- src/common/slurm_accounting_storage.c | 36 +++++----- src/common/slurm_accounting_storage.h | 12 ++-- .../mysql/accounting_storage_mysql.c | 12 ++-- .../accounting_storage/mysql/mysql_rollup.c | 30 ++++---- src/sreport/cluster_reports.c | 72 ++++++++++--------- src/sreport/common.c | 14 ++-- src/sreport/sreport.h | 2 +- 11 files changed, 190 insertions(+), 90 deletions(-) diff --git a/src/common/pack.c b/src/common/pack.c index 53902756f17..d48f3c0413b 100644 --- a/src/common/pack.c +++ b/src/common/pack.c @@ -47,7 +47,6 @@ #include <string.h> #include <time.h> #include <inttypes.h> - #include <slurm/slurm_errno.h> #include "src/common/pack.h" @@ -65,6 +64,8 @@ strong_alias(init_buf, slurm_init_buf); strong_alias(xfer_buf_data, slurm_xfer_buf_data); strong_alias(pack_time, slurm_pack_time); strong_alias(unpack_time, slurm_unpack_time); +strong_alias(pack64, slurm_pack64); +strong_alias(unpack64, slurm_unpack64); strong_alias(pack32, slurm_pack32); strong_alias(unpack32, slurm_unpack32); strong_alias(pack16, slurm_pack16); @@ -174,6 +175,39 @@ int unpack_time(time_t * valp, Buf buffer) } +/* + * Given a 64-bit integer in host byte order, convert to network byte order + * store in buffer, and adjust buffer counters. + */ +void pack64(uint64_t val, Buf buffer) +{ + uint64_t nl = HTON_uint64(val); + + if (remaining_buf(buffer) < sizeof(nl)) { + buffer->size += BUF_SIZE; + xrealloc(buffer->head, buffer->size); + } + + memcpy(&buffer->head[buffer->processed], &nl, sizeof(nl)); + buffer->processed += sizeof(nl); +} + +/* + * Given a buffer containing a network byte order 64-bit integer, + * store a host integer at 'valp', and adjust buffer counters. + */ +int unpack64(uint64_t * valp, Buf buffer) +{ + uint64_t nl; + if (remaining_buf(buffer) < sizeof(nl)) + return SLURM_ERROR; + + memcpy(&nl, &buffer->head[buffer->processed], sizeof(nl)); + *valp = NTOH_uint64(nl); + buffer->processed += sizeof(nl); + return SLURM_SUCCESS; +} + /* * Given a 32-bit integer in host byte order, convert to network byte order * store in buffer, and adjust buffer counters. diff --git a/src/common/pack.h b/src/common/pack.h index 4b18355e9c4..3e537b6e075 100644 --- a/src/common/pack.h +++ b/src/common/pack.h @@ -83,6 +83,9 @@ void *xfer_buf_data(Buf my_buf); void pack_time(time_t val, Buf buffer); int unpack_time(time_t *valp, Buf buffer); +void pack64(uint64_t val, Buf buffer); +int unpack64(uint64_t *valp, Buf buffer); + void pack32(uint32_t val, Buf buffer); int unpack32(uint32_t *valp, Buf buffer); @@ -124,6 +127,20 @@ int unpackmem_array(char *valp, uint32_t size_valp, Buf buffer); goto unpack_error; \ } while (0) +#define safe_pack64(val,buf) do { \ + assert(sizeof(val) == sizeof(uint64_t)); \ + assert(buf->magic == BUF_MAGIC); \ + pack64(val,buf); \ +} while (0) + +#define safe_unpack64(valp,buf) do { \ + assert((valp) != NULL); \ + assert(sizeof(*valp) == sizeof(uint64_t)); \ + assert(buf->magic == BUF_MAGIC); \ + if (unpack64(valp,buf)) \ + goto unpack_error; \ +} while (0) + #define safe_pack32(val,buf) do { \ assert(sizeof(val) == sizeof(uint32_t)); \ assert(buf->magic == BUF_MAGIC); \ diff --git a/src/common/print_fields.c b/src/common/print_fields.c index 92592eedba8..5648eb67ba1 100644 --- a/src/common/print_fields.c +++ b/src/common/print_fields.c @@ -121,7 +121,8 @@ extern void print_fields_str(type_t type, print_field_t *field, char *value) } } -extern void print_fields_uint(type_t type, print_field_t *field, uint32_t value) +extern void print_fields_uint32(type_t type, print_field_t *field, + uint32_t value) { switch(type) { case SLURM_PRINT_HEADLINE: @@ -158,6 +159,44 @@ extern void print_fields_uint(type_t type, print_field_t *field, uint32_t value) } } +extern void print_fields_uint64(type_t type, print_field_t *field, + uint64_t value) +{ + switch(type) { + case SLURM_PRINT_HEADLINE: + if(print_fields_parsable_print) + printf("%s|", field->name); + else + printf("%-*.*s ", field->len, field->len, field->name); + break; + case SLURM_PRINT_UNDERSCORE: + if(!print_fields_parsable_print) + printf("%-*.*s ", field->len, field->len, + "---------------------------------------"); + break; + case SLURM_PRINT_VALUE: + /* (value == unset) || (value == cleared) */ + if((value == NO_VAL) || (value == INFINITE)) { + if(print_fields_parsable_print) + printf("|"); + else + printf("%-*s ", field->len, " "); + } else { + if(print_fields_parsable_print) + printf("%llu|", value); + else + printf("%*llu ", field->len, value); + } + break; + default: + if(print_fields_parsable_print) + printf("%s|", "n/a"); + else + printf("%-*.*s ", field->len, field->len, "n/a"); + break; + } +} + extern void print_fields_time(type_t type, print_field_t *field, uint32_t value) { switch(type) { @@ -194,3 +233,4 @@ extern void print_fields_time(type_t type, print_field_t *field, uint32_t value) break; } } + diff --git a/src/common/print_fields.h b/src/common/print_fields.h index 6d840ebc846..0a4e2c04657 100644 --- a/src/common/print_fields.h +++ b/src/common/print_fields.h @@ -87,9 +87,12 @@ extern void destroy_print_field(void *object); extern void print_fields_header(List print_fields_list); extern void print_fields_date(void); extern void print_fields_str(type_t type, print_field_t *field, char *value); -extern void print_fields_uint(type_t type, print_field_t *field, - uint32_t value); +extern void print_fields_uint32(type_t type, print_field_t *field, + uint32_t value); +extern void print_fields_uint64(type_t type, print_field_t *field, + uint64_t value); extern void print_fields_time(type_t type, print_field_t *field, uint32_t value); +#define print_fields_uint print_fields_uint32 #endif diff --git a/src/common/slurm_accounting_storage.c b/src/common/slurm_accounting_storage.c index 1567b39f494..1323cbf8514 100644 --- a/src/common/slurm_accounting_storage.c +++ b/src/common/slurm_accounting_storage.c @@ -757,23 +757,23 @@ extern void pack_cluster_accounting_rec(void *in, Buf buffer) cluster_accounting_rec_t *object = (cluster_accounting_rec_t *)in; if(!object) { + pack64(0, buffer); pack32(0, buffer); - pack32(0, buffer); - pack32(0, buffer); - pack32(0, buffer); - pack32(0, buffer); + pack64(0, buffer); + pack64(0, buffer); + pack64(0, buffer); pack_time(0, buffer); - pack32(0, buffer); + pack64(0, buffer); return; } - pack32(object->alloc_secs, buffer); + pack64(object->alloc_secs, buffer); pack32(object->cpu_count, buffer); - pack32(object->down_secs, buffer); - pack32(object->idle_secs, buffer); - pack32(object->over_secs, buffer); + pack64(object->down_secs, buffer); + pack64(object->idle_secs, buffer); + pack64(object->over_secs, buffer); pack_time(object->period_start, buffer); - pack32(object->resv_secs, buffer); + pack64(object->resv_secs, buffer); } extern int unpack_cluster_accounting_rec(void **object, Buf buffer) @@ -782,13 +782,13 @@ extern int unpack_cluster_accounting_rec(void **object, Buf buffer) xmalloc(sizeof(cluster_accounting_rec_t)); *object = object_ptr; - safe_unpack32(&object_ptr->alloc_secs, buffer); + safe_unpack64(&object_ptr->alloc_secs, buffer); safe_unpack32(&object_ptr->cpu_count, buffer); - safe_unpack32(&object_ptr->down_secs, buffer); - safe_unpack32(&object_ptr->idle_secs, buffer); - safe_unpack32(&object_ptr->over_secs, buffer); + safe_unpack64(&object_ptr->down_secs, buffer); + safe_unpack64(&object_ptr->idle_secs, buffer); + safe_unpack64(&object_ptr->over_secs, buffer); safe_unpack_time(&object_ptr->period_start, buffer); - safe_unpack32(&object_ptr->resv_secs, buffer); + safe_unpack64(&object_ptr->resv_secs, buffer); return SLURM_SUCCESS; @@ -885,13 +885,13 @@ extern void pack_acct_accounting_rec(void *in, Buf buffer) acct_accounting_rec_t *object = (acct_accounting_rec_t *)in; if(!object) { - pack32(0, buffer); + pack64(0, buffer); pack32(0, buffer); pack_time(0, buffer); return; } - pack32(object->alloc_secs, buffer); + pack64(object->alloc_secs, buffer); pack32(object->assoc_id, buffer); pack_time(object->period_start, buffer); } @@ -902,7 +902,7 @@ extern int unpack_acct_accounting_rec(void **object, Buf buffer) xmalloc(sizeof(acct_accounting_rec_t)); *object = object_ptr; - safe_unpack32(&object_ptr->alloc_secs, buffer); + safe_unpack64(&object_ptr->alloc_secs, buffer); safe_unpack32(&object_ptr->assoc_id, buffer); safe_unpack_time(&object_ptr->period_start, buffer); diff --git a/src/common/slurm_accounting_storage.h b/src/common/slurm_accounting_storage.h index b20d0e3898d..ccee9c173b5 100644 --- a/src/common/slurm_accounting_storage.h +++ b/src/common/slurm_accounting_storage.h @@ -112,7 +112,7 @@ typedef struct { } acct_account_rec_t; typedef struct { - uint32_t alloc_secs; /* number of cpu seconds allocated */ + uint64_t alloc_secs; /* number of cpu seconds allocated */ uint32_t assoc_id; /* association ID */ time_t period_start; } acct_accounting_rec_t; @@ -218,13 +218,13 @@ typedef struct { } shares_used_object_t; typedef struct { - uint32_t alloc_secs; /* number of cpu seconds allocated */ + uint64_t alloc_secs; /* number of cpu seconds allocated */ uint32_t cpu_count; /* number of cpus during time period */ - uint32_t down_secs; /* number of cpu seconds down */ - uint32_t idle_secs; /* number of cpu seconds idle */ - uint32_t over_secs; /* number of cpu seconds overcommitted */ + uint64_t down_secs; /* number of cpu seconds down */ + uint64_t idle_secs; /* number of cpu seconds idle */ + uint64_t over_secs; /* number of cpu seconds overcommitted */ time_t period_start; /* when this record was started */ - uint32_t resv_secs; /* number of cpu seconds reserved */ + uint64_t resv_secs; /* number of cpu seconds reserved */ } cluster_accounting_rec_t; extern void destroy_acct_user_rec(void *object); diff --git a/src/plugins/accounting_storage/mysql/accounting_storage_mysql.c b/src/plugins/accounting_storage/mysql/accounting_storage_mysql.c index 3a2964dd7f6..95d36b226a6 100644 --- a/src/plugins/accounting_storage/mysql/accounting_storage_mysql.c +++ b/src/plugins/accounting_storage/mysql/accounting_storage_mysql.c @@ -4575,7 +4575,7 @@ extern int acct_storage_p_get_usage(mysql_conn_t *mysql_conn, xmalloc(sizeof(acct_accounting_rec_t)); accounting_rec->assoc_id = atoi(row[ASSOC_ID]); accounting_rec->period_start = atoi(row[ASSOC_START]); - accounting_rec->alloc_secs = atoi(row[ASSOC_ACPU]); + accounting_rec->alloc_secs = atoll(row[ASSOC_ACPU]); list_append(acct_assoc->accounting_list, accounting_rec); } mysql_free_result(result); @@ -5130,11 +5130,11 @@ extern int clusteracct_storage_p_get_usage( while((row = mysql_fetch_row(result))) { cluster_accounting_rec_t *accounting_rec = xmalloc(sizeof(cluster_accounting_rec_t)); - accounting_rec->alloc_secs = atoi(row[CLUSTER_ACPU]); - accounting_rec->down_secs = atoi(row[CLUSTER_DCPU]); - accounting_rec->idle_secs = atoi(row[CLUSTER_ICPU]); - accounting_rec->over_secs = atoi(row[CLUSTER_OCPU]); - accounting_rec->resv_secs = atoi(row[CLUSTER_RCPU]); + accounting_rec->alloc_secs = atoll(row[CLUSTER_ACPU]); + accounting_rec->down_secs = atoll(row[CLUSTER_DCPU]); + accounting_rec->idle_secs = atoll(row[CLUSTER_ICPU]); + accounting_rec->over_secs = atoll(row[CLUSTER_OCPU]); + accounting_rec->resv_secs = atoll(row[CLUSTER_RCPU]); accounting_rec->cpu_count = atoi(row[CLUSTER_CPU_COUNT]); accounting_rec->period_start = atoi(row[CLUSTER_START]); list_append(cluster_rec->accounting_list, accounting_rec); diff --git a/src/plugins/accounting_storage/mysql/mysql_rollup.c b/src/plugins/accounting_storage/mysql/mysql_rollup.c index 335cb0b3de7..8d89e795563 100644 --- a/src/plugins/accounting_storage/mysql/mysql_rollup.c +++ b/src/plugins/accounting_storage/mysql/mysql_rollup.c @@ -498,16 +498,17 @@ extern int mysql_hourly_rollup(mysql_conn_t *mysql_conn, c_usage->o_cpu, c_usage->r_cpu); } } - xstrfmtcat(query, - " on duplicate key update " - "mod_time=%d, cpu_count=VALUES(cpu_count), " - "alloc_cpu_secs=VALUES(alloc_cpu_secs), " - "down_cpu_secs=VALUES(down_cpu_secs), " - "idle_cpu_secs=VALUES(idle_cpu_secs), " - "over_cpu_secs=VALUES(over_cpu_secs), " - "resv_cpu_secs=VALUES(resv_cpu_secs)", - now); + if(query) { + xstrfmtcat(query, + " on duplicate key update " + "mod_time=%d, cpu_count=VALUES(cpu_count), " + "alloc_cpu_secs=VALUES(alloc_cpu_secs), " + "down_cpu_secs=VALUES(down_cpu_secs), " + "idle_cpu_secs=VALUES(idle_cpu_secs), " + "over_cpu_secs=VALUES(over_cpu_secs), " + "resv_cpu_secs=VALUES(resv_cpu_secs)", + now); rc = mysql_db_query(mysql_conn->acct_mysql_db, query); xfree(query); if(rc != SLURM_SUCCESS) { @@ -538,12 +539,13 @@ extern int mysql_hourly_rollup(mysql_conn_t *mysql_conn, a_usage->a_cpu); } } - xstrfmtcat(query, - " on duplicate key update " - "mod_time=%d, alloc_cpu_secs=VALUES(alloc_cpu_secs)", - now); - if(query) { + xstrfmtcat(query, + " on duplicate key update " + "mod_time=%d, " + "alloc_cpu_secs=VALUES(alloc_cpu_secs)", + now); + debug3("%d query\n%s", mysql_conn->conn, query); rc = mysql_db_query(mysql_conn->acct_mysql_db, query); xfree(query); diff --git a/src/sreport/cluster_reports.c b/src/sreport/cluster_reports.c index 411f2e0d8af..7741d04e5b8 100644 --- a/src/sreport/cluster_reports.c +++ b/src/sreport/cluster_reports.c @@ -172,49 +172,49 @@ static int _setup_print_fields_list(List format_list) field->type = PRINT_CLUSTER_ACPU; field->name = xstrdup("Allocated"); if(time_format == SREPORT_TIME_SECS_PER) - field->len = 20; + field->len = 18; else - field->len = 12; + field->len = 10; field->print_routine = sreport_print_time; } else if(!strncasecmp("down", object, 1)) { field->type = PRINT_CLUSTER_DCPU; field->name = xstrdup("Down"); if(time_format == SREPORT_TIME_SECS_PER) - field->len = 20; + field->len = 18; else - field->len = 12; + field->len = 10; field->print_routine = sreport_print_time; } else if(!strncasecmp("idle", object, 1)) { field->type = PRINT_CLUSTER_ICPU; field->name = xstrdup("Idle"); if(time_format == SREPORT_TIME_SECS_PER) - field->len = 20; + field->len = 18; else - field->len = 12; - field->print_routine = sreport_print_time; - } else if(!strncasecmp("reserved", object, 1)) { - field->type = PRINT_CLUSTER_RCPU; - field->name = xstrdup("Reserved"); - if(time_format == SREPORT_TIME_SECS_PER) - field->len = 20; - else - field->len = 12; + field->len = 10; field->print_routine = sreport_print_time; } else if(!strncasecmp("overcommited", object, 1)) { field->type = PRINT_CLUSTER_OCPU; field->name = xstrdup("Over Comm"); if(time_format == SREPORT_TIME_SECS_PER) - field->len = 20; + field->len = 18; else - field->len = 12; + field->len = 10; field->print_routine = sreport_print_time; - } else if(!strncasecmp("total", object, 1)) { + } else if(!strncasecmp("reported", object, 3)) { field->type = PRINT_CLUSTER_TOTAL; - field->name = xstrdup("Total Time"); + field->name = xstrdup("Reported"); + if(time_format == SREPORT_TIME_SECS_PER) + field->len = 18; + else + field->len = 10; + field->print_routine = sreport_print_time; + } else if(!strncasecmp("reserved", object, 3)) { + field->type = PRINT_CLUSTER_RCPU; + field->name = xstrdup("Reserved"); if(time_format == SREPORT_TIME_SECS_PER) - field->len = 20; + field->len = 18; else - field->len = 12; + field->len = 10; field->print_routine = sreport_print_time; } else { printf("Unknown field '%s'\n", object); @@ -228,7 +228,7 @@ static int _setup_print_fields_list(List format_list) return SLURM_SUCCESS; } -static List _get_cluster_list(int argc, char *argv[], +static List _get_cluster_list(int argc, char *argv[], uint64_t *total_time, char *report_name, List format_list) { acct_cluster_cond_t *cluster_cond = @@ -251,11 +251,13 @@ static List _get_cluster_list(int argc, char *argv[], end_char, sizeof(end_char)); printf("----------------------------------------" "----------------------------------------\n"); - printf("\t%s %s - %s\n", - report_name, start_char, end_char); + printf(" %s %s - %s (%d secs)\n", + report_name, start_char, end_char, + (cluster_cond->usage_end - cluster_cond->usage_start)); printf("----------------------------------------" "----------------------------------------\n"); } + (*total_time) = cluster_cond->usage_end - cluster_cond->usage_start; destroy_acct_cluster_cond(cluster_cond); if(!cluster_list) @@ -273,6 +275,7 @@ extern int cluster_utilization(int argc, char *argv[]) acct_cluster_rec_t *cluster = NULL; print_field_t *field = NULL; + uint64_t total_time = 0; List cluster_list = NULL; List format_list = list_create(slurm_destroy_char); @@ -280,11 +283,12 @@ extern int cluster_utilization(int argc, char *argv[]) print_fields_list = list_create(destroy_print_field); if(!list_count(format_list)) - addto_char_list(format_list, "Cl,a,d,i,r,o"); + addto_char_list(format_list, "Cl,a,d,i,res,o,rep"); if(!(cluster_list = _get_cluster_list( - argc, argv, "Cluster Utilization Report", format_list))) { + argc, argv, &total_time, + "Cluster Utilization Report", format_list))) { rc = SLURM_ERROR; goto end_it; } @@ -299,7 +303,7 @@ extern int cluster_utilization(int argc, char *argv[]) while((cluster = list_next(itr))) { cluster_accounting_rec_t *accting = NULL; cluster_accounting_rec_t total_acct; - uint32_t total_time = 0; + uint64_t total_reported = 0; if(!cluster->accounting_list || !list_count(cluster->accounting_list)) @@ -319,8 +323,8 @@ extern int cluster_utilization(int argc, char *argv[]) list_iterator_destroy(itr3); total_acct.cpu_count /= list_count(cluster->accounting_list); - - total_time = total_acct.alloc_secs + total_acct.down_secs + total_time *= total_acct.cpu_count; + total_reported = total_acct.alloc_secs + total_acct.down_secs + total_acct.idle_secs + total_acct.resv_secs; while((field = list_next(itr2))) { @@ -339,36 +343,36 @@ extern int cluster_utilization(int argc, char *argv[]) field->print_routine(SLURM_PRINT_VALUE, field, total_acct.alloc_secs, - total_time); + total_reported); break; case PRINT_CLUSTER_DCPU: field->print_routine(SLURM_PRINT_VALUE, field, total_acct.down_secs, - total_time); + total_reported); break; case PRINT_CLUSTER_ICPU: field->print_routine(SLURM_PRINT_VALUE, field, total_acct.idle_secs, - total_time); + total_reported); break; case PRINT_CLUSTER_RCPU: field->print_routine(SLURM_PRINT_VALUE, field, total_acct.resv_secs, - total_time); + total_reported); break; case PRINT_CLUSTER_OCPU: field->print_routine(SLURM_PRINT_VALUE, field, total_acct.over_secs, - total_time); + total_reported); break; case PRINT_CLUSTER_TOTAL: field->print_routine(SLURM_PRINT_VALUE, field, - total_time, + total_reported, total_time); break; default: diff --git a/src/sreport/common.c b/src/sreport/common.c index 358581425d0..fa753664fa7 100644 --- a/src/sreport/common.c +++ b/src/sreport/common.c @@ -40,7 +40,7 @@ #include "sreport.h" extern void sreport_print_time(type_t type, print_field_t *field, - uint32_t value, uint32_t total_time) + uint64_t value, uint64_t total_time) { if(!total_time) total_time = 1; @@ -66,25 +66,25 @@ extern void sreport_print_time(type_t type, print_field_t *field, printf("%-*s ", field->len, " "); } else { char *output = NULL; - float percent = (float)value; + double percent = (double)value; switch(time_format) { case SREPORT_TIME_SECS: - output = xstrdup_printf("%u", value); + output = xstrdup_printf("%llu", value); break; case SREPORT_TIME_PERCENT: percent /= total_time; percent *= 100; - output = xstrdup_printf("%.2f%%", percent); + output = xstrdup_printf("%.2lf%%", percent); break; case SREPORT_TIME_SECS_PER: - percent /= (float)total_time; + percent /= total_time; percent *= 100; - output = xstrdup_printf("%u(%.2f%%)", + output = xstrdup_printf("%llu(%.2lf%%)", value, percent); break; default: - output = xstrdup_printf("%u", value); + output = xstrdup_printf("%llu", value); break; } diff --git a/src/sreport/sreport.h b/src/sreport/sreport.h index 97f113b7f62..f7ea93d615a 100644 --- a/src/sreport/sreport.h +++ b/src/sreport/sreport.h @@ -102,7 +102,7 @@ extern void *db_conn; extern uint32_t my_uid; extern void sreport_print_time(type_t type, print_field_t *field, - uint32_t value, uint32_t total_time); + uint64_t value, uint64_t total_time); extern int parse_option_end(char *option); extern char *strip_quotes(char *option, int *increased); extern void addto_char_list(List char_list, char *names); -- GitLab