diff --git a/doc/man/man1/slurmctld.1 b/doc/man/man1/slurmctld.1
index 6815073ab5802d4c98196a78dd7d2aa5db346d55..5f99463ec68a4a60f224b7574465aab757a182bc 100644
--- a/doc/man/man1/slurmctld.1
+++ b/doc/man/man1/slurmctld.1
@@ -33,6 +33,10 @@ Write log messages to the specified file.
\fB-s <errlev>\fR
Set syslog logging to the specified level.
.TP
+\fB-r\fR
+Recover \fBslurmctld\fR state from its last checkpoint
+(performed periodically and at time of graceful shutdown).
+.TP
ARGUMENTS
.TP
\fIerrlev\fP
diff --git a/src/common/xstring.h b/src/common/xstring.h
index 8dfdb5bce6105c08b98425615c0de0ef96abc89e..6e0c0375dd66ce099a10a07dcc47ebf97f58eeff 100644
--- a/src/common/xstring.h
+++ b/src/common/xstring.h
@@ -1,7 +1,7 @@
/*****************************************************************************\
* xstring.h - "safe" string processing functions with automatic memory
- * management
-******************************************************************************
+ * management
+ ******************************************************************************
* Copyright (C) 2002 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
* Written by Jim Garlick <garlick@llnl.gov>, et. al.
diff --git a/src/slurmctld/controller.c b/src/slurmctld/controller.c
index 05392f38e401900023b9630aa974900356a4772a..b7fe84aa1e86a14cd9efaf8f1f850f857edadc48 100644
--- a/src/slurmctld/controller.c
+++ b/src/slurmctld/controller.c
@@ -67,6 +67,7 @@ pthread_t thread_id_main = (pthread_t)0;
pthread_t thread_id_rpc = (pthread_t)0;
extern slurm_ssl_key_ctx_t sign_ctx ;
int daemonize = 0;
+int recover = 0;
int msg_from_root (void);
void slurmctld_req ( slurm_msg_t * msg );
@@ -75,6 +76,7 @@ void init_ctld_conf ( slurm_ctl_conf_t * build_ptr );
void parse_commandline( int argc, char* argv[], slurm_ctl_conf_t * );
void *process_rpc ( void * req );
void report_locks_set ( void );
+inline static void save_all_state ( void );
void *slurmctld_background ( void * no_data );
void slurmctld_cleanup (void *context);
void *slurmctld_rpc_mgr( void * no_data );
@@ -133,7 +135,7 @@ main (int argc, char *argv[])
}
init_locks ( );
- if ( ( error_code = read_slurm_conf ()) )
+ if ( ( error_code = read_slurm_conf (recover)) )
fatal ("read_slurm_conf error %d reading %s", error_code, SLURM_CONFIG_FILE);
if ( ( error_code = getnodename (node_name, MAX_NAME_LEN) ) )
fatal ("getnodename errno %d", error_code);
@@ -203,7 +205,7 @@ main (int argc, char *argv[])
break;
case SIGHUP: /* kill -1 */
info ("Reconfigure signal (SIGHUP) received\n");
- error_code = read_slurm_conf ( );
+ error_code = read_slurm_conf (0);
if (error_code)
error ("read_slurm_conf error %d", error_code);
break;
@@ -362,12 +364,12 @@ slurmctld_background ( void * no_data )
report_locks_set ( );
last_checkpoint_time = now;
/* don't lock to insure checkpoint never blocks */
- /* issue call to save state */
+ save_all_state ( );
}
else {
last_checkpoint_time = now;
lock_slurmctld (state_write_lock);
- /* issue call to save state */
+ save_all_state ( );
unlock_slurmctld (state_write_lock);
}
}
@@ -377,6 +379,13 @@ slurmctld_background ( void * no_data )
pthread_exit ((void *)0);
}
+/* save_all_state - save slurmctld state for later recovery */
+void
+save_all_state ( void )
+{
+ dump_all_node_state ( );
+}
+
/* report_locks_set - report any slurmctld locks left set */
void
report_locks_set ( void )
@@ -1116,7 +1125,7 @@ slurm_rpc_reconfigure_controller ( slurm_msg_t * msg )
/* must be user root */
/* do RPC call */
- error_code = read_slurm_conf ( );
+ error_code = read_slurm_conf (0);
/* return result */
if (error_code)
@@ -1388,7 +1397,7 @@ parse_commandline( int argc, char* argv[], slurm_ctl_conf_t * conf_ptr )
char *log_file = NULL;
opterr = 0;
- while ((c = getopt (argc, argv, "de:f:hl:L:s:")) != -1)
+ while ((c = getopt (argc, argv, "de:f:hl:L:rs:")) != -1)
switch (c)
{
case 'd':
@@ -1425,6 +1434,9 @@ parse_commandline( int argc, char* argv[], slurm_ctl_conf_t * conf_ptr )
case 'L':
log_file = optarg;
break;
+ case 'r':
+ recover = 1;
+ break;
case 's':
errlev = strtol (optarg, (char **) NULL, 10);
if ((errlev < LOG_LEVEL_QUIET) ||
@@ -1455,6 +1467,7 @@ usage (char *prog_name)
printf (" -l <errlev> Set logfile logging to the specified level\n");
printf (" -L <file> Set logfile to the supplied file name\n");
printf (" -s <errlev> Set syslog logging to the specified level\n");
+ printf (" -r Recover state from last checkpoint\n");
printf ("<errlev> is an integer between 0 and 7 with higher numbers providing more detail.\n");
}
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index f24a104f5bd88f136dec9d501c9d886fff1f5bd4..d7f5eaae5a885dddccca0c25cd092f472d7715f8 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -74,8 +74,6 @@ void delete_job_desc_files (uint32_t job_id);
void list_delete_job (void *job_entry);
int list_find_job_id (void *job_entry, void *key);
int list_find_job_old (void *job_entry, void *key);
-int mkdir2 (char * path, int modes);
-int rmdir2 (char * path);
int top_priority (struct job_record *job_ptr);
int validate_job_desc ( job_desc_msg_t * job_desc_msg , int allocate ) ;
int write_data_to_file ( char * file_name, char * data ) ;
@@ -768,8 +766,6 @@ copy_job_desc_to_file ( job_desc_msg_t * job_desc , uint32_t job_id )
dir_name = xstrdup (slurmctld_conf . state_save_location);
if (dir_name == NULL)
fatal ("Memory exhausted");
- if (stat (dir_name, &sbuf) == -1) /* create base directory as needed */
- (void) mkdir2 (dir_name, 0744);
/* Create job_id specific directory */
sprintf (job_dir, "/job.%d", job_id);
diff --git a/src/slurmctld/node_mgr.c b/src/slurmctld/node_mgr.c
index a3b63a622d886d0a770cdc9b5d794dc3353bafe2..5d60ceac52ba094bc3f35b7effbc3ae4e8ebcf61 100644
--- a/src/slurmctld/node_mgr.c
+++ b/src/slurmctld/node_mgr.c
@@ -35,16 +35,21 @@
#include <errno.h>
#include <stdio.h>
#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
#include <src/common/hostlist.h>
+#include <src/common/xstring.h>
#include <src/slurmctld/locks.h>
#include <src/slurmctld/slurmctld.h>
-#define BUF_SIZE 1024
+#define BUF_SIZE 4096
List config_list = NULL; /* list of config_record entries */
struct node_record *node_record_table_ptr = NULL; /* location of the node records */
-int *hash_table = NULL; /* table of hashed indicies into node_record */
+int *hash_table = NULL; /* table of hashed indicies into node_record */
struct config_record default_config_record;
struct node_record default_node_record;
time_t last_bitmap_update = (time_t) NULL; /* time of last node creation or deletion */
@@ -55,7 +60,9 @@ bitstr_t *idle_node_bitmap = NULL; /* bitmap of nodes are idle */
int delete_config_record ();
void dump_hash ();
+void dump_node_state (struct node_record *dump_node_ptr, void **buf_ptr, int *buf_len);
int hash_index (char *name);
+void pack_node (struct node_record *dump_node_ptr, void **buf_ptr, int *buf_len);
void split_node_name (char *name, char *prefix, char *suffix, int *index, int *digits);
#if DEBUG_MODULE
@@ -516,6 +523,170 @@ dump_hash ()
}
+/*
+ * dump_all_node_state - save the state of all nodes to file
+ */
+int
+dump_all_node_state ( void )
+{
+ int buf_len, buffer_allocated, buffer_offset = 0, error_code = 0, inx, log_fd;
+ char *buffer;
+ char *old_file, *new_file, *reg_file;
+ void *buf_ptr;
+
+ old_file = xstrdup (slurmctld_conf.state_save_location);
+ xstrcat (old_file, "/node_state.old");
+ new_file = xstrdup (slurmctld_conf.state_save_location);
+ xstrcat (new_file, "/node_state");
+
+ buffer_allocated = (BUF_SIZE*16);
+ buffer = xmalloc(buffer_allocated);
+ buf_ptr = buffer;
+ buf_len = buffer_allocated;
+
+ /* write header: time */
+ pack32 ((uint32_t) time (NULL), &buf_ptr, &buf_len);
+
+ /* write node records to buffer */
+ for (inx = 0; inx < node_record_count; inx++) {
+ if ((node_record_table_ptr[inx].magic != NODE_MAGIC) ||
+ (node_record_table_ptr[inx].config_ptr->magic != CONFIG_MAGIC))
+ fatal ("pack_all_node: data integrity is bad");
+
+ dump_node_state(&node_record_table_ptr[inx], &buf_ptr, &buf_len);
+ if (buf_len > BUF_SIZE)
+ continue;
+ buffer_allocated += (BUF_SIZE*16);
+ buf_len += (BUF_SIZE*16);
+ buffer_offset = (char *)buf_ptr - buffer;
+ xrealloc(buffer, buffer_allocated);
+ buf_ptr = buffer + buffer_offset;
+ }
+
+ /* write the buffer to file */
+ old_file = xstrdup (slurmctld_conf.state_save_location);
+ xstrcat (old_file, "/node_state.old");
+ reg_file = xstrdup (slurmctld_conf.state_save_location);
+ xstrcat (reg_file, "/node_state");
+ new_file = xstrdup (slurmctld_conf.state_save_location);
+ xstrcat (new_file, "/node_state.new");
+ lock_state_files ();
+ log_fd = creat (new_file, 0600);
+ if (log_fd == 0) {
+ error ("Create error %d on file %s, can't save state", errno, new_file);
+ error_code = errno;
+ }
+ else {
+ buf_len = buffer_allocated - buf_len;
+ if (write (log_fd, buffer, buf_len) != buf_len) {
+ error ("Write error %d on file %s, can't save state", errno, new_file);
+ error_code = errno;
+ }
+ close (log_fd);
+ }
+ if (error_code)
+ (void) unlink (new_file);
+ else { /* file shuffle */
+ (void) unlink (old_file);
+ (void) link (reg_file, old_file);
+ (void) unlink (reg_file);
+ (void) link (new_file, reg_file);
+ (void) unlink (new_file);
+ }
+ xfree (old_file);
+ xfree (new_file);
+ unlock_state_files ();
+
+ xfree (buffer);
+ return error_code;
+}
+
+/*
+ * dump_node_state - dump the state of a specific node to a buffer
+ * input: dump_node_ptr - pointer to node for which information is requested
+ * buf_ptr - buffer for node information
+ * buf_len - byte size of buffer
+ * output: buf_ptr - advanced to end of data written
+ * buf_len - byte size remaining in buffer
+ */
+void
+dump_node_state (struct node_record *dump_node_ptr, void **buf_ptr, int *buf_len)
+{
+ packstr (dump_node_ptr->name, buf_ptr, buf_len);
+ pack16 (dump_node_ptr->node_state, buf_ptr, buf_len);
+ pack32 (dump_node_ptr->config_ptr->cpus, buf_ptr, buf_len);
+ pack32 (dump_node_ptr->config_ptr->real_memory, buf_ptr, buf_len);
+ pack32 (dump_node_ptr->config_ptr->tmp_disk, buf_ptr, buf_len);
+}
+
+/*
+ * load_node_state - load the node state from file, recover from slurmctld restart.
+ * execute this after loading the configuration file data.
+ */
+int
+load_node_state ( void )
+{
+ char *node_name, *state_file;
+ int buffer_allocated, buffer_used = 0, error_code = 0;
+ int state_fd;
+ uint16_t node_state, name_len;
+ uint32_t time, cpus, real_memory, tmp_disk;
+ struct node_record *node_ptr;
+ uint32_t buffer_size = 0;
+ char *buffer;
+ void *buf_ptr;
+
+ /* read the file */
+ state_file = xstrdup (slurmctld_conf.state_save_location);
+ xstrcat (state_file, "/node_state");
+ lock_state_files ();
+ state_fd = open (state_file, O_RDONLY);
+ if (state_fd < 0) {
+ info ("No node_state file (%s) to recover", state_file);
+ error_code = ENOENT;
+ }
+ else {
+ buffer_allocated = BUF_SIZE;
+ buffer = xmalloc(buffer_allocated);
+ buf_ptr = buffer;
+ while ((buffer_used = read (state_fd, buf_ptr, BUF_SIZE)) == BUF_SIZE) {
+ buffer_size += buffer_used;
+ buffer_allocated += BUF_SIZE;
+ xrealloc(buffer, buffer_allocated);
+ buf_ptr = (void *) (buffer + buffer_size);
+ }
+ buffer_size += buffer_used;
+ close (state_fd);
+ if (buffer_used < 0)
+ error ("Read error %d on %s", errno, state_file);
+ }
+ xfree (state_file);
+ unlock_state_files ();
+
+ if (buffer_size > sizeof (uint32_t))
+ unpack32 (&time, &buf_ptr, &buffer_size);
+
+ while (buffer_size >= (4 *sizeof (uint32_t))) {
+ unpackstr_xmalloc (&node_name, &name_len, &buf_ptr, &buffer_size);
+ unpack16 (&node_state, &buf_ptr, &buffer_size);
+ unpack32 (&cpus, &buf_ptr, &buffer_size);
+ unpack32 (&real_memory, &buf_ptr, &buffer_size);
+ unpack32 (&tmp_disk, &buf_ptr, &buffer_size);
+
+ /* find record and perform update */
+ node_ptr = find_node_record (node_name);
+ if (node_ptr) {
+ node_ptr->node_state = node_state;
+ node_ptr->cpus = cpus;
+ node_ptr->real_memory = real_memory;
+ node_ptr->tmp_disk = tmp_disk;
+ }
+ if (node_name)
+ xfree (node_name);
+ }
+ return error_code;
+}
+
/*
* find_node_record - find a record for node with specified name
* input: name - name of the desired node
@@ -785,7 +956,6 @@ node_name2bitmap (char *node_names, bitstr_t **bitmap)
* update_time - set to time partition records last updated
* global: node_record_table_ptr - pointer to global node table
* NOTE: the caller must xfree the buffer at *buffer_ptr when no longer required
- * NOTE: change NODE_STRUCT_VERSION in common/slurmlib.h whenever the format changes
* NOTE: change slurm_load_node() in api/node_info.c whenever the data format changes
*/
void
@@ -809,7 +979,7 @@ pack_all_node (char **buffer_ptr, int *buffer_size, time_t * update_time)
buf_ptr = buffer;
buf_len = buffer_allocated;
- /* write haeader: version and time */
+ /* write header: version and time */
nodes_packed = 0 ;
pack32 ((uint32_t) nodes_packed, &buf_ptr, &buf_len);
pack32 ((uint32_t) last_node_update, &buf_ptr, &buf_len);
@@ -821,17 +991,14 @@ pack_all_node (char **buffer_ptr, int *buffer_size, time_t * update_time)
fatal ("pack_all_node: data integrity is bad");
pack_node(&node_record_table_ptr[inx], &buf_ptr, &buf_len);
+ nodes_packed ++ ;
if (buf_len > BUF_SIZE)
- {
- nodes_packed ++ ;
continue;
- }
buffer_allocated += (BUF_SIZE*16);
buf_len += (BUF_SIZE*16);
buffer_offset = (char *)buf_ptr - buffer;
xrealloc(buffer, buffer_allocated);
buf_ptr = buffer + buffer_offset;
- nodes_packed ++ ;
}
unlock_slurmctld (node_read_lock);
@@ -857,8 +1024,8 @@ pack_all_node (char **buffer_ptr, int *buffer_size, time_t * update_time)
* buf_len - byte size of buffer
* output: buf_ptr - advanced to end of data written
* buf_len - byte size remaining in buffer
- * NOTE: if you make any changes here be sure to increment the value of NODE_STRUCT_VERSION
- * and make the corresponding changes to load_node_config in api/node_info.c
+ * NOTE: if you make any changes here be sure to make the corresponding changes to
+ * load_node_config in api/node_info.c
*/
void
pack_node (struct node_record *dump_node_ptr, void **buf_ptr, int *buf_len)
diff --git a/src/slurmctld/partition_mgr.c b/src/slurmctld/partition_mgr.c
index b4bda9fd4c963f2d01f9f4c4eda7243fd782ab70..9728cde7ece2d1f7fd7e1939c85698530d86d3b0 100644
--- a/src/slurmctld/partition_mgr.c
+++ b/src/slurmctld/partition_mgr.c
@@ -50,6 +50,7 @@ struct part_record *default_part_loc = NULL; /* location of default partition */
time_t last_part_update; /* time of last update to partition records */
int build_part_bitmap (struct part_record *part_record_point);
+void dump_part_state (struct part_record *part_record_point, void **buf_ptr, int *buf_len);
void list_delete_part (void *part_entry);
int list_find_part (void *part_entry, void *key);
@@ -316,6 +317,148 @@ delete_part_record (char *name)
}
+/*
+ * dump_all_part_state - save the state of all partitions to file
+ */
+int
+dump_all_part_state ( void )
+{
+ ListIterator part_record_iterator;
+ struct part_record *part_record_point;
+ int buf_len, buffer_allocated, buffer_offset = 0;
+ char *buffer;
+ void *buf_ptr;
+ /* Locks: Read partition */
+ slurmctld_lock_t part_read_lock = { NO_LOCK, NO_LOCK, NO_LOCK, READ_LOCK };
+
+ lock_slurmctld (part_read_lock);
+
+ buffer_allocated = (BUF_SIZE*16);
+ buffer = xmalloc(buffer_allocated);
+ buf_ptr = buffer;
+ buf_len = buffer_allocated;
+
+ /* write header: time */
+ pack32 ((uint32_t) time (NULL), &buf_ptr, &buf_len);
+
+ /* write partition records to buffer */
+ part_record_iterator = list_iterator_create (part_list);
+ while ((part_record_point = (struct part_record *) list_next (part_record_iterator))) {
+ if (part_record_point->magic != PART_MAGIC)
+ fatal ("pack_all_part: data integrity is bad");
+
+ dump_part_state (part_record_point, &buf_ptr, &buf_len);
+ if (buf_len > BUF_SIZE)
+ continue;
+ buffer_allocated += (BUF_SIZE*16);
+ buf_len += (BUF_SIZE*16);
+ buffer_offset = (char *)buf_ptr - buffer;
+ xrealloc(buffer, buffer_allocated);
+ buf_ptr = buffer + buffer_offset;
+ }
+ list_iterator_destroy (part_record_iterator);
+ unlock_slurmctld (part_read_lock);
+
+ /* write the buffer to file */
+ lock_state_files ();
+
+ unlock_state_files ();
+
+ xfree (buffer);
+ return 0;
+}
+
+/*
+ * dump_part_state - dump the state of a specific partition to a buffer
+ * input: part_record_point - pointer to partition for which information is requested
+ * buf_ptr - buffer for node information
+ * buf_len - byte size of buffer
+ * output: buf_ptr - advanced to end of data written
+ * buf_len - byte size remaining in buffer
+ */
+void
+dump_part_state (struct part_record *part_record_point, void **buf_ptr, int *buf_len)
+{
+ uint16_t default_part_flag;
+
+ if (default_part_loc == part_record_point)
+ default_part_flag = 1;
+ else
+ default_part_flag = 0;
+
+ packstr (part_record_point->name, buf_ptr, buf_len);
+ pack32 (part_record_point->max_time, buf_ptr, buf_len);
+ pack32 (part_record_point->max_nodes, buf_ptr, buf_len);
+
+ pack16 (default_part_flag, buf_ptr, buf_len);
+ pack16 ((uint16_t)part_record_point->root_only, buf_ptr, buf_len);
+ pack16 ((uint16_t)part_record_point->shared, buf_ptr, buf_len);
+
+ pack16 ((uint16_t)part_record_point->state_up, buf_ptr, buf_len);
+ packstr (part_record_point->allow_groups, buf_ptr, buf_len);
+ packstr (part_record_point->nodes, buf_ptr, buf_len);
+}
+
+/*
+ * load_part_state - load the partition state from file, recover from slurmctld restart.
+ * execute this after loading the configuration file data.
+ */
+int
+load_part_state ( void )
+{
+ char *part_name, *allow_groups, *nodes;
+ uint32_t dump_time, max_time, max_nodes;
+ uint16_t name_len, def_part_flag, root_only, shared, state_up;
+ struct part_record *part_ptr;
+ uint32_t buffer_size;
+ void *buf_ptr;
+
+ /* read the file */
+ lock_state_files ();
+ unpack32 (&dump_time, &buf_ptr, &buffer_size);
+
+ while (1) {
+ unpackstr_xmalloc (&part_name, &name_len, &buf_ptr, &buffer_size);
+ unpack32 (&max_time, &buf_ptr, &buffer_size);
+ unpack32 (&max_nodes, &buf_ptr, &buffer_size);
+ unpack16 (&def_part_flag, &buf_ptr, &buffer_size);
+ unpack16 (&root_only, &buf_ptr, &buffer_size);
+ unpack16 (&shared, &buf_ptr, &buffer_size);
+ unpack16 (&state_up, &buf_ptr, &buffer_size);
+ unpackstr_xmalloc (&allow_groups, &name_len, &buf_ptr, &buffer_size);
+ unpackstr_xmalloc (&nodes, &name_len, &buf_ptr, &buffer_size);
+
+ /* find record and perform update */
+ part_ptr = list_find_first (part_list, &list_find_part, part_name);
+
+ if (part_ptr == NULL) {
+ error ("load_part_state: partition %s removed from configuration file.",
+ part_name);
+ }
+ else {
+ part_ptr->max_time = max_time;
+ part_ptr->max_nodes = max_nodes;
+ if (def_part_flag) {
+ strcpy (default_part_name, part_name);
+ default_part_loc = part_ptr;
+ }
+ part_ptr->root_only = root_only;
+ part_ptr->shared = shared;
+ part_ptr->state_up = state_up;
+ if (part_ptr->allow_groups)
+ xfree (part_ptr->allow_groups);
+ part_ptr->allow_groups = allow_groups;
+ if (part_ptr->nodes)
+ xfree (part_ptr->nodes);
+ part_ptr->allow_groups = nodes;
+ }
+
+ if (part_name)
+ xfree (part_name);
+ }
+ unlock_state_files ();
+}
+
/*
* find_part_record - find a record for partition with specified name
* input: name - name of the desired partition
@@ -475,17 +618,14 @@ pack_all_part (char **buffer_ptr, int *buffer_size, time_t * update_time)
fatal ("pack_all_part: data integrity is bad");
pack_part(part_record_point, &buf_ptr, &buf_len);
+ parts_packed ++ ;
if (buf_len > BUF_SIZE)
- {
- parts_packed ++ ;
continue;
- }
buffer_allocated += (BUF_SIZE*16);
buf_len += (BUF_SIZE*16);
buffer_offset = (char *)buf_ptr - buffer;
xrealloc(buffer, buffer_allocated);
buf_ptr = buffer + buffer_offset;
- parts_packed ++ ;
}
list_iterator_destroy (part_record_iterator);
diff --git a/src/slurmctld/read_config.c b/src/slurmctld/read_config.c
index bce28bb8cb3a1e59d9852cb6f11e89579726148e..297fffbeafa67f36b3383899d9875483298a1c62 100644
--- a/src/slurmctld/read_config.c
+++ b/src/slurmctld/read_config.c
@@ -1,5 +1,5 @@
/*****************************************************************************\
- * read_config.c - read the overall slurm configuration file
+ * read_config.c - read the overall slurm configuration file
*****************************************************************************
* Copyright (C) 2002 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
@@ -34,6 +34,8 @@
#include <stdio.h>
#include <string.h>
#include <syslog.h>
+#include <sys/types.h>
+#include <sys/stat.h>
#include <unistd.h>
#include <src/common/hostlist.h>
@@ -73,7 +75,7 @@ main (int argc, char *argv[]) {
exit (1);
}
- error_code = read_slurm_conf ( );
+ error_code = read_slurm_conf (0);
if (error_code) {
printf ("ERROR %d from read_slurm_conf\n", error_code);
exit (error_code);
@@ -149,7 +151,7 @@ main (int argc, char *argv[]) {
sleep (5);
}
for (i = 0; i < cycles; i++) {
- error_code = read_slurm_conf ( );
+ error_code = read_slurm_conf (0);
if (error_code) {
printf ("ERROR %d from read_slurm_conf\n",
error_code);
@@ -361,6 +363,7 @@ parse_config_spec (char *in_line)
char *job_credential_private_key = NULL , *job_credential_public_certificate = NULL;
long first_job_id = 0;
struct servent *servent;
+ struct stat sbuf;
error_code = slurm_parser(in_line,
"BackupController=", 's', &backup_controller,
@@ -455,6 +458,8 @@ parse_config_spec (char *in_line)
if ( slurmctld_conf.state_save_location )
xfree (slurmctld_conf.state_save_location);
slurmctld_conf.state_save_location = state_save_location;
+ if (stat (state_save_location, &sbuf) == -1) /* create directory as needed */
+ (void) mkdir2 (state_save_location, 0744);
}
if ( tmp_fs ) {
@@ -831,10 +836,11 @@ parse_part_spec (char *in_line) {
/*
* read_slurm_conf - load the slurm configuration from the configured file.
* read_slurm_conf can be called more than once if so desired.
+ * input: recover - set to use state saved from last slurmctld shutdown
* output: return - 0 if no error, otherwise an error code
*/
int
-read_slurm_conf ( ) {
+read_slurm_conf (int recover) {
clock_t start_time;
FILE *slurm_spec_file; /* pointer to input data file */
int line_num; /* line number in input file */
@@ -940,6 +946,11 @@ read_slurm_conf ( ) {
rehash ();
set_slurmd_addr ();
+
+ if (recover) {
+ (void) load_node_state ();
+ }
+
if ((error_code = build_bitmaps ())) {
unlock_slurmctld (config_write_lock);
return error_code;
diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h
index ef4d0a3d56c664d821e2dc81d539220993a0021c..0982e9d35630a710f848550cfa92e299b493dd00 100644
--- a/src/slurmctld/slurmctld.h
+++ b/src/slurmctld/slurmctld.h
@@ -259,6 +259,12 @@ extern int delete_part_record (char *name);
/* delete_step_record - delete record for job step for specified job_ptr and step_id */
extern int delete_step_record (struct job_record *job_ptr, uint32_t step_id);
+/* dump_all_node_state - save the state of all nodes to file */
+extern int dump_all_node_state ( void );
+
+/* dump_all_part_state - save the state of all partitions to file */
+extern int dump_all_part_state ( void );
+
/* dump_job_desc - dump the incoming job submit request message */
extern void dump_job_desc(job_desc_msg_t * job_specs);
@@ -342,12 +348,21 @@ extern void list_delete_part (void *part_entry);
/* list_find_part - find an entry in the partition list */
extern int list_find_part (void *part_entry, void *key);
+/* load_node_state - load the node state from file, recover from slurmctld restart */
+extern int load_node_state ( void );
+
+/* load_part_state - load the partition state from file, recover from slurmctld restart */
+extern int load_node_state ( void );
+
/* match_feature - determine if the desired feature (seek) is one of those available */
extern int match_feature (char *seek, char *available);
/* match_group - determine if the user is a member of any groups permitted to use this partition */
extern int match_group (char *allow_groups, char *user_groups);
+/* mkdir2 - issues system calls for mkdir (if root) */
+int mkdir2 (char * path, int modes);
+
/* node_name2bitmap - given a node name regular expression, build a bitmap representation */
extern int node_name2bitmap (char *node_names, bitstr_t **bitmap);
@@ -389,11 +404,6 @@ extern void pack_all_part (char **buffer_ptr, int *buffer_size, time_t * update_
*/
extern void pack_job (struct job_record *dump_job_ptr, void **buf_ptr, int *buf_len);
-/* pack_node - dump all configuration information about a specific node in
- * machine independent form (for network transmission)
- */
-extern void pack_node (struct node_record *dump_node_ptr, void **buf_ptr, int *buf_len);
-
/*
* pack_part - dump all configuration information about a specific partition in
* machine independent form (for network transmission)
@@ -407,7 +417,7 @@ extern void pack_part (struct part_record *part_record_point, void **buf_ptr, in
void purge_old_job (void);
/* read_slurm_conf - load the slurm configuration from the configured file */
-extern int read_slurm_conf ( );
+extern int read_slurm_conf (int recover);
/* rehash - build a hash table of the node_record entries */
extern void rehash ();
@@ -415,6 +425,9 @@ extern void rehash ();
/* reset_job_bitmaps - reestablish bitmaps for existing jobs */
extern void reset_job_bitmaps ();
+/* rmdir2 - issues system call to rmdir (if root) */
+extern int rmdir2 (char * path);
+
/* schedule - attempt to schedule all pending jobs */
void schedule();