diff --git a/NEWS b/NEWS
index 825008e2f5ec88716934ead5c17df1d05d25f93b..168d3f984caa5738e0e962e163a378fd2cc990de 100644
--- a/NEWS
+++ b/NEWS
@@ -5,8 +5,9 @@ documents those changes that are of interest to users and admins.
=============================
-- Removed sched/gang plugin and moved the logic directly into the slurmctld
daemon so that job preemption and gang scheduling can be used with the
- sched/backfill plugin. Added PreemptMode=off|suspend|kill configuration
- parameter to enable/disable the job preemption logic (disabled by default).
+ sched/backfill plugin. Added configuration parameter:
+ PreemptMode=off|suspend|kill|requeue|checkpoint
+ to enable/disable the job preemption logic (disabled by default).
-- If the --partition option is used with the sinfo or squeue command then
print information about even hidden partitions.
-- Replaced misc cpu allocation members in job_info_t with select_job_res_t
diff --git a/doc/html/preempt.shtml b/doc/html/preempt.shtml
index c4c8993105af99baea02b62885d9c057f261b817..a181809966204926891adfa462b0c36b0952e74c 100644
--- a/doc/html/preempt.shtml
+++ b/doc/html/preempt.shtml
@@ -26,6 +26,9 @@ or be preempted by other jobs.-->
SLURM version 2.1 offers the option of requeuing low priority jobs
rather than suspending them, which may permit the preempted jobs to restart
faster by using different resources.
+SLURM version 2.1 also offers the option of checkpointing low priority jobs
+rather than suspending them
+Checkpointed jobs are not automatically requeued or restarted.
Requeuing jobs also releases their memory space for use by other jobs.
In SLURM version 2.1, the job preemption logic was moved directly into the
main code bases to permit use of both job preemption plus the backfill
@@ -73,12 +76,15 @@ limits, it will be canceled in order to prevent it from adversely effecting
other jobs sharing the same resources.
</LI>
<LI>
-<B>PreemptMode</B>: Configure to <I>SUSPEND</I> or <I>KILL</I> depending on
-the desired action for low priority jobs.
+<B>PreemptMode</B>: Configure to <I>CHECKPOINT</I>, <I>SUSPEND</I> or
+<I>REQUEUE</I> depending on the desired action for low priority jobs.
+A value of <I>CHECKPOINT</I> will checkpoint (if possible) or kill low
+priority jobs.
+Checkpointed jobs are not automatically restarted.
+A value of <I>REQUEUE</I> will requeue (if possible) or kill low priority jobs.
+Requeued jobs are permitted to be restarted on different resources.
A value of <I>SUSPEND</I> will suspend and automatically resume the low
priority jobs.
-A value of <I>KILL</I> will requeue (if possible) or kill low priority jobs.
-Requeued jobs are permitted to be restarted on different resources.
</LI>
<LI>
<B>Priority</B>: Configure the partition's <I>Priority</I> setting relative to
@@ -281,6 +287,16 @@ order to support ideal placements such as this, which can quickly complicate
the design. Any and all help is welcome here!
</P>
+<P>
+<B>Better memory management</B>: Some additional work is required to
+better track memory when preempted jobs are checkpointed, requeued or
+killed.
+In those cases, the memory originally assigned to the job is freed.
+Some addition logic is required to properly track memory being freed
+by these preempted jobs, especially for the select/cons_res plugin,
+which typically runs multiple jobs per node.
+</P>
+
<P>
<B>Job preemption based upon Quality of Service (QOS) rather than
partition priority</B>: Granting user's a QOS with preemption capabilities
@@ -293,6 +309,6 @@ It would also necessitate major changes to the shadow bitmap data structures
which are currently used.
</P>
-<p style="text-align:center;">Last modified 9 July 2009</p>
+<p style="text-align:center;">Last modified 14 July 2009</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index f3b2fd49acfff5214c621a7740a8e131c4b42543..e84dcf0405034de18e1b51943ca940098dfc59ef 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -740,8 +740,11 @@ This is the only option compatible with \fBSchedulerType=sched/wiki\fR
or \fBSchedulerType=sched/wiki2\fR (used by Maui and Moab respecitvely,
which provide their own job preemption functionality).
.TP
-\fBKILL\fR
-preempts jobs by killing or requeuing them.
+\fBCHECKPOINT\fR
+preempts jobs by checkpointing them (if possible) or killing them.
+.TP
+\fBREQUEUE\fR
+preempts jobs by requeuing them (if possible) or killing them.
.TP
\fBSUSPEND\fR
preempts jobs by suspending them.
diff --git a/slurm/slurm.h.in b/slurm/slurm.h.in
index 7017cf4738579b6c3e4bc512a69fbb44dcd9a532..a606fdfbf86bea806ddb20e4caef8f80b913632f 100644
--- a/slurm/slurm.h.in
+++ b/slurm/slurm.h.in
@@ -1106,7 +1106,9 @@ typedef struct reservation_name_msg {
#define PREEMPT_MODE_OFF 0x0000 /* disable job preemption */
#define PREEMPT_MODE_SUSPEND 0x0001 /* suspend jobs to preempt */
-#define PREEMPT_MODE_KILL 0x0002 /* requeue or kill jobs to preempt */
+#define PREEMPT_MODE_REQUEUE 0x0002 /* requeue or kill jobs to preempt */
+#define PREEMPT_MODE_CHECKPOINT 0x0003 /* checkpoint job to preempt,
+ * no automatic restart */
typedef struct slurm_ctl_conf {
time_t last_update; /* last update time of the build parameters */
diff --git a/src/api/config_info.c b/src/api/config_info.c
index e846e9212d9d00e9a22515a6bda1b2b72b035f25..8245f212951125fe046184dca76f0fd497bdba66 100644
--- a/src/api/config_info.c
+++ b/src/api/config_info.c
@@ -303,8 +303,10 @@ void slurm_print_ctl_conf ( FILE* out,
fprintf(out, "PreemptMode = OFF\n");
else if (slurm_ctl_conf_ptr->preempt_mode == PREEMPT_MODE_SUSPEND)
fprintf(out, "PreemptMode = SUSPEND\n");
- else if (slurm_ctl_conf_ptr->preempt_mode == PREEMPT_MODE_KILL)
- fprintf(out, "PreemptMode = KILL\n");
+ else if (slurm_ctl_conf_ptr->preempt_mode == PREEMPT_MODE_CHECKPOINT)
+ fprintf(out, "PreemptMode = CHECKPOINT\n");
+ else if (slurm_ctl_conf_ptr->preempt_mode == PREEMPT_MODE_REQUEUE)
+ fprintf(out, "PreemptMode = REQUEUE\n");
if (strcmp(slurm_ctl_conf_ptr->priority_type, "priority/basic") == 0) {
fprintf(out, "PriorityType = %s\n",
diff --git a/src/common/read_config.c b/src/common/read_config.c
index cb915607d1030ab400c48921c20e2bef12eecb17..d8bc858d00b50b6cc820a7d14783398cb69ca933 100644
--- a/src/common/read_config.c
+++ b/src/common/read_config.c
@@ -2126,7 +2126,9 @@ _validate_and_set_defaults(slurm_ctl_conf_t *conf, s_p_hashtbl_t *hashtbl)
conf->preempt_mode = PREEMPT_MODE_SUSPEND;
else if ((strcasecmp(temp_str, "kill") == 0) ||
(strcasecmp(temp_str, "requeue") == 0))
- conf->preempt_mode = PREEMPT_MODE_KILL;
+ conf->preempt_mode = PREEMPT_MODE_REQUEUE;
+ else if (strcasecmp(temp_str, "checkpoint") == 0)
+ conf->preempt_mode = PREEMPT_MODE_CHECKPOINT;
else
fatal("Invalid PreemptMode: %s", temp_str);
}
diff --git a/src/plugins/select/cons_res/select_cons_res.c b/src/plugins/select/cons_res/select_cons_res.c
index 5fe5b5012a498a4f25755e04b15427401fe07a54..da8f24d55d35914479fdb47752cfe6cbe5abc736 100644
--- a/src/plugins/select/cons_res/select_cons_res.c
+++ b/src/plugins/select/cons_res/select_cons_res.c
@@ -255,7 +255,8 @@ extern bool cr_preemption_enabled(void)
uint16_t mode = slurm_get_preempt_mode();
if (mode == PREEMPT_MODE_SUSPEND)
job_preemption_enabled = true;
- else if (mode == PREEMPT_MODE_KILL) {
+ else if ((mode == PREEMPT_MODE_CHECKPOINT) ||
+ (mode == PREEMPT_MODE_REQUEUE)) {
job_preemption_enabled = true;
job_preemption_killing = true;
}
diff --git a/src/plugins/select/linear/select_linear.c b/src/plugins/select/linear/select_linear.c
index 74338d340ae9dec0d00ed92c71bdd1cdc1782182..fcfdd09fc3cbb3ca9f131745a4bf7997bb3012d8 100644
--- a/src/plugins/select/linear/select_linear.c
+++ b/src/plugins/select/linear/select_linear.c
@@ -279,7 +279,8 @@ static inline bool _job_preemption_enabled(void)
uint16_t mode = slurm_get_preempt_mode();
if (mode == PREEMPT_MODE_SUSPEND)
job_preemption_enabled = true;
- else if (mode == PREEMPT_MODE_KILL) {
+ else if ((mode == PREEMPT_MODE_CHECKPOINT) ||
+ (mode == PREEMPT_MODE_REQUEUE)) {
job_preemption_enabled = true;
job_preemption_killing = true;
}
diff --git a/src/slurmctld/gang.c b/src/slurmctld/gang.c
index 4dff244076483f9e4221c635b323ec0d38778e21..90695359f3df597958605a1f69ca0b25c87bfd5a 100644
--- a/src/slurmctld/gang.c
+++ b/src/slurmctld/gang.c
@@ -640,7 +640,7 @@ static void _add_job_to_active(struct job_record *job_ptr,
p_ptr->jobs_active += 1;
}
-static void _suspend_job(uint32_t job_id)
+static int _suspend_job(uint32_t job_id)
{
int rc;
suspend_msg_t msg;
@@ -653,6 +653,7 @@ static void _suspend_job(uint32_t job_id)
error("gang: suspending job %u: %s",
job_id, slurm_strerror(rc));
}
+ return rc;
}
static void _resume_job(uint32_t job_id)
@@ -670,6 +671,39 @@ static void _resume_job(uint32_t job_id)
}
}
+static int _checkpoint_job(uint32_t job_id)
+{
+ int rc;
+ checkpoint_msg_t ckpt_msg;
+
+ /* NOTE: job_checkpoint(VACATE) eventually calls gs_job_fini(),
+ * so we can't process this request in real-time */
+ memset(&ckpt_msg, 0, sizeof(checkpoint_msg_t));
+ ckpt_msg.op = CHECK_VACATE;
+ rc = job_checkpoint(&ckpt_msg, 0, -1);
+ if (rc == SLURM_SUCCESS) {
+ info("gang: preempted job %u has been checkpointed",
+ job_id);
+ }
+
+ return rc;
+}
+
+static int _requeue_job(uint32_t job_id)
+{
+ int rc;
+
+ /* NOTE: job_requeue eventually calls gs_job_fini(),
+ * so we can't process this request in real-time */
+ rc = job_requeue(0, job_id, -1);
+ if (rc == SLURM_SUCCESS) {
+ info("gang: preempted job %u has been requeued",
+ job_id);
+ }
+
+ return rc;
+}
+
void _preempt_job_list_del(void *x)
{
xfree(x);
@@ -684,33 +718,32 @@ static void _preempt_job_queue(uint32_t job_id)
static void _preempt_job_dequeue(void)
{
- int rc;
+ int rc = 0;
uint32_t job_id, *tmp_id;
+ uint16_t preempt_mode = slurm_get_preempt_mode();
while ((tmp_id = list_pop(preempt_job_list))) {
job_id = *tmp_id;
xfree(tmp_id);
- if (slurm_get_preempt_mode() == PREEMPT_MODE_SUSPEND) {
- _suspend_job(job_id);
- continue;
- }
-
- /* NOTE: job_requeue eventually calls gs_job_fini(),
- * so we can't process the request in real-time */
- rc = job_requeue(0, job_id, -1);
- if (rc == SLURM_SUCCESS) {
- info("gang: preempted job %u has been requeued",
- job_id);
- continue;
- }
-
- rc = job_signal(job_id, SIGKILL, 0, 0);
- if (rc == SLURM_SUCCESS)
- info("gang: preempted job %u has been killed", job_id);
- else {
- info("gang: preempted job %u kill failure %s",
- job_id, slurm_strerror(rc));
+ if (preempt_mode == PREEMPT_MODE_SUSPEND)
+ rc = _suspend_job(job_id);
+ else if (preempt_mode == PREEMPT_MODE_REQUEUE)
+ rc = _requeue_job(job_id);
+ else if (preempt_mode == PREEMPT_MODE_CHECKPOINT)
+ rc = _checkpoint_job(job_id);
+ else
+ fatal("Invalid preempt_mode: %u", preempt_mode);
+
+ if (rc != SLURM_SUCCESS) {
+ rc = job_signal(job_id, SIGKILL, 0, 0);
+ if (rc == SLURM_SUCCESS)
+ info("gang: preempted job %u had to be killed",
+ job_id);
+ else {
+ info("gang: preempted job %u kill failure %s",
+ job_id, slurm_strerror(rc));
+ }
}
}
return;