diff --git a/src/slurmctld/controller.c b/src/slurmctld/controller.c
index 4a73ba504abbfa26bb8a945675cd5c14c8d8deae..18872ff6ebaccac4365f197c4a00eb82bebf1301 100644
--- a/src/slurmctld/controller.c
+++ b/src/slurmctld/controller.c
@@ -966,8 +966,7 @@ static void _slurm_rpc_job_step_kill(slurm_msg_t * msg)
(long) (clock() - start_time));
slurm_send_rc_msg(msg, SLURM_SUCCESS);
- /* Below functions provide their own locking */
- schedule();
+ /* Below function provides its own locking */
(void) dump_all_job_state();
}
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index 3915097c1a1854f9a71c6894cdd4a80f11649d38..cb10f0e4c635e2d3ff48d966519124f517000d70 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -458,16 +458,16 @@ static int _load_job_state(Buf buffer)
job_id, job_state, batch_flag);
goto unpack_error;
}
- if ((kill_on_node_fail > 1) || (kill_on_step_done > 1)) {
- error("Invalid data for job %u: kill_on_node_fail=%u",
- job_id, kill_on_node_fail);
- goto unpack_error;
- }
- if ((kill_on_node_fail > 1) || (kill_on_step_done > 1)) {
+ if (kill_on_step_done > KILL_IN_PROGRESS) {
error("Invalid data for job %u: kill_on_step_done=%u",
job_id, kill_on_step_done);
goto unpack_error;
}
+ if (kill_on_node_fail > 1) {
+ error("Invalid data for job %u: kill_on_node_fail=%u",
+ job_id, kill_on_node_fail);
+ goto unpack_error;
+ }
if ((nodes) && (node_name2bitmap(nodes, &node_bitmap))) {
error("_load_job_state: invalid nodes (%s) for job_id %u",
nodes, job_id);
@@ -1113,6 +1113,7 @@ int job_allocate(job_desc_msg_t * job_specs, uint32_t * new_job_id,
int job_signal(uint32_t job_id, uint16_t signal, uid_t uid)
{
struct job_record *job_ptr;
+ time_t now = time(NULL);
job_ptr = find_job_record(job_id);
if (job_ptr == NULL) {
@@ -1122,7 +1123,8 @@ int job_signal(uint32_t job_id, uint16_t signal, uid_t uid)
if ((job_ptr->job_state == JOB_FAILED) ||
(job_ptr->job_state == JOB_COMPLETE) ||
- (job_ptr->job_state == JOB_TIMEOUT))
+ (job_ptr->job_state == JOB_TIMEOUT) ||
+ (job_ptr->kill_on_step_done & KILL_IN_PROGRESS))
return ESLURM_ALREADY_DONE;
if ((job_ptr->user_id != uid) && (uid != 0) && (uid != getuid())) {
@@ -1133,7 +1135,7 @@ int job_signal(uint32_t job_id, uint16_t signal, uid_t uid)
if ((job_ptr->job_state == JOB_PENDING) &&
(signal == SIGKILL)) {
- last_job_update = time(NULL);
+ last_job_update = now;
job_ptr->job_state = JOB_FAILED;
job_ptr->start_time = job_ptr->end_time = time(NULL);
delete_job_details(job_ptr);
@@ -1155,14 +1157,16 @@ int job_signal(uint32_t job_id, uint16_t signal, uid_t uid)
}
list_iterator_destroy (step_record_iterator);
- if (signal == SIGKILL) {
- job_ptr->kill_on_step_done = 1;
- last_job_update = time(NULL);
+ if ((signal == SIGKILL) &&
+ ((job_ptr->kill_on_step_done & KILL_IN_PROGRESS) == 0)) {
+ job_ptr->kill_on_step_done = KILL_IN_PROGRESS;
+ job_ptr->time_last_active = now;
+ last_job_update = now;
}
if ((signal == SIGKILL) && (step_cnt == 0)) {
/* kill job with no active steps */
job_ptr->job_state = JOB_COMPLETE;
- job_ptr->end_time = time(NULL);
+ job_ptr->end_time = now;
deallocate_nodes(job_ptr);
delete_job_details(job_ptr);
}
@@ -1190,6 +1194,7 @@ job_complete(uint32_t job_id, uid_t uid, bool requeue,
uint32_t job_return_code)
{
struct job_record *job_ptr;
+ time_t now = time(NULL);
job_ptr = find_job_record(job_id);
if (job_ptr == NULL) {
@@ -1224,13 +1229,15 @@ job_complete(uint32_t job_id, uid_t uid, bool requeue,
} else {
if (job_return_code)
job_ptr->job_state = JOB_FAILED;
+ else if (job_ptr->end_time < now)
+ job_ptr->job_state = JOB_TIMEOUT;
else
job_ptr->job_state = JOB_COMPLETE;
- job_ptr->end_time = time(NULL);
+ job_ptr->end_time = now;
delete_job_details(job_ptr);
delete_all_step_records(job_ptr);
}
- last_job_update = time(NULL);
+ last_job_update = now;
return SLURM_SUCCESS;
}
@@ -1861,6 +1868,22 @@ void job_time_limit(void)
(job_ptr->job_state == JOB_TIMEOUT) ||
(job_ptr->job_state == JOB_NODE_FAIL))
continue;
+
+ if ((job_ptr->kill_on_step_done & KILL_IN_PROGRESS) &&
+ (difftime(now, job_ptr->time_last_active) >
+ JOB_KILL_TIMEOUT)) {
+ info("Job_id %u not properly terminating, forcing it",
+ job_ptr->job_id);
+ last_job_update = now;
+ job_ptr->job_state = JOB_TIMEOUT;
+ job_ptr->end_time = time(NULL);
+ deallocate_nodes(job_ptr);
+ delete_all_step_records(job_ptr);
+ delete_job_details(job_ptr);
+ }
+ if (job_ptr->kill_on_step_done & KILL_IN_PROGRESS)
+ continue;
+
if (slurmctld_conf.inactive_limit) {
if (job_ptr->step_list &&
(list_count(job_ptr->step_list) > 0))
@@ -1876,13 +1899,11 @@ void job_time_limit(void)
(job_ptr->end_time > now))
continue;
last_job_update = now;
- info("Time limit exhausted for job_id %u, terminated",
+ info("Time limit exhausted for job_id %u, terminating",
job_ptr->job_id);
- job_ptr->job_state = JOB_TIMEOUT;
- job_ptr->end_time = time(NULL);
- deallocate_nodes(job_ptr);
- delete_all_step_records(job_ptr);
- delete_job_details(job_ptr);
+ job_signal(job_ptr->job_id, SIGKILL, 0);
+ if (job_ptr->job_state == JOB_COMPLETE)
+ job_ptr->job_state = JOB_TIMEOUT;
}
list_iterator_destroy(job_record_iterator);
diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h
index c8d8577bfd28736f0b7dcb2a1603f7678ef4a599..fb3315ed094acca7b4ce98e0eaefb63e5faf5f65 100644
--- a/src/slurmctld/slurmctld.h
+++ b/src/slurmctld/slurmctld.h
@@ -85,6 +85,10 @@
/* Check for jobs reaching their time limit every PERIODIC_TIMEOUT seconds */
#define PERIODIC_TIMEOUT 60
+/* Release a job's allocation if it does not terminate gracefully in
+ * JOB_KILL_TIMEOUT seconds */
+#define JOB_KILL_TIMEOUT 60
+
/* Pathname of group file record for checking update times */
#define GROUP_FILE "/etc/group"
@@ -192,6 +196,8 @@ extern time_t last_job_update; /* time of last update to part records */
#define DETAILS_MAGIC 0xdea84e7
#define JOB_MAGIC 0xf0b7392c
#define STEP_MAGIC 0xce593bc1
+#define KILL_ON_STEP_DONE 1
+#define KILL_IN_PROGRESS 2
extern int job_count; /* number of jobs in the system */
@@ -234,7 +240,8 @@ struct job_record {
uint16_t kill_on_node_fail; /* 1 if job should be killed on
node failure */
uint16_t kill_on_step_done; /* 1 if job should be killed when
- the job step completes */
+ the job step completes, 2 if kill
+ in progress */
char *nodes; /* list of nodes allocated to job */
bitstr_t *node_bitmap; /* bitmap of nodes allocated to job */
uint32_t time_limit; /* time_limit minutes or INFINITE */