From 3b5ad76736a48d3d7e8acb6c992f13fb90493d8e Mon Sep 17 00:00:00 2001
From: Moe Jette <jette1@llnl.gov>
Date: Tue, 27 Nov 2007 19:59:55 +0000
Subject: [PATCH] Added support for richer job dependency specification.
---
NEWS | 2 +
RELEASE_NOTES | 3 +-
doc/html/news.shtml | 4 +-
doc/man/man1/salloc.1 | 29 +++-
doc/man/man1/sbatch.1 | 27 +++-
doc/man/man1/srun.1 | 34 ++++-
slurm/slurm.h.in | 4 +-
src/api/init_msg.c | 2 +-
src/api/job_info.c | 33 +++--
src/common/slurm_errno.c | 2 +-
src/common/slurm_protocol_defs.c | 2 +
src/common/slurm_protocol_pack.c | 24 ++-
src/plugins/sched/wiki/job_modify.c | 43 ++----
src/plugins/sched/wiki2/get_jobs.c | 2 +-
src/plugins/sched/wiki2/job_modify.c | 43 ++----
src/salloc/opt.c | 14 +-
src/salloc/opt.h | 2 +-
src/salloc/salloc.c | 3 +-
src/sbatch/opt.c | 14 +-
src/sbatch/opt.h | 2 +-
src/sbatch/sbatch.c | 3 +-
src/scontrol/update_job.c | 4 +-
src/slurmctld/job_mgr.c | 74 +++++-----
src/slurmctld/job_scheduler.c | 213 +++++++++++++++++++++++++++
src/slurmctld/read_config.c | 29 ++++
src/slurmctld/slurmctld.h | 35 ++++-
src/squeue/print.c | 9 +-
src/srun/opt.c | 17 +--
src/srun/opt.h | 2 +-
src/sview/job_info.c | 21 +--
testsuite/expect/test1.42 | 6 +-
testsuite/expect/test15.14 | 6 +-
testsuite/expect/test17.18 | 6 +-
33 files changed, 515 insertions(+), 199 deletions(-)
diff --git a/NEWS b/NEWS
index c5c4b663c18..6d5c8e37bae 100644
--- a/NEWS
+++ b/NEWS
@@ -22,6 +22,8 @@ documents those changes that are of interest to users and admins.
information is not found in the "slurm.conf" file, but a scheduler plugin
specific configuration (e.g. "wiki.conf").
-- sview partition information reported now includes partition priority.
+ -- Expand job dependency specification to support concurrent execution,
+ testing of job exit status and multiple job IDs.
* Changes in SLURM 1.3.0-pre4
=============================
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 6ba5ad4c19d..57e4daac9ca 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -40,6 +40,8 @@ COMMAND CHANGES
* sacct -c can now be used to view job completion data
* scontrol "notify" command added to send message to stdout of srun for
specified job id.
+* Support has been added for a much richer job dependency specification
+ including testing of exit codes and multiple dependencies.
CONFIGURATION FILE CHANGES
@@ -73,7 +75,6 @@ CONFIGURATION FILE CHANGES
* The partition MaxTime format now accepts minutes, minutes:seconds,
hours:minutes:seconds, days-hours, days-hours:minutes,
days-hours:minutes:seconds or "UNLIMITED".
-
* See "man slurm.conf" for more information.
OTHER CHANGES
diff --git a/doc/html/news.shtml b/doc/html/news.shtml
index 0b2f792e767..9d3a675c198 100644
--- a/doc/html/news.shtml
+++ b/doc/html/news.shtml
@@ -80,6 +80,8 @@ without an external scheduler).</li>
<i>salloc</i>, <i>sattach</i> or <i>sbatch</i> commands instead).</li>
<li><i>srun --pty</i> option added to support remote pseudo terminial for
spawned tasks.</li>
+<li>Support added for a much richer job dependency specification
+including testing of exit codes and multiple dependencies.</li>
</ul>
<h2><a name="14">Major Updates in SLURM Version 1.4 and beyond</a></h2>
@@ -93,6 +95,6 @@ to coordinate activies. Future development plans includes:
and refresh.</li>
</ul>
-<p style="text-align:center;">Last modified 24 October 2007</p>
+<p style="text-align:center;">Last modified 27 November 2007</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/man/man1/salloc.1 b/doc/man/man1/salloc.1
index 97679079d54..31e63022595 100644
--- a/doc/man/man1/salloc.1
+++ b/doc/man/man1/salloc.1
@@ -1,4 +1,4 @@
-.TH "salloc" "1" "SLURM 1.3" "September 2007" "SLURM Commands"
+.TH "salloc" "1" "SLURM 1.3" "November 2007" "SLURM Commands"
.SH "NAME"
.LP
salloc \- Obtain a SLURM job allocation (a set of nodes), execute a command, and then release the allocation when the command is finished.
@@ -117,11 +117,32 @@ the \-\-cpus\-per\-task=3 options, the controller knows that each task requires
of 4 nodes, one for each of the 4 tasks.
.TP
-\fB\-d\fR, \fB\-\-dependency\fR[=]<\fIjobid\fR>
-Defer the start of this job until the specified \fIjobid\fR has completed.
+\fB\-d\fR, \fB\-\-dependency\fR[=]<\fIdependency_list\fR>
+Defer the start of this job until the specified dependencies have been
+satisfied completed.
+<\fIdependency_list\fR> is of the form
+<\fItype:job_id[:job_id][,type:job_id[:job_id]]\fR>.
Many jobs can share the same dependency and these jobs may even belong to
-different users. The value may be changed after job submission using the
+different users. The value may be changed after job submission using the
scontrol command.
+.PD
+.RS
+.TP
+\fBafter:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have begun
+execution.
+.TP
+\fBafterany:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have terminated.
+.TP
+\fBafternotok:job_id[:jobid...\fR
+This job can begin execution after the specified jobs have terminated
+in some failed state (non-zero exit code, node failure, timed out, etc).
+.TP
+\fBafterok:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have successfully
+executed (ran to completion with non-zero exit code).
+.RE
.TP
\fB\-\-exclusive\fR
diff --git a/doc/man/man1/sbatch.1 b/doc/man/man1/sbatch.1
index 382717546d8..f2410b8c666 100644
--- a/doc/man/man1/sbatch.1
+++ b/doc/man/man1/sbatch.1
@@ -125,11 +125,32 @@ Set the working directory of the batch script to \fIdirectory\fR before
it it executed.
.TP
-\fB\-d\fR, \fB\-\-dependency\fR[=]<\fIjobid\fR>
-Defer the start of this job until the specified \fIjobid\fR has completed.
+\fB\-d\fR, \fB\-\-dependency\fR[=]<\fIdependency_list\fR>
+Defer the start of this job until the specified dependencies have been
+satisfied completed.
+<\fIdependency_list\fR> is of the form
+<\fItype:job_id[:job_id][,type:job_id[:job_id]]\fR>.
Many jobs can share the same dependency and these jobs may even belong to
-different users. The value may be changed after job submission using the
+different users. The value may be changed after job submission using the
scontrol command.
+.PD
+.RS
+.TP
+\fBafter:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have begun
+execution.
+.TP
+\fBafterany:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have terminated.
+.TP
+\fBafternotok:job_id[:jobid...\fR
+This job can begin execution after the specified jobs have terminated
+in some failed state (non-zero exit code, node failure, timed out, etc).
+.TP
+\fBafterok:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have successfully
+executed (ran to completion with non-zero exit code).
+.RE
.TP
\fB\-e\fR, \fB\-\-error\fR[=]<\fIfilename pattern\fR>
diff --git a/doc/man/man1/srun.1 b/doc/man/man1/srun.1
index 2fd5e5dca8b..438014bc6c4 100644
--- a/doc/man/man1/srun.1
+++ b/doc/man/man1/srun.1
@@ -1,6 +1,6 @@
\." $Id$
.\"
-.TH SRUN "1" "September 2007" "srun 1.3" "slurm components"
+.TH SRUN "1" "November 2007" "srun 1.3" "slurm components"
.SH "NAME"
srun \- run parallel jobs
@@ -657,13 +657,33 @@ If the specified file already exists, it will be overwritten.
If \fB\-\-error\fR is not also specified on the command line, both
stdout and stderr will directed to the file specified by \fB\-\-output\fR.
+.TP
+\fB\-P\fR, \fB\-\-dependency\fR[=]<\fIdependency_list\fR>
+Defer the start of this job until the specified dependencies have been
+satisfied completed.
+<\fIdependency_list\fR> is of the form
+<\fItype:job_id[:job_id][,type:job_id[:job_id]]\fR>.
+Many jobs can share the same dependency and these jobs may even belong to
+different users. The value may be changed after job submission using the
+scontrol command.
+.PD
+.RS
.TP
-\fB\-P\fR, \fB\-\-dependency\fR=\fIjobid\fR
-Defer initiation of this job until the specified jobid
-has completed execution. Many jobs can share the same
-dependency and these jobs may belong to different users.
-The value may be changed after job submission using the
-\fBscontrol\fR command.
+\fBafter:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have begun
+execution.
+.TP
+\fBafterany:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have terminated.
+.TP
+\fBafternotok:job_id[:jobid...\fR
+This job can begin execution after the specified jobs have terminated
+in some failed state (non-zero exit code, node failure, timed out, etc).
+.TP
+\fBafterok:job_id[:jobid...]\fR
+This job can begin execution after the specified jobs have successfully
+executed (ran to completion with non-zero exit code).
+.RE
.TP
\fB\-p\fR, \fB\-\-partition\fR=\fIpartition\fR
diff --git a/slurm/slurm.h.in b/slurm/slurm.h.in
index 6941a01c38e..7fe25cb1773 100644
--- a/slurm/slurm.h.in
+++ b/slurm/slurm.h.in
@@ -527,7 +527,7 @@ typedef struct job_descriptor { /* For submit, allocate, and update requests */
uint16_t alloc_resp_port;
uint16_t other_port;
- uint32_t dependency; /* defer until specified job completes */
+ char *dependency; /* syncrhonize job execution with other jobs */
uint16_t overcommit; /* over subscribe resources, for batch only */
uint32_t num_tasks; /* number of tasks to be started, for batch only */
uint16_t nice; /* requested priority change,
@@ -623,7 +623,7 @@ typedef struct job_info {
* start_range_1, end_range_1,
* start_range_2, .., -1 */
char *features; /* comma separated list of required features */
- uint32_t dependency; /* defer until specified job completes */
+ char *dependency; /* syncrhonize job execution with other jobs */
uint32_t exit_code; /* exit code for job (status from wait call) */
char *account; /* charge to specified account */
uint16_t state_reason; /* reason job still pending or failed, see
diff --git a/src/api/init_msg.c b/src/api/init_msg.c
index 96a7044495f..bca6a1b800c 100644
--- a/src/api/init_msg.c
+++ b/src/api/init_msg.c
@@ -64,7 +64,7 @@ void slurm_init_job_desc_msg(job_desc_msg_t * job_desc_msg)
job_desc_msg->ntasks_per_node = (uint16_t) NO_VAL;
job_desc_msg->ntasks_per_socket = (uint16_t) NO_VAL;
job_desc_msg->ntasks_per_core = (uint16_t) NO_VAL;
- job_desc_msg->dependency = NO_VAL;
+ job_desc_msg->dependency = NULL;
job_desc_msg->environment = ((char **) NULL);
job_desc_msg->env_size = 0;
job_desc_msg->features = NULL;
diff --git a/src/api/job_info.c b/src/api/job_info.c
index ea19c5b38ff..6e755812f2b 100644
--- a/src/api/job_info.c
+++ b/src/api/job_info.c
@@ -378,8 +378,17 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
/****** Line 11 ******/
snprintf(tmp_line, sizeof(tmp_line),
- "Dependency=%u Account=%s Reason=%s Network=%s",
- job_ptr->dependency, job_ptr->account,
+ "Dependency=%s Account=%s",
+ job_ptr->dependency, job_ptr->account);
+ xstrcat(out, tmp_line);
+ if (one_liner)
+ xstrcat(out, " ");
+ else
+ xstrcat(out, "\n ");
+
+ /****** Line 12 ******/
+ snprintf(tmp_line, sizeof(tmp_line),
+ "Reason=%s Network=%s",
job_reason_string(job_ptr->state_reason),
job_ptr->network);
xstrcat(out, tmp_line);
@@ -388,7 +397,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
else
xstrcat(out, "\n ");
- /****** Line 12 ******/
+ /****** Line 13 ******/
snprintf(tmp_line, sizeof(tmp_line), "Req%s=%s Req%sIndices=",
nodelist, job_ptr->req_nodes, nodelist);
xstrcat(out, tmp_line);
@@ -405,7 +414,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
else
xstrcat(out, "\n ");
- /****** Line 13 ******/
+ /****** Line 14 ******/
snprintf(tmp_line, sizeof(tmp_line), "Exc%s=%s Exc%sIndices=",
nodelist, job_ptr->exc_nodes, nodelist);
xstrcat(out, tmp_line);
@@ -422,7 +431,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
else
xstrcat(out, "\n ");
- /****** Line 14 ******/
+ /****** Line 15 ******/
slurm_make_time_str((time_t *)&job_ptr->submit_time, time_str,
sizeof(time_str));
sprintf(tmp_line, "SubmitTime=%s ", time_str);
@@ -437,7 +446,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
time_str, (long int)job_ptr->pre_sus_time);
xstrcat(out, tmp_line);
- /****** Lines 15, 16 (optional, batch only) ******/
+ /****** Lines 16, 17 (optional, batch only) ******/
if (job_ptr->batch_flag) {
if (one_liner)
xstrcat(out, " ");
@@ -454,7 +463,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
xstrcat(out, tmp_line);
}
- /****** Line 17 (optional) ******/
+ /****** Line 18 (optional) ******/
if (job_ptr->comment) {
if (one_liner)
xstrcat(out, " ");
@@ -464,7 +473,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
xstrcat(out, tmp_line);
}
- /****** Line 18 (optional) ******/
+ /****** Line 19 (optional) ******/
select_g_sprint_jobinfo(job_ptr->select_jobinfo,
select_buf, sizeof(select_buf),
SELECT_PRINT_MIXED);
@@ -475,7 +484,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
xstrcat(out, "\n ");
xstrcat(out, select_buf);
}
- /****** Line 19 (optional) ******/
+ /****** Line 20 (optional) ******/
select_g_sprint_jobinfo(job_ptr->select_jobinfo,
select_buf, sizeof(select_buf),
SELECT_PRINT_BLRTS_IMAGE);
@@ -488,7 +497,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
"BlrtsImage=%s", select_buf);
xstrcat(out, tmp_line);
}
- /****** Line 20 (optional) ******/
+ /****** Line 21 (optional) ******/
select_g_sprint_jobinfo(job_ptr->select_jobinfo,
select_buf, sizeof(select_buf),
SELECT_PRINT_LINUX_IMAGE);
@@ -501,7 +510,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
"LinuxImage=%s", select_buf);
xstrcat(out, tmp_line);
}
- /****** Line 21 (optional) ******/
+ /****** Line 22 (optional) ******/
select_g_sprint_jobinfo(job_ptr->select_jobinfo,
select_buf, sizeof(select_buf),
SELECT_PRINT_MLOADER_IMAGE);
@@ -514,7 +523,7 @@ slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner )
"MloaderImage=%s", select_buf);
xstrcat(out, tmp_line);
}
- /****** Line 22 (optional) ******/
+ /****** Line 23 (optional) ******/
select_g_sprint_jobinfo(job_ptr->select_jobinfo,
select_buf, sizeof(select_buf),
SELECT_PRINT_RAMDISK_IMAGE);
diff --git a/src/common/slurm_errno.c b/src/common/slurm_errno.c
index 8977b79a352..c30dfb8e0c1 100644
--- a/src/common/slurm_errno.c
+++ b/src/common/slurm_errno.c
@@ -189,7 +189,7 @@ static slurm_errtab_t slurm_errtab[] = {
{ ESLURM_DISABLED,
"Requested operation is presently disabled" },
{ ESLURM_DEPENDENCY,
- "Immediate execution impossible, job dependency problem"},
+ "Job dependency problem" },
{ ESLURM_BATCH_ONLY,
"Only batch jobs are accepted or processed" },
{ ESLURM_TASKDIST_ARBITRARY_UNSUPPORTED,
diff --git a/src/common/slurm_protocol_defs.c b/src/common/slurm_protocol_defs.c
index 9ce114419b7..aa77dbcc285 100644
--- a/src/common/slurm_protocol_defs.c
+++ b/src/common/slurm_protocol_defs.c
@@ -200,6 +200,7 @@ void slurm_free_job_desc_msg(job_desc_msg_t * msg)
xfree(msg->account);
xfree(msg->network);
xfree(msg->comment);
+ xfree(msg->dependency);
xfree(msg->resp_host);
xfree(msg->blrtsimage);
xfree(msg->linuximage);
@@ -267,6 +268,7 @@ void slurm_free_job_info_members(job_info_t * job)
xfree(job->exc_node_inx);
xfree(job->network);
xfree(job->comment);
+ xfree(job->dependency);
xfree(job->work_dir);
xfree(job->command);
}
diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index 85f18c10b72..c84028281d0 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -2035,10 +2035,9 @@ _unpack_job_info_members(job_info_t * job, Buf buffer)
safe_unpackstr_xmalloc(&job->account, &uint16_tmp, buffer);
safe_unpackstr_xmalloc(&job->network, &uint16_tmp, buffer);
safe_unpackstr_xmalloc(&job->comment, &uint16_tmp, buffer);
+ safe_unpackstr_xmalloc(&job->dependency, &uint16_tmp, buffer);
- safe_unpack32(&job->dependency, buffer);
safe_unpack32(&job->exit_code, buffer);
-
safe_unpack16(&job->num_cpu_groups, buffer);
safe_unpack32_array(&job->cpus_per_node, &uint32_tmp, buffer);
safe_unpack32_array(&job->cpu_count_reps, &uint32_tmp, buffer);
@@ -2117,17 +2116,23 @@ unpack_error:
xfree(job->nodes);
xfree(job->partition);
xfree(job->account);
+ xfree(job->network);
+ xfree(job->comment);
+ xfree(job->dependency);
+ xfree(job->cpus_per_node);
+ xfree(job->cpu_count_reps);
xfree(job->name);
xfree(job->alloc_node);
xfree(job->node_inx);
select_g_free_jobinfo(&job->select_jobinfo);
xfree(job->features);
+ xfree(job->work_dir);
+ xfree(job->command);
xfree(job->req_nodes);
xfree(job->req_node_inx);
xfree(job->exc_nodes);
xfree(job->exc_node_inx);
- xfree(job->network);
- xfree(job->comment);
+
return SLURM_ERROR;
}
@@ -2495,7 +2500,7 @@ _pack_job_desc_msg(job_desc_msg_t * job_desc_ptr, Buf buffer)
packstr(job_desc_ptr->partition, buffer);
pack32(job_desc_ptr->priority, buffer);
- pack32(job_desc_ptr->dependency, buffer);
+ packstr(job_desc_ptr->dependency, buffer);
packstr(job_desc_ptr->account, buffer);
packstr(job_desc_ptr->comment, buffer);
pack16(job_desc_ptr->nice, buffer);
@@ -2624,7 +2629,7 @@ _unpack_job_desc_msg(job_desc_msg_t ** job_desc_buffer_ptr, Buf buffer)
safe_unpackstr_xmalloc(&job_desc_ptr->partition, &uint16_tmp, buffer);
safe_unpack32(&job_desc_ptr->priority, buffer);
- safe_unpack32(&job_desc_ptr->dependency, buffer);
+ safe_unpackstr_xmalloc(&job_desc_ptr->dependency, &uint16_tmp, buffer);
safe_unpackstr_xmalloc(&job_desc_ptr->account, &uint16_tmp, buffer);
safe_unpackstr_xmalloc(&job_desc_ptr->comment, &uint16_tmp, buffer);
safe_unpack16(&job_desc_ptr->nice, buffer);
@@ -2688,11 +2693,15 @@ _unpack_job_desc_msg(job_desc_msg_t ** job_desc_buffer_ptr, Buf buffer)
return SLURM_SUCCESS;
unpack_error:
- select_g_free_jobinfo(&job_desc_ptr->select_jobinfo);
+
xfree(job_desc_ptr->features);
xfree(job_desc_ptr->name);
xfree(job_desc_ptr->partition);
+ xfree(job_desc_ptr->dependency);
+ xfree(job_desc_ptr->account);
+ xfree(job_desc_ptr->comment);
xfree(job_desc_ptr->req_nodes);
+ xfree(job_desc_ptr->exc_nodes);
xfree(job_desc_ptr->environment);
xfree(job_desc_ptr->script);
xfree(job_desc_ptr->argv);
@@ -2702,6 +2711,7 @@ unpack_error:
xfree(job_desc_ptr->work_dir);
xfree(job_desc_ptr->network);
xfree(job_desc_ptr->mail_user);
+ select_g_free_jobinfo(&job_desc_ptr->select_jobinfo);
xfree(job_desc_ptr);
*job_desc_buffer_ptr = NULL;
return SLURM_ERROR;
diff --git a/src/plugins/sched/wiki/job_modify.c b/src/plugins/sched/wiki/job_modify.c
index e3a62654951..72030e90aea 100644
--- a/src/plugins/sched/wiki/job_modify.c
+++ b/src/plugins/sched/wiki/job_modify.c
@@ -53,24 +53,8 @@ static void _null_term(char *str)
}
}
-/* return -1 on error */
-static int32_t _get_depend_id(char *str)
-{
- /* stand-alone job_id */
- if (isdigit(str[0]))
- return (int32_t) atol(str);
-
- if (strncasecmp(str, "afterany:", 9) != 0) /* invalid spec */
- return (int32_t) -1;
-
- str += 9;
- if (!isdigit(str[0]))
- return (int32_t) -1;
- return (int32_t) atol(str);
-}
-
static int _job_modify(uint32_t jobid, char *bank_ptr,
- int32_t depend_id, char *new_hostlist,
+ char *depend_ptr, char *new_hostlist,
uint32_t new_node_cnt, char *part_name_ptr,
uint32_t new_time_limit)
{
@@ -86,9 +70,16 @@ static int _job_modify(uint32_t jobid, char *bank_ptr,
return ESLURM_DISABLED;
}
- if (depend_id != -1) {
- info("wiki: changing job dependency to %d", depend_id);
- job_ptr->dependency = depend_id;
+ if (depend_ptr) {
+ int rc = update_job_dependency(job_ptr, depend_ptr);
+ if (rc == SLURM_SUCCESS) {
+ info("wiki: changed job %u dependency to %s",
+ jobid, depend_ptr);
+ } else {
+ error("wiki: changing job %u dependency to %s",
+ jobid, depend_ptr);
+ return EINVAL;
+ }
}
if (new_time_limit) {
@@ -207,7 +198,6 @@ extern int job_modify_wiki(char *cmd_ptr, int *err_code, char **err_msg)
char *arg_ptr, *bank_ptr, *depend_ptr, *nodes_ptr;
char *host_ptr, *part_ptr, *time_ptr, *tmp_char;
int slurm_rc;
- int depend_id = -1;
uint32_t jobid, new_node_cnt = 0, new_time_limit = 0;
static char reply_msg[128];
/* Locks: write job, read node and partition info */
@@ -246,14 +236,7 @@ extern int job_modify_wiki(char *cmd_ptr, int *err_code, char **err_msg)
if (depend_ptr) {
depend_ptr[6] = ':';
depend_ptr += 7;
- depend_id = _get_depend_id(depend_ptr);
- if (depend_id == -1) {
- *err_code = -300;
- *err_msg = "MODIFYJOB has invalid DEPEND specificiation";
- error("wiki: MODIFYJOB has invalid DEPEND spec: %s",
- depend_ptr);
- return -1;
- }
+ _null_term(depend_ptr);
}
if (host_ptr) {
host_ptr[8] = ':';
@@ -286,7 +269,7 @@ extern int job_modify_wiki(char *cmd_ptr, int *err_code, char **err_msg)
}
lock_slurmctld(job_write_lock);
- slurm_rc = _job_modify(jobid, bank_ptr, depend_id, host_ptr,
+ slurm_rc = _job_modify(jobid, bank_ptr, depend_ptr, host_ptr,
new_node_cnt, part_ptr, new_time_limit);
unlock_slurmctld(job_write_lock);
if (slurm_rc != SLURM_SUCCESS) {
diff --git a/src/plugins/sched/wiki2/get_jobs.c b/src/plugins/sched/wiki2/get_jobs.c
index 847cb86ff36..bd51ec31b39 100644
--- a/src/plugins/sched/wiki2/get_jobs.c
+++ b/src/plugins/sched/wiki2/get_jobs.c
@@ -372,7 +372,7 @@ static void _get_job_comment(struct job_record *job_ptr,
if (job_ptr->dependency) {
/* Kludge for job dependency set via srun */
size += snprintf((buffer + size), (buf_size - size),
- "DEPEND=afterany:%u", job_ptr->dependency);
+ "DEPEND=%s", job_ptr->dependency);
field_sep = "?";
}
diff --git a/src/plugins/sched/wiki2/job_modify.c b/src/plugins/sched/wiki2/job_modify.c
index e3a62654951..72030e90aea 100644
--- a/src/plugins/sched/wiki2/job_modify.c
+++ b/src/plugins/sched/wiki2/job_modify.c
@@ -53,24 +53,8 @@ static void _null_term(char *str)
}
}
-/* return -1 on error */
-static int32_t _get_depend_id(char *str)
-{
- /* stand-alone job_id */
- if (isdigit(str[0]))
- return (int32_t) atol(str);
-
- if (strncasecmp(str, "afterany:", 9) != 0) /* invalid spec */
- return (int32_t) -1;
-
- str += 9;
- if (!isdigit(str[0]))
- return (int32_t) -1;
- return (int32_t) atol(str);
-}
-
static int _job_modify(uint32_t jobid, char *bank_ptr,
- int32_t depend_id, char *new_hostlist,
+ char *depend_ptr, char *new_hostlist,
uint32_t new_node_cnt, char *part_name_ptr,
uint32_t new_time_limit)
{
@@ -86,9 +70,16 @@ static int _job_modify(uint32_t jobid, char *bank_ptr,
return ESLURM_DISABLED;
}
- if (depend_id != -1) {
- info("wiki: changing job dependency to %d", depend_id);
- job_ptr->dependency = depend_id;
+ if (depend_ptr) {
+ int rc = update_job_dependency(job_ptr, depend_ptr);
+ if (rc == SLURM_SUCCESS) {
+ info("wiki: changed job %u dependency to %s",
+ jobid, depend_ptr);
+ } else {
+ error("wiki: changing job %u dependency to %s",
+ jobid, depend_ptr);
+ return EINVAL;
+ }
}
if (new_time_limit) {
@@ -207,7 +198,6 @@ extern int job_modify_wiki(char *cmd_ptr, int *err_code, char **err_msg)
char *arg_ptr, *bank_ptr, *depend_ptr, *nodes_ptr;
char *host_ptr, *part_ptr, *time_ptr, *tmp_char;
int slurm_rc;
- int depend_id = -1;
uint32_t jobid, new_node_cnt = 0, new_time_limit = 0;
static char reply_msg[128];
/* Locks: write job, read node and partition info */
@@ -246,14 +236,7 @@ extern int job_modify_wiki(char *cmd_ptr, int *err_code, char **err_msg)
if (depend_ptr) {
depend_ptr[6] = ':';
depend_ptr += 7;
- depend_id = _get_depend_id(depend_ptr);
- if (depend_id == -1) {
- *err_code = -300;
- *err_msg = "MODIFYJOB has invalid DEPEND specificiation";
- error("wiki: MODIFYJOB has invalid DEPEND spec: %s",
- depend_ptr);
- return -1;
- }
+ _null_term(depend_ptr);
}
if (host_ptr) {
host_ptr[8] = ':';
@@ -286,7 +269,7 @@ extern int job_modify_wiki(char *cmd_ptr, int *err_code, char **err_msg)
}
lock_slurmctld(job_write_lock);
- slurm_rc = _job_modify(jobid, bank_ptr, depend_id, host_ptr,
+ slurm_rc = _job_modify(jobid, bank_ptr, depend_ptr, host_ptr,
new_node_cnt, part_ptr, new_time_limit);
unlock_slurmctld(job_write_lock);
if (slurm_rc != SLURM_SUCCESS) {
diff --git a/src/salloc/opt.c b/src/salloc/opt.c
index 24b7ca9ebf2..0c7c846c369 100644
--- a/src/salloc/opt.c
+++ b/src/salloc/opt.c
@@ -235,7 +235,7 @@ static void _opt_default()
opt.job_name = NULL;
opt.jobid = NO_VAL;
- opt.dependency = NO_VAL;
+ opt.dependency = NULL;
opt.account = NULL;
opt.comment = NULL;
@@ -564,7 +564,8 @@ void set_options(const int argc, char **argv)
opt.constraints = xstrdup(optarg);
break;
case 'd':
- opt.dependency = _get_int(optarg, "dependency");
+ xfree(opt.dependency);
+ opt.dependency = xstrdup(optarg);
break;
case 'F':
xfree(opt.nodelist);
@@ -1215,10 +1216,7 @@ static void _opt_list()
info("nice : %d", opt.nice);
info("account : %s", opt.account);
info("comment : %s", opt.comment);
- if (opt.dependency == NO_VAL)
- info("dependency : none");
- else
- info("dependency : %u", opt.dependency);
+ info("dependency : %s", opt.dependency);
str = print_constraints();
info("constraints : %s", str);
xfree(str);
@@ -1270,7 +1268,7 @@ static void _usage(void)
" [--verbose] [--gid=group] [--uid=user]\n"
" [-W sec] [--minsockets=n] [--mincores=n] [--minthreads=n]\n"
" [--contiguous] [--mincpus=n] [--mem=MB] [--tmp=MB] [-C list]\n"
-" [--account=name] [--dependency=jobid] [--comment=name]\n"
+" [--account=name] [--dependency=type:jobid] [--comment=name]\n"
#ifdef HAVE_BG /* Blue gene specific options */
" [--geometry=XxYxZ] [--conn-type=type] [--no-rotate] [ --reboot]\n"
" [--blrts-image=path] [--linux-image=path]\n"
@@ -1310,7 +1308,7 @@ static void _help(void)
" immediately available\n"
" -v, --verbose verbose mode (multiple -v's increase verbosity)\n"
" -q, --quiet quiet mode (suppress informational messages)\n"
-" -d, --dependency=jobid defer job until specified jobid completes\n"
+" -d, --dependency=type:jobid defer job until condition on jobid is satisfied\n"
" --nice[=value] decrease secheduling priority by value\n"
" -U, --account=name charge job to specified account\n"
" --begin=time defer job until HH:MM DD/MM/YY\n"
diff --git a/src/salloc/opt.h b/src/salloc/opt.h
index ea5fb69d57e..bdf773d9248 100644
--- a/src/salloc/opt.h
+++ b/src/salloc/opt.h
@@ -83,7 +83,7 @@ typedef struct salloc_options {
* plane> */
char *job_name; /* --job-name=, -J name */
unsigned int jobid; /* --jobid=jobid */
- unsigned int dependency;/* --dependency, -P jobid */
+ char *dependency; /* --dependency, -P type:jobid */
int nice; /* --nice */
char *account; /* --account, -U acct_name */
char *comment; /* --comment */
diff --git a/src/salloc/salloc.c b/src/salloc/salloc.c
index 25262bdb010..f08d2752dd0 100644
--- a/src/salloc/salloc.c
+++ b/src/salloc/salloc.c
@@ -250,7 +250,8 @@ static int fill_job_desc_from_opts(job_desc_msg_t *desc)
desc->max_nodes = opt.max_nodes;
desc->user_id = opt.uid;
desc->group_id = opt.gid;
- desc->dependency = opt.dependency;
+ if (opt.dependency)
+ desc->dependency = xstrdup(opt.dependency);
desc->task_dist = opt.distribution;
if (opt.plane_size != NO_VAL)
desc->plane_size = opt.plane_size;
diff --git a/src/sbatch/opt.c b/src/sbatch/opt.c
index 0caba426496..7eecac2d9b7 100644
--- a/src/sbatch/opt.c
+++ b/src/sbatch/opt.c
@@ -234,7 +234,7 @@ static void _opt_default()
opt.job_name = NULL;
opt.jobid = NO_VAL;
opt.jobid_set = false;
- opt.dependency = NO_VAL;
+ opt.dependency = NULL;
opt.account = NULL;
opt.comment = NULL;
@@ -892,7 +892,8 @@ static void _set_options(int argc, char **argv)
opt.constraints = xstrdup(optarg);
break;
case 'd':
- opt.dependency = _get_int(optarg, "dependency");
+ xfree(opt.dependency);
+ opt.dependency = xstrdup(optarg);
break;
case 'D':
xfree(opt.cwd);
@@ -1928,10 +1929,7 @@ static void _opt_list()
info("nice : %d", opt.nice);
info("account : %s", opt.account);
info("comment : %s", opt.comment);
- if (opt.dependency == NO_VAL)
- info("dependency : none");
- else
- info("dependency : %u", opt.dependency);
+ info("dependency : %s", opt.dependency);
str = print_constraints();
info("constraints : %s", str);
xfree(str);
@@ -1986,7 +1984,7 @@ static void _usage(void)
" [--jobid=id] [--verbose] [--gid=group] [--uid=user]\n"
" [-W sec] [--minsockets=n] [--mincores=n] [--minthreads=n]\n"
" [--contiguous] [--mincpus=n] [--mem=MB] [--tmp=MB] [-C list]\n"
-" [--account=name] [--dependency=jobid] [--comment=name]\n"
+" [--account=name] [--dependency=type:jobid] [--comment=name]\n"
#ifdef HAVE_BG /* Blue gene specific options */
" [--geometry=XxYxZ] [--conn-type=type] [--no-rotate] [ --reboot]\n"
" [--blrts-image=path] [--linux-image=path]\n"
@@ -2024,7 +2022,7 @@ static void _help(void)
" --jobid=id run under already allocated job\n"
" -v, --verbose verbose mode (multiple -v's increase verbosity)\n"
" -q, --quiet quiet mode (suppress informational messages)\n"
-" -d, --dependency=jobid defer job until specified jobid completes\n"
+" -d, --dependency=type:jobid defer job until condition on jobid is satisfied\n"
" -D, --workdir=directory set working directory for batch script\n"
" --nice[=value] decrease secheduling priority by value\n"
" -O, --overcommit overcommit resources\n"
diff --git a/src/sbatch/opt.h b/src/sbatch/opt.h
index 1c572e5f6e3..cb8097a05ad 100644
--- a/src/sbatch/opt.h
+++ b/src/sbatch/opt.h
@@ -87,7 +87,7 @@ typedef struct sbatch_options {
unsigned int jobid; /* --jobid=jobid */
bool jobid_set; /* true of jobid explicitly set */
char *mpi_type; /* --mpi=type */
- unsigned int dependency;/* --dependency, -P jobid */
+ char *dependency; /* --dependency, -P type:jobid */
int nice; /* --nice */
char *account; /* --account, -U acct_name */
char *comment; /* --comment */
diff --git a/src/sbatch/sbatch.c b/src/sbatch/sbatch.c
index f608b538739..75f0082e0de 100644
--- a/src/sbatch/sbatch.c
+++ b/src/sbatch/sbatch.c
@@ -127,7 +127,8 @@ static int fill_job_desc_from_opts(job_desc_msg_t *desc)
desc->max_nodes = opt.max_nodes;
desc->user_id = opt.uid;
desc->group_id = opt.gid;
- desc->dependency = opt.dependency;
+ if (opt.dependency)
+ desc->dependency = xstrdup(opt.dependency);
desc->task_dist = opt.distribution;
if (opt.plane_size != NO_VAL)
desc->plane_size = opt.plane_size;
diff --git a/src/scontrol/update_job.c b/src/scontrol/update_job.c
index 1a03924bb6e..fed7bcc7d51 100644
--- a/src/scontrol/update_job.c
+++ b/src/scontrol/update_job.c
@@ -366,9 +366,7 @@ scontrol_update_job (int argc, char *argv[])
update_cnt++;
}
else if (strncasecmp(argv[i], "Dependency=", 11) == 0) {
- job_msg.dependency =
- (uint32_t) strtol(&argv[i][11],
- (char **) NULL, 10);
+ job_msg.dependency = &argv[i][11];
update_cnt++;
}
#ifdef HAVE_BG
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index 74d69516e46..ce91288fd33 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -357,7 +357,7 @@ int dump_all_job_state(void)
* checkpoint. Execute this after loading the configuration file data.
* RET 0 or error code
*/
-int load_all_job_state(void)
+extern int load_all_job_state(void)
{
int data_allocated, data_read = 0, error_code = 0;
uint32_t data_size = 0;
@@ -484,7 +484,6 @@ static void _dump_job_state(struct job_record *dump_job_ptr, Buf buffer)
pack32(dump_job_ptr->time_limit, buffer);
pack32(dump_job_ptr->priority, buffer);
pack32(dump_job_ptr->alloc_sid, buffer);
- pack32(dump_job_ptr->dependency, buffer);
pack32(dump_job_ptr->num_procs, buffer);
pack32(dump_job_ptr->exit_code, buffer);
pack32(dump_job_ptr->db_index, buffer);
@@ -520,6 +519,7 @@ static void _dump_job_state(struct job_record *dump_job_ptr, Buf buffer)
packstr(dump_job_ptr->alloc_node, buffer);
packstr(dump_job_ptr->account, buffer);
packstr(dump_job_ptr->comment, buffer);
+ packstr(dump_job_ptr->dependency, buffer);
packstr(dump_job_ptr->network, buffer);
packstr(dump_job_ptr->mail_user, buffer);
@@ -550,7 +550,7 @@ static void _dump_job_state(struct job_record *dump_job_ptr, Buf buffer)
static int _load_job_state(Buf buffer)
{
uint32_t job_id, user_id, group_id, time_limit, priority, alloc_sid;
- uint32_t dependency, exit_code, num_procs, db_index;
+ uint32_t exit_code, num_procs, db_index;
time_t start_time, end_time, suspend_time, pre_sus_time;
uint16_t job_state, next_step_id, details, batch_flag, step_flag;
uint16_t kill_on_node_fail, kill_on_step_done, name_len;
@@ -558,6 +558,7 @@ static int _load_job_state(Buf buffer)
char *nodes = NULL, *partition = NULL, *name = NULL, *resp_host = NULL;
char *account = NULL, *network = NULL, *mail_user = NULL;
char *comment = NULL, *nodes_completing = NULL, *alloc_node = NULL;
+ char *dependency = NULL;
struct job_record *job_ptr;
struct part_record *part_ptr;
int error_code;
@@ -569,7 +570,6 @@ static int _load_job_state(Buf buffer)
safe_unpack32(&time_limit, buffer);
safe_unpack32(&priority, buffer);
safe_unpack32(&alloc_sid, buffer);
- safe_unpack32(&dependency, buffer);
safe_unpack32(&num_procs, buffer);
safe_unpack32(&exit_code, buffer);
safe_unpack32(&db_index, buffer);
@@ -601,6 +601,7 @@ static int _load_job_state(Buf buffer)
safe_unpackstr_xmalloc(&alloc_node, &name_len, buffer);
safe_unpackstr_xmalloc(&account, &name_len, buffer);
safe_unpackstr_xmalloc(&comment, &name_len, buffer);
+ safe_unpackstr_xmalloc(&dependency, &name_len, buffer);
safe_unpackstr_xmalloc(&network, &name_len, buffer);
safe_unpackstr_xmalloc(&mail_user, &name_len, buffer);
@@ -738,6 +739,7 @@ unpack_error:
xfree(alloc_node);
xfree(account);
xfree(comment);
+ xfree(dependency);
xfree(resp_host);
xfree(mail_user);
select_g_free_jobinfo(&select_jobinfo);
@@ -1133,7 +1135,7 @@ void dump_job_desc(job_desc_msg_t * job_specs)
long job_min_procs, job_min_sockets, job_min_cores, job_min_threads;
long job_min_memory, job_max_memory, job_min_tmp_disk, num_procs;
long time_limit, priority, contiguous;
- long kill_on_node_fail, shared, immediate, dependency;
+ long kill_on_node_fail, shared, immediate;
long cpus_per_task, no_requeue, num_tasks, overcommit;
long ntasks_per_node, ntasks_per_socket, ntasks_per_core;
char buf[100];
@@ -1239,13 +1241,12 @@ void dump_job_desc(job_desc_msg_t * job_specs)
job_specs->work_dir,
job_specs->alloc_node, job_specs->alloc_sid);
- dependency = (job_specs->dependency != NO_VAL) ?
- (long) job_specs->dependency : -1L;
debug3(" resp_host=%s alloc_resp_port=%u other_port=%u",
job_specs->resp_host,
job_specs->alloc_resp_port, job_specs->other_port);
- debug3(" dependency=%ld account=%s comment=%s",
- dependency, job_specs->account, job_specs->comment);
+ debug3(" dependency=%s account=%s comment=%s",
+ job_specs->dependency, job_specs->account,
+ job_specs->comment);
num_tasks = (job_specs->num_tasks != (uint16_t) NO_VAL) ?
(long) job_specs->num_tasks : -1L;
@@ -1926,10 +1927,9 @@ static int _job_create(job_desc_msg_t * job_desc, int allocate, int will_run,
goto cleanup;
}
-
if ((error_code =_validate_job_create_req(job_desc)))
goto cleanup;
-
+
if ((error_code = _copy_job_desc_to_job_record(job_desc,
job_pptr,
part_ptr,
@@ -1938,10 +1938,9 @@ static int _job_create(job_desc_msg_t * job_desc, int allocate, int will_run,
error_code = ESLURM_ERROR_ON_DESC_TO_RECORD_COPY;
goto cleanup;
}
-
+
job_ptr = *job_pptr;
- if (job_ptr->dependency == job_ptr->job_id) {
- info("User specified self as dependent job");
+ if (update_job_dependency(job_ptr, job_desc->dependency)) {
error_code = ESLURM_DEPENDENCY;
goto cleanup;
}
@@ -2434,8 +2433,6 @@ _copy_job_desc_to_job_record(job_desc_msg_t * job_desc,
job_ptr->account = xstrdup(job_desc->account);
job_ptr->network = xstrdup(job_desc->network);
job_ptr->comment = xstrdup(job_desc->comment);
- if (job_desc->dependency != NO_VAL) /* leave as zero */
- job_ptr->dependency = job_desc->dependency;
if (job_desc->priority != NO_VAL) /* already confirmed submit_uid==0 */
job_ptr->priority = job_desc->priority;
@@ -2516,7 +2513,6 @@ _copy_job_desc_to_job_record(job_desc_msg_t * job_desc,
job_ptr->select_jobinfo =
select_g_copy_jobinfo(job_desc->select_jobinfo);
detail_ptr->mc_ptr = _set_multi_core_data(job_desc);
-
*job_rec_ptr = job_ptr;
return SLURM_SUCCESS;
}
@@ -2781,6 +2777,9 @@ static void _list_delete_job(void *job_entry)
xfree(job_ptr->alloc_lps);
xfree(job_ptr->used_lps);
xfree(job_ptr->comment);
+ xfree(job_ptr->dependency);
+ if (job_ptr->depend_list)
+ list_destroy(job_ptr->depend_list);
select_g_free_jobinfo(&job_ptr->select_jobinfo);
if (job_ptr->step_list) {
delete_step_records(job_ptr, 0);
@@ -2952,8 +2951,8 @@ void pack_job(struct job_record *dump_job_ptr, Buf buffer)
packstr(dump_job_ptr->account, buffer);
packstr(dump_job_ptr->network, buffer);
packstr(dump_job_ptr->comment, buffer);
+ packstr(dump_job_ptr->dependency, buffer);
- pack32(dump_job_ptr->dependency, buffer);
pack32(dump_job_ptr->exit_code, buffer);
pack16(dump_job_ptr->num_cpu_groups, buffer);
@@ -3788,14 +3787,14 @@ int update_job(job_desc_msg_t * job_specs, uid_t uid)
}
}
- if (job_specs->dependency != NO_VAL) {
+ if (job_specs->dependency) {
if (!IS_JOB_PENDING(job_ptr))
error_code = ESLURM_DISABLED;
- else if (job_specs->dependency == job_ptr->job_id)
+ else if (update_job_dependency(job_ptr, job_specs->dependency)
+ != SLURM_SUCCESS) {
error_code = ESLURM_DEPENDENCY;
- else {
- job_ptr->dependency = job_specs->dependency;
- info("update_job: setting dependency to %u for "
+ } else {
+ info("update_job: setting dependency to %s for "
"job_id %u", job_ptr->dependency,
job_ptr->job_id);
}
@@ -4424,27 +4423,30 @@ extern void job_completion_logger(struct job_record *job_ptr)
*/
extern bool job_independent(struct job_record *job_ptr)
{
- struct job_record *dep_ptr;
struct job_details *detail_ptr = job_ptr->details;
+ int rc;
if (detail_ptr && (detail_ptr->begin_time > time(NULL))) {
job_ptr->state_reason = WAIT_TIME;
return false; /* not yet time */
}
-
- if (job_ptr->dependency == 0)
- return true;
- dep_ptr = find_job_record(job_ptr->dependency);
- if (dep_ptr == NULL)
+ rc = test_job_dependency(job_ptr);
+ if (rc == 0)
return true;
-
- if (((dep_ptr->job_state & JOB_COMPLETING) == 0) &&
- (dep_ptr->job_state >= JOB_COMPLETE))
- return true;
-
- job_ptr->state_reason = WAIT_DEPENDENCY;
- return false; /* job exists and incomplete */
+ else if (rc == 1) {
+ job_ptr->state_reason = WAIT_DEPENDENCY;
+ return false;
+ } else { /* rc == 2 */
+ time_t now = time(NULL);
+ info("Job dependency can't be satisfied, cancelling job %u",
+ job_ptr->job_id);
+ job_ptr->job_state = JOB_CANCELLED;
+ job_ptr->start_time = now;
+ job_ptr->end_time = now;
+ job_completion_logger(job_ptr);
+ return false;
+ }
}
/*
* determine if job is ready to execute per the node select plugin
diff --git a/src/slurmctld/job_scheduler.c b/src/slurmctld/job_scheduler.c
index 5b341537a47..9e99c7530ee 100644
--- a/src/slurmctld/job_scheduler.c
+++ b/src/slurmctld/job_scheduler.c
@@ -58,6 +58,7 @@
#include "src/slurmctld/slurmctld.h"
#include "src/slurmctld/srun_comm.h"
+#define _DEBUG 0
#define MAX_RETRIES 10
struct job_queue {
@@ -67,6 +68,7 @@ struct job_queue {
};
static int _build_job_queue(struct job_queue **job_queue);
+static void _depend_list_del(void *dep_ptr);
static void _launch_job(struct job_record *job_ptr);
static void _sort_job_queue(struct job_queue *job_queue,
int job_queue_size);
@@ -440,3 +442,214 @@ extern int make_batch_job_cred(batch_job_launch_msg_t *launch_msg_ptr)
error("slurm_cred_create failure for batch job %u", cred_arg.jobid);
return SLURM_ERROR;
}
+
+static void _depend_list_del(void *dep_ptr)
+{
+ xfree(dep_ptr);
+}
+
+/* Print a job's dependency information based upon job_ptr->depend_list */
+extern void print_job_dependency(struct job_record *job_ptr)
+{
+ ListIterator depend_iter;
+ struct depend_spec *dep_ptr;
+ char *dep_str;
+
+ info("Dependency information for job %u", job_ptr->job_id);
+ if (!job_ptr->depend_list)
+ return;
+
+ depend_iter = list_iterator_create(job_ptr->depend_list);
+ if (!depend_iter)
+ fatal("list_iterator_create memory allocation failure");
+ while ((dep_ptr = list_next(depend_iter))) {
+ if (dep_ptr->depend_type == SLURM_DEPEND_AFTER)
+ dep_str = "after";
+ else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER_ANY)
+ dep_str = "afterany";
+ else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER_NOT_OK)
+ dep_str = "afternotok";
+ else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER_OK)
+ dep_str = "afterok";
+ else
+ dep_str = "unknown";
+ info(" %s:%u", dep_str, dep_ptr->job_id);
+ }
+ list_iterator_destroy(depend_iter);
+}
+
+/*
+ * Determine if a job's dependencies are met
+ * RET: 0 = no dependencies
+ * 1 = dependencies remain
+ * 2 = failure (job completion code not per dependency), delete the job
+ */
+extern int test_job_dependency(struct job_record *job_ptr)
+{
+ ListIterator depend_iter;
+ struct depend_spec *dep_ptr;
+ bool failure = false;
+
+ if (!job_ptr->depend_list)
+ return 0;
+
+ depend_iter = list_iterator_create(job_ptr->depend_list);
+ if (!depend_iter)
+ fatal("list_iterator_create memory allocation failure");
+ while ((dep_ptr = list_next(depend_iter))) {
+ if (dep_ptr->job_ptr->job_id != dep_ptr->job_id) {
+ /* job is gone, dependency lifted */
+ list_delete_item(depend_iter);
+ } else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER) {
+ if (!IS_JOB_PENDING(dep_ptr->job_ptr))
+ list_delete_item(depend_iter);
+ else
+ break;
+ } else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER_ANY) {
+ if (IS_JOB_FINISHED(dep_ptr->job_ptr))
+ list_delete_item(depend_iter);
+ else
+ break;
+ } else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER_NOT_OK) {
+ if (!IS_JOB_FINISHED(dep_ptr->job_ptr))
+ break;
+ if ((dep_ptr->job_ptr->job_state & (~JOB_COMPLETING))
+ != JOB_COMPLETE)
+ list_delete_item(depend_iter);
+ else {
+ failure = true;
+ break;
+ }
+ } else if (dep_ptr->depend_type == SLURM_DEPEND_AFTER_OK) {
+ if (!IS_JOB_FINISHED(dep_ptr->job_ptr))
+ break;
+ if ((dep_ptr->job_ptr->job_state & (~JOB_COMPLETING))
+ == JOB_COMPLETE)
+ list_delete_item(depend_iter);
+ else {
+ failure = true;
+ break;
+ }
+ } else
+ failure = true;
+ }
+ list_iterator_destroy(depend_iter);
+
+ if (failure)
+ return 2;
+ if (dep_ptr)
+ return 1;
+ return 0;
+}
+
+/*
+ * Parse a job dependency string and use it to establish a "depend_spec"
+ * list of dependencies. We accept both old format (a single job ID) and
+ * new format (e.g. "afterok:123:124,after:128").
+ * IN job_ptr - job record to have dependency and depend_list updated
+ * IN new_depend - new dependency description
+ * RET returns an error code from slurm_errno.h
+ */
+extern int update_job_dependency(struct job_record *job_ptr, char *new_depend)
+{
+ int rc = SLURM_SUCCESS;
+ uint16_t depend_type = 0;
+ uint32_t job_id = 0;
+ char *tok = new_depend, *sep_ptr, *sep_ptr2;
+ List new_depend_list = NULL;
+ struct depend_spec *dep_ptr;
+ struct job_record *dep_job_ptr;
+ char dep_buf[32];
+
+ /* Clear dependencies on NULL or empty dependency input */
+ if ((new_depend == NULL) || (new_depend[0] == '\0')) {
+ xfree(job_ptr->dependency);
+ if (job_ptr->depend_list)
+ list_destroy(job_ptr->depend_list);
+ return rc;
+
+ }
+
+ new_depend_list = list_create(_depend_list_del);
+ /* validate new dependency string */
+ while (rc == SLURM_SUCCESS) {
+ sep_ptr = strchr(tok, ':');
+ if ((sep_ptr == NULL) && (job_id == 0)) {
+ job_id = strtol(tok, &sep_ptr, 10);
+ if ((sep_ptr == NULL) || (sep_ptr[0] != '\0') ||
+ (job_id <= 0) || (job_id == job_ptr->job_id)) {
+ rc = EINVAL;
+ break;
+ }
+ /* old format, just a single job_id */
+ dep_job_ptr = find_job_record(job_id);
+ if (!dep_job_ptr) /* assume already done */
+ break;
+ snprintf(dep_buf, sizeof(dep_buf), "afterany:%u", job_id);
+ new_depend = dep_buf;
+ dep_ptr = xmalloc(sizeof(struct depend_spec));
+ dep_ptr->depend_type = SLURM_DEPEND_AFTER_ANY;
+ dep_ptr->job_id = job_id;
+ dep_ptr->job_ptr = dep_job_ptr;
+ if (!list_append(new_depend_list, dep_ptr))
+ fatal("list_append memory allocation failure");
+ break;
+ }
+
+ if (strncasecmp(tok, "afternotok", 10) == 0)
+ depend_type = SLURM_DEPEND_AFTER_NOT_OK;
+ else if (strncasecmp(tok, "afterany", 8) == 0)
+ depend_type = SLURM_DEPEND_AFTER_ANY;
+ else if (strncasecmp(tok, "afterok", 7) == 0)
+ depend_type = SLURM_DEPEND_AFTER_OK;
+ else if (strncasecmp(tok, "after", 5) == 0)
+ depend_type = SLURM_DEPEND_AFTER;
+ else {
+ rc = EINVAL;
+ break;
+ }
+ sep_ptr++; /* skip over ":" */
+ while (rc == SLURM_SUCCESS) {
+ job_id = strtol(sep_ptr, &sep_ptr2, 10);
+ if ((sep_ptr2 == NULL) ||
+ (job_id <= 0) || (job_id == job_ptr->job_id) ||
+ ((sep_ptr2[0] != '\0') && (sep_ptr2[0] != ',') &&
+ (sep_ptr2[0] != ':'))) {
+ rc = EINVAL;
+ break;
+ }
+ dep_job_ptr = find_job_record(job_id);
+ if (dep_job_ptr) { /* job still active */
+ dep_ptr = xmalloc(sizeof(struct depend_spec));
+ dep_ptr->depend_type = depend_type;
+ dep_ptr->job_id = job_id;
+ dep_ptr->job_ptr = dep_job_ptr;
+ if (!list_append(new_depend_list, dep_ptr)) {
+ fatal("list_append memory allocation "
+ "failure");
+ }
+ }
+ if (sep_ptr2[0] != ':')
+ break;
+ sep_ptr = sep_ptr2 + 1; /* skip over ":" */
+ }
+ if (sep_ptr2[0] == ',')
+ tok = sep_ptr2 + 1;
+ else
+ break;
+ }
+
+ if (rc == SLURM_SUCCESS) {
+ xfree(job_ptr->dependency);
+ job_ptr->dependency = xstrdup(new_depend);
+ if (job_ptr->depend_list)
+ list_destroy(job_ptr->depend_list);
+ job_ptr->depend_list = new_depend_list;
+#if _DEBUG
+ print_job_dependency(job_ptr);
+#endif
+ } else {
+ list_destroy(new_depend_list);
+ }
+ return rc;
+}
diff --git a/src/slurmctld/read_config.c b/src/slurmctld/read_config.c
index 514c56a3fec..f2097b66e87 100644
--- a/src/slurmctld/read_config.c
+++ b/src/slurmctld/read_config.c
@@ -76,6 +76,7 @@ static int _build_bitmaps(void);
static int _init_all_slurm_conf(void);
static void _purge_old_node_state(struct node_record *old_node_table_ptr,
int old_node_record_count);
+static int _restore_job_dependencies(void);
static void _restore_node_state(struct node_record *old_node_table_ptr,
int old_node_record_count);
static int _preserve_select_type_param(slurm_ctl_conf_t * ctl_conf_ptr,
@@ -784,6 +785,7 @@ int read_slurm_conf(int recover)
if ((error_code = _build_bitmaps()))
return error_code;
+ _restore_job_dependencies();
restore_node_features();
#ifdef HAVE_ELAN
_validate_node_proc_count();
@@ -1114,3 +1116,30 @@ static void _validate_node_proc_count(void)
}
#endif
+/*
+ * _restore_job_dependencies - Build depend_list for every job
+ */
+static int _restore_job_dependencies(void)
+{
+ int error_code = SLURM_SUCCESS, rc;
+ struct job_record *job_ptr;
+ ListIterator job_iterator;
+ char *new_depend;
+
+ job_iterator = list_iterator_create(job_list);
+ while ((job_ptr = (struct job_record *) list_next(job_iterator))) {
+ if (job_ptr->dependency == NULL)
+ continue;
+ new_depend = job_ptr->dependency;
+ job_ptr->dependency = NULL;
+ rc = update_job_dependency(job_ptr, new_depend);
+ if (rc != SLURM_SUCCESS) {
+ error("Invalid dependencies discarded for job %u: %s",
+ job_ptr->job_id, new_depend);
+ error_code = rc;
+ }
+ xfree(new_depend);
+ }
+ list_iterator_destroy(job_iterator);
+ return error_code;
+}
diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h
index 66360f0b17b..f8ffb35c278 100644
--- a/src/slurmctld/slurmctld.h
+++ b/src/slurmctld/slurmctld.h
@@ -374,7 +374,8 @@ struct job_record {
uint16_t other_port; /* port for client communications */
char *account; /* account number to charge */
char *comment; /* arbitrary comment */
- uint32_t dependency; /* defer until this job completes */
+ char *dependency; /* wait for other jobs */
+ List depend_list; /* list of job_ptr:state pairs */
char *network; /* network/switch requirement spec */
struct job_record *job_next; /* next entry with same hash index */
uint16_t cr_enabled; /* specify if if Consumable
@@ -406,6 +407,17 @@ struct job_record {
plugins */
};
+/* Job dependency specification, used in "depend_list" within job_record */
+#define SLURM_DEPEND_AFTER 1
+#define SLURM_DEPEND_AFTER_ANY 2
+#define SLURM_DEPEND_AFTER_NOT_OK 3
+#define SLURM_DEPEND_AFTER_OK 4
+struct depend_spec {
+ uint16_t depend_type; /* SLURM_DEPEND_* type */
+ uint32_t job_id; /* SLURM job_id */
+ struct job_record *job_ptr; /* pointer to this job */
+};
+
struct step_record {
struct job_record* job_ptr; /* ptr to the job that owns the step */
uint16_t step_id; /* step number */
@@ -1153,6 +1165,9 @@ extern void part_filter_set(uid_t uid);
/* part_fini - free all memory associated with partition records */
void part_fini (void);
+/* Print a job's dependency information based upon job_ptr->depend_list */
+extern void print_job_dependency(struct job_record *job_ptr);
+
/*
* purge_old_job - purge old job records.
* the jobs must have completed at least MIN_JOB_AGE minutes ago
@@ -1328,6 +1343,14 @@ extern void suspend_job_step(struct job_record *job_ptr);
*/
extern int sync_job_files(void);
+/*
+ * Determine if a job's dependencies are met
+ * RET: 0 = no dependencies
+ * 1 = dependencies remain
+ * 2 = failure (job completion code not per dependency), delete the job
+ */
+extern int test_job_dependency(struct job_record *job_ptr);
+
/*
* update_job - update a job's parameters per the supplied specifications
* IN job_specs - a job's specification
@@ -1338,6 +1361,16 @@ extern int sync_job_files(void);
*/
extern int update_job (job_desc_msg_t * job_specs, uid_t uid);
+/*
+ * Parse a job dependency string and use it to establish a "depend_spec"
+ * list of dependencies. We accept both old format (a single job ID) and
+ * new format (e.g. "afterok:123:124,after:128").
+ * IN job_ptr - job record to have dependency and depend_list updated
+ * IN new_depend - new dependency description
+ * RET returns an error code from slurm_errno.h
+ */
+extern int update_job_dependency(struct job_record *job_ptr, char *new_depend);
+
/* Reset nodes_completing field for all jobs */
extern void update_job_nodes_completing(void);
diff --git a/src/squeue/print.c b/src/squeue/print.c
index d15d0424dd5..904a60f58a3 100644
--- a/src/squeue/print.c
+++ b/src/squeue/print.c
@@ -1022,11 +1022,10 @@ int _print_job_dependency(job_info_t * job, int width, bool right_justify,
{
if (job == NULL) /* Print the Header instead */
_print_str("DEPENDENCY", width, right_justify, true);
- else {
- char id[FORMAT_STRING_SIZE];
- snprintf(id, FORMAT_STRING_SIZE, "%u", job->dependency);
- _print_str(id, width, right_justify, true);
- }
+ else if (job->dependency)
+ _print_str(job->dependency, width, right_justify, true);
+ else
+ _print_str("", width, right_justify, true);
if (suffix)
printf("%s", suffix);
return SLURM_SUCCESS;
diff --git a/src/srun/opt.c b/src/srun/opt.c
index a68e04c88b9..c882777aca8 100644
--- a/src/srun/opt.c
+++ b/src/srun/opt.c
@@ -612,7 +612,7 @@ static void _opt_default()
opt.job_name_set = false;
opt.jobid = NO_VAL;
opt.jobid_set = false;
- opt.dependency = NO_VAL;
+ opt.dependency = NULL;
opt.account = NULL;
opt.comment = NULL;
@@ -728,7 +728,7 @@ env_vars_t env_vars[] = {
{"SLURM_CORE_FORMAT", OPT_CORE, NULL, NULL },
{"SLURM_CPU_BIND", OPT_CPU_BIND, NULL, NULL },
{"SLURM_MEM_BIND", OPT_MEM_BIND, NULL, NULL },
-{"SLURM_DEPENDENCY", OPT_INT, &opt.dependency, NULL },
+{"SLURM_DEPENDENCY", OPT_STRING, &opt.dependency, NULL },
{"SLURM_DISTRIBUTION", OPT_DISTRIB, NULL, NULL },
{"SLURM_GEOMETRY", OPT_GEOMETRY, NULL, NULL },
{"SLURM_IMMEDIATE", OPT_INT, &opt.immediate, NULL },
@@ -1175,7 +1175,8 @@ static void set_options(const int argc, char **argv)
opt.partition = xstrdup(optarg);
break;
case (int)'P':
- opt.dependency = _get_int(optarg, "dependency", true);
+ xfree(opt.dependency);
+ opt.dependency = xstrdup(optarg);
break;
case (int)'q':
opt.quit_on_intr = true;
@@ -2068,10 +2069,8 @@ static void _opt_list()
info("nice : %d", opt.nice);
info("account : %s", opt.account);
info("comment : %s", opt.comment);
- if (opt.dependency == NO_VAL)
- info("dependency : none");
- else
- info("dependency : %u", opt.dependency);
+
+ info("dependency : %s", opt.dependency);
info("exclusive : %s", tf_(opt.exclusive));
if (opt.shared != (uint16_t) NO_VAL)
info("shared : %u", opt.shared);
@@ -2143,7 +2142,7 @@ static void _usage(void)
" [--jobid=id] [--verbose] [--slurmd_debug=#]\n"
" [--core=type] [-T threads] [-W sec] [--checkpoint=time]\n"
" [--contiguous] [--mincpus=n] [--mem=MB] [--tmp=MB] [-C list]\n"
-" [--mpi=type] [--account=name] [--dependency=jobid]\n"
+" [--mpi=type] [--account=name] [--dependency=type:jobid]\n"
" [--kill-on-bad-exit] [--propagate[=rlimits] [--comment=name]\n"
" [--cpu_bind=...] [--mem_bind=...]\n"
" [--ntasks-per-node=n] [--ntasks-per-socket=n]\n"
@@ -2204,7 +2203,7 @@ static void _help(void)
" -d, --slurmd-debug=level slurmd debug level\n"
" --core=type change default corefile format type\n"
" (type=\"list\" to list of valid formats)\n"
-" -P, --dependency=jobid defer job until specified jobid completes\n"
+" -P, --dependency=type:jobid defer job until condition on jobid is satisfied\n"
" --nice[=value] decrease secheduling priority by value\n"
" -U, --account=name charge job to specified account\n"
" --comment=name arbitrary comment\n"
diff --git a/src/srun/opt.h b/src/srun/opt.h
index 2e150aaf2e8..2eb2e6713be 100644
--- a/src/srun/opt.h
+++ b/src/srun/opt.h
@@ -123,7 +123,7 @@ typedef struct srun_options {
unsigned int jobid; /* --jobid=jobid */
bool jobid_set; /* true if jobid explicitly set */
char *mpi_type; /* --mpi=type */
- unsigned int dependency;/* --dependency, -P jobid */
+ char *dependency; /* --dependency, -P type:jobid */
int nice; /* --nice */
char *account; /* --account, -U acct_name */
char *comment; /* --comment */
diff --git a/src/sview/job_info.c b/src/sview/job_info.c
index e92bfd1888e..d9085eeb080 100644
--- a/src/sview/job_info.c
+++ b/src/sview/job_info.c
@@ -696,12 +696,8 @@ static const char *_set_job_msg(job_desc_msg_t *job_msg, const char *new_text,
type = "account";
break;
case SORTID_DEPENDENCY:
- temp_int = strtol(new_text, (char **)NULL, 10);
-
+ job_msg->dependency = xstrdup(new_text);
type = "dependency";
- if(temp_int <= 0)
- goto return_error;
- job_msg->dependency = (uint32_t)temp_int;
break;
#ifdef HAVE_BG
case SORTID_GEOMETRY:
@@ -1396,14 +1392,10 @@ static void _layout_job_record(GtkTreeView *treeview,
SORTID_FEATURES),
job_ptr->features);
- if(job_ptr->dependency > 0)
- sprintf(tmp_char, "%u", job_ptr->dependency);
- else
- sprintf(tmp_char, " ");
add_display_treestore_line(update, treestore, &iter,
find_col_name(display_data_job,
SORTID_DEPENDENCY),
- tmp_char);
+ job_ptr->dependency);
add_display_treestore_line(update, treestore, &iter,
find_col_name(display_data_job,
@@ -1689,11 +1681,10 @@ static void _update_job_record(sview_job_info_t *sview_job_info_ptr,
gtk_tree_store_set(treestore, iter,
SORTID_ACCOUNT, job_ptr->account, -1);
- if(job_ptr->dependency > 0) {
- sprintf(tmp_char, "%u", job_ptr->dependency);
- gtk_tree_store_set(treestore, iter,
- SORTID_DEPENDENCY, tmp_char, -1);
- }
+
+ gtk_tree_store_set(treestore, iter,
+ SORTID_DEPENDENCY, job_ptr->dependency, -1);
+
sprintf(tmp_char, "%u", job_ptr->priority);
gtk_tree_store_set(treestore, iter,
SORTID_PRIORITY, tmp_char, -1);
diff --git a/testsuite/expect/test1.42 b/testsuite/expect/test1.42
index be5795cef2f..f090db99e38 100755
--- a/testsuite/expect/test1.42
+++ b/testsuite/expect/test1.42
@@ -8,7 +8,7 @@
# "FAILURE: ..." otherwise with an explanation of the failure, OR
# anything else indicates a failure mode that must be investigated.
############################################################################
-# Copyright (C) 2004 The Regents of the University of California.
+# Copyright (C) 2004-2007 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette1@llnl.gov>
# UCRL-CODE-226842.
@@ -77,7 +77,7 @@ if {$job_id1 == 0} {
#
set match_acct 0
set match_state 0
-set srun_pid [spawn $srun -v --dependency=$job_id1 $scontrol show job $job_id1]
+set srun_pid [spawn $srun -v --dependency=afterany:$job_id1 $scontrol show job $job_id1]
expect {
-re "launching ($number).0" {
set job_id2 $expect_out(1,string)
@@ -117,7 +117,7 @@ set match_acct 0
set match_jobid 0
spawn $scontrol show job $job_id2
expect {
- -re "Dependency=($number)" {
+ -re "Dependency=afterany:($number)" {
set match_jobid $expect_out(1,string)
exp_continue
}
diff --git a/testsuite/expect/test15.14 b/testsuite/expect/test15.14
index 3a775ace17b..c3761158d18 100755
--- a/testsuite/expect/test15.14
+++ b/testsuite/expect/test15.14
@@ -8,7 +8,7 @@
# "FAILURE: ..." otherwise with an explanation of the failure, OR
# anything else indicates a failure mode that must be investigated.
############################################################################
-# Copyright (C) 2004-2006 The Regents of the University of California.
+# Copyright (C) 2004-2007 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette1@llnl.gov>
# UCRL-CODE-226842.
@@ -76,7 +76,7 @@ if {$job_id1 == 0} {
#
set match_acct 0
set match_state 0
-set salloc_pid [spawn $salloc --dependency=$job_id1 $srun $scontrol show job $job_id1]
+set salloc_pid [spawn $salloc --dependency=afterany:$job_id1 $srun $scontrol show job $job_id1]
expect {
-re "Granted job allocation ($number)" {
set job_id2 $expect_out(1,string)
@@ -119,7 +119,7 @@ set match_acct 0
set match_jobid 0
spawn $scontrol show job $job_id2
expect {
- -re "Dependency=($number)" {
+ -re "Dependency=afterany:($number)" {
set match_jobid $expect_out(1,string)
exp_continue
}
diff --git a/testsuite/expect/test17.18 b/testsuite/expect/test17.18
index 78974a8c0c6..def936b5fe9 100755
--- a/testsuite/expect/test17.18
+++ b/testsuite/expect/test17.18
@@ -8,7 +8,7 @@
# "FAILURE: ..." otherwise with an explanation of the failure, OR
# anything else indicates a failure mode that must be investigated.
############################################################################
-# Copyright (C) 2004-2006 The Regents of the University of California.
+# Copyright (C) 2004-2007 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette1@llnl.gov>
# UCRL-CODE-226842.
@@ -82,7 +82,7 @@ make_bash_script $file_in "$scontrol show job $job_id1"
set match_acct 0
set match_state 0
set timeout 30
-spawn $sbatch --dependency=$job_id1 --output=$file_out $file_in
+spawn $sbatch --dependency=afterany:$job_id1 --output=$file_out $file_in
expect {
-re "Submitted batch job ($number)" {
set job_id2 $expect_out(1,string)
@@ -145,7 +145,7 @@ set match_acct 0
set match_jobid 0
spawn $scontrol show job $job_id2
expect {
- -re "Dependency=($number)" {
+ -re "Dependency=afterany:($number)" {
set match_jobid $expect_out(1,string)
exp_continue
}
--
GitLab