diff --git a/doc/man/man1/salloc.1 b/doc/man/man1/salloc.1
index 6913d29e9783caf199db2aea0085ead7bc3ead08..7d0347840b79a9a5fadd0a6295964658b6aff5dd 100644
--- a/doc/man/man1/salloc.1
+++ b/doc/man/man1/salloc.1
@@ -608,6 +608,16 @@ License names can be followed by a colon and count
Multiple license names should be comma separated (e.g.
"\-\-licenses=foo:4,bar").
+.TP
+\fB\-M\fR, \fB\-\-clusters\fR=<\fIstring\fR>
+Clusters to issue commands to. Multiple cluster names may be comma separated.
+The job will be submitted to the one cluster providing the earliest expected
+job initiation time. The default value is the current cluster. A value of
+\(aq\fIall\fR' will query to run on all clusters. Note the
+\fB\-\-export\fR option to control environment variables exported
+between clusters.
+Note that the SlurmDBD must be up for this option to work properly.
+
.TP
\fB\-m\fR, \fB\-\-distribution\fR=
\fIarbitrary\fR|<\fIblock\fR|\fIcyclic\fR|\fIplane=<options>\fR[:\fIblock\fR|\fIcyclic\fR|\fIfcyclic\fR]>
@@ -1476,6 +1486,9 @@ Same as \fB\-\-bell\fR
\fBSALLOC_BURST_BUFFER\fR
Same as \fB\-\-bb\fR
.TP
+\fBSALLOC_CLUSTERS\fR or \fBSLURM_CLUSTERS\fR
+Same as \fB\-\-clusters\fR
+.TP
\fBSALLOC_CONN_TYPE\fR
Same as \fB\-\-conn\-type\fR
.TP
diff --git a/doc/man/man1/srun.1 b/doc/man/man1/srun.1
index 6b497a0242f84ab13ff31ce86d00ddcd2a91df20..8a657adf4ba5b59083d7b8d9447cc7822c7e2a1e 100644
--- a/doc/man/man1/srun.1
+++ b/doc/man/man1/srun.1
@@ -911,6 +911,17 @@ License names can be followed by a colon and count
Multiple license names should be comma separated (e.g.
"\-\-licenses=foo:4,bar"). This option applies to job allocations.
+.TP
+\fB\-M\fR, \fB\-\-clusters\fR=<\fIstring\fR>
+Clusters to issue commands to. Multiple cluster names may be comma separated.
+The job will be submitted to the one cluster providing the earliest expected
+job initiation time. The default value is the current cluster. A value of
+\(aq\fIall\fR' will query to run on all clusters. Note the
+\fB\-\-export\fR option to control environment variables exported
+between clusters.
+This option applies only to job allocations.
+Note that the SlurmDBD must be up for this option to work properly.
+
.TP
.na
\fB\-m\fR, \fB\-\-distribution\fR=
diff --git a/src/salloc/opt.c b/src/salloc/opt.c
index 9151518f15d546b31044707cd90d3ef9b517b72c..089f5727b7268afecc7f115408e2d3200e362a54 100644
--- a/src/salloc/opt.c
+++ b/src/salloc/opt.c
@@ -286,6 +286,7 @@ static void _opt_default()
opt.uid = uid;
opt.gid = getgid();
+ opt.clusters = NULL;
opt.cwd = NULL;
opt.progname = NULL;
@@ -408,6 +409,8 @@ env_vars_t env_vars[] = {
{"SALLOC_ACCTG_FREQ", OPT_STRING, &opt.acctg_freq, NULL },
{"SALLOC_BELL", OPT_BELL, NULL, NULL },
{"SALLOC_BURST_BUFFER", OPT_STRING, &opt.burst_buffer, NULL },
+ {"SALLOC_CLUSTERS", OPT_STRING, &opt.clusters, NULL },
+ {"SLURM_CLUSTERS", OPT_STRING, &opt.clusters, NULL },
{"SALLOC_CONN_TYPE", OPT_CONN_TYPE, NULL, NULL },
{"SALLOC_CORE_SPEC", OPT_INT, &opt.core_spec, NULL },
{"SALLOC_CPU_FREQ_REQ", OPT_CPU_FREQ, NULL, NULL },
@@ -682,6 +685,8 @@ void set_options(const int argc, char **argv)
{"kill-command", optional_argument, 0, 'K'},
{"licenses", required_argument, 0, 'L'},
{"distribution", required_argument, 0, 'm'},
+ {"cluster", required_argument, 0, 'M'},
+ {"clusters", required_argument, 0, 'M'},
{"tasks", required_argument, 0, 'n'},
{"ntasks", required_argument, 0, 'n'},
{"nodes", required_argument, 0, 'N'},
@@ -763,7 +768,7 @@ void set_options(const int argc, char **argv)
{NULL, 0, 0, 0}
};
char *opt_string =
- "+A:B:c:C:d:D:F:g:hHI::J:kK::L:m:n:N:Op:P:QRsS:t:uU:vVw:W:x:";
+ "+A:B:c:C:d:D:F:g:hHI::J:kK::L:m:M:n:N:Op:P:QRsS:t:uU:vVw:W:x:";
char *pos_delimit;
struct option *optz = spank_option_table_create(long_options);
@@ -882,6 +887,10 @@ void set_options(const int argc, char **argv)
exit(error_exit);
}
break;
+ case 'M':
+ xfree(opt.clusters);
+ opt.clusters = xstrdup(optarg);
+ break;
case 'n':
opt.ntasks_set = true;
opt.ntasks =
@@ -2076,6 +2085,7 @@ static void _usage(void)
" [--immediate[=secs]] [--no-kill] [--overcommit] [-D path]\n"
" [--oversubscribe] [-J jobname] [--jobid=id]\n"
" [--verbose] [--gid=group] [--uid=user] [--licenses=names]\n"
+" [--clusters=cluster_names]\n"
" [--contiguous] [--mincpus=n] [--mem=MB] [--tmp=MB] [-C list]\n"
" [--account=name] [--dependency=type:jobid] [--comment=name]\n"
#ifdef HAVE_BG /* Blue gene specific options */
@@ -2128,6 +2138,10 @@ static void _help(void)
" -k, --no-kill do not kill job on node failure\n"
" -K, --kill-command[=signal] signal to send terminating job\n"
" -L, --licenses=names required license, comma separated\n"
+" -M, --clusters=names Comma separated list of clusters to issue\n"
+" commands to. Default is current cluster.\n"
+" Name of 'all' will submit to run on all clusters.\n"
+" NOTE: SlurmDBD must up.\n"
" -m, --distribution=type distribution method for processes to nodes\n"
" (type = block|cyclic|arbitrary)\n"
" --mail-type=type notify on state change: BEGIN, END, FAIL or ALL\n"
diff --git a/src/salloc/opt.h b/src/salloc/opt.h
index 8b96d15441f123c42b53ce583f079dc26376645a..21bc42504ffbc6201bf519b31e1e60cf518aabd4 100644
--- a/src/salloc/opt.h
+++ b/src/salloc/opt.h
@@ -60,7 +60,7 @@
typedef enum {BELL_NEVER, BELL_AFTER_DELAY, BELL_ALWAYS} bell_flag_t;
typedef struct salloc_options {
-
+ char *clusters; /* cluster to run this on. */
char *progname; /* argv[0] of this program or
* configuration file if multi_prog */
char* user; /* local username */
diff --git a/src/salloc/salloc.c b/src/salloc/salloc.c
index 817f90ab0b76ea94c5363ad5e4274911aef6891b..a772993dc4d4644b7bf19cbd380a8d9a5e97f52d 100644
--- a/src/salloc/salloc.c
+++ b/src/salloc/salloc.c
@@ -59,6 +59,7 @@
#include "src/common/cpu_frequency.h"
#include "src/common/env.h"
#include "src/common/plugstack.h"
+#include "src/common/proc_args.h"
#include "src/common/read_config.h"
#include "src/common/slurm_rlimits_info.h"
#include "src/common/slurm_time.h"
@@ -305,6 +306,16 @@ int main(int argc, char *argv[])
}
}
+ /* If can run on multiple clusters find the earliest run time
+ * and run it there */
+ desc.clusters = xstrdup(opt.clusters);
+ if (opt.clusters &&
+ slurmdb_get_first_avail_cluster(&desc, opt.clusters,
+ &working_cluster_rec) != SLURM_SUCCESS) {
+ print_db_notok(opt.clusters, 0);
+ exit(error_exit);
+ }
+
callbacks.ping = _ping_handler;
callbacks.timeout = _timeout_handler;
callbacks.job_complete = _job_complete_handler;
@@ -560,6 +571,8 @@ relinquish:
}
}
}
+
+ xfree(desc.clusters);
return rc;
}
diff --git a/src/srun/libsrun/allocate.c b/src/srun/libsrun/allocate.c
index d409eb98658c42f838395f6c18d7cd57a5151bc4..45db52f08cc858ecad47a9cbfdf0b8d9a97928e8 100644
--- a/src/srun/libsrun/allocate.c
+++ b/src/srun/libsrun/allocate.c
@@ -50,6 +50,7 @@
#include "src/common/forward.h"
#include "src/common/log.h"
#include "src/common/macros.h"
+#include "src/common/proc_args.h"
#include "src/common/slurm_auth.h"
#include "src/common/slurm_protocol_api.h"
#include "src/common/slurm_time.h"
@@ -869,6 +870,16 @@ job_desc_msg_create_from_opts (void)
if (opt.mcs_label)
j->mcs_label = opt.mcs_label;
+ /* If can run on multiple clusters find the earliest run time
+ * and run it there */
+ j->clusters = xstrdup(opt.clusters);
+ if (opt.clusters &&
+ slurmdb_get_first_avail_cluster(j, opt.clusters,
+ &working_cluster_rec) != SLURM_SUCCESS) {
+ print_db_notok(opt.clusters, 0);
+ exit(error_exit);
+ }
+
return j;
}
diff --git a/src/srun/libsrun/opt.c b/src/srun/libsrun/opt.c
index f60f7831ceda7eae2003aee56c3fff3be3ab964c..7c5a801f9f69d69f5a34aee073cf04d00290d797 100644
--- a/src/srun/libsrun/opt.c
+++ b/src/srun/libsrun/opt.c
@@ -394,6 +394,7 @@ static void _opt_default(void)
opt.cwd = xstrdup(buf);
opt.cwd_set = false;
+ opt.clusters = NULL;
opt.progname = NULL;
opt.ntasks = 1;
@@ -579,6 +580,7 @@ env_vars_t env_vars[] = {
{"SLURM_BCAST", OPT_BCAST, NULL, NULL },
{"SLURM_BLRTS_IMAGE", OPT_STRING, &opt.blrtsimage, NULL },
{"SLURM_BURST_BUFFER", OPT_STRING, &opt.burst_buffer, NULL },
+{"SLURM_CLUSTERS", OPT_STRING, &opt.clusters, NULL },
{"SLURM_CHECKPOINT", OPT_STRING, &opt.ckpt_interval_str, NULL },
{"SLURM_CHECKPOINT_DIR",OPT_STRING, &opt.ckpt_dir, NULL },
{"SLURM_CNLOAD_IMAGE", OPT_STRING, &opt.linuximage, NULL },
@@ -935,6 +937,8 @@ static void _set_options(const int argc, char **argv)
{"kill-on-bad-exit", optional_argument, 0, 'K'},
{"label", no_argument, 0, 'l'},
{"licenses", required_argument, 0, 'L'},
+ {"cluster", required_argument, 0, 'M'},
+ {"clusters", required_argument, 0, 'M'},
{"distribution", required_argument, 0, 'm'},
{"ntasks", required_argument, 0, 'n'},
{"nodes", required_argument, 0, 'N'},
@@ -1045,7 +1049,7 @@ static void _set_options(const int argc, char **argv)
{"wckey", required_argument, 0, LONG_OPT_WCKEY},
{NULL, 0, 0, 0}
};
- char *opt_string = "+A:B:c:C:d:D:e:Eg:hHi:I::jJ:kK::lL:m:n:N:"
+ char *opt_string = "+A:B:c:C:d:D:e:Eg:hHi:I::jJ:kK::lL:m:M:n:N:"
"o:Op:P:qQr:RsS:t:T:uU:vVw:W:x:XZ";
char *pos_delimit;
bool ntasks_set_opt = false;
@@ -1185,6 +1189,10 @@ static void _set_options(const int argc, char **argv)
xfree(opt.licenses);
opt.licenses = xstrdup(optarg);
break;
+ case 'M':
+ xfree(opt.clusters);
+ opt.clusters = xstrdup(optarg);
+ break;
case (int)'m':
opt.distribution = verify_dist_type(optarg,
&opt.plane_size);
@@ -2742,7 +2750,7 @@ static void _usage(void)
" [--oversubscribe] [--label] [--unbuffered] [-m dist] [-J jobname]\n"
" [--jobid=id] [--verbose] [--slurmd_debug=#] [--gres=list]\n"
" [-T threads] [-W sec] [--checkpoint=time] [--gres-flags=opts]\n"
-" [--checkpoint-dir=dir] [--licenses=names]\n"
+" [--checkpoint-dir=dir] [--licenses=names] [--clusters=cluster_names]\n"
" [--restart-dir=dir] [--qos=qos] [--time-min=minutes]\n"
" [--contiguous] [--mincpus=n] [--mem=MB] [--tmp=MB] [-C list]\n"
" [--mpi=type] [--account=name] [--dependency=type:jobid]\n"
@@ -2819,10 +2827,14 @@ static void _help(void)
" -K, --kill-on-bad-exit kill the job if any task terminates with a\n"
" non-zero exit code\n"
" -l, --label prepend task number to lines of stdout/err\n"
-" -L, --licenses=names required license, comma separated\n"
" --launch-cmd print external launcher command line if not SLURM\n"
" --launcher-opts= options for the external launcher command if not\n"
" SLURM\n"
+" -L, --licenses=names required license, comma separated\n"
+" -M, --clusters=names Comma separated list of clusters to issue\n"
+" commands to. Default is current cluster.\n"
+" Name of 'all' will submit to run on all clusters.\n"
+" NOTE: SlurmDBD must up.\n"
" -m, --distribution=type distribution method for processes to nodes\n"
" (type = block|cyclic|arbitrary)\n"
" --mail-type=type notify on state change: BEGIN, END, FAIL or ALL\n"
diff --git a/src/srun/libsrun/opt.h b/src/srun/libsrun/opt.h
index 1380867a8726dbd69c877bf1ce5b4b287eab8398..1c8e9aa0fe6df4e0794992fc5e20dc94ca301d1a 100644
--- a/src/srun/libsrun/opt.h
+++ b/src/srun/libsrun/opt.h
@@ -66,7 +66,7 @@ extern int _verbose;
extern enum modes mode;
typedef struct srun_options {
-
+ char *clusters; /* cluster to run this on. */
char *progname; /* argv[0] of this program or
* configuration file if multi_prog */
bool multi_prog; /* multiple programs to execute */