From 7a48b58f4696f82d0a307ded07232b49ef3358fb Mon Sep 17 00:00:00 2001
From: Michael Hinton <hinton@schedmd.com>
Date: Mon, 10 Dec 2018 07:33:37 -0700
Subject: [PATCH] GPU frequencies now reset after job is finished
Add step_unconfigure_hardware() to GRES plugin API
Update test39.18 regarding links.
Update GRES docs.
Update docs related to links.
Document GPU frequency resetting behavior.
Specify what the default is for GpuFreqDef.
Move NVML init and shutdown to configure() and unconfigure().
Get rid of superfluous `!= 0`-style statements.
Print note when GPU index != minor number.
Clean up various formatting and other errors.
bug 5520
---
doc/html/gres_plugins.shtml | 17 +-
doc/man/man1/salloc.1 | 4 +-
doc/man/man1/sbatch.1 | 4 +-
doc/man/man1/srun.1 | 4 +-
doc/man/man5/gres.conf.5 | 25 ++-
doc/man/man5/slurm.conf.5 | 4 +-
src/common/gres.c | 21 ++-
src/common/gres.h | 5 +
src/plugins/gres/gpu/gres_gpu.c | 192 ++++++++++++--------
src/plugins/gres/mic/gres_mic.c | 7 +-
src/plugins/gres/mps/gres_mps.c | 5 +
src/plugins/gres/nic/gres_nic.c | 7 +-
src/slurmd/slurmstepd/mgr.c | 26 ++-
testsuite/expect/test39.18 | 313 ++++++++++++++++----------------
testsuite/expect/test39.9 | 2 +-
15 files changed, 377 insertions(+), 259 deletions(-)
diff --git a/doc/html/gres_plugins.shtml b/doc/html/gres_plugins.shtml
index f7dfeb9885c..6fe3cf1f3f4 100644
--- a/doc/html/gres_plugins.shtml
+++ b/doc/html/gres_plugins.shtml
@@ -207,8 +207,8 @@ void step_configure_hardware(bitstr_t *usable_gres, char *settings)
<p style="margin-left:.2in"><b>Description</b>:<br>
Configure device hardware corresponding to all the GRES devices of the plugin
type. The <i>slurmstepd</i> calls this function while privileged and before
-tasks are forked and executed. For example, the
-<span class="commandline"><i>gres/gpu</i></span> plugin will set GPU frequencies
+tasks are forked and executed. The
+<i>gres/gpu</i> plugin sets GPU frequencies
here.</p>
<p style="margin-left:.2in"><b>Arguments</b>: <br>
<span class="commandline">usable_gres</span>
@@ -219,6 +219,17 @@ to the step.<br>
hardware devices.<br>
<p style="margin-left:.2in"><b>Returns</b>: None.</p>
-<p style="text-align:center;">Last modified 30 November 2018</p>
+<p class="commandline">
+void step_unconfigure_hardware(void)
+<p style="margin-left:.2in"><b>Description</b>:<br>
+Do hardware configuration after the step is finished while privileged. This is
+meant to allow Slurm to undo hardware configuration changes performed by
+<span class="commandline">step_configure_hardware()</span>.
+The <i>slurmstepd</i> calls this function while privileged and after tasks
+complete. The <i>gres/gpu</i> plugin resets GPU frequencies to high here.</p>
+<p style="margin-left:.2in"><b>Arguments</b>: None.</p>
+<p style="margin-left:.2in"><b>Returns</b>: None.</p>
+
+<p style="text-align:center;">Last modified 7 December 2018</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/man/man1/salloc.1 b/doc/man/man1/salloc.1
index 8845d62c359..a3c5c0cb1f0 100644
--- a/doc/man/man1/salloc.1
+++ b/doc/man/man1/salloc.1
@@ -1,4 +1,4 @@
-.TH salloc "1" "Slurm Commands" "November 2018" "Slurm Commands"
+.TH salloc "1" "Slurm Commands" "December 2018" "Slurm Commands"
.SH "NAME"
salloc \- Obtain a Slurm job allocation (a set of nodes), execute a command,
@@ -583,6 +583,8 @@ Request that GPUs allocated to the job are configured with specific frequency
values.
This option can be used to independently configure the GPU and its memory
frequencies.
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
In some cases, system power caps may override the requested values.
The field \fItype\fR can be "memory".
If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/doc/man/man1/sbatch.1 b/doc/man/man1/sbatch.1
index 7fd07e820d4..51802421b7c 100644
--- a/doc/man/man1/sbatch.1
+++ b/doc/man/man1/sbatch.1
@@ -1,4 +1,4 @@
-.TH sbatch "1" "Slurm Commands" "November 2018" "Slurm Commands"
+.TH sbatch "1" "Slurm Commands" "December 2018" "Slurm Commands"
.SH "NAME"
sbatch \- Submit a batch script to Slurm.
@@ -676,6 +676,8 @@ Request that GPUs allocated to the job are configured with specific frequency
values.
This option can be used to independently configure the GPU and its memory
frequencies.
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
In some cases, system power caps may override the requested values.
The field \fItype\fR can be "memory".
If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/doc/man/man1/srun.1 b/doc/man/man1/srun.1
index d4a0b635bdb..5b21799b4c0 100644
--- a/doc/man/man1/srun.1
+++ b/doc/man/man1/srun.1
@@ -1,4 +1,4 @@
-.TH srun "1" "Slurm Commands" "November 2018" "Slurm Commands"
+.TH srun "1" "Slurm Commands" "December 2018" "Slurm Commands"
.SH "NAME"
srun \- Run parallel jobs
@@ -856,6 +856,8 @@ Request that GPUs allocated to the job are configured with specific frequency
values.
This option can be used to independently configure the GPU and its memory
frequencies.
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
In some cases, system power caps may override the requested values.
The field \fItype\fR can be "memory".
If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/doc/man/man5/gres.conf.5 b/doc/man/man5/gres.conf.5
index 52b134d5c5e..93010316c1e 100644
--- a/doc/man/man5/gres.conf.5
+++ b/doc/man/man5/gres.conf.5
@@ -1,4 +1,4 @@
-.TH "gres.conf" "5" "Slurm Configuration File" "November 2018" "Slurm Configuration File"
+.TH "gres.conf" "5" "Slurm Configuration File" "December 2018" "Slurm Configuration File"
.SH "NAME"
gres.conf \- Slurm configuration file for generic resource management.
@@ -101,15 +101,24 @@ file.
.TP
\fBLinks\fR
-Specify a comma delimited list of numbers identifying the number of connections
-between the devices so that better connected devices can be co\-scheduled.
+A comma\-delimited list of numbers identifying the number of connections
+between this device and other devices to allow coscheduling of
+better connected devices.
This is an ordered list in which the number of connections this specific
device has to device number 0 would be in the first position, the number of
-connections it has to device 1 in the second position, etc.
-Specify a count of -1 for the number of connections this device has with itself.
-A typical use case would be to identify GPUs having LNLink connectivity.
-This is an optional value and is typically automatically generated using
-NVIDIA libraries.
+connections it has to device number 1 in the second position, etc.
+A \-1 indicates the device itself and a 0 indicates no connection.
+If specified, then this line can only contain a single GRES device (i.e. can
+only contain a single file via \fBFile\fR).
+
+
+This is an optional value and is typically automatically determined using
+the NVIDIA NVML library.
+A typical use case would be to identify GPUs having NVLink connectivity.
+Note that for GPUs, the minor number assigned by the OS and used in the device
+file (i.e. the X in \fI/dev/nvidiaX\fR) is not necessarily the same as the
+device number/index. The device number is created by sorting the GPUs by PCI bus
+ID and then numbering them starting from the smallest bus ID.
.TP
\fBName\fR
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index f791c6b21c0..e0a5e20b656 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -909,7 +909,9 @@ Also see the \fBGroupUpdateForce\fR parameter.
Default GPU frequency to use when running a job step if it
has not been explicitly set using the \-\-gpu\-freq option.
This option can be used to independently configure the GPU and its memory
-frequencies.
+frequencies. Defaults to "high,memory=high".
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
In some cases, system power caps may override the requested values.
The field \fItype\fR can be "memory".
If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/src/common/gres.c b/src/common/gres.c
index a4173cefcf2..9b12253cd93 100644
--- a/src/common/gres.c
+++ b/src/common/gres.c
@@ -135,6 +135,7 @@ typedef struct slurm_gres_ops {
void *data);
List (*get_devices) ( void );
void (*step_configure_hardware) ( bitstr_t *, char * );
+ void (*step_unconfigure_hardware) ( void );
} slurm_gres_ops_t;
/* Gres plugin context, one for each gres type */
@@ -390,6 +391,7 @@ static int _load_gres_plugin(char *plugin_name,
"step_info",
"get_devices",
"step_configure_hardware",
+ "step_unconfigure_hardware",
};
int n_syms = sizeof(syms) / sizeof(char *);
@@ -10049,8 +10051,6 @@ extern void gres_plugin_step_configure_hardware(char *settings)
{
int i;
(void) gres_plugin_init();
- debug2("gres_plugin_step_configure_hardware(%s)", settings);
-
slurm_mutex_lock(&gres_context_lock);
for (i = 0; i < gres_context_cnt; i++) {
bitstr_t *devices = NULL;
@@ -10070,6 +10070,23 @@ extern void gres_plugin_step_configure_hardware(char *settings)
slurm_mutex_unlock(&gres_context_lock);
}
+/*
+ * Optionally undo GRES hardware configuration while privileged
+ */
+extern void gres_plugin_step_unconfigure_hardware(void)
+{
+ int i;
+ (void) gres_plugin_init();
+ slurm_mutex_lock(&gres_context_lock);
+ for (i = 0; i < gres_context_cnt; i++) {
+ if (gres_context[i].ops.step_unconfigure_hardware == NULL) {
+ continue;
+ }
+ (*(gres_context[i].ops.step_unconfigure_hardware)) ();
+ }
+ slurm_mutex_unlock(&gres_context_lock);
+}
+
/*
* Given a set GRES maps and the local process ID, return the bitmap of
* GRES that should be available to this task.
diff --git a/src/common/gres.h b/src/common/gres.h
index 5797526fff5..3097ee3d7e4 100644
--- a/src/common/gres.h
+++ b/src/common/gres.h
@@ -1056,6 +1056,11 @@ extern uint64_t gres_plugin_step_count(List step_gres_list, char *gres_name);
*/
extern void gres_plugin_step_configure_hardware(char *settings);
+/*
+ * Optionally undo GRES hardware configuration while privileged
+ */
+extern void gres_plugin_step_unconfigure_hardware(void);
+
/*
* Set environment as required for all tasks of a job step
* IN/OUT job_env_ptr - environment variable array
diff --git a/src/plugins/gres/gpu/gres_gpu.c b/src/plugins/gres/gpu/gres_gpu.c
index 3f14728962f..5c5e40459f3 100644
--- a/src/plugins/gres/gpu/gres_gpu.c
+++ b/src/plugins/gres/gpu/gres_gpu.c
@@ -117,6 +117,8 @@ static List gres_devices = NULL;
#define FREQS_SIZE 512
#define FREQS_CONCISE 5 // This must never be smaller than 5, or error
+static bitstr_t *saved_gpus = NULL;
+
#ifdef HAVE_NVML
/*
@@ -550,7 +552,7 @@ static void _get_nearest_freq(unsigned int *freq, unsigned int freqs_size,
return;
case GPU_MEDIUM:
- *freq = freqs[(freqs_size - 1)/2];
+ *freq = freqs[(freqs_size - 1) / 2];
debug2("Setting to GPU_MEDIUM:");
debug2("Freq: %u MHz", *freq);
return;
@@ -577,13 +579,13 @@ static void _get_nearest_freq(unsigned int *freq, unsigned int freqs_size,
/* check if freq is out of bounds of freqs */
if (*freq > freqs[0]) {
- log_var(log_lvl, "Rounding requested frequency %u MHz down to %u MHz "
- "(highest available)", *freq, freqs[0]);
+ log_var(log_lvl, "Rounding requested frequency %u MHz down to "
+ "%u MHz (highest available)", *freq, freqs[0]);
*freq = freqs[0];
return;
} else if (*freq < freqs[freqs_size - 1]) {
- log_var(log_lvl, "Rounding requested frequency %u MHz up to %u MHz "
- "(lowest available)", *freq, freqs[freqs_size - 1]);
+ log_var(log_lvl, "Rounding requested frequency %u MHz up to %u "
+ "MHz (lowest available)", *freq, freqs[freqs_size - 1]);
*freq = freqs[freqs_size - 1];
return;
}
@@ -600,14 +602,15 @@ static void _get_nearest_freq(unsigned int *freq, unsigned int freqs_size,
* Safe to advance due to bounds checks above here
*/
if (*freq > freqs[i]) {
- log_var(log_lvl, ("Rounding requested frequency %u MHz up to %u MHz "
- "(next available)", *freq, freqs[i-1]);
- *freq = freqs[i-1];
+ log_var(log_lvl, "Rounding requested frequency %u MHz "
+ "up to %u MHz (next available)", *freq,
+ freqs[i - 1]);
+ *freq = freqs[i - 1];
return;
}
}
- error("%s: Got to the end of the function. This shouldn't happen. Freq: %u MHz",
- __func__, *freq);
+ error("%s: Got to the end of the function. This shouldn't happen. Freq:"
+ " %u MHz", __func__, *freq);
}
/*
@@ -674,11 +677,42 @@ static bool _nvml_set_freqs(nvmlDevice_t *device, unsigned int mem_freq,
#endif // HAVE_NVML
+
/*
- * Set GPU frequencies for this job
- * gpus (IN) GPUs on which to operate
- * gpu_freq (IN) requested frequencies
- * log_lvl (IN) The log level at which to print
+ * Convert a frequency value to a string
+ * Returned string must be xfree()'ed
+ */
+static char *_freq_value_to_string(unsigned int freq)
+{
+ switch (freq) {
+ case GPU_LOW:
+ return xstrdup("low");
+ break;
+ case GPU_MEDIUM:
+ return xstrdup("medium");
+ break;
+ case GPU_HIGH:
+ return xstrdup("high");
+ break;
+ case GPU_HIGH_M1:
+ return xstrdup("highm1");
+ break;
+ default:
+ return xstrdup_printf("%u", freq);
+ break;
+ }
+}
+
+
+/*
+ * Set the frequencies of each GPU specified for the step
+ *
+ * gpus (IN) A bitmap specifying the GPUs on which to operate.
+ * gpu_freq (IN) The frequencies to set each of the GPUs to. If NULL, then
+ * defaults to GpuFreqDef, which defaults to "high,memory=high" if
+ * not set.
+ * log_lvl (IN) The log level at which to print
+ * NOTE: NVML must be initialized beforehand
*/
static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
{
@@ -689,10 +723,6 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
bool freq_set = false, freq_logged = false;
char *tmp = NULL;
-#ifdef HAVE_NVML
- _nvml_init();
-#endif
-
// TODO: Overwrite gpus if env var is specified? Do this earlier?
/*
@@ -703,28 +733,12 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
if (verbose_flag)
debug2("verbose_flag ON");
- // TODO: Pull out keyword detection code into a single location
- if (mem_freq_num == GPU_LOW)
- debug2("User requested mem=low");
- else if (mem_freq_num == GPU_MEDIUM)
- debug2("User requested mem=medium");
- else if (mem_freq_num == GPU_HIGH)
- debug2("User requested mem=high");
- else if (mem_freq_num == GPU_HIGH_M1)
- debug2("User requested mem=highm1");
- else
- debug2("User requested mem=%u", mem_freq_num);
-
- if (gpu_freq_num == GPU_LOW)
- debug2("User requested gfx=low");
- else if (gpu_freq_num == GPU_MEDIUM)
- debug2("User requested gfx=medium");
- else if (gpu_freq_num == GPU_HIGH)
- debug2("User requested gfx=high");
- else if (gpu_freq_num == GPU_HIGH_M1)
- debug2("User requested gfx=highm1");
- else
- debug2("User requested gfx=%u", gpu_freq_num);
+ tmp = _freq_value_to_string(mem_freq_num);
+ debug2("Requested GPU memory frequency: %s", tmp);
+ xfree(tmp);
+ tmp = _freq_value_to_string(gpu_freq_num);
+ debug2("Requested GPU graphics frequency: %s", tmp);
+ xfree(tmp);
if (!mem_freq_num || !gpu_freq_num) {
debug2("%s: No frequencies to set", __func__);
@@ -732,7 +746,7 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
}
/*
- * Set the frequency for each device allocated to the step
+ * Set the frequency of each device allocated to the step
*/
while (++i < gpu_len) {
char *sep = "";
@@ -758,7 +772,6 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
"Can not set GPU frequency\n");
return;
#endif
- // TODO: Will these always be set?
if (mem_freq_num) {
xstrfmtcat(tmp, "%smemory_freq:%u", sep, mem_freq_num);
sep = ",";
@@ -789,9 +802,6 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
fprintf(stderr, "Could not set frequencies for all GPUs. "
"Set %d/%d total GPUs\n", count_set, count);
}
-#ifdef HAVE_NVML
- _nvml_shutdown();
-#endif
}
extern void step_configure_hardware(bitstr_t *usable_gpus, char *tres_freq)
@@ -812,6 +822,12 @@ extern void step_configure_hardware(bitstr_t *usable_gpus, char *tres_freq)
if ((tmp = strchr(freq, ';')))
tmp[0] = '\0';
+ // Save a copy of the GPUs affected, so we can reset things afterwards
+ saved_gpus = bit_copy(usable_gpus);
+
+#ifdef HAVE_NVML
+ _nvml_init();
+#endif
// Set the frequency of each GPU index specified in the bitstr
if (debug_flags & DEBUG_FLAG_GRES)
log_lvl = LOG_LEVEL_INFO;
@@ -821,6 +837,25 @@ extern void step_configure_hardware(bitstr_t *usable_gpus, char *tres_freq)
xfree(freq);
}
+extern void step_unconfigure_hardware(void)
+{
+ log_level_t log_lvl;
+ if (!saved_gpus) {
+ return;
+ }
+
+ // Reset the frequency back to the maximum
+ if (debug_flags & DEBUG_FLAG_GRES)
+ log_lvl = LOG_LEVEL_INFO;
+ else
+ log_lvl = LOG_LEVEL_DEBUG;
+ _set_freq(saved_gpus, "high,memory=high", log_lvl);
+ FREE_NULL_BITMAP(saved_gpus);
+#ifdef HAVE_NVML
+ _nvml_shutdown();
+#endif
+}
+
static void _set_env(char ***env_ptr, void *gres_ptr, int node_inx,
bitstr_t *usable_gres,
bool *already_seen, int *local_inx,
@@ -863,17 +898,16 @@ static void _set_env(char ***env_ptr, void *gres_ptr, int node_inx,
}
/*
- * A one-liner version of _print_gres_conf_full()
+ * Print the GRES conf record on a single line
*/
static void _print_gres_conf(gres_slurmd_conf_t *gres_slurmd_conf,
log_level_t log_lvl)
{
- log_var(log_lvl,
- " GRES[%s](%"PRIu64"): %8s | Cores(%d): %6s | Links: %6s | %15s%s",
- gres_slurmd_conf->name, gres_slurmd_conf->count,
- gres_slurmd_conf->type_name, gres_slurmd_conf->cpu_cnt,
- gres_slurmd_conf->cpus, gres_slurmd_conf->links,
- gres_slurmd_conf->file,
+ log_var(log_lvl, " GRES[%s](%"PRIu64"): %8s | Cores(%d): %6s | "
+ "Links: %6s | %15s%s", gres_slurmd_conf->name,
+ gres_slurmd_conf->count, gres_slurmd_conf->type_name,
+ gres_slurmd_conf->cpu_cnt, gres_slurmd_conf->cpus,
+ gres_slurmd_conf->links, gres_slurmd_conf->file,
gres_slurmd_conf->ignore ? " | IGNORE":"");
}
@@ -1006,9 +1040,8 @@ static int _compare_number_range_str(char *a, char *b)
a_range = _strip_brackets_xmalloc(a_range_brackets);
b_range = _strip_brackets_xmalloc(b_range_brackets);
- if (xstrcmp(a_range, b_range) != 0) {
+ if (xstrcmp(a_range, b_range))
rc = 1;
- }
hostlist_destroy(hl_a);
hostlist_destroy(hl_b);
xfree(a_range_brackets);
@@ -1055,15 +1088,15 @@ static int _find_gres_in_list(void *x, void *key)
gres_slurmd_conf_t *item_b = (gres_slurmd_conf_t *) key;
// Check if type names are equal
- if (xstrcmp(item_a->type_name, item_b->type_name) != 0)
+ if (xstrcmp(item_a->type_name, item_b->type_name))
return 0;
// Check if cpus are equal
- if (_compare_number_range_str(item_a->cpus, item_b->cpus) != 0)
+ if (_compare_number_range_str(item_a->cpus, item_b->cpus))
return 0;
// Check if links are equal
- if (_compare_number_range_str(item_a->links, item_b->links) != 0)
+ if (_compare_number_range_str(item_a->links, item_b->links))
return 0;
// Found!
return 1;
@@ -1078,16 +1111,16 @@ static int _find_gres_device_file_in_list(void *a, void *b)
gres_slurmd_conf_t *item_b = (gres_slurmd_conf_t *) b;
if (item_a->count > item_b->count) {
- if (_key_in_number_range_str(item_a->file, item_b->file) != 0)
+ if (_key_in_number_range_str(item_a->file, item_b->file))
return 0;
} else if (item_a->count < item_b->count) {
- if (_key_in_number_range_str(item_b->file, item_a->file) != 0)
+ if (_key_in_number_range_str(item_b->file, item_a->file))
return 0;
} else {
// a and b have the same number of devices
if (item_a->count == 1) {
// Only a strcmp should be necessary for single devices
- if (xstrcmp(item_a->file, item_b->file) != 0)
+ if (xstrcmp(item_a->file, item_b->file))
return 0;
} else {
if (_key_in_number_range_str(item_a->file, item_b->file)
@@ -1113,10 +1146,10 @@ static int _find_gres_device_in_list(void *a, void *b)
return 0;
// Make sure we have the right record before checking the devices
- if (_find_gres_in_list(a, b) != 1)
+ if (!_find_gres_in_list(a, b))
return 0;
- if (_find_gres_device_file_in_list(a, b) != 1)
+ if (!_find_gres_device_file_in_list(a, b))
return 0;
// Found!
@@ -1148,7 +1181,7 @@ static void _add_gres_to_list(List gres_list, char *name, int device_cnt,
gpu_record = xmalloc(sizeof(gres_slurmd_conf_t));
gpu_record->cpu_cnt = cpu_cnt;
gpu_record->cpus_bitmap = bit_alloc(gpu_record->cpu_cnt);
- if (bit_unfmt(gpu_record->cpus_bitmap, cpu_aff_abs_range) != 0) {
+ if (bit_unfmt(gpu_record->cpus_bitmap, cpu_aff_abs_range)) {
error("%s: bit_unfmt(dst_bitmap, src_str) failed", __func__);
error(" Is the CPU range larger than the CPU count allows?");
error(" src_str: %s", cpu_aff_abs_range);
@@ -1408,7 +1441,7 @@ static void _normalize_gres_conf(List gres_list_conf, List gres_list_system)
char **file_array;
char *hl_name;
// Just move this GRES record if it's not a GPU GRES
- if (xstrcasecmp(gres_record->name, "gpu") != 0) {
+ if (xstrcasecmp(gres_record->name, "gpu")) {
debug2("%s: preserving original `%s` GRES record",
__func__, gres_record->name);
_add_gres_to_list(non_gpu_list,
@@ -1517,7 +1550,7 @@ static void _normalize_gres_conf(List gres_list_conf, List gres_list_system)
list_iterator_destroy(itr_single);
debug2("GPUs detected on the node, but not specified in gres.conf:");
- if (gres_list_system != NULL) {
+ if (gres_list_system) {
itr_system = list_iterator_create(gres_list_system);
while ((gres_record = list_next(itr_system))) {
if (!_device_exists_in_list(gres_list_conf_single,
@@ -1903,7 +1936,7 @@ static void _set_cpu_set_bitstr(bitstr_t *cpu_set_bitstr,
// If this isn't -1, then something went horribly wrong
if (bit_cur != -1)
- fatal("%s: bit_cur(%d) != 0", __func__, bit_cur);
+ fatal("%s: bit_cur(%d) != -1", __func__, bit_cur);
}
#ifdef HAVE_NVML
@@ -2001,7 +2034,7 @@ static List _get_system_gpu_list_nvml(node_config_load_t *node_config)
// Convert cpu range str from machine to abstract(slurm) format
if (node_config->xcpuinfo_mac_to_abs(cpu_aff_mac_range,
- &cpu_aff_abs_range) != 0) {
+ &cpu_aff_abs_range)) {
error(" Conversion from machine to abstract failed");
xfree(cpu_aff_mac_range);
continue;
@@ -2012,15 +2045,18 @@ static List _get_system_gpu_list_nvml(node_config_load_t *node_config)
device_brand = _nvml_get_device_brand(&device);
xstrfmtcat(device_file, "/dev/nvidia%u", minor_number);
- debug2("Device index %d:", i);
- debug2(" GPU Name: %s", device_name);
- debug2(" GPU Brand/Type: %s", device_brand);
- debug2(" GPU UUID: %s", uuid);
- debug2(" GPU PCI Domain/Bus/Device: %u:%u:%u",
- pci_info.domain, pci_info.bus, pci_info.device);
- debug2(" GPU PCI.busId: %s", pci_info.busId);
- debug2(" GPU NV Links: %s", nvlinks);
- debug2(" GPU Device File (minor number): %s", device_file);
+ debug2("GPU index %u:", i);
+ debug2(" Name: %s", device_name);
+ debug2(" Brand/Type: %s", device_brand);
+ debug2(" UUID: %s", uuid);
+ debug2(" PCI Domain/Bus/Device: %u:%u:%u", pci_info.domain,
+ pci_info.bus, pci_info.device);
+ debug2(" PCI Bus ID: %s", pci_info.busId);
+ debug2(" NVLinks: %s", nvlinks);
+ debug2(" Device File (minor number): %s", device_file);
+ if (minor_number != i)
+ debug("Note: GPU index %u is different from minor "
+ "number %u", i, minor_number);
debug2(" CPU Affinity Range: %s", cpu_aff_mac_range);
debug2(" CPU Affinity Range Abstract: %s",cpu_aff_abs_range);
// Print out possible memory frequencies for this device
@@ -2243,7 +2279,7 @@ static void _add_fake_gpus_from_file(List gres_list_system,
}
// Loop through each line of the file
- while (fgets(buffer, 256, f) != NULL) {
+ while (fgets(buffer, 256, f)) {
char *save_ptr = NULL;
char *tok;
int i = 0;
@@ -2390,7 +2426,7 @@ extern int node_config_load(List gres_conf_list,
}
gres_list_system = _get_system_gpu_list_fake();
- if (gres_list_system != NULL)
+ if (gres_list_system)
using_fake_system = true;
#ifdef HAVE_NVML
// Only query real system devices if there is no fake override
diff --git a/src/plugins/gres/mic/gres_mic.c b/src/plugins/gres/mic/gres_mic.c
index 37f24f95c5f..6514d0f96b9 100644
--- a/src/plugins/gres/mic/gres_mic.c
+++ b/src/plugins/gres/mic/gres_mic.c
@@ -253,4 +253,9 @@ extern List get_devices(void)
extern void step_configure_hardware(bitstr_t *usable_gres, char *settings)
{
-}
\ No newline at end of file
+}
+
+extern void step_unconfigure_hardware(void)
+{
+
+}
diff --git a/src/plugins/gres/mps/gres_mps.c b/src/plugins/gres/mps/gres_mps.c
index 87d39b3cada..67abed9998c 100644
--- a/src/plugins/gres/mps/gres_mps.c
+++ b/src/plugins/gres/mps/gres_mps.c
@@ -488,3 +488,8 @@ extern void step_configure_hardware(bitstr_t *usable_gres, char *settings)
{
}
+
+extern void step_unconfigure_hardware(void)
+{
+
+}
\ No newline at end of file
diff --git a/src/plugins/gres/nic/gres_nic.c b/src/plugins/gres/nic/gres_nic.c
index 47383b92467..dc40a447371 100644
--- a/src/plugins/gres/nic/gres_nic.c
+++ b/src/plugins/gres/nic/gres_nic.c
@@ -265,4 +265,9 @@ extern List get_devices(void)
extern void step_configure_hardware(bitstr_t *usable_gres, char *settings)
{
-}
\ No newline at end of file
+}
+
+extern void step_unconfigure_hardware(void)
+{
+
+}
diff --git a/src/slurmd/slurmstepd/mgr.c b/src/slurmd/slurmstepd/mgr.c
index 4aa18130e84..3662a44bcf6 100644
--- a/src/slurmd/slurmstepd/mgr.c
+++ b/src/slurmd/slurmstepd/mgr.c
@@ -1401,7 +1401,18 @@ fail2:
if ((job->cpu_freq_min != NO_VAL) || (job->cpu_freq_max != NO_VAL) ||
(job->cpu_freq_gov != NO_VAL))
cpu_freq_reset(job);
- // TODO: Reset TRES frequencies?
+
+ /*
+ * Reset GRES hardware, if needed. This is where GPU frequency is reset.
+ * Make sure stepd is root. If not, emit error.
+ */
+ if (!job->batch && job->tres_freq) {
+ if (getuid() == (uid_t) 0)
+ gres_plugin_step_unconfigure_hardware();
+ else
+ error("step_unconfigure_hardware() invalid permissions:"
+ " Slurmd was not started as root");
+ }
/*
* Notify srun of completion AFTER frequency reset to avoid race
@@ -1712,18 +1723,15 @@ _fork_all_tasks(stepd_step_rec_t *job, bool *io_initialized)
/*
* Now that errors will be copied back to srun, set the frequencies of
* the GPUs allocated to the step (and eventually other GRES hardware
- * config options)
+ * config options). Make sure stepd is root. If not, emit error.
* TODO: generic "settings" parameter rather than tres_freq
*/
if (!job->batch && job->tres_freq) {
- // Make sure stepd is root. If not, emit error
- // TODO: Leave privilege checking to step_configure_hardware()?
- if (getuid() == (uid_t) 0) {
+ if (getuid() == (uid_t) 0)
gres_plugin_step_configure_hardware(job->tres_freq);
- } else {
- error("Slurmd started with insufficient permissions: "
- "Cannot configure GRES hardware unless privileged");
- }
+ else
+ error("step_configure_hardware() invalid permissions:"
+ " Slurmd was not started as root");
}
/*
diff --git a/testsuite/expect/test39.18 b/testsuite/expect/test39.18
index d6d1cb94bb7..a6193a6b75f 100755
--- a/testsuite/expect/test39.18
+++ b/testsuite/expect/test39.18
@@ -75,6 +75,7 @@ generate_file $slurm_conf $cfgpath/slurm.conf
# Set up dummy device files for testing. They just need to exist
set dev "$cfgpath/nvidia"
+set dev0 "${dev}0"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
@@ -83,6 +84,7 @@ set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
+touch_file $dev0
touch_file $dev1
touch_file $dev2
touch_file $dev3
@@ -217,14 +219,14 @@ proc check_configuration {test_minor gres_conf fake_gpus_conf expected_output} {
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-1|0,1|${dev}1
-tesla|4|0-1|0,1|${dev}2
-tesla|4|2-3|0,1|${dev}3
-tesla|4|2-3|0,1|${dev}4
+tesla|4|0-1|(null)|${dev}1
+tesla|4|0-1|(null)|${dev}2
+tesla|4|2-3|(null)|${dev}3
+tesla|4|2-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-1|0,1|${dev}\[1-2\]|
-GRES_PARSABLE\[gpu\](2):tesla|4|2-3|0,1|${dev}\[3-4\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|0-1|(null)|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|2-3|(null)|${dev}\[3-4\]|
"
check_configuration 0 $gres_conf $fake_gpus_conf $expected_output
@@ -234,17 +236,17 @@ check_configuration 0 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|2-3|0,1|${dev}1
-tesla|4|2-3|0,1|${dev}2
-tesla|4|2-3|0,1|${dev}3
-tesla|4|2-3|0,1|${dev}4
+tesla|4|2-3|(null)|${dev}1
+tesla|4|2-3|(null)|${dev}2
+tesla|4|2-3|(null)|${dev}3
+tesla|4|2-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|2-3|0,1|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|2-3|(null)|${dev}\[1-2\]|
"
check_configuration 1 $gres_conf $fake_gpus_conf $expected_output
@@ -254,19 +256,19 @@ check_configuration 1 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0-1 Links=0,1
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0-1
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-1|0,1|${dev}1
-tesla|4|0-1|0,1|${dev}2
-tesla|4|2-3|0,1|${dev}3
-tesla|4|2-3|0,1|${dev}4
+tesla|4|0-1|(null)|${dev}1
+tesla|4|0-1|(null)|${dev}2
+tesla|4|2-3|(null)|${dev}3
+tesla|4|2-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](1):tesla|4|0-1|0,1|${dev}1|
-GRES_PARSABLE\[gpu\](2):tesla|4|2-3|0,1|${dev}\[3-4\]|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
+GRES_PARSABLE\[gpu\](2):tesla|4|2-3|(null)|${dev}\[3-4\]|
"
check_configuration 2 $gres_conf $fake_gpus_conf $expected_output
@@ -276,18 +278,18 @@ check_configuration 2 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=2-3 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Cores=2-3 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Cores=2-3 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=2-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=2-3
+Name=gpu Type=tesla File=${dev}2 Cores=2-3
+Name=gpu Type=tesla File=${dev}3 Cores=2-3
+Name=gpu Type=tesla File=${dev}4 Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-1|0,1|${dev}5
-tesla|4|0-1|0,1|${dev}6
+tesla|4|0-1|(null)|${dev}5
+tesla|4|0-1|(null)|${dev}6
"
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|2-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|2-3|(null)|${dev}\[1-4\]|
"
check_configuration 4 $gres_conf $fake_gpus_conf $expected_output
@@ -295,24 +297,30 @@ check_configuration 4 $gres_conf $fake_gpus_conf $expected_output
# # Test 5 - Different links, different records
# ##############################################################################
+# Devices 0-2 are all doubly linked to each other
+# Device 5 is singly linked to 3-4
+# Devices 7-8 are doubly linked to each other
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0
-Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=1
-Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,2
-Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,3
-Name=gpu Type=tesla File=${dev}8 Cores=0-3 Links=0,3
+Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
+Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
+Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
+Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
+Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
+Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
+Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
+Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0|${dev}\[1-2\]|
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|1|${dev}\[3-4\]|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1|${dev}5|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,2|${dev}6|
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0,3|${dev}\[7-8\]|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|
"
check_configuration 5 $gres_conf $fake_gpus_conf $expected_output
@@ -331,11 +339,11 @@ check_configuration 100 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3 Links=0,1
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|2-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|2-3|(null)|${dev}\[1-4\]|
"
check_configuration 101 $gres_conf $fake_gpus_conf $expected_output
@@ -475,15 +483,15 @@ check_configuration 108 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=a File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu Type=a File=${dev}2 Cores=0-3 Links=0,1
-Name=gpu Type=b File=${dev}5 Cores=0-3 Links=0,1
-Name=gpu Type=b File=${dev}6 Cores=0-3 Links=0,1
+Name=gpu Type=a File=${dev}1 Cores=0-3
+Name=gpu Type=a File=${dev}2 Cores=0-3
+Name=gpu Type=b File=${dev}5 Cores=0-3
+Name=gpu Type=b File=${dev}6 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](2):a|4|0-3|0,1|${dev}\[1-2\]|
-GRES_PARSABLE\[gpu\](2):b|4|0-3|0,1|${dev}\[5-6\]|
+GRES_PARSABLE\[gpu\](2):a|4|0-3|(null)|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):b|4|0-3|(null)|${dev}\[5-6\]|
"
check_configuration 6 $gres_conf $fake_gpus_conf $expected_output
@@ -493,32 +501,32 @@ check_configuration 6 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-NodeName=$nodename1 Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1 Ignore=true
+NodeName=$nodename1 Name=gpu Type=tesla File=${dev}1 Cores=0-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
"
check_configuration 7 $gres_conf $fake_gpus_conf $expected_output
# ##############################################################################
-# # Test 8 - Ignore records for gres-specified nodes take no effect
+# # Test 8 - Ignore records for GRESs in gres.conf should take no effect
# ##############################################################################
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
+Name=gpu Type=tesla File=${dev}2 Cores=0-3 Ignore=true
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
"
check_configuration 8 $gres_conf $fake_gpus_conf $expected_output
@@ -528,17 +536,17 @@ check_configuration 8 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
"
check_configuration 9 $gres_conf $fake_gpus_conf $expected_output
@@ -548,17 +556,17 @@ check_configuration 9 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla_a File=${dev}5 Cores=0-3 Links=0,1
-Name=gpu Type=tesla_b File=${dev}6 Cores=0-3 Links=0,1
-Name=gpu Type=tesla_a File=${dev}7 Cores=0-3 Links=0,1
-Name=gpu Type=tesla_b File=${dev}8 Cores=0-3 Links=0,1
+Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
+Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
+Name=gpu Type=tesla_a File=${dev}5 Cores=0-3
+Name=gpu Type=tesla_b File=${dev}6 Cores=0-3
+Name=gpu Type=tesla_a File=${dev}7 Cores=0-3
+Name=gpu Type=tesla_b File=${dev}8 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla_a|4|0-3|0,1|${dev}\[1-2,5,7\]|
-GRES_PARSABLE\[gpu\](4):tesla_b|4|0-3|0,1|${dev}\[3-4,6,8\]|
+GRES_PARSABLE\[gpu\](4):tesla_a|4|0-3|(null)|${dev}\[1-2,5,7\]|
+GRES_PARSABLE\[gpu\](4):tesla_b|4|0-3|(null)|${dev}\[3-4,6,8\]|
"
check_configuration 10 $gres_conf $fake_gpus_conf $expected_output
@@ -567,22 +575,22 @@ check_configuration 10 $gres_conf $fake_gpus_conf $expected_output
# ##############################################################################
set gres_conf "
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}3 Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}5 Cores=0-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
-tesla|4|0-3|0,1|${dev}5
-tesla|4|0-3|0,1|${dev}6
-tesla|4|0-3|0,1|${dev}7
-tesla|4|0-3|0,1|${dev}8
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
+tesla|4|0-3|(null)|${dev}5
+tesla|4|0-3|(null)|${dev}6
+tesla|4|0-3|(null)|${dev}7
+tesla|4|0-3|(null)|${dev}8
"
set expected_output "
-GRES_PARSABLE\[gpu\](6):tesla|4|0-3|0,1|${dev}\[1-2,4,6-8\]|
+GRES_PARSABLE\[gpu\](6):tesla|4|0-3|(null)|${dev}\[1-2,4,6-8\]|
"
check_configuration 11 $gres_conf $fake_gpus_conf $expected_output
@@ -592,17 +600,17 @@ check_configuration 11 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-NodeName=$nodename Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1 Ignore=true
+NodeName=$nodename Name=gpu Type=tesla File=${dev}2 Cores=0-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](3):tesla|4|0-3|0,1|${dev}\[1,3-4\]|
+GRES_PARSABLE\[gpu\](3):tesla|4|0-3|(null)|${dev}\[1,3-4\]|
"
check_configuration 12 $gres_conf $fake_gpus_conf $expected_output
@@ -612,21 +620,21 @@ check_configuration 12 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}3 Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}3 Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}4 Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}4 Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
"
set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0,1|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|0-3|(null)|${dev}\[1-2\]|
"
check_configuration 13 $gres_conf $fake_gpus_conf $expected_output
@@ -636,15 +644,15 @@ check_configuration 13 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}3 Cores=0-3
+Name=gpu Type=tesla File=${dev}3 Cores=0-3
+Name=gpu Type=tesla File=${dev}4 Cores=0-3
+Name=gpu Type=tesla File=${dev}4 Cores=0-3
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0,1|${dev}\[3-4\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|0-3|(null)|${dev}\[3-4\]|
"
check_configuration 14 $gres_conf $fake_gpus_conf $expected_output
@@ -654,14 +662,14 @@ check_configuration 14 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
"
check_configuration 15 $gres_conf $fake_gpus_conf $expected_output
@@ -671,14 +679,14 @@ check_configuration 15 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}4 Cores=0-3
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[4,3,2,1\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[4,3,2,1\]|
"
check_configuration 16 $gres_conf $fake_gpus_conf $expected_output
@@ -688,17 +696,17 @@ check_configuration 16 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Cores=0-1 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Cores=0-2 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0
+Name=gpu Type=tesla File=${dev}2 Cores=0-1
+Name=gpu Type=tesla File=${dev}3 Cores=0-2
+Name=gpu Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](1):tesla|4|0|0,1|${dev}1|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-1|0,1|${dev}2|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-2|0,1|${dev}3|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 17 $gres_conf $fake_gpus_conf $expected_output
@@ -708,14 +716,14 @@ check_configuration 17 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0 Links=0,1
-Name=gpu Type=tesla File=${dev}1 Cores=0-1 Links=0,1
-Name=gpu Type=tesla File=${dev}1 Cores=0-2 Links=0,1
-Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0
+Name=gpu Type=tesla File=${dev}1 Cores=0-1
+Name=gpu Type=tesla File=${dev}1 Cores=0-2
+Name=gpu Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](1):tesla|4|0|0,1|${dev}1|
+GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
"
check_configuration 18 $gres_conf $fake_gpus_conf $expected_output
@@ -744,15 +752,15 @@ check_configuration 19 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
-Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1,2
+Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](2):tesla|4|0-3|(null)|${dev}\[1,3\]|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1|${dev}2|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1,2|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|
"
check_configuration 20 $gres_conf $fake_gpus_conf $expected_output
@@ -762,16 +770,16 @@ check_configuration 20 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-# Name=gpu File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu File=${dev}2 Cores=0-3 Links=0,1 Type=\"\"
-Name=gpu File=${dev}3 Cores=0-3 Links=0,1 Type=null
-Name=gpu File=${dev}4 Cores=0-3 Links=0,1 Type=0
+# Name=gpu File=${dev}1 Cores=0-3
+Name=gpu File=${dev}2 Cores=0-3 Type=\"\"
+Name=gpu File=${dev}3 Cores=0-3 Type=null
+Name=gpu File=${dev}4 Cores=0-3 Type=0
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](1):|4|0-3|0,1|${dev}2|
-GRES_PARSABLE\[gpu\](1):null|4|0-3|0,1|${dev}3|
-GRES_PARSABLE\[gpu\](1):0|4|0-3|0,1|${dev}4|
+GRES_PARSABLE\[gpu\](1):|4|0-3|(null)|${dev}2|
+GRES_PARSABLE\[gpu\](1):null|4|0-3|(null)|${dev}3|
+GRES_PARSABLE\[gpu\](1):0|4|0-3|(null)|${dev}4|
"
check_configuration 21 $gres_conf $fake_gpus_conf $expected_output
@@ -781,15 +789,15 @@ check_configuration 21 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu File=${dev}2 Cores=0-3 Links=0,1
-Name=gpu File=${dev}3 Cores=0-3 Links=0,1
-Name=gpu File=${dev}4 Cores=0-3 Links=0,1
+Name=gpu File=${dev}1 Cores=0-3
+Name=gpu File=${dev}2 Cores=0-3
+Name=gpu File=${dev}3 Cores=0-3
+Name=gpu File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](4):(null)|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):(null)|4|0-3|(null)|${dev}\[1-4\]|
"
check_configuration 22 $gres_conf $fake_gpus_conf $expected_output
@@ -799,17 +807,17 @@ check_configuration 22 $gres_conf $fake_gpus_conf $expected_output
set gres_conf "
# This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Links=0,1 Cores=\"\"
-Name=gpu Type=tesla File=${dev}4 Links=0,1 Cores=null
-Name=gpu Type=tesla File=${dev}5 Links=0,1 Cores=0
+Name=gpu Type=tesla File=${dev}1
+Name=gpu Type=tesla File=${dev}2
+Name=gpu Type=tesla File=${dev}3 Cores=\"\"
+Name=gpu Type=tesla File=${dev}4 Cores=null
+Name=gpu Type=tesla File=${dev}5 Cores=0
"
set fake_gpus_conf ""
set expected_output "
-GRES_PARSABLE\[gpu\](3):tesla|4|(null)|0,1|${dev}\[1-3\]|
-GRES_PARSABLE\[gpu\](1):tesla|4|null|0,1|${dev}4|
-GRES_PARSABLE\[gpu\](1):tesla|4|0|0,1|${dev}5|
+GRES_PARSABLE\[gpu\](3):tesla|4|(null)|(null)|${dev}\[1-3\]|
+GRES_PARSABLE\[gpu\](1):tesla|4|null|(null)|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|
"
check_configuration 23 $gres_conf $fake_gpus_conf $expected_output
@@ -1002,6 +1010,7 @@ file delete $cfgpath/fake_gpus.conf
file delete $cfgpath/slurm.conf
file delete $test_prog
file delete $test_ulong_prog
+file delete $dev0
file delete $dev1
file delete $dev2
file delete $dev3
diff --git a/testsuite/expect/test39.9 b/testsuite/expect/test39.9
index 9e899c570f2..3a2f1f50d43 100755
--- a/testsuite/expect/test39.9
+++ b/testsuite/expect/test39.9
@@ -37,7 +37,7 @@ set number_commas "\[0-9_,\]+"
set freq_parse "GpuFreq=memory_freq:($number),graphics_freq:($number)"
set nvml_parse "Slurm is not configured with NVIDIA NVML support"
set nvml_warning "\nWARNING: Slurm is not configured with NVML support. Could not test setting GPU frequencies\n"
-set permission_parse "Cannot configure GRES hardware unless privileged"
+set permission_parse "step_configure_hardware() invalid permissions"
set permission_warning "\nWARNING: Slurmd is not root. Could not test setting GPU frequencies due to insufficient permissions\n"
proc get_node_config {} {
--
GitLab