diff --git a/doc/html/gres_plugins.shtml b/doc/html/gres_plugins.shtml
index f7dfeb9885cd754fe99a5bc893a9f8894f356587..6fe3cf1f3f461f4e09ff23551caa335affedd1dd 100644
--- a/doc/html/gres_plugins.shtml
+++ b/doc/html/gres_plugins.shtml
@@ -207,8 +207,8 @@ void step_configure_hardware(bitstr_t *usable_gres, char *settings)
 <p style="margin-left:.2in"><b>Description</b>:<br>
 Configure device hardware corresponding to all the GRES devices of the plugin
 type. The <i>slurmstepd</i> calls this function while privileged and before
-tasks are forked and executed. For example, the
-<span class="commandline"><i>gres/gpu</i></span> plugin will set GPU frequencies
+tasks are forked and executed. The
+<i>gres/gpu</i> plugin sets GPU frequencies
 here.</p>
 <p style="margin-left:.2in"><b>Arguments</b>: <br>
 <span class="commandline">usable_gres</span>
@@ -219,6 +219,17 @@ to the step.<br>
 hardware devices.<br>
 <p style="margin-left:.2in"><b>Returns</b>: None.</p>
 
-<p style="text-align:center;">Last modified 30 November 2018</p>
+<p class="commandline">
+void step_unconfigure_hardware(void)
+<p style="margin-left:.2in"><b>Description</b>:<br>
+Do hardware configuration after the step is finished while privileged. This is
+meant to allow Slurm to undo hardware configuration changes performed by
+<span class="commandline">step_configure_hardware()</span>.
+The <i>slurmstepd</i> calls this function while privileged and after tasks
+complete. The <i>gres/gpu</i> plugin resets GPU frequencies to high here.</p>
+<p style="margin-left:.2in"><b>Arguments</b>: None.</p>
+<p style="margin-left:.2in"><b>Returns</b>: None.</p>
+
+<p style="text-align:center;">Last modified 7 December 2018</p>
 
 <!--#include virtual="footer.txt"-->
diff --git a/doc/man/man1/salloc.1 b/doc/man/man1/salloc.1
index 8845d62c35982e96efe444e889256f6e0400ea4b..a3c5c0cb1f0b732018045b997f29484f01154e13 100644
--- a/doc/man/man1/salloc.1
+++ b/doc/man/man1/salloc.1
@@ -1,4 +1,4 @@
-.TH salloc "1" "Slurm Commands" "November 2018" "Slurm Commands"
+.TH salloc "1" "Slurm Commands" "December 2018" "Slurm Commands"
 
 .SH "NAME"
 salloc \- Obtain a Slurm job allocation (a set of nodes), execute a command,
@@ -583,6 +583,8 @@ Request that GPUs allocated to the job are configured with specific frequency
 values.
 This option can be used to independently configure the GPU and its memory
 frequencies.
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
 In some cases, system power caps may override the requested values.
 The field \fItype\fR can be "memory".
 If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/doc/man/man1/sbatch.1 b/doc/man/man1/sbatch.1
index 7fd07e820d4fd7805c69fa84cc487adaea4d4bed..51802421b7caf3ea54aa02b88dd1624610a9a178 100644
--- a/doc/man/man1/sbatch.1
+++ b/doc/man/man1/sbatch.1
@@ -1,4 +1,4 @@
-.TH sbatch "1" "Slurm Commands" "November 2018" "Slurm Commands"
+.TH sbatch "1" "Slurm Commands" "December 2018" "Slurm Commands"
 
 .SH "NAME"
 sbatch \- Submit a batch script to Slurm.
@@ -676,6 +676,8 @@ Request that GPUs allocated to the job are configured with specific frequency
 values.
 This option can be used to independently configure the GPU and its memory
 frequencies.
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
 In some cases, system power caps may override the requested values.
 The field \fItype\fR can be "memory".
 If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/doc/man/man1/srun.1 b/doc/man/man1/srun.1
index d4a0b635bdb44cf787b09908edce159cd409e76e..5b21799b4c003f5b1765e33b9e7c1e7f42a62f6f 100644
--- a/doc/man/man1/srun.1
+++ b/doc/man/man1/srun.1
@@ -1,4 +1,4 @@
-.TH srun "1" "Slurm Commands" "November 2018" "Slurm Commands"
+.TH srun "1" "Slurm Commands" "December 2018" "Slurm Commands"
 
 .SH "NAME"
 srun \- Run parallel jobs
@@ -856,6 +856,8 @@ Request that GPUs allocated to the job are configured with specific frequency
 values.
 This option can be used to independently configure the GPU and its memory
 frequencies.
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
 In some cases, system power caps may override the requested values.
 The field \fItype\fR can be "memory".
 If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/doc/man/man5/gres.conf.5 b/doc/man/man5/gres.conf.5
index 52b134d5c5ee7724c97eb9f26e4dd26d5dbedecb..93010316c1eb0366e9ae7c329fb36aeaa81a3b63 100644
--- a/doc/man/man5/gres.conf.5
+++ b/doc/man/man5/gres.conf.5
@@ -1,4 +1,4 @@
-.TH "gres.conf" "5" "Slurm Configuration File" "November 2018" "Slurm Configuration File"
+.TH "gres.conf" "5" "Slurm Configuration File" "December 2018" "Slurm Configuration File"
 
 .SH "NAME"
 gres.conf \- Slurm configuration file for generic resource management.
@@ -101,15 +101,24 @@ file.
 
 .TP
 \fBLinks\fR
-Specify a comma delimited list of numbers identifying the number of connections
-between the devices so that better connected devices can be co\-scheduled.
+A comma\-delimited list of numbers identifying the number of connections
+between this device and other devices to allow coscheduling of
+better connected devices.
 This is an ordered list in which the number of connections this specific
 device has to device number 0 would be in the first position, the number of
-connections it has to device 1 in the second position, etc.
-Specify a count of -1 for the number of connections this device has with itself.
-A typical use case would be to identify GPUs having LNLink connectivity.
-This is an optional value and is typically automatically generated using
-NVIDIA libraries.
+connections it has to device number 1 in the second position, etc.
+A \-1 indicates the device itself and a 0 indicates no connection.
+If specified, then this line can only contain a single GRES device (i.e. can
+only contain a single file via \fBFile\fR).
+
+
+This is an optional value and is typically automatically determined using
+the NVIDIA NVML library.
+A typical use case would be to identify GPUs having NVLink connectivity.
+Note that for GPUs, the minor number assigned by the OS and used in the device
+file (i.e. the X in \fI/dev/nvidiaX\fR) is not necessarily the same as the
+device number/index. The device number is created by sorting the GPUs by PCI bus
+ID and then numbering them starting from the smallest bus ID.
 
 .TP
 \fBName\fR
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index f791c6b21c0e9f1d6d2284c148a03120c8acb94d..e0a5e20b6564c90bdc3e76113c7641945c902f6c 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -909,7 +909,9 @@ Also see the \fBGroupUpdateForce\fR parameter.
 Default GPU frequency to use when running a job step if it
 has not been explicitly set using the \-\-gpu\-freq option.
 This option can be used to independently configure the GPU and its memory
-frequencies.
+frequencies. Defaults to "high,memory=high".
+After the job is completed, the frequencies of all affected GPUs will be reset
+to the highest possible values.
 In some cases, system power caps may override the requested values.
 The field \fItype\fR can be "memory".
 If \fItype\fR is not specified, the GPU frequency is implied.
diff --git a/src/common/gres.c b/src/common/gres.c
index a4173cefcf2d5299dde8a7ac21b74e956991fb2c..9b12253cd93bf9f2c59f9b63e6da72f854ccbd31 100644
--- a/src/common/gres.c
+++ b/src/common/gres.c
@@ -135,6 +135,7 @@ typedef struct slurm_gres_ops {
 						  void *data);
 	List            (*get_devices)		( void );
 	void            (*step_configure_hardware)	( bitstr_t *, char * );
+	void            (*step_unconfigure_hardware)	( void );
 } slurm_gres_ops_t;
 
 /* Gres plugin context, one for each gres type */
@@ -390,6 +391,7 @@ static int _load_gres_plugin(char *plugin_name,
 		"step_info",
 		"get_devices",
 		"step_configure_hardware",
+		"step_unconfigure_hardware",
 	};
 	int n_syms = sizeof(syms) / sizeof(char *);
 
@@ -10049,8 +10051,6 @@ extern void gres_plugin_step_configure_hardware(char *settings)
 {
 	int i;
 	(void) gres_plugin_init();
-	debug2("gres_plugin_step_configure_hardware(%s)", settings);
-
 	slurm_mutex_lock(&gres_context_lock);
 	for (i = 0; i < gres_context_cnt; i++) {
 		bitstr_t *devices = NULL;
@@ -10070,6 +10070,23 @@ extern void gres_plugin_step_configure_hardware(char *settings)
 	slurm_mutex_unlock(&gres_context_lock);
 }
 
+/*
+ * Optionally undo GRES hardware configuration while privileged
+ */
+extern void gres_plugin_step_unconfigure_hardware(void)
+{
+	int i;
+	(void) gres_plugin_init();
+	slurm_mutex_lock(&gres_context_lock);
+	for (i = 0; i < gres_context_cnt; i++) {
+		if (gres_context[i].ops.step_unconfigure_hardware == NULL) {
+			continue;
+		}
+		(*(gres_context[i].ops.step_unconfigure_hardware)) ();
+	}
+	slurm_mutex_unlock(&gres_context_lock);
+}
+
 /*
  * Given a set GRES maps and the local process ID, return the bitmap of
  * GRES that should be available to this task.
diff --git a/src/common/gres.h b/src/common/gres.h
index 5797526fff5d9b82e0407c4176d3e7d78cc129a6..3097ee3d7e496fba5169e6f325b5e9656692f1b2 100644
--- a/src/common/gres.h
+++ b/src/common/gres.h
@@ -1056,6 +1056,11 @@ extern uint64_t gres_plugin_step_count(List step_gres_list, char *gres_name);
  */
 extern void gres_plugin_step_configure_hardware(char *settings);
 
+/*
+ * Optionally undo GRES hardware configuration while privileged
+ */
+extern void gres_plugin_step_unconfigure_hardware(void);
+
 /*
  * Set environment as required for all tasks of a job step
  * IN/OUT job_env_ptr - environment variable array
diff --git a/src/plugins/gres/gpu/gres_gpu.c b/src/plugins/gres/gpu/gres_gpu.c
index 3f14728962f00ad590841d3976e49b502b35b441..5c5e40459f3fd473532b11861e6cd0035a8042d0 100644
--- a/src/plugins/gres/gpu/gres_gpu.c
+++ b/src/plugins/gres/gpu/gres_gpu.c
@@ -117,6 +117,8 @@ static List	gres_devices		= NULL;
 #define FREQS_SIZE	512
 #define FREQS_CONCISE	5 // This must never be smaller than 5, or error
 
+static bitstr_t	*saved_gpus		= NULL;
+
 #ifdef HAVE_NVML
 
 /*
@@ -550,7 +552,7 @@ static void _get_nearest_freq(unsigned int *freq, unsigned int freqs_size,
 		return;
 
 	case GPU_MEDIUM:
-		*freq = freqs[(freqs_size - 1)/2];
+		*freq = freqs[(freqs_size - 1) / 2];
 		debug2("Setting to GPU_MEDIUM:");
 		debug2("Freq: %u MHz", *freq);
 		return;
@@ -577,13 +579,13 @@ static void _get_nearest_freq(unsigned int *freq, unsigned int freqs_size,
 
 	/* check if freq is out of bounds of freqs */
 	if (*freq > freqs[0]) {
-		log_var(log_lvl, "Rounding requested frequency %u MHz down to %u MHz "
-			"(highest available)", *freq, freqs[0]);
+		log_var(log_lvl, "Rounding requested frequency %u MHz down to "
+			"%u MHz (highest available)", *freq, freqs[0]);
 		*freq = freqs[0];
 		return;
 	} else if (*freq < freqs[freqs_size - 1]) {
-		log_var(log_lvl, "Rounding requested frequency %u MHz up to %u MHz "
-			"(lowest available)", *freq, freqs[freqs_size - 1]);
+		log_var(log_lvl, "Rounding requested frequency %u MHz up to %u "
+			"MHz (lowest available)", *freq, freqs[freqs_size - 1]);
 		*freq = freqs[freqs_size - 1];
 		return;
 	}
@@ -600,14 +602,15 @@ static void _get_nearest_freq(unsigned int *freq, unsigned int freqs_size,
 		 * Safe to advance due to bounds checks above here
 		 */
 		if (*freq > freqs[i]) {
-			log_var(log_lvl, ("Rounding requested frequency %u MHz up to %u MHz "
-				"(next available)", *freq, freqs[i-1]);
-			*freq = freqs[i-1];
+			log_var(log_lvl, "Rounding requested frequency %u MHz "
+				"up to %u MHz (next available)", *freq,
+				freqs[i - 1]);
+			*freq = freqs[i - 1];
 			return;
 		}
 	}
-	error("%s: Got to the end of the function. This shouldn't happen. Freq: %u MHz",
-	      __func__, *freq);
+	error("%s: Got to the end of the function. This shouldn't happen. Freq:"
+	      " %u MHz", __func__, *freq);
 }
 
 /*
@@ -674,11 +677,42 @@ static bool _nvml_set_freqs(nvmlDevice_t *device, unsigned int mem_freq,
 
 #endif // HAVE_NVML
 
+
 /*
- * Set GPU frequencies for this job
- * gpus			(IN) GPUs on which to operate
- * gpu_freq		(IN) requested frequencies
- * log_lvl		(IN) The log level at which to print
+ * Convert a frequency value to a string
+ * Returned string must be xfree()'ed
+ */
+static char *_freq_value_to_string(unsigned int freq)
+{
+	switch (freq) {
+	case GPU_LOW:
+		return xstrdup("low");
+		break;
+	case GPU_MEDIUM:
+		return xstrdup("medium");
+		break;
+	case GPU_HIGH:
+		return xstrdup("high");
+		break;
+	case GPU_HIGH_M1:
+		return xstrdup("highm1");
+		break;
+	default:
+		return xstrdup_printf("%u", freq);
+		break;
+	}
+}
+
+
+/*
+ * Set the frequencies of each GPU specified for the step
+ *
+ * gpus		(IN) A bitmap specifying the GPUs on which to operate.
+ * gpu_freq	(IN) The frequencies to set each of the GPUs to. If NULL, then
+ * 		defaults to GpuFreqDef, which defaults to "high,memory=high" if
+ * 		not set.
+ * log_lvl	(IN) The log level at which to print
+ * NOTE: NVML must be initialized beforehand
  */
 static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
 {
@@ -689,10 +723,6 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
 	bool freq_set = false, freq_logged = false;
 	char *tmp = NULL;
 
-#ifdef HAVE_NVML
-	_nvml_init();
-#endif
-
 	// TODO: Overwrite gpus if env var is specified? Do this earlier?
 
 	/*
@@ -703,28 +733,12 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
 	if (verbose_flag)
 		debug2("verbose_flag ON");
 
-	// TODO: Pull out keyword detection code into a single location
-	if (mem_freq_num == GPU_LOW)
-		debug2("User requested mem=low");
-	else if (mem_freq_num == GPU_MEDIUM)
-		debug2("User requested mem=medium");
-	else if (mem_freq_num == GPU_HIGH)
-		debug2("User requested mem=high");
-	else if (mem_freq_num == GPU_HIGH_M1)
-		debug2("User requested mem=highm1");
-	else
-		debug2("User requested mem=%u", mem_freq_num);
-
-	if (gpu_freq_num == GPU_LOW)
-		debug2("User requested gfx=low");
-	else if (gpu_freq_num == GPU_MEDIUM)
-		debug2("User requested gfx=medium");
-	else if (gpu_freq_num == GPU_HIGH)
-		debug2("User requested gfx=high");
-	else if (gpu_freq_num == GPU_HIGH_M1)
-		debug2("User requested gfx=highm1");
-	else
-		debug2("User requested gfx=%u", gpu_freq_num);
+	tmp = _freq_value_to_string(mem_freq_num);
+	debug2("Requested GPU memory frequency: %s", tmp);
+	xfree(tmp);
+	tmp = _freq_value_to_string(gpu_freq_num);
+	debug2("Requested GPU graphics frequency: %s", tmp);
+	xfree(tmp);
 
 	if (!mem_freq_num || !gpu_freq_num) {
 		debug2("%s: No frequencies to set", __func__);
@@ -732,7 +746,7 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
 	}
 
 	/*
-	 * Set the frequency for each device allocated to the step
+	 * Set the frequency of each device allocated to the step
 	 */
 	while (++i < gpu_len) {
 		char *sep = "";
@@ -758,7 +772,6 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
 			"Can not set GPU frequency\n");
 		return;
 #endif
-		// TODO: Will these always be set?
 		if (mem_freq_num) {
 			xstrfmtcat(tmp, "%smemory_freq:%u", sep, mem_freq_num);
 			sep = ",";
@@ -789,9 +802,6 @@ static void _set_freq(bitstr_t *gpus, char *gpu_freq, log_level_t log_lvl)
 		fprintf(stderr, "Could not set frequencies for all GPUs. "
 			"Set %d/%d total GPUs\n", count_set, count);
 	}
-#ifdef HAVE_NVML
-	_nvml_shutdown();
-#endif
 }
 
 extern void step_configure_hardware(bitstr_t *usable_gpus, char *tres_freq)
@@ -812,6 +822,12 @@ extern void step_configure_hardware(bitstr_t *usable_gpus, char *tres_freq)
 	if ((tmp = strchr(freq, ';')))
 		tmp[0] = '\0';
 
+	// Save a copy of the GPUs affected, so we can reset things afterwards
+	saved_gpus = bit_copy(usable_gpus);
+
+#ifdef HAVE_NVML
+	_nvml_init();
+#endif
 	// Set the frequency of each GPU index specified in the bitstr
 	if (debug_flags & DEBUG_FLAG_GRES)
 		log_lvl = LOG_LEVEL_INFO;
@@ -821,6 +837,25 @@ extern void step_configure_hardware(bitstr_t *usable_gpus, char *tres_freq)
 	xfree(freq);
 }
 
+extern void step_unconfigure_hardware(void)
+{
+	log_level_t log_lvl;
+	if (!saved_gpus) {
+		return;
+	}
+
+	// Reset the frequency back to the maximum
+	if (debug_flags & DEBUG_FLAG_GRES)
+		log_lvl = LOG_LEVEL_INFO;
+	else
+		log_lvl = LOG_LEVEL_DEBUG;
+	_set_freq(saved_gpus, "high,memory=high", log_lvl);
+	FREE_NULL_BITMAP(saved_gpus);
+#ifdef HAVE_NVML
+	_nvml_shutdown();
+#endif
+}
+
 static void _set_env(char ***env_ptr, void *gres_ptr, int node_inx,
 		     bitstr_t *usable_gres,
 		     bool *already_seen, int *local_inx,
@@ -863,17 +898,16 @@ static void _set_env(char ***env_ptr, void *gres_ptr, int node_inx,
 }
 
 /*
- * A one-liner version of _print_gres_conf_full()
+ * Print the GRES conf record on a single line
  */
 static void _print_gres_conf(gres_slurmd_conf_t *gres_slurmd_conf,
 			     log_level_t log_lvl)
 {
-	log_var(log_lvl,
-		"    GRES[%s](%"PRIu64"): %8s | Cores(%d): %6s | Links: %6s | %15s%s",
-		gres_slurmd_conf->name, gres_slurmd_conf->count,
-		gres_slurmd_conf->type_name, gres_slurmd_conf->cpu_cnt,
-		gres_slurmd_conf->cpus, gres_slurmd_conf->links,
-		gres_slurmd_conf->file,
+	log_var(log_lvl, "    GRES[%s](%"PRIu64"): %8s | Cores(%d): %6s | "
+		"Links: %6s | %15s%s", gres_slurmd_conf->name,
+		gres_slurmd_conf->count, gres_slurmd_conf->type_name,
+		gres_slurmd_conf->cpu_cnt, gres_slurmd_conf->cpus,
+		gres_slurmd_conf->links, gres_slurmd_conf->file,
 		gres_slurmd_conf->ignore ? " | IGNORE":"");
 }
 
@@ -1006,9 +1040,8 @@ static int _compare_number_range_str(char *a, char *b)
 	a_range = _strip_brackets_xmalloc(a_range_brackets);
 	b_range = _strip_brackets_xmalloc(b_range_brackets);
 
-	if (xstrcmp(a_range, b_range) != 0) {
+	if (xstrcmp(a_range, b_range))
 		rc = 1;
-	}
 	hostlist_destroy(hl_a);
 	hostlist_destroy(hl_b);
 	xfree(a_range_brackets);
@@ -1055,15 +1088,15 @@ static int _find_gres_in_list(void *x, void *key)
 	gres_slurmd_conf_t *item_b = (gres_slurmd_conf_t *) key;
 
 	// Check if type names are equal
-	if (xstrcmp(item_a->type_name, item_b->type_name) != 0)
+	if (xstrcmp(item_a->type_name, item_b->type_name))
 		return 0;
 
 	// Check if cpus are equal
-	if (_compare_number_range_str(item_a->cpus, item_b->cpus) != 0)
+	if (_compare_number_range_str(item_a->cpus, item_b->cpus))
 		return 0;
 
 	// Check if links are equal
-	if (_compare_number_range_str(item_a->links, item_b->links) != 0)
+	if (_compare_number_range_str(item_a->links, item_b->links))
 		return 0;
 	// Found!
 	return 1;
@@ -1078,16 +1111,16 @@ static int _find_gres_device_file_in_list(void *a, void *b)
 	gres_slurmd_conf_t *item_b = (gres_slurmd_conf_t *) b;
 
 	if (item_a->count > item_b->count) {
-		if (_key_in_number_range_str(item_a->file, item_b->file) != 0)
+		if (_key_in_number_range_str(item_a->file, item_b->file))
 			return 0;
 	} else if (item_a->count < item_b->count) {
-		if (_key_in_number_range_str(item_b->file, item_a->file) != 0)
+		if (_key_in_number_range_str(item_b->file, item_a->file))
 			return 0;
 	} else {
 		// a and b have the same number of devices
 		if (item_a->count == 1) {
 			// Only a strcmp should be necessary for single devices
-			if (xstrcmp(item_a->file, item_b->file) != 0)
+			if (xstrcmp(item_a->file, item_b->file))
 				return 0;
 		} else {
 			if (_key_in_number_range_str(item_a->file, item_b->file)
@@ -1113,10 +1146,10 @@ static int _find_gres_device_in_list(void *a, void *b)
 		return 0;
 
 	// Make sure we have the right record before checking the devices
-	if (_find_gres_in_list(a, b) != 1)
+	if (!_find_gres_in_list(a, b))
 		return 0;
 
-	if (_find_gres_device_file_in_list(a, b) != 1)
+	if (!_find_gres_device_file_in_list(a, b))
 		return 0;
 
 	// Found!
@@ -1148,7 +1181,7 @@ static void _add_gres_to_list(List gres_list, char *name, int device_cnt,
 		gpu_record = xmalloc(sizeof(gres_slurmd_conf_t));
 	gpu_record->cpu_cnt = cpu_cnt;
 	gpu_record->cpus_bitmap = bit_alloc(gpu_record->cpu_cnt);
-	if (bit_unfmt(gpu_record->cpus_bitmap, cpu_aff_abs_range) != 0) {
+	if (bit_unfmt(gpu_record->cpus_bitmap, cpu_aff_abs_range)) {
 		error("%s: bit_unfmt(dst_bitmap, src_str) failed", __func__);
 		error("    Is the CPU range larger than the CPU count allows?");
 		error("    src_str: %s", cpu_aff_abs_range);
@@ -1408,7 +1441,7 @@ static void _normalize_gres_conf(List gres_list_conf, List gres_list_system)
 		char **file_array;
 		char *hl_name;
 		// Just move this GRES record if it's not a GPU GRES
-		if (xstrcasecmp(gres_record->name, "gpu") != 0) {
+		if (xstrcasecmp(gres_record->name, "gpu")) {
 			debug2("%s: preserving original `%s` GRES record",
 			       __func__, gres_record->name);
 			_add_gres_to_list(non_gpu_list,
@@ -1517,7 +1550,7 @@ static void _normalize_gres_conf(List gres_list_conf, List gres_list_system)
 	list_iterator_destroy(itr_single);
 
 	debug2("GPUs detected on the node, but not specified in gres.conf:");
-	if (gres_list_system != NULL) {
+	if (gres_list_system) {
 		itr_system = list_iterator_create(gres_list_system);
 		while ((gres_record = list_next(itr_system))) {
 			if (!_device_exists_in_list(gres_list_conf_single,
@@ -1903,7 +1936,7 @@ static void _set_cpu_set_bitstr(bitstr_t *cpu_set_bitstr,
 
 	// If this isn't -1, then something went horribly wrong
 	if (bit_cur != -1)
-		fatal("%s: bit_cur(%d) != 0", __func__, bit_cur);
+		fatal("%s: bit_cur(%d) != -1", __func__, bit_cur);
 }
 
 #ifdef HAVE_NVML
@@ -2001,7 +2034,7 @@ static List _get_system_gpu_list_nvml(node_config_load_t *node_config)
 
 		// Convert cpu range str from machine to abstract(slurm) format
 		if (node_config->xcpuinfo_mac_to_abs(cpu_aff_mac_range,
-						     &cpu_aff_abs_range) != 0) {
+						     &cpu_aff_abs_range)) {
 			error("    Conversion from machine to abstract failed");
 			xfree(cpu_aff_mac_range);
 			continue;
@@ -2012,15 +2045,18 @@ static List _get_system_gpu_list_nvml(node_config_load_t *node_config)
 		device_brand = _nvml_get_device_brand(&device);
 		xstrfmtcat(device_file, "/dev/nvidia%u", minor_number);
 
-		debug2("Device index %d:", i);
-		debug2("    GPU Name: %s", device_name);
-		debug2("    GPU Brand/Type: %s", device_brand);
-		debug2("    GPU UUID: %s", uuid);
-		debug2("    GPU PCI Domain/Bus/Device: %u:%u:%u",
-		       pci_info.domain, pci_info.bus, pci_info.device);
-		debug2("    GPU PCI.busId: %s", pci_info.busId);
-		debug2("    GPU NV Links: %s", nvlinks);
-		debug2("    GPU Device File (minor number): %s", device_file);
+		debug2("GPU index %u:", i);
+		debug2("    Name: %s", device_name);
+		debug2("    Brand/Type: %s", device_brand);
+		debug2("    UUID: %s", uuid);
+		debug2("    PCI Domain/Bus/Device: %u:%u:%u", pci_info.domain,
+		       pci_info.bus, pci_info.device);
+		debug2("    PCI Bus ID: %s", pci_info.busId);
+		debug2("    NVLinks: %s", nvlinks);
+		debug2("    Device File (minor number): %s", device_file);
+		if (minor_number != i)
+			debug("Note: GPU index %u is different from minor "
+			      "number %u", i, minor_number);
 		debug2("    CPU Affinity Range: %s", cpu_aff_mac_range);
 		debug2("    CPU Affinity Range Abstract: %s",cpu_aff_abs_range);
 		// Print out possible memory frequencies for this device
@@ -2243,7 +2279,7 @@ static void _add_fake_gpus_from_file(List gres_list_system,
 	}
 
 	// Loop through each line of the file
-	while (fgets(buffer, 256, f) != NULL) {
+	while (fgets(buffer, 256, f)) {
 		char *save_ptr = NULL;
 		char *tok;
 		int i = 0;
@@ -2390,7 +2426,7 @@ extern int node_config_load(List gres_conf_list,
 	}
 
 	gres_list_system = _get_system_gpu_list_fake();
-	if (gres_list_system != NULL)
+	if (gres_list_system)
 		using_fake_system = true;
 #ifdef HAVE_NVML
 	// Only query real system devices if there is no fake override
diff --git a/src/plugins/gres/mic/gres_mic.c b/src/plugins/gres/mic/gres_mic.c
index 37f24f95c5fab4d6c72c213eb840bd79226b6a85..6514d0f96b9f6d4e655dc7fe5eae293e75eb5c14 100644
--- a/src/plugins/gres/mic/gres_mic.c
+++ b/src/plugins/gres/mic/gres_mic.c
@@ -253,4 +253,9 @@ extern List get_devices(void)
 extern void step_configure_hardware(bitstr_t *usable_gres, char *settings)
 {
 
-}
\ No newline at end of file
+}
+
+extern void step_unconfigure_hardware(void)
+{
+
+}
diff --git a/src/plugins/gres/mps/gres_mps.c b/src/plugins/gres/mps/gres_mps.c
index 87d39b3cada6fd2d6bbec88455567da6f6f1a18c..67abed9998cebca0735960eff1edf323378b94a1 100644
--- a/src/plugins/gres/mps/gres_mps.c
+++ b/src/plugins/gres/mps/gres_mps.c
@@ -488,3 +488,8 @@ extern void step_configure_hardware(bitstr_t *usable_gres, char *settings)
 {
 
 }
+
+extern void step_unconfigure_hardware(void)
+{
+
+}
\ No newline at end of file
diff --git a/src/plugins/gres/nic/gres_nic.c b/src/plugins/gres/nic/gres_nic.c
index 47383b92467e6a11d49a12d38fcde4b0368e758f..dc40a44737186f64dd5edfa77a7ae9e12870a19e 100644
--- a/src/plugins/gres/nic/gres_nic.c
+++ b/src/plugins/gres/nic/gres_nic.c
@@ -265,4 +265,9 @@ extern List get_devices(void)
 extern void step_configure_hardware(bitstr_t *usable_gres, char *settings)
 {
 
-}
\ No newline at end of file
+}
+
+extern void step_unconfigure_hardware(void)
+{
+
+}
diff --git a/src/slurmd/slurmstepd/mgr.c b/src/slurmd/slurmstepd/mgr.c
index 4aa18130e84d72de4046f3a2b212c7793e4db03f..3662a44bcf6c5c85a33bffe04ad1abff74890015 100644
--- a/src/slurmd/slurmstepd/mgr.c
+++ b/src/slurmd/slurmstepd/mgr.c
@@ -1401,7 +1401,18 @@ fail2:
 	if ((job->cpu_freq_min != NO_VAL) || (job->cpu_freq_max != NO_VAL) ||
 	    (job->cpu_freq_gov != NO_VAL))
 		cpu_freq_reset(job);
-	// TODO: Reset TRES frequencies?
+
+	/*
+	 * Reset GRES hardware, if needed. This is where GPU frequency is reset.
+	 * Make sure stepd is root. If not, emit error.
+	 */
+	if (!job->batch && job->tres_freq) {
+		if (getuid() == (uid_t) 0)
+			gres_plugin_step_unconfigure_hardware();
+		else
+			error("step_unconfigure_hardware() invalid permissions:"
+			      " Slurmd was not started as root");
+	}
 
 	/*
 	 * Notify srun of completion AFTER frequency reset to avoid race
@@ -1712,18 +1723,15 @@ _fork_all_tasks(stepd_step_rec_t *job, bool *io_initialized)
 	/*
 	 * Now that errors will be copied back to srun, set the frequencies of
 	 * the GPUs allocated to the step (and eventually other GRES hardware
-	 * config options)
+	 * config options). Make sure stepd is root. If not, emit error.
 	 * TODO: generic "settings" parameter rather than tres_freq
 	 */
 	if (!job->batch && job->tres_freq) {
-		// Make sure stepd is root. If not, emit error
-		// TODO: Leave privilege checking to step_configure_hardware()?
-		if (getuid() == (uid_t) 0) {
+		if (getuid() == (uid_t) 0)
 			gres_plugin_step_configure_hardware(job->tres_freq);
-		} else {
-			error("Slurmd started with insufficient permissions: "
-			      "Cannot configure GRES hardware unless privileged");
-		}
+		else
+			error("step_configure_hardware() invalid permissions:"
+			      " Slurmd was not started as root");
 	}
 
 	/*
diff --git a/testsuite/expect/test39.18 b/testsuite/expect/test39.18
index d6d1cb94bb7e38c66f3da3dad930b2a416be397b..a6193a6b75f96170a39d81033d16c36d4470cf31 100755
--- a/testsuite/expect/test39.18
+++ b/testsuite/expect/test39.18
@@ -75,6 +75,7 @@ generate_file $slurm_conf $cfgpath/slurm.conf
 
 # Set up dummy device files for testing. They just need to exist
 set dev "$cfgpath/nvidia"
+set dev0 "${dev}0"
 set dev1 "${dev}1"
 set dev2 "${dev}2"
 set dev3 "${dev}3"
@@ -83,6 +84,7 @@ set dev5 "${dev}5"
 set dev6 "${dev}6"
 set dev7 "${dev}7"
 set dev8 "${dev}8"
+touch_file $dev0
 touch_file $dev1
 touch_file $dev2
 touch_file $dev3
@@ -217,14 +219,14 @@ proc check_configuration {test_minor gres_conf fake_gpus_conf expected_output} {
 set gres_conf ""
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-1|0,1|${dev}1
-tesla|4|0-1|0,1|${dev}2
-tesla|4|2-3|0,1|${dev}3
-tesla|4|2-3|0,1|${dev}4
+tesla|4|0-1|(null)|${dev}1
+tesla|4|0-1|(null)|${dev}2
+tesla|4|2-3|(null)|${dev}3
+tesla|4|2-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-1|0,1|${dev}\[1-2\]|
-GRES_PARSABLE\[gpu\](2):tesla|4|2-3|0,1|${dev}\[3-4\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|0-1|(null)|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|2-3|(null)|${dev}\[3-4\]|
 "
 check_configuration 0 $gres_conf $fake_gpus_conf $expected_output
 
@@ -234,17 +236,17 @@ check_configuration 0 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Ignore=true
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|2-3|0,1|${dev}1
-tesla|4|2-3|0,1|${dev}2
-tesla|4|2-3|0,1|${dev}3
-tesla|4|2-3|0,1|${dev}4
+tesla|4|2-3|(null)|${dev}1
+tesla|4|2-3|(null)|${dev}2
+tesla|4|2-3|(null)|${dev}3
+tesla|4|2-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|2-3|0,1|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|2-3|(null)|${dev}\[1-2\]|
 "
 check_configuration 1 $gres_conf $fake_gpus_conf $expected_output
 
@@ -254,19 +256,19 @@ check_configuration 1 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0-1 Links=0,1
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0-1
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-1|0,1|${dev}1
-tesla|4|0-1|0,1|${dev}2
-tesla|4|2-3|0,1|${dev}3
-tesla|4|2-3|0,1|${dev}4
+tesla|4|0-1|(null)|${dev}1
+tesla|4|0-1|(null)|${dev}2
+tesla|4|2-3|(null)|${dev}3
+tesla|4|2-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](1):tesla|4|0-1|0,1|${dev}1|
-GRES_PARSABLE\[gpu\](2):tesla|4|2-3|0,1|${dev}\[3-4\]|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
+GRES_PARSABLE\[gpu\](2):tesla|4|2-3|(null)|${dev}\[3-4\]|
 "
 check_configuration 2 $gres_conf $fake_gpus_conf $expected_output
 
@@ -276,18 +278,18 @@ check_configuration 2 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=2-3 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Cores=2-3 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Cores=2-3 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=2-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=2-3
+Name=gpu Type=tesla File=${dev}2 Cores=2-3
+Name=gpu Type=tesla File=${dev}3 Cores=2-3
+Name=gpu Type=tesla File=${dev}4 Cores=2-3
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-1|0,1|${dev}5
-tesla|4|0-1|0,1|${dev}6
+tesla|4|0-1|(null)|${dev}5
+tesla|4|0-1|(null)|${dev}6
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|2-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|2-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 4 $gres_conf $fake_gpus_conf $expected_output
 
@@ -295,24 +297,30 @@ check_configuration 4 $gres_conf $fake_gpus_conf $expected_output
 # # Test 5 - Different links, different records
 # ##############################################################################
 
+# Devices 0-2 are all doubly linked to each other
+# Device 5 is singly linked to 3-4
+# Devices 7-8 are doubly linked to each other
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0
-Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=1
-Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,2
-Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,3
-Name=gpu Type=tesla File=${dev}8 Cores=0-3 Links=0,3
+Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
+Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
+Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
+Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
+Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
+Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
+Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
+Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0|${dev}\[1-2\]|
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|1|${dev}\[3-4\]|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1|${dev}5|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,2|${dev}6|
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0,3|${dev}\[7-8\]|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|
 "
 check_configuration 5 $gres_conf $fake_gpus_conf $expected_output
 
@@ -331,11 +339,11 @@ check_configuration 100 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3 Links=0,1
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|2-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|2-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 101 $gres_conf $fake_gpus_conf $expected_output
 
@@ -475,15 +483,15 @@ check_configuration 108 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=a File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu Type=a File=${dev}2 Cores=0-3 Links=0,1
-Name=gpu Type=b File=${dev}5 Cores=0-3 Links=0,1
-Name=gpu Type=b File=${dev}6 Cores=0-3 Links=0,1
+Name=gpu Type=a File=${dev}1 Cores=0-3
+Name=gpu Type=a File=${dev}2 Cores=0-3
+Name=gpu Type=b File=${dev}5 Cores=0-3
+Name=gpu Type=b File=${dev}6 Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](2):a|4|0-3|0,1|${dev}\[1-2\]|
-GRES_PARSABLE\[gpu\](2):b|4|0-3|0,1|${dev}\[5-6\]|
+GRES_PARSABLE\[gpu\](2):a|4|0-3|(null)|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):b|4|0-3|(null)|${dev}\[5-6\]|
 "
 check_configuration 6 $gres_conf $fake_gpus_conf $expected_output
 
@@ -493,32 +501,32 @@ check_configuration 6 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-NodeName=$nodename1 Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1 Ignore=true
+NodeName=$nodename1 Name=gpu Type=tesla File=${dev}1 Cores=0-3 Ignore=true
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 7 $gres_conf $fake_gpus_conf $expected_output
 
 # ##############################################################################
-# # Test 8 - Ignore records for gres-specified nodes take no effect
+# # Test 8 - Ignore records for GRESs in gres.conf should take no effect
 # ##############################################################################
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
+Name=gpu Type=tesla File=${dev}2       Cores=0-3 Ignore=true
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 8 $gres_conf $fake_gpus_conf $expected_output
 
@@ -528,17 +536,17 @@ check_configuration 8 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 9 $gres_conf $fake_gpus_conf $expected_output
 
@@ -548,17 +556,17 @@ check_configuration 9 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla_a File=${dev}5 Cores=0-3 Links=0,1
-Name=gpu Type=tesla_b File=${dev}6 Cores=0-3 Links=0,1
-Name=gpu Type=tesla_a File=${dev}7 Cores=0-3 Links=0,1
-Name=gpu Type=tesla_b File=${dev}8 Cores=0-3 Links=0,1
+Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
+Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
+Name=gpu Type=tesla_a File=${dev}5       Cores=0-3
+Name=gpu Type=tesla_b File=${dev}6       Cores=0-3
+Name=gpu Type=tesla_a File=${dev}7       Cores=0-3
+Name=gpu Type=tesla_b File=${dev}8       Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla_a|4|0-3|0,1|${dev}\[1-2,5,7\]|
-GRES_PARSABLE\[gpu\](4):tesla_b|4|0-3|0,1|${dev}\[3-4,6,8\]|
+GRES_PARSABLE\[gpu\](4):tesla_a|4|0-3|(null)|${dev}\[1-2,5,7\]|
+GRES_PARSABLE\[gpu\](4):tesla_b|4|0-3|(null)|${dev}\[3-4,6,8\]|
 "
 check_configuration 10 $gres_conf $fake_gpus_conf $expected_output
 
@@ -567,22 +575,22 @@ check_configuration 10 $gres_conf $fake_gpus_conf $expected_output
 # ##############################################################################
 
 set gres_conf "
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}3 Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}5 Cores=0-3 Ignore=true
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
-tesla|4|0-3|0,1|${dev}5
-tesla|4|0-3|0,1|${dev}6
-tesla|4|0-3|0,1|${dev}7
-tesla|4|0-3|0,1|${dev}8
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
+tesla|4|0-3|(null)|${dev}5
+tesla|4|0-3|(null)|${dev}6
+tesla|4|0-3|(null)|${dev}7
+tesla|4|0-3|(null)|${dev}8
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](6):tesla|4|0-3|0,1|${dev}\[1-2,4,6-8\]|
+GRES_PARSABLE\[gpu\](6):tesla|4|0-3|(null)|${dev}\[1-2,4,6-8\]|
 "
 check_configuration 11 $gres_conf $fake_gpus_conf $expected_output
 
@@ -592,17 +600,17 @@ check_configuration 11 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-NodeName=$nodename Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1 Ignore=true
+NodeName=$nodename Name=gpu Type=tesla File=${dev}2 Cores=0-3 Ignore=true
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](3):tesla|4|0-3|0,1|${dev}\[1,3-4\]|
+GRES_PARSABLE\[gpu\](3):tesla|4|0-3|(null)|${dev}\[1,3-4\]|
 "
 check_configuration 12 $gres_conf $fake_gpus_conf $expected_output
 
@@ -612,21 +620,21 @@ check_configuration 12 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1 Ignore=true
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1 Ignore=true
+Name=gpu Type=tesla File=${dev}3       Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}3       Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}4       Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}4       Cores=0-3 Ignore=true
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Ignore=true
 "
 set fake_gpus_conf "
 # This file was autogenerated by test$test_id
-tesla|4|0-3|0,1|${dev}1
-tesla|4|0-3|0,1|${dev}2
-tesla|4|0-3|0,1|${dev}3
-tesla|4|0-3|0,1|${dev}4
+tesla|4|0-3|(null)|${dev}1
+tesla|4|0-3|(null)|${dev}2
+tesla|4|0-3|(null)|${dev}3
+tesla|4|0-3|(null)|${dev}4
 "
 set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0,1|${dev}\[1-2\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|0-3|(null)|${dev}\[1-2\]|
 "
 check_configuration 13 $gres_conf $fake_gpus_conf $expected_output
 
@@ -636,15 +644,15 @@ check_configuration 13 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}3       Cores=0-3
+Name=gpu Type=tesla File=${dev}3       Cores=0-3
+Name=gpu Type=tesla File=${dev}4       Cores=0-3
+Name=gpu Type=tesla File=${dev}4       Cores=0-3
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](2):tesla|4|0-3|0,1|${dev}\[3-4\]|
+GRES_PARSABLE\[gpu\](2):tesla|4|0-3|(null)|${dev}\[3-4\]|
 "
 check_configuration 14 $gres_conf $fake_gpus_conf $expected_output
 
@@ -654,14 +662,14 @@ check_configuration 14 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1       Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 15 $gres_conf $fake_gpus_conf $expected_output
 
@@ -671,14 +679,14 @@ check_configuration 15 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3 Links=0,1
-Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}4       Cores=0-3
+Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
+Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](4):tesla|4|0-3|0,1|${dev}\[4,3,2,1\]|
+GRES_PARSABLE\[gpu\](4):tesla|4|0-3|(null)|${dev}\[4,3,2,1\]|
 "
 check_configuration 16 $gres_conf $fake_gpus_conf $expected_output
 
@@ -688,17 +696,17 @@ check_configuration 16 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Cores=0-1 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Cores=0-2 Links=0,1
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0
+Name=gpu Type=tesla File=${dev}2 Cores=0-1
+Name=gpu Type=tesla File=${dev}3 Cores=0-2
+Name=gpu Type=tesla File=${dev}4 Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](1):tesla|4|0|0,1|${dev}1|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-1|0,1|${dev}2|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-2|0,1|${dev}3|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
 "
 check_configuration 17 $gres_conf $fake_gpus_conf $expected_output
 
@@ -708,14 +716,14 @@ check_configuration 17 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Cores=0 Links=0,1
-Name=gpu Type=tesla File=${dev}1 Cores=0-1 Links=0,1
-Name=gpu Type=tesla File=${dev}1 Cores=0-2 Links=0,1
-Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}1 Cores=0
+Name=gpu Type=tesla File=${dev}1 Cores=0-1
+Name=gpu Type=tesla File=${dev}1 Cores=0-2
+Name=gpu Type=tesla File=${dev}1 Cores=0-3
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](1):tesla|4|0|0,1|${dev}1|
+GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
 "
 check_configuration 18 $gres_conf $fake_gpus_conf $expected_output
 
@@ -744,15 +752,15 @@ check_configuration 19 $gres_conf $fake_gpus_conf $expected_output
 set gres_conf "
 # This file was autogenerated by test$test_id
 Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
-Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1
+Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
 Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
-Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1,2
+Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
 "
 set fake_gpus_conf ""
 set expected_output "
 GRES_PARSABLE\[gpu\](2):tesla|4|0-3|(null)|${dev}\[1,3\]|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1|${dev}2|
-GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,1,2|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|
+GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|
 "
 check_configuration 20 $gres_conf $fake_gpus_conf $expected_output
 
@@ -762,16 +770,16 @@ check_configuration 20 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-# Name=gpu File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu File=${dev}2 Cores=0-3 Links=0,1 Type=\"\"
-Name=gpu File=${dev}3 Cores=0-3 Links=0,1 Type=null
-Name=gpu File=${dev}4 Cores=0-3 Links=0,1 Type=0
+# Name=gpu File=${dev}1 Cores=0-3
+Name=gpu File=${dev}2 Cores=0-3 Type=\"\"
+Name=gpu File=${dev}3 Cores=0-3 Type=null
+Name=gpu File=${dev}4 Cores=0-3 Type=0
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](1):|4|0-3|0,1|${dev}2|
-GRES_PARSABLE\[gpu\](1):null|4|0-3|0,1|${dev}3|
-GRES_PARSABLE\[gpu\](1):0|4|0-3|0,1|${dev}4|
+GRES_PARSABLE\[gpu\](1):|4|0-3|(null)|${dev}2|
+GRES_PARSABLE\[gpu\](1):null|4|0-3|(null)|${dev}3|
+GRES_PARSABLE\[gpu\](1):0|4|0-3|(null)|${dev}4|
 "
 check_configuration 21 $gres_conf $fake_gpus_conf $expected_output
 
@@ -781,15 +789,15 @@ check_configuration 21 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu File=${dev}1 Cores=0-3 Links=0,1
-Name=gpu File=${dev}2 Cores=0-3 Links=0,1
-Name=gpu File=${dev}3 Cores=0-3 Links=0,1
-Name=gpu File=${dev}4 Cores=0-3 Links=0,1
+Name=gpu File=${dev}1 Cores=0-3
+Name=gpu File=${dev}2 Cores=0-3
+Name=gpu File=${dev}3 Cores=0-3
+Name=gpu File=${dev}4 Cores=0-3
 
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](4):(null)|4|0-3|0,1|${dev}\[1-4\]|
+GRES_PARSABLE\[gpu\](4):(null)|4|0-3|(null)|${dev}\[1-4\]|
 "
 check_configuration 22 $gres_conf $fake_gpus_conf $expected_output
 
@@ -799,17 +807,17 @@ check_configuration 22 $gres_conf $fake_gpus_conf $expected_output
 
 set gres_conf "
 # This file was autogenerated by test$test_id
-Name=gpu Type=tesla File=${dev}1 Links=0,1
-Name=gpu Type=tesla File=${dev}2 Links=0,1
-Name=gpu Type=tesla File=${dev}3 Links=0,1 Cores=\"\"
-Name=gpu Type=tesla File=${dev}4 Links=0,1 Cores=null
-Name=gpu Type=tesla File=${dev}5 Links=0,1 Cores=0
+Name=gpu Type=tesla File=${dev}1
+Name=gpu Type=tesla File=${dev}2
+Name=gpu Type=tesla File=${dev}3 Cores=\"\"
+Name=gpu Type=tesla File=${dev}4 Cores=null
+Name=gpu Type=tesla File=${dev}5 Cores=0
 "
 set fake_gpus_conf ""
 set expected_output "
-GRES_PARSABLE\[gpu\](3):tesla|4|(null)|0,1|${dev}\[1-3\]|
-GRES_PARSABLE\[gpu\](1):tesla|4|null|0,1|${dev}4|
-GRES_PARSABLE\[gpu\](1):tesla|4|0|0,1|${dev}5|
+GRES_PARSABLE\[gpu\](3):tesla|4|(null)|(null)|${dev}\[1-3\]|
+GRES_PARSABLE\[gpu\](1):tesla|4|null|(null)|${dev}4|
+GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|
 "
 check_configuration 23 $gres_conf $fake_gpus_conf $expected_output
 
@@ -1002,6 +1010,7 @@ file delete $cfgpath/fake_gpus.conf
 file delete $cfgpath/slurm.conf
 file delete $test_prog
 file delete $test_ulong_prog
+file delete $dev0
 file delete $dev1
 file delete $dev2
 file delete $dev3
diff --git a/testsuite/expect/test39.9 b/testsuite/expect/test39.9
index 9e899c570f2e60a64b79b5c885b670d74efca057..3a2f1f50d43340e3eabb3f12c92b303f7df0d4a4 100755
--- a/testsuite/expect/test39.9
+++ b/testsuite/expect/test39.9
@@ -37,7 +37,7 @@ set number_commas  "\[0-9_,\]+"
 set freq_parse "GpuFreq=memory_freq:($number),graphics_freq:($number)"
 set nvml_parse "Slurm is not configured with NVIDIA NVML support"
 set nvml_warning "\nWARNING: Slurm is not configured with NVML support. Could not test setting GPU frequencies\n"
-set permission_parse "Cannot configure GRES hardware unless privileged"
+set permission_parse "step_configure_hardware() invalid permissions"
 set permission_warning "\nWARNING: Slurmd is not root. Could not test setting GPU frequencies due to insufficient permissions\n"
 
 proc get_node_config {} {