Added the scancel option --nodelist to cancel any jobs running on a

    given list of nodes.

NEWS

@@ -9,6 +9,8 @@ documents those changes that are of interest to users and admins.
  -- Added environment variable SLURM_RESTART_COUNT to batch jobs to
     indicated the count of job restarts made.
  -- Added sacctmgr command "show config".
+ -- Added the scancel option --nodelist to cancel any jobs running on a
+    given list of nodes.
 
 * Changes in SLURM 1.4.0-pre7
 =============================
@@ -34,7 +36,7 @@ documents those changes that are of interest to users and admins.
     CPUs on a node when a partition's Shared value is YES or FORCE.
  -- Added configure options "--enable-cray-xt" and "--with-apbasil=PATH" for
     eventual support of Cray-XT systems.
- 
+
 * Changes in SLURM 1.4.0-pre6
 =============================
  -- Fix job preemption when sched/gang and select/linear are configured with

doc/man/man1/scancel.1

-.TH SCANCEL "1" "November 2008" "scancel 1.2" "Slurm components"
+.TH SCANCEL "1" "February 2009" "scancel 1.4" "Slurm components"
 
 .SH "NAME"
 scancel \- Used to signal jobs or job steps that are under the control of Slurm.
@@ -78,6 +78,13 @@ This option is incompatible with the \fB\-\-quiet\fR option.
 \fB\-V\fR, \fB\-\-Version\fR
 Print the version number of the scancel command. 
 
+.TP
+\fB\-w\fR, \fB\-\-nodelist=\fIhost1,host2,...\fR
+Cancel any jobs using any of the given hosts.  The list may be specified as 
+a comma\-separated list of hosts, a range of hosts (host[1\-5,7,...] for 
+example), or a filename. The host list will be assumed to be a filename only 
+if it contains a "/" character. 
+
 .TP
 ARGUMENTS
 

src/scancel/opt.c

 /*****************************************************************************\
  *  opt.c - options processing for scancel
  *****************************************************************************
- *  Copyright (C) 2002 The Regents of the University of California.
+ *  Copyright (C) 2002-2007 The Regents of the University of California.
+ *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Mark Grondona <grondona1@llnl.gov>, et. al.
  *  LLNL-CODE-402394.
@@ -71,6 +72,7 @@
 #define OPT_LONG_HELP  0x100
 #define OPT_LONG_USAGE 0x101
 #define OPT_LONG_CTLD  0x102
+#define OPT_LONG_WCKEY 0x103
 
 #define SIZE(a) (sizeof(a)/sizeof(a[0]))
 
@@ -230,6 +232,7 @@ static void _opt_default()
 	opt.user_id	= 0;
 	opt.verbose	= 0;
 	opt.wckey	= NULL;
+	opt.nodelist	= NULL;
 }
 
 /*
@@ -328,12 +331,14 @@ static void _opt_args(int argc, char **argv)
 		{"user",        required_argument, 0, 'u'},
 		{"verbose",     no_argument,       0, 'v'},
 		{"version",     no_argument,       0, 'V'},
+		{"nodelist",    required_argument, 0, 'w'},
+		{"wckey",       required_argument, 0, OPT_LONG_WCKEY},
 		{"help",        no_argument,       0, OPT_LONG_HELP},
 		{"usage",       no_argument,       0, OPT_LONG_USAGE},
 		{NULL,          0,                 0, 0}
 	};
 
-	while((opt_char = getopt_long(argc, argv, "bin:p:qs:t:u:vVw",
+	while((opt_char = getopt_long(argc, argv, "bin:p:qs:t:u:vVw:",
 			long_options, &option_index)) != -1) {
 		switch (opt_char) {
 			case (int)'?':
@@ -375,6 +380,9 @@ static void _opt_args(int argc, char **argv)
 				_print_version();
 				exit(0);
 			case (int)'w':
+				opt.nodelist = xstrdup(optarg);
+				break;
+			case OPT_LONG_WCKEY:
 				opt.wckey = xstrdup(optarg);
 				break;
 			case OPT_LONG_HELP:
@@ -463,7 +471,8 @@ _opt_verify(void)
 	    (opt.state == JOB_END) &&
 	    (opt.user_name == NULL) &&
 	    (opt.wckey == NULL) &&
-	    (opt.job_cnt == 0)) {
+	    (opt.job_cnt == 0) &&
+	    (opt.nodelist == NULL)) {
 		error("No job identification provided");
 		verified = false;	/* no job specification */
 	}
@@ -488,6 +497,7 @@ static void _opt_list(void)
 	info("user_name      : %s", opt.user_name);
 	info("verbose        : %d", opt.verbose);
 	info("wckey          : %s", opt.wckey);
+	info("nodelist       : %s", opt.nodelist);
 
 	for (i=0; i<opt.job_cnt; i++) {
 		info("job_steps      : %u.%u ", opt.job_id[i], opt.step_id[i]);
@@ -498,7 +508,7 @@ static void _usage(void)
 {
 	printf("Usage: scancel [-n job_name] [-u user] [-p partition] [-q] [-s name | integer]\n");
 	printf("               [--batch] [-t PENDING | RUNNING | SUSPENDED] [--usage] [-v] [-V]\n");
-	printf("               [job_id[.step_id]]\n");
+	printf("               [-w hosts...] [job_id[.step_id]]\n");
 }
 
 static void _help(void)
@@ -511,12 +521,13 @@ static void _help(void)
 	printf("  -p, --partition=partition       name of job's partition\n");
 	printf("  -q, --quiet                     disable warnings\n");
 	printf("  -s, --signal=name | integer     signal to send to job, default is SIGKILL\n");
-	printf("  -t, --state=state               state of the jobs to be signaled\n");
-	printf("                                  valid options are either pending,\n");
-	printf("                                  running, or suspended\n");
-	printf("  -u, --user=user                 name or id of user to have jobs signaled\n");
+	printf("  -t, --state=states              states of jobs to be signaled,\n");
+	printf("                                  default is pending, running, and\n");
+	printf("                                  suspended\n");
+	printf("  -u, --user=user                 name or id of user to have jobs cancelled\n");
 	printf("  -v, --verbose                   verbosity level\n");
 	printf("  -V, --version                   output version information and exit\n");
+	printf("  -w, --nodelist                  cancel jobs using any of these nodes\n");
 	printf("\nHelp options:\n");
 	printf("  --help                          show this help message\n");
 	printf("  --usage                         display brief usage message\n");

src/scancel/scancel.c

@@ -2,6 +2,7 @@
  *  scancel - cancel specified job(s) and/or job step(s)
  *****************************************************************************
  *  Copyright (C) 2002-2007 The Regents of the University of California.
+ *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Morris Jette <jette1@llnl.gov>
  *  LLNL-CODE-402394.
@@ -45,6 +46,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <signal.h>
+#include <pthread.h>
 
 #if HAVE_INTTYPES_H
 #  include <inttypes.h>
@@ -59,20 +61,34 @@
 #include "src/common/log.h"
 #include "src/common/xstring.h"
 #include "src/common/xmalloc.h"
+#include "src/common/hostlist.h"
 #include "src/scancel/scancel.h"
 
 #define MAX_CANCEL_RETRY 10
+#define MAX_THREADS 20
+
 
 static void _cancel_jobs (void);
-static void _cancel_job_id (uint32_t job_id, uint16_t sig);
-static void _cancel_step_id (uint32_t job_id, uint32_t step_id, 
-			     uint16_t sig);
+static void *_cancel_job_id (void *cancel_info);
+static void *_cancel_step_id (void *cancel_info);
+
 static int  _confirmation (int i, uint32_t step_id);
 static void _filter_job_records (void);
 static void _load_job_records (void);
+static void _verify_job_ids (void);
 
 static job_info_msg_t * job_buffer_ptr = NULL;
 
+typedef struct job_cancel_info {
+	uint32_t job_id;
+	uint32_t step_id;
+	uint16_t sig;
+	int             *num_active_threads;
+	pthread_mutex_t *num_active_threads_lock;
+	pthread_cond_t  *num_active_threads_cond;
+} job_cancel_info_t;
+
+
 int
 main (int argc, char *argv[]) 
 {
@@ -85,12 +101,15 @@ main (int argc, char *argv[])
 		log_alter (log_opts, SYSLOG_FACILITY_DAEMON, NULL);
 	} 
 	
+	_load_job_records();
+	_verify_job_ids();
+
 	if ((opt.interactive) ||
 	    (opt.job_name) ||
 	    (opt.partition) ||
 	    (opt.state != JOB_END) ||
-	    (opt.user_name)) {
-		_load_job_records ();
+	    (opt.user_name) ||
+	    (opt.nodelist)) {
 		_filter_job_records ();
 	}
 	_cancel_jobs ();
@@ -114,6 +133,33 @@ _load_job_records (void)
 }
 
 
+static void
+_verify_job_ids (void)
+{
+	/* If a list of jobs was given, make sure each job is actually in
+         * our list of job records. */
+	int i, j;
+	job_info_t *job_ptr = job_buffer_ptr->job_array;
+
+	for (j = 0; j < opt.job_cnt; j++ ) {
+		for (i = 0; i < job_buffer_ptr->record_count; i++) {
+			if (job_ptr[i].job_id == opt.job_id[j])
+				break;
+		}
+		if (i >= job_buffer_ptr->record_count) {
+			if (opt.step_id[j] == SLURM_BATCH_SCRIPT)
+				error("Kill job error on job id %u: %s", 
+				      opt.job_id[j], 
+				      slurm_strerror(ESLURM_INVALID_JOB_ID));
+			else
+				error("Kill job error on job step id %u.%u: %s",
+				      opt.job_id[j], opt.step_id[j],
+				      slurm_strerror(ESLURM_INVALID_JOB_ID));
+		}
+	}
+}
+
+
 /* _filter_job_records - filtering job information per user specification */
 static void 
 _filter_job_records (void)
@@ -163,6 +209,28 @@ _filter_job_records (void)
 			continue;
 		}
 
+		if (opt.nodelist != NULL) {
+			/* If nodelist contains a '/', treat it as a file name */
+			if (strchr(opt.nodelist, '/') != NULL) {
+				char *reallist;
+				reallist = slurm_read_hostfile(opt.nodelist,
+							       NO_VAL);
+				if (reallist) {
+					xfree(opt.nodelist);
+					opt.nodelist = reallist;
+				}
+			}
+
+			hostset_t hs = hostset_create(job_ptr[i].nodes);
+			if (!hostset_intersects(hs, opt.nodelist)) {
+				job_ptr[i].job_id = 0;
+				hostset_destroy(hs);
+				continue;
+			} else {
+				hostset_destroy(hs);
+			}
+		}
+
 		if (opt.job_cnt == 0)
 			continue;
 		for (j = 0; j < opt.job_cnt; j++) {
@@ -181,62 +249,137 @@ _filter_job_records (void)
 static void
 _cancel_jobs (void)
 {
-	int i, j;
+	int i, j, err;
 	job_info_t *job_ptr = NULL;
+	pthread_attr_t  attr;
+	job_cancel_info_t *cancel_info;
+	pthread_t  dummy;
+	int num_active_threads = 0;
+	pthread_mutex_t  num_active_threads_lock;
+	pthread_cond_t   num_active_threads_cond;
+
+	slurm_attr_init(&attr);
+	if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
+		error("pthread_attr_setdetachstate error %m");
+
+	slurm_mutex_init(&num_active_threads_lock);
+
+	if (pthread_cond_init(&num_active_threads_cond, NULL))
+		error("pthread_cond_init error %m");
+
+	job_ptr = job_buffer_ptr->job_array ;
+
+	/* Spawn a thread to cancel each job or job step marked for cancellation */
+	for (i = 0; i < job_buffer_ptr->record_count; i++) {
+		if (job_ptr[i].job_id == 0) 
+			continue;
 
-	if (opt.job_cnt && opt.interactive) {	/* confirm cancel */
-		job_ptr = job_buffer_ptr->job_array ;
-		for (j = 0; j < opt.job_cnt; j++ ) {
-			for (i = 0; i < job_buffer_ptr->record_count; i++) {
-				if (job_ptr[i].job_id != opt.job_id[j]) 
+		/* If cancelling a list of jobs, see if the current job 
+		 * included a step id */
+		if (opt.job_cnt) {
+			for (j = 0; j < opt.job_cnt; j++ ) {
+				if (job_ptr[i].job_id != opt.job_id[j])
 					continue;
+
 				if (opt.interactive && 
 				    (_confirmation(i, opt.step_id[j]) == 0))
+					continue;
+
+				cancel_info = 
+					(job_cancel_info_t *) 
+					xmalloc(sizeof(job_cancel_info_t));
+				cancel_info->job_id  = job_ptr[i].job_id;
+				cancel_info->sig     = opt.signal;
+				cancel_info->num_active_threads = 
+					&num_active_threads;
+				cancel_info->num_active_threads_lock = 
+					&num_active_threads_lock;
+				cancel_info->num_active_threads_cond = 
+					&num_active_threads_cond;
+
+				pthread_mutex_lock( &num_active_threads_lock );
+				num_active_threads++;
+				while (num_active_threads > MAX_THREADS) {
+					pthread_cond_wait(&num_active_threads_cond,
+							  &num_active_threads_lock);
+				}
+				pthread_mutex_unlock( &num_active_threads_lock );
+
+				if (opt.step_id[j] == SLURM_BATCH_SCRIPT) {
+					err = pthread_create(&dummy, &attr, 
+							     _cancel_job_id,
+							     cancel_info);
+					if (err)
+						_cancel_job_id(cancel_info);
 					break;
-				if (opt.step_id[j] == SLURM_BATCH_SCRIPT)
-					_cancel_job_id (opt.job_id[j], 
-							opt.signal);
-				else
-					_cancel_step_id (opt.job_id[j], 
-					                opt.step_id[j],
-							opt.signal);
-				break;
+				} else {
+					cancel_info->step_id = opt.step_id[j];
+					err = pthread_create(&dummy, &attr, 
+							     _cancel_step_id,
+							     cancel_info);
+					if (err)
+						_cancel_step_id(cancel_info);
+					/* Don't break here.  Keep looping in 
+					 * case other steps from the same job 
+					 * are cancelled. */
+				}
 			}
-			if (i >= job_buffer_ptr->record_count)
-				fprintf (stderr, "Job %u not found\n", 
-				         opt.job_id[j]);
-		}
-
-	} else if (opt.job_cnt) {	/* delete specific jobs */
-		for (j = 0; j < opt.job_cnt; j++ ) {
-			if (opt.step_id[j] == SLURM_BATCH_SCRIPT)
-				_cancel_job_id (opt.job_id[j], 
-						opt.signal);
-			else
-				_cancel_step_id (opt.job_id[j], 
-				                opt.step_id[j], 
-						opt.signal);
-		}
-
-	} else {		/* delete all jobs per filtering */
-		job_ptr = job_buffer_ptr->job_array ;
-		for (i = 0; i < job_buffer_ptr->record_count; i++) {
-			if (job_ptr[i].job_id == 0) 
-				continue;
+		} else {
 			if (opt.interactive && 
 			    (_confirmation(i, SLURM_BATCH_SCRIPT) == 0))
 				continue;
-			_cancel_job_id (job_ptr[i].job_id, opt.signal);
+
+			cancel_info = 
+				(job_cancel_info_t *)
+				xmalloc(sizeof(job_cancel_info_t));
+			cancel_info->job_id  = job_ptr[i].job_id;
+			cancel_info->sig     = opt.signal;
+			cancel_info->num_active_threads = &num_active_threads;
+			cancel_info->num_active_threads_lock = 
+				&num_active_threads_lock;
+			cancel_info->num_active_threads_cond = 
+				&num_active_threads_cond;
+
+			pthread_mutex_lock( &num_active_threads_lock );
+			num_active_threads++;
+			while (num_active_threads > MAX_THREADS) {
+				pthread_cond_wait( &num_active_threads_cond,
+						   &num_active_threads_lock );
+			}
+			pthread_mutex_unlock( &num_active_threads_lock );
+
+			err = pthread_create(&dummy, &attr, 
+					     _cancel_job_id,
+					     cancel_info);
+			if (err)
+				_cancel_job_id(cancel_info);
 		}
 	}
+
+	/* Wait for any spawned threads that have not finished */
+	pthread_mutex_lock( &num_active_threads_lock );
+	while (num_active_threads > 0) {
+		pthread_cond_wait( &num_active_threads_cond,
+				   &num_active_threads_lock );
+	}
+	pthread_mutex_unlock( &num_active_threads_lock );
+
+	slurm_attr_destroy(&attr);
+	slurm_mutex_destroy(&num_active_threads_lock);
+	if (pthread_cond_destroy(&num_active_threads_cond))
+		error("pthread_cond_destroy error %m");
 }
 
-static void
-_cancel_job_id (uint32_t job_id, uint16_t sig)
+static void *
+_cancel_job_id (void *ci)
 {
 	int error_code = SLURM_SUCCESS, i;
 	bool sig_set = true;
 
+	job_cancel_info_t *cancel_info = (job_cancel_info_t *)ci;
+	uint32_t job_id = cancel_info->job_id;
+	uint16_t sig    = cancel_info->sig;
+
 	if (sig == (uint16_t)-1) {
 		sig = SIGKILL;
 		sig_set = false;
@@ -274,12 +417,27 @@ _cancel_job_id (uint32_t job_id, uint16_t sig)
 			error("Kill job error on job id %u: %s", 
 				job_id, slurm_strerror(slurm_get_errno()));
 	}
+
+	/* Purposely free the struct passed in here, so the caller doesn't have
+	 * to keep track of it, but don't destroy the mutex and condition 
+	 * variables contained. */ 
+	pthread_mutex_lock(   cancel_info->num_active_threads_lock );
+	(*(cancel_info->num_active_threads))--;
+	pthread_cond_signal(  cancel_info->num_active_threads_cond );
+	pthread_mutex_unlock( cancel_info->num_active_threads_lock );
+
+	xfree(cancel_info);
+	return NULL;
 }
 
-static void
-_cancel_step_id (uint32_t job_id, uint32_t step_id, uint16_t sig)
+static void *
+_cancel_step_id (void *ci)
 {
 	int error_code = SLURM_SUCCESS, i;
+	job_cancel_info_t *cancel_info = (job_cancel_info_t *)ci;
+	uint32_t job_id  = cancel_info->job_id;
+	uint32_t step_id = cancel_info->step_id;
+	uint16_t sig     = cancel_info->sig;
 
 	if (sig == (uint16_t)-1)
 		sig = SIGKILL;
@@ -313,6 +471,17 @@ _cancel_step_id (uint32_t job_id, uint32_t step_id, uint16_t sig)
 		 		job_id, step_id, 
 				slurm_strerror(slurm_get_errno()));
 	}
+
+	/* Purposely free the struct passed in here, so the caller doesn't have
+	 * to keep track of it, but don't destroy the mutex and condition 
+	 * variables contained. */ 
+	pthread_mutex_lock(   cancel_info->num_active_threads_lock );
+	(*(cancel_info->num_active_threads))--;
+	pthread_cond_signal(  cancel_info->num_active_threads_cond );
+	pthread_mutex_unlock( cancel_info->num_active_threads_lock );
+
+	xfree(cancel_info);
+	return NULL;
 }
 
 /* _confirmation - Confirm job cancel request interactively */

src/scancel/scancel.h

@@ -57,7 +57,8 @@ typedef struct scancel_options {
 	uint16_t job_cnt;	/* count of job_id's specified	*/
 	uint32_t *job_id;	/* list of job_id's		*/
 	uint32_t *step_id;	/* list of job step id's	*/
-	char *wckey;		/* --name=n, -nn		*/
+	char *wckey;		/* --wckey			*/
+	char *nodelist;		/* --nodelist, -w		*/
 } opt_t;
 
 opt_t opt;

testsuite/expect/Makefile.am

@@ -148,6 +148,7 @@ EXTRA_DIST = \
 	test6.12			\
 	test6.13			\
 	test6.13.prog.c			\
+	test6.14			\
 	test7.1				\
 	test7.2				\
 	test7.2.prog.c			\

testsuite/expect/Makefile.in

@@ -387,6 +387,7 @@ EXTRA_DIST = \
 	test6.12			\
 	test6.13			\
 	test6.13.prog.c			\
+	test6.14			\
 	test7.1				\
 	test7.2				\
 	test7.2.prog.c			\

testsuite/expect/README

@@ -268,6 +268,7 @@ test6.11   Validate scancel quiet option, no warning if job gone
 test6.12   Test scancel signal to batch script (--batch option)
 test6.13   Test routing all signals through slurmctld rather than directly 
            to slurmd (undocumented --ctld option).
+test6.14   Test scancel nodelist option (--nodelist or -w)
 
 test7.#    Testing of other functionality.
 ==========================================

testsuite/expect/test6.14

+#!/usr/bin/expect
+############################################################################
+# Purpose: Test of SLURM functionality
+#          Test scancel --nodelist option.
+#
+# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
+#          "FAILURE: ..." otherwise with an explanation of the failure, OR
+#          anything else indicates a failure mode that must be investigated.
+############################################################################
+# Copyright (C) 2008 Lawrence Livermore National Security
+# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+# Written by David Bremer <dbremer@llnl.gov>
+# LLNL-CODE-402394.
+# 
+# This file is part of SLURM, a resource management program.
+# For details, see <http://www.llnl.gov/linux/slurm/>.
+#  
+# SLURM is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+# 
+# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with SLURM; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
+############################################################################
+source ./globals
+
+set test_id        "6.14"
+set exit_code      0
+set file_in        "test$test_id.input"
+set num_procs      10
+set ii             0
+set job_id         ""
+set job_map        {}
+set found          0
+set tmp_job_list   {}
+set tmp_map_entry  {}
+set submitted_jobs {}
+set job_list       {}
+set job_index      -1
+
+print_header $test_id
+
+#
+# Build input script file
+#
+make_bash_script $file_in "$srun $bin_sleep 600"
+
+
+
+
+#
+# Submit some jobs so we have something to work with
+#
+for {set ii 0} {$ii < $num_procs} {incr ii} {
+	set sbatch_pid [spawn $sbatch --output=/dev/null --error=/dev/null -n1 -N1 $file_in]
+	expect {
+		-re "Submitted batch job ($number)" {
+			set job_id $expect_out(1,string)
+			lappend submitted_jobs $job_id
+			exp_continue
+		}
+		timeout {
+			send_user "\nFAILURE: srun not responding\n"
+			slow_kill $sbatch_pid
+			set exit_code 1
+		}
+		eof {
+			wait
+		}
+	}
+	if {$job_id == 0} {
+		send_user "\nFAILURE: job submit failure\n"
+		exit 1
+	}
+}
+
+#
+# Run squeue and build a map, implemented as a list of list of lists, like so:
+# { {node1  {job1 job2 job3}}
+#   {node2  {job4 job5}}
+# }
+#
+# Only put jobs into the map if they were submitted by this test.
+#
+
+spawn $squeue -h -t running -u $env(USER) -o "%10i %40N"
+expect {
+	-re "($number) *($alpha_numeric) *\r\n" {
+		set job_id $expect_out(1,string)
+		set node_name $expect_out(2,string)
+		
+		#This test doesn't need to track jobs that it didn't submit.
+		if { [lsearch $submitted_jobs $job_id] == -1 } {
+			exp_continue
+		}
+		#send_user "job $job_id: node $node_name\n"
+		#Insert into a table with node_name as the key, job_id as the value
+		set found 0
+		for {set ii 0} {$ii < [llength $job_map]} {incr ii} {
+			if { [lindex [lindex $job_map $ii] 0] == $node_name } {
+				set tmp_map_entry [list $node_name [concat [lindex [lindex $job_map $ii] 1] $job_id]]
+				set job_map [lreplace $job_map $ii $ii $tmp_map_entry]
+				set found 1
+				break
+			}
+		}
+		if {$found == 0} {
+			lappend job_map [list $node_name [list $job_id] ]
+		}
+		exp_continue
+	}
+}
+#send_user "job map: $job_map\n"
+
+
+#
+# Issue an scancel command against each node in the map described above.
+# Remove entries from the internal list, and ensure that the list is
+# empty at the end of the scancel call.
+#
+
+for {set ii 0} {$ii < [llength $job_map]} {incr ii} {
+	set node_name [lindex [lindex $job_map $ii] 0]
+	set job_list  [lindex [lindex $job_map $ii] 1]
+
+	if {$ii == 0} {
+		spawn $scancel -v -u $env(USER) --nodelist $node_name
+	} else {
+		spawn $scancel -v -u $env(USER) -w $node_name
+	}
+	expect {
+		-re "scancel: Terminating job ($number)" {
+			#Search for the terminated job in the list recently
+			#returned from squeue. Don't worry if an unknown job
+			#gets cancelled, because maybe one of our submitted
+			#jobs will start running while we cancel other jobs
+			#Issue cancel commands node by node until all the 
+			#jobs submitted for this test are gone.
+
+			set job_id $expect_out(1,string)
+			set job_index [lsearch $job_list $job_id]
+			if {$job_index != -1} {
+				set job_list [lreplace $job_list $job_index $job_index]
+			} 
+			set job_index [lsearch $submitted_jobs $job_id]
+			if {$job_index != -1} {
+				set submitted_jobs [lreplace $submitted_jobs $job_index $job_index]
+			}
+			exp_continue
+		}
+		timeout {
+			send_user "\nFAILURE: scancel not responding while cancelling for node $node_name\n"
+			set exit_code 1
+		}
+		eof {
+			wait
+		}
+	}
+	if { [llength $job_list] != 0 } {
+		send_user "\nFAILURE: scancel did not remove jobs $job_list from node $node_name\n"
+		set exit_code 1
+	}
+}
+
+#
+# Clean up any jobs submitted by this test, which were not mapped to a node,
+# and thus not cancelled in the previous block of code
+#
+
+foreach job_id $submitted_jobs {
+	spawn $scancel $job_id
+	expect {
+		timeout {
+			send_user "\nFAILURE: scancel not responding while cancelling job $job_id\n"
+			set exit_code 1
+		}
+		eof {
+			wait
+		}
+	}
+}
+
+
+
+if {$exit_code == 0} {
+	send_user "\nSUCCESS\n"
+}
+exit $exit_code
+