Fix several tests to behave better if slurmctld is down

testsuite/expect/globals

@@ -721,9 +721,20 @@ proc wait_for_all_jobs { job_name incr_sleep } {
 				}
 			}
 		}
+		if {$matches == -1} {
+			break
+		}
 	}
 	if {$matches != 0} {
-		exec $scancel -n $job_name
+		spawn $scancel -n $job_name
+		expect {
+			timeout {
+				send_user "No response from scancel\n"
+			}
+			eof {
+				wait
+			}
+		}
 	}
 	return $matches
 }

testsuite/expect/test1.53

@@ -54,7 +54,8 @@ exec $bin_cc -o $file_prog $file_prog.c
 # submit job to run 4 mins, but get signaled 2 mins before time limit
 # so it should execute between 1 and 2 minutes before signal
 #
-set run_time 0
+set began_flag 0
+set run_time   0
 set timeout [expr $max_job_delay + 120]
 set srun_pid [spawn $srun -t4 --signal=2@120 ./$file_prog]
 expect {

testsuite/expect/test15.8

@@ -42,6 +42,7 @@ print_header $test_id
 # Spawn a srun immediate execution job with hold (priority==0) option,
 # The job can't run immediately with a priority of zero
 #
+set job_id  0
 set timeout 10
 set salloc_pid [spawn $salloc --immediate=3 --hold -t1 -v $bin_pwd]
 expect {

testsuite/expect/test17.32

@@ -125,7 +125,7 @@ if {$matches != 2} {
 }
 if {$tasks != $task_cnt} {
 	send_user "\nFAILURE: Did not get proper number of tasks: "
-	send_user "($task != $task_cnt)\n"
+	send_user "($tasks != $task_cnt)\n"
 	set exit_code 1
 }
 

testsuite/expect/test27.4

@@ -67,6 +67,8 @@ proc sdiag_all { } {
 }
 
 sdiag_all
+set jobs_started   0
+set jobs_submitted 0
 set submit_begin $jobs_submitted
 set start_begin  $jobs_started
 

testsuite/expect/test27.5

@@ -68,7 +68,9 @@ proc sdiag_all { } {
 
 sdiag_all
 
-set perm_denied 0
+set jobs_started   0
+set jobs_submitted 0
+set perm_denied    0
 spawn $sdiag --reset
 expect {
 	-re "permission denied" {