Add check in tests for cancel_job failures

In the course of testing some scancel changes, a bunch of tests generated
"FAILURE" messages due to job cancellation failures, but the tests reported
"SUCCESS" at the end and an exit code of zero. This patch adds a checks
for the return value of the "cancel_job" procedure.

testsuite/expect/test1.58

@@ -98,7 +98,10 @@ if {$got_pattern == 0} {
 #
 # Release the allocation by killing salloc (really it kills the "sleep")
 #
-cancel_job $jobid
+if {[cancel_job $jobid] != 0} {
+	set exit_code 1
+}
+
 set spawn_id $srun_alloc_sid
 exec kill -9 $srun_alloc_pid
 expect {
@@ -110,7 +113,7 @@ expect {
 	}
 }
 
-if {$got_pattern == 1} {
+if {$exit_code == 0} {
 	send_user "\nSUCCESS\n"
 }
 exit $exit_code

testsuite/expect/test15.7

@@ -126,12 +126,13 @@ expect {
 		wait
 	}
 }
-cancel_job $job_id
-
 if {$matches != 5} {
 	send_user "\nFAILURE: Did not get constraints ($matches of 5)\n"
 	set exit_code 1
 }
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 if {$exit_code == 0} {
 	send_user "\nSUCCESS\n"
 }

testsuite/expect/test17.10

@@ -129,12 +129,13 @@ expect {
 		wait
 	}
 }
-cancel_job $job_id
-
 if {$matches != 5} {
 	send_user "\nFAILURE: Did not get proper constraints\n"
 	set exit_code 1
 }
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 if {$exit_code == 0} {
 	exec $bin_rm -f $file_in
 	send_user "\nSUCCESS\n"

testsuite/expect/test17.11

@@ -103,13 +103,15 @@ if {$job_id != 0} {
 		}
 		timeout {
 			send_user "\nFAILURE: scontrol not responding\n"
-			set exit_code   1
+			set exit_code 1
 		}
 		eof {
 			wait
 		}
 	}
-	cancel_job $job_id
+	if {[cancel_job $job_id] != 0} {
+		set exit_code 1
+	}
 	set job_id 0
 } else {
 	set exit_code   1
@@ -169,7 +171,9 @@ if {$job_id != 0} {
 			wait
 		}
 	}
-	cancel_job $job_id
+	if {[cancel_job $job_id] != 0} {
+		set exit_code 1
+	}
 } else {
 	set exit_code 1
 }

testsuite/expect/test17.19

@@ -103,11 +103,13 @@ expect {
 		wait
 	}
 }
-cancel_job $job_id
 if {$matches != 1} {
 	send_user "\nFAILURE: sbatch failed to set partition properly\n"
 	exit 1
 }
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 if {$exit_code != 0} {
 	exit $exit_code
 }
@@ -140,7 +142,9 @@ expect {
 }
 if {$job_id != 0} {
 	send_user "\nFAILURE: sbatch submitted job to invalid partition name\n"
-	cancel_job $job_id
+	if {[cancel_job $job_id] != 0} {
+		set exit_code 1
+	}
 	set exit_code 1
 }
 if {$matches != 1} {
@@ -203,11 +207,13 @@ expect {
 		wait
 	}
 }
-cancel_job $job_id
 if {$matches != 1} {
 	send_user "\nFAILURE: sbatch failed to set partition properly\n"
 	exit 1
 }
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 
 if {$exit_code == 0} {
 	exec $bin_rm -f $file_in

testsuite/expect/test17.37

@@ -136,8 +136,9 @@ if { $match != 2 } {
 	send_user "should have DependencyNeverSatisfied for a reason\n"
 	set exit_code 1
 }
-
-cancel_job $job_id2
+if {[cancel_job $job_id2] != 0} {
+	set exit_code 1
+}
 
 if {$exit_code == 0} {
 	exec $bin_rm -f $script

testsuite/expect/test20.6

@@ -119,7 +119,9 @@ expect {
 # 1 for "y" and 0 for "n"
 check_rerun 0 $job_id
 
-cancel_job $job_id
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 
 if {$exit_code == 0} {
 	exec $bin_rm -f $file_in

testsuite/expect/test20.7

@@ -115,7 +115,9 @@ expect {
 
 check_output $out_path $job_id
 
-cancel_job $job_id
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 
 if {$exit_code == 0} {
 	exec $bin_rm -f $file_in

testsuite/expect/test20.8

@@ -108,7 +108,9 @@ expect {
 }
 
 check_name $job_id $job_name
-cancel_job $job_id
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 
 if {$exit_code == 0} {
 	exec $bin_rm -f $file_in

testsuite/expect/test28.3

@@ -94,7 +94,9 @@ expect {
 }
 if {$job_id != 0} {
 	check_ids $job_id
-	cancel_job $job_id
+	if {[cancel_job $job_id] != 0} {
+		set exit_code 1
+	}
 }
 
 if {$exit_code == 0} {

testsuite/expect/test28.4

@@ -136,7 +136,9 @@ set new_job_id [check_update ${job_id}_2 4]
 # Make sure other tasks remain unchanged
 check_update ${job_id}_1 1
 
-cancel_job $job_id
+if {[cancel_job $job_id] != 0} {
+	set exit_code 1
+}
 
 if {$exit_code == 0} {
 	exec $bin_rm -f $file_script

testsuite/expect/test28.5

@@ -184,7 +184,9 @@ if {$job_id != 0} {
 	release_job ${job_id}_$array_in
 	check_release_job ${job_id}_$array_in
 
-	cancel_job $job_id
+	if {[cancel_job $job_id] != 0} {
+		set exit_code 1
+	}
 }
 
 if {$exit_code == 0} {

testsuite/expect/test5.10

@@ -195,7 +195,7 @@ expect {
 
 if {[string length $nodes] == 0} {
 	send_user "\nFAILURE: did not get a valid node list\n"
-	exit 0
+	exit 1
 }
 
 # Create 2 test partitions

testsuite/expect/test5.4

@@ -240,8 +240,12 @@ if {[test_alps] == 0 && $step_found == 0} {
 	set exit_code 1
 }
 
-cancel_job $job_id1
-cancel_job $job_id2
+if {[cancel_job $job_id1] != 0} {
+	set exit_code 1
+}
+if {[cancel_job $job_id2] != 0} {
+	set exit_code 1
+}
 if {$exit_code == 0} {
 	send_user "\nSUCCESS\n"
 }

testsuite/expect/test5.5

@@ -119,8 +119,12 @@ expect {
 	}
 }
 
-cancel_job $job_id2
-cancel_job $job_id1
+if {[cancel_job $job_id1] != 0} {
+	set exit_code 1
+}
+if {[cancel_job $job_id2] != 0} {
+	set exit_code 1
+}
 if {$exit_code == 0} {
 	send_user "\nSUCCESS\n"
 }

testsuite/expect/test5.6

@@ -420,8 +420,12 @@ if {$job_found == 0} {
 #
 # Cancel jobs and remove files
 #
-cancel_job $job_id2
-cancel_job $job_id1
+if {[cancel_job $job_id1] != 0} {
+	set exit_code 1
+}
+if {[cancel_job $job_id2] != 0} {
+	set exit_code 1
+}
 if {$exit_code == 0} {
 	send_user "\nSUCCESS\n"
 	exec $bin_rm -f $file_in

testsuite/expect/test7.1

@@ -127,7 +127,9 @@ if {$job_id1 != 0} {
 			wait
 		}
 	}
-	cancel_job $job_id1
+        if {[cancel_job $job_id1] != 0} {
+		set exit_code   1
+	}
 } else {
 	set exit_code   1
 }
@@ -147,7 +149,9 @@ if {$job_id2 != 0} {
 			wait
                 }
         }
-        cancel_job $job_id2
+        if {[cancel_job $job_id2] != 0} {
+		set exit_code   1
+	}
 } else {
         set exit_code   1
 }
@@ -167,7 +171,9 @@ if {$job_id3 != 0} {
 			wait
                 }
         }
-        cancel_job $job_id3
+        if {[cancel_job $job_id3] != 0} {
+		set exit_code   1
+	}
 } else {
         set exit_code   1
 }