diff --git a/testsuite/expect/test1.1 b/testsuite/expect/test1.1 index c4dc9727633a5ec451f9bb4fb61b7daab4728709..b71a9ae45feea97c7f8cb3583bfddb6c3e9e40de 100755 --- a/testsuite/expect/test1.1 +++ b/testsuite/expect/test1.1 @@ -81,7 +81,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.10 b/testsuite/expect/test1.10 index 2eb02babb4e23f9e4affad97cd374b724f42bce6..1e7cfb4c47b8b601378d78cbc79bef815564eaba 100755 --- a/testsuite/expect/test1.10 +++ b/testsuite/expect/test1.10 @@ -54,7 +54,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } } diff --git a/testsuite/expect/test1.11 b/testsuite/expect/test1.11 index 5449706987d7a2d15549684312f32dc57192de4d..5bc44158016c288de949a07a6d87283d8d504a35 100755 --- a/testsuite/expect/test1.11 +++ b/testsuite/expect/test1.11 @@ -78,7 +78,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -189,7 +188,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.12 b/testsuite/expect/test1.12 index efe66e7bab10af285a2effb4958f70bbd5b577e0..200c1827209bdb4f3a23ef1df72c5a80e82614e0 100755 --- a/testsuite/expect/test1.12 +++ b/testsuite/expect/test1.12 @@ -68,7 +68,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.13 b/testsuite/expect/test1.13 index 9bf489e21a7720558734abd2d33f60cd02cd6549..ee38994e831e8b4968bd557237e5c8e399fda51e 100755 --- a/testsuite/expect/test1.13 +++ b/testsuite/expect/test1.13 @@ -53,7 +53,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -85,7 +84,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.14 b/testsuite/expect/test1.14 index 56f6a86a6efdab03d544b99ce83d841df1ff25b2..fa81dbdeaee5e02e10f6df17bb7ae93ccee263c1 100755 --- a/testsuite/expect/test1.14 +++ b/testsuite/expect/test1.14 @@ -52,7 +52,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.15 b/testsuite/expect/test1.15 index c49e9292663dcca851c54bab3bc51004d8003423..6257c34b691b60034686e54e773a0ff0bba36ba1 100755 --- a/testsuite/expect/test1.15 +++ b/testsuite/expect/test1.15 @@ -69,7 +69,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.16 b/testsuite/expect/test1.16 index fb5f06b4a45d49c521361e48890a47da1032b58b..e706b9d89d9bb10e1a5a615a7ecafd97afdd49b1 100755 --- a/testsuite/expect/test1.16 +++ b/testsuite/expect/test1.16 @@ -49,7 +49,7 @@ exec $bin_touch $scratch_file # The --unbuffered option will send the message which lacks a '\n' # set timeout $max_job_delay -spawn $srun --unbuffered --verbose -t1 $bin_rm -i $scratch_file +set srun_pid [spawn $srun --unbuffered --verbose -t1 $bin_rm -i $scratch_file] expect { -re "launching ($number).0 on host" { set job_id $expect_out(1,string) @@ -62,9 +62,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.17 b/testsuite/expect/test1.17 index ab8d10783a086353951465de56d7abf29e7960ae..b5689a0bdebc4df9a9bcd9b4c63a93b5552d5302 100755 --- a/testsuite/expect/test1.17 +++ b/testsuite/expect/test1.17 @@ -44,7 +44,7 @@ print_header $test_id # set timeout $max_job_delay set match 0 -spawn $srun --allocate --verbose -t1 +set srun_pid [spawn $srun --allocate --verbose -t1] expect { -re "jobid ($number).*" { set job_id $expect_out(1,string) @@ -68,9 +68,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.18 b/testsuite/expect/test1.18 index ce22cf2a27edfa533802e2c4472c27dee5f869ea..bb769e4b4c9a4472f6fb45ec888f8b398d246c49 100755 --- a/testsuite/expect/test1.18 +++ b/testsuite/expect/test1.18 @@ -53,7 +53,7 @@ exec $bin_chmod 700 $file_prog # Spawn initial program via srun # set timeout $max_job_delay -spawn $srun -N1-4 -v -t5 $file_prog +set srun_pid [spawn $srun -N1-4 -v -t5 -l $file_prog] set init_id $spawn_id expect { -i $init_id @@ -66,6 +66,7 @@ expect { } timeout { send_user "\nFAILURE: srun (launch) not responding\n" + slow_kill $srun_pid set exit_code 1 } eof { @@ -86,7 +87,7 @@ if {$matches == 0} { # set matches 0 set timeout 10 -spawn $srun -vv --attach=$job_id --join +set attach_pid [spawn $srun -vv -l --attach=$job_id --join] set attach_id $spawn_id expect { -i $attach_id @@ -96,9 +97,8 @@ expect { } timeout { send_user "\nFAILURE: srun (attach) not responding\n" - kill_srun + slow_kill $attach_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.19 b/testsuite/expect/test1.19 index cb74d07fc164ff532057bfce9dc6acc33d0f25dc..7e396042987e073995da0deec98be5a3afb4204f 100755 --- a/testsuite/expect/test1.19 +++ b/testsuite/expect/test1.19 @@ -54,7 +54,7 @@ for {set task_id 0} {$task_id < $task_cnt} {incr task_id} { exec $bin_rm -f $file_out_t_glob } set timeout $max_job_delay -spawn $srun --output=$file_out_t -N1 -n$task_cnt -O -v -t1 $bin_id +set srun_pid [spawn $srun --output=$file_out_t -N1 -n$task_cnt -O -v -t1 $bin_id] expect { -re "jobid ($number).*" { set job_id $expect_out(1,string) @@ -62,7 +62,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { @@ -94,7 +94,7 @@ if {$file_cnt != $task_cnt} { # set job_id 0 set srun_exit 0 -spawn $srun --error=$file_err_j --output=/dev/null -N1 -n$task_cnt -O -v -t1 $bin_sleep aaa +set srun_pid [spawn $srun --error=$file_err_j --output=/dev/null -N1 -n$task_cnt -O -v -t1 $bin_sleep aaa] expect { -re "jobid ($number).*" { set job_id $expect_out(1,string) @@ -107,7 +107,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { @@ -136,7 +136,7 @@ if {[wait_for_file $file_err_j_glob] == 0} { # file name and confirm it is created # set job_id 0 -spawn $srun --output=$file_out_J -N1 -v -t1 $bin_hostname +set srun_pid [spawn $srun --output=$file_out_J -N1 -v -t1 $bin_hostname] expect { -re "jobid ($number).*" { set job_id $expect_out(1,string) @@ -144,7 +144,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { @@ -173,7 +173,7 @@ set file_out_n_glob "test$test_id.n.$node_id.output" exec $bin_rm -f $file_out_n_glob set job_id 0 -spawn $srun --output=$file_out_n -N1 -n2 -O -v -t1 $bin_hostname +set srun_pid [spawn $srun --output=$file_out_n -N1 -n2 -O -v -t1 $bin_hostname] expect { -re "jobid ($number).*" { set job_id $expect_out(1,string) @@ -181,7 +181,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { @@ -241,7 +241,7 @@ if { [test_bluegene] } { } set job_id 0 -spawn $srun --batch --output=/dev/null -N$node_cnt -n$task_cnt -O -t1 $file_in +set srun_pid [spawn $srun --batch --output=/dev/null -N$node_cnt -n$task_cnt -O -t1 $file_in] expect { -re "jobid ($number) submitted" { set job_id $expect_out(1,string) @@ -249,7 +249,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { diff --git a/testsuite/expect/test1.2 b/testsuite/expect/test1.2 index d6b90a0a487a34bb0d8efbdf87db348917b550dd..2bde95c63497b13fdbc8ac6daec2a93ec6456c53 100755 --- a/testsuite/expect/test1.2 +++ b/testsuite/expect/test1.2 @@ -53,7 +53,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.20 b/testsuite/expect/test1.20 index 661875ce48b4671323f63bc717ae9a16492551c4..f04116e20a43fa6ad43388e092a0074b9afbe3bc 100755 --- a/testsuite/expect/test1.20 +++ b/testsuite/expect/test1.20 @@ -41,7 +41,7 @@ print_header $test_id # Spawn a program via srun with stdout forwarding disabled # set timeout $max_job_delay -spawn $srun --output=none -t1 $bin_id +set srun_pid [spawn $srun --output=none -t1 $bin_id] expect { -re "uid=" { send_user "\nFAILURE: srun improperly forwarded stdout\n" @@ -55,9 +55,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -68,7 +67,7 @@ expect { # Spawn a program via srun with stderr forwarding disabled # set matches 0 -spawn $srun --error=none -t1 $bin_sleep aaa +set srun_pid [spawn $srun --error=none -t1 $bin_sleep aaa] expect { -re "invalid" { send_user "\nFAILURE: srun improperly forwarded stderr\n" @@ -82,9 +81,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.21 b/testsuite/expect/test1.21 index 794b7268dda82e30603a102fb37cb10d338401e5..401d2f0719c298b3b407b79c537a6933e83748db 100755 --- a/testsuite/expect/test1.21 +++ b/testsuite/expect/test1.21 @@ -44,7 +44,7 @@ print_header $test_id # Spawn a shell via srun with stdout forwarding disabled # set timeout $max_job_delay -spawn $srun --input=$task_id -N1 -n10 -O -v --wait=2 -t1 $bin_bash +set srun_pid [spawn $srun --input=$task_id -N1 -n10 -O -v --wait=2 -t1 $bin_bash] expect { -re "launching ($number).0 on host" { set job_id $expect_out(1,string) @@ -68,9 +68,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.22 b/testsuite/expect/test1.22 index 9ab76e5903a944395aa84f2be2888953213f755e..2cf22b19a2d2030d77c11e762f55c7ec82e11040 100755 --- a/testsuite/expect/test1.22 +++ b/testsuite/expect/test1.22 @@ -53,7 +53,7 @@ if { [test_bluegene] } { } } -spawn $srun -N$node_cnt -l --threads=1 -t1 $bin_hostname +set srun_pid [spawn $srun -N$node_cnt -l --threads=1 -t1 $bin_hostname] expect { -re "0: ($alpha_numeric)" { set host_0 $expect_out(1,string) @@ -61,9 +61,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -83,7 +82,7 @@ if {[string compare $host_0 ""] == 0} { # set host_0 "" set timeout $max_job_delay -spawn $srun -N$node_cnt -n32 -O -l --threads=32 -t1 $bin_hostname +set srun_pid [spawn $srun -N$node_cnt -n32 -O -l --threads=32 -t1 $bin_hostname] expect { -re "0: ($alpha_numeric)" { set host_0 $expect_out(1,string) @@ -91,9 +90,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.23 b/testsuite/expect/test1.23 index ea1f5588a5e29ad55c267089a98e336fce8c32e1..0eefd47374878ae8d2e2885530bd5745976109ac 100755 --- a/testsuite/expect/test1.23 +++ b/testsuite/expect/test1.23 @@ -44,7 +44,7 @@ print_header $test_id set err_msg 0 set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --mincpus=999999 -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --mincpus=999999 -t1 $bin_hostname] expect { -re "configuration is not available" { send_user "This error is expected, no worries\n" @@ -57,9 +57,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -80,7 +79,7 @@ if {$err_msg != 1} { # set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --mincpus=1 -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --mincpus=1 -t1 $bin_hostname] expect { -re "0: ($alpha_numeric)" { set host_0 $expect_out(1,string) @@ -88,9 +87,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -108,7 +106,7 @@ if {[string compare $host_0 ""] == 0} { set err_msg 0 set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --mem=999999 -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --mem=999999 -t1 $bin_hostname] expect { -re "configuration is not available" { send_user "This error is expected, no worries\n" @@ -121,9 +119,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -144,7 +141,7 @@ if {$err_msg != 1} { # set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --mem=1 -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --mem=1 -t1 $bin_hostname] expect { -re "0: ($alpha_numeric)" { set host_0 $expect_out(1,string) @@ -152,9 +149,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -172,7 +168,7 @@ if {[string compare $host_0 ""] == 0} { set err_msg 0 set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --tmp=999999999 -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --tmp=999999999 -t1 $bin_hostname] expect { -re "configuration is not available" { send_user "This error is expected, no worries\n" @@ -185,9 +181,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -208,7 +203,7 @@ if {$err_msg != 1} { # set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --tmp=1 -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --tmp=1 -t1 $bin_hostname] expect { -re "0: ($alpha_numeric)" { set host_0 $expect_out(1,string) @@ -216,9 +211,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.24 b/testsuite/expect/test1.24 index 03a3783546c858282758d6c2b7fd78d509ba36f5..829cf84e80e13b2f7aa087c1ac6c1b35a8209ec3 100755 --- a/testsuite/expect/test1.24 +++ b/testsuite/expect/test1.24 @@ -43,7 +43,7 @@ print_header $test_id set err_msg 0 set host_0 "" set timeout $max_job_delay -spawn $srun -N1 -l --constraint=invalid,constraint -t1 $bin_hostname +set srun_pid [spawn $srun -N1 -l --constraint=invalid,constraint -t1 $bin_hostname] expect { -re "configuration is not available" { send_user "This error is expected, no worries\n" @@ -56,9 +56,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.25 b/testsuite/expect/test1.25 index 7c6d06f589205feac0f8ed5396ee31ec1daa4d89..c590dbe530804055869b79183c76c3adfd4b0cd4 100755 --- a/testsuite/expect/test1.25 +++ b/testsuite/expect/test1.25 @@ -54,7 +54,7 @@ if { [test_bluegene] } { } } -spawn $srun -N$node_cnt --no-kill -t1 $bin_sleep $sleep_time +set srun_pid [spawn $srun -N$node_cnt --no-kill -t1 $bin_sleep $sleep_time] expect { -re "error" { send_user "\nFAILURE: some error occurred\n" @@ -63,9 +63,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.26 b/testsuite/expect/test1.26 index bf90382ddfc2f985fb2eeafdd8ec1cc12dc0bf85..9e3d5c53db252932100884088982342e6eb123b2 100755 --- a/testsuite/expect/test1.26 +++ b/testsuite/expect/test1.26 @@ -54,7 +54,7 @@ if {[is_super_user] == 0} { set host_0 "" set nodelist_name "" set timeout $max_job_delay -spawn $srun -v -N1 -l $bin_printenv SLURMD_NODENAME +set srun_pid [spawn $srun -v -N1 -l $bin_printenv SLURMD_NODENAME] expect { -re "on host ($alpha_numeric)," { set nodelist_name $expect_out(1,string) @@ -70,7 +70,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 } eof { @@ -89,7 +89,7 @@ if {[string compare $nodelist_name ""] == 0} { send_user "\nFAILURE: Did not get nodelist_name of task 0\n" exit 1 } -set include_node $nodelist_name +set include_node $host_0 # # Submit a job directly to that node @@ -97,7 +97,7 @@ set include_node $nodelist_name set host_1 "" set slurm_user 1 set timeout 10 -spawn $srun -N1 -l --nodelist=$include_node --no-allocate -t1 $bin_printenv SLURMD_NODENAME +set srun_pid [spawn $srun -N1 -l --nodelist=$include_node --no-allocate -t1 $bin_printenv SLURMD_NODENAME] expect { -re "Invalid job credential" { send_user "\nWARNING: Not SlurmUser or root.\n" @@ -124,6 +124,7 @@ expect { timeout { send_user "\nWARNING: srun not responding, " send_user "expected if not SlurmUser or root.\n" + slow_kill $srun_pid set slurm_user 0 } eof { @@ -133,23 +134,11 @@ expect { if {$slurm_user == 0} { exit 0; } -if {[string compare $host_0 $host_1]} { - send_user "\nFAILURE: host name value wrong $host_0 $host_1\n" +if {[string compare $host_1 $include_node]} { + send_user "\nFAILURE: Allocation lacked an included node\n" set exit_code 1 } -# -# Verify node count and name -# -if {[string compare $host_0 ""] == 0} { - send_user "\nFAILURE: Did not get hostname of task 0\n" - set exit_code 1 -} -#if {[string compare $host_0 $include_node] != 0} { -# send_user "\nFAILURE: Allocation lacked an included node\n" -# set exit_code 1 -#} - # # Run three tasks at a time on some node and do so repeatedly # This checks for slurmd race conditions @@ -164,13 +153,13 @@ for {set inx 0} {$inx < $interations} {incr inx} { exec $bin_usleep 250000 set failures 0 - spawn $srun -N1 --nodelist=$include_node -t1 -l $bin_printenv SLURMD_NODENAME + set srun_pid [spawn $srun -N1 --nodelist=$include_node -t1 -l $bin_printenv SLURMD_NODENAME] set alloc $spawn_id - spawn $srun -N1 --nodelist=$include_node -Z $bin_usleep 500000 + set srun_pid1 [spawn $srun -N1 --nodelist=$include_node -Z $bin_usleep 500000] set noalloc1 $spawn_id - spawn $srun -N1 --nodelist=$include_node -Z $bin_usleep 250000 + set srun_pid2 [spawn $srun -N1 --nodelist=$include_node -Z $bin_usleep 250000] set noalloc2 $spawn_id set timeout 20 @@ -203,6 +192,7 @@ for {set inx 0} {$inx < $interations} {incr inx} { } timeout { send_user "\nFAILURE: srun not responding\n" + slow_kill $srun_pid2 set failures 1 } eof { @@ -239,6 +229,7 @@ for {set inx 0} {$inx < $interations} {incr inx} { } timeout { send_user "\nFAILURE: srun not responding\n" + slow_kill $srun_pid1 set failures 1 } eof { @@ -278,6 +269,7 @@ for {set inx 0} {$inx < $interations} {incr inx} { } timeout { send_user "\nFAILURE: srun not responding\n" + slow_kill $srun_pid set failures 1 } eof { diff --git a/testsuite/expect/test1.27 b/testsuite/expect/test1.27 index 6d8fe3a0b5bf638fe981e320d55416c23a99ed67..40df8849375e33d9a3b93b3984742277c0ddff47 100755 --- a/testsuite/expect/test1.27 +++ b/testsuite/expect/test1.27 @@ -63,7 +63,7 @@ array set good_vars { # Spawn a job via srun to print environment variables # set timeout $max_job_delay -spawn $srun -N1 -n1 -t1 $bin_env +set srun_pid [spawn $srun -N1 -n1 -t1 $bin_env] expect { -re "(SLURM_$alpha_under)=($alpha_numeric)" { set found_vars($expect_out(1,string)) "$expect_out(2,string)" @@ -71,9 +71,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.28 b/testsuite/expect/test1.28 index 2aae4a492bca69f237d1fab4ab1ebb3de54209fb..3bd362a7110fc44a48bd7ed3c2994989fd13d427 100755 --- a/testsuite/expect/test1.28 +++ b/testsuite/expect/test1.28 @@ -47,7 +47,7 @@ print_header $test_id # set timeout $max_job_delay set env($test_env_name) $test_env_val -spawn $srun -N1 -t1 $bin_env +set srun_pid [spawn $srun -N1 -t1 $bin_env] expect { -re "$test_env_name=($number)" { if {$expect_out(1,string) == $test_env_val} { @@ -57,9 +57,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.29 b/testsuite/expect/test1.29 index 5ed500f6efe42c9bf64836e8eb6151a95e39db30..44730f107f35a553503976024e7e2f4f3a83dfd0 100755 --- a/testsuite/expect/test1.29 +++ b/testsuite/expect/test1.29 @@ -133,7 +133,7 @@ make_bash_script $file_in " set timeout $max_job_delay -spawn $srun --batch --output=$file_out --error=$file_err -t1 ./$file_in +set srun_pid [spawn $srun --batch --output=$file_out --error=$file_err -t1 ./$file_in] expect { -re "jobid ($number) submitted" { set job_id $expect_out(1,string) @@ -141,9 +141,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.3 b/testsuite/expect/test1.3 index 23838281a66e2277f4c8463e5d03a9ac78cc0227..57ec23bb0cb2ae1cefd970243818f1c69c8e5cb8 100755 --- a/testsuite/expect/test1.3 +++ b/testsuite/expect/test1.3 @@ -50,7 +50,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.30 b/testsuite/expect/test1.30 index bb149952d1d96176c5439d009b8c9361cf521a19..30337c0c6750e96ddab2eebe0b96b14d1dff6d24 100755 --- a/testsuite/expect/test1.30 +++ b/testsuite/expect/test1.30 @@ -55,7 +55,7 @@ if { [test_bluegene] } { # set timeout $max_job_delay for {set node_cnt 1} {$node_cnt > 0} {set node_cnt [expr $node_cnt * 2]} { - spawn $srun --immediate -N$node_cnt $srun_opts -t2 $bin_hostname + set srun_pid [spawn $srun --immediate -N$node_cnt $srun_opts -t2 $bin_hostname] expect { -re "Immediate execution impossible" { send_user "This error is expected, no worries\n" @@ -85,9 +85,8 @@ for {set node_cnt 1} {$node_cnt > 0} {set node_cnt [expr $node_cnt * 2]} { } timeout { send_user "\nFAILURE: srun not responding\n" - set exit_code 1 - kill_srun - exp_continue + slow_kill $srun_pid + set exit_code 1 } eof { wait diff --git a/testsuite/expect/test1.31 b/testsuite/expect/test1.31 index fca0bfaaa39acc4a7137be61c63e9239e0de1631..d4693de681b50f823109b9023f3258ab22714d73 100755 --- a/testsuite/expect/test1.31 +++ b/testsuite/expect/test1.31 @@ -77,7 +77,7 @@ exec $bin_rm -f $env_valu_stdout # # Spawn a job via srun using these environment variables # -spawn $srun -t1 $bin_env +set srun_pid [spawn $srun -t1 $bin_env] expect { -re "jobid ($number)" { set job_id $expect_out(1,string) @@ -86,9 +86,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.32 b/testsuite/expect/test1.32 index 6fd60f4feed998acf2d513b1d030a36550100a3c..197a05ccca97dfb6a1383dba55750ee77a88ff63 100755 --- a/testsuite/expect/test1.32 +++ b/testsuite/expect/test1.32 @@ -79,13 +79,10 @@ expect { # Expect have difficulties handling unbuffered srun output # set timeout $max_job_delay -spawn $srun -N1 -t1 --unbuffered $file_prog -set srun_pid [exp_pid] +set srun_pid [spawn $srun -N1 -t1 --unbuffered $file_prog] exec $bin_sleep 5 exec $bin_kill -USR1 $srun_pid -#exec $bin_pkill -USR1 -n -u $uid srun exec $bin_kill -USR2 $srun_pid -#exec $bin_pkill -USR2 -n -u $uid srun expect { -re "WAITING" { incr matches @@ -105,9 +102,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { send_user "\nEOF\n" diff --git a/testsuite/expect/test1.33 b/testsuite/expect/test1.33 index 63f6d778c52a67cd4ed651f0db77861c334c3e9e..a42c1bfb4a8ad60394588453afcff0c0778f9d1b 100755 --- a/testsuite/expect/test1.33 +++ b/testsuite/expect/test1.33 @@ -47,7 +47,6 @@ make_bash_script $exit_script " make_bash_script $test_script " $srun -N1 -n2 -O $exit_script -# RC=\$? echo srun_exit_code_\$? " @@ -56,7 +55,7 @@ make_bash_script $test_script " # set sum 0 set timeout $max_job_delay -spawn $srun -N1 -n2 -O $exit_script +set srun_pid [spawn $srun -N1 -n2 -O $exit_script] expect { -re "exit code ($number)" { send_user "This error is expected, no worries\n" @@ -65,9 +64,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -104,7 +102,5 @@ if {$matches != 1} { if {$exit_code == 0} { exec $bin_rm -f $exit_script $test_script send_user "\nSUCCESS\n" -} else { - kill_srun } exit $exit_code diff --git a/testsuite/expect/test1.34 b/testsuite/expect/test1.34 index b97deefd8913efc8821eb52cc94b88cf0a48d513..13f9dee363233a1b6ab70d73f609f61a3141ab09 100755 --- a/testsuite/expect/test1.34 +++ b/testsuite/expect/test1.34 @@ -58,7 +58,7 @@ make_bash_script $file_in "$bin_echo \$1,\$2" # Spawn a srun batch job with arguments # set timeout $max_job_delay -spawn $srun --batch --output=$file_out --error=$file_err -t1 $file_in $arg1 $arg2 +set srun_pid [spawn $srun --batch --output=$file_out --error=$file_err -t1 $file_in $arg1 $arg2] expect { -re "jobid ($number) submitted" { set job_id $expect_out(1,string) @@ -66,9 +66,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -85,6 +84,7 @@ if {$job_id == 0} { # if {[wait_for_job $job_id "DONE"] != 0} { send_user "\nFAILURE: waiting for job to complete\n" + cancel_job $job_id set exit_code 1 } diff --git a/testsuite/expect/test1.35 b/testsuite/expect/test1.35 index ba277667aba1f628725b0e2fd7b14a552700b1f2..6d76bfb9de48eef4841cb3c43e6bdd2de6cd431a 100755 --- a/testsuite/expect/test1.35 +++ b/testsuite/expect/test1.35 @@ -6,9 +6,6 @@ # Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR # "FAILURE: ..." otherwise with an explanation of the failure, OR # anything else indicates a failure mode that must be investigated. -# -# Note: This script generates and then deletes files in the working directory -# named test1.35.input, test1.35.output, and test1.35.error ############################################################################ # Copyright (C) 2002-2006 The Regents of the University of California. # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). @@ -73,7 +70,7 @@ if { [test_bluegene] } { } } -spawn $srun --batch -N$node_cnt --output=$file_out --error=$file_err -t1 $file_in +set srun_pid [spawn $srun --batch -N$node_cnt --output=$file_out --error=$file_err -t1 $file_in] expect { -re "jobid ($number) submitted" { set job_id $expect_out(1,string) @@ -81,9 +78,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -100,6 +96,7 @@ if {$job_id == 0} { # if {[wait_for_job $job_id "DONE"] != 0} { send_user "\nFAILURE: waiting for job to complete\n" + cancel_job $job_id set exit_code 1 } diff --git a/testsuite/expect/test1.36 b/testsuite/expect/test1.36 index e1cd00443ed9d55c2a565e99ea0ebc22c167f520..3e3632a17722d8181df3665ac5e39f984efde0a2 100755 --- a/testsuite/expect/test1.36 +++ b/testsuite/expect/test1.36 @@ -58,7 +58,7 @@ if { [test_bluegene] } { } } -spawn $srun -N$node_cnt -n$task_cnt -O -t1 $srun -l -n$mult -O $bin_id +set srun_pid [spawn $srun -N$node_cnt -n$task_cnt -O -t1 $srun -l -n$mult -O $bin_id] expect { -re "($number): uid=" { incr task_output @@ -66,9 +66,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.37 b/testsuite/expect/test1.37 index bf8e57c10af0b6307c0b12dfb606ba1314e6fd85..5915a40baf38b310e14168858d4367fc616d0e01 100755 --- a/testsuite/expect/test1.37 +++ b/testsuite/expect/test1.37 @@ -48,7 +48,7 @@ print_header $test_id # Submit a job and get the node's NodeName from the nodelist # set timeout $max_job_delay -spawn $srun -v -N1 -l -t1 $bin_hostname +set srun_pid [spawn $srun -v -N1 -l -t1 $bin_hostname] expect { -re "on host ($alpha_numeric)," { set nodelist_name $expect_out(1,string) @@ -60,9 +60,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -89,7 +88,7 @@ make_bash_script $file_in "$srun $bin_sleep 5" # with sharing permitted. Insure the first job completes before the # second job is started. # -spawn $srun --batch -N1 --exclusive --nodelist=$nodelist_name -t1 --output=$file_out --error=$file_err $file_in +set srun_pid [spawn $srun --batch -N1 --exclusive --nodelist=$nodelist_name -t1 --output=$file_out --error=$file_err $file_in] expect { -re "jobid ($number) submitted" { set job_id1 $expect_out(1,string) @@ -97,7 +96,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { @@ -113,7 +112,7 @@ if {$job_id1 == 0} { set partition "dummy" set waited 1 set timeout [expr $timeout + 5] -spawn $srun -N1 --nodelist=$nodelist_name -t1 --share $scontrol -o show job $job_id1 +set srun_pid [spawn $srun -N1 --nodelist=$nodelist_name -t1 --share $scontrol -o show job $job_id1] expect { -re "Partition=($alpha_numeric)" { set partition $expect_out(1,string) @@ -125,7 +124,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid exit 1 } eof { diff --git a/testsuite/expect/test1.38 b/testsuite/expect/test1.38 index 07a093b1ce3d54736453cdaa9e0a92c9a2c25ebc..e3084acf3f22bb5bd4523c50b5acba28b5eae487 100755 --- a/testsuite/expect/test1.38 +++ b/testsuite/expect/test1.38 @@ -74,8 +74,7 @@ expect { set timeout $max_job_delay set matches 0 set job_id 0 -spawn $srun -v -N1 -t1 --unbuffered $file_in -set srun_pid [exp_pid] +set srun_pid [spawn $srun -v -N1 -t1 --unbuffered $file_in] expect { -re "launching ($number).0" { set job_id $expect_out(1,string) @@ -84,7 +83,6 @@ expect { -re "WAITING" { incr matches exec $bin_kill -INT $srun_pid -# exec $bin_pkill -INT -n -u $uid srun exp_continue } -re "srun: interrupt" { @@ -98,9 +96,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -118,8 +115,7 @@ if {$matches != 3} { # set matches 0 set job_id 0 -spawn $srun -v -N1 -t1 --unbuffered --quit-on-interrupt $file_in -set srun_pid [exp_pid] +set srun_pid [spawn $srun -v -N1 -t1 --unbuffered --quit-on-interrupt $file_in] expect { -re "launching ($number).0" { set job_id $expect_out(1,string) @@ -128,7 +124,6 @@ expect { -re "WAITING" { incr matches exec $bin_kill -INT $srun_pid -# exec $bin_pkill -INT -n -u $uid srun exp_continue } -re "srun: interrupt" { @@ -145,9 +140,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.39 b/testsuite/expect/test1.39 index 34885f0eda663f378df90e50b37fcb25d85cf0a2..4d8cfd1351dd22f7a92ac9ffe82438a222ab7e73 100755 --- a/testsuite/expect/test1.39 +++ b/testsuite/expect/test1.39 @@ -44,12 +44,17 @@ print_header $test_id # Test if "light" core file supported # set matches 0 -spawn $srun --core=list +set srun_pid [spawn $srun --core=list] expect { -re " light " { set matches 1 exp_continue } + timeout { + send_user "\nFAILURE: srun not responding\n" + slow_kill $srun_pid + exit 1 + } eof { wait } @@ -77,7 +82,7 @@ foreach filename [glob -nocomplain *core*] { # Spawn initial program via srun # set timeout $max_job_delay -spawn $srun -N1 -t1 --core=light $file_prog +set srun_pid [spawn $srun -N1 -t1 --core=light $file_prog] expect { -re "Segmentation fault" { send_user "\nNo worries, this error is expected.\n" @@ -85,9 +90,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.4 b/testsuite/expect/test1.4 index 177f1edb896ccb6388b6e66d8166016c3ab7f9f2..cd70baea80e73d799427318fc75029a019995d43 100755 --- a/testsuite/expect/test1.4 +++ b/testsuite/expect/test1.4 @@ -50,7 +50,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.5 b/testsuite/expect/test1.5 index 179da4a4245ee7ceea5a58e4b0614a7cf0ea04da..fae47de242525a095c6d674f1417d3694932d872 100755 --- a/testsuite/expect/test1.5 +++ b/testsuite/expect/test1.5 @@ -58,7 +58,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.6 b/testsuite/expect/test1.6 index 2863c290832dfc5c834d4c009a2d3bb7074a801d..f4311e80ff22e2237136714e7d0b896b4ab0f183 100755 --- a/testsuite/expect/test1.6 +++ b/testsuite/expect/test1.6 @@ -59,7 +59,6 @@ expect { send_user "\nFAILURE: srun not responding\n" slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.7 b/testsuite/expect/test1.7 index 3e1c12230ed8683e95eb72189c99f0bbdfee5d75..ed0515be2b90368dc175567a8235b9bf5961a834 100755 --- a/testsuite/expect/test1.7 +++ b/testsuite/expect/test1.7 @@ -6,9 +6,6 @@ # Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR # "FAILURE: ..." otherwise with an explanation of the failure, OR # anything else indicates a failure mode that must be investigated. -# -# Note: This script generates and then deletes files in the working directory -# named test1.7.input, test1.7.output, and test1.7.error ############################################################################ # Copyright (C) 2002-2006 The Regents of the University of California. # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). @@ -110,7 +107,7 @@ make_bash_script $file_in " # set timeout [expr $max_job_delay + $sleep_time] set timed_out 0 -spawn $srun -t1 $bin_sleep $sleep_time +set srun_pid [spawn $srun -t1 $bin_sleep $sleep_time] expect { -re "job exceeded timelimit" { set timed_out 1 @@ -126,9 +123,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -142,7 +138,7 @@ if {$timed_out == 1} { } set completions 0 -spawn $srun -t4 $bin_sleep $sleep_time +set srun_pid [spawn $srun -t4 $bin_sleep $sleep_time] expect { -re "job exceeded timelimit" { set completions -1 @@ -162,9 +158,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -180,7 +175,7 @@ if {$completions != 1} { # Spawn a srun batch job with arguments # set timeout $max_job_delay -spawn $srun --batch --output=$file_out --error=$file_err -t4 $file_in +set srun_pid [spawn $srun --batch --output=$file_out --error=$file_err -t4 ./$file_in] expect { -re "jobid ($number) submitted" { set job_id $expect_out(1,string) @@ -188,7 +183,7 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 } eof { @@ -224,7 +219,8 @@ if {[wait_for_file $file_out] == 0} { } if {$output_fini == 0} { - send_user "\nFAILURE: Unexpected batch job output, possible premature job termination\n" + send_user "\nFAILURE: Unexpected batch job output, " + send_user "possible premature job termination\n" set exit_code 1 } diff --git a/testsuite/expect/test1.8 b/testsuite/expect/test1.8 index 29b9d88c6766965558f2cafa53d2b191073d284d..ee9303b490690a95ce43c444b276fb1d515b3ea5 100755 --- a/testsuite/expect/test1.8 +++ b/testsuite/expect/test1.8 @@ -63,7 +63,7 @@ exec $bin_chmod 700 $file_in # Spawn a shell via srun that uses stdin/out/err and confirm their contents # set timeout $max_job_delay -spawn $srun --input=$file_in --output=$file_out --error=$file_err -t1 $bin_bash +set srun_pid [spawn $srun --input=$file_in --output=$file_out --error=$file_err -t1 $bin_bash] expect { -re "Unable to contact" { send_user "\nFAILURE: slurm appears to be down\n" @@ -71,9 +71,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait @@ -150,7 +149,7 @@ if {$got_sleep_err == 0} { # # Spawn a program to run for a while with no input, output or error # -spawn $srun --input=none --output=none --error=none -t1 -N1 od -c $srun +set srun_pid [spawn $srun --input=none --output=none --error=none -t1 -N1 od -c $srun] expect { -re "Unable to contact" { send_user "\nFAILURE: slurm appears to be down\n" @@ -168,9 +167,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test1.9 b/testsuite/expect/test1.9 index e91341879e919b249d52deb5351259c1f74c4d08..22be915a3e7c3875aadf139ad6fa61e45d3439f7 100755 --- a/testsuite/expect/test1.9 +++ b/testsuite/expect/test1.9 @@ -43,7 +43,7 @@ print_header $test_id # Submit a slurm job that will execute 'id' on 1 node and over task_cnt tasks # set timeout $max_job_delay -spawn $srun -N1 -n$task_cnt -O -v -t1 $bin_id +set srun_pid [spawn $srun -N1 -n$task_cnt -O -v -t1 $bin_id] expect { -re "jobid" { set verbosity 1 @@ -59,9 +59,8 @@ expect { } timeout { send_user "\nFAILURE: srun not responding\n" - kill_srun + slow_kill $srun_pid set exit_code 1 - exp_continue } eof { wait diff --git a/testsuite/expect/test15.10 b/testsuite/expect/test15.10 index 27c715d21adb0cbc0a61e3fd4f71792d34c773b3..fc880c863ae98d1c5092765251d7326dfa0f422a 100755 --- a/testsuite/expect/test15.10 +++ b/testsuite/expect/test15.10 @@ -59,13 +59,11 @@ for {set inx 1} {$inx < 4} {set inx [expr $inx * 2]} { } timeout { send_user "\nFAILURE: salloc not responding\n" - if {$job_id == 0} { - slow_kill $salloc_pid - } else { + if {$job_id != 0} { cancel_job $job_id } + slow_kill $salloc_pid set exit_code 1 - exp_continue } eof { if {$tasks_get < $tasks_set} {