diff --git a/testsuite/expect/globals_federation b/testsuite/expect/globals_federation index 6914a1578eb7eac340c5a1ad964ce5717606c50a..eb650f87a6cc8dc4ed9d899c4d6d62cca465fb95 100644 --- a/testsuite/expect/globals_federation +++ b/testsuite/expect/globals_federation @@ -184,10 +184,10 @@ proc delete_federations { names } { } proc get_clusterfed_info { fed_name } { - global sacctmgr fedc1 fedc2 fedc3 eol + global sacctmgr eol set matches 0 array set clusters {} - set my_pid [spawn $sacctmgr show cluster $fedc1 $fedc2 $fedc3 \ + set my_pid [spawn $sacctmgr show cluster federation=$fed_name \ format="cluster%20,federation%20,index,controlhost,controlport,weight,fedstate"] expect { -re "Cluster\\s+Federation\\s+Index\\s+ControlHost\\s+ControlPort\\s+Weight\\s+FedState $eol" { @@ -212,11 +212,102 @@ proc get_clusterfed_info { fed_name } { wait } } - if {$matches != 4} { - send_user "$matches FAILURE: didn't match enough clustesr for\ - $fed_name.\n" + if {$matches < 2} { + send_user "$matches FAILURE: didn't match enough clustesr for $fed_name.\n" exit 1 } return [array get clusters] } + +# +# Add a single cluster to the given federation. +# IN: cname - name of cluster to add to federation. +# IN: fed_name - name of federation to add cluster to. +# RET: returns 0 on sucess, 1 on failure. +# +proc add_cluster_to_fed {cname fed_name} { + global sacctmgr eol + + set rc 0 + set matches 0 + set my_pid [spawn $sacctmgr -i modify federation $fed_name set clusters+=$cname] + expect { + -re "Setting$eol" { + incr matches + exp_continue + } + -re "Cluster\\s+ \\+= $cname$eol" { + incr matches + exp_continue + } + -re "^\\s+Modified federation...$eol" { + incr matches + exp_continue + } + -re "\\s+$fed_name$eol" { + incr matches + exp_continue + } + timeout { + send_user "\nFAILURE: sacctmgr add not responding\n" + slow_kill $my_pid + set rc 1 + } + eof { + wait + } + } + if {$rc || $matches != 4} { + send_user "$matches FAILURE: failed to add $cname to $fed_name.\n" + set $rc 1 + } + + return $rc +} + +# +# Remove a single cluster from the given federation. +# IN: cname - name of cluster to remove from the federation. +# IN: fed_name - name of federation to remove cluster from. +# RET: returns 0 on sucess, 1 on failure. +# +proc remove_cluster_from_fed {cname fed_name} { + global sacctmgr eol + + set rc 0 + set matches 0 + set my_pid [spawn $sacctmgr -i modify federation $fed_name set clusters-=$cname] + expect { + -re "Setting$eol" { + incr matches + exp_continue + } + -re "Cluster\\s+ -= $cname$eol" { + incr matches + exp_continue + } + -re "^\\s+Modified federation...$eol" { + incr matches + exp_continue + } + -re "\\s+$fed_name$eol" { + incr matches + exp_continue + } + timeout { + send_user "\nFAILURE: sacctmgr add not responding\n" + slow_kill $my_pid + set rc 1 + } + eof { + wait + } + } + if {$rc || $matches != 4} { + send_user "$matches FAILURE: failed to remove $cname from $fed_name.\n" + set $rc 1 + } + + return $rc +} diff --git a/testsuite/expect/test37.1 b/testsuite/expect/test37.1 index 769066138813b7723e27a6a70839db40eb1d0920..e69f756e1455236535674434b2afc2d69a90b1d9 100755 --- a/testsuite/expect/test37.1 +++ b/testsuite/expect/test37.1 @@ -2,6 +2,7 @@ ############################################################################ # Purpose: Test federation functionality # Test scontrol show federations +# Test that persistent connections are made # # Reqs: 1. Using slurmdbd accounting storage type and is up # 2. fed_slurm_base is defined in globals.local - set to directory that @@ -87,26 +88,36 @@ proc test_fed_status {cname cluster_list} { # Match db info with what cluster has. global fed_slurm_base fed_name eol array set clusters $cluster_list - parray clusters set matches 0 + set lines 0 set my_scontrol "${fed_slurm_base}/$cname/bin/scontrol" set my_pid [spawn $my_scontrol show federation] + set expected_matches [array size clusters] + if {$expected_matches} { + # Match the fedname line + incr expected_matches + } expect { - -re "Federation: $fed_name$eol" { - incr matches - exp_continue - } - -re "Sibling:\\s+(\\S+):(\\S+):(\\d+) Index:(\\d+) FedState:(\\S*) Weight:(\\d+) PersistConn:(\\S+)$eol" { - set name $expect_out(1,string) - if {![string compare [dict get $clusters($name) host] $expect_out(2,string)] && - ![string compare [dict get $clusters($name) state] $expect_out(5,string)] && - [dict get $clusters($name) port] == $expect_out(3,string) && - [dict get $clusters($name) index] == $expect_out(4,string) && - [dict get $clusters($name) weight] == $expect_out(6,string) && - ((![string compare $name $cname] && - ![string compare "Self" $expect_out(7,string)]) || - ![string compare "Connected" $expect_out(7,string)])} { + -re "$eol" { + incr lines + set line $expect_out(buffer) + #send_user "line: $line\n" + + if {[regexp {Federation: feda} $line match]} { + #send_user "matched: $match\n" incr matches + } elseif {[regexp {Sibling:\s+(\S+):(\S+):(\d+) Index:(\d+) FedState:(\S*) Weight:(\d+) PersistConn:(\S+)} \ + $line match name host port index state weight conn]} { + #send_user "matched: $match\n" + if {![string compare [dict get $clusters($name) host] $host] && + ![string compare [dict get $clusters($name) state] $state] && + ![string compare [dict get $clusters($name) conn] $conn] && + [dict get $clusters($name) port] == $port && + [dict get $clusters($name) index] == $index && + [dict get $clusters($name) weight] == $weight} { + #send_user "matched: $name\n" + incr matches + } } exp_continue } @@ -119,8 +130,13 @@ proc test_fed_status {cname cluster_list} { wait } } - if {$matches != 4} { - send_user "$matches FAILURE: unexpected error.\n" + if {$matches != $expected_matches} { + send_user "FAILURE: failed to match $matches != $expected_matches.\n" + end_it 1 + } + + if {$lines != $matches} { + send_user "FAILURE: more or less lines found than matches: $ilines != $matches.\n" end_it 1 } } @@ -152,8 +168,77 @@ if {[setup_federation $fed_name]} { # Get cluster/fed info from db array set clusters [get_clusterfed_info $fed_name] +send_user "\n\ntest from $fedc1\n" +dict set clusters($fedc1) conn Self +dict set clusters($fedc2) conn Connected +dict set clusters($fedc3) conn Connected test_fed_status $fedc1 [array get clusters] + +send_user "\n\ntest from $fedc2\n" +dict set clusters($fedc1) conn Connected +dict set clusters($fedc2) conn Self +dict set clusters($fedc3) conn Connected test_fed_status $fedc2 [array get clusters] + +send_user "\n\ntest from $fedc3\n" +dict set clusters($fedc1) conn Connected +dict set clusters($fedc2) conn Connected +dict set clusters($fedc3) conn Self test_fed_status $fedc3 [array get clusters] +send_user "\n\nremove $fedc3 and test from $fedc1 and $fedc2\n" +if {[remove_cluster_from_fed $fedc3 $fed_name]} { + end_it 1 +} +array unset clusters $fedc3 +dict set clusters($fedc1) conn Self +dict set clusters($fedc2) conn Connected +test_fed_status $fedc1 [array get clusters] + +dict set clusters($fedc1) conn Connected +dict set clusters($fedc2) conn Self +test_fed_status $fedc2 [array get clusters] + +send_user "\n\nremove $fedc1 and test from $fedc2\n" +if {[remove_cluster_from_fed $fedc1 $fed_name]} { + end_it 1 +} +array unset clusters $fedc1 +dict set clusters($fedc2) conn Self +test_fed_status $fedc2 [array get clusters] + +send_user "\n\nremove $fedc2 and test from $fedc2\n" +array unset clusters +if {[remove_cluster_from_fed $fedc2 $fed_name]} { + end_it 1 +} +test_fed_status $fedc2 [array get clusters] + +send_user "\n\nadd $fedc1 and test from $fedc1\n" +if {[add_cluster_to_fed $fedc1 $fed_name]} { + end_it 1 +} +array set clusters [get_clusterfed_info $fed_name] +dict set clusters($fedc1) conn Self +test_fed_status $fedc1 [array get clusters] + +send_user "\n\nadd $fedc2 and test from $fedc1\n" +if {[add_cluster_to_fed $fedc2 $fed_name]} { + end_it 1 +} +array set clusters [get_clusterfed_info $fed_name] +dict set clusters($fedc1) conn Self +dict set clusters($fedc2) conn Connected +test_fed_status $fedc1 [array get clusters] + +send_user "\n\nadd $fedc3 and test from $fedc1\n" +if {[add_cluster_to_fed $fedc3 $fed_name]} { + end_it 1 +} +array set clusters [get_clusterfed_info $fed_name] +dict set clusters($fedc1) conn Self +dict set clusters($fedc2) conn Connected +dict set clusters($fedc3) conn Connected +test_fed_status $fedc1 [array get clusters] + end_it 0