Update test to validate persistent connections

Update "scontrol show federation test to validate persistent connections
after adding and removing clusters from federation.

testsuite/expect/globals_federation

@@ -184,10 +184,10 @@ proc delete_federations { names } {
 }
 
 proc get_clusterfed_info { fed_name } {
-	global sacctmgr fedc1 fedc2 fedc3 eol
+	global sacctmgr eol
 	set matches 0
 	array set clusters {}
-	set my_pid [spawn $sacctmgr show cluster $fedc1 $fedc2 $fedc3 \
+	set my_pid [spawn $sacctmgr show cluster federation=$fed_name \
 		    format="cluster%20,federation%20,index,controlhost,controlport,weight,fedstate"]
 	expect {
 		-re "Cluster\\s+Federation\\s+Index\\s+ControlHost\\s+ControlPort\\s+Weight\\s+FedState $eol" {
@@ -212,11 +212,102 @@ proc get_clusterfed_info { fed_name } {
 			wait
 		}
 	}
-	if {$matches != 4} {
-		send_user "$matches FAILURE: didn't match enough clustesr for\
-			$fed_name.\n"
+	if {$matches < 2} {
+		send_user "$matches FAILURE: didn't match enough clustesr for $fed_name.\n"
 		exit 1
 	}
 
 	return [array get clusters]
 }
+
+#
+# Add a single cluster to the given federation.
+# IN:  cname    - name of cluster to add to federation.
+# IN:  fed_name - name of federation to add cluster to.
+# RET: returns 0 on sucess, 1 on failure.
+#
+proc add_cluster_to_fed {cname fed_name} {
+	global sacctmgr eol
+
+	set rc 0
+	set matches 0
+	set my_pid [spawn $sacctmgr -i modify federation $fed_name set clusters+=$cname]
+	expect {
+		-re "Setting$eol" {
+			incr matches
+			exp_continue
+		}
+		-re "Cluster\\s+ \\+= $cname$eol" {
+			incr matches
+			exp_continue
+		}
+		-re "^\\s+Modified federation...$eol" {
+			incr matches
+			exp_continue
+		}
+		-re "\\s+$fed_name$eol" {
+			incr matches
+			exp_continue
+		}
+		timeout {
+			send_user "\nFAILURE: sacctmgr add not responding\n"
+			slow_kill $my_pid
+			set rc 1
+		}
+		eof {
+			wait
+		}
+	}
+	if {$rc || $matches != 4} {
+		send_user "$matches FAILURE: failed to add $cname to $fed_name.\n"
+		set $rc 1
+	}
+
+	return $rc
+}
+
+#
+# Remove a single cluster from the given federation.
+# IN:  cname    - name of cluster to remove from the federation.
+# IN:  fed_name - name of federation to remove cluster from.
+# RET: returns 0 on sucess, 1 on failure.
+#
+proc remove_cluster_from_fed {cname fed_name} {
+	global sacctmgr eol
+
+	set rc 0
+	set matches 0
+	set my_pid [spawn $sacctmgr -i modify federation $fed_name set clusters-=$cname]
+	expect {
+		-re "Setting$eol" {
+			incr matches
+			exp_continue
+		}
+		-re "Cluster\\s+ -= $cname$eol" {
+			incr matches
+			exp_continue
+		}
+		-re "^\\s+Modified federation...$eol" {
+			incr matches
+			exp_continue
+		}
+		-re "\\s+$fed_name$eol" {
+			incr matches
+			exp_continue
+		}
+		timeout {
+			send_user "\nFAILURE: sacctmgr add not responding\n"
+			slow_kill $my_pid
+			set rc 1
+		}
+		eof {
+			wait
+		}
+	}
+	if {$rc || $matches != 4} {
+		send_user "$matches FAILURE: failed to remove $cname from $fed_name.\n"
+		set $rc 1
+	}
+
+	return $rc
+}

testsuite/expect/test37.1

@@ -2,6 +2,7 @@
 ############################################################################
 # Purpose: Test federation functionality
 #          Test scontrol show federations
+#	   Test that persistent connections are made
 #
 # Reqs:    1. Using slurmdbd accounting storage type and is up
 #          2. fed_slurm_base is defined in globals.local - set to directory that
@@ -87,26 +88,36 @@ proc test_fed_status {cname cluster_list} {
 	# Match db info with what cluster has.
 	global fed_slurm_base fed_name eol
 	array set clusters $cluster_list
-	parray clusters
 	set matches 0
+	set lines 0
 	set my_scontrol "${fed_slurm_base}/$cname/bin/scontrol"
 	set my_pid [spawn $my_scontrol show federation]
+	set expected_matches [array size clusters]
+	if {$expected_matches} {
+		# Match the fedname line
+		incr expected_matches
+	}
 	expect {
-		-re "Federation: $fed_name$eol" {
-			incr matches
-			exp_continue
-		}
-		-re "Sibling:\\s+(\\S+):(\\S+):(\\d+) Index:(\\d+) FedState:(\\S*) Weight:(\\d+) PersistConn:(\\S+)$eol" {
-			set name $expect_out(1,string)
-			if {![string compare [dict get $clusters($name) host]  $expect_out(2,string)] &&
-			    ![string compare [dict get $clusters($name) state] $expect_out(5,string)] &&
-			    [dict get $clusters($name) port]   == $expect_out(3,string) &&
-			    [dict get $clusters($name) index]  == $expect_out(4,string) &&
-			    [dict get $clusters($name) weight] == $expect_out(6,string) &&
-			    ((![string compare $name $cname] &&
-			      ![string compare "Self" $expect_out(7,string)]) ||
-			     ![string compare "Connected" $expect_out(7,string)])} {
+		-re "$eol" {
+			incr lines
+			set line $expect_out(buffer)
+			#send_user "line: $line\n"
+
+			if {[regexp {Federation: feda} $line match]} {
+				#send_user "matched: $match\n"
 				incr matches
+			} elseif {[regexp {Sibling:\s+(\S+):(\S+):(\d+) Index:(\d+) FedState:(\S*) Weight:(\d+) PersistConn:(\S+)} \
+					  $line match name host port index state weight conn]} {
+				#send_user "matched: $match\n"
+				if {![string compare [dict get $clusters($name) host]  $host]  &&
+				    ![string compare [dict get $clusters($name) state] $state] &&
+				    ![string compare [dict get $clusters($name) conn]  $conn]  &&
+				    [dict get $clusters($name) port]   == $port &&
+				    [dict get $clusters($name) index]  == $index &&
+				    [dict get $clusters($name) weight] == $weight} {
+					#send_user "matched: $name\n"
+					incr matches
+				}
 			}
 			exp_continue
 		}
@@ -119,8 +130,13 @@ proc test_fed_status {cname cluster_list} {
 			wait
 		}
 	}
-	if {$matches != 4} {
-		send_user "$matches FAILURE: unexpected error.\n"
+	if {$matches != $expected_matches} {
+		send_user "FAILURE: failed to match $matches != $expected_matches.\n"
+		end_it 1
+	}
+
+	if {$lines != $matches} {
+		send_user "FAILURE: more or less lines found than matches: $ilines  != $matches.\n"
 		end_it 1
 	}
 }
@@ -152,8 +168,77 @@ if {[setup_federation $fed_name]} {
 # Get cluster/fed info from db
 array set clusters [get_clusterfed_info $fed_name]
 
+send_user "\n\ntest from $fedc1\n"
+dict set clusters($fedc1) conn Self
+dict set clusters($fedc2) conn Connected
+dict set clusters($fedc3) conn Connected
 test_fed_status $fedc1 [array get clusters]
+
+send_user "\n\ntest from $fedc2\n"
+dict set clusters($fedc1) conn Connected
+dict set clusters($fedc2) conn Self
+dict set clusters($fedc3) conn Connected
 test_fed_status $fedc2 [array get clusters]
+
+send_user "\n\ntest from $fedc3\n"
+dict set clusters($fedc1) conn Connected
+dict set clusters($fedc2) conn Connected
+dict set clusters($fedc3) conn Self
 test_fed_status $fedc3 [array get clusters]
 
+send_user "\n\nremove $fedc3 and test from $fedc1 and $fedc2\n"
+if {[remove_cluster_from_fed $fedc3 $fed_name]} {
+	end_it 1
+}
+array unset clusters $fedc3
+dict set clusters($fedc1) conn Self
+dict set clusters($fedc2) conn Connected
+test_fed_status $fedc1 [array get clusters]
+
+dict set clusters($fedc1) conn Connected
+dict set clusters($fedc2) conn Self
+test_fed_status $fedc2 [array get clusters]
+
+send_user "\n\nremove $fedc1 and test from $fedc2\n"
+if {[remove_cluster_from_fed $fedc1 $fed_name]} {
+	end_it 1
+}
+array unset clusters $fedc1
+dict set clusters($fedc2) conn Self
+test_fed_status $fedc2 [array get clusters]
+
+send_user "\n\nremove $fedc2 and test from $fedc2\n"
+array unset clusters
+if {[remove_cluster_from_fed $fedc2 $fed_name]} {
+	end_it 1
+}
+test_fed_status $fedc2 [array get clusters]
+
+send_user "\n\nadd $fedc1 and test from $fedc1\n"
+if {[add_cluster_to_fed $fedc1 $fed_name]} {
+	end_it 1
+}
+array set clusters [get_clusterfed_info $fed_name]
+dict set clusters($fedc1) conn Self
+test_fed_status $fedc1 [array get clusters]
+
+send_user "\n\nadd $fedc2 and test from $fedc1\n"
+if {[add_cluster_to_fed $fedc2 $fed_name]} {
+	end_it 1
+}
+array set clusters [get_clusterfed_info $fed_name]
+dict set clusters($fedc1) conn Self
+dict set clusters($fedc2) conn Connected
+test_fed_status $fedc1 [array get clusters]
+
+send_user "\n\nadd $fedc3 and test from $fedc1\n"
+if {[add_cluster_to_fed $fedc3 $fed_name]} {
+	end_it 1
+}
+array set clusters [get_clusterfed_info $fed_name]
+dict set clusters($fedc1) conn Self
+dict set clusters($fedc2) conn Connected
+dict set clusters($fedc3) conn Connected
+test_fed_status $fedc1 [array get clusters]
+
 end_it 0