Add test to make sure job array head record purged last

bug 5693, 6397

testsuite/expect/Makefile.am

@@ -549,6 +549,7 @@ EXTRA_DIST = \
 	test28.8			\
 	test28.9			\
 	test28.10			\
+	test28.11			\
 	test29.1                        \
 	test29.2                        \
 	test29.3                        \

testsuite/expect/Makefile.in

@@ -966,6 +966,7 @@ EXTRA_DIST = \
 	test28.8			\
 	test28.9			\
 	test28.10			\
+	test28.11			\
 	test29.1                        \
 	test29.2                        \
 	test29.3                        \

testsuite/expect/README

@@ -687,6 +687,7 @@ test28.7   Confirms job array dependencies.
 test28.8   Test of job array suspend/resume.
 test28.9   Test of maximum running task count in a job array.
 test28.10  Test change to maximum running task count in a job array.
+test28.11  Verify head job isn't purged before other jobs in the array.
 
 
 test29.#   Testing of smd command and option.

testsuite/expect/test28.11

+#!/usr/bin/env expect
+############################################################################
+# Purpose: Test of Slurm functionality
+#          Verify head job isn't purged before other jobs in the array.
+#
+#
+# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
+#          "FAILURE: ..." otherwise with an explanation of the failure, OR
+#          anything else indicates a failure mode that must be investigated.
+############################################################################
+# Copyright (C) 2019 SchedMD LLC
+# Written by Morris Jette
+#
+# This file is part of Slurm, a resource management program.
+# For details, see <https://slurm.schedmd.com/>.
+# Please also read the included file: DISCLAIMER.
+#
+# Slurm is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along
+# with Slurm; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
+############################################################################
+source ./globals
+
+set test_id     "28.11"
+set exit_code   0
+set file_script "test$test_id.bash"
+set job_id      0
+
+print_header $test_id
+
+set min_job_age [get_min_job_age]
+if {$min_job_age > 60} {
+	send_user "\nWARNING: MinJobAge too large for this test\n"
+	exit 0
+}
+
+make_bash_script $file_script "
+if \[ \$SLURM_ARRAY_TASK_ID == 4 \]
+then
+	exit 0
+fi
+sleep 60
+exit 0"
+
+# submit a batch job array
+spawn $sbatch -n1 --array=1-4 --output=/dev/null $file_script
+expect {
+	-re "Submitted batch job ($number)" {
+		set job_id $expect_out(1,string)
+		send_user "\njob $job_id was submitted\n"
+		exp_continue
+	}
+	-re "error" {
+		send_user "\nFAILURE: sbatch did not submit jobs\n"
+		exit 1
+	}
+	timeout {
+		send_user "\nFAILURE: sbatch not responding\n"
+		set exit_code 1
+	}
+	eof {
+		wait
+	}
+}
+
+# wait for head job to complete
+if {[wait_for_job $job_id "DONE"] != 0} {
+	send_user "\nFAILURE: error waiting for job $job_id to complete\n"
+	cancel_job $job_id
+	exit 1
+}
+
+# wait a while for job purge to happen
+send_user "\nWaiting for job record purge to happen...\n"
+sleep 65
+
+# check that head job still exists
+set match 0
+spawn $squeue -tall --job=$job_id
+expect {
+	-re "${job_id}_4" {
+		incr match
+		exp_continue
+	}
+	-re "error" {
+		send_user "\nFAILURE: error running squeue\n"
+		set exit_code 1
+		exp_continue
+	}
+	timeout {
+		send_user "\nFAILURE: squeue not responding\n"
+		set exit_code 1
+	}
+	eof {
+		wait
+	}
+}
+if {$match != 1} {
+	send_user "\nFAILURE: head job not found\n"
+	set exit_code 1
+}
+
+cancel_job $job_id
+if {$exit_code == 0} {
+	file delete $file_script
+	send_user "\nSUCCESS\n"
+}
+exit $exit_code