tweak user env loading logic to prevent indefinite hang of slurmd

or salloc and Moab daemons

src/common/env.c

@@ -1446,10 +1446,21 @@ char **env_array_user_default(const char *username, int timeout, int mode)
 		}
 	}
 	close(fildes[0]);
-	kill(-child, 9);
-	if (waitpid((pid_t)-1, &rc, WNOHANG))
-		waitpid((pid_t)-1, &rc, WNOHANG); /* left from previous runs */
-
+	for (config_timeout=0; ; config_timeout++) {
+		kill(-child, 9);
+		if (config_timeout)
+			sleep(1);
+		if (waitpid(child, &rc, WNOHANG) > 0)
+			break;
+		if (config_timeout > 2) {
+			/* Non-killable processes are indicative of file system
+			 * problems. The process will remain as a zombie, but 
+			 * slurmd/salloc/moab will not otherwise be effected. */
+			error("Failed to kill program loading user environment");
+			break;
+		}
+	}
+	
 	if (!found) {
 		error("Failed to load current user environment variables");
 		xfree(buffer);