From e12ba6fe5ea952248bf1838e94345db5527f83e3 Mon Sep 17 00:00:00 2001
From: Moe Jette <jette1@llnl.gov>
Date: Wed, 25 Mar 2009 23:00:24 +0000
Subject: [PATCH] Set "/proc/self/oom_adj" for slurmd and slurmstepd daemons
based upon the values of SLURMD_OOM_ADJ and SLURMSTEPD_OOM_ADJ environment
variables. This can be used to prevent daemons being killed when a node's
memory is exhausted.
---
NEWS | 4 ++++
RELEASE_NOTES | 4 ++++
src/slurmd/common/set_oomadj.c | 5 ++---
src/slurmd/slurmd/slurmd.c | 7 ++++++-
src/slurmd/slurmstepd/mgr.c | 14 ++++++++++++--
src/slurmd/slurmstepd/slurmstepd.c | 3 ---
6 files changed, 28 insertions(+), 9 deletions(-)
diff --git a/NEWS b/NEWS
index aa6b11198b8..b55f3ee9d84 100644
--- a/NEWS
+++ b/NEWS
@@ -19,6 +19,10 @@ documents those changes that are of interest to users and admins.
set TopologyType=topology/tree and add configuration information
to a new file called topology.conf. See "man topology.conf" or
topology.html web page for details.
+ -- Set "/proc/self/oom_adj" for slurmd and slurmstepd daemons based upon
+ the values of SLURMD_OOM_ADJ and SLURMSTEPD_OOM_ADJ environment
+ variables. This can be used to prevent daemons being killed when
+ a node's memory is exhausted.
* Changes in SLURM 1.4.0-pre9
=============================
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index a41283bcbe0..c401cc3000d 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -139,3 +139,7 @@ OTHER CHANGES
* Modify PMI_Get_clique_ranks() to return an array of integers rather
than a char * to satisfy PMI standard. Correct logic in
PMI_Get_clique_size() for when srun --overcommit option is used.
+* Set "/proc/self/oom_adj" for slurmd and slurmstepd daemons based upon
+ the values of SLURMD_OOM_ADJ and SLURMSTEPD_OOM_ADJ environment
+ variables. This can be used to prevent daemons being killed when
+ a node's memory is exhausted.
diff --git a/src/slurmd/common/set_oomadj.c b/src/slurmd/common/set_oomadj.c
index 9db9d51f74b..26d0a87cdc4 100644
--- a/src/slurmd/common/set_oomadj.c
+++ b/src/slurmd/common/set_oomadj.c
@@ -46,7 +46,6 @@
extern int set_oom_adj(int adj)
{
-#if 0
int fd;
char oom_adj[16];
@@ -55,7 +54,7 @@ extern int set_oom_adj(int adj)
if (errno == ENOENT)
debug("failed to open /proc/self/oom_adj: %m");
else
- verbose("failed to open /proc/self/oom_adj: %m");
+ error("failed to open /proc/self/oom_adj: %m");
return -1;
}
if (snprintf(oom_adj, 16, "%d", adj) >= 16) {
@@ -64,7 +63,7 @@ extern int set_oom_adj(int adj)
while ((write(fd, oom_adj, strlen(oom_adj)) < 0) && (errno == EINTR))
;
close(fd);
-#endif
+
return 0;
}
diff --git a/src/slurmd/slurmd/slurmd.c b/src/slurmd/slurmd/slurmd.c
index ac82a966e24..b499bee28ff 100644
--- a/src/slurmd/slurmd/slurmd.c
+++ b/src/slurmd/slurmd/slurmd.c
@@ -153,6 +153,7 @@ main (int argc, char *argv[])
{
int i, pidfd;
int blocked_signals[] = {SIGPIPE, 0};
+ char *oom_value;
/*
* Make sure we have no extra open files which
@@ -201,7 +202,11 @@ main (int argc, char *argv[])
info("slurmd version %s started", SLURM_VERSION);
debug3("finished daemonize");
- set_oom_adj(OOM_DISABLE);
+ if ((oom_value = getenv("SLURMD_OOM_ADJ"))) {
+ i = atoi(oom_value);
+ debug("Setting slurmd oom_adj to %d", i);
+ set_oom_adj(i);
+ }
_kill_old_slurmd();
diff --git a/src/slurmd/slurmstepd/mgr.c b/src/slurmd/slurmstepd/mgr.c
index 9ea0b62d733..8f27e440f94 100644
--- a/src/slurmd/slurmstepd/mgr.c
+++ b/src/slurmd/slurmstepd/mgr.c
@@ -877,6 +877,7 @@ _fork_all_tasks(slurmd_job_t *job)
int fdpair[2];
struct priv_state sprivs;
jobacct_id_t jobacct_id;
+ char *oom_value;
xassert(job != NULL);
@@ -929,6 +930,9 @@ _fork_all_tasks(slurmd_job_t *job)
writefds[i] = fdpair[1];
}
+ error("setting user oom to zero");
+ set_oom_adj(0); /* the tasks may be killed by OOM */
+
/* Temporarily drop effective privileges, except for the euid.
* We need to wait until after pam_setup() to drop euid.
*/
@@ -971,8 +975,6 @@ _fork_all_tasks(slurmd_job_t *job)
} else if (pid == 0) { /* child */
int j;
- set_oom_adj(0); /* the tasks may be killed by OOM */
-
#ifdef HAVE_AIX
(void) mkcrid(0);
#endif
@@ -1029,6 +1031,14 @@ _fork_all_tasks(slurmd_job_t *job)
/* Don't bother erroring out here */
}
+ if ((oom_value = getenv("SLURMSTEPD_OOM_ADJ"))) {
+ int i = atoi(oom_value);
+ debug("Setting slurmstepd oom_adj to %d", i);
+ set_oom_adj(i);
+ } else
+ error("NO SLURMSTEPD_OOM_ADJ");
+
+
if (chdir (sprivs.saved_cwd) < 0) {
error ("Unable to return to working directory");
}
diff --git a/src/slurmd/slurmstepd/slurmstepd.c b/src/slurmd/slurmstepd/slurmstepd.c
index 653e55f2858..5a870de8da4 100644
--- a/src/slurmd/slurmstepd/slurmstepd.c
+++ b/src/slurmd/slurmstepd/slurmstepd.c
@@ -58,7 +58,6 @@
#include "src/slurmd/common/slurmstepd_init.h"
#include "src/slurmd/common/setproctitle.h"
#include "src/slurmd/common/proctrack.h"
-#include "src/slurmd/common/set_oomadj.h"
#include "src/slurmd/slurmstepd/slurmstepd.h"
#include "src/slurmd/slurmstepd/mgr.h"
#include "src/slurmd/slurmstepd/req.h"
@@ -102,8 +101,6 @@ main (int argc, char *argv[])
exit(0);
}
- set_oom_adj(OOM_DISABLE);
-
xsignal_block(slurmstepd_blocked_signals);
conf = xmalloc(sizeof(*conf));
conf->argv = &argv;
--
GitLab