lmkd: Implement watchdog thread

To detect lmkd being stuck on a syscall for prolonged period of time,
introduce a watchdog thread which gets set when lmkd starts handling of
events and is reset after handling is done. If it takes more than the
timeout period (2 sec) to handle an event, watchdog wakes up and kills
the least important process to prevent mounting memory pressure caused
by lmkd lockup. After a kill, watchdog will wait for the reset for
another timeout period and kill again. This repeats until lmkd unlocks
and resets the watchdog.

Bug: 201671997
Test: induce random sleep in lmkd main handler and observe watchdog kills
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Change-Id: I56a55834582e11c06cc6cf9da3bc7380e634b301

Android.bp

@@ -15,6 +15,7 @@ cc_binary {
     srcs: [
         "lmkd.cpp",
         "reaper.cpp",
+        "watchdog.cpp",
     ],
     shared_libs: [
         "libcutils",

lmkd.cpp

@@ -34,6 +34,8 @@
 #include <time.h>
 #include <unistd.h>
 
+#include <shared_mutex>
+
 #include <cutils/properties.h>
 #include <cutils/sockets.h>
 #include <liblmkd_utils.h>
@@ -46,6 +48,7 @@
 
 #include "reaper.h"
 #include "statslog.h"
+#include "watchdog.h"
 
 #define BPF_FD_JUST_USE_INT
 #include "BpfSyscallWrappers.h"
@@ -161,6 +164,8 @@ static inline void trace_kill_end() {}
 
 #define LMKD_REINIT_PROP "lmkd.reinit"
 
+#define WATCHDOG_TIMEOUT_SEC 2
+
 /* default to old in-kernel interface if no memory pressure events */
 static bool use_inkernel_interface = true;
 static bool has_inkernel_module;
@@ -523,6 +528,11 @@ static struct proc *pidhash[PIDHASH_SZ];
 
 #define ADJTOSLOT(adj) ((adj) + -OOM_SCORE_ADJ_MIN)
 #define ADJTOSLOT_COUNT (ADJTOSLOT(OOM_SCORE_ADJ_MAX) + 1)
+
+// protects procadjslot_list from concurrent access
+static std::shared_mutex adjslot_list_lock;
+// procadjslot_list should be modified only from the main thread while exclusively holding
+// adjslot_list_lock. Readers from non-main threads should hold adjslot_list_lock shared lock.
 static struct adjslot_list procadjslot_list[ADJTOSLOT_COUNT];
 
 #define MAX_DISTINCT_OOM_ADJ 32
@@ -912,13 +922,18 @@ static struct adjslot_list *adjslot_tail(struct adjslot_list *head) {
     return asl == head ? NULL : asl;
 }
 
+// Should be modified only from the main thread.
 static void proc_slot(struct proc *procp) {
     int adjslot = ADJTOSLOT(procp->oomadj);
+    std::scoped_lock lock(adjslot_list_lock);
 
     adjslot_insert(&procadjslot_list[adjslot], &procp->asl);
 }
 
+// Should be modified only from the main thread.
 static void proc_unslot(struct proc *procp) {
+    std::scoped_lock lock(adjslot_list_lock);
+
     adjslot_remove(&procp->asl);
 }
 
@@ -1955,16 +1970,24 @@ static void killinfo_log(struct proc* procp, int min_oom_score, int rss_kb,
 
     /* log meminfo fields */
     for (int field_idx = 0; field_idx < MI_FIELD_COUNT; field_idx++) {
-        android_log_write_int32(ctx, (int32_t)min(mi->arr[field_idx] * page_k, INT32_MAX));
+        android_log_write_int32(ctx, mi ? (int32_t)min(mi->arr[field_idx] * page_k, INT32_MAX): 0);
     }
 
     /* log lmkd wakeup information */
+    if (wi) {
         android_log_write_int32(ctx, (int32_t)get_time_diff_ms(&wi->last_event_tm, tm));
         android_log_write_int32(ctx, (int32_t)get_time_diff_ms(&wi->prev_wakeup_tm, tm));
         android_log_write_int32(ctx, wi->wakeups_since_event);
         android_log_write_int32(ctx, wi->skipped_wakeups);
+    } else {
+        android_log_write_int32(ctx, 0);
+        android_log_write_int32(ctx, 0);
+        android_log_write_int32(ctx, 0);
+        android_log_write_int32(ctx, 0);
+    }
+
     android_log_write_int32(ctx, (int32_t)min(swap_kb, INT32_MAX));
-    android_log_write_int32(ctx, (int32_t)mi->field.total_gpu_kb);
+    android_log_write_int32(ctx, mi ? (int32_t)mi->field.total_gpu_kb : 0);
     if (ki) {
         android_log_write_int32(ctx, ki->thrashing);
         android_log_write_int32(ctx, ki->max_thrashing);
@@ -1977,10 +2000,33 @@ static void killinfo_log(struct proc* procp, int min_oom_score, int rss_kb,
     android_log_reset(ctx);
 }
 
-static struct proc *proc_adj_lru(int oomadj) {
+// Note: returned entry is only an anchor and does not hold a valid process info.
+// When called from a non-main thread, adjslot_list_lock read lock should be taken.
+static struct proc *proc_adj_head(int oomadj) {
+    return (struct proc *)&procadjslot_list[ADJTOSLOT(oomadj)];
+}
+
+// When called from a non-main thread, adjslot_list_lock read lock should be taken.
+static struct proc *proc_adj_tail(int oomadj) {
     return (struct proc *)adjslot_tail(&procadjslot_list[ADJTOSLOT(oomadj)]);
 }
 
+// When called from a non-main thread, adjslot_list_lock read lock should be taken.
+static struct proc *proc_adj_prev(int oomadj, int pid) {
+    struct adjslot_list *head = &procadjslot_list[ADJTOSLOT(oomadj)];
+    struct adjslot_list *curr = adjslot_tail(&procadjslot_list[ADJTOSLOT(oomadj)]);
+
+    while (curr != head) {
+        if (((struct proc *)curr)->pid == pid) {
+            return (struct proc *)curr->prev;
+        }
+        curr = curr->prev;
+    }
+
+    return NULL;
+}
+
+// When called from a non-main thread, adjslot_list_lock read lock should be taken.
 static struct proc *proc_get_heaviest(int oomadj) {
     struct adjslot_list *head = &procadjslot_list[ADJTOSLOT(oomadj)];
     struct adjslot_list *curr = head->next;
@@ -2004,6 +2050,55 @@ static struct proc *proc_get_heaviest(int oomadj) {
     return maxprocp;
 }
 
+static bool find_victim(int oom_score, int prev_pid, struct proc &target_proc) {
+    struct proc *procp;
+    std::shared_lock lock(adjslot_list_lock);
+
+    if (!prev_pid) {
+        procp = proc_adj_tail(oom_score);
+    } else {
+        procp = proc_adj_prev(oom_score, prev_pid);
+        if (!procp) {
+            // pid was removed, restart at the tail
+            procp = proc_adj_tail(oom_score);
+        }
+    }
+
+    // the list is empty at this oom_score or we looped through it
+    if (!procp || procp == proc_adj_head(oom_score)) {
+        return false;
+    }
+
+    // make a copy because original might be destroyed after adjslot_list_lock is released
+    target_proc = *procp;
+
+    return true;
+}
+
+static void watchdog_callback() {
+    int prev_pid = 0;
+
+    ALOGW("lmkd watchdog timed out!");
+    for (int oom_score = OOM_SCORE_ADJ_MAX; oom_score >= 0;) {
+        struct proc target;
+
+        if (!find_victim(oom_score, prev_pid, target)) {
+            oom_score--;
+            prev_pid = 0;
+            continue;
+        }
+
+        if (reaper.kill({ target.pidfd, target.pid }, true) == 0) {
+            ALOGW("lmkd watchdog killed process %d, oom_score_adj %d", target.pid, oom_score);
+            killinfo_log(&target, 0, 0, 0, NULL, NULL, NULL, NULL);
+            break;
+        }
+        prev_pid = target.pid;
+    }
+}
+
+static Watchdog watchdog(WATCHDOG_TIMEOUT_SEC, watchdog_callback);
+
 static bool is_kill_pending(void) {
     char buf[24];
 
@@ -2165,7 +2260,7 @@ static int kill_one_process(struct proc* procp, int min_oom_score, struct kill_i
     trace_kill_start(pid, desc);
 
     start_wait_for_proc_kill(pidfd < 0 ? pid : pidfd);
-    kill_result = reaper.kill({ pidfd, pid });
+    kill_result = reaper.kill({ pidfd, pid }, false);
 
     trace_kill_end();
 
@@ -2241,7 +2336,7 @@ static int find_and_kill_process(int min_score_adj, struct kill_info *ki, union
 
         while (true) {
             procp = choose_heaviest_task ?
-                proc_get_heaviest(i) : proc_adj_lru(i);
+                proc_get_heaviest(i) : proc_adj_tail(i);
 
             if (!procp)
                 break;
@@ -3297,6 +3392,7 @@ static void call_handler(struct event_handler_info* handler_info,
                          struct polling_params *poll_params, uint32_t events) {
     struct timespec curr_tm;
 
+    watchdog.start();
     poll_params->update = POLLING_DO_NOT_CHANGE;
     handler_info->handler(handler_info->data, events, poll_params);
     clock_gettime(CLOCK_MONOTONIC_COARSE, &curr_tm);
@@ -3328,6 +3424,7 @@ static void call_handler(struct event_handler_info* handler_info,
         }
         break;
     }
+    watchdog.stop();
 }
 
 static void mainloop(void) {
@@ -3409,7 +3506,9 @@ static void mainloop(void) {
             if ((evt->events & EPOLLHUP) && evt->data.ptr) {
                 ALOGI("lmkd data connection dropped");
                 handler_info = (struct event_handler_info*)evt->data.ptr;
+                watchdog.start();
                 ctrl_data_close(handler_info->data);
+                watchdog.stop();
             }
         }
 
@@ -3550,6 +3649,10 @@ int main(int argc, char **argv) {
                 reaper.thread_cnt());
         }
 
+        if (!watchdog.init()) {
+            ALOGE("Failed to initialize the watchdog");
+        }
+
         mainloop();
     }
 

reaper.cpp

@@ -206,18 +206,23 @@ bool Reaper::async_kill(const struct target_proc& target) {
     return true;
 }
 
-int Reaper::kill(const struct target_proc& target) {
+int Reaper::kill(const struct target_proc& target, bool synchronous) {
     /* CAP_KILL required */
     if (target.pidfd < 0) {
         return ::kill(target.pid, SIGKILL);
     }
 
-    if (async_kill(target)) {
+    if (!synchronous && async_kill(target)) {
         // we assume the kill will be successful and if it fails we will be notified
         return 0;
     }
 
-    return pidfd_send_signal(target.pidfd, SIGKILL, NULL, 0);
+    int result = pidfd_send_signal(target.pidfd, SIGKILL, NULL, 0);
+    if (result) {
+        return result;
+    }
+
+    return is_reaping_supported() ? process_mrelease(target.pidfd, 0) : 0;
 }
 
 Reaper::target_proc Reaper::dequeue_request() {

reaper.h

@@ -51,7 +51,7 @@ public:
     bool debug_enabled() const { return debug_enabled_; }
 
     // return 0 on success or error code returned by the syscall
-    int kill(const struct target_proc& target);
+    int kill(const struct target_proc& target, bool synchronous);
     // below members are used only by reaper_main
     target_proc dequeue_request();
     void request_complete();

watchdog.cpp

@@ -0,0 +1,123 @@
+/*
+ *  Copyright 2021 Google, Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#define LOG_TAG "lowmemorykiller"
+
+#include <errno.h>
+#include <log/log.h>
+#include <string.h>
+
+#include <processgroup/processgroup.h>
+
+#include "watchdog.h"
+
+static void* watchdog_main(void* param) {
+    Watchdog *watchdog = static_cast<Watchdog*>(param);
+    sigset_t sigset;
+    int signum;
+
+    // Ensure the thread does not use little cores
+    if (!SetTaskProfiles(gettid(), {"CPUSET_SP_FOREGROUND"}, true)) {
+        ALOGE("Failed to assign cpuset to the watchdog thread");
+    }
+
+    if (!watchdog->create_timer(sigset)) {
+        ALOGE("Watchdog timer creation failed!");
+        return NULL;
+    }
+
+    while (true) {
+        if (sigwait(&sigset, &signum) == -1) {
+            ALOGE("sigwait failed: %s", strerror(errno));
+        }
+
+        watchdog->bite();
+    }
+
+    return NULL;
+}
+
+bool Watchdog::init() {
+    pthread_t thread;
+
+    if (pthread_create(&thread, NULL, watchdog_main, this)) {
+        ALOGE("pthread_create failed: %s", strerror(errno));
+        return false;
+    }
+    if (pthread_setname_np(thread, "lmkd_watchdog")) {
+        ALOGW("pthread_setname_np failed: %s", strerror(errno));
+    }
+
+    return true;
+}
+
+bool Watchdog::start() {
+    // Start the timer and keep it active until it's disarmed
+    struct itimerspec new_timer;
+
+    if (!timer_created_) {
+        return false;
+    }
+
+    new_timer.it_value.tv_sec = timeout_;
+    new_timer.it_value.tv_nsec = 0;
+    new_timer.it_interval.tv_sec = timeout_;
+    new_timer.it_interval.tv_nsec = 0;
+
+    if (timer_settime(timer_, 0, &new_timer, NULL)) {
+        ALOGE("timer_settime failed: %s", strerror(errno));
+        return false;
+    }
+
+    return true;
+}
+
+bool Watchdog::stop() {
+    struct itimerspec new_timer = {};
+
+    if (!timer_created_) {
+        return false;
+    }
+
+    if (timer_settime(timer_, 0, &new_timer, NULL)) {
+        ALOGE("timer_settime failed: %s", strerror(errno));
+        return false;
+    }
+
+    return true;
+}
+
+bool Watchdog::create_timer(sigset_t &sigset) {
+    struct sigevent sevent;
+
+    sigemptyset(&sigset);
+    sigaddset(&sigset, SIGALRM);
+    if (sigprocmask(SIG_BLOCK, &sigset, NULL)) {
+        ALOGE("sigprocmask failed: %s", strerror(errno));
+        return false;
+    }
+
+    sevent.sigev_notify = SIGEV_THREAD_ID;
+    sevent.sigev_notify_thread_id = gettid();
+    sevent.sigev_signo = SIGALRM;
+    if (timer_create(CLOCK_MONOTONIC, &sevent, &timer_)) {
+        ALOGE("timer_create failed: %s", strerror(errno));
+        return false;
+    }
+
+    timer_created_ = true;
+    return true;
+}

watchdog.h

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2021 Google, Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#pragma once
+
+#include <atomic>
+#include <signal.h>
+#include <time.h>
+
+class Watchdog {
+private:
+    int timeout_;
+    timer_t timer_;
+    std::atomic<bool> timer_created_;
+    void (*callback_)();
+public:
+    Watchdog(int timeout, void (*callback)()) :
+        timeout_(timeout), timer_created_(false), callback_(callback) {}
+
+    bool init();
+    bool start();
+    bool stop();
+    // used by the watchdog_main
+    bool create_timer(sigset_t &sigset);
+    void bite() const { if (callback_) callback_(); }
+};