lmkd: Introduce update zoneinfo memevent

This change adds support for the zoneinfo memevent, allowing lmkd
to react to memory zone changes and update watermarks dynamically,
reducing the need for polling.

Polling is retained as a fallback for kernels without support.

Bug: 396115949
Test: build and verify on kernel with/without memevent supported
Change-Id: I7bc746a0e9dbdc48cfaf80c38c2ead5b5116d20b
Merged-In: I7bc746a0e9dbdc48cfaf80c38c2ead5b5116d20b
Signed-off-by: Martin Liu <liumartin@google.com>

Android.bp

@@ -88,6 +88,10 @@ cc_library_static {
         "liblog",
         "libprocessgroup",
     ],
+    header_libs: [
+        "libmemevents_headers",
+        "libbase_headers",
+    ],
 }
 
 cc_library_static {

README.md

@@ -60,14 +60,13 @@ properties:
                                  any eligible task (fast decision). Default = false
 
   - `ro.lmk.kill_timeout_ms`:    duration in ms after a kill when no additional
-                                 kill will be done. Default = 0 (disabled)
+                                 kill will be done. Default = 100
 
   - `ro.lmk.debug`:              enable lmkd debug logs, Default = false
 
   - `ro.lmk.swap_free_low_percentage`: level of free swap as a percentage of the
                                  total swap space used as a threshold to consider
-                                 the system as swap space starved. Default for
-                                 low-RAM devices = 10, for high-end devices = 20
+                                 the system as swap space starved. Default = 10
 
   - `ro.lmk.thrashing_limit`:    number of workingset refaults as a percentage of
                                 the file-backed pagecache size used as a threshold

lmkd.cpp

@@ -94,6 +94,7 @@ static inline void trace_kill_end() {}
 #define PROC_STATUS_TGID_FIELD "Tgid:"
 #define PROC_STATUS_RSS_FIELD "VmRSS:"
 #define PROC_STATUS_SWAP_FIELD "VmSwap:"
+#define NODE_STATS_MARKER "  per-node stats"
 
 #define PERCEPTIBLE_APP_ADJ 200
 #define PREVIOUS_APP_ADJ 700
@@ -210,8 +211,10 @@ static int mpevfd[VMPRESS_LEVEL_COUNT] = { -1, -1, -1 };
 static bool pidfd_supported;
 static int last_kill_pid_or_fd = -1;
 static struct timespec last_kill_tm;
+enum vmpressure_level prev_level = VMPRESS_LEVEL_LOW;
 static bool monitors_initialized;
 static bool boot_completed_handled = false;
+static bool mem_event_update_zoneinfo_supported;
 
 /* lmkd configurable parameters */
 static bool debug_process_killing;
@@ -245,6 +248,8 @@ static struct psi_threshold psi_thresholds[VMPRESS_LEVEL_COUNT] = {
     { PSI_FULL, 70 },    /* 70ms out of 1sec for complete stall */
 };
 
+static uint64_t mp_event_count;
+
 static android_log_context ctx;
 static Reaper reaper;
 static int reaper_comm_fd[2];
@@ -1859,6 +1864,15 @@ static int zoneinfo_parse(struct zoneinfo *zi) {
         int node_id;
         if (sscanf(line, "Node %d, zone %" STRINGIFY(LINE_MAX) "s", &node_id, zone_name) == 2) {
             if (!node || node->id != node_id) {
+                line = strtok_r(NULL, "\n", &save_ptr);
+                if (strncmp(line, NODE_STATS_MARKER, strlen(NODE_STATS_MARKER)) != 0) {
+                    /*
+                     * per-node stats are only present in the first non-empty zone of
+                     * the node.
+                     */
+                    continue;
+                }
+
                 /* new node is found */
                 if (node) {
                     node->zone_count = zone_idx + 1;
@@ -2220,6 +2234,10 @@ static struct proc *proc_get_heaviest(int oomadj) {
     struct adjslot_list *curr = head->next;
     struct proc *maxprocp = NULL;
     int maxsize = 0;
+    if ((curr != head) && (curr->next == head)) {
+        // Our list only has one process.  No need to access procfs for its size.
+        return (struct proc *)curr;
+    }
     while (curr != head) {
         int pid = ((struct proc *)curr)->pid;
         int tasksize = proc_get_size(pid);
@@ -2625,6 +2643,8 @@ struct zone_watermarks {
     long min_wmark;
 };
 
+static struct zone_watermarks watermarks;
+
 /*
  * Returns lowest breached watermark or WMARK_NONE.
  */
@@ -2664,6 +2684,15 @@ void calc_zone_watermarks(struct zoneinfo *zi, struct zone_watermarks *watermark
     }
 }
 
+static int update_zoneinfo_watermarks(struct zoneinfo *zi) {
+    if (zoneinfo_parse(zi) < 0) {
+        ALOGE("Failed to parse zoneinfo!");
+        return -1;
+    }
+    calc_zone_watermarks(zi, &watermarks);
+    return 0;
+}
+
 static int calc_swap_utilization(union meminfo *mi) {
     int64_t swap_used = mi->field.total_swap - get_free_swap(mi);
     int64_t total_swappable = mi->field.active_anon + mi->field.inactive_anon +
@@ -2671,7 +2700,18 @@ static int calc_swap_utilization(union meminfo *mi) {
     return total_swappable > 0 ? (swap_used * 100) / total_swappable : 0;
 }
 
-static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_params) {
+enum event_source {
+    PSI,
+    VENDOR,
+};
+
+union psi_event_data {
+    enum vmpressure_level level;
+    mem_event_t vendor_event;
+};
+
+static void __mp_event_psi(enum event_source source, union psi_event_data data,
+                           uint32_t events, struct polling_params *poll_params) {
     enum reclaim_state {
         NO_RECLAIM = 0,
         KSWAPD_RECLAIM,
@@ -2685,7 +2725,6 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_
     static int64_t init_pgrefill;
     static bool killing;
     static int thrashing_limit = thrashing_limit_pct;
-    static struct zone_watermarks watermarks;
     static struct timespec wmark_update_tm;
     static struct wakeup_info wi;
     static struct timespec thrashing_reset_tm;
@@ -2699,7 +2738,7 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_
     struct timespec curr_tm;
     int64_t thrashing = 0;
     bool swap_is_low = false;
-    enum vmpressure_level level = (enum vmpressure_level)data;
+    enum vmpressure_level level = (source == PSI) ? data.level: (enum vmpressure_level)0;
     enum kill_reasons kill_reason = NONE;
     bool cycle_after_kill = false;
     enum reclaim_state reclaim = NO_RECLAIM;
@@ -2716,12 +2755,37 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_
     long direct_reclaim_duration_ms;
     bool in_kswapd_reclaim;
 
+    mp_event_count++;
+    if (debug_process_killing) {
+        if (source == PSI)
+            ALOGI("%s memory pressure event #%" PRIu64 " is triggered",
+                  level_name[level], mp_event_count);
+        else
+            ALOGI("vendor kill event #%" PRIu64 " is triggered", mp_event_count);
+    }
+
     if (clock_gettime(CLOCK_MONOTONIC_COARSE, &curr_tm) != 0) {
         ALOGE("Failed to get current time");
         return;
     }
 
+    if (source == PSI) {
+        if (events > 0 ) {
+            /* Ignore a lower event within the first polling window. */
+            if (level < prev_level) {
+                if (debug_process_killing)
+                    ALOGI("Ignoring %s pressure event; occurred too soon.",
+                           level_name[level]);
+                return;
+            }
+            prev_level = level;
+        } else {
+            /* Reset event level after the first polling window. */
+            prev_level = VMPRESS_LEVEL_LOW;
+        }
+
         record_wakeup_time(&curr_tm, events ? Event : Polling, &wi);
+    }
 
     bool kill_pending = is_kill_pending();
     if (kill_pending && (kill_timeout_ms == 0 ||
@@ -2788,7 +2852,8 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_
         init_pgscan_kswapd = vs.field.pgscan_kswapd;
         init_pgrefill = vs.field.pgrefill;
         reclaim = KSWAPD_RECLAIM;
-    } else if (workingset_refault_file == prev_workingset_refault) {
+    } else if ((workingset_refault_file == prev_workingset_refault) &&
+                (source == PSI)) {
         /*
          * Device is not thrashing and not reclaiming, bail out early until we see these stats
          * changing
@@ -2842,19 +2907,18 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_
 
 update_watermarks:
     /*
-     * Refresh watermarks once per min in case user updated one of the margins.
-     * TODO: b/140521024 replace this periodic update with an API for AMS to notify LMKD
-     * that zone watermarks were changed by the system software.
+     * Refresh watermarks:
+     * 1. watermarks haven't been initialized (high_wmark == 0)
+     * 2. per min in case user updated one of the margins if mem_event update_zoneinfo is NOT
+     *    supported.
      */
-    if (watermarks.high_wmark == 0 || get_time_diff_ms(&wmark_update_tm, &curr_tm) > 60000) {
+    if (watermarks.high_wmark == 0 || (!mem_event_update_zoneinfo_supported &&
+        get_time_diff_ms(&wmark_update_tm, &curr_tm) > 60000)) {
         struct zoneinfo zi;
 
-        if (zoneinfo_parse(&zi) < 0) {
-            ALOGE("Failed to parse zoneinfo!");
+        if (update_zoneinfo_watermarks(&zi) < 0) {
             return;
         }
-
-        calc_zone_watermarks(&zi, &watermarks);
         wmark_update_tm = curr_tm;
     }
 
@@ -2868,7 +2932,23 @@ update_watermarks:
      * TODO: move this logic into a separate function
      * Decide if killing a process is necessary and record the reason
      */
-    if (cycle_after_kill && wmark < WMARK_LOW) {
+    if (source == VENDOR) {
+        int vendor_kill_reason = data.vendor_event.event_data.vendor_kill.reason;
+        short vendor_kill_min_oom_score_adj =
+            data.vendor_event.event_data.vendor_kill.min_oom_score_adj;
+        if (vendor_kill_reason < 0 ||
+            vendor_kill_reason > VENDOR_KILL_REASON_END ||
+            vendor_kill_min_oom_score_adj < 0) {
+            ALOGE("Invalid vendor kill reason %d, min_oom_score_adj %d",
+                  vendor_kill_reason, vendor_kill_min_oom_score_adj);
+            return;
+        }
+
+        kill_reason = (enum kill_reasons)(vendor_kill_reason + VENDOR_KILL_REASON_BASE);
+        min_score_adj = vendor_kill_min_oom_score_adj;
+        snprintf(kill_desc, sizeof(kill_desc),
+            "vendor kill with the reason %d, min_score_adj %d", kill_reason, min_score_adj);
+    } else if (cycle_after_kill && wmark < WMARK_LOW) {
         /*
          * Prevent kills not freeing enough memory which might lead to OOM kill.
          * This might happen when a process is consuming memory faster than reclaim can
@@ -2877,6 +2957,7 @@ update_watermarks:
         min_score_adj = pressure_after_kill_min_score;
         kill_reason = PRESSURE_AFTER_KILL;
         strncpy(kill_desc, "min watermark is breached even after kill", sizeof(kill_desc));
+        kill_desc[sizeof(kill_desc) - 1] = '\0';
     } else if (level == VMPRESS_LEVEL_CRITICAL && events != 0) {
         /*
          * Device is too busy reclaiming memory which might lead to ANR.
@@ -2885,6 +2966,7 @@ update_watermarks:
          */
         kill_reason = NOT_RESPONDING;
         strncpy(kill_desc, "device is not responding", sizeof(kill_desc));
+        kill_desc[sizeof(kill_desc) - 1] = '\0';
     } else if (swap_is_low && thrashing > thrashing_limit_pct) {
         /* Page cache is thrashing while swap is low */
         kill_reason = LOW_SWAP_AND_THRASHING;
@@ -3032,6 +3114,11 @@ no_kill:
     }
 }
 
+static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_params) {
+    union psi_event_data event_data = {.level = (enum vmpressure_level)data};
+    __mp_event_psi(PSI, event_data, events, poll_params);
+}
+
 static std::string GetCgroupAttributePath(const char* attr) {
     std::string path;
     if (!CgroupGetAttributePath(attr, &path)) {
@@ -3066,8 +3153,10 @@ static void mp_event_common(int data, uint32_t events, struct polling_params *po
     };
     static struct wakeup_info wi;
 
+    mp_event_count++;
     if (debug_process_killing) {
-        ALOGI("%s memory pressure event is triggered", level_name[level]);
+        ALOGI("%s memory pressure event #%" PRIu64 " is triggered",
+              level_name[level], mp_event_count);
     }
 
     if (!use_psi_monitors) {
@@ -3354,7 +3443,7 @@ static MemcgVersion memcg_version() {
 }
 
 static void memevent_listener_notification(int data __unused, uint32_t events __unused,
-                                           struct polling_params* poll_params __unused) {
+                                           struct polling_params* poll_params) {
     struct timespec curr_tm;
     std::vector<mem_event_t> mem_events;
 
@@ -3391,6 +3480,16 @@ static void memevent_listener_notification(int data __unused, uint32_t events __
                 kswapd_start_tm.tv_sec = 0;
                 kswapd_start_tm.tv_nsec = 0;
                 break;
+            case MEM_EVENT_VENDOR_LMK_KILL: {
+                union psi_event_data event_data = {.vendor_event = mem_event};
+                 __mp_event_psi(VENDOR, event_data, 0, poll_params);
+                break;
+            }
+            case MEM_EVENT_UPDATE_ZONEINFO: {
+                struct zoneinfo zi;
+                update_zoneinfo_watermarks(&zi);
+                break;
+            }
         }
     }
 }
@@ -3425,6 +3524,17 @@ static bool init_memevent_listener_monitoring() {
         return false;
     }
 
+    if (!memevent_listener->registerEvent(MEM_EVENT_VENDOR_LMK_KILL)) {
+        ALOGI("Failed to register android_vendor_kill memevents");
+    }
+
+    if (!memevent_listener->registerEvent(MEM_EVENT_UPDATE_ZONEINFO)) {
+        mem_event_update_zoneinfo_supported = false;
+        ALOGI("update_zoneinfo memevents are not supported");
+    } else {
+        mem_event_update_zoneinfo_supported = true;
+    }
+
     int memevent_listener_fd = memevent_listener->getRingBufferFd();
     if (memevent_listener_fd < 0) {
         memevent_listener.reset();
@@ -3582,6 +3692,8 @@ static void kernel_event_handler(int data __unused, uint32_t events __unused,
 }
 
 static bool init_monitors() {
+    ALOGI("Wakeup counter is reset from %" PRIu64 " to 0", mp_event_count);
+    mp_event_count = 0;
     /* Try to use psi monitor first if kernel has it */
     use_psi_monitors = GET_LMK_PROPERTY(bool, "use_psi", true) &&
         init_psi_monitors();
@@ -3815,6 +3927,7 @@ static void call_handler(struct event_handler_info* handler_info,
          */
         poll_params->poll_start_tm = curr_tm;
         poll_params->poll_handler = handler_info;
+        poll_params->last_poll_tm = curr_tm;
         break;
     case POLLING_PAUSE:
         poll_params->paused_handler = handler_info;
@@ -3911,13 +4024,17 @@ static void mainloop(void) {
          */
         for (i = 0, evt = &events[0]; i < nevents; ++i, evt++) {
             if ((evt->events & EPOLLHUP) && evt->data.ptr) {
-                ALOGI("lmkd data connection dropped");
                 handler_info = (struct event_handler_info*)evt->data.ptr;
+                if (handler_info->handler == kill_done_handler) {
+                    call_handler(handler_info, &poll_params, evt->events);
+                } else {
+                    ALOGI("lmkd data connection dropped");
                     watchdog.start();
                     ctrl_data_close(handler_info->data);
                     watchdog.stop();
                 }
             }
+        }
 
         /* Second pass to handle all other events */
         for (i = 0, evt = &events[0]; i < nevents; ++i, evt++) {

statslog.cpp

@@ -158,13 +158,12 @@ struct memory_stat *stats_read_memory_stat(bool per_app_memcg, int pid, uid_t ui
         if (memory_stat_from_cgroup(&mem_st, pid, uid) == 0) {
             return &mem_st;
         }
-    } else {
+    }
     if (memory_stat_from_procfs(&mem_st, pid) == 0) {
         mem_st.rss_in_bytes = rss_bytes;
         mem_st.swap_in_bytes = swap_bytes;
         return &mem_st;
     }
-    }
 
     return NULL;
 }

statslog.h

@@ -26,6 +26,7 @@
 #include <sys/types.h>
 
 #include <cutils/properties.h>
+#include <memevents/memevents.h>
 
 __BEGIN_DECLS
 
@@ -66,6 +67,9 @@ enum kill_reasons {
     LOW_FILECACHE_AFTER_THRASHING,
     LOW_MEM,
     DIRECT_RECL_STUCK,
+    /* reserve aosp kill 0 ~ 999 */
+    VENDOR_KILL_REASON_BASE = 1000,
+    VENDOR_KILL_REASON_END = VENDOR_KILL_REASON_BASE + NUM_VENDOR_LMK_KILL_REASON - 1,
     KILL_REASON_COUNT
 };
 

tests/lmkd_tests.cpp

@@ -30,10 +30,6 @@
 
 using namespace android::base;
 
-#ifndef __NR_process_mrelease
-#define __NR_process_mrelease 448
-#endif
-
 #define INKERNEL_MINFREE_PATH "/sys/module/lowmemorykiller/parameters/minfree"
 
 #define LMKD_LOGCAT_MARKER "lowmemorykiller"