From 9e136285a6a9aee5b2aa6cfb285456b533561ca8 Mon Sep 17 00:00:00 2001 From: Carlos Galo Date: Tue, 12 Mar 2024 21:18:12 +0000 Subject: [PATCH] Adding direct reclaim state monitoring Replacing mechanism of reading vmstats to detect direct reclaim with memevents instead. Maintain vmstats mechanism if bpf ring buffer is not supported by current kernel. Test: Verified lmkd receives direct reclaim state changes Test: m Bug: 244232958 Change-Id: I59ee7657da1240355d611dfa129c4d50bed2c330 Signed-off-by: Carlos Galo --- Android.bp | 25 ++++++----- lmkd.cpp | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 12 deletions(-) diff --git a/Android.bp b/Android.bp index e4f5e97..9d70642 100644 --- a/Android.bp +++ b/Android.bp @@ -19,19 +19,19 @@ lmkd_hooks_cc_defaults { soong_config_variables: { use_hooks: { cflags: [ - "-DLMKD_USE_HOOKS" + "-DLMKD_USE_HOOKS", ], static_libs: [ - "liblmkdhooks" - ] - } - } + "liblmkdhooks", + ], + }, + }, } cc_defaults { name: "stats_defaults", cflags: [ - "-DLMKD_LOG_STATS" + "-DLMKD_LOG_STATS", ], } @@ -44,8 +44,10 @@ cc_binary { "watchdog.cpp", ], shared_libs: [ + "libbase", "libcutils", "liblog", + "libmemevents", "libprocessgroup", "libpsi", ], @@ -54,17 +56,20 @@ cc_binary { "liblmkd_utils", ], header_libs: [ - "bpf_syscall_wrappers", + "bpf_headers", ], local_include_dirs: ["include"], cflags: [ "-Wall", "-Werror", "-Wextra", - "-DLMKD_TRACE_KILLS" + "-DLMKD_TRACE_KILLS", ], init_rc: ["lmkd.rc"], - defaults: ["stats_defaults", "lmkd_hooks_defaults"], + defaults: [ + "stats_defaults", + "lmkd_hooks_defaults", + ], logtags: ["event.logtags"], afdo: true, } @@ -98,5 +103,5 @@ cc_library_static { "-g", "-Wall", "-Werror", - ] + ], } diff --git a/lmkd.cpp b/lmkd.cpp index 767f5d4..ded3510 100644 --- a/lmkd.cpp +++ b/lmkd.cpp @@ -36,8 +36,12 @@ #include #include +#include #include +#include +#include +#include #include #include #include @@ -46,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -189,6 +194,10 @@ struct psi_threshold { int threshold_ms; }; +/* Listener for direct reclaim state changes */ +static std::unique_ptr memevent_listener(nullptr); +static struct timespec direct_reclaim_start_tm; + static int level_oomadj[VMPRESS_LEVEL_COUNT]; static int mpevfd[VMPRESS_LEVEL_COUNT] = { -1, -1, -1 }; static bool pidfd_supported; @@ -278,8 +287,9 @@ static struct event_handler_info vmpressure_hinfo[VMPRESS_LEVEL_COUNT]; /* * 1 ctrl listen socket, 3 ctrl data socket, 3 memory pressure levels, * 1 lmk events + 1 fd to wait for process death + 1 fd to receive kill failure notifications + * + 1 fd to receive direct reclaim state change notifications */ -#define MAX_EPOLL_EVENTS (1 + MAX_DATA_CONN + VMPRESS_LEVEL_COUNT + 1 + 1 + 1) +#define MAX_EPOLL_EVENTS (1 + MAX_DATA_CONN + VMPRESS_LEVEL_COUNT + 1 + 1 + 1 + 1) static int epollfd; static int maxevents; @@ -2620,6 +2630,7 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_ long since_thrashing_reset_ms; int64_t workingset_refault_file; bool critical_stall = false; + bool in_direct_reclaim; if (clock_gettime(CLOCK_MONOTONIC_COARSE, &curr_tm) != 0) { ALOGE("Failed to get current time"); @@ -2672,8 +2683,12 @@ static void mp_event_psi(int data, uint32_t events, struct polling_params *poll_ swap_low_threshold = 0; } + in_direct_reclaim = memevent_listener ? (direct_reclaim_start_tm.tv_sec != 0 || + direct_reclaim_start_tm.tv_nsec != 0) + : (vs.field.pgscan_direct != init_pgscan_direct); + /* Identify reclaim state */ - if (vs.field.pgscan_direct != init_pgscan_direct) { + if (in_direct_reclaim) { init_pgscan_direct = vs.field.pgscan_direct; init_pgscan_kswapd = vs.field.pgscan_kswapd; init_pgrefill = vs.field.pgrefill; @@ -3232,6 +3247,103 @@ static MemcgVersion memcg_version() { return version; } +static void direct_reclaim_state_change(int data __unused, uint32_t events __unused, + struct polling_params* poll_params __unused) { + struct timespec curr_tm; + std::vector mem_events; + + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &curr_tm) != 0) { + direct_reclaim_start_tm.tv_sec = 0; + direct_reclaim_start_tm.tv_nsec = 0; + ALOGE("Failed to get current time for direct reclaim state change."); + return; + } + + if (!memevent_listener->getMemEvents(mem_events)) { + direct_reclaim_start_tm.tv_sec = 0; + direct_reclaim_start_tm.tv_nsec = 0; + ALOGE("Failed fetching direct reclaim events."); + return; + } + + /* + * `mem_events` is ordered from oldest to newest, therefore we use + * the last/latest direct reclaim event as the current direct reclaim + * state. + */ + for (const mem_event_t mem_event : mem_events) { + if (mem_event.type == MEM_EVENT_DIRECT_RECLAIM_BEGIN) { + direct_reclaim_start_tm = curr_tm; + } else if (mem_event.type == MEM_EVENT_DIRECT_RECLAIM_END) { + direct_reclaim_start_tm.tv_sec = 0; + direct_reclaim_start_tm.tv_nsec = 0; + } + } +} + +static bool init_direct_reclaim_monitoring() { + static struct event_handler_info direct_reclaim_poll_hinfo = {0, direct_reclaim_state_change}; + + if (!memevent_listener) { + // Make sure bpf programs are loaded + android::bpf::waitForProgsLoaded(); + memevent_listener = std::make_unique( + android::bpf::memevents::MemEventClient::LMKD); + } + + if (!memevent_listener->ok()) { + ALOGE("Failed to initialize memevents listener"); + memevent_listener.reset(); + return false; + } + + if (!memevent_listener->registerEvent(MEM_EVENT_DIRECT_RECLAIM_BEGIN) || + !memevent_listener->registerEvent(MEM_EVENT_DIRECT_RECLAIM_END)) { + ALOGE("Failed to register direct reclaim memevents"); + memevent_listener.reset(); + return false; + } + + int memevent_listener_fd = memevent_listener->getRingBufferFd(); + if (memevent_listener_fd < 0) { + memevent_listener.reset(); + ALOGE("Invalid memevent_listener fd: %d", memevent_listener_fd); + return false; + } + + struct epoll_event epev; + epev.events = EPOLLIN; + epev.data.ptr = (void*)&direct_reclaim_poll_hinfo; + if (epoll_ctl(epollfd, EPOLL_CTL_ADD, memevent_listener_fd, &epev) < 0) { + ALOGE("Failed registering direct reclaim fd: %d; errno=%d", memevent_listener_fd, errno); + /* + * Reset the fd to let `destroy_direct_reclaim_monitoring` know we failed adding this fd, + * therefore it won't try to close the `memevent_listener_fd`. + */ + memevent_listener.reset(); + return false; + } + + direct_reclaim_start_tm.tv_sec = 0; + direct_reclaim_start_tm.tv_nsec = 0; + + maxevents++; + return true; +} + +static void destroy_direct_reclaim_monitoring() { + if (!memevent_listener) return; + + if (epoll_ctl(epollfd, EPOLL_CTL_DEL, memevent_listener->getRingBufferFd(), NULL) < 0) { + ALOGE("Failed to unregister direct reclaim monitoring; errno=%d", errno); + } + + maxevents--; + memevent_listener.reset(); + direct_reclaim_start_tm.tv_sec = 0; + direct_reclaim_start_tm.tv_nsec = 0; +} + static bool init_psi_monitors() { /* * When PSI is used on low-ram devices or on high-end devices without memfree levels @@ -3382,6 +3494,13 @@ static bool init_monitors() { } else { ALOGI("Using vmpressure for memory pressure detection"); } + + if (init_direct_reclaim_monitoring()) { + ALOGI("Using memevents for direct reclaim detection"); + } else { + ALOGI("Using vmstats for direct reclaim detection"); + } + monitors_initialized = true; return true; } @@ -3396,6 +3515,7 @@ static void destroy_monitors() { destroy_mp_common(VMPRESS_LEVEL_MEDIUM); destroy_mp_common(VMPRESS_LEVEL_LOW); } + destroy_direct_reclaim_monitoring(); } static void drop_reaper_comm() {