初始化cgroups
task_profiles
task_profiles.json文件中记录了两种类型数据 -- Attributes和profiles。Attributes的格式为:
{
"Name": "LowCapacityCPUs",
"Controller": "cpuset",
"File": "background/cpus"
},
Name指当前的Attribute的名字,Controller是指所归属于哪个cgroups的subsystem,File是指该controller的VFS目录下的子节点路径。
profiles的格式如下:
{
"Name": "HighEnergySaving",
"Actions": [
{
"Name": "JoinCgroup",
"Params":
{
"Controller": "schedtune",
"Path": "background"
}
}
]
},
{
"Name": "TimerSlackHigh",
"Actions": [
{
"Name": "SetTimerSlack",
"Params":
{
"Slack": "40000000"
}
}
]
},
{
"Name": "LowMemoryUsage",
"Actions": [
{
"Name": "SetAttribute",
"Params":
{
"Name": "MemSoftLimit",
"Value": "16MB"
}
},
{
"Name": "SetAttribute",
"Params":
{
"Name": "MemSwappiness",
"Value": "150"
}
}
]
},
{
"Name": "PerfBoost",
"Actions": [
{
"Name": "SetClamps",
"Params":
{
"Boost": "50%",
"Clamp": "0"
}
}
]
},
每个profile中都有一个name,然后剩下的字段就是Actions,每个profile可以有多个Actions,profile支持4种Action -- JoinCgroup、SetTimerSlack、SetAttribute以及SetClamps。
JoinCgroup
JoinCgroup只有两个参数 -- controller和Path,Controller仍是指cgroups的subsystem,path则是指该subsystem下的路径,也就是子group。这个Action的含义很明显,就是将设置成这个profile的process加入到该subsystem的子group中,受这个group的资源限制。
而在源码中,其对应的处理方法是SetCgroupAction类。
bool SetCgroupAction::AddTidToCgroup(int tid, int fd) {
if (tid <= 0) {
return true;
}
std::string value = std::to_string(tid);
if (TEMP_FAILURE_RETRY(write(fd, value.c_str(), value.length())) < 0) {
// If the thread is in the process of exiting, don't flag an error
if (errno != ESRCH) {
PLOG(ERROR) << "AddTidToCgroup failed to write '" << value << "'; fd=" << fd;
return false;
}
}
return true;
}
设置进程的cgroups也是很简单,只要将pid/tid写入到对应的cgroups下的cgroup.procs级可。
SetTimerSlack
setTimerSlack只有一个Slack参数值,这个参数对应了/proc/<pid>/timerslack_ns。TimerSlack是Linux系统为了降低系统功耗,避免timer时间参差不齐,过于的频繁的唤醒cpu,而设置的一种对齐策略。也就是说,这个值是关系到进程的定时器,诸如select、epoll_wait、sleep等API的唤醒时间,具体可参考下面的案例:https://cloud.tencent.com/developer/article/1836285
SetAttribute
SetAttribute则跟task_profiles.json中的Attributes挂钩起来,对应了SetAttributeAction,SetAttribute有两个参数 -- Name是指前面Attributes所定义过的Attribute名字,Value则是往Attribute对应group子节点写入的值。
bool SetAttributeAction::ExecuteForTask(int tid) const {
std::string path;
if (!attribute_->GetPathForTask(tid, &path)) {
LOG(ERROR) << "Failed to find cgroup for tid " << tid;
return false;
}
if (!WriteStringToFile(value_, path)) {
PLOG(ERROR) << "Failed to write '" << value_ << "' to " << path;
return false;
}
return true;
}
SetClamps
预留的接口,未实现,待内核支持util_clamp。
profiles的设置过程
Start thread
set_sched_policy: #00 pc 00000000000253ec /system/lib64/libprocessgroup.so (set_sched_policy+92)
set_sched_policy: #00 pc 00000000000253ec /system/lib64/libprocessgroup.so (set_sched_policy+92)
set_sched_policy: #01 pc 000000000001301c /system/lib64/libutils.so (thread_data_t::trampoline(thread_data_t const*)+92)
set_sched_policy: #02 pc 00000000000e1100 /system/lib64/bootstrap/libc.so (__pthread_start(void*)+36)
set_sched_policy: #03 pc 0000000000083ab0 /system/lib64/bootstrap/libc.so (__start_thread+64)
android在创建线程时,根据创建线程时设置的priority(默认是ANDROID_PRIORITY_BACKGROUND)来设置processgroup:
int androidCreateRawThreadEtc(android_thread_func_t entryFunction,
void *userData,
const char* threadName __android_unused,
int32_t threadPriority,
size_t threadStackSize,
android_thread_id_t *threadId)
{
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
#if defined(__ANDROID__) /* valgrind is rejecting RT-priority create reqs */
if (threadPriority != PRIORITY_DEFAULT || threadName != NULL) {
// Now that the pthread_t has a method to find the associated
// android_thread_id_t (pid) from pthread_t, it would be possible to avoid
// this trampoline in some cases as the parent could set the properties
// for the child. However, there would be a race condition because the
// child becomes ready immediately, and it doesn't work for the name.
// prctl(PR_SET_NAME) only works for self; prctl(PR_SET_THREAD_NAME) was
// proposed but not yet accepted.
thread_data_t* t = new thread_data_t;
t->priority = threadPriority;
t->threadName = threadName ? strdup(threadName) : NULL;
t->entryFunction = entryFunction;
t->userData = userData;
entryFunction = (android_thread_func_t)&thread_data_t::trampoline;
userData = t;
}
#endif
……
}
static int trampoline(const thread_data_t* t) {
thread_func_t f = t->entryFunction;
void* u = t->userData;
int prio = t->priority;
char * name = t->threadName;
delete t;
setpriority(PRIO_PROCESS, 0, prio);
if (prio >= ANDROID_PRIORITY_BACKGROUND) {
set_sched_policy(0, SP_BACKGROUND);
} else {
set_sched_policy(0, SP_FOREGROUND);
}
if (name) {
androidSetThreadName(name);
free(name);
}
return f(u);
}
Zygote SpecializeCommon
05-26 15:26:08.641 3147 3147 D set_sched_policy: #00 pc 00000000000253ec /system/lib64/libprocessgroup.so (set_sched_policy+92)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #01 pc 00000000001ae994 /system/lib64/libandroid_runtime.so ((anonymous namespace)::SpecializeCommon(_JNIEnv*, unsigned int, unsigned int, _jintArray*, int, _jobjectArray*, long, long, int, _jstring*, _jstring*, bool, bool, _jstring*, _jstring*)+5728)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #02 pc 00000000001aaec8 /system/lib64/libandroid_runtime.so (android::com_android_internal_os_Zygote_nativeForkAndSpecialize(_JNIEnv*, _jclass*, int, int, _jintArray*, int, _jobjectArray*, int, _jstring*, _jstring*, _jintArray*, _jintArray*, unsigned char, _jstring*, _jstring*)+740)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #03 pc 00000000002c4300 /system/framework/arm64/boot-framework.oat (art_jni_trampoline+416)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #04 pc 00000000009aa8b8 /system/framework/arm64/boot-framework.oat (com.android.internal.os.Zygote.forkAndSpecialize+200)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #05 pc 00000000009aeb14 /system/framework/arm64/boot-framework.oat (com.android.internal.os.ZygoteConnection.processOneCommand+1844)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #06 pc 00000000009b4090 /system/framework/arm64/boot-framework.oat (com.android.internal.os.ZygoteServer.runSelectLoop+1600)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #07 pc 00000000009b0a14 /system/framework/arm64/boot-framework.oat (com.android.internal.os.ZygoteInit.main+2884)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #08 pc 00000000001365b8 /apex/com.android.runtime/lib64/libart.so (art_quick_invoke_static_stub+568)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #09 pc 000000000014508c /apex/com.android.runtime/lib64/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+276)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #10 pc 00000000004a9b0c /apex/com.android.runtime/lib64/libart.so (art::(anonymous namespace)::InvokeWithArgArray(art::ScopedObjectAccessAlreadyRunnable const&, art::ArtMethod*, art::(anonymous namespace)::ArgArray*, art::JValue*, char const*)+104)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #11 pc 00000000004a9778 /apex/com.android.runtime/lib64/libart.so (art::InvokeWithVarArgs(art::ScopedObjectAccessAlreadyRunnable const&, _jobject*, _jmethodID*, std::__va_list)+408)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #12 pc 00000000003b67f0 /apex/com.android.runtime/lib64/libart.so (art::JNI::CallStaticVoidMethodV(_JNIEnv*, _jclass*, _jmethodID*, std::__va_list)+628)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #13 pc 00000000000bf560 /system/lib64/libandroid_runtime.so (_JNIEnv::CallStaticVoidMethod(_jclass*, _jmethodID*, ...)+116)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #14 pc 00000000000c246c /system/lib64/libandroid_runtime.so (android::AndroidRuntime::start(char const*, android::Vector<android::String8> const&, bool)+912)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #15 pc 00000000000034e0 /system/bin/app_process64 (main+1168)
05-26 15:26:08.641 3147 3147 D set_sched_policy: #16 pc 000000000007d798 /apex/com.android.runtime/lib64/bionic/libc.so (__libc_init+108)
zygote在fork一个应用出来时,就会默认设置了应用的profile值。
ThreadPriorityBooster
set_sched_policy: #00 pc 00000000000253ec /system/lib64/libprocessgroup.so (set_sched_policy+92)
set_sched_policy: #01 pc 000000000001319c /system/lib64/libutils.so (androidSetThreadPriority+76)
set_sched_policy: #02 pc 000000000014d068 /system/lib64/libandroid_runtime.so (android_os_Process_setThreadPriority(_JNIEnv*, _jobject*, int, int)+32)
set_sched_policy: #03 pc 00000000002ab060 /system/framework/arm64/boot-framework.oat (art_jni_trampoline+160)
set_sched_policy: #04 pc 00000000012ced1c /system/framework/oat/arm64/services.odex (com.android.server.ThreadPriorityBooster.boost+188)
set_sched_policy: #05 pc 00000000015e65e0 /system/framework/oat/arm64/services.odex (com.android.server.am.ActivityManagerService.boostPriorityForLockedSection+64)
set_sched_policy: #06 pc 0000000001612478 /system/framework/oat/arm64/services.odex (com.android.server.am.ActivityManagerService.registerReceiver+184)
set_sched_policy: #07 pc 00000000004deb64 /system/framework/arm64/boot-framework.oat (android.app.ContextImpl.registerReceiverInternal+660)
set_sched_policy: #08 pc 00000000004e4aa0 /system/framework/arm64/boot-framework.oat (android.app.ContextImpl.registerReceiver+128)
set_sched_policy: #09 pc 00000000004e49ec /system/framework/arm64/boot-framework.oat (android.app.ContextImpl.registerReceiver+60)
set_sched_policy: #10 pc 000000000004ad08 /system/priv-app/SettingsProvider/oat/arm64/SettingsProvider.odex (com.android.providers.settings.SettingsProvider.registerBroadcastReceivers+232)
set_sched_policy: #11 pc 000000000004e95c /system/priv-app/SettingsProvider/oat/arm64/SettingsProvider.odex (com.android.providers.settings.SettingsProvider.lambda$onCreate$0$SettingsProvider+44)
set_sched_policy: #12 pc 00000000000170e4 /system/priv-app/SettingsProvider/oat/arm64/SettingsProvider.odex (com.android.providers.settings.-$$Lambda$SettingsProvider$h_zJ8TmggsGxbxfR60fhjb7Ynw4.run+52)
set_sched_policy: #13 pc 000000000073237c /system/framework/arm64/boot-framework.oat (android.os.Handler.dispatchMessage+76)
set_sched_policy: #14 pc 0000000000735990 /system/framework/arm64/boot-framework.oat (android.os.Looper.loop+1440)
set_sched_policy: #15 pc 00000000007343d0 /system/framework/arm64/boot-framework.oat (android.os.HandlerThread.run+544)
set_sched_policy: #16 pc 0000000000136334 /apex/com.android.runtime/lib64/libart.so (art_quick_invoke_stub+548)
set_sched_policy: #17 pc 000000000014506c /apex/com.android.runtime/lib64/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+244)
set_sched_policy: #18 pc 00000000004a9b0c /apex/com.android.runtime/lib64/libart.so (art::(anonymous namespace)::InvokeWithArgArray(art::ScopedObjectAccessAlreadyRunnable const&, art::ArtMethod*, art::(anonymous namespace)::ArgArray*, art::JValue*, char const*)+104)
set_sched_policy: #19 pc 00000000004aaba0 /apex/com.android.runtime/lib64/libart.so (art::InvokeVirtualOrInterfaceWithJValues(art::ScopedObjectAccessAlreadyRunnable const&, _jobject*, _jmethodID*, jvalue const*)+416)
set_sched_policy: #20 pc 00000000004ea93c /apex/com.android.runtime/lib64/libart.so (art::Thread::CreateCallback(void*)+1176)
set_sched_policy: #21 pc 00000000000e1100 /apex/com.android.runtime/lib64/bionic/libc.so (__pthread_start(void*)+36)
set_sched_policy: #22 pc 0000000000083ab0 /apex/com.android.runtime/lib64/bionic/libc.so (__start_thread+64)
根据堆栈对照源码来看,实际上我们是看不到相关的调用的,而是通过ASM插桩的方式加入到字节码中的,可参考:https://zhuanlan.zhihu.com/p/348548041
通过反编译services.core.ja(out/soong/.intermediates/frameworks/base/services/core/services.core/android_common/combined/services.core.jar),可以看到插桩后的代码如下:
public Intent registerReceiver(final IApplicationThread caller, String callerPackage, final IIntentReceiver receiver, final IntentFilter filter, final String permission, int userId, final int flags) {
this.enforceNotIsolatedCaller("registerReceiver");
ArrayList<Intent> stickyIntents = null;
ProcessRecord callerApp = null;
final boolean visibleToInstantApps = (flags & 0x1) != 0x0;
// monitorenter(this)
int callingUid;
int callingPid;
boolean instantApp;
try {
boostPriorityForLockedSection();
……
}
finally {
// monitorexit(this)
resetPriorityAfterLockedSection();
}
}
在所有synchronized(ActivityManagerService.this)的调用前后均使用try-finally包裹,并在执行前调用boostPriorityForLockedSection,执行后调用resetPriorityAfterLockedSection。
applyOomAdjLocked
set_sched_policy: #00 pc 00000000000253ec /system/lib64/libprocessgroup.so (set_sched_policy+92)
set_sched_policy: #01 pc 000000000001301c /system/lib64/libutils.so (thread_data_t::trampoline(thread_data_t const*)+92)
set_sched_policy: #02 pc 00000000000e1100 /apex/com.android.runtime/lib64/bionic/libc.so (__pthread_start(void*)+36)
set_sched_policy: #03 pc 0000000000083ab0 /apex/com.android.runtime/lib64/bionic/libc.so (__start_thread+64)
set_sched_policy: #00 pc 00000000000253ec /system/lib64/libprocessgroup.so (set_sched_policy+92)
set_sched_policy: #01 pc 000000000014c4b4 /system/lib64/libandroid_runtime.so (android_os_Process_setProcessGroup(_JNIEnv*, _jobject*, int, int)+328)
set_sched_policy: #02 pc 00000000002ab060 /system/framework/arm64/boot-framework.oat (art_jni_trampoline+160)
set_sched_policy: #03 pc 00000000006bb4e4 /system/framework/oat/arm64/services.odex (com.android.server.am.OomAdjuster.lambda$new$0+116)
set_sched_policy: #04 pc 000000000067d554 /system/framework/oat/arm64/services.odex (com.android.server.am.-$$Lambda$OomAdjuster$OVkqAAacT5-taN3pgDzyZj3Ymvk.handleMessage+52)
set_sched_policy: #05 pc 00000000007323b8 /system/framework/arm64/boot-framework.oat (android.os.Handler.dispatchMessage+136)
set_sched_policy: #06 pc 0000000000735990 /system/framework/arm64/boot-framework.oat (android.os.Looper.loop+1440)
set_sched_policy: #07 pc 00000000007343d0 /system/framework/arm64/boot-framework.oat (android.os.HandlerThread.run+544)
set_sched_policy: #08 pc 00000000012ad6e4 /system/framework/oat/arm64/services.odex (com.android.server.ServiceThread.run+100)
set_sched_policy: #09 pc 0000000000136334 /apex/com.android.runtime/lib64/libart.so (art_quick_invoke_stub+548)
set_sched_policy: #10 pc 000000000014506c /apex/com.android.runtime/lib64/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+244)
set_sched_policy: #11 pc 00000000004a9b0c /apex/com.android.runtime/lib64/libart.so (art::(anonymous namespace)::InvokeWithArgArray(art::ScopedObjectAccessAlreadyRunnable const&, art::ArtMethod*, art::(anonymous namespace)::ArgArray*, art::JValue*, char const*)+104)
set_sched_policy: #12 pc 00000000004aaba0 /apex/com.android.runtime/lib64/libart.so (art::InvokeVirtualOrInterfaceWithJValues(art::ScopedObjectAccessAlreadyRunnable const&, _jobject*, _jmethodID*, jvalue const*)+416)
set_sched_policy: #13 pc 00000000004ea93c /apex/com.android.runtime/lib64/libart.so (art::Thread::CreateCallback(void*)+1176)
set_sched_policy: #14 pc 00000000000e1100 /apex/com.android.runtime/lib64/bionic/libc.so (__pthread_start(void*)+36)
set_sched_policy: #15 pc 0000000000083ab0 /apex/com.android.runtime/lib64/bionic/libc.so (__start_thread+64)
AMS对于应用状态发生变化时,会触发OomAdj的值的修改,在此过程中,还会修改process的SchedulingGroup。
新增一组profile
针对一些场景,例如我们不想让某类进程(java层的应用)消耗过多的CPU资源,那么可以在cpuset上再创建一个子group,限制
1.为cgroups的子系统创建一个子节点,在cpuset下创建一个test的子group:
on init
mkdir /dev/cpuset/test
copy /dev/cpuset/cpus /dev/cpuset/test/cpus
copy /dev/cpuset/mems /dev/cpuset/test/mems
chown system system /dev/cpuset/test
chown system system /dev/cpuset/test/tasks
chmod 0664 /dev/cpuset/test/tasks
2.修改task_profiles.json,增加一个profile类型:
{
"Name": "TestCapacity",
"Actions": [
{
"Name": "JoinCgroup",
"Params":
{
"Controller": "cpuset",
"Path": "test"
}
}
]
},
3.libprocessgroup中增加对新增profile的处理:
// system/core/libprocessgroup/include/processgroup/sched_policy.h
typedef enum {
SP_DEFAULT = -1,
SP_BACKGROUND = 0,
SP_FOREGROUND = 1,
SP_SYSTEM = 2, // can't be used with set_sched_policy()
SP_AUDIO_APP = 3,
SP_AUDIO_SYS = 4,
SP_TOP_APP = 5,
SP_RT_APP = 6,
SP_RESTRICTED = 7,
SP_TEST = 8,
SP_CNT,
SP_MAX = SP_CNT - 1,
SP_SYSTEM_DEFAULT = SP_FOREGROUND,
} SchedPolicy;
// system/core/libprocessgroup/sched_policy.cpp
int set_cpuset_policy(int tid, SchedPolicy policy) {
if (tid == 0) {
tid = GetThreadId();
}
policy = _policy(policy);
switch (policy) {
case SP_BACKGROUND:
return SetTaskProfiles(tid,
{"HighEnergySaving", "ProcessCapacityLow", "LowIoPriority",
"TimerSlackHigh"},
true)
? 0
: -1;
case SP_FOREGROUND:
case SP_AUDIO_APP:
case SP_AUDIO_SYS:
return SetTaskProfiles(tid,
{"HighPerformance", "ProcessCapacityHigh", "HighIoPriority",
"TimerSlackNormal"},
true)
? 0
: -1;
case SP_TOP_APP:
return SetTaskProfiles(tid,
{"MaxPerformance", "ProcessCapacityMax", "MaxIoPriority",
"TimerSlackNormal"},
true)
? 0
: -1;
case SP_SYSTEM:
return SetTaskProfiles(tid, {"ServiceCapacityLow", "TimerSlackNormal"}, true) ? 0 : -1;
case SP_RESTRICTED:
return SetTaskProfiles(tid, {"ServiceCapacityRestricted", "TimerSlackNormal"}, true)
? 0
: -1;
case SP_STEST:
return SetTaskProfiles(tid, {"HighEnergySaving", "TestCapacity", "TimerSlackNormal"}, true) ? 0 : -1;
default:
break;
}
return 0;
}
4.修改AMS设置sched group的地方:
// frameworks/base/core/java/android/os/Process.java
public static final int THREAD_GROUP_TEST = 8;
// frameworks/base/services/core/java/com/android/server/am/ProcessList.java
static final int SCHED_GROUP_TEST_APP = 5;
// frameworks/base/services/core/java/com/android/server/am/ProcessRecord.java
void setCurrentSchedulingGroup(int curSchedGroup) {
if (mTestApp && curSchedGroup < ProcessList.SCHED_GROUP_TOP_APP)
curSchedGroup = ProcessList.SCHED_GROUP_TEST_APP;
mCurSchedGroup = curSchedGroup;
mWindowProcessController.setCurrentSchedulingGroup(curSchedGroup);
}
// frameworks/base/services/core/java/com/android/server/am/OomAdjuster.java
private final boolean applyOomAdjLocked(ProcessRecord app, boolean doingAll, long now,
long nowElapsed) {
final int curSchedGroup = app.getCurrentSchedulingGroup();
int processGroup;
switch (curSchedGroup) {
case ProcessList.SCHED_GROUP_BACKGROUND:
processGroup = THREAD_GROUP_BG_NONINTERACTIVE;
break;
case ProcessList.SCHED_GROUP_TOP_APP:
case ProcessList.SCHED_GROUP_TOP_APP_BOUND:
processGroup = THREAD_GROUP_TOP_APP;
break;
case ProcessList.SCHED_GROUP_RESTRICTED:
processGroup = THREAD_GROUP_RESTRICTED;
break;
case ProcessList.SCHED_GROUP_TEST_APP:
processGroup = THREAD_GROUP_TEST;
break;
default:
processGroup = THREAD_GROUP_DEFAULT;
break;
}
mProcessGroupHandler.sendMessage(mProcessGroupHandler.obtainMessage(
0 /* unused */, app.pid, processGroup));
}
参考
https://tech.meituan.com/2015/03/31/cgroups.html
https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/cgroups.html