Android Framework 常见解决方案（25-1）定制CPUSET解决方案-framework部分修改

1 原理说明

这个方案有如下基本需求：

• 构建自定义CPUSET，/dev/cpuset中包含一个全新的cpuset分组。且可以通过set_cpuset_policy和set_sched_policy接口可以设置自定义CPUSET。
• 开机启动后可以通过zygote判定来对特定的应用进程设置CPUSET，并一直保持，且保证自定义CPUSET不受其他CPUSET影响，持续独立。

原理上因为修改代码涉及部分较多，因此共分3个部分：

1. framework修改：添加SCHED GROUP和THREAD GROUP（THREAD_GROUP_对应SP_）的支持，且支持开机启动后直接设置。applyOomAdjLSP的判定保持不变。
2. system修改：添加SP_CUSTOM的支持及set_cpuset_policy和set_sched_policy接口等支持，同时修改task_profiles.json，添加SP_CUSTOM CPUSET的支持。
3. init.rc修改及编译部分调整：对自定义cpuset节点进行操作，vndk部分编译需要重新调整方案以及不修改VNDK如何保证编译通过。

由于修改中涉及代码量过大，这里拆分成两节进行展示。本章节主要针对第1部分修改进行说明。下一篇文章 👇

Android Framework 常见解决方案（25-2）定制CPUSET解决方案-system修改及编译部分调整

主要对第2和第3部分修改进行说明。

2 修改方案-framework部分（Android S）

这里需要添加THREAD_GROUP_CUSTOM，与system中修改的SP_CUSTOM同步，数值需一致，需要在$AOSP/frameworks/base/core/java/android/os/Process.java文件中修改：

public class Process {
    private static final String LOG_TAG = "Process";

    /**
     * An invalid UID value.
     */
    public static final int INVALID_UID = -1;
    //...
    /**
     * Thread group for RT app.
     * @hide
     **/
    public static final int THREAD_GROUP_RT_APP = 6;

    /**
     * Thread group for bound foreground services that should
     * have additional CPU restrictions during screen off
     * @hide
     **/
    public static final int THREAD_GROUP_RESTRICTED = 7;

    /**
     * Thread Group for CUSTOM
     * @hide
     **/
    public static final int THREAD_GROUP_CUSTOM = 8;
    //...
}

接下来需要添加SCHED_GROUP_CUSTOM相关配置。在$AOSP/frameworks/base/services/core/java/com/android/server/am/ProcessList.java文件中修改：

public final class ProcessList {
    //...
    // Activity manager's version of Process.THREAD_GROUP_BACKGROUND
    static final int SCHED_GROUP_BACKGROUND = 0;
      // Activity manager's version of Process.THREAD_GROUP_RESTRICTED
    static final int SCHED_GROUP_RESTRICTED = 1;
    // Activity manager's version of Process.THREAD_GROUP_DEFAULT
    static final int SCHED_GROUP_DEFAULT = 2;
    // Activity manager's version of Process.THREAD_GROUP_TOP_APP
    public static final int SCHED_GROUP_TOP_APP = 3;
    // Activity manager's version of Process.THREAD_GROUP_TOP_APP
    // Disambiguate between actual top app and processes bound to the top app
    static final int SCHED_GROUP_TOP_APP_BOUND = 4;
+    //add custom schedule group
+    static final int SCHED_GROUP_CUSTOM = 10;
    //...
    private static boolean writeProcessOomListToProto(ProtoOutputStream proto, long fieldId,
            ActivityManagerService service, List<ProcessRecord> origList,
            boolean inclDetails, String dumpPackage) {
        //...
            switch (state.getSetSchedGroup()) {
                case SCHED_GROUP_BACKGROUND:
                    schedGroup = ProcessOomProto.SCHED_GROUP_BACKGROUND;
                    break;
                case SCHED_GROUP_DEFAULT:
                    schedGroup = ProcessOomProto.SCHED_GROUP_DEFAULT;
                    break;
                case SCHED_GROUP_TOP_APP:
                    schedGroup = ProcessOomProto.SCHED_GROUP_TOP_APP;
                    break;
                case SCHED_GROUP_TOP_APP_BOUND:
                    schedGroup = ProcessOomProto.SCHED_GROUP_TOP_APP_BOUND;
+                case SCHED_GROUP_CUSTOM:
+                    schedGroup = ProcessOomProto.SCHED_GROUP_CUSTOM;
+                    break;
            }
	//...
    }
    //...
    private static boolean dumpProcessOomList(PrintWriter pw,
            ActivityManagerService service, List<ProcessRecord> origList,
            String prefix, String normalLabel, String persistentLabel,
            boolean inclDetails, String dumpPackage) {
        //...    
        for (int i = list.size() - 1; i >= 0; i--) {
            //...
            switch (state.getSetSchedGroup()) {
                case SCHED_GROUP_BACKGROUND:
                    schedGroup = 'b';
                    break;
                case SCHED_GROUP_DEFAULT:
                    schedGroup = 'F';
                    break;
                case SCHED_GROUP_TOP_APP:
                    schedGroup = 'T';
                    break;
                case SCHED_GROUP_RESTRICTED:
                    schedGroup = 'R';
                    break;
                case SCHED_GROUP_TOP_APP_BOUND:
                    schedGroup = 'B';
                    break;
+                case SCHED_GROUP_CUSTOM:
+                    schedGroup = 'C';
+                    break;
                default:
                    schedGroup = '?';
                    break;
            }

    }

添加SCHED_GROUP_CUSTOM，与ProcessList.java中同步。在$AOSP/frameworks/base/core/proto/android/server/activitymanagerservice.proto文件中修改：

//...
message ProcessOomProto {
    //...
    enum SchedGroup {
        SCHED_GROUP_UNKNOWN = -1;
        SCHED_GROUP_BACKGROUND = 0;
        SCHED_GROUP_DEFAULT = 1;
        SCHED_GROUP_TOP_APP = 2;
        SCHED_GROUP_TOP_APP_BOUND = 3;
+        SCHED_GROUP_CUSTOM = 10;
    }

?为适配之前的适配SP_CUSTOM，同时契合之前1中修改的task_profile.json文件中的配置，在AOSP/frameworks/base/core/jni/android_util_Process.cpp文件中修改：

//修改android_os_Process_setProcessGroup，适配SP_CUSTOM
void android_os_Process_setProcessGroup(JNIEnv* env, jobject clazz, int pid, jint grp)
{
    ALOGV("%s pid=%d grp=%" PRId32, __func__, pid, grp);
    //...
    while ((de = readdir(d))) {
        //...

        // grp != SP_BACKGROUND. Only change the cpuset cgroup for low priority thread, so it could
        // preserve it sched policy profile setting.
	if (t_pri >= ANDROID_PRIORITY_BACKGROUND) {
            switch (grp) {
                case SP_SYSTEM:
                    taskprofile = "ServiceCapacityLow";
                    break;
                case SP_RESTRICTED:
                    taskprofile = "ServiceCapacityRestricted";
                    break;
                case SP_FOREGROUND:
                case SP_AUDIO_APP:
                case SP_AUDIO_SYS:
                    taskprofile = "ProcessCapacityHigh";
                    break;
                case SP_TOP_APP:
                    taskprofile = "ProcessCapacityMax";
                    break;
+                case SP_CUSTOM:
+                    taskprofile = "CustomPerformance";
+                    break;
                default:
                    taskprofile = "ProcessCapacityNormal";
                    break;
            }
            if (!SetTaskProfiles(t_pid, {taskprofile}, true)) {
                signalExceptionForGroupError(env, errno ? errno : EPERM, t_pid);
                break;
            }
        // Change the cpuset policy profile for non-low priority thread according to the grp
        } else {
            if (!SetTaskProfiles(t_pid, {get_cpuset_policy_profile_name((SchedPolicy)grp)}, true)) {
                signalExceptionForGroupError(env, errno ? errno : EPERM, t_pid);
                break;
            }
        }
    }
    closedir(d);
}

//修改get_cpuset_cores_for_policy，适配SP_CUSTOM
static void get_cpuset_cores_for_policy(SchedPolicy policy, cpu_set_t *cpu_set)
{
    FILE *file;
    std::string filename;

    CPU_ZERO(cpu_set);

    switch (policy) {
        case SP_BACKGROUND:
            if (!CgroupGetAttributePath("LowCapacityCPUs", &filename)) {
                return;
            }
            break;
        case SP_FOREGROUND:
            if (!CgroupGetAttributePath("HighCapacityCPUs", &filename)) {
                return;
            }
            break;
        case SP_AUDIO_APP:
        case SP_AUDIO_SYS:
            if (!CgroupGetAttributePath("AudioAppCapacityCPUs", &filename)) {
                return;
            }
            if (access(filename.c_str(), F_OK) != 0) {
                if (!CgroupGetAttributePath("HighCapacityCPUs", &filename)) {
                    return;
                }
            }
            break;
        case SP_RT_APP:
            if (!CgroupGetAttributePath("HighCapacityCPUs", &filename)) {
                return;
            }
            break;
        case SP_TOP_APP:
            if (!CgroupGetAttributePath("MaxCapacityCPUs", &filename)) {
                return;
            }
            break;
+        case SP_CUSTOM:
+            if (!CgroupGetAttributePath("CustomCPUs", &filename)) {
+                return;
+            }
+            break;
        default:
            return;
    }
    file = fopen(filename.c_str(), "re");
    if (file != NULL) {
        // Parse cpus string
        char *line = NULL;
        size_t len = 0;
        ssize_t num_read = getline(&line, &len, file);
        fclose (file);
        if (num_read > 0) {
            parse_cpuset_cpus(line, cpu_set);
        } else {
            ALOGE("Failed to read %s", filename.c_str());
        }
        free(line);
    }
    return;
}

在这里使用com.ags.test应用demo进行测试后验证。根据自己需要调整即可。主要保证该应用始终在SCHED_GROUP_CUSTOM中，不切换到其它schedule group中，同时建立SCHED_GROUP_CUSTOM和THREAD_GROUP_CUSTOM之间的联系。在$AOSP/frameworks/base/services/core/java/com/android/server/am/OomAdjuster.java文件中修改：

//...
+import static android.os.Process.THREAD_GROUP_CUSTOM;
//...
public class OomAdjuster {
    //...
    private boolean applyOomAdjLSP(ProcessRecord app, boolean doingAll, long now,
            long nowElapsed) {
        boolean success = true;
        final ProcessStateRecord state = app.mState;

        if (state.getCurRawAdj() != state.getSetRawAdj()) {
            state.setSetRawAdj(state.getCurRawAdj());
        }
        //...
                int processGroup;
                switch (curSchedGroup) {
                    case ProcessList.SCHED_GROUP_BACKGROUND:
                        processGroup = THREAD_GROUP_BACKGROUND;
                        break;
                    case ProcessList.SCHED_GROUP_TOP_APP:
                    case ProcessList.SCHED_GROUP_TOP_APP_BOUND:
                        processGroup = THREAD_GROUP_TOP_APP;
                        break;
                    case ProcessList.SCHED_GROUP_RESTRICTED:
                        processGroup = THREAD_GROUP_RESTRICTED;
                        break;
+                    case ProcessList.SCHED_GROUP_CUSTOM:
+                        processGroup = THREAD_GROUP_CUSTOM;
+                        break;
                    default:
                        processGroup = THREAD_GROUP_DEFAULT;
                        break;
                }
         //...
    }
    //...
    private boolean computeOomAdjLSP(ProcessRecord app, int cachedAdj,
            ProcessRecord topApp, boolean doingAll, long now, boolean cycleReEval,
            boolean computeClients) {
        //...
+        if(app.processName.equals("com.ags.test")){
+            schedGroup = ProcessList.SCHED_GROUP_CUSTOM;
+        }
        // Do final modification to adj.  Everything we do between here and applying
        // the final setAdj must be done in this function, because we will also use
        // it when computing the final cached adj later.  Note that we don't need to
        // worry about this for max adj above, since max adj will always be used to
        // keep it out of the cached vaues.
        state.setCurAdj(psr.modifyRawOomAdj(adj));
        state.setCurCapability(capability);
        state.setCurrentSchedulingGroup(schedGroup);
        state.setCurProcState(procState);
        state.setCurRawProcState(procState);
        state.updateLastInvisibleTime(hasVisibleActivities);
        state.setHasForegroundActivities(foregroundActivities);
        state.setCompletedAdjSeq(mAdjSeq);

        // if curAdj or curProcState improved, then this process was promoted
        return state.getCurAdj() < prevAppAdj || state.getCurProcState() < prevProcState
                || state.getCurCapability() != prevCapability;
    }
    //... 
}

修改SpecializeCommon，主要是在创建进程时直接设置cpuset，在$AOSP/frameworks/base/core/jni/android_util_Process.cpp文件中修改：

static void SpecializeCommon(JNIEnv* env, uid_t uid, gid_t gid, jintArray gids, jint runtime_flags,
                             jobjectArray rlimits, jlong permitted_capabilities,
                             jlong effective_capabilities, jint mount_external,
                             jstring managed_se_info, jstring managed_nice_name,
                             bool is_system_server, bool is_child_zygote,
                             jstring managed_instruction_set, jstring managed_app_data_dir,
                             bool is_top_app, jobjectArray pkg_data_info_list,
                             jobjectArray allowlisted_data_info_list, bool mount_data_dirs,
                             bool mount_storage_dirs) {
    const char* process_name = is_system_server ? "system_server" : "zygote";
    auto fail_fn = std::bind(ZygoteFailure, env, process_name, managed_nice_name, _1);
    auto extract_fn = std::bind(ExtractJString, env, process_name, managed_nice_name, _1);
    //...
    if (setresgid(gid, gid, gid) == -1) {
        fail_fn(CREATE_ERROR("setresgid(%d) failed: %s", gid, strerror(errno)));
    }

    // Must be called when the new process still has CAP_SYS_ADMIN, in this case,
    // before changing uid from 0, which clears capabilities.  The other
    // alternative is to call prctl(PR_SET_NO_NEW_PRIVS, 1) afterward, but that
    // breaks SELinux domain transition (see b/71859146).  As the result,
    // privileged syscalls used below still need to be accessible in app process.
    SetUpSeccompFilter(uid, is_child_zygote);

    // Must be called before losing the permission to set scheduler policy.
    SetSchedulerPolicy(fail_fn, is_top_app);

+    if (nice_name.has_value()) {
+        if(strncmp(nice_name.value().c_str(),"com.ags.test",sizeof("com.ags.test"))==0){
+#if 1 //set_cpuset_policy方案
+            int ret = set_cpuset_policy(0, SP_CUSTOM);
+            if (ret != 0) {
+                ALOGE("set_cpuset_policy call default failure,ret:%d,(%s)", ret,nice_name.value().c_str());
+            }else{
+                ALOGD("set_cpuset_policy call default success,ret==0,(%s)", nice_name.value().c_str());
+            }
+#else //set_sched_policy兼容方案
+            int ret = set_sched_policy(0, SP_CUSTOM);
+            if (ret != 0) {
+                ALOGE("set_sched_policy call default failure,ret:%d,(%s)", ret,nice_name.value().c_str());
+            }
+           else{
+               ALOGD("set_sched_policy call default success,ret==0,(%s)", nice_name.value().c_str());
+            }
+#endif
+        }
+    }
     //...
 }

至此，framework层相关修改就结束了。主要添加SCHED_GROUP_CUSTOM、THREAD_GROUP_CUSTOM（THREAD_GROUP_对应SP_）即以和对应SP_CUSTOM之间的联系建立，且支持开机启动后直接过滤包，进行CPUSET的设置。同时applyOomAdjLSP的判定保持不变，不受其他CPUSET的影响。