From de2306bd128ed328d2085419dd9a113c66eda919 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Tue, 7 Sep 2021 19:35:28 -0700 Subject: [PATCH] Proper incremental builds Auto generate flag.h for precise rebuilding --- build.py | 75 +++++++++++++++++++++--------------- native/jni/Android.mk | 6 --- native/jni/Application.mk | 6 --- native/jni/core/daemon.cpp | 2 +- native/jni/core/magisk.cpp | 2 +- native/jni/include/flags.hpp | 20 ---------- native/jni/init/init.cpp | 3 +- native/jni/su/su.cpp | 2 +- native/jni/utils/Android.mk | 6 ++- native/jni/utils/logging.cpp | 4 +- native/jni/zygisk/hook.cpp | 4 +- 11 files changed, 56 insertions(+), 74 deletions(-) delete mode 100644 native/jni/include/flags.hpp diff --git a/build.py b/build.py index 3cf303028..3ccac1ff2 100755 --- a/build.py +++ b/build.py @@ -10,6 +10,7 @@ import shutil import stat import subprocess import sys +import textwrap import urllib.request import zipfile from distutils.dir_util import copy_tree @@ -67,6 +68,7 @@ ndk_path = op.join(ndk_root, 'magisk') ndk_build = op.join(ndk_path, 'ndk-build') gradlew = op.join('.', 'gradlew' + ('.bat' if is_windows else '')) adb_path = op.join(sdk_path, 'platform-tools', 'adb' + ('.exe' if is_windows else '')) +native_gen_path = op.join('native', 'out', 'generated') # Global vars config = {} @@ -244,33 +246,60 @@ def sign_zip(unsigned): error('Signing failed!') -def binary_dump(src, out, var_name): - out.write(f'constexpr unsigned char {var_name}[] = {{') +def binary_dump(src, var_name): + out_str = f'constexpr unsigned char {var_name}[] = {{' for i, c in enumerate(xz(src.read())): if i % 16 == 0: - out.write('\n') - out.write(f'0x{c:02X},') - out.write('\n};\n') - out.flush() + out_str += '\n' + out_str += f'0x{c:02X},' + out_str += '\n};\n' + return out_str def run_ndk_build(flags): os.chdir('native') - proc = system(f'{ndk_build} {base_flags} {flags} -j{cpu_count}') + proc = system(f'{ndk_build} {flags} -j{cpu_count}') if proc.returncode != 0: error('Build binary failed!') os.chdir('..') collect_binary() -def dump_bin_headers(): +def write_if_diff(file_name, text): + do_write = True + if op.exists(file_name): + with open(file_name, 'r') as f: + orig = f.read() + do_write = orig != text + if do_write: + with open(file_name, 'w') as f: + f.write(text) + + +def dump_bin_header(): stub = op.join(config['outdir'], 'stub-release.apk') if not op.exists(stub): error('Build stub APK before building "magiskinit"') - mkdir_p(op.join('native', 'out')) - with open(op.join('native', 'out', 'binaries.h'), 'w') as out: - with open(stub, 'rb') as src: - binary_dump(src, out, 'manager_xz') + mkdir_p(native_gen_path) + with open(stub, 'rb') as src: + text = binary_dump(src, 'manager_xz') + write_if_diff(op.join(native_gen_path, 'binaries.h'), text) + + +def dump_flag_header(): + flag_txt = textwrap.dedent('''\ + #pragma once + #define quote(s) #s + #define str(s) quote(s) + #define MAGISK_FULL_VER MAGISK_VERSION "(" str(MAGISK_VER_CODE) ")" + #define NAME_WITH_VER(name) str(name) " " MAGISK_FULL_VER + ''') + flag_txt += f'#define MAGISK_VERSION "{config["version"]}"\n' + flag_txt += f'#define MAGISK_VER_CODE {config["versionCode"]}\n' + flag_txt += f'#define MAGISK_DEBUG {0 if args.release else 1}\n' + + mkdir_p(native_gen_path) + write_if_diff(op.join(native_gen_path, 'flags.h'), flag_txt) def build_binary(args): @@ -291,25 +320,7 @@ def build_binary(args): header('* Building binaries: ' + ' '.join(args.target)) - update_flags = False - flags = op.join('native', 'jni', 'include', 'flags.hpp') - flags_stat = os.stat(flags) - - if op.exists(args.config): - if os.stat(args.config).st_mtime_ns > flags_stat.st_mtime_ns: - update_flags = True - - if os.stat('gradle.properties').st_mtime_ns > flags_stat.st_mtime_ns: - update_flags = True - - if update_flags: - os.utime(flags) - - # Basic flags - global base_flags - base_flags = f'MAGISK_VERSION={config["version"]} MAGISK_VER_CODE={config["versionCode"]}' - if not args.release: - base_flags += ' MAGISK_DEBUG=1' + dump_flag_header() flag = '' @@ -320,7 +331,7 @@ def build_binary(args): flag += ' B_TEST=1' if 'magiskinit' in args.target: - dump_bin_headers() + dump_bin_header() flag += ' B_INIT=1' if 'magiskpolicy' in args.target: diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 8c3e52b48..fe69cee5c 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -9,7 +9,6 @@ ifdef B_MAGISK include $(CLEAR_VARS) LOCAL_MODULE := magisk LOCAL_STATIC_LIBRARIES := libnanopb libsystemproperties libutils libphmap libxhook -LOCAL_C_INCLUDES := jni/include LOCAL_SRC_FILES := \ core/applets.cpp \ @@ -49,7 +48,6 @@ ifdef B_INIT LOCAL_MODULE := magiskinit LOCAL_STATIC_LIBRARIES := libsepol libxz libutils -LOCAL_C_INCLUDES := jni/include out LOCAL_SRC_FILES := \ init/init.cpp \ @@ -76,7 +74,6 @@ ifdef B_BOOT include $(CLEAR_VARS) LOCAL_MODULE := magiskboot LOCAL_STATIC_LIBRARIES := libmincrypt liblzma liblz4 libbz2 libfdt libutils libz libzopfli -LOCAL_C_INCLUDES := jni/include LOCAL_SRC_FILES := \ magiskboot/main.cpp \ @@ -99,7 +96,6 @@ ifdef B_POLICY include $(CLEAR_VARS) LOCAL_MODULE := magiskpolicy LOCAL_STATIC_LIBRARIES := libsepol libutils -LOCAL_C_INCLUDES := jni/include LOCAL_SRC_FILES := \ core/applet_stub.cpp \ @@ -120,7 +116,6 @@ ifdef B_PROP include $(CLEAR_VARS) LOCAL_MODULE := resetprop LOCAL_STATIC_LIBRARIES := libnanopb libsystemproperties libutils -LOCAL_C_INCLUDES := jni/include LOCAL_SRC_FILES := \ core/applet_stub.cpp \ @@ -139,7 +134,6 @@ ifneq (,$(wildcard jni/test.cpp)) include $(CLEAR_VARS) LOCAL_MODULE := test LOCAL_STATIC_LIBRARIES := libutils libphmap -LOCAL_C_INCLUDES := jni/include LOCAL_SRC_FILES := test.cpp include $(BUILD_EXECUTABLE) diff --git a/native/jni/Application.mk b/native/jni/Application.mk index e8d99c2ed..13d2e8a4b 100644 --- a/native/jni/Application.mk +++ b/native/jni/Application.mk @@ -10,10 +10,4 @@ APP_STRIP_MODE := --strip-all # Busybox should use stock libc.a ifdef B_BB APP_PLATFORM := android-22 -else -# Make Busybox cflag stable -APP_CFLAGS += -D__MVSTR=${MAGISK_VERSION} -D__MCODE=${MAGISK_VER_CODE} -ifdef MAGISK_DEBUG -APP_CFLAGS += -D__MDBG -endif endif diff --git a/native/jni/core/daemon.cpp b/native/jni/core/daemon.cpp index 1c3caad34..dde4f7577 100644 --- a/native/jni/core/daemon.cpp +++ b/native/jni/core/daemon.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "core.hpp" diff --git a/native/jni/core/magisk.cpp b/native/jni/core/magisk.cpp index b2616aa23..a8d47b83f 100644 --- a/native/jni/core/magisk.cpp +++ b/native/jni/core/magisk.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include "core.hpp" diff --git a/native/jni/include/flags.hpp b/native/jni/include/flags.hpp deleted file mode 100644 index 001b4b6fd..000000000 --- a/native/jni/include/flags.hpp +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -/* Include this header anywhere accessing MAGISK_DEBUG, MAGISK_VERSION, MAGISK_VER_CODE. - * - * This file is for precise incremental builds. We can make sure code that uses - * external flags are re-compiled by updating the timestamp of this file - * */ - -#define quote(s) #s -#define str(s) quote(s) - -#define MAGISK_VERSION str(__MVSTR) -#define MAGISK_VER_CODE __MCODE -#define MAGISK_FULL_VER MAGISK_VERSION "(" str(MAGISK_VER_CODE) ")" - -#define NAME_WITH_VER(name) str(name) " " MAGISK_FULL_VER - -#ifdef __MDBG -#define MAGISK_DEBUG -#endif diff --git a/native/jni/init/init.cpp b/native/jni/init/init.cpp index 940242d8e..720b9fabc 100644 --- a/native/jni/init/init.cpp +++ b/native/jni/init/init.cpp @@ -6,10 +6,11 @@ #include #include + #include #include +#include -#include "binaries.h" #include "init.hpp" using namespace std; diff --git a/native/jni/su/su.cpp b/native/jni/su/su.cpp index 5b9f6156a..3ec0738ee 100644 --- a/native/jni/su/su.cpp +++ b/native/jni/su/su.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include "su.hpp" #include "pts.hpp" diff --git a/native/jni/utils/Android.mk b/native/jni/utils/Android.mk index bfda5436c..631915e95 100644 --- a/native/jni/utils/Android.mk +++ b/native/jni/utils/Android.mk @@ -1,9 +1,11 @@ LOCAL_PATH := $(call my-dir) +# All Magisk common code lives here + include $(CLEAR_VARS) LOCAL_MODULE:= libutils -LOCAL_C_INCLUDES := jni/include $(LOCAL_PATH)/include -LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/include +LOCAL_C_INCLUDES := jni/include $(LOCAL_PATH)/include out/generated +LOCAL_EXPORT_C_INCLUDES := $(LOCAL_C_INCLUDES) LOCAL_EXPORT_STATIC_LIBRARIES := libcxx LOCAL_STATIC_LIBRARIES := libcxx LOCAL_SRC_FILES := \ diff --git a/native/jni/utils/logging.cpp b/native/jni/utils/logging.cpp index 717ae95cf..d4f291ba9 100644 --- a/native/jni/utils/logging.cpp +++ b/native/jni/utils/logging.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "logging.hpp" @@ -47,7 +47,7 @@ void cmdline_logging() { } // LTO will optimize out the NOP function -#ifdef MAGISK_DEBUG +#if MAGISK_DEBUG void LOGD(const char *fmt, ...) { LOG_BODY(d) } #else void LOGD(const char *fmt, ...) {} diff --git a/native/jni/zygisk/hook.cpp b/native/jni/zygisk/hook.cpp index 67af77292..529e77f50 100644 --- a/native/jni/zygisk/hook.cpp +++ b/native/jni/zygisk/hook.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include "inject.hpp" @@ -356,7 +356,7 @@ static int hook_register(const char *path, const char *symbol, void *new_func, v #define APP_PROCESS "^/system/bin/app_process.*" void hook_functions() { -#ifdef MAGISK_DEBUG +#if MAGISK_DEBUG xhook_enable_debug(1); xhook_enable_sigsegv_protection(0); #endif