From 5b8b48ccc1a6b72bdea3c9ea157a65896c1d6846 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Fri, 26 May 2023 14:07:11 -0700 Subject: [PATCH] Properly support streamable input --- app/build.gradle.kts | 2 +- .../magisk/core/tasks/MagiskInstaller.kt | 331 ++++++++++-------- native/src/base/files.rs | 36 +- native/src/boot/main.cpp | 2 +- native/src/boot/payload.rs | 43 ++- 5 files changed, 239 insertions(+), 175 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index c2d4f6dea..3b3abad74 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -80,7 +80,7 @@ dependencies { implementation("dev.rikka.rikkax.recyclerview:recyclerview-ktx:1.3.1") implementation("io.noties.markwon:core:4.6.2") - val vLibsu = "5.0.5" + val vLibsu = "5.1.0" implementation("com.github.topjohnwu.libsu:core:${vLibsu}") implementation("com.github.topjohnwu.libsu:service:${vLibsu}") implementation("com.github.topjohnwu.libsu:nio:${vLibsu}") diff --git a/app/src/main/java/com/topjohnwu/magisk/core/tasks/MagiskInstaller.kt b/app/src/main/java/com/topjohnwu/magisk/core/tasks/MagiskInstaller.kt index 6c2e27b79..dd39f9b90 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/tasks/MagiskInstaller.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/tasks/MagiskInstaller.kt @@ -1,10 +1,10 @@ package com.topjohnwu.magisk.core.tasks import android.net.Uri -import android.os.Build -import android.os.ParcelFileDescriptor import android.system.ErrnoException import android.system.Os +import android.system.OsConstants +import android.system.OsConstants.O_WRONLY import android.widget.Toast import androidx.annotation.WorkerThread import androidx.core.os.postDelayed @@ -18,7 +18,6 @@ import com.topjohnwu.magisk.core.ktx.toast import com.topjohnwu.magisk.core.ktx.withStreams import com.topjohnwu.magisk.core.ktx.writeTo import com.topjohnwu.magisk.core.utils.MediaStoreUtils -import com.topjohnwu.magisk.core.utils.MediaStoreUtils.fileDescriptor import com.topjohnwu.magisk.core.utils.MediaStoreUtils.inputStream import com.topjohnwu.magisk.core.utils.MediaStoreUtils.outputStream import com.topjohnwu.magisk.core.utils.RootUtils @@ -170,151 +169,61 @@ abstract class MagiskInstallImpl protected constructor( return true } - private fun InputStream.cleanPump(out: OutputStream) = withStreams(this, out) { src, _ -> - src.copyTo(out) - } + private fun InputStream.copyAndClose(out: OutputStream) = out.use { copyTo(it) } private fun newTarEntry(name: String, size: Long): TarEntry { console.add("-- Writing: $name") return TarEntry(TarHeader.createHeader(name, size, 0, false, 420 /* 0644 */)) } - @Throws(IOException::class) - private fun processZip(input: InputStream): ExtendedFile { - val boot = installDir.getChildFile("boot.img") - val initBoot = installDir.getChildFile("init_boot.img") - ZipInputStream(input).use { zipIn -> - lateinit var entry: ZipEntry - while (zipIn.nextEntry?.also { entry = it } != null) { - if (entry.isDirectory) continue - when (entry.name.substringAfterLast('/')) { - "payload.bin" -> { - console.add("- Extracting payload") - val dest = File(installDir, "payload.bin") - FileOutputStream(dest).use { zipIn.copyTo(it) } - try { - return processPayload(Uri.fromFile(dest)) - } catch (e: IOException) { - // No boot image in payload.bin, continue to find boot images - } - } - "init_boot.img" -> { - console.add("- Extracting init_boot image") - initBoot.newOutputStream().use { zipIn.copyTo(it) } - return initBoot - } - "boot.img" -> { - console.add("- Extracting boot image") - boot.newOutputStream().use { zipIn.copyTo(it) } - // no break here since there might be an init_boot.img - } - } - } - } - if (boot.exists()) { - return boot - } else { - console.add("! No boot image found") - throw IOException() - } + private class LZ4InputStream(s: InputStream) : LZ4FrameInputStream(s) { + // Workaround bug in LZ4FrameInputStream + override fun available() = 0 } - @Throws(IOException::class) - @Synchronized - private fun processPayload(input: Uri): ExtendedFile { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { - throw IOException("Payload is only supported on Android Oreo or above") - } - try { - console.add("- Processing payload.bin") - console.add("-- Extracting boot.img") - input.fileDescriptor("r").use { fd -> - val bk = ParcelFileDescriptor.fromFd(0) - try { - Os.dup2(fd.fileDescriptor, 0) - val process = ProcessBuilder() - .redirectInput(ProcessBuilder.Redirect.INHERIT) - .command( - "$installDir/magiskboot", - "extract", - "-" - ) - .start() - if (process.waitFor() != 0) { - throw IOException( - "magiskboot extract failed with code ${ - process.errorStream.readBytes().toString(Charsets.UTF_8) - }" - ) - } - } finally { - Os.dup2(bk.fileDescriptor, 0) - } - } - val boot = installDir.getChildFile("boot.img") - val initBoot = installDir.getChildFile("init_boot.img") - return when { - initBoot.exists() -> initBoot - boot.exists() -> boot - else -> { - console.add("! No boot image found") - throw IOException() - } - } - } catch (e: ErrnoException) { - throw IOException(e) - } - } + private class NoBootException : IOException() @Throws(IOException::class) - private fun processTar(input: InputStream, output: OutputStream): OutputStream { + private fun processTar(tarIn: TarInputStream, tarOut: TarOutputStream): ExtendedFile { console.add("- Processing tar file") - val tarOut = TarOutputStream(output) - TarInputStream(input).use { tarIn -> - lateinit var entry: TarEntry + lateinit var entry: TarEntry - fun decompressedStream(): InputStream { - val src = if (entry.name.endsWith(".lz4")) LZ4FrameInputStream(tarIn) else tarIn - return object : FilterInputStream(src) { - override fun available() = 0 /* Workaround bug in LZ4FrameInputStream */ - override fun close() { /* Never close src stream */ } - } - } + fun decompressedStream(): InputStream { + return if (entry.name.endsWith(".lz4")) LZ4InputStream(tarIn) else tarIn + } - while (tarIn.nextEntry?.let { entry = it } != null) { - if (entry.name.startsWith("boot.img") || - entry.name.startsWith("init_boot.img") || - (Config.recovery && entry.name.contains("recovery.img"))) { - val name = entry.name.replace(".lz4", "") - console.add("-- Extracting: $name") + while (tarIn.nextEntry?.let { entry = it } != null) { + if (entry.name.startsWith("boot.img") || + entry.name.startsWith("init_boot.img") || + (Config.recovery && entry.name.contains("recovery.img"))) { + val name = entry.name.replace(".lz4", "") + console.add("-- Extracting: $name") - val extract = installDir.getChildFile(name) - decompressedStream().cleanPump(extract.newOutputStream()) - } else if (entry.name.contains("vbmeta.img")) { - val rawData = decompressedStream().readBytes() - // Valid vbmeta.img should be at least 256 bytes - if (rawData.size < 256) - continue + val extract = installDir.getChildFile(name) + decompressedStream().copyAndClose(extract.newOutputStream()) + } else if (entry.name.contains("vbmeta.img")) { + val rawData = decompressedStream().readBytes() + // Valid vbmeta.img should be at least 256 bytes + if (rawData.size < 256) + continue - // Patch flags to AVB_VBMETA_IMAGE_FLAGS_HASHTREE_DISABLED | - // AVB_VBMETA_IMAGE_FLAGS_VERIFICATION_DISABLED - console.add("-- Patching: vbmeta.img") - ByteBuffer.wrap(rawData).putInt(120, 3) - tarOut.putNextEntry(newTarEntry("vbmeta.img", rawData.size.toLong())) - tarOut.write(rawData) - } else { - console.add("-- Copying: ${entry.name}") - tarOut.putNextEntry(entry) - tarIn.copyTo(tarOut, bufferSize = 1024 * 1024) - } + // Patch flags to AVB_VBMETA_IMAGE_FLAGS_HASHTREE_DISABLED | + // AVB_VBMETA_IMAGE_FLAGS_VERIFICATION_DISABLED + console.add("-- Patching: vbmeta.img") + ByteBuffer.wrap(rawData).putInt(120, 3) + tarOut.putNextEntry(newTarEntry("vbmeta.img", rawData.size.toLong())) + tarOut.write(rawData) + } else { + console.add("-- Copying: ${entry.name}") + tarOut.putNextEntry(entry) + tarIn.copyTo(tarOut, bufferSize = 1024 * 1024) } } + val boot = installDir.getChildFile("boot.img") val initBoot = installDir.getChildFile("init_boot.img") val recovery = installDir.getChildFile("recovery.img") if (Config.recovery && recovery.exists() && boot.exists()) { - // Install to recovery - srcBoot = recovery // Repack boot image to prevent auto restore arrayOf( "cd $installDir", @@ -330,31 +239,130 @@ abstract class MagiskInstallImpl protected constructor( it.copyTo(tarOut) } boot.delete() + // Install to recovery + return recovery } else { - srcBoot = when { + return when { initBoot.exists() -> initBoot boot.exists() -> boot else -> { - console.add("! No boot image found") - throw IOException() + throw NoBootException() } } } - return tarOut + } + + @Throws(IOException::class) + private fun processZip(zipIn: ZipInputStream): ExtendedFile { + console.add("- Processing zip file") + val boot = installDir.getChildFile("boot.img") + val initBoot = installDir.getChildFile("init_boot.img") + lateinit var entry: ZipEntry + while (zipIn.nextEntry?.also { entry = it } != null) { + if (entry.isDirectory) continue + when (entry.name.substringAfterLast('/')) { + "payload.bin" -> { + try { + return processPayload(zipIn) + } catch (e: IOException) { + // No boot image in payload.bin, continue to find boot images + } + } + "init_boot.img" -> { + console.add("- Extracting init_boot.img") + zipIn.copyAndClose(initBoot.newOutputStream()) + return initBoot + } + "boot.img" -> { + console.add("- Extracting boot.img") + zipIn.copyAndClose(boot.newOutputStream()) + // Don't return here since there might be an init_boot.img + } + } + } + if (boot.exists()) { + return boot + } else { + throw NoBootException() + } + } + + @Throws(IOException::class) + private fun processPayload(input: InputStream): ExtendedFile { + var fifo: File? = null + try { + console.add("- Processing payload.bin") + fifo = File.createTempFile("payload-fifo-", null, installDir) + fifo.delete() + Os.mkfifo(fifo.path, 420 /* 0644 */) + + // Enqueue the shell command first, or the subsequent FIFO open will block + val future = arrayOf( + "cd $installDir", + "./magiskboot extract $fifo", + "cd /" + ).eq() + + val fd = Os.open(fifo.path, O_WRONLY, 0) + try { + val buf = ByteBuffer.allocate(1024 * 1024) + buf.position(input.read(buf.array()).coerceAtLeast(0)).flip() + while (buf.hasRemaining()) { + try { + Os.write(fd, buf) + } catch (e: ErrnoException) { + if (e.errno != OsConstants.EPIPE) + throw e + // If SIGPIPE, then the other side is closed, we're done + break + } + if (!buf.hasRemaining()) { + buf.position(input.read(buf.array()).coerceAtLeast(0)).flip() + } + } + } finally { + Os.close(fd) + } + + val success = try { future.get().isSuccess } catch (e: Exception) { false } + if (!success) { + console.add("! Error while extracting payload.bin") + throw IOException() + } + val boot = installDir.getChildFile("boot.img") + val initBoot = installDir.getChildFile("init_boot.img") + return when { + initBoot.exists() -> { + console.add("-- Extract init_boot.img") + initBoot + } + boot.exists() -> { + console.add("-- Extract boot.img") + boot + } + else -> { + throw NoBootException() + } + } + } catch (e: ErrnoException) { + throw IOException(e) + } finally { + fifo?.delete() + } } private fun handleFile(uri: Uri): Boolean { val outStream: OutputStream - var outFile: MediaStoreUtils.UriFile? = null + val outFile: MediaStoreUtils.UriFile // Process input file try { uri.inputStream().buffered().use { src -> src.mark(500) - val magic = ByteArray(5) - val headMagic = ByteArray(4) - if (src.read(headMagic) != headMagic.size || src.skip(253) != 253L || - src.read(magic) != magic.size + val magic = ByteArray(4) + val tarMagic = ByteArray(5) + if (src.read(magic) != magic.size || src.skip(253) != 253L || + src.read(tarMagic) != tarMagic.size ) { console.add("! Invalid input file") return false @@ -371,36 +379,52 @@ abstract class MagiskInstallImpl protected constructor( toString() } - outStream = if (magic.contentEquals("ustar".toByteArray())) { + srcBoot = if (tarMagic.contentEquals("ustar".toByteArray())) { // tar file outFile = MediaStoreUtils.getFile("$filename.tar", true) - processTar(src, outFile!!.uri.outputStream()) - } else { - srcBoot = if (headMagic.contentEquals("CrAU".toByteArray())) { - processPayload(uri) - } else if (headMagic.contentEquals("PK\u0003\u0004".toByteArray())) { - processZip(src) - } else { - val boot = installDir.getChildFile("boot.img") - console.add("- Copying image to cache") - src.cleanPump(boot.newOutputStream()) - boot + outStream = TarOutputStream(outFile.uri.outputStream()) + + try { + processTar(TarInputStream(src), outStream) + } catch (e: IOException) { + outStream.close() + outFile.delete() + throw e } + } else { // raw image outFile = MediaStoreUtils.getFile("$filename.img", true) - outFile!!.uri.outputStream() + outStream = outFile.uri.outputStream() + + try { + if (magic.contentEquals("CrAU".toByteArray())) { + processPayload(src) + } else if (magic.contentEquals("PK\u0003\u0004".toByteArray())) { + processZip(ZipInputStream(src)) + } else { + console.add("- Copying image to cache") + installDir.getChildFile("boot.img").also { + src.copyAndClose(it.newOutputStream()) + } + } + } catch (e: IOException) { + outStream.close() + outFile.delete() + throw e + } } } } catch (e: IOException) { + if (e is NoBootException) + console.add("! No boot image found") console.add("! Process error") - outFile?.delete() Timber.e(e) return false } // Patch file if (!patchBoot()) { - outFile!!.delete() + outFile.delete() return false } @@ -417,7 +441,7 @@ abstract class MagiskInstallImpl protected constructor( } outStream.putNextEntry(newTarEntry(name, newBoot.length())) } - newBoot.newInputStream().cleanPump(outStream) + newBoot.newInputStream().copyAndClose(outStream) newBoot.delete() console.add("") @@ -427,7 +451,7 @@ abstract class MagiskInstallImpl protected constructor( console.add("****************************") } catch (e: IOException) { console.add("! Failed to output to $outFile") - outFile!!.delete() + outFile.delete() Timber.e(e) return false } @@ -516,6 +540,7 @@ abstract class MagiskInstallImpl protected constructor( return true } + private fun Array.eq() = shell.newJob().add(*this).to(console, logs).enqueue() private fun String.sh() = shell.newJob().add(this).to(console, logs).exec() private fun Array.sh() = shell.newJob().add(*this).to(console, logs).exec() private fun String.fsh() = ShellUtils.fastCmd(shell, this) diff --git a/native/src/base/files.rs b/native/src/base/files.rs index 9308bec80..f74ded79a 100644 --- a/native/src/base/files.rs +++ b/native/src/base/files.rs @@ -1,8 +1,9 @@ +use mem::MaybeUninit; use std::cmp::min; use std::ffi::CStr; -use std::io; -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Seek, SeekFrom, Write}; use std::os::unix::io::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::{io, mem}; use libc::{c_char, c_uint, mode_t, EEXIST, ENOENT, O_CLOEXEC, O_PATH}; @@ -135,6 +136,37 @@ pub extern "C" fn mkdirs(path: *const c_char, mode: mode_t) -> i32 { } } +pub trait ReadExt { + fn skip(&mut self, len: usize) -> io::Result<()>; +} + +impl ReadExt for T { + fn skip(&mut self, mut len: usize) -> io::Result<()> { + let mut buf = MaybeUninit::<[u8; 4096]>::uninit(); + let buf = unsafe { buf.assume_init_mut() }; + while len > 0 { + let l = min(buf.len(), len); + self.read_exact(&mut buf[..l])?; + len -= l; + } + Ok(()) + } +} + +pub trait ReadSeekExt { + fn skip(&mut self, len: usize) -> io::Result<()>; +} + +impl ReadSeekExt for T { + fn skip(&mut self, len: usize) -> io::Result<()> { + if self.seek(SeekFrom::Current(len as i64)).is_err() { + // If the file is not actually seekable, fallback to read + ReadExt::skip(self, len)?; + } + Ok(()) + } +} + pub trait BufReadExt { fn foreach_lines bool>(&mut self, f: F); fn foreach_props bool>(&mut self, f: F); diff --git a/native/src/boot/main.cpp b/native/src/boot/main.cpp index 3be37bdc1..f7d274024 100644 --- a/native/src/boot/main.cpp +++ b/native/src/boot/main.cpp @@ -51,7 +51,7 @@ Supported actions: If [partition] is not specified, then attempt to extract either 'init_boot' or 'boot'. Which partition was chosen can be determined by whichever 'init_boot.img' or 'boot.img' exists. - /[outfile] can be '-' to be STDIN/STDOUT. + can be '-' to be STDIN. hexpatch Search in , and replace it with diff --git a/native/src/boot/payload.rs b/native/src/boot/payload.rs index 54cf97d5c..2691c83a5 100644 --- a/native/src/boot/payload.rs +++ b/native/src/boot/payload.rs @@ -7,7 +7,7 @@ use byteorder::{BigEndian, ReadBytesExt}; use protobuf::{EnumFull, Message}; use base::libc::c_char; -use base::{ptr_to_str_result, StrErr}; +use base::{ptr_to_str_result, ReadSeekExt, StrErr}; use base::{ResultExt, WriteExt}; use crate::ffi; @@ -27,7 +27,7 @@ const PAYLOAD_MAGIC: &str = "CrAU"; fn do_extract_boot_from_payload( in_path: &str, - partition: Option<&str>, + partition_name: Option<&str>, out_path: Option<&str>, ) -> anyhow::Result<()> { let mut reader = BufReader::new(if in_path == "-" { @@ -74,46 +74,50 @@ fn do_extract_boot_from_payload( let block_size = manifest.block_size() as u64; - let part = match partition { + let partition = match partition_name { None => { let boot = manifest .partitions .iter() - .find(|partition| partition.partition_name() == "init_boot"); + .find(|p| p.partition_name() == "init_boot"); let boot = match boot { Some(boot) => Some(boot), None => manifest .partitions .iter() - .find(|partition| partition.partition_name() == "boot"), + .find(|p| p.partition_name() == "boot"), }; boot.ok_or(anyhow!("boot partition not found"))? } - Some(partition) => manifest + Some(name) => manifest .partitions .iter() - .find(|p| p.partition_name() == partition) - .ok_or(anyhow!("partition '{partition}' not found"))?, + .find(|p| p.partition_name() == name) + .ok_or(anyhow!("partition '{name}' not found"))?, }; let out_str: String; let out_path = match out_path { None => { - out_str = format!("{}.img", part.partition_name()); + out_str = format!("{}.img", partition.partition_name()); out_str.as_str() } - Some(p) => p, + Some(s) => s, }; - let mut out_file = if out_path == "-" { - unsafe { File::from_raw_fd(1) } - } else { - File::create(out_path).with_context(|| format!("cannot write to '{out_path}'"))? - }; + let mut out_file = + File::create(out_path).with_context(|| format!("cannot write to '{out_path}'"))?; - let base_offset = reader.stream_position()? + manifest_sig_len as u64; + // Skip the manifest signature + reader.skip(manifest_sig_len as usize)?; - for operation in part.operations.iter() { + // Sort the install operations with data_offset so we will only ever need to seek forward + // This makes it possible to support non-seekable input file descriptors + let mut operations = partition.operations.clone(); + operations.sort_by_key(|e| e.data_offset.unwrap_or(0)); + let mut curr_data_offset: u64 = 0; + + for operation in operations.iter() { let data_len = operation .data_length .ok_or(bad_payload!("data length not found"))? as usize; @@ -131,8 +135,11 @@ fn do_extract_boot_from_payload( buf.resize(data_len, 0u8); let data = &mut buf[..data_len]; - reader.seek(SeekFrom::Start(base_offset + data_offset))?; + // Skip to the next offset and read data + let skip = data_offset - curr_data_offset; + reader.skip(skip as usize)?; reader.read_exact(data)?; + curr_data_offset = data_offset + data_len as u64; let out_offset = operation .dst_extents