Properly support streamable input

app/build.gradle.kts

@@ -80,7 +80,7 @@ dependencies {
     implementation("dev.rikka.rikkax.recyclerview:recyclerview-ktx:1.3.1")
     implementation("io.noties.markwon:core:4.6.2")
 
-    val vLibsu = "5.0.5"
+    val vLibsu = "5.1.0"
     implementation("com.github.topjohnwu.libsu:core:${vLibsu}")
     implementation("com.github.topjohnwu.libsu:service:${vLibsu}")
     implementation("com.github.topjohnwu.libsu:nio:${vLibsu}")

app/src/main/java/com/topjohnwu/magisk/core/tasks/MagiskInstaller.kt

@@ -1,10 +1,10 @@
 package com.topjohnwu.magisk.core.tasks
 
 import android.net.Uri
-import android.os.Build
-import android.os.ParcelFileDescriptor
 import android.system.ErrnoException
 import android.system.Os
+import android.system.OsConstants
+import android.system.OsConstants.O_WRONLY
 import android.widget.Toast
 import androidx.annotation.WorkerThread
 import androidx.core.os.postDelayed
@@ -18,7 +18,6 @@ import com.topjohnwu.magisk.core.ktx.toast
 import com.topjohnwu.magisk.core.ktx.withStreams
 import com.topjohnwu.magisk.core.ktx.writeTo
 import com.topjohnwu.magisk.core.utils.MediaStoreUtils
-import com.topjohnwu.magisk.core.utils.MediaStoreUtils.fileDescriptor
 import com.topjohnwu.magisk.core.utils.MediaStoreUtils.inputStream
 import com.topjohnwu.magisk.core.utils.MediaStoreUtils.outputStream
 import com.topjohnwu.magisk.core.utils.RootUtils
@@ -170,151 +169,61 @@ abstract class MagiskInstallImpl protected constructor(
         return true
     }
 
-    private fun InputStream.cleanPump(out: OutputStream) = withStreams(this, out) { src, _ ->
+    private fun InputStream.copyAndClose(out: OutputStream) = out.use { copyTo(it) }
-        src.copyTo(out)
-    }
 
     private fun newTarEntry(name: String, size: Long): TarEntry {
         console.add("-- Writing: $name")
         return TarEntry(TarHeader.createHeader(name, size, 0, false, 420 /* 0644 */))
     }
 
-    @Throws(IOException::class)
+    private class LZ4InputStream(s: InputStream) : LZ4FrameInputStream(s) {
-    private fun processZip(input: InputStream): ExtendedFile {
+        // Workaround bug in LZ4FrameInputStream
-        val boot = installDir.getChildFile("boot.img")
+        override fun available() = 0
-        val initBoot = installDir.getChildFile("init_boot.img")
-        ZipInputStream(input).use { zipIn ->
-            lateinit var entry: ZipEntry
-            while (zipIn.nextEntry?.also { entry = it } != null) {
-                if (entry.isDirectory) continue
-                when (entry.name.substringAfterLast('/')) {
-                    "payload.bin" -> {
-                        console.add("- Extracting payload")
-                        val dest = File(installDir, "payload.bin")
-                        FileOutputStream(dest).use { zipIn.copyTo(it) }
-                        try {
-                            return processPayload(Uri.fromFile(dest))
-                        } catch (e: IOException) {
-                            // No boot image in payload.bin, continue to find boot images
-                        }
-                    }
-                    "init_boot.img" -> {
-                        console.add("- Extracting init_boot image")
-                        initBoot.newOutputStream().use { zipIn.copyTo(it) }
-                        return initBoot
-                    }
-                    "boot.img" -> {
-                        console.add("- Extracting boot image")
-                        boot.newOutputStream().use { zipIn.copyTo(it) }
-                        // no break here since there might be an init_boot.img
-                    }
-                }
-            }
-        }
-        if (boot.exists()) {
-            return boot
-        } else {
-            console.add("! No boot image found")
-            throw IOException()
-        }
     }
 
-    @Throws(IOException::class)
+    private class NoBootException : IOException()
-    @Synchronized
-    private fun processPayload(input: Uri): ExtendedFile {
-        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
-            throw IOException("Payload is only supported on Android Oreo or above")
-        }
-        try {
-            console.add("- Processing payload.bin")
-            console.add("-- Extracting boot.img")
-            input.fileDescriptor("r").use { fd ->
-                val bk = ParcelFileDescriptor.fromFd(0)
-                try {
-                    Os.dup2(fd.fileDescriptor, 0)
-                    val process = ProcessBuilder()
-                        .redirectInput(ProcessBuilder.Redirect.INHERIT)
-                        .command(
-                            "$installDir/magiskboot",
-                            "extract",
-                            "-"
-                        )
-                        .start()
-                    if (process.waitFor() != 0) {
-                        throw IOException(
-                            "magiskboot extract failed with code ${
-                                process.errorStream.readBytes().toString(Charsets.UTF_8)
-                            }"
-                        )
-                    }
-                } finally {
-                    Os.dup2(bk.fileDescriptor, 0)
-                }
-            }
-            val boot = installDir.getChildFile("boot.img")
-            val initBoot = installDir.getChildFile("init_boot.img")
-            return when {
-                initBoot.exists() -> initBoot
-                boot.exists() -> boot
-                else -> {
-                    console.add("! No boot image found")
-                    throw IOException()
-                }
-            }
-        } catch (e: ErrnoException) {
-            throw IOException(e)
-        }
-    }
 
     @Throws(IOException::class)
-    private fun processTar(input: InputStream, output: OutputStream): OutputStream {
+    private fun processTar(tarIn: TarInputStream, tarOut: TarOutputStream): ExtendedFile {
         console.add("- Processing tar file")
-        val tarOut = TarOutputStream(output)
+        lateinit var entry: TarEntry
-        TarInputStream(input).use { tarIn ->
-            lateinit var entry: TarEntry
 
-            fun decompressedStream(): InputStream {
+        fun decompressedStream(): InputStream {
-                val src = if (entry.name.endsWith(".lz4")) LZ4FrameInputStream(tarIn) else tarIn
+            return if (entry.name.endsWith(".lz4")) LZ4InputStream(tarIn) else tarIn
-                return object : FilterInputStream(src) {
+        }
-                    override fun available() = 0  /* Workaround bug in LZ4FrameInputStream */
-                    override fun close() { /* Never close src stream */ }
-                }
-            }
 
-            while (tarIn.nextEntry?.let { entry = it } != null) {
+        while (tarIn.nextEntry?.let { entry = it } != null) {
-                if (entry.name.startsWith("boot.img") ||
+            if (entry.name.startsWith("boot.img") ||
-                    entry.name.startsWith("init_boot.img") ||
+                entry.name.startsWith("init_boot.img") ||
-                    (Config.recovery && entry.name.contains("recovery.img"))) {
+                (Config.recovery && entry.name.contains("recovery.img"))) {
-                    val name = entry.name.replace(".lz4", "")
+                val name = entry.name.replace(".lz4", "")
-                    console.add("-- Extracting: $name")
+                console.add("-- Extracting: $name")
 
-                    val extract = installDir.getChildFile(name)
+                val extract = installDir.getChildFile(name)
-                    decompressedStream().cleanPump(extract.newOutputStream())
+                decompressedStream().copyAndClose(extract.newOutputStream())
-                } else if (entry.name.contains("vbmeta.img")) {
+            } else if (entry.name.contains("vbmeta.img")) {
-                    val rawData = decompressedStream().readBytes()
+                val rawData = decompressedStream().readBytes()
-                    // Valid vbmeta.img should be at least 256 bytes
+                // Valid vbmeta.img should be at least 256 bytes
-                    if (rawData.size < 256)
+                if (rawData.size < 256)
-                        continue
+                    continue
 
-                    // Patch flags to AVB_VBMETA_IMAGE_FLAGS_HASHTREE_DISABLED |
+                // Patch flags to AVB_VBMETA_IMAGE_FLAGS_HASHTREE_DISABLED |
-                    // AVB_VBMETA_IMAGE_FLAGS_VERIFICATION_DISABLED
+                // AVB_VBMETA_IMAGE_FLAGS_VERIFICATION_DISABLED
-                    console.add("-- Patching: vbmeta.img")
+                console.add("-- Patching: vbmeta.img")
-                    ByteBuffer.wrap(rawData).putInt(120, 3)
+                ByteBuffer.wrap(rawData).putInt(120, 3)
-                    tarOut.putNextEntry(newTarEntry("vbmeta.img", rawData.size.toLong()))
+                tarOut.putNextEntry(newTarEntry("vbmeta.img", rawData.size.toLong()))
-                    tarOut.write(rawData)
+                tarOut.write(rawData)
-                } else {
+            } else {
-                    console.add("-- Copying: ${entry.name}")
+                console.add("-- Copying: ${entry.name}")
-                    tarOut.putNextEntry(entry)
+                tarOut.putNextEntry(entry)
-                    tarIn.copyTo(tarOut, bufferSize = 1024 * 1024)
+                tarIn.copyTo(tarOut, bufferSize = 1024 * 1024)
-                }
             }
         }
+
         val boot = installDir.getChildFile("boot.img")
         val initBoot = installDir.getChildFile("init_boot.img")
         val recovery = installDir.getChildFile("recovery.img")
         if (Config.recovery && recovery.exists() && boot.exists()) {
-            // Install to recovery
-            srcBoot = recovery
             // Repack boot image to prevent auto restore
             arrayOf(
                 "cd $installDir",
@@ -330,31 +239,130 @@ abstract class MagiskInstallImpl protected constructor(
                 it.copyTo(tarOut)
             }
             boot.delete()
+            // Install to recovery
+            return recovery
         } else {
-            srcBoot = when {
+            return when {
                 initBoot.exists() -> initBoot
                 boot.exists() -> boot
                 else -> {
-                    console.add("! No boot image found")
+                    throw NoBootException()
-                    throw IOException()
                 }
             }
         }
-        return tarOut
+    }
+
+    @Throws(IOException::class)
+    private fun processZip(zipIn: ZipInputStream): ExtendedFile {
+        console.add("- Processing zip file")
+        val boot = installDir.getChildFile("boot.img")
+        val initBoot = installDir.getChildFile("init_boot.img")
+        lateinit var entry: ZipEntry
+        while (zipIn.nextEntry?.also { entry = it } != null) {
+            if (entry.isDirectory) continue
+            when (entry.name.substringAfterLast('/')) {
+                "payload.bin" -> {
+                    try {
+                        return processPayload(zipIn)
+                    } catch (e: IOException) {
+                        // No boot image in payload.bin, continue to find boot images
+                    }
+                }
+                "init_boot.img" -> {
+                    console.add("- Extracting init_boot.img")
+                    zipIn.copyAndClose(initBoot.newOutputStream())
+                    return initBoot
+                }
+                "boot.img" -> {
+                    console.add("- Extracting boot.img")
+                    zipIn.copyAndClose(boot.newOutputStream())
+                    // Don't return here since there might be an init_boot.img
+                }
+            }
+        }
+        if (boot.exists()) {
+            return boot
+        } else {
+            throw NoBootException()
+        }
+    }
+
+    @Throws(IOException::class)
+    private fun processPayload(input: InputStream): ExtendedFile {
+        var fifo: File? = null
+        try {
+            console.add("- Processing payload.bin")
+            fifo = File.createTempFile("payload-fifo-", null, installDir)
+            fifo.delete()
+            Os.mkfifo(fifo.path, 420 /* 0644 */)
+
+            // Enqueue the shell command first, or the subsequent FIFO open will block
+            val future = arrayOf(
+                "cd $installDir",
+                "./magiskboot extract $fifo",
+                "cd /"
+            ).eq()
+
+            val fd = Os.open(fifo.path, O_WRONLY, 0)
+            try {
+                val buf = ByteBuffer.allocate(1024 * 1024)
+                buf.position(input.read(buf.array()).coerceAtLeast(0)).flip()
+                while (buf.hasRemaining()) {
+                    try {
+                        Os.write(fd, buf)
+                    } catch (e: ErrnoException) {
+                        if (e.errno != OsConstants.EPIPE)
+                            throw e
+                        // If SIGPIPE, then the other side is closed, we're done
+                        break
+                    }
+                    if (!buf.hasRemaining()) {
+                        buf.position(input.read(buf.array()).coerceAtLeast(0)).flip()
+                    }
+                }
+            } finally {
+                Os.close(fd)
+            }
+
+            val success = try { future.get().isSuccess } catch (e: Exception) { false }
+            if (!success) {
+                console.add("! Error while extracting payload.bin")
+                throw IOException()
+            }
+            val boot = installDir.getChildFile("boot.img")
+            val initBoot = installDir.getChildFile("init_boot.img")
+            return when {
+                initBoot.exists() -> {
+                    console.add("-- Extract init_boot.img")
+                    initBoot
+                }
+                boot.exists() -> {
+                    console.add("-- Extract boot.img")
+                    boot
+                }
+                else -> {
+                    throw NoBootException()
+                }
+            }
+        } catch (e: ErrnoException) {
+            throw IOException(e)
+        } finally {
+            fifo?.delete()
+        }
     }
 
     private fun handleFile(uri: Uri): Boolean {
         val outStream: OutputStream
-        var outFile: MediaStoreUtils.UriFile? = null
+        val outFile: MediaStoreUtils.UriFile
 
         // Process input file
         try {
             uri.inputStream().buffered().use { src ->
                 src.mark(500)
-                val magic = ByteArray(5)
+                val magic = ByteArray(4)
-                val headMagic = ByteArray(4)
+                val tarMagic = ByteArray(5)
-                if (src.read(headMagic) != headMagic.size || src.skip(253) != 253L ||
+                if (src.read(magic) != magic.size || src.skip(253) != 253L ||
-                    src.read(magic) != magic.size
+                    src.read(tarMagic) != tarMagic.size
                 ) {
                     console.add("! Invalid input file")
                     return false
@@ -371,36 +379,52 @@ abstract class MagiskInstallImpl protected constructor(
                     toString()
                 }
 
-                outStream = if (magic.contentEquals("ustar".toByteArray())) {
+                srcBoot = if (tarMagic.contentEquals("ustar".toByteArray())) {
                     // tar file
                     outFile = MediaStoreUtils.getFile("$filename.tar", true)
-                    processTar(src, outFile!!.uri.outputStream())
+                    outStream = TarOutputStream(outFile.uri.outputStream())
-                } else {
+
-                    srcBoot = if (headMagic.contentEquals("CrAU".toByteArray())) {
+                    try {
-                        processPayload(uri)
+                        processTar(TarInputStream(src), outStream)
-                    } else if (headMagic.contentEquals("PK\u0003\u0004".toByteArray())) {
+                    } catch (e: IOException) {
-                        processZip(src)
+                        outStream.close()
-                    } else {
+                        outFile.delete()
-                        val boot = installDir.getChildFile("boot.img")
+                        throw e
-                        console.add("- Copying image to cache")
-                        src.cleanPump(boot.newOutputStream())
-                        boot
                     }
+                } else {
                     // raw image
                     outFile = MediaStoreUtils.getFile("$filename.img", true)
-                    outFile!!.uri.outputStream()
+                    outStream = outFile.uri.outputStream()
+
+                    try {
+                        if (magic.contentEquals("CrAU".toByteArray())) {
+                            processPayload(src)
+                        } else if (magic.contentEquals("PK\u0003\u0004".toByteArray())) {
+                            processZip(ZipInputStream(src))
+                        } else {
+                            console.add("- Copying image to cache")
+                            installDir.getChildFile("boot.img").also {
+                                src.copyAndClose(it.newOutputStream())
+                            }
+                        }
+                    } catch (e: IOException) {
+                        outStream.close()
+                        outFile.delete()
+                        throw e
+                    }
                 }
             }
         } catch (e: IOException) {
+            if (e is NoBootException)
+                console.add("! No boot image found")
             console.add("! Process error")
-            outFile?.delete()
             Timber.e(e)
             return false
         }
 
         // Patch file
         if (!patchBoot()) {
-            outFile!!.delete()
+            outFile.delete()
             return false
         }
 
@@ -417,7 +441,7 @@ abstract class MagiskInstallImpl protected constructor(
                 }
                 outStream.putNextEntry(newTarEntry(name, newBoot.length()))
             }
-            newBoot.newInputStream().cleanPump(outStream)
+            newBoot.newInputStream().copyAndClose(outStream)
             newBoot.delete()
 
             console.add("")
@@ -427,7 +451,7 @@ abstract class MagiskInstallImpl protected constructor(
             console.add("****************************")
         } catch (e: IOException) {
             console.add("! Failed to output to $outFile")
-            outFile!!.delete()
+            outFile.delete()
             Timber.e(e)
             return false
         }
@@ -516,6 +540,7 @@ abstract class MagiskInstallImpl protected constructor(
         return true
     }
 
+    private fun Array<String>.eq() = shell.newJob().add(*this).to(console, logs).enqueue()
     private fun String.sh() = shell.newJob().add(this).to(console, logs).exec()
     private fun Array<String>.sh() = shell.newJob().add(*this).to(console, logs).exec()
     private fun String.fsh() = ShellUtils.fastCmd(shell, this)

native/src/base/files.rs

@@ -1,8 +1,9 @@
+use mem::MaybeUninit;
 use std::cmp::min;
 use std::ffi::CStr;
-use std::io;
+use std::io::{BufRead, Read, Seek, SeekFrom, Write};
-use std::io::{BufRead, Write};
 use std::os::unix::io::{AsRawFd, FromRawFd, OwnedFd, RawFd};
+use std::{io, mem};
 
 use libc::{c_char, c_uint, mode_t, EEXIST, ENOENT, O_CLOEXEC, O_PATH};
 
@@ -135,6 +136,37 @@ pub extern "C" fn mkdirs(path: *const c_char, mode: mode_t) -> i32 {
     }
 }
 
+pub trait ReadExt {
+    fn skip(&mut self, len: usize) -> io::Result<()>;
+}
+
+impl<T: Read> ReadExt for T {
+    fn skip(&mut self, mut len: usize) -> io::Result<()> {
+        let mut buf = MaybeUninit::<[u8; 4096]>::uninit();
+        let buf = unsafe { buf.assume_init_mut() };
+        while len > 0 {
+            let l = min(buf.len(), len);
+            self.read_exact(&mut buf[..l])?;
+            len -= l;
+        }
+        Ok(())
+    }
+}
+
+pub trait ReadSeekExt {
+    fn skip(&mut self, len: usize) -> io::Result<()>;
+}
+
+impl<T: Read + Seek> ReadSeekExt for T {
+    fn skip(&mut self, len: usize) -> io::Result<()> {
+        if self.seek(SeekFrom::Current(len as i64)).is_err() {
+            // If the file is not actually seekable, fallback to read
+            ReadExt::skip(self, len)?;
+        }
+        Ok(())
+    }
+}
+
 pub trait BufReadExt {
     fn foreach_lines<F: FnMut(&mut String) -> bool>(&mut self, f: F);
     fn foreach_props<F: FnMut(&str, &str) -> bool>(&mut self, f: F);

native/src/boot/main.cpp

@@ -51,7 +51,7 @@ Supported actions:
     If [partition] is not specified, then attempt to extract either
     'init_boot' or 'boot'. Which partition was chosen can be determined
     by whichever 'init_boot.img' or 'boot.img' exists.
-    <payload.bin>/[outfile] can be '-' to be STDIN/STDOUT.
+    <payload.bin> can be '-' to be STDIN.
 
   hexpatch <file> <hexpattern1> <hexpattern2>
     Search <hexpattern1> in <file>, and replace it with <hexpattern2>

native/src/boot/payload.rs

@@ -7,7 +7,7 @@ use byteorder::{BigEndian, ReadBytesExt};
 use protobuf::{EnumFull, Message};
 
 use base::libc::c_char;
-use base::{ptr_to_str_result, StrErr};
+use base::{ptr_to_str_result, ReadSeekExt, StrErr};
 use base::{ResultExt, WriteExt};
 
 use crate::ffi;
@@ -27,7 +27,7 @@ const PAYLOAD_MAGIC: &str = "CrAU";
 
 fn do_extract_boot_from_payload(
     in_path: &str,
-    partition: Option<&str>,
+    partition_name: Option<&str>,
     out_path: Option<&str>,
 ) -> anyhow::Result<()> {
     let mut reader = BufReader::new(if in_path == "-" {
@@ -74,46 +74,50 @@ fn do_extract_boot_from_payload(
 
     let block_size = manifest.block_size() as u64;
 
-    let part = match partition {
+    let partition = match partition_name {
         None => {
             let boot = manifest
                 .partitions
                 .iter()
-                .find(|partition| partition.partition_name() == "init_boot");
+                .find(|p| p.partition_name() == "init_boot");
             let boot = match boot {
                 Some(boot) => Some(boot),
                 None => manifest
                     .partitions
                     .iter()
-                    .find(|partition| partition.partition_name() == "boot"),
+                    .find(|p| p.partition_name() == "boot"),
             };
             boot.ok_or(anyhow!("boot partition not found"))?
         }
-        Some(partition) => manifest
+        Some(name) => manifest
             .partitions
             .iter()
-            .find(|p| p.partition_name() == partition)
+            .find(|p| p.partition_name() == name)
-            .ok_or(anyhow!("partition '{partition}' not found"))?,
+            .ok_or(anyhow!("partition '{name}' not found"))?,
     };
 
     let out_str: String;
     let out_path = match out_path {
         None => {
-            out_str = format!("{}.img", part.partition_name());
+            out_str = format!("{}.img", partition.partition_name());
             out_str.as_str()
         }
-        Some(p) => p,
+        Some(s) => s,
     };
 
-    let mut out_file = if out_path == "-" {
+    let mut out_file =
-        unsafe { File::from_raw_fd(1) }
+        File::create(out_path).with_context(|| format!("cannot write to '{out_path}'"))?;
-    } else {
-        File::create(out_path).with_context(|| format!("cannot write to '{out_path}'"))?
-    };
 
-    let base_offset = reader.stream_position()? + manifest_sig_len as u64;
+    // Skip the manifest signature
+    reader.skip(manifest_sig_len as usize)?;
 
-    for operation in part.operations.iter() {
+    // Sort the install operations with data_offset so we will only ever need to seek forward
+    // This makes it possible to support non-seekable input file descriptors
+    let mut operations = partition.operations.clone();
+    operations.sort_by_key(|e| e.data_offset.unwrap_or(0));
+    let mut curr_data_offset: u64 = 0;
+
+    for operation in operations.iter() {
         let data_len = operation
             .data_length
             .ok_or(bad_payload!("data length not found"))? as usize;
@@ -131,8 +135,11 @@ fn do_extract_boot_from_payload(
         buf.resize(data_len, 0u8);
         let data = &mut buf[..data_len];
 
-        reader.seek(SeekFrom::Start(base_offset + data_offset))?;
+        // Skip to the next offset and read data
+        let skip = data_offset - curr_data_offset;
+        reader.skip(skip as usize)?;
         reader.read_exact(data)?;
+        curr_data_offset = data_offset + data_len as u64;
 
         let out_offset = operation
             .dst_extents