Switch to new repo format

app/src/main/java/com/topjohnwu/magisk/core/Const.kt

@@ -54,6 +54,7 @@ object Const {
         const val GITHUB_API_URL = "https://api.github.com/"
         const val GITHUB_PAGE_URL = "https://topjohnwu.github.io/magisk_files/"
         const val JS_DELIVR_URL = "https://cdn.jsdelivr.net/gh/"
+        const val OFFICIAL_REPO = "https://magisk-modules-repo.github.io/submission/modules.json"
     }
 
     object Key {

app/src/main/java/com/topjohnwu/magisk/core/SplashActivity.kt

@@ -5,7 +5,7 @@ import android.content.Context
 import android.os.Bundle
 import com.topjohnwu.magisk.BuildConfig
 import com.topjohnwu.magisk.R
-import com.topjohnwu.magisk.data.network.GithubRawServices
+import com.topjohnwu.magisk.data.network.RawServices
 import com.topjohnwu.magisk.ktx.get
 import com.topjohnwu.magisk.ui.MainActivity
 import com.topjohnwu.magisk.view.Notifications
@@ -55,7 +55,7 @@ open class SplashActivity : Activity() {
         Shortcuts.setupDynamic(this)
 
         // Pre-fetch network stuffs
-        get<GithubRawServices>()
+        get<RawServices>()
 
         DONE = true
 

app/src/main/java/com/topjohnwu/magisk/core/download/Subject.kt

@@ -29,7 +29,7 @@ sealed class Subject : Parcelable {
         val module: Repo,
         override val action: Action
     ) : Subject() {
-        override val url: String get() = module.zipUrl
+        override val url: String get() = module.zip_url
         override val title: String get() = module.downloadFilename
 
         @IgnoredOnParcel

app/src/main/java/com/topjohnwu/magisk/core/model/UpdateInfo.kt

@@ -42,6 +42,22 @@ data class StubJson(
     val link: String = ""
 ) : Parcelable
 
+@JsonClass(generateAdapter = true)
+data class ModuleJson(
+    val id: String,
+    val last_update: Long,
+    val prop_url: String,
+    val zip_url: String,
+    val notes_url: String
+)
+
+@JsonClass(generateAdapter = true)
+data class RepoJson(
+    val name: String,
+    val last_update: Long,
+    val modules: List<ModuleJson>
+)
+
 @JsonClass(generateAdapter = true)
 data class CommitInfo(
     val sha: String

app/src/main/java/com/topjohnwu/magisk/core/model/module/Repo.kt

@@ -3,7 +3,7 @@ package com.topjohnwu.magisk.core.model.module
 import android.os.Parcelable
 import androidx.room.Entity
 import androidx.room.PrimaryKey
-import com.topjohnwu.magisk.core.Const
+import com.topjohnwu.magisk.core.model.ModuleJson
 import com.topjohnwu.magisk.data.repository.NetworkService
 import com.topjohnwu.magisk.ktx.get
 import com.topjohnwu.magisk.ktx.legalFilename
@@ -15,34 +15,40 @@ import java.util.*
 @Parcelize
 data class Repo(
     @PrimaryKey override var id: String,
-    override var name: String,
+    override var name: String = "",
-    override var author: String,
+    override var author: String = "",
-    override var version: String,
+    override var version: String = "",
-    override var versionCode: Int,
+    override var versionCode: Int = -1,
-    override var description: String,
+    override var description: String = "",
-    var last_update: Long
+    val last_update: Long,
+    val prop_url: String,
+    val zip_url: String,
+    val notes_url: String
 ) : BaseModule(), Parcelable {
 
     private val svc: NetworkService get() = get()
 
+    constructor(info: ModuleJson) : this(
+        id = info.id,
+        last_update = info.last_update,
+        prop_url = info.prop_url,
+        zip_url = info.zip_url,
+        notes_url = info.notes_url
+    )
+
     val lastUpdate get() = Date(last_update)
+    val lastUpdateString get() = DATE_FORMAT.format(lastUpdate)
+    val downloadFilename get() = "$name-$version($versionCode).zip".legalFilename()
 
-    val lastUpdateString: String get() = dateFormat.format(lastUpdate)
+    suspend fun notes() = svc.fetchString(notes_url)
-
-    val downloadFilename: String get() = "$name-$version($versionCode).zip".legalFilename()
-
-    suspend fun readme() = svc.fetchReadme(this)
-
-    val zipUrl: String get() = Const.Url.ZIP_URL.format(id)
-
-    constructor(id: String) : this(id, "", "", "", -1, "", 0)
 
     @Throws(IllegalRepoException::class)
-    private fun loadProps(props: String) {
+    suspend fun load() {
+        val props = svc.fetchString(prop_url)
         props.split("\\n".toRegex()).dropLastWhile { it.isEmpty() }.runCatching {
             parseProps(this)
         }.onFailure {
-            throw IllegalRepoException("Repo [$id] parse error: " + it.message)
+            throw IllegalRepoException("Repo [$id] parse error: ", it)
         }
 
         if (versionCode < 0) {
@@ -50,15 +56,10 @@ data class Repo(
         }
     }
 
-    @Throws(IllegalRepoException::class)
+    class IllegalRepoException(msg: String, cause: Throwable? = null) : Exception(msg, cause)
-    suspend fun update(lastUpdate: Date? = null) {
-        lastUpdate?.let { last_update = it.time }
-        loadProps(svc.fetchMetadata(this))
-    }
-
-    class IllegalRepoException(message: String) : Exception(message)
 
     companion object {
-        val dateFormat = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM)
+        private val DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM)
     }
+
 }

app/src/main/java/com/topjohnwu/magisk/core/tasks/RepoUpdater.kt

@@ -1,119 +1,41 @@
 package com.topjohnwu.magisk.core.tasks
 
-import com.squareup.moshi.JsonClass
-import com.topjohnwu.magisk.core.Const
 import com.topjohnwu.magisk.core.model.module.Repo
 import com.topjohnwu.magisk.data.database.RepoDao
 import com.topjohnwu.magisk.data.repository.NetworkService
 import com.topjohnwu.magisk.ktx.synchronized
-import kotlinx.coroutines.*
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
 import timber.log.Timber
-import java.net.HttpURLConnection
-import java.text.SimpleDateFormat
 import java.util.*
-import kotlin.collections.HashSet
 
 class RepoUpdater(
     private val svc: NetworkService,
     private val repoDB: RepoDao
 ) {
 
-    private fun String.trimEtag() = substring(indexOf('\"'), lastIndexOf('\"') + 1)
-
-    private suspend fun forcedReload(cached: MutableSet<String>) = coroutineScope {
-        cached.forEach {
-            launch {
-                val repo = repoDB.getRepo(it)!!
-                try {
-                    repo.update()
-                    repoDB.addRepo(repo)
-                } catch (e: Repo.IllegalRepoException) {
-                    Timber.e(e)
-                }
-            }
-        }
-    }
-
-    private suspend fun loadRepos(
-        repos: List<GithubRepoInfo>,
-        cached: MutableSet<String>
-    ) = coroutineScope {
-        repos.forEach {
-            // Skip submission
-            if (it.id == "submission")
-                return@forEach
-            launch {
-                val repo = repoDB.getRepo(it.id)?.apply { cached.remove(it.id) } ?: Repo(it.id)
-                try {
-                    repo.update(it.pushDate)
-                    repoDB.addRepo(repo)
-                } catch (e: Repo.IllegalRepoException) {
-                    Timber.e(e)
-                }
-            }
-        }
-    }
-
-    private enum class PageResult {
-        SUCCESS,
-        CACHED,
-        ERROR
-    }
-
-    private suspend fun loadPage(
-        cached: MutableSet<String>,
-        page: Int = 1,
-        etag: String = ""
-    ): PageResult = coroutineScope {
-        runCatching {
-            val result = svc.fetchRepos(page, etag)
-            result.run {
-                if (code() == HttpURLConnection.HTTP_NOT_MODIFIED)
-                    return@coroutineScope PageResult.CACHED
-
-                if (!isSuccessful)
-                    return@coroutineScope PageResult.ERROR
-
-                if (page == 1)
-                    repoDB.etagKey = headers()[Const.Key.ETAG_KEY].orEmpty().trimEtag()
-
-                val repoLoad = async { loadRepos(body()!!, cached) }
-                val next = if (headers()[Const.Key.LINK_KEY].orEmpty().contains("next")) {
-                    async { loadPage(cached, page + 1) }
-                } else {
-                    async { PageResult.SUCCESS }
-                }
-                repoLoad.await()
-                return@coroutineScope next.await()
-            }
-        }.getOrElse {
-            Timber.e(it)
-            PageResult.ERROR
-        }
-    }
-
     suspend fun run(forced: Boolean) = withContext(Dispatchers.IO) {
-        val cached = HashSet(repoDB.repoIDList).synchronized()
+        val cachedMap = HashMap<String, Date>().also { map ->
-        when (loadPage(cached, etag = repoDB.etagKey)) {
+            repoDB.getRepoStubs().forEach { map[it.id] = Date(it.last_update) }
-            PageResult.CACHED -> if (forced) forcedReload(cached)
+        }.synchronized()
-            PageResult.SUCCESS -> repoDB.removeRepos(cached)
+        val info = svc.fetchRepoInfo()
-            PageResult.ERROR -> Unit
+        coroutineScope {
+            info.modules.forEach {
+                launch {
+                    val lastUpdated = cachedMap.remove(it.id)
+                    if (forced || lastUpdated?.before(Date(it.last_update)) != false) {
+                        try {
+                            val repo = Repo(it).apply { load() }
+                            repoDB.addRepo(repo)
+                        } catch (e: Repo.IllegalRepoException) {
+                            Timber.e(e)
+                        }
+                    }
+                }
+            }
         }
+        repoDB.removeRepos(cachedMap.keys)
     }
 }
-
-private val dateFormat: SimpleDateFormat =
-    SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US).apply {
-        timeZone = TimeZone.getTimeZone("UTC")
-    }
-
-@JsonClass(generateAdapter = true)
-data class GithubRepoInfo(
-    val name: String,
-    val pushed_at: String
-) {
-    val id get() = name
-
-    @Transient
-    val pushDate = dateFormat.parse(pushed_at)!!
-}

app/src/main/java/com/topjohnwu/magisk/data/database/RepoDao.kt

@@ -6,7 +6,7 @@ import com.topjohnwu.magisk.core.model.module.Repo
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 
-@Database(version = 6, entities = [Repo::class, RepoEtag::class], exportSchema = false)
+@Database(version = 7, entities = [Repo::class], exportSchema = false)
 abstract class RepoDatabase : RoomDatabase() {
 
     abstract fun repoDao() : RepoDao
@@ -17,17 +17,11 @@ abstract class RepoDatabase : RoomDatabase() {
 @Dao
 abstract class RepoDao(private val db: RepoDatabase) {
 
-    val repoIDList get() = getRepoID().map { it.id }
-
     val repos: List<Repo> get() = when (Config.repoOrder) {
             Config.Value.ORDER_NAME -> getReposNameOrder()
             else -> getReposDateOrder()
         }
 
-    var etagKey: String
-        set(value) = addEtagRaw(RepoEtag(0, value))
-        get() = etagRaw()?.key.orEmpty()
-
     suspend fun clear() = withContext(Dispatchers.IO) { db.clearAllTables() }
 
     @Query("SELECT * FROM repos ORDER BY last_update DESC")
@@ -42,8 +36,8 @@ abstract class RepoDao(private val db: RepoDatabase) {
     @Query("SELECT * FROM repos WHERE id = :id")
     abstract fun getRepo(id: String): Repo?
 
-    @Query("SELECT id FROM repos")
+    @Query("SELECT id, last_update FROM repos")
-    protected abstract fun getRepoID(): List<RepoID>
+    abstract fun getRepoStubs(): List<RepoStub>
 
     @Delete
     abstract fun removeRepo(repo: Repo)
@@ -53,21 +47,9 @@ abstract class RepoDao(private val db: RepoDatabase) {
 
     @Query("DELETE FROM repos WHERE id IN (:idList)")
     abstract fun removeRepos(idList: Collection<String>)
-
-    @Query("SELECT * FROM etag")
-    protected abstract fun etagRaw(): RepoEtag?
-
-    @Insert(onConflict = OnConflictStrategy.REPLACE)
-    protected abstract fun addEtagRaw(etag: RepoEtag)
 }
 
-data class RepoID(
+data class RepoStub(
-    @PrimaryKey val id: String
+    @PrimaryKey val id: String,
+    val last_update: Long
 )
-
-@Entity(tableName = "etag")
-data class RepoEtag(
-    @PrimaryKey val id: Int,
-    val key: String
-)
-

app/src/main/java/com/topjohnwu/magisk/data/network/NetworkServices.kt

@@ -2,22 +2,17 @@ package com.topjohnwu.magisk.data.network
 
 import com.topjohnwu.magisk.core.Const
 import com.topjohnwu.magisk.core.model.BranchInfo
+import com.topjohnwu.magisk.core.model.RepoJson
 import com.topjohnwu.magisk.core.model.UpdateInfo
-import com.topjohnwu.magisk.core.tasks.GithubRepoInfo
 import okhttp3.ResponseBody
-import retrofit2.Response
 import retrofit2.http.*
 
 private const val REVISION = "revision"
-private const val MODULE = "module"
-private const val FILE = "file"
-private const val IF_NONE_MATCH = "If-None-Match"
 private const val BRANCH = "branch"
 private const val REPO = "repo"
 
 const val MAGISK_FILES = "topjohnwu/magisk_files"
 const val MAGISK_MAIN = "topjohnwu/Magisk"
-private const val MAGISK_MODULES = "Magisk-Modules-Repo"
 
 interface GithubPageServices {
 
@@ -46,13 +41,13 @@ interface JSDelivrServices {
     suspend fun fetchInstaller(@Path(REVISION) revision: String): ResponseBody
 }
 
-interface GithubRawServices {
+interface RawServices {
 
     @GET
     suspend fun fetchCustomUpdate(@Url url: String): UpdateInfo
 
-    @GET("$MAGISK_MODULES/{$MODULE}/master/{$FILE}")
+    @GET
-    suspend fun fetchModuleFile(@Path(MODULE) id: String, @Path(FILE) file: String): String
+    suspend fun fetchRepoInfo(@Url url: String): RepoJson
 
     @GET
     @Streaming
@@ -65,15 +60,6 @@ interface GithubRawServices {
 
 interface GithubApiServices {
 
-    @GET("users/$MAGISK_MODULES/repos")
-    @Headers("Accept: application/vnd.github.v3+json")
-    suspend fun fetchRepos(
-        @Query("page") page: Int,
-        @Header(IF_NONE_MATCH) etag: String,
-        @Query("sort") sort: String = "pushed",
-        @Query("per_page") count: Int = 100
-    ): Response<List<GithubRepoInfo>>
-
     @GET("repos/{$REPO}/branches/{$BRANCH}")
     @Headers("Accept: application/vnd.github.v3+json")
     suspend fun fetchBranch(

app/src/main/java/com/topjohnwu/magisk/data/repository/NetworkService.kt

@@ -9,7 +9,6 @@ import com.topjohnwu.magisk.core.Config.Value.STABLE_CHANNEL
 import com.topjohnwu.magisk.core.Const
 import com.topjohnwu.magisk.core.Info
 import com.topjohnwu.magisk.core.model.*
-import com.topjohnwu.magisk.core.model.module.Repo
 import com.topjohnwu.magisk.data.network.*
 import okhttp3.ResponseBody
 import retrofit2.HttpException
@@ -18,7 +17,7 @@ import java.io.IOException
 
 class NetworkService(
     private val pages: GithubPageServices,
-    private val raw: GithubRawServices,
+    private val raw: RawServices,
     private val jsd: JSDelivrServices,
     private val api: GithubApiServices
 ) {
@@ -68,7 +67,10 @@ class NetworkService(
         )
     }
 
-    // Byte streams
+    // Modules related
+    suspend fun fetchRepoInfo(url: String = Const.Url.OFFICIAL_REPO) = raw.fetchRepoInfo(url)
+
+    // Fetch files
     suspend fun fetchSafetynet() = jsd.fetchSafetynet()
     suspend fun fetchBootctl() = jsd.fetchBootctl()
     suspend fun fetchInstaller(): ResponseBody {
@@ -76,14 +78,8 @@ class NetworkService(
         return jsd.fetchInstaller(sha)
     }
     suspend fun fetchFile(url: String) = raw.fetchFile(url)
-
-    // Strings
-    suspend fun fetchMetadata(repo: Repo) = raw.fetchModuleFile(repo.id, "module.prop")
-    suspend fun fetchReadme(repo: Repo) = raw.fetchModuleFile(repo.id, "README.md")
     suspend fun fetchString(url: String) = raw.fetchString(url)
 
-    // API calls
-    suspend fun fetchRepos(page: Int, etag: String) = api.fetchRepos(page, etag)
     private suspend fun fetchCanaryVersion() = api.fetchBranch(MAGISK_FILES, "canary").commit.sha
     private suspend fun fetchMainVersion() = api.fetchBranch(MAGISK_MAIN, "master").commit.sha
 }

app/src/main/java/com/topjohnwu/magisk/di/NetworkingModule.kt

@@ -8,8 +8,8 @@ import com.topjohnwu.magisk.core.Const
 import com.topjohnwu.magisk.core.Info
 import com.topjohnwu.magisk.data.network.GithubApiServices
 import com.topjohnwu.magisk.data.network.GithubPageServices
-import com.topjohnwu.magisk.data.network.GithubRawServices
 import com.topjohnwu.magisk.data.network.JSDelivrServices
+import com.topjohnwu.magisk.data.network.RawServices
 import com.topjohnwu.magisk.ktx.precomputedText
 import com.topjohnwu.magisk.net.Networking
 import com.topjohnwu.magisk.net.NoSSLv3SocketFactory
@@ -30,7 +30,7 @@ import java.net.UnknownHostException
 val networkingModule = module {
     single { createOkHttpClient(get()) }
     single { createRetrofit(get()) }
-    single { createApiService<GithubRawServices>(get(), Const.Url.GITHUB_RAW_URL) }
+    single { createApiService<RawServices>(get(), Const.Url.GITHUB_RAW_URL) }
     single { createApiService<GithubApiServices>(get(), Const.Url.GITHUB_API_URL) }
     single { createApiService<GithubPageServices>(get(), Const.Url.GITHUB_PAGE_URL) }
     single { createApiService<JSDelivrServices>(get(), Const.Url.JS_DELIVR_URL) }

app/src/main/java/com/topjohnwu/magisk/events/ViewEvents.kt

@@ -22,10 +22,10 @@ class ViewActionEvent(val action: BaseActivity.() -> Unit) : ViewEvent(), Activi
     override fun invoke(activity: BaseUIActivity<*, *>) = action(activity)
 }
 
-class OpenChangelogEvent(val item: Repo) : ViewEventWithScope(), ContextExecutor {
+class OpenReadmeEvent(val item: Repo) : ViewEventWithScope(), ContextExecutor {
     override fun invoke(context: Context) {
         scope.launch {
-            MarkDownWindow.show(context, null, item::readme)
+            MarkDownWindow.show(context, null, item::notes)
         }
     }
 }

app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt

@@ -14,8 +14,8 @@ import com.topjohnwu.magisk.core.tasks.RepoUpdater
 import com.topjohnwu.magisk.data.database.RepoByNameDao
 import com.topjohnwu.magisk.data.database.RepoByUpdatedDao
 import com.topjohnwu.magisk.databinding.RvItem
+import com.topjohnwu.magisk.events.OpenReadmeEvent
 import com.topjohnwu.magisk.events.SelectModuleEvent
-import com.topjohnwu.magisk.events.OpenChangelogEvent
 import com.topjohnwu.magisk.events.SnackbarEvent
 import com.topjohnwu.magisk.events.dialog.ModuleInstallDialog
 import com.topjohnwu.magisk.ktx.addOnListChangedCallback
@@ -315,14 +315,14 @@ class ModuleViewModel(
     }
 
     fun infoPressed(item: RepoItem) =
-        if (isConnected.get()) OpenChangelogEvent(item.item).publish()
+        if (isConnected.get()) OpenReadmeEvent(item.item).publish()
         else SnackbarEvent(R.string.no_connection).publish()
 
 
     fun infoPressed(item: ModuleItem) {
         item.repo?.also {
             if (isConnected.get())
-                OpenChangelogEvent(it).publish()
+                OpenReadmeEvent(it).publish()
             else
                 SnackbarEvent(R.string.no_connection).publish()
         } ?: return

build.gradle.kts

@@ -18,7 +18,7 @@ buildscript {
     extra["vNav"] = vNav
 
     dependencies {
-        classpath("com.android.tools.build:gradle:4.0.1")
+        classpath("com.android.tools.build:gradle:4.0.2")
         classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.10")
         classpath("androidx.navigation:navigation-safe-args-gradle-plugin:${vNav}")