Reverted removing UpdateRepos temporarily

app/src/main/java/com/topjohnwu/magisk/Const.kt

@@ -59,6 +59,9 @@ object Const {
     }
 
     object Url {
+        @Deprecated("This shouldn't be used. There's literally no need for it")
+        const val REPO_URL =
+            "https://api.github.com/users/Magisk-Modules-Repo/repos?per_page=100&sort=pushed&page=%d"
         const val FILE_URL = "https://raw.githubusercontent.com/Magisk-Modules-Repo/%s/master/%s"
         const val ZIP_URL = "https://github.com/Magisk-Modules-Repo/%s/archive/master.zip"
         const val MODULE_INSTALLER =

app/src/main/java/com/topjohnwu/magisk/data/database/RepoDatabaseHelper.java

@@ -0,0 +1,118 @@
+package com.topjohnwu.magisk.data.database;
+
+import android.content.Context;
+import android.database.Cursor;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+
+import com.topjohnwu.magisk.Config;
+import com.topjohnwu.magisk.model.entity.Repo;
+
+import java.util.HashSet;
+import java.util.Set;
+
+@Deprecated
+public class RepoDatabaseHelper extends SQLiteOpenHelper {
+
+    private static final int DATABASE_VER = 5;
+    private static final String TABLE_NAME = "repos";
+
+    private final SQLiteDatabase mDb;
+
+    @Deprecated
+    public RepoDatabaseHelper(Context context) {
+        super(context, "repo.db", null, DATABASE_VER);
+        mDb = getWritableDatabase();
+    }
+
+    @Override
+    public void onCreate(SQLiteDatabase db) {
+        onUpgrade(db, 0, DATABASE_VER);
+    }
+
+    @Override
+    public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+        if (oldVersion != newVersion) {
+            // Nuke old DB and create new table
+            db.execSQL("DROP TABLE IF EXISTS " + TABLE_NAME);
+            db.execSQL(
+                    "CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " " +
+                            "(id TEXT, name TEXT, version TEXT, versionCode INT, " +
+                            "author TEXT, description TEXT, last_update INT, PRIMARY KEY(id))");
+            Config.remove(Config.Key.ETAG_KEY);
+        }
+    }
+
+    @Override
+    public void onDowngrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+        onUpgrade(db, 0, DATABASE_VER);
+    }
+
+    @Deprecated
+    public void clearRepo() {
+        mDb.delete(TABLE_NAME, null, null);
+    }
+
+
+    @Deprecated
+    public void removeRepo(String id) {
+        mDb.delete(TABLE_NAME, "id=?", new String[]{id});
+    }
+
+    @Deprecated
+    public void removeRepo(Repo repo) {
+        removeRepo(repo.getId());
+    }
+
+    @Deprecated
+    public void removeRepo(Iterable<String> list) {
+        for (String id : list) {
+            if (id == null) continue;
+            mDb.delete(TABLE_NAME, "id=?", new String[]{id});
+        }
+    }
+
+    @Deprecated
+    public void addRepo(Repo repo) {
+        mDb.replace(TABLE_NAME, null, repo.getContentValues());
+    }
+
+    @Deprecated
+    public Repo getRepo(String id) {
+        try (Cursor c = mDb.query(TABLE_NAME, null, "id=?", new String[]{id}, null, null, null)) {
+            if (c.moveToNext()) {
+                return new Repo(c);
+            }
+        }
+        return null;
+    }
+
+    @Deprecated
+    public Cursor getRawCursor() {
+        return mDb.query(TABLE_NAME, null, null, null, null, null, null);
+    }
+
+    @Deprecated
+    public Cursor getRepoCursor() {
+        String orderBy = null;
+        switch ((int) Config.get(Config.Key.REPO_ORDER)) {
+            case Config.Value.ORDER_NAME:
+                orderBy = "name COLLATE NOCASE";
+                break;
+            case Config.Value.ORDER_DATE:
+                orderBy = "last_update DESC";
+        }
+        return mDb.query(TABLE_NAME, null, null, null, null, null, orderBy);
+    }
+
+    @Deprecated
+    public Set<String> getRepoIDSet() {
+        HashSet<String> set = new HashSet<>(300);
+        try (Cursor c = mDb.query(TABLE_NAME, null, null, null, null, null, null)) {
+            while (c.moveToNext()) {
+                set.add(c.getString(c.getColumnIndex("id")));
+            }
+        }
+        return set;
+    }
+}

app/src/main/java/com/topjohnwu/magisk/di/DatabaseModule.kt

@@ -3,6 +3,7 @@ package com.topjohnwu.magisk.di
 import android.content.Context
 import androidx.room.Room
 import com.topjohnwu.magisk.data.database.*
+import com.topjohnwu.magisk.tasks.UpdateRepos
 import org.koin.dsl.module
 
 
@@ -13,6 +14,8 @@ val databaseModule = module {
     single { SettingsDao() }
     single { StringDao() }
     single { createRepositoryDao(get()) }
+    single { RepoDatabaseHelper(get()) }
+    single { UpdateRepos(get()) }
 }
 
 fun createDatabase(context: Context): AppDatabase =

app/src/main/java/com/topjohnwu/magisk/di/ViewModelsModule.kt

@@ -18,7 +18,7 @@ val viewModelModules = module {
     viewModel { HomeViewModel(get()) }
     viewModel { SuperuserViewModel(get(), get(), get(), get()) }
     viewModel { HideViewModel(get(), get()) }
-    viewModel { ModuleViewModel(get(), get()) }
+    viewModel { ModuleViewModel(get(), get(), get(), get()) }
     viewModel { LogViewModel(get(), get()) }
     viewModel { (action: String, uri: Uri?) -> FlashViewModel(action, uri, get()) }
     viewModel { SuRequestViewModel(get(), get(), get(SUTimeout), get()) }

app/src/main/java/com/topjohnwu/magisk/tasks/UpdateRepos.java

@@ -0,0 +1,185 @@
+package com.topjohnwu.magisk.tasks;
+
+import android.database.Cursor;
+import android.util.Pair;
+
+import com.topjohnwu.magisk.App;
+import com.topjohnwu.magisk.Config;
+import com.topjohnwu.magisk.Const;
+import com.topjohnwu.magisk.data.database.RepoDatabaseHelper;
+import com.topjohnwu.magisk.model.entity.Repo;
+import com.topjohnwu.magisk.utils.Logger;
+import com.topjohnwu.magisk.utils.Utils;
+import com.topjohnwu.net.Networking;
+import com.topjohnwu.net.Request;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.net.HttpURLConnection;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Queue;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import androidx.annotation.NonNull;
+import io.reactivex.Single;
+
+@Deprecated
+public class UpdateRepos {
+    private static final DateFormat DATE_FORMAT;
+
+    static {
+        DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+        DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
+    }
+
+    @NonNull
+    private final RepoDatabaseHelper repoDB;
+    private Set<String> cached;
+    private Queue<Pair<String, Date>> moduleQueue;
+
+    public UpdateRepos(@NonNull RepoDatabaseHelper repoDatabase) {
+        repoDB = repoDatabase;
+    }
+
+    private void runTasks(Runnable task) {
+        Future[] futures = new Future[App.THREAD_POOL.getMaximumPoolSize() - 1];
+        for (int i = 0; i < futures.length; ++i) {
+            futures[i] = App.THREAD_POOL.submit(task);
+        }
+        for (Future f : futures) {
+            while (true) {
+                try {
+                    f.get();
+                } catch (InterruptedException e) {
+                    continue;
+                } catch (ExecutionException ignored) {
+                }
+                break;
+            }
+        }
+    }
+
+    /* We sort repos by last push, which means that we only need to check whether the
+     * first page is updated to determine whether the online repo database is changed
+     */
+    private boolean parsePage(int page) {
+        Request req = Networking.get(Utils.fmt(Const.Url.REPO_URL, page + 1));
+        if (page == 0) {
+            String etag = Config.get(Config.Key.ETAG_KEY);
+            if (etag != null)
+                req.addHeaders(Const.Key.IF_NONE_MATCH, etag);
+        }
+        Request.Result<JSONArray> res = req.execForJSONArray();
+        // JSON not updated
+        if (res.getCode() == HttpURLConnection.HTTP_NOT_MODIFIED)
+            return false;
+        // Network error
+        if (res.getResult() == null) {
+            cached.clear();
+            return true;
+        }
+        // Current page is the last page
+        if (res.getResult().length() == 0)
+            return true;
+
+        try {
+            for (int i = 0; i < res.getResult().length(); i++) {
+                JSONObject rawRepo = res.getResult().getJSONObject(i);
+                String id = rawRepo.getString("name");
+                Date date = DATE_FORMAT.parse(rawRepo.getString("pushed_at"));
+                moduleQueue.offer(new Pair<>(id, date));
+            }
+        } catch (JSONException | ParseException e) {
+            // Should not happen, but if exception occurs, page load fails
+            return false;
+        }
+
+        // Update ETAG
+        if (page == 0) {
+            String etag = res.getConnection().getHeaderField(Config.Key.ETAG_KEY);
+            if (etag != null) {
+                etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1);
+                Config.set(Config.Key.ETAG_KEY, etag);
+            }
+        }
+
+        String links = res.getConnection().getHeaderField(Const.Key.LINK_KEY);
+        return links == null || !links.contains("next") || parsePage(page + 1);
+    }
+
+    private boolean loadPages() {
+        if (!parsePage(0))
+            return false;
+        runTasks(() -> {
+            while (true) {
+                Pair<String, Date> pair = moduleQueue.poll();
+                if (pair == null)
+                    return;
+                Repo repo = repoDB.getRepo(pair.first);
+                try {
+                    if (repo == null)
+                        repo = new Repo(pair.first);
+                    else
+                        cached.remove(pair.first);
+                    repo.update(pair.second);
+                    repoDB.addRepo(repo);
+                } catch (Repo.IllegalRepoException e) {
+                    Logger.debug(e.getMessage());
+                    repoDB.removeRepo(pair.first);
+                }
+            }
+        });
+        return true;
+    }
+
+    private void fullReload() {
+        Cursor c = repoDB.getRawCursor();
+        runTasks(() -> {
+            while (true) {
+                Repo repo;
+                synchronized (c) {
+                    if (!c.moveToNext())
+                        return;
+                    repo = new Repo(c);
+                }
+                try {
+                    repo.update();
+                    repoDB.addRepo(repo);
+                } catch (Repo.IllegalRepoException e) {
+                    Logger.debug(e.getMessage());
+                    repoDB.removeRepo(repo);
+                }
+            }
+        });
+    }
+
+    public Single<Boolean> exec(boolean force) {
+        return Single.fromCallable(() -> {
+            cached = Collections.synchronizedSet(repoDB.getRepoIDSet());
+            moduleQueue = new ConcurrentLinkedQueue<>();
+
+            if (loadPages()) {
+                // The leftover cached means they are removed from online repo
+                repoDB.removeRepo(cached);
+            } else if (force) {
+                fullReload();
+            }
+            return force; // not important
+        });
+    }
+
+    public Single<Boolean> exec() {
+        return exec(false);
+    }
+}

app/src/main/java/com/topjohnwu/magisk/ui/SplashActivity.kt

@@ -5,11 +5,15 @@ import android.os.Bundle
 import android.text.TextUtils
 import androidx.appcompat.app.AlertDialog
 import androidx.appcompat.app.AppCompatActivity
+import com.skoumal.teanity.extensions.subscribeK
 import com.topjohnwu.magisk.*
+import com.topjohnwu.magisk.tasks.UpdateRepos
 import com.topjohnwu.magisk.utils.Utils
 import com.topjohnwu.magisk.view.Notifications
 import com.topjohnwu.magisk.view.Shortcuts
+import com.topjohnwu.net.Networking
 import com.topjohnwu.superuser.Shell
+import org.koin.android.ext.android.get
 
 open class SplashActivity : AppCompatActivity() {
 
@@ -62,8 +66,9 @@ open class SplashActivity : AppCompatActivity() {
             // Load modules
             //Utils.loadModules(false)
             // Load repos
-            //if (Networking.checkNetworkStatus(this))
+            if (Networking.checkNetworkStatus(this)) {
-            //UpdateRepos().exec()
+                get<UpdateRepos>().exec().subscribeK()
+            }
         }
 
         val intent = Intent(this, ClassMap[MainActivity::class.java])

app/src/main/java/com/topjohnwu/magisk/ui/base/BasePreferenceFragment.kt

@@ -15,6 +15,7 @@ import com.topjohnwu.magisk.App
 import com.topjohnwu.magisk.Config
 import com.topjohnwu.magisk.KConfig
 import com.topjohnwu.magisk.R
+import com.topjohnwu.magisk.data.database.RepoDatabaseHelper
 import com.topjohnwu.magisk.data.repository.ModuleRepository
 import com.topjohnwu.magisk.data.repository.SettingRepository
 import org.koin.android.ext.android.inject
@@ -22,6 +23,7 @@ import org.koin.android.ext.android.inject
 abstract class BasePreferenceFragment : PreferenceFragmentCompat(),
     SharedPreferences.OnSharedPreferenceChangeListener {
 
+    protected val repoDatabase: RepoDatabaseHelper by inject()
     protected val prefs: SharedPreferences by inject()
     protected val app: App by inject()
     protected val settingRepo: SettingRepository by inject()

app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt

@@ -1,24 +1,30 @@
 package com.topjohnwu.magisk.ui.module
 
 import android.content.res.Resources
+import android.database.Cursor
 import com.skoumal.teanity.databinding.ComparableRvItem
 import com.skoumal.teanity.extensions.addOnPropertyChangedCallback
+import com.skoumal.teanity.extensions.doOnSuccessUi
 import com.skoumal.teanity.extensions.subscribeK
 import com.skoumal.teanity.util.DiffObservableList
 import com.skoumal.teanity.util.KObservableField
 import com.topjohnwu.magisk.BR
 import com.topjohnwu.magisk.R
+import com.topjohnwu.magisk.data.database.RepoDatabaseHelper
 import com.topjohnwu.magisk.data.repository.ModuleRepository
+import com.topjohnwu.magisk.model.entity.Repo
 import com.topjohnwu.magisk.model.entity.recycler.ModuleRvItem
 import com.topjohnwu.magisk.model.entity.recycler.RepoRvItem
 import com.topjohnwu.magisk.model.entity.recycler.SectionRvItem
 import com.topjohnwu.magisk.model.events.InstallModuleEvent
 import com.topjohnwu.magisk.model.events.OpenChangelogEvent
 import com.topjohnwu.magisk.model.events.OpenFilePickerEvent
+import com.topjohnwu.magisk.tasks.UpdateRepos
 import com.topjohnwu.magisk.ui.base.MagiskViewModel
 import com.topjohnwu.magisk.utils.toSingle
 import com.topjohnwu.magisk.utils.update
 import com.topjohnwu.magisk.utils.zip
+import io.reactivex.Single
 import io.reactivex.android.schedulers.AndroidSchedulers
 import io.reactivex.disposables.Disposable
 import io.reactivex.schedulers.Schedulers
@@ -26,7 +32,9 @@ import me.tatarka.bindingcollectionadapter2.OnItemBind
 
 class ModuleViewModel(
     private val resources: Resources,
-    private val moduleRepo: ModuleRepository
+    private val moduleRepo: ModuleRepository,
+    private val repoDatabase: RepoDatabaseHelper,
+    private val repoUpdater: UpdateRepos
 ) : MagiskViewModel() {
 
     val query = KObservableField("")
@@ -54,7 +62,7 @@ class ModuleViewModel(
     fun repoPressed(item: RepoRvItem) = OpenChangelogEvent(item.item).publish()
     fun downloadPressed(item: RepoRvItem) = InstallModuleEvent(item.item).publish()
 
-    fun refresh(forceReload: Boolean) {
+    fun refreshNew(forceReload: Boolean) {
         val updateInstalled = moduleRepo.fetchInstalledModules()
             .flattenAsFlowable { it }
             .map { ModuleRvItem(it) }
@@ -79,6 +87,24 @@ class ModuleViewModel(
             .subscribeK { itemsRemote.update(it.first, it.second) }
     }
 
+    fun refresh/*Old*/(force: Boolean) {
+        moduleRepo.fetchInstalledModules()
+            .flattenAsFlowable { it }
+            .map { ModuleRvItem(it) }
+            .toList()
+            .map { it to itemsInstalled.calculateDiff(it) }
+            .doOnSuccessUi { itemsInstalled.update(it.first, it.second) }
+            .flatMap { repoUpdater.exec() }
+            .flatMap { Single.fromCallable { repoDatabase.repoCursor.toList { Repo(it) } } }
+            .flattenAsFlowable { it }
+            .map { RepoRvItem(it) }
+            .toList()
+            .doOnSuccess { allItems.update(it) }
+            .flatMap { queryRaw() }
+            .applyViewModel(this)
+            .subscribeK { itemsRemote.update(it.first, it.second) }
+    }
+
     private fun query() = queryRaw()
         .subscribeK { itemsRemote.update(it.first, it.second) }
 
@@ -112,6 +138,12 @@ class ModuleViewModel(
                 groupedItems.getOrElse(MODULE_REMOTE) { listOf() }.withTitle(R.string.not_installed)
     }
 
+    private fun <Result> Cursor.toList(transformer: (Cursor) -> Result): List<Result> {
+        val out = mutableListOf<Result>()
+        while (moveToNext()) out.add(transformer(this))
+        return out
+    }
+
     companion object {
         protected const val MODULE_INSTALLED = 0
         protected const val MODULE_REMOTE = 1

app/src/main/java/com/topjohnwu/magisk/ui/settings/SettingsFragment.java

@@ -107,12 +107,12 @@ public final class SettingsFragment extends BasePreferenceFragment {
             DownloadApp.restore();
             return true;
         });
-        Preference clear = findPreference("clear");
+        findPreference("clear").setOnPreferenceClickListener(pref -> {
-        clear.setOnPreferenceClickListener(pref -> {
             getPrefs().edit().remove(Config.Key.ETAG_KEY).apply();
-            getModuleRepo().deleteAllCached().subscribeOn(Schedulers.io()).subscribe(() -> {
+            getRepoDatabase().clearRepo();
-            }, throwable -> {
+            //getModuleRepo().deleteAllCached().subscribeOn(Schedulers.io()).subscribe(() -> {
-            });
+            //}, throwable -> {
+            //});
             Utils.toast(R.string.repo_cache_cleared, Toast.LENGTH_SHORT);
             return true;
         });