Massive improvement on Online Repo fetching

app/src/main/java/com/topjohnwu/magisk/MagiskManager.java

@@ -284,7 +284,7 @@ public class MagiskManager extends Application {
                     .setPeriodic(8 * 60 * 60 * 1000)
                     .build();
             ((JobScheduler) getSystemService(Context.JOB_SCHEDULER_SERVICE)).schedule(jobInfo);
-            loadModuleTask.setCallBack(() -> new UpdateRepos(this).exec());
+            loadModuleTask.setCallBack(() -> new UpdateRepos(this, false).exec());
         }
         // Fire asynctasks
         loadModuleTask.exec();

app/src/main/java/com/topjohnwu/magisk/ReposFragment.java

@@ -47,7 +47,7 @@ public class ReposFragment extends Fragment implements Topic.Subscriber {
         mSwipeRefreshLayout.setOnRefreshListener(() -> {
             recyclerView.setVisibility(View.VISIBLE);
             emptyRv.setVisibility(View.GONE);
-            new UpdateRepos(getActivity()).exec();
+            new UpdateRepos(getActivity(), true).exec();
         });
 
         getActivity().setTitle(R.string.downloads);

app/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java

@@ -2,11 +2,13 @@ package com.topjohnwu.magisk.asyncs;
 
 import android.content.Context;
 import android.content.SharedPreferences;
+import android.os.AsyncTask;
 
 import com.topjohnwu.magisk.MagiskManager;
 import com.topjohnwu.magisk.ReposFragment;
 import com.topjohnwu.magisk.container.Repo;
 import com.topjohnwu.magisk.database.RepoDatabaseHelper;
+import com.topjohnwu.magisk.utils.Logger;
 import com.topjohnwu.magisk.utils.WebService;
 
 import org.json.JSONArray;
@@ -38,8 +40,11 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
     private List<String> cached, etags, newEtags = new ArrayList<>();
     private RepoDatabaseHelper repoDB;
     private SharedPreferences prefs;
+    private boolean forceUpdate;
 
-    public UpdateRepos(Context context) {
+    private int tasks = 0;
+
+    public UpdateRepos(Context context, boolean force) {
         super(context);
         prefs = getMagiskManager().prefs;
         repoDB = getMagiskManager().repoDB;
@@ -51,6 +56,7 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
             prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply();
             repoDB.clearRepo();
         }
+        forceUpdate = force;
     }
 
     private void loadJSON(String jsonString) throws Exception {
@@ -66,20 +72,37 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
             String lastUpdate = jsonobject.getString("pushed_at");
             SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
             Date updatedDate = format.parse(lastUpdate);
+            ++tasks;
+            AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
                 Repo repo = repoDB.getRepo(id);
                 Boolean updated;
+                try {
                     if (repo == null) {
                         repo = new Repo(name, updatedDate);
                         updated = true;
                     } else {
                         // Popout from cached
                         cached.remove(id);
+                        if (forceUpdate) {
+                            repo.update();
+                            updated = true;
+                        } else {
                             updated = repo.update(updatedDate);
                         }
+                    }
                     if (updated) {
                         repoDB.addRepo(repo);
                         publishProgress();
                     }
+                    if (!id.equals(repo.getId())) {
+                        Logger.error("Repo [" + name + "] id=[" + repo.getId() + "] has illegal repo id");
+                    }
+                } catch (Repo.IllegalRepoException e) {
+                    Logger.error(e.getMessage());
+                    repoDB.removeRepo(id);
+                }
+                --tasks;
+            });
         }
     }
 
@@ -126,6 +149,17 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
         return true;
     }
 
+    private Void waitTasks() {
+        while (tasks > 0) {
+            try {
+                Thread.sleep(5);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+        return null;
+    }
+
     @Override
     protected void onProgressUpdate(Void... values) {
         if (ReposFragment.adapter != null)
@@ -138,9 +172,29 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
         cached = repoDB.getRepoIDList();
 
         if (!loadPage(0, CHECK_ETAG)) {
-            // Nothing changed
+            // Nothing changed online
-            return null;
+            if (forceUpdate) {
+                for (String id : cached) {
+                    if (id == null) continue;
+                    ++tasks;
+                    AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
+                        Repo repo = repoDB.getRepo(id);
+                        try {
+                            repo.update();
+                            repoDB.addRepo(repo);
+                        } catch (Repo.IllegalRepoException e) {
+                            Logger.error(e.getMessage());
+                            repoDB.removeRepo(repo);
                         }
+                        --tasks;
+                    });
+                }
+            }
+            return waitTasks();
+        }
+
+        // Wait till all tasks are done
+        waitTasks();
 
         // The leftover cached means they are removed from online repo
         repoDB.removeRepo(cached);

app/src/main/java/com/topjohnwu/magisk/container/BaseModule.java

@@ -9,8 +9,8 @@ import java.util.List;
 
 public abstract class BaseModule implements Comparable<BaseModule> {
 
-    private String mId, mName, mVersion, mAuthor, mDescription;
+    private String mId = null, mName, mVersion, mAuthor, mDescription;
-    private int mVersionCode = 0, templateVersion = 0;
+    private int mVersionCode = -1, templateVersion = -1;
 
     protected BaseModule() {}
 
@@ -38,7 +38,7 @@ public abstract class BaseModule implements Comparable<BaseModule> {
 
     protected void parseProps(List<String> props) { parseProps(props.toArray(new String[0])); }
 
-    protected void parseProps(String[] props) {
+    protected void parseProps(String[] props) throws NumberFormatException {
         for (String line : props) {
             String[] prop = line.split("=", 2);
             if (prop.length != 2)
@@ -59,9 +59,7 @@ public abstract class BaseModule implements Comparable<BaseModule> {
                     mVersion = prop[1];
                     break;
                 case "versionCode":
-                    try {
                     mVersionCode = Integer.parseInt(prop[1]);
-                    } catch (NumberFormatException ignored) {}
                     break;
                 case "author":
                     mAuthor = prop[1];
@@ -70,9 +68,7 @@ public abstract class BaseModule implements Comparable<BaseModule> {
                     mDescription = prop[1];
                     break;
                 case "template":
-                    try {
                     templateVersion = Integer.parseInt(prop[1]);
-                    } catch (NumberFormatException ignored) {}
                     break;
                 default:
                     break;

app/src/main/java/com/topjohnwu/magisk/container/Module.java

@@ -10,7 +10,9 @@ public class Module extends BaseModule {
 
     public Module(Shell shell, String path) {
 
+        try {
             parseProps(Utils.readFile(shell, path + "/module.prop"));
+        } catch (NumberFormatException ignored) {}
 
         mRemoveFile = path + "/remove";
         mDisableFile = path + "/disable";

app/src/main/java/com/topjohnwu/magisk/container/Repo.java

@@ -9,13 +9,15 @@ import java.util.Date;
 
 public class Repo extends BaseModule {
 
+    public static final int MIN_TEMPLATE_VER = 4;
+
     private static final String FILE_URL = "https://raw.githubusercontent.com/Magisk-Modules-Repo/%s/master/%s";
     private static final String ZIP_URL = "https://github.com/Magisk-Modules-Repo/%s/archive/master.zip";
 
     private String repoName;
     private Date mLastUpdate;
 
-    public Repo(String name, Date lastUpdate) {
+    public Repo(String name, Date lastUpdate) throws IllegalRepoException {
         mLastUpdate = lastUpdate;
         repoName = name;
         update();
@@ -27,13 +29,27 @@ public class Repo extends BaseModule {
         mLastUpdate = new Date(c.getLong(c.getColumnIndex("last_update")));
     }
 
-    public void update() {
+    public void update() throws IllegalRepoException {
         String props = WebService.getString(getManifestUrl());
         String lines[] = props.split("\\n");
+        try {
             parseProps(lines);
+        } catch (NumberFormatException e) {
+            throw new IllegalRepoException("Repo [" + repoName + "] parse error: " + e.getMessage());
         }
 
-    public boolean update(Date lastUpdate) {
+        if (getId() == null) {
+            throw new IllegalRepoException("Repo [" + repoName + "] does not contain id");
+        }
+        if (getVersionCode() < 0) {
+            throw new IllegalRepoException("Repo [" + repoName + "] does not contain versionCode");
+        }
+        if (getTemplateVersion() < MIN_TEMPLATE_VER) {
+            throw new IllegalRepoException("Repo [" + repoName + "] is outdated");
+        }
+    }
+
+    public boolean update(Date lastUpdate) throws IllegalRepoException {
         if (lastUpdate.after(mLastUpdate)) {
             mLastUpdate = lastUpdate;
             update();
@@ -49,6 +65,10 @@ public class Repo extends BaseModule {
         return values;
     }
 
+    public String getRepoName() {
+        return repoName;
+    }
+
     public String getZipUrl() {
         return String.format(ZIP_URL, repoName);
     }
@@ -64,4 +84,10 @@ public class Repo extends BaseModule {
     public Date getLastUpdate() {
         return mLastUpdate;
     }
+
+    public class IllegalRepoException extends Exception {
+        IllegalRepoException(String message) {
+            super(message);
+        }
+    }
 }

app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java

@@ -7,7 +7,6 @@ import android.database.sqlite.SQLiteOpenHelper;
 
 import com.topjohnwu.magisk.MagiskManager;
 import com.topjohnwu.magisk.container.Repo;
-import com.topjohnwu.magisk.utils.Logger;
 import com.topjohnwu.magisk.utils.Utils;
 
 import java.util.LinkedList;
@@ -17,7 +16,6 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
 
     private static final int DATABASE_VER = 2;
     private static final String TABLE_NAME = "repos";
-    private static final int MIN_TEMPLATE_VER = 3;
 
     private SQLiteDatabase mDb;
     private MagiskManager mm;
@@ -26,6 +24,10 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         super(context, "repo.db", null, DATABASE_VER);
         mDb = getWritableDatabase();
         mm = Utils.getMagiskManager(context);
+
+        // Clear bad repos
+        mDb.delete(TABLE_NAME, "template<?",
+                new String[] { String.valueOf(Repo.MIN_TEMPLATE_VER) });
     }
 
     @Override
@@ -53,9 +55,18 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         mDb.delete(TABLE_NAME, null, null);
     }
 
+
+    public void removeRepo(String id) {
+        mDb.delete(TABLE_NAME, "id=?", new String[] { id });
+    }
+
+    public void removeRepo(Repo repo) {
+        mDb.delete(TABLE_NAME, "repo_name=?", new String[] { repo.getRepoName() });
+    }
+
     public void removeRepo(List<String> list) {
         for (String id : list) {
-            Logger.dev("Remove from DB: " + id);
+            if (id == null) continue;
             mDb.delete(TABLE_NAME, "id=?", new String[] { id });
         }
     }
@@ -74,7 +85,9 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
     }
 
     public Cursor getRepoCursor() {
-        return mDb.query(TABLE_NAME, null, "template>=? AND template<=?", new String[] { String.valueOf(MIN_TEMPLATE_VER), String.valueOf(mm.magiskVersionCode) }, null, null, "name COLLATE NOCASE");
+        return mDb.query(TABLE_NAME, null, "template<=?",
+                new String[] { String.valueOf(mm.magiskVersionCode) },
+                null, null, "name COLLATE NOCASE");
     }
 
     public List<String> getRepoIDList() {