Optimize repo list fetching

app-core/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java

@@ -19,7 +19,6 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
     private static final String TABLE_NAME = "repos";
 
     private SQLiteDatabase mDb;
-    private Runnable adapterCb;
 
     public RepoDatabaseHelper(Context context) {
         super(context, "repo.db", null, DATABASE_VER);
@@ -54,13 +53,11 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
 
     public void clearRepo() {
         mDb.delete(TABLE_NAME, null, null);
-        notifyAdapter();
     }
 
 
     public void removeRepo(String id) {
         mDb.delete(TABLE_NAME, "id=?", new String[] { id });
-        notifyAdapter();
     }
 
     public void removeRepo(Repo repo) {
@@ -72,12 +69,10 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
             if (id == null) continue;
             mDb.delete(TABLE_NAME, "id=?", new String[] { id });
         }
-        notifyAdapter();
     }
 
     public void addRepo(Repo repo) {
         mDb.replace(TABLE_NAME, null, repo.getContentValues());
-        notifyAdapter();
     }
 
     public Repo getRepo(String id) {
@@ -116,18 +111,4 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         }
         return set;
     }
-
-    public void registerAdapterCallback(Runnable cb) {
-        adapterCb = cb;
-    }
-
-    public void unregisterAdapterCallback() {
-        adapterCb = null;
-    }
-
-    private void notifyAdapter() {
-        if (adapterCb != null) {
-            UiThreadHandler.run(adapterCb);
-        }
-    }
 }

app-core/src/main/java/com/topjohnwu/magisk/tasks/UpdateRepos.java

@@ -1,6 +1,7 @@
 package com.topjohnwu.magisk.tasks;
 
 import android.database.Cursor;
+import android.util.Pair;
 
 import com.topjohnwu.magisk.App;
 import com.topjohnwu.magisk.Config;
@@ -23,62 +24,46 @@ import java.text.SimpleDateFormat;
 import java.util.Collections;
 import java.util.Date;
 import java.util.Locale;
+import java.util.Queue;
 import java.util.Set;
 import java.util.TimeZone;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Future;
 
 public class UpdateRepos {
-    private static final int CPU_COUNT = Runtime.getRuntime().availableProcessors();
+    private static final DateFormat DATE_FORMAT;
-    private static final int CORE_POOL_SIZE = Math.max(2, CPU_COUNT - 1);
-    private static final DateFormat dateFormat;
 
     private App app = App.self;
     private Set<String> cached;
-    private ExecutorService threadPool;
+    private Queue<Pair<String, Date>> moduleQueue;
 
     static {
-        dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+        DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
-        dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+        DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
     }
 
-    private void waitTasks() {
+    private void runTasks(Runnable task) {
-        threadPool.shutdown();
+        Future[] futures = new Future[App.THREAD_POOL.getMaximumPoolSize() - 1];
-        while (true) {
+        for (int i = 0; i < futures.length; ++i) {
-            try {
+            futures[i] = App.THREAD_POOL.submit(task);
-                if (threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS))
-                    break;
-            } catch (InterruptedException ignored) {}
         }
-    }
+        for (Future f : futures) {
-
+            while (true) {
-    private void loadJSON(JSONArray array) throws JSONException, ParseException {
-        for (int i = 0; i < array.length(); i++) {
-            JSONObject rawRepo = array.getJSONObject(i);
-            String id = rawRepo.getString("name");
-            Date date = dateFormat.parse(rawRepo.getString("pushed_at"));
-            threadPool.execute(() -> {
-                Repo repo = app.repoDB.getRepo(id);
                 try {
-                    if (repo == null)
+                    f.get();
-                        repo = new Repo(id);
+                } catch (InterruptedException e) {
-                    else
+                    continue;
-                        cached.remove(id);
+                } catch (ExecutionException ignored) {}
-                    repo.update(date);
+                break;
-                    app.repoDB.addRepo(repo);
+            }
-                } catch (Repo.IllegalRepoException e) {
-                    Logger.debug(e.getMessage());
-                    app.repoDB.removeRepo(id);
-                }
-            });
         }
     }
 
     /* We sort repos by last push, which means that we only need to check whether the
      * first page is updated to determine whether the online repo database is changed
      */
-    private boolean loadPage(int page) {
+    private boolean parsePage(int page) {
         Request req = Networking.get(Utils.fmt(Const.Url.REPO_URL, page + 1));
         if (page == 0) {
             String etag = Config.get(Config.Key.ETAG_KEY);
@@ -99,7 +84,12 @@ public class UpdateRepos {
             return true;
 
         try {
-            loadJSON(res.getResult());
+            for (int i = 0; i < res.getResult().length(); i++) {
+                JSONObject rawRepo = res.getResult().getJSONObject(i);
+                String id = rawRepo.getString("name");
+                Date date = DATE_FORMAT.parse(rawRepo.getString("pushed_at"));
+                moduleQueue.offer(new Pair<>(id, date));
+            }
         } catch (JSONException | ParseException e) {
             // Should not happen, but if exception occurs, page load fails
             return false;
@@ -115,18 +105,44 @@ public class UpdateRepos {
         }
 
         String links = res.getConnection().getHeaderField(Const.Key.LINK_KEY);
-        return links == null || !links.contains("next") || loadPage(page + 1);
+        return links == null || !links.contains("next") || parsePage(page + 1);
     }
 
     private boolean loadPages() {
-        return loadPage(0);
+        if (!parsePage(0))
+            return false;
+        runTasks(() -> {
+            while (true) {
+                Pair<String, Date> pair = moduleQueue.poll();
+                if (pair == null)
+                    return;
+                Repo repo = app.repoDB.getRepo(pair.first);
+                try {
+                    if (repo == null)
+                        repo = new Repo(pair.first);
+                    else
+                        cached.remove(pair.first);
+                    repo.update(pair.second);
+                    app.repoDB.addRepo(repo);
+                } catch (Repo.IllegalRepoException e) {
+                    Logger.debug(e.getMessage());
+                    app.repoDB.removeRepo(pair.first);
+                }
+            }
+        });
+        return true;
     }
 
     private void fullReload() {
         Cursor c = app.repoDB.getRawCursor();
-        while (c.moveToNext()) {
+        runTasks(() -> {
-            Repo repo = new Repo(c);
+            while (true) {
-            threadPool.execute(() -> {
+                Repo repo;
+                synchronized (c) {
+                    if (!c.moveToNext())
+                        return;
+                    repo = new Repo(c);
+                }
                 try {
                     repo.update();
                     app.repoDB.addRepo(repo);
@@ -134,19 +150,17 @@ public class UpdateRepos {
                     Logger.debug(e.getMessage());
                     app.repoDB.removeRepo(repo);
                 }
-            });
+            }
-        }
+        });
-        waitTasks();
     }
 
     public void exec(boolean force) {
         Topic.reset(Topic.REPO_LOAD_DONE);
         App.THREAD_POOL.execute(() -> {
             cached = Collections.synchronizedSet(app.repoDB.getRepoIDSet());
-            threadPool = Executors.newFixedThreadPool(CORE_POOL_SIZE);
+            moduleQueue = new ConcurrentLinkedQueue<>();
 
             if (loadPages()) {
-                waitTasks();
                 // The leftover cached means they are removed from online repo
                 app.repoDB.removeRepo(cached);
             } else if (force) {

app-core/src/main/java/com/topjohnwu/magisk/utils/Topic.java

@@ -87,6 +87,10 @@ public class Topic {
         return true;
     }
 
+    public static boolean isPublished(AutoSubscriber sub) {
+        return isPublished(sub.getSubscribedTopics());
+    }
+
     private static class Store {
         boolean published = false;
         Set<Subscriber> subscribers = new HashSet<>();

app/src/full/java/com/topjohnwu/magisk/fragments/ReposFragment.java

@@ -50,11 +50,7 @@ public class ReposFragment extends BaseFragment implements Topic.Subscriber {
         mSwipeRefreshLayout.setRefreshing(true);
         recyclerView.setVisibility(View.GONE);
 
-        mSwipeRefreshLayout.setOnRefreshListener(() -> {
+        mSwipeRefreshLayout.setOnRefreshListener(() -> new UpdateRepos().exec(true));
-            recyclerView.setVisibility(View.VISIBLE);
-            emptyRv.setVisibility(View.GONE);
-            new UpdateRepos().exec(true);
-        });
 
         requireActivity().setTitle(R.string.downloads);
 
@@ -68,17 +64,21 @@ public class ReposFragment extends BaseFragment implements Topic.Subscriber {
 
     @Override
     public void onPublish(int topic, Object[] result) {
-        if (topic == Topic.MODULE_LOAD_DONE) {
+        switch (topic) {
-            adapter = new ReposAdapter(app.repoDB, (Map<String, Module>) result[0]);
+            case Topic.MODULE_LOAD_DONE:
-            app.repoDB.registerAdapterCallback(adapter::notifyDBChanged);
+                adapter = new ReposAdapter(app.repoDB, (Map<String, Module>) result[0]);
-            recyclerView.setAdapter(adapter);
+                recyclerView.setAdapter(adapter);
-            recyclerView.setVisibility(View.VISIBLE);
+                break;
-            emptyRv.setVisibility(View.GONE);
+            case Topic.REPO_LOAD_DONE:
+                if (adapter != null)
+                    adapter.notifyDBChanged();
+                break;
         }
-        if (Topic.isPublished(getSubscribedTopics())) {
+        if (Topic.isPublished(this)) {
             mSwipeRefreshLayout.setRefreshing(false);
-            recyclerView.setVisibility(adapter.getItemCount() == 0 ? View.GONE : View.VISIBLE);
+            boolean empty = adapter.getItemCount() == 0;
-            emptyRv.setVisibility(adapter.getItemCount() == 0 ? View.VISIBLE : View.GONE);
+            recyclerView.setVisibility(empty ? View.GONE : View.VISIBLE);
+            emptyRv.setVisibility(empty ? View.VISIBLE : View.GONE);
         }
     }
 
@@ -114,10 +114,4 @@ public class ReposFragment extends BaseFragment implements Topic.Subscriber {
         }
         return true;
     }
-
-    @Override
-    public void onDestroyView() {
-        super.onDestroyView();
-        app.repoDB.unregisterAdapterCallback();
-    }
 }

app/src/full/res/layout/fragment_repos.xml

@@ -20,8 +20,7 @@
             android:gravity="center"
             android:text="@string/no_modules_found"
             android:textSize="20sp"
-            android:textStyle="italic"
+            android:textStyle="italic" />
-            android:visibility="gone" />
 
         <androidx.recyclerview.widget.RecyclerView
             android:id="@+id/recyclerView"