From 153d0f55056f69eb42f7eedd8286d0da2941a2a5 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Wed, 4 Jul 2018 20:13:12 +0800 Subject: [PATCH] Small optimization to UpdateRepos --- .../magisk/asyncs/ProcessRepoZip.java | 1 - .../topjohnwu/magisk/asyncs/UpdateRepos.java | 116 +++++++----------- .../com/topjohnwu/magisk/container/Repo.java | 14 +-- .../magisk/database/RepoDatabaseHelper.java | 18 +-- .../topjohnwu/magisk/utils/WebService.java | 57 +++++---- 5 files changed, 89 insertions(+), 117 deletions(-) diff --git a/src/full/java/com/topjohnwu/magisk/asyncs/ProcessRepoZip.java b/src/full/java/com/topjohnwu/magisk/asyncs/ProcessRepoZip.java index 405193a3b..f0f564b68 100644 --- a/src/full/java/com/topjohnwu/magisk/asyncs/ProcessRepoZip.java +++ b/src/full/java/com/topjohnwu/magisk/asyncs/ProcessRepoZip.java @@ -90,7 +90,6 @@ public class ProcessRepoZip extends ParallelTask { HttpURLConnection conn; do { conn = WebService.request(mLink, null); - if (conn == null) return null; total = conn.getContentLength(); if (total < 0) conn.disconnect(); diff --git a/src/full/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java b/src/full/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java index 000444df1..55b7685f5 100644 --- a/src/full/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java +++ b/src/full/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java @@ -1,6 +1,8 @@ package com.topjohnwu.magisk.asyncs; +import android.database.Cursor; import android.os.AsyncTask; +import android.text.TextUtils; import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.ReposFragment; @@ -11,20 +13,22 @@ import com.topjohnwu.magisk.utils.Utils; import com.topjohnwu.magisk.utils.WebService; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; -import java.io.File; import java.net.HttpURLConnection; import java.text.DateFormat; +import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; public class UpdateRepos extends ParallelTask { @@ -35,7 +39,8 @@ public class UpdateRepos extends ParallelTask { private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); private MagiskManager mm; - private List cached, etags, newEtags = new LinkedList<>(); + private List etags, newEtags = new LinkedList<>(); + private Set cached; private boolean forceUpdate; private AtomicInteger taskCount = new AtomicInteger(0); final private Object allDone = new Object(); @@ -43,13 +48,6 @@ public class UpdateRepos extends ParallelTask { public UpdateRepos(boolean force) { mm = MagiskManager.get(); mm.repoLoadDone.reset(); - // Legacy data cleanup - File old = new File(mm.getApplicationInfo().dataDir + "/shared_prefs", "RepoMap.xml"); - if (old.exists() || mm.prefs.getString("repomap", null) != null) { - old.delete(); - mm.prefs.edit().remove("version").remove("repomap").remove(Const.Key.ETAG_KEY).apply(); - mm.repoDB.clearRepo(); - } forceUpdate = force; } @@ -82,75 +80,61 @@ public class UpdateRepos extends ParallelTask { } } - private void loadJSON(String jsonString) throws Exception { + private boolean loadJSON(String jsonString) throws JSONException, ParseException { JSONArray jsonArray = new JSONArray(jsonString); - // Empty page, throw error + // Empty page, halt if (jsonArray.length() == 0) - throw new Exception(); + return false; for (int i = 0; i < jsonArray.length(); i++) { JSONObject rawRepo = jsonArray.getJSONObject(i); String id = rawRepo.getString("description"); String name = rawRepo.getString("name"); Date date = dateFormat.parse(rawRepo.getString("pushed_at")); - final List c = cached; + Set set = Collections.synchronizedSet(cached); queueTask(() -> { Repo repo = mm.repoDB.getRepo(id); - Boolean updated; try { - if (repo == null) { - repo = new Repo(name, date); - updated = true; - } else { - // Popout from cached - synchronized (c) { - c.remove(id); - } - if (forceUpdate) { - repo.update(); - updated = true; - } else { - updated = repo.update(date); - } - } - if (updated) { - mm.repoDB.addRepo(repo); - publishProgress(); - } + if (repo == null) + repo = new Repo(name); + else + set.remove(id); + repo.update(date); + mm.repoDB.addRepo(repo); + publishProgress(); } catch (Repo.IllegalRepoException e) { Logger.debug(e.getMessage()); mm.repoDB.removeRepo(id); } }); } + return true; } private boolean loadPage(int page, int mode) { Map header = new HashMap<>(); - String etag = (mode == CHECK_ETAG && page < etags.size()) ? etags.get(page) : ""; - header.put(Const.Key.IF_NONE_MATCH, etag); + if (mode == CHECK_ETAG && page < etags.size()) + header.put(Const.Key.IF_NONE_MATCH, etags.get(page)); String url = Utils.fmt(Const.Url.REPO_URL, page + 1); - HttpURLConnection conn = WebService.request(url, header); try { - if (conn == null) - throw new Exception(); + HttpURLConnection conn = WebService.request(url, header); if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { // Current page is not updated, check the next page - return page + 1 < etags.size() && loadPage(page + 1, CHECK_ETAG); + return loadPage(page + 1, CHECK_ETAG); } - loadJSON(WebService.getString(conn)); + if (!loadJSON(WebService.getString(conn))) + return mode != CHECK_ETAG; } catch (Exception e) { e.printStackTrace(); - // Don't continue - return true; + return false; } /* If one page is updated, we force update all pages */ // Update ETAG - etag = header.get(Const.Key.ETAG_KEY); + String etag = header.get(Const.Key.ETAG_KEY); etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1); if (mode == LOAD_PREV) { // We are loading a previous page, push the new tag to the front @@ -188,40 +172,32 @@ public class UpdateRepos extends ParallelTask { @Override protected Void doInBackground(Void... voids) { - etags = new ArrayList<>(Arrays.asList(mm.prefs.getString(Const.Key.ETAG_KEY, "").split(","))); - cached = mm.repoDB.getRepoIDList(); + etags = Arrays.asList(mm.prefs.getString(Const.Key.ETAG_KEY, "").split(",")); + cached = mm.repoDB.getRepoIDSet(); - if (!loadPage(0, CHECK_ETAG)) { - // Nothing changed online - if (forceUpdate) { - for (String id : cached) { - if (id == null) continue; - queueTask(() -> { - Repo repo = mm.repoDB.getRepo(id); - try { - repo.update(); - mm.repoDB.addRepo(repo); - } catch (Repo.IllegalRepoException e) { - Logger.debug(e.getMessage()); - mm.repoDB.removeRepo(repo); - } - }); - } - } - waitTasks(); - } else { + if (loadPage(0, CHECK_ETAG)) { waitTasks(); // The leftover cached means they are removed from online repo mm.repoDB.removeRepo(cached); // Update ETag - StringBuilder etagBuilder = new StringBuilder(); - for (int i = 0; i < newEtags.size(); ++i) { - if (i != 0) etagBuilder.append(","); - etagBuilder.append(newEtags.get(i)); + mm.prefs.edit().putString(Const.Key.ETAG_KEY, TextUtils.join(",", newEtags)).apply(); + } else if (forceUpdate) { + Cursor c = mm.repoDB.getRawCursor(); + while (c.moveToNext()) { + Repo repo = new Repo(c); + queueTask(() -> { + try { + repo.update(); + mm.repoDB.addRepo(repo); + } catch (Repo.IllegalRepoException e) { + Logger.debug(e.getMessage()); + mm.repoDB.removeRepo(repo); + } + }); } - mm.prefs.edit().putString(Const.Key.ETAG_KEY, etagBuilder.toString()).apply(); + waitTasks(); } return null; } diff --git a/src/full/java/com/topjohnwu/magisk/container/Repo.java b/src/full/java/com/topjohnwu/magisk/container/Repo.java index fc99fe790..a422983a6 100644 --- a/src/full/java/com/topjohnwu/magisk/container/Repo.java +++ b/src/full/java/com/topjohnwu/magisk/container/Repo.java @@ -17,10 +17,8 @@ public class Repo extends BaseModule { private String repoName; private Date mLastUpdate; - public Repo(String name, Date lastUpdate) throws IllegalRepoException { - mLastUpdate = lastUpdate; + public Repo(String name) { repoName = name; - update(); } public Repo(Cursor c) { @@ -48,13 +46,9 @@ public class Repo extends BaseModule { } } - public boolean update(Date lastUpdate) throws IllegalRepoException { - if (lastUpdate.after(mLastUpdate)) { - mLastUpdate = lastUpdate; - update(); - return true; - } - return false; + public void update(Date lastUpdate) throws IllegalRepoException { + mLastUpdate = lastUpdate; + update(); } @Override diff --git a/src/full/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java b/src/full/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java index 442526d84..acc4fecb0 100644 --- a/src/full/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java +++ b/src/full/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java @@ -10,8 +10,8 @@ import com.topjohnwu.magisk.container.Repo; import com.topjohnwu.magisk.utils.Const; import com.topjohnwu.magisk.utils.Utils; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; public class RepoDatabaseHelper extends SQLiteOpenHelper { @@ -74,7 +74,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { mDb.delete(TABLE_NAME, "repo_name=?", new String[] { repo.getRepoName() }); } - public void removeRepo(List list) { + public void removeRepo(Iterable list) { for (String id : list) { if (id == null) continue; mDb.delete(TABLE_NAME, "id=?", new String[] { id }); @@ -94,6 +94,10 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { return null; } + public Cursor getRawCursor() { + return mDb.query(TABLE_NAME, null, null, null, null, null, null); + } + public Cursor getRepoCursor() { String orderBy = null; switch (mm.repoOrder) { @@ -108,13 +112,13 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { null, null, orderBy); } - public List getRepoIDList() { - LinkedList ret = new LinkedList<>(); + public Set getRepoIDSet() { + HashSet set = new HashSet<>(300); try (Cursor c = mDb.query(TABLE_NAME, null, null, null, null, null, null)) { while (c.moveToNext()) { - ret.add(c.getString(c.getColumnIndex("id"))); + set.add(c.getString(c.getColumnIndex("id"))); } } - return ret; + return set; } } diff --git a/src/main/java/com/topjohnwu/magisk/utils/WebService.java b/src/main/java/com/topjohnwu/magisk/utils/WebService.java index bc4073a34..df578b0e6 100644 --- a/src/main/java/com/topjohnwu/magisk/utils/WebService.java +++ b/src/main/java/com/topjohnwu/magisk/utils/WebService.java @@ -15,9 +15,13 @@ public class WebService { } public static String getString(String url, Map header) { - HttpURLConnection conn = request(url, header); - if (conn == null) return ""; - return getString(conn); + try { + HttpURLConnection conn = request(url, header); + return getString(conn); + } catch (IOException e) { + e.printStackTrace(); + return ""; + } } public static String getString(HttpURLConnection conn) { @@ -40,34 +44,29 @@ public class WebService { } } - public static HttpURLConnection request(String address, Map header) { - try { - URL url = new URL(address); + public static HttpURLConnection request(String address, Map header) throws IOException { + URL url = new URL(address); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(15000); - conn.setConnectTimeout(15000); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(15000); + conn.setConnectTimeout(15000); - if (header != null) { - for (Map.Entry entry : header.entrySet()) { - conn.setRequestProperty(entry.getKey(), entry.getValue()); - } + if (header != null) { + for (Map.Entry entry : header.entrySet()) { + conn.setRequestProperty(entry.getKey(), entry.getValue()); } - - conn.connect(); - - if (header != null) { - header.clear(); - for (Map.Entry> entry : conn.getHeaderFields().entrySet()) { - List l = entry.getValue(); - header.put(entry.getKey(), l.get(l.size() - 1)); - } - } - - return conn; - } catch (Exception e) { - e.printStackTrace(); - return null; } + + conn.connect(); + + if (header != null) { + header.clear(); + for (Map.Entry> entry : conn.getHeaderFields().entrySet()) { + List l = entry.getValue(); + header.put(entry.getKey(), l.get(l.size() - 1)); + } + } + + return conn; } -} \ No newline at end of file +}