From 3e17ab2b06f06d13cc057aacee478f83ae3b0f96 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 23 Oct 2024 13:18:29 +1030 Subject: [PATCH 1/2] Optimise and test IP2Country (#1684) * Move ipv4Int to top level * Remove redundant fun calls in ipv4ToCountry * Add null safety to loadFile * Close streams on failure * Simplify cacheCountryForIP * Add IP2CountryTest * Generate binary * Simplify ipv4Int * Fix companion object visibility * Use array instead of Treemap * Synchronize OnionApi#paths * Move csv * Deduplicate locations csv * Move ipToCode to gradle * Use std lib binarySearch --------- Co-authored-by: bemusementpark --- app/build.gradle | 13 ++ .../csv => }/geolite2_country_blocks_ipv4.csv | 0 app/ipToCode.gradle.kts | 41 ++++++ .../thoughtcrime/securesms/util/IP2Country.kt | 136 +++++++----------- .../securesms/util/IP2CountryTest.kt | 40 ++++++ .../libsession/snode/OnionRequestAPI.kt | 4 +- 6 files changed, 150 insertions(+), 84 deletions(-) rename app/{src/main/assets/csv => }/geolite2_country_blocks_ipv4.csv (100%) create mode 100644 app/ipToCode.gradle.kts create mode 100644 app/src/test/java/org/thoughtcrime/securesms/util/IP2CountryTest.kt diff --git a/app/build.gradle b/app/build.gradle index da0d2f3fa5..37673102a9 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -105,6 +105,13 @@ android { String sharedTestDir = 'src/sharedTest/java' test.java.srcDirs += sharedTestDir androidTest.java.srcDirs += sharedTestDir + main { + assets.srcDirs += "$buildDir/generated/binary" + } + test { + resources.srcDirs += "$buildDir/generated/binary" + resources.srcDirs += "$projectDir/src/main/assets" + } } buildTypes { @@ -242,6 +249,12 @@ android { } } +apply { + from("ipToCode.gradle.kts") +} + +preBuild.dependsOn ipToCode + dependencies { implementation project(':content-descriptions') diff --git a/app/src/main/assets/csv/geolite2_country_blocks_ipv4.csv b/app/geolite2_country_blocks_ipv4.csv similarity index 100% rename from app/src/main/assets/csv/geolite2_country_blocks_ipv4.csv rename to app/geolite2_country_blocks_ipv4.csv diff --git a/app/ipToCode.gradle.kts b/app/ipToCode.gradle.kts new file mode 100644 index 0000000000..9ec2b29806 --- /dev/null +++ b/app/ipToCode.gradle.kts @@ -0,0 +1,41 @@ +import java.io.File +import java.io.DataOutputStream +import java.io.FileOutputStream + +task("ipToCode") { + val inputFile = File("${projectDir}/geolite2_country_blocks_ipv4.csv") + + val outputDir = "${buildDir}/generated/binary" + val outputFile = File(outputDir, "geolite2_country_blocks_ipv4.bin").apply { parentFile.mkdirs() } + + outputs.file(outputFile) + + doLast { + + // Ensure the input file exists + if (!inputFile.exists()) { + throw IllegalArgumentException("Input file does not exist: ${inputFile.absolutePath}") + } + + // Create a DataOutputStream to write binary data + DataOutputStream(FileOutputStream(outputFile)).use { out -> + inputFile.useLines { lines -> + var prevCode = -1 + lines.drop(1).forEach { line -> + runCatching { + val ints = line.split(".", "/", ",") + val code = ints[5].toInt().also { if (it == prevCode) return@forEach } + val ip = ints.take(4).fold(0) { acc, s -> acc shl 8 or s.toInt() } + + out.writeInt(ip) + out.writeInt(code) + + prevCode = code + } + } + } + } + + println("Processed data written to: ${outputFile.absolutePath}") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/IP2Country.kt b/app/src/main/java/org/thoughtcrime/securesms/util/IP2Country.kt index 446f0286c5..d6b71b3151 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/IP2Country.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/util/IP2Country.kt @@ -9,52 +9,46 @@ import com.opencsv.CSVReader import org.session.libsession.snode.OnionRequestAPI import org.session.libsignal.utilities.Log import org.session.libsignal.utilities.ThreadUtils -import java.io.File -import java.io.FileOutputStream -import java.io.FileReader -import java.util.SortedMap -import java.util.TreeMap +import java.io.DataInputStream +import java.io.InputStream +import java.io.InputStreamReader +import kotlin.math.absoluteValue -class IP2Country private constructor(private val context: Context) { - private val pathsBuiltEventReceiver: BroadcastReceiver +private fun ipv4Int(ip: String): UInt = + ip.split(".", "/", ",").take(4).fold(0U) { acc, s -> acc shl 8 or s.toUInt() } + +@OptIn(ExperimentalUnsignedTypes::class) +class IP2Country internal constructor( + private val context: Context, + private val openStream: (String) -> InputStream = context.assets::open +) { val countryNamesCache = mutableMapOf() - private fun Ipv4Int(ip: String): Int { - var result = 0L - var currentValue = 0L - var octetIndex = 0 + private val ips: UIntArray by lazy { ipv4ToCountry.first } + private val codes: IntArray by lazy { ipv4ToCountry.second } - for (char in ip) { - if (char == '.' || char == '/') { - result = result or (currentValue shl (8 * (3 - octetIndex))) - currentValue = 0 - octetIndex++ - if (char == '/') break - } else { - currentValue = currentValue * 10 + (char - '0') + private val ipv4ToCountry: Pair by lazy { + openStream("geolite2_country_blocks_ipv4.bin") + .let(::DataInputStream) + .use { + val size = it.available() / 8 + + val ips = UIntArray(size) + val codes = IntArray(size) + var i = 0 + + while (it.available() > 0) { + ips[i] = it.readInt().toUInt() + codes[i] = it.readInt() + i++ + } + + ips to codes } - } - - // Handle the last octet - result = result or (currentValue shl (8 * (3 - octetIndex))) - - return result.toInt() - } - - private val ipv4ToCountry: TreeMap by lazy { - val file = loadFile("geolite2_country_blocks_ipv4.csv") - CSVReader(FileReader(file.absoluteFile)).use { csv -> - csv.skip(1) - - csv.asSequence().associateTo(TreeMap()) { cols -> - Ipv4Int(cols[0]).toInt() to cols[1].toIntOrNull() - } - } } private val countryToNames: Map by lazy { - val file = loadFile("geolite2_country_locations_english.csv") - CSVReader(FileReader(file.absoluteFile)).use { csv -> + CSVReader(InputStreamReader(openStream("csv/geolite2_country_locations_english.csv"))).use { csv -> csv.skip(1) csv.asSequence() @@ -68,81 +62,57 @@ class IP2Country private constructor(private val context: Context) { // region Initialization companion object { - public lateinit var shared: IP2Country + lateinit var shared: IP2Country - public val isInitialized: Boolean get() = Companion::shared.isInitialized + val isInitialized: Boolean get() = Companion::shared.isInitialized - public fun configureIfNeeded(context: Context) { + fun configureIfNeeded(context: Context) { if (isInitialized) { return; } shared = IP2Country(context.applicationContext) + + val pathsBuiltEventReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + shared.populateCacheIfNeeded() + } + } + LocalBroadcastManager.getInstance(context).registerReceiver(pathsBuiltEventReceiver, IntentFilter("pathsBuilt")) } } init { populateCacheIfNeeded() - pathsBuiltEventReceiver = object : BroadcastReceiver() { - override fun onReceive(context: Context, intent: Intent) { - populateCacheIfNeeded() - } - } - LocalBroadcastManager.getInstance(context).registerReceiver(pathsBuiltEventReceiver, IntentFilter("pathsBuilt")) } // TODO: Deinit? // endregion // region Implementation - private fun loadFile(fileName: String): File { - val directory = File(context.applicationInfo.dataDir) - val file = File(directory, fileName) - if (directory.list().contains(fileName)) { return file } - val inputStream = context.assets.open("csv/$fileName") - val outputStream = FileOutputStream(file) - val buffer = ByteArray(1024) - while (true) { - val count = inputStream.read(buffer) - if (count < 0) { break } - outputStream.write(buffer, 0, count) - } - inputStream.close() - outputStream.close() - return file - } - - private fun cacheCountryForIP(ip: String): String? { - + internal fun cacheCountryForIP(ip: String): String? { // return early if cached countryNamesCache[ip]?.let { return it } - val ipInt = Ipv4Int(ip) - val bestMatchCountry = ipv4ToCountry.floorEntry(ipInt)?.let { (_, code) -> - if (code != null) { - countryToNames[code] - } else { - null - } - } + val ipInt = ipv4Int(ip) + val index = ips.binarySearch(ipInt).let { it.takeIf { it >= 0 } ?: (it.absoluteValue - 2) } + val code = codes.getOrNull(index) + val bestMatchCountry = countryToNames[code] - if (bestMatchCountry != null) { - countryNamesCache[ip] = bestMatchCountry - return bestMatchCountry - } else { - Log.d("Loki","Country name for $ip couldn't be found") - } - return null + if (bestMatchCountry != null) countryNamesCache[ip] = bestMatchCountry + else Log.d("Loki","Country name for $ip couldn't be found") + + return bestMatchCountry } private fun populateCacheIfNeeded() { ThreadUtils.queue { + val start = System.currentTimeMillis() OnionRequestAPI.paths.iterator().forEach { path -> path.iterator().forEach { snode -> cacheCountryForIP(snode.ip) // Preload if needed } } + Log.d("Loki","Cache populated in ${System.currentTimeMillis() - start}ms") Broadcaster(context).broadcast("onionRequestPathCountriesLoaded") - Log.d("Loki", "Finished preloading onion request path countries.") } } // endregion } - diff --git a/app/src/test/java/org/thoughtcrime/securesms/util/IP2CountryTest.kt b/app/src/test/java/org/thoughtcrime/securesms/util/IP2CountryTest.kt new file mode 100644 index 0000000000..d13ecb4c9b --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/util/IP2CountryTest.kt @@ -0,0 +1,40 @@ +package org.thoughtcrime.securesms.util + +import android.content.Context +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import org.mockito.Mockito.mock + +@RunWith(Parameterized::class) +class IP2CountryTest( + private val ip: String, + private val country: String +) { + private val context: Context = mock(Context::class.java) + private val ip2Country = IP2Country(context, this::class.java.classLoader!!::getResourceAsStream) + + @Test + fun getCountryNamesCache() { + assertEquals(country, ip2Country.cacheCountryForIP(ip)) + } + + companion object { + @JvmStatic + @Parameterized.Parameters + fun data(): Collection> = listOf( + arrayOf("223.121.64.0", "Hong Kong"), + arrayOf("223.121.64.1", "Hong Kong"), + arrayOf("223.121.127.0", "Hong Kong"), + arrayOf("223.121.128.0", "China"), + arrayOf("223.121.129.0", "China"), + arrayOf("223.122.0.0", "Hong Kong"), + arrayOf("223.123.0.0", "Pakistan"), + arrayOf("223.123.128.0", "China"), + arrayOf("223.124.0.0", "China"), + arrayOf("223.128.0.0", "China"), + arrayOf("223.130.0.0", "Singapore") + ) + } +} diff --git a/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt b/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt index 9ff541a9d5..e6721958a7 100644 --- a/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt @@ -44,7 +44,8 @@ object OnionRequestAPI { var guardSnodes = setOf() var _paths: AtomicReference?> = AtomicReference(null) - var paths: List // Not a set to ensure we consistently show the same path to the user + var paths: List // Not a Set to ensure we consistently show the same path to the user + @Synchronized get() { val paths = _paths.get() @@ -57,6 +58,7 @@ object OnionRequestAPI { _paths.set(result) return result } + @Synchronized set(newValue) { if (newValue.isEmpty()) { database.clearOnionRequestPaths() From 16cca2d3cc961644a743de1428fc0ed5bb913b6f Mon Sep 17 00:00:00 2001 From: ThomasSession Date: Wed, 23 Oct 2024 13:48:52 +1100 Subject: [PATCH 2/2] Getting the recipient from the VM (#1694) --- .../components/ProfilePictureView.kt | 11 +++++++- .../securesms/preferences/SettingsActivity.kt | 17 ++++++++---- .../preferences/SettingsViewModel.kt | 26 ++++++++++++++++++- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/ProfilePictureView.kt b/app/src/main/java/org/thoughtcrime/securesms/components/ProfilePictureView.kt index 9511bddb6a..d390d82ec3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/ProfilePictureView.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/ProfilePictureView.kt @@ -36,6 +36,7 @@ class ProfilePictureView @JvmOverloads constructor( var displayName: String? = null var additionalPublicKey: String? = null var additionalDisplayName: String? = null + var recipient: Recipient? = null private val profilePicturesCache = mutableMapOf() private val resourcePadding by lazy { @@ -51,6 +52,7 @@ class ProfilePictureView @JvmOverloads constructor( } fun update(recipient: Recipient) { + this.recipient = recipient recipient.run { update(address, isClosedGroupRecipient, isOpenGroupInboxRecipient) } } @@ -121,7 +123,14 @@ class ProfilePictureView @JvmOverloads constructor( private fun setProfilePictureIfNeeded(imageView: ImageView, publicKey: String, displayName: String?) { if (publicKey.isNotEmpty()) { - val recipient = Recipient.from(context, Address.fromSerialized(publicKey), false) + // if we already have a recipient that matches the current key, reuse it + val recipient = if(this.recipient != null && this.recipient?.address?.serialize() == publicKey){ + this.recipient!! + } + else { + this.recipient = Recipient.from(context, Address.fromSerialized(publicKey), false) + this.recipient!! + } if (profilePicturesCache[imageView] == recipient) return profilePicturesCache[imageView] = recipient val signalProfilePicture = recipient.contactPhoto diff --git a/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsActivity.kt b/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsActivity.kt index eaebcb57f6..c3ecc3cdac 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsActivity.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsActivity.kt @@ -173,11 +173,6 @@ class SettingsActivity : PassphraseRequiredActionBarActivity() { } binding.run { - profilePictureView.apply { - publicKey = viewModel.hexEncodedPublicKey - displayName = viewModel.getDisplayName() - update() - } profilePictureView.setOnClickListener { binding.avatarDialog.isVisible = true showAvatarDialog = true @@ -209,6 +204,18 @@ class SettingsActivity : PassphraseRequiredActionBarActivity() { binding.profilePictureView.update() } } + + lifecycleScope.launch { + viewModel.avatarData.collect { + if(it == null) return@collect + + binding.profilePictureView.apply { + publicKey = it.publicKey + displayName = it.displayName + update(it.recipient) + } + } + } } override fun onStart() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsViewModel.kt index 5b6fa78d44..07240e2404 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/preferences/SettingsViewModel.kt @@ -26,6 +26,7 @@ import org.session.libsession.utilities.Address import org.session.libsession.utilities.ProfileKeyUtil import org.session.libsession.utilities.ProfilePictureUtilities import org.session.libsession.utilities.TextSecurePreferences +import org.session.libsession.utilities.recipients.Recipient import org.session.libsession.utilities.truncateIdForDisplay import org.session.libsignal.utilities.ExternalStorageUtil.getImageDir import org.session.libsignal.utilities.Log @@ -52,7 +53,7 @@ class SettingsViewModel @Inject constructor( private var tempFile: File? = null - val hexEncodedPublicKey: String get() = prefs.getLocalNumber() ?: "" + val hexEncodedPublicKey: String = prefs.getLocalNumber() ?: "" private val userAddress = Address.fromSerialized(hexEncodedPublicKey) @@ -70,6 +71,10 @@ class SettingsViewModel @Inject constructor( val recoveryHidden: StateFlow get() = _recoveryHidden + private val _avatarData: MutableStateFlow = MutableStateFlow(null) + val avatarData: StateFlow + get() = _avatarData + /** * Refreshes the avatar on the main settings page */ @@ -77,6 +82,19 @@ class SettingsViewModel @Inject constructor( val refreshAvatar: SharedFlow get() = _refreshAvatar.asSharedFlow() + init { + viewModelScope.launch(Dispatchers.Default) { + val recipient = Recipient.from(context, Address.fromSerialized(hexEncodedPublicKey), false) + _avatarData.update { + AvatarData( + publicKey = hexEncodedPublicKey, + displayName = getDisplayName(), + recipient = recipient + ) + } + } + } + fun getDisplayName(): String = prefs.getProfileName() ?: truncateIdForDisplay(hexEncodedPublicKey) @@ -249,4 +267,10 @@ class SettingsViewModel @Inject constructor( val hasAvatar: Boolean // true if the user has an avatar set already but is in this temp state because they are trying out a new avatar ) : AvatarDialogState() } + + data class AvatarData( + val publicKey: String, + val displayName: String, + val recipient: Recipient + ) } \ No newline at end of file