From cc0ced9a81d3fc0490cd8d7d71de891a5a29b816 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Thu, 26 Sep 2019 10:12:51 -0400 Subject: [PATCH] Add internal pre-alpha support for storage service. --- app/build.gradle | 2 + .../service/protobuf/SignalService.proto | 6 +- .../service/protobuf/SignalStorage.proto | 78 +++ .../api/SignalServiceAccountManager.java | 121 ++++ .../signalservice/api/crypto/CryptoUtil.java | 22 + .../api/messages/multidevice/KeysMessage.java | 17 + .../messages/multidevice/RequestMessage.java | 4 + .../multidevice/SignalServiceSyncMessage.java | 69 +- .../ContactManifestMismatchException.java | 14 + .../api/storage/SignalContactRecord.java | 198 ++++++ .../api/storage/SignalStorageCipher.java | 65 ++ .../api/storage/SignalStorageManifest.java | 21 + .../api/storage/SignalStorageModels.java | 146 +++++ .../api/storage/SignalStorageRecord.java | 61 ++ .../api/storage/SignalStorageUtil.java | 13 + .../api/storage/StorageAuthResponse.java | 27 + .../SignalServiceConfiguration.java | 10 +- .../configuration/SignalStorageUrl.java | 17 + .../internal/push/PushServiceSocket.java | 148 ++++- .../signalservice/internal/util/Util.java | 8 +- .../securesms/ApplicationContext.java | 1 + .../contacts/sync/DirectoryHelper.java | 19 +- .../contacts/sync/DirectoryHelperV1.java | 3 +- .../contacts/sync/StorageSyncHelper.java | 537 ++++++++++++++++ .../securesms/database/DatabaseFactory.java | 6 + .../securesms/database/IdentityDatabase.java | 67 +- .../securesms/database/RecipientDatabase.java | 587 +++++++++++++++--- .../database/StorageKeyDatabase.java | 115 ++++ .../database/helpers/SQLCipherOpenHelper.java | 41 +- .../securesms/jobs/JobManagerFactories.java | 4 + .../jobs/MultiDeviceKeysUpdateJob.java | 91 +++ .../MultiDeviceProfileContentUpdateJob.java | 2 +- .../MultiDeviceStorageSyncRequestJob.java | 74 +++ .../securesms/jobs/PushDecryptJob.java | 4 + .../securesms/jobs/StorageForcePushJob.java | 142 +++++ .../securesms/jobs/StorageSyncJob.java | 228 +++++++ .../push/SignalServiceNetworkAccess.java | 26 +- .../securesms/recipients/Recipient.java | 23 + .../recipients/RecipientDetails.java | 15 +- .../securesms/util/FeatureFlags.java | 5 +- .../thoughtcrime/securesms/util/SetUtil.java | 31 + .../securesms/util/TextSecurePreferences.java | 10 + .../contacts/sync/StorageSyncHelperTest.java | 323 ++++++++++ 43 files changed, 3238 insertions(+), 163 deletions(-) create mode 100644 libsignal/service/protobuf/SignalStorage.proto create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/KeysMessage.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/push/exceptions/ContactManifestMismatchException.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalContactRecord.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageCipher.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageModels.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageUtil.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/StorageAuthResponse.java create mode 100644 libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalStorageUrl.java create mode 100644 src/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelper.java create mode 100644 src/org/thoughtcrime/securesms/database/StorageKeyDatabase.java create mode 100644 src/org/thoughtcrime/securesms/jobs/MultiDeviceKeysUpdateJob.java create mode 100644 src/org/thoughtcrime/securesms/jobs/MultiDeviceStorageSyncRequestJob.java create mode 100644 src/org/thoughtcrime/securesms/jobs/StorageForcePushJob.java create mode 100644 src/org/thoughtcrime/securesms/jobs/StorageSyncJob.java create mode 100644 src/org/thoughtcrime/securesms/util/SetUtil.java create mode 100644 test/unitTest/java/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelperTest.java diff --git a/app/build.gradle b/app/build.gradle index 364335b9b7..730343307b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -227,6 +227,7 @@ android { buildConfigField "long", "BUILD_TIMESTAMP", getLastCommitTimestamp() + "L" buildConfigField "String", "SIGNAL_URL", "\"https://textsecure-service.whispersystems.org\"" + buildConfigField "String", "STORAGE_URL", "\"https://storage.signal.org\"" buildConfigField "String", "SIGNAL_CDN_URL", "\"https://cdn.signal.org\"" buildConfigField "String", "SIGNAL_CONTACT_DISCOVERY_URL", "\"https://api.directory.signal.org\"" buildConfigField "String", "SIGNAL_SERVICE_STATUS_URL", "\"uptime.signal.org\"" @@ -302,6 +303,7 @@ android { initWith debug buildConfigField "String", "SIGNAL_URL", "\"https://textsecure-service-staging.whispersystems.org\"" + buildConfigField "String", "STORAGE_URL", "\"https://storage-staging.signal.org\"" buildConfigField "String", "SIGNAL_CDN_URL", "\"https://cdn-staging.signal.org\"" buildConfigField "String", "SIGNAL_CONTACT_DISCOVERY_URL", "\"https://api-staging.directory.signal.org\"" buildConfigField "String", "SIGNAL_KEY_BACKUP_URL", "\"https://api-staging.backup.signal.org\"" diff --git a/libsignal/service/protobuf/SignalService.proto b/libsignal/service/protobuf/SignalService.proto index 43c8ded269..42ae35c84f 100644 --- a/libsignal/service/protobuf/SignalService.proto +++ b/libsignal/service/protobuf/SignalService.proto @@ -289,6 +289,7 @@ message SyncMessage { GROUPS = 2; BLOCKED = 3; CONFIGURATION = 4; + KEYS = 5; } optional Type type = 1; @@ -325,7 +326,6 @@ message SyncMessage { optional uint64 timestamp = 2; } - message FetchLatest { enum Type { UNKNOWN = 0; @@ -336,6 +336,9 @@ message SyncMessage { optional Type type = 1; } + message Keys { + optional bytes storageService = 1; + } optional Sent sent = 1; optional Contacts contacts = 2; @@ -349,6 +352,7 @@ message SyncMessage { repeated StickerPackOperation stickerPackOperation = 10; optional ViewOnceOpen viewOnceOpen = 11; optional FetchLatest fetchLatest = 12; + optional Keys keys = 13; } message AttachmentPointer { diff --git a/libsignal/service/protobuf/SignalStorage.proto b/libsignal/service/protobuf/SignalStorage.proto new file mode 100644 index 0000000000..19fa066607 --- /dev/null +++ b/libsignal/service/protobuf/SignalStorage.proto @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2019 Open Whisper Systems + * + * Licensed according to the LICENSE file in this repository. + */ +syntax = "proto2"; + +package signalservice; + +option java_package = "org.whispersystems.signalservice.internal.storage.protos"; +option java_multiple_files = true; + + +message StorageItem { + optional bytes key = 1; + optional bytes value = 2; +} + +message StorageItems { + repeated StorageItem items = 1; +} + +message StorageManifest { + optional uint64 version = 1; + optional bytes value = 2; +} + +message ReadOperation { + repeated bytes readKey = 1; +} + +message WriteOperation { + optional StorageManifest manifest = 1; + repeated StorageItem insertItem = 2; + repeated bytes deleteKey = 3; +} + +message StorageRecord { + enum Type { + UNKNOWN = 0; + CONTACT = 1; + } + + optional uint32 type = 1; + optional ContactRecord contact = 2; +} + +message ContactRecord { + message Identity { + enum State { + DEFAULT = 0; + VERIFIED = 1; + UNVERIFIED = 2; + } + + optional bytes key = 1; + optional State state = 2; + } + + message Profile { + optional string name = 1; + optional bytes key = 2; + optional string username = 3; + } + + optional string serviceUuid = 1; + optional string serviceE164 = 2; + optional Profile profile = 3; + optional Identity identity = 4; + optional bool blocked = 5; + optional bool whitelisted = 6; + optional string nickname = 7; +} + +message ManifestRecord { + optional uint64 version = 1; + repeated bytes keys = 2; +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/SignalServiceAccountManager.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/SignalServiceAccountManager.java index d6606efbb4..10d0b8efc9 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/SignalServiceAccountManager.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/SignalServiceAccountManager.java @@ -24,6 +24,12 @@ import org.whispersystems.signalservice.api.messages.calls.TurnServerInfo; import org.whispersystems.signalservice.api.messages.multidevice.DeviceInfo; import org.whispersystems.signalservice.api.push.ContactTokenDetails; import org.whispersystems.signalservice.api.push.SignedPreKeyEntity; +import org.whispersystems.signalservice.api.push.exceptions.NotFoundException; +import org.whispersystems.signalservice.api.storage.SignalStorageCipher; +import org.whispersystems.signalservice.api.storage.SignalContactRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageModels; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageManifest; import org.whispersystems.signalservice.api.util.CredentialsProvider; import org.whispersystems.signalservice.api.util.StreamDetails; import org.whispersystems.signalservice.internal.configuration.SignalServiceConfiguration; @@ -39,6 +45,12 @@ import org.whispersystems.signalservice.internal.push.ProfileAvatarData; import org.whispersystems.signalservice.internal.push.PushServiceSocket; import org.whispersystems.signalservice.internal.push.RemoteAttestationUtil; import org.whispersystems.signalservice.internal.push.http.ProfileCipherOutputStreamFactory; +import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord; +import org.whispersystems.signalservice.internal.storage.protos.ReadOperation; +import org.whispersystems.signalservice.internal.storage.protos.StorageItem; +import org.whispersystems.signalservice.internal.storage.protos.StorageItems; +import org.whispersystems.signalservice.internal.storage.protos.StorageManifest; +import org.whispersystems.signalservice.internal.storage.protos.WriteOperation; import org.whispersystems.signalservice.internal.util.StaticCredentialsProvider; import org.whispersystems.signalservice.internal.util.Util; import org.whispersystems.util.Base64; @@ -48,6 +60,7 @@ import java.security.KeyStore; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.SignatureException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -386,6 +399,113 @@ public class SignalServiceAccountManager { } } + public long getStorageManifestVersion() throws IOException { + try { + String authToken = this.pushServiceSocket.getStorageAuth(); + StorageManifest storageManifest = this.pushServiceSocket.getStorageManifest(authToken); + + return storageManifest.getVersion(); + } catch (NotFoundException e) { + return 0; + } + } + + public Optional getStorageManifest(byte[] storageServiceKey) throws IOException, InvalidKeyException { + try { + SignalStorageCipher cipher = new SignalStorageCipher(storageServiceKey); + String authToken = this.pushServiceSocket.getStorageAuth(); + StorageManifest storageManifest = this.pushServiceSocket.getStorageManifest(authToken); + byte[] rawRecord = cipher.decrypt(storageManifest.getValue().toByteArray()); + ManifestRecord manifestRecord = ManifestRecord.parseFrom(rawRecord); + List keys = new ArrayList<>(manifestRecord.getKeysCount()); + + for (ByteString key : manifestRecord.getKeysList()) { + keys.add(key.toByteArray()); + } + + return Optional.of(new SignalStorageManifest(manifestRecord.getVersion(), keys)); + } catch (NotFoundException e) { + return Optional.absent(); + } + } + + public List readStorageRecords(byte[] storageServiceKey, List storageKeys) throws IOException, InvalidKeyException { + ReadOperation.Builder operation = ReadOperation.newBuilder(); + + for (byte[] key : storageKeys) { + operation.addReadKey(ByteString.copyFrom(key)); + } + + String authToken = this.pushServiceSocket.getStorageAuth(); + StorageItems items = this.pushServiceSocket.readStorageItems(authToken, operation.build()); + + SignalStorageCipher storageCipher = new SignalStorageCipher(storageServiceKey); + List result = new ArrayList<>(items.getItemsCount()); + + for (StorageItem item : items.getItemsList()) { + if (item.hasKey()) { + result.add(SignalStorageModels.remoteToLocalStorageRecord(item, storageCipher)); + } else { + Log.w(TAG, "Encountered a StorageItem with no key! Skipping."); + } + } + + return result; + } + + /** + * @return If there was a conflict, the latest {@link SignalStorageManifest}. Otherwise absent. + */ + public Optional writeStorageRecords(byte[] storageServiceKey, + SignalStorageManifest manifest, + List inserts, + List deletes) + throws IOException, InvalidKeyException + { + ManifestRecord.Builder manifestRecordBuilder = ManifestRecord.newBuilder().setVersion(manifest.getVersion()); + + for (byte[] key : manifest.getStorageKeys()) { + manifestRecordBuilder.addKeys(ByteString.copyFrom(key)); + } + + String authToken = this.pushServiceSocket.getStorageAuth(); + SignalStorageCipher cipher = new SignalStorageCipher(storageServiceKey); + byte[] encryptedRecord = cipher.encrypt(manifestRecordBuilder.build().toByteArray()); + StorageManifest storageManifest = StorageManifest.newBuilder() + .setVersion(manifest.getVersion()) + .setValue(ByteString.copyFrom(encryptedRecord)) + .build(); + WriteOperation.Builder writeBuilder = WriteOperation.newBuilder().setManifest(storageManifest); + + for (SignalStorageRecord insert : inserts) { + writeBuilder.addInsertItem(SignalStorageModels.localToRemoteStorageRecord(insert, cipher)); + } + + for (byte[] delete : deletes) { + writeBuilder.addDeleteKey(ByteString.copyFrom(delete)); + } + + Optional conflict = this.pushServiceSocket.writeStorageContacts(authToken, writeBuilder.build()); + + if (conflict.isPresent()) { + byte[] rawManifestRecord = cipher.decrypt(conflict.get().getValue().toByteArray()); + ManifestRecord record = ManifestRecord.parseFrom(rawManifestRecord); + List keys = new ArrayList<>(record.getKeysCount()); + + for (ByteString key : record.getKeysList()) { + keys.add(key.toByteArray()); + } + + SignalStorageManifest conflictManifest = new SignalStorageManifest(record.getVersion(), keys); + + return Optional.of(conflictManifest); + } else { + return Optional.absent(); + } + } + + + public String getNewDeviceVerificationCode() throws IOException { return this.pushServiceSocket.getNewDeviceVerificationCode(); } @@ -495,4 +615,5 @@ public class SignalServiceAccountManager { return tokenMap; } + } diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java new file mode 100644 index 0000000000..0d4e2861b8 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/crypto/CryptoUtil.java @@ -0,0 +1,22 @@ +package org.whispersystems.signalservice.api.crypto; + +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +public final class CryptoUtil { + + private CryptoUtil () { } + + public static byte[] computeHmacSha256(byte[] key, byte[] data) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(data); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + throw new AssertionError(e); + } + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/KeysMessage.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/KeysMessage.java new file mode 100644 index 0000000000..04cb024939 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/KeysMessage.java @@ -0,0 +1,17 @@ +package org.whispersystems.signalservice.api.messages.multidevice; + + +import org.whispersystems.libsignal.util.guava.Optional; + +public class KeysMessage { + + private final Optional storageService; + + public KeysMessage(Optional storageService) { + this.storageService = storageService; + } + + public Optional getStorageService() { + return storageService; + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/RequestMessage.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/RequestMessage.java index 823d00cbd4..93108f7130 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/RequestMessage.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/RequestMessage.java @@ -31,4 +31,8 @@ public class RequestMessage { public boolean isConfigurationRequest() { return request.getType() == Request.Type.CONFIGURATION; } + + public boolean isKeysRequest() { + return request.getType() == Request.Type.KEYS; + } } diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/SignalServiceSyncMessage.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/SignalServiceSyncMessage.java index 1a6eb5d51e..d1f1533d47 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/SignalServiceSyncMessage.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/messages/multidevice/SignalServiceSyncMessage.java @@ -25,6 +25,7 @@ public class SignalServiceSyncMessage { private final Optional configuration; private final Optional> stickerPackOperations; private final Optional fetchType; + private final Optional keys; private SignalServiceSyncMessage(Optional sent, Optional contacts, @@ -36,7 +37,8 @@ public class SignalServiceSyncMessage { Optional verified, Optional configuration, Optional> stickerPackOperations, - Optional fetchType) + Optional fetchType, + Optional keys) { this.sent = sent; this.contacts = contacts; @@ -49,6 +51,7 @@ public class SignalServiceSyncMessage { this.configuration = configuration; this.stickerPackOperations = stickerPackOperations; this.fetchType = fetchType; + this.keys = keys; } public static SignalServiceSyncMessage forSentTranscript(SentTranscriptMessage sent) { @@ -62,7 +65,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forContacts(ContactsMessage contacts) { @@ -76,7 +80,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forGroups(SignalServiceAttachment groups) { @@ -90,7 +95,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forRequest(RequestMessage request) { @@ -104,7 +110,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forRead(List reads) { @@ -118,7 +125,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forViewOnceOpen(ViewOnceOpenMessage timerRead) { @@ -132,7 +140,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forRead(ReadMessage read) { @@ -149,7 +158,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forVerified(VerifiedMessage verifiedMessage) { @@ -163,7 +173,8 @@ public class SignalServiceSyncMessage { Optional.of(verifiedMessage), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forBlocked(BlockedListMessage blocked) { @@ -177,7 +188,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forConfiguration(ConfigurationMessage configuration) { @@ -191,7 +203,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.of(configuration), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forStickerPackOperations(List stickerPackOperations) { @@ -205,7 +218,8 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.of(stickerPackOperations), - Optional.absent()); + Optional.absent(), + Optional.absent()); } public static SignalServiceSyncMessage forFetchLatest(FetchType fetchType) { @@ -219,13 +233,14 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.of(fetchType)); + Optional.of(fetchType), + Optional.absent()); } - public static SignalServiceSyncMessage empty() { + public static SignalServiceSyncMessage forKeys(KeysMessage keys) { return new SignalServiceSyncMessage(Optional.absent(), - Optional.absent(), - Optional.absent(), + Optional.absent(), + Optional.absent(), Optional.absent(), Optional.absent(), Optional.>absent(), @@ -233,7 +248,23 @@ public class SignalServiceSyncMessage { Optional.absent(), Optional.absent(), Optional.>absent(), - Optional.absent()); + Optional.absent(), + Optional.of(keys)); + } + + public static SignalServiceSyncMessage empty() { + return new SignalServiceSyncMessage(Optional.absent(), + Optional.absent(), + Optional.absent(), + Optional.absent(), + Optional.absent(), + Optional.>absent(), + Optional.absent(), + Optional.absent(), + Optional.absent(), + Optional.>absent(), + Optional.absent(), + Optional.absent()); } public Optional getSent() { @@ -280,6 +311,10 @@ public class SignalServiceSyncMessage { return fetchType; } + public Optional getKeys() { + return keys; + } + public enum FetchType { LOCAL_PROFILE, STORAGE_MANIFEST diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/push/exceptions/ContactManifestMismatchException.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/push/exceptions/ContactManifestMismatchException.java new file mode 100644 index 0000000000..a739152247 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/push/exceptions/ContactManifestMismatchException.java @@ -0,0 +1,14 @@ +package org.whispersystems.signalservice.api.push.exceptions; + +public class ContactManifestMismatchException extends NonSuccessfulResponseCodeException { + + private final byte[] responseBody; + + public ContactManifestMismatchException(byte[] responseBody) { + this.responseBody = responseBody; + } + + public byte[] getResponseBody() { + return responseBody; + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalContactRecord.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalContactRecord.java new file mode 100644 index 0000000000..c6d6cb1038 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalContactRecord.java @@ -0,0 +1,198 @@ +package org.whispersystems.signalservice.api.storage; + +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.push.SignalServiceAddress; + +import java.util.Arrays; +import java.util.Objects; + +public class SignalContactRecord { + + private final byte[] key; + private final SignalServiceAddress address; + private final Optional profileName; + private final Optional profileKey; + private final Optional username; + private final Optional identityKey; + private final IdentityState identityState; + private final boolean blocked; + private final boolean profileSharingEnabled; + private final Optional nickname; + private final int protoVersion; + + private SignalContactRecord(byte[] key, + SignalServiceAddress address, + String profileName, + byte[] profileKey, + String username, + byte[] identityKey, + IdentityState identityState, + boolean blocked, + boolean profileSharingEnabled, + String nickname, + int protoVersion) + { + this.key = key; + this.address = address; + this.profileName = Optional.fromNullable(profileName); + this.profileKey = Optional.fromNullable(profileKey); + this.username = Optional.fromNullable(username); + this.identityKey = Optional.fromNullable(identityKey); + this.identityState = identityState != null ? identityState : IdentityState.DEFAULT; + this.blocked = blocked; + this.profileSharingEnabled = profileSharingEnabled; + this.nickname = Optional.fromNullable(nickname); + this.protoVersion = protoVersion; + } + + public byte[] getKey() { + return key; + } + + public SignalServiceAddress getAddress() { + return address; + } + + public Optional getProfileName() { + return profileName; + } + + public Optional getProfileKey() { + return profileKey; + } + + public Optional getUsername() { + return username; + } + + public Optional getIdentityKey() { + return identityKey; + } + + public IdentityState getIdentityState() { + return identityState; + } + + public boolean isBlocked() { + return blocked; + } + + public boolean isProfileSharingEnabled() { + return profileSharingEnabled; + } + + public Optional getNickname() { + return nickname; + } + + public int getProtoVersion() { + return protoVersion; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SignalContactRecord contact = (SignalContactRecord) o; + return blocked == contact.blocked && + profileSharingEnabled == contact.profileSharingEnabled && + Arrays.equals(key, contact.key) && + Objects.equals(address, contact.address) && + Objects.equals(profileName, contact.profileName) && + Objects.equals(profileKey, contact.profileKey) && + Objects.equals(username, contact.username) && + Objects.equals(identityKey, contact.identityKey) && + identityState == contact.identityState && + Objects.equals(nickname, contact.nickname); + } + + @Override + public int hashCode() { + int result = Objects.hash(address, profileName, profileKey, username, identityKey, identityState, blocked, profileSharingEnabled, nickname); + result = 31 * result + Arrays.hashCode(key); + return result; + } + + public static final class Builder { + private final byte[] key; + private final SignalServiceAddress address; + + private String profileName; + private byte[] profileKey; + private String username; + private byte[] identityKey; + private IdentityState identityState; + private boolean blocked; + private boolean profileSharingEnabled; + private String nickname; + private int version; + + public Builder(byte[] key, SignalServiceAddress address) { + this.key = key; + this.address = address; + } + + public Builder setProfileName(String profileName) { + this.profileName = profileName; + return this; + } + + public Builder setProfileKey(byte[] profileKey) { + this.profileKey= profileKey; + return this; + } + + public Builder setUsername(String username) { + this.username = username; + return this; + } + + public Builder setIdentityKey(byte[] identityKey) { + this.identityKey = identityKey; + return this; + } + + public Builder setIdentityState(IdentityState identityState) { + this.identityState = identityState; + return this; + } + + public Builder setBlocked(boolean blocked) { + this.blocked = blocked; + return this; + } + + public Builder setProfileSharingEnabled(boolean profileSharingEnabled) { + this.profileSharingEnabled = profileSharingEnabled; + return this; + } + + public Builder setNickname(String nickname) { + this.nickname = nickname; + return this; + } + + Builder setProtoVersion(int version) { + this.version = version; + return this; + } + + public SignalContactRecord build() { + return new SignalContactRecord(key, + address, + profileName, + profileKey, + username, + identityKey, + identityState, + blocked, + profileSharingEnabled, + nickname, + version); + } + } + + public enum IdentityState { + DEFAULT, VERIFIED, UNVERIFIED + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageCipher.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageCipher.java new file mode 100644 index 0000000000..03ae3020e6 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageCipher.java @@ -0,0 +1,65 @@ +package org.whispersystems.signalservice.api.storage; + +import org.whispersystems.libsignal.InvalidKeyException; +import org.whispersystems.libsignal.util.ByteUtil; +import org.whispersystems.signalservice.api.crypto.CryptoUtil; +import org.whispersystems.signalservice.internal.util.Util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.NoSuchAlgorithmException; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.CipherOutputStream; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +/** + * Encrypts and decrypts data from the storage service. + */ +public class SignalStorageCipher { + + private final byte[] key; + + public SignalStorageCipher(byte[] storageServiceKey) { + this.key = storageServiceKey; + } + + public byte[] encrypt(byte[] data) { + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + byte[] iv = Util.getSecretBytes(16); + + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(key, "AES"), new GCMParameterSpec(128, iv)); + byte[] ciphertext = cipher.doFinal(data); + + return ByteUtil.combine(iv, ciphertext); + } catch (NoSuchAlgorithmException | java.security.InvalidKeyException | InvalidAlgorithmParameterException | NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException e) { + throw new AssertionError(e); + } + } + + public byte[] decrypt(byte[] data) throws InvalidKeyException { + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + byte[][] split = Util.split(data, 16, data.length - 16); + byte[] iv = split[0]; + byte[] cipherText = split[1]; + + cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key, "AES"), new GCMParameterSpec(128, iv)); + return cipher.doFinal(cipherText); + } catch (java.security.InvalidKeyException | BadPaddingException | IllegalBlockSizeException e) { + throw new InvalidKeyException(e); + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidAlgorithmParameterException e) { + throw new AssertionError(e); + } + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.java new file mode 100644 index 0000000000..b997ef3b2a --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.java @@ -0,0 +1,21 @@ +package org.whispersystems.signalservice.api.storage; + +import java.util.List; + +public class SignalStorageManifest { + private final long version; + private final List storageKeys; + + public SignalStorageManifest(long version, List storageKeys) { + this.version = version; + this.storageKeys = storageKeys; + } + + public long getVersion() { + return version; + } + + public List getStorageKeys() { + return storageKeys; + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageModels.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageModels.java new file mode 100644 index 0000000000..15807dbbea --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageModels.java @@ -0,0 +1,146 @@ +package org.whispersystems.signalservice.api.storage; + +import com.google.protobuf.ByteString; + +import org.whispersystems.libsignal.InvalidKeyException; +import org.whispersystems.signalservice.api.push.SignalServiceAddress; +import org.whispersystems.signalservice.api.util.UuidUtil; +import org.whispersystems.signalservice.internal.storage.protos.ContactRecord; +import org.whispersystems.signalservice.internal.storage.protos.StorageItem; +import org.whispersystems.signalservice.internal.storage.protos.StorageRecord; + +import java.io.IOException; + +public final class SignalStorageModels { + + public static SignalStorageRecord remoteToLocalStorageRecord(StorageItem item, SignalStorageCipher cipher) throws IOException, InvalidKeyException { + byte[] rawRecord = cipher.decrypt(item.getValue().toByteArray()); + StorageRecord record = StorageRecord.parseFrom(rawRecord); + byte[] storageKey = item.getKey().toByteArray(); + + if (record.getType() == StorageRecord.Type.CONTACT_VALUE && record.hasContact()) { + return SignalStorageRecord.forContact(storageKey, remoteToLocalContactRecord(storageKey, record.getContact())); + } else { + return SignalStorageRecord.forUnknown(storageKey, record.getType()); + } + } + + public static StorageItem localToRemoteStorageRecord(SignalStorageRecord record, SignalStorageCipher cipher) throws IOException { + StorageRecord.Builder builder = StorageRecord.newBuilder(); + + if (record.getContact().isPresent()) { + builder.setContact(localToRemoteContactRecord(record.getContact().get())); + } else { + throw new InvalidStorageWriteError(); + } + + builder.setType(record.getType()); + + StorageRecord remoteRecord = builder.build(); + byte[] encryptedRecord = cipher.encrypt(remoteRecord.toByteArray()); + + return StorageItem.newBuilder() + .setKey(ByteString.copyFrom(record.getKey())) + .setValue(ByteString.copyFrom(encryptedRecord)) + .build(); + } + + public static SignalContactRecord remoteToLocalContactRecord(byte[] storageKey, ContactRecord contact) throws IOException { + SignalServiceAddress address = new SignalServiceAddress(UuidUtil.parseOrNull(contact.getServiceUuid()), contact.getServiceE164()); + SignalContactRecord.Builder builder = new SignalContactRecord.Builder(storageKey, address); + + if (contact.hasBlocked()) { + builder.setBlocked(contact.getBlocked()); + } + + if (contact.hasWhitelisted()) { + builder.setProfileSharingEnabled(contact.getWhitelisted()); + } + + if (contact.hasNickname()) { + builder.setNickname(contact.getNickname()); + } + + if (contact.hasProfile()) { + if (contact.getProfile().hasKey()) { + builder.setProfileKey(contact.getProfile().getKey().toByteArray()); + } + + if (contact.getProfile().hasName()) { + builder.setProfileName(contact.getProfile().getName()); + } + + if (contact.getProfile().hasUsername()) { + builder.setUsername(contact.getProfile().getUsername()); + } + } + + if (contact.hasIdentity()) { + if (contact.getIdentity().hasKey()) { + builder.setIdentityKey(contact.getIdentity().getKey().toByteArray()); + } + + if (contact.getIdentity().hasState()) { + switch (contact.getIdentity().getState()) { + case VERIFIED: builder.setIdentityState(SignalContactRecord.IdentityState.VERIFIED); + case UNVERIFIED: builder.setIdentityState(SignalContactRecord.IdentityState.UNVERIFIED); + default: builder.setIdentityState(SignalContactRecord.IdentityState.VERIFIED); + } + } + } + + return builder.build(); + } + + public static ContactRecord localToRemoteContactRecord(SignalContactRecord contact) { + ContactRecord.Builder contactRecordBuilder = ContactRecord.newBuilder() + .setBlocked(contact.isBlocked()) + .setWhitelisted(contact.isProfileSharingEnabled()); + if (contact.getAddress().getNumber().isPresent()) { + contactRecordBuilder.setServiceE164(contact.getAddress().getNumber().get()); + } + + if (contact.getAddress().getUuid().isPresent()) { + contactRecordBuilder.setServiceUuid(contact.getAddress().getUuid().get().toString()); + } + + if (contact.getNickname().isPresent()) { + contactRecordBuilder.setNickname(contact.getNickname().get()); + } + + ContactRecord.Identity.Builder identityBuilder = ContactRecord.Identity.newBuilder(); + + switch (contact.getIdentityState()) { + case VERIFIED: identityBuilder.setState(ContactRecord.Identity.State.VERIFIED); + case UNVERIFIED: identityBuilder.setState(ContactRecord.Identity.State.UNVERIFIED); + case DEFAULT: identityBuilder.setState(ContactRecord.Identity.State.DEFAULT); + } + + if (contact.getIdentityKey().isPresent()) { + identityBuilder.setKey(ByteString.copyFrom(contact.getIdentityKey().get())); + } + + contactRecordBuilder.setIdentity(identityBuilder.build()); + + ContactRecord.Profile.Builder profileBuilder = ContactRecord.Profile.newBuilder(); + + if (contact.getProfileKey().isPresent()) { + profileBuilder.setKey(ByteString.copyFrom(contact.getProfileKey().get())); + } + + if (contact.getProfileName().isPresent()) { + profileBuilder.setName(contact.getProfileName().get()); + } + + if (contact.getUsername().isPresent()) { + profileBuilder.setUsername(contact.getUsername().get()); + } + + contactRecordBuilder.setProfile(profileBuilder.build()); + + return contactRecordBuilder.build(); + } + + private static class InvalidStorageWriteError extends Error { + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.java new file mode 100644 index 0000000000..c73b94bad1 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageRecord.java @@ -0,0 +1,61 @@ +package org.whispersystems.signalservice.api.storage; + +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.internal.storage.protos.StorageRecord; + +import java.util.Arrays; +import java.util.Objects; + +public class SignalStorageRecord { + + private final byte[] key; + private final int type; + private final Optional contact; + + public static SignalStorageRecord forContact(byte[] key, SignalContactRecord contact) { + return new SignalStorageRecord(key, StorageRecord.Type.CONTACT_VALUE, Optional.of(contact)); + } + + public static SignalStorageRecord forUnknown(byte[] key, int type) { + return new SignalStorageRecord(key, type, Optional.absent()); + } + + private SignalStorageRecord(byte key[], int type, Optional contact) { + this.key = key; + this.type = type; + this.contact = contact; + } + + public byte[] getKey() { + return key; + } + + public int getType() { + return type; + } + + public Optional getContact() { + return contact; + } + + public boolean isUnknown() { + return !contact.isPresent(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SignalStorageRecord record = (SignalStorageRecord) o; + return type == record.type && + Arrays.equals(key, record.key) && + contact.equals(record.contact); + } + + @Override + public int hashCode() { + int result = Objects.hash(type, contact); + result = 31 * result + Arrays.hashCode(key); + return result; + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageUtil.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageUtil.java new file mode 100644 index 0000000000..aa42fd930b --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageUtil.java @@ -0,0 +1,13 @@ +package org.whispersystems.signalservice.api.storage; + +import org.whispersystems.signalservice.api.crypto.CryptoUtil; + +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; + +public final class SignalStorageUtil { + + public static byte[] computeStorageServiceKey(byte[] kbsMasterKey) { + return CryptoUtil.computeHmacSha256(kbsMasterKey, "Storage Service Encryption".getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/StorageAuthResponse.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/StorageAuthResponse.java new file mode 100644 index 0000000000..829c3e27b5 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/api/storage/StorageAuthResponse.java @@ -0,0 +1,27 @@ +package org.whispersystems.signalservice.api.storage; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class StorageAuthResponse { + + @JsonProperty + private String username; + + @JsonProperty + private String password; + + public StorageAuthResponse() { } + + public StorageAuthResponse(String username, String password) { + this.username = username; + this.password = password; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalServiceConfiguration.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalServiceConfiguration.java index fdd528acac..f284f84b5e 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalServiceConfiguration.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalServiceConfiguration.java @@ -7,15 +7,19 @@ public class SignalServiceConfiguration { private final SignalCdnUrl[] signalCdnUrls; private final SignalContactDiscoveryUrl[] signalContactDiscoveryUrls; private final SignalKeyBackupServiceUrl[] signalKeyBackupServiceUrls; + private final SignalStorageUrl[] signalStorageUrls; public SignalServiceConfiguration(SignalServiceUrl[] signalServiceUrls, SignalCdnUrl[] signalCdnUrls, SignalContactDiscoveryUrl[] signalContactDiscoveryUrls, - SignalKeyBackupServiceUrl[] signalKeyBackupServiceUrls) { + SignalKeyBackupServiceUrl[] signalKeyBackupServiceUrls, + SignalStorageUrl[] signalStorageUrls) + { this.signalServiceUrls = signalServiceUrls; this.signalCdnUrls = signalCdnUrls; this.signalContactDiscoveryUrls = signalContactDiscoveryUrls; this.signalKeyBackupServiceUrls = signalKeyBackupServiceUrls; + this.signalStorageUrls = signalStorageUrls; } public SignalServiceUrl[] getSignalServiceUrls() { @@ -33,4 +37,8 @@ public class SignalServiceConfiguration { public SignalKeyBackupServiceUrl[] getSignalKeyBackupServiceUrls() { return signalKeyBackupServiceUrls; } + + public SignalStorageUrl[] getSignalStorageUrls() { + return signalStorageUrls; + } } diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalStorageUrl.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalStorageUrl.java new file mode 100644 index 0000000000..35db1516e3 --- /dev/null +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/configuration/SignalStorageUrl.java @@ -0,0 +1,17 @@ +package org.whispersystems.signalservice.internal.configuration; + + +import org.whispersystems.signalservice.api.push.TrustStore; + +import okhttp3.ConnectionSpec; + +public class SignalStorageUrl extends SignalUrl { + + public SignalStorageUrl(String url, TrustStore trustStore) { + super(url, trustStore); + } + + public SignalStorageUrl(String url, String hostHeader, TrustStore trustStore, ConnectionSpec connectionSpec) { + super(url, hostHeader, trustStore, connectionSpec); + } +} diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java index 9bf00a11ee..7c93e97eca 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java @@ -27,6 +27,7 @@ import org.whispersystems.signalservice.api.push.SignalServiceAddress; import org.whispersystems.signalservice.api.push.SignedPreKeyEntity; import org.whispersystems.signalservice.api.push.exceptions.AuthorizationFailedException; import org.whispersystems.signalservice.api.push.exceptions.CaptchaRequiredException; +import org.whispersystems.signalservice.api.push.exceptions.ContactManifestMismatchException; import org.whispersystems.signalservice.api.push.exceptions.ExpectationFailedException; import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException; import org.whispersystems.signalservice.api.push.exceptions.NotFoundException; @@ -36,6 +37,7 @@ import org.whispersystems.signalservice.api.push.exceptions.RemoteAttestationRes import org.whispersystems.signalservice.api.push.exceptions.UnregisteredUserException; import org.whispersystems.signalservice.api.push.exceptions.UsernameMalformedException; import org.whispersystems.signalservice.api.push.exceptions.UsernameTakenException; +import org.whispersystems.signalservice.api.storage.StorageAuthResponse; import org.whispersystems.signalservice.api.util.CredentialsProvider; import org.whispersystems.signalservice.api.util.Tls12SocketFactory; import org.whispersystems.signalservice.api.util.UuidUtil; @@ -50,6 +52,10 @@ import org.whispersystems.signalservice.internal.push.exceptions.MismatchedDevic import org.whispersystems.signalservice.internal.push.exceptions.StaleDevicesException; import org.whispersystems.signalservice.internal.push.http.DigestingRequestBody; import org.whispersystems.signalservice.internal.push.http.OutputStreamFactory; +import org.whispersystems.signalservice.internal.storage.protos.ReadOperation; +import org.whispersystems.signalservice.internal.storage.protos.StorageItems; +import org.whispersystems.signalservice.internal.storage.protos.StorageManifest; +import org.whispersystems.signalservice.internal.storage.protos.WriteOperation; import org.whispersystems.signalservice.internal.util.BlacklistingTrustManager; import org.whispersystems.signalservice.internal.util.Hex; import org.whispersystems.signalservice.internal.util.JsonUtil; @@ -79,15 +85,14 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; -import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; import okhttp3.Call; import okhttp3.ConnectionSpec; +import okhttp3.Credentials; import okhttp3.MediaType; import okhttp3.MultipartBody; import okhttp3.OkHttpClient; @@ -158,6 +163,7 @@ public class PushServiceSocket { private final ConnectionHolder[] cdnClients; private final ConnectionHolder[] contactDiscoveryClients; private final ConnectionHolder[] keyBackupServiceClients; + private final ConnectionHolder[] storageClients; private final OkHttpClient attachmentClient; private final CredentialsProvider credentialsProvider; @@ -171,6 +177,7 @@ public class PushServiceSocket { this.cdnClients = createConnectionHolders(signalServiceConfiguration.getSignalCdnUrls()); this.contactDiscoveryClients = createConnectionHolders(signalServiceConfiguration.getSignalContactDiscoveryUrls()); this.keyBackupServiceClients = createConnectionHolders(signalServiceConfiguration.getSignalKeyBackupServiceUrls()); + this.storageClients = createConnectionHolders(signalServiceConfiguration.getSignalStorageUrls()); this.attachmentClient = createAttachmentClient(); this.random = new SecureRandom(); } @@ -203,7 +210,7 @@ public class PushServiceSocket { } else if (challenge.isPresent()) { path += "?challenge=" + challenge.get(); } - + makeServiceRequest(path, "GET", null, headers, new ResponseCodeHandler() { @Override public void handle(int responseCode) throws NonSuccessfulResponseCodeException { @@ -426,8 +433,7 @@ public class PushServiceSocket { public PreKeyBundle getPreKey(SignalServiceAddress destination, int deviceId) throws IOException { try { - String path = String.format(PREKEY_DEVICE_PATH, destination.getIdentifier(), - String.valueOf(deviceId)); + String path = String.format(PREKEY_DEVICE_PATH, destination.getIdentifier(), String.valueOf(deviceId)); if (destination.getRelay().isPresent()) { path = path + "?relay=" + destination.getRelay().get(); @@ -543,7 +549,7 @@ public class PushServiceSocket { } public void retrieveProfileAvatar(String path, File destination, int maxSizeBytes) - throws NonSuccessfulResponseCodeException, PushNetworkException + throws NonSuccessfulResponseCodeException, PushNetworkException { downloadFromCdn(destination, path, maxSizeBytes, null); } @@ -688,6 +694,42 @@ public class PushServiceSocket { return JsonUtil.fromJson(response, TurnServerInfo.class); } + public String getStorageAuth() throws IOException { + String response = makeServiceRequest("/v1/storage/auth", "GET", null); + StorageAuthResponse authResponse = JsonUtil.fromJson(response, StorageAuthResponse.class); + + return Credentials.basic(authResponse.getUsername(), authResponse.getPassword()); + } + + public StorageManifest getStorageManifest(String authToken) throws IOException { + Response response = makeStorageRequest(authToken, "/v1/storage/manifest", "GET", null); + + if (response.body() == null) { + throw new IOException("Missing body!"); + } + + return StorageManifest.parseFrom(response.body().bytes()); + } + + public StorageItems readStorageItems(String authToken, ReadOperation operation) throws IOException { + Response response = makeStorageRequest(authToken, "/v1/storage/read", "PUT", operation.toByteArray()); + + if (response.body() == null) { + throw new IOException("Missing body!"); + } + + return StorageItems.parseFrom(response.body().bytes()); + } + + public Optional writeStorageContacts(String authToken, WriteOperation writeOperation) throws IOException { + try { + makeStorageRequest(authToken, "/v1/storage", "PUT", writeOperation.toByteArray()); + return Optional.absent(); + } catch (ContactManifestMismatchException e) { + return Optional.of(StorageManifest.parseFrom(e.getResponseBody())); + } + } + public void setSoTimeoutMillis(long soTimeoutMillis) { this.soTimeoutMillis = soTimeoutMillis; } @@ -812,17 +854,17 @@ public class PushServiceSocket { DigestingRequestBody file = new DigestingRequestBody(data, outputStreamFactory, contentType, length, progressListener); RequestBody requestBody = new MultipartBody.Builder() - .setType(MultipartBody.FORM) - .addFormDataPart("acl", acl) - .addFormDataPart("key", key) - .addFormDataPart("policy", policy) - .addFormDataPart("Content-Type", contentType) - .addFormDataPart("x-amz-algorithm", algorithm) - .addFormDataPart("x-amz-credential", credential) - .addFormDataPart("x-amz-date", date) - .addFormDataPart("x-amz-signature", signature) - .addFormDataPart("file", "file", file) - .build(); + .setType(MultipartBody.FORM) + .addFormDataPart("acl", acl) + .addFormDataPart("key", key) + .addFormDataPart("policy", policy) + .addFormDataPart("Content-Type", contentType) + .addFormDataPart("x-amz-algorithm", algorithm) + .addFormDataPart("x-amz-credential", credential) + .addFormDataPart("x-amz-date", date) + .addFormDataPart("x-amz-signature", signature) + .addFormDataPart("file", "file", file) + .build(); Request.Builder request = new Request.Builder() .url(connectionHolder.getUrl() + "/" + path) @@ -967,8 +1009,7 @@ public class PushServiceSocket { } if (responseCode != 200 && responseCode != 204) { - throw new NonSuccessfulResponseCodeException("Bad response: " + responseCode + " " + - responseMessage); + throw new NonSuccessfulResponseCodeException("Bad response: " + responseCode + " " + responseMessage); } return responseBody; @@ -1118,6 +1159,75 @@ public class PushServiceSocket { throw new NonSuccessfulResponseCodeException("Response: " + response); } + private Response makeStorageRequest(String authorization, String path, String method, byte[] body) + throws PushNetworkException, NonSuccessfulResponseCodeException + { + ConnectionHolder connectionHolder = getRandom(storageClients, random); + OkHttpClient okHttpClient = connectionHolder.getClient() + .newBuilder() + .connectTimeout(soTimeoutMillis, TimeUnit.MILLISECONDS) + .readTimeout(soTimeoutMillis, TimeUnit.MILLISECONDS) + .build(); + + Request.Builder request = new Request.Builder().url(connectionHolder.getUrl() + path); + + if (body != null) { + request.method(method, RequestBody.create(MediaType.parse("application/x-protobuf"), body)); + } else { + request.method(method, null); + } + + if (connectionHolder.getHostHeader().isPresent()) { + request.addHeader("Host", connectionHolder.getHostHeader().get()); + } + + if (authorization != null) { + request.addHeader("Authorization", authorization); + } + + Call call = okHttpClient.newCall(request.build()); + + synchronized (connections) { + connections.add(call); + } + + Response response; + + try { + response = call.execute(); + + if (response.isSuccessful()) { + return response; + } + } catch (IOException e) { + throw new PushNetworkException(e); + } finally { + synchronized (connections) { + connections.remove(call); + } + } + + switch (response.code()) { + case 401: + case 403: + throw new AuthorizationFailedException("Authorization failed!"); + case 404: + throw new NotFoundException("Not found"); + case 409: + if (response.body() != null) { + try { + throw new ContactManifestMismatchException(response.body().bytes()); + } catch (IOException e) { + throw new PushNetworkException(e); + } + } + case 429: + throw new RateLimitException("Rate limit exceeded: " + response.code()); + } + + throw new NonSuccessfulResponseCodeException("Response: " + response); + } + private ServiceConnectionHolder[] createServiceConnectionHolders(SignalUrl[] urls) { List serviceConnectionHolders = new LinkedList<>(); diff --git a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/util/Util.java b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/util/Util.java index b1a3f5b4af..2fd0d7e70f 100644 --- a/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/util/Util.java +++ b/libsignal/service/src/main/java/org/whispersystems/signalservice/internal/util/Util.java @@ -82,7 +82,7 @@ public class Util { return result; } - public static String readFully(InputStream in) throws IOException { + public static byte[] readFullyAsBytes(InputStream in) throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int read; @@ -93,7 +93,11 @@ public class Util { in.close(); - return new String(bout.toByteArray()); + return bout.toByteArray(); + } + + public static String readFully(InputStream in) throws IOException { + return new String(readFullyAsBytes(in)); } public static void readFully(InputStream in, byte[] buffer) throws IOException { diff --git a/src/org/thoughtcrime/securesms/ApplicationContext.java b/src/org/thoughtcrime/securesms/ApplicationContext.java index 7ab5714a75..232714471a 100644 --- a/src/org/thoughtcrime/securesms/ApplicationContext.java +++ b/src/org/thoughtcrime/securesms/ApplicationContext.java @@ -45,6 +45,7 @@ import org.thoughtcrime.securesms.insights.InsightsOptOut; import org.thoughtcrime.securesms.jobmanager.JobManager; import org.thoughtcrime.securesms.jobs.CreateSignedPreKeyJob; import org.thoughtcrime.securesms.jobs.FcmRefreshJob; +import org.thoughtcrime.securesms.jobs.StorageSyncJob; import org.thoughtcrime.securesms.jobs.MultiDeviceContactUpdateJob; import org.thoughtcrime.securesms.jobs.PushNotificationReceiveJob; import org.thoughtcrime.securesms.logging.AndroidLogger; diff --git a/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelper.java b/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelper.java index a63ea6610e..982cb6dcc7 100644 --- a/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelper.java +++ b/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelper.java @@ -6,6 +6,8 @@ import androidx.annotation.NonNull; import androidx.annotation.WorkerThread; import org.thoughtcrime.securesms.database.RecipientDatabase.RegisteredState; +import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; +import org.thoughtcrime.securesms.jobs.StorageSyncJob; import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.util.FeatureFlags; @@ -21,15 +23,28 @@ public class DirectoryHelper { } else { DirectoryHelperV1.refreshDirectory(context, notifyOfNewUsers); } + + if (FeatureFlags.STORAGE_SERVICE) { + ApplicationDependencies.getJobManager().add(new StorageSyncJob()); + } } @WorkerThread public static RegisteredState refreshDirectoryFor(@NonNull Context context, @NonNull Recipient recipient, boolean notifyOfNewUsers) throws IOException { + RegisteredState originalRegisteredState = recipient.resolve().getRegistered(); + RegisteredState newRegisteredState = null; + if (FeatureFlags.UUIDS) { // TODO [greyson] Create a DirectoryHelperV2 when appropriate. - return DirectoryHelperV1.refreshDirectoryFor(context, recipient, notifyOfNewUsers); + newRegisteredState = DirectoryHelperV1.refreshDirectoryFor(context, recipient, notifyOfNewUsers); } else { - return DirectoryHelperV1.refreshDirectoryFor(context, recipient, notifyOfNewUsers); + newRegisteredState = DirectoryHelperV1.refreshDirectoryFor(context, recipient, notifyOfNewUsers); } + + if (FeatureFlags.STORAGE_SERVICE && newRegisteredState != originalRegisteredState) { + ApplicationDependencies.getJobManager().add(new StorageSyncJob()); + } + + return newRegisteredState; } } diff --git a/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelperV1.java b/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelperV1.java index a011608f82..ba53296eba 100644 --- a/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelperV1.java +++ b/src/org/thoughtcrime/securesms/contacts/sync/DirectoryHelperV1.java @@ -20,7 +20,6 @@ import androidx.annotation.WorkerThread; import com.annimon.stream.Collectors; import com.annimon.stream.Stream; -import org.thoughtcrime.securesms.ApplicationContext; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.contacts.ContactAccessor; import org.thoughtcrime.securesms.crypto.SessionUtil; @@ -155,7 +154,7 @@ class DirectoryHelperV1 { DatabaseFactory.getContactsDatabase(context).setRegisteredUsers(account.get().getAccount(), activeAddresses, removeMissing); Cursor cursor = ContactAccessor.getInstance().getAllSystemContacts(context); - RecipientDatabase.BulkOperationsHandle handle = DatabaseFactory.getRecipientDatabase(context).resetAllSystemContactInfo(); + RecipientDatabase.BulkOperationsHandle handle = DatabaseFactory.getRecipientDatabase(context).beginBulkSystemContactUpdate(); try { while (cursor != null && cursor.moveToNext()) { diff --git a/src/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelper.java b/src/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelper.java new file mode 100644 index 0000000000..04cb4c754d --- /dev/null +++ b/src/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelper.java @@ -0,0 +1,537 @@ +package org.thoughtcrime.securesms.contacts.sync; + +import android.content.Context; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; + +import com.annimon.stream.Stream; + +import org.thoughtcrime.securesms.database.DatabaseFactory; +import org.thoughtcrime.securesms.database.IdentityDatabase; +import org.thoughtcrime.securesms.database.RecipientDatabase; +import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings; +import org.thoughtcrime.securesms.database.StorageKeyDatabase; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.recipients.RecipientId; +import org.thoughtcrime.securesms.util.SetUtil; +import org.thoughtcrime.securesms.util.Util; +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.push.SignalServiceAddress; +import org.whispersystems.signalservice.api.storage.SignalContactRecord; +import org.whispersystems.signalservice.api.storage.SignalContactRecord.IdentityState; +import org.whispersystems.signalservice.api.storage.SignalStorageManifest; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +public final class StorageSyncHelper { + + private static final String TAG = Log.tag(StorageSyncHelper.class); + + private static final KeyGenerator KEY_GENERATOR = () -> Util.getSecretBytes(16); + + private static KeyGenerator testKeyGenerator = null; + + /** + * Given the local state of pending storage mutatations, this will generate a result that will + * include that data that needs to be written to the storage service, as well as any changes you + * need to write back to local storage (like storage keys that might have changed for updated + * contacts). + * + * @param currentManifestVersion What you think the version is locally. + * @param currentLocalKeys All local keys you have. This assumes that 'inserts' were given keys + * already, and that deletes still have keys. + * @param updates Contacts that have been altered. + * @param inserts Contacts that have been inserted (or newly marked as registered). + * @param deletes Contacts that are no longer registered. + * + * @return If changes need to be written, then it will return those changes. If no changes need + * to be written, this will return {@link Optional#absent()}. + */ + public static @NonNull Optional buildStorageUpdatesForLocal(long currentManifestVersion, + @NonNull List currentLocalKeys, + @NonNull List updates, + @NonNull List inserts, + @NonNull List deletes) + { + Set completeKeys = new LinkedHashSet<>(Stream.of(currentLocalKeys).map(ByteBuffer::wrap).toList()); + Set contactInserts = new LinkedHashSet<>(); + Set contactDeletes = new LinkedHashSet<>(); + Map storageKeyUpdates = new HashMap<>(); + + for (RecipientSettings insert : inserts) { + contactInserts.add(localToRemoteContact(insert)); + } + + for (RecipientSettings delete : deletes) { + byte[] key = Objects.requireNonNull(delete.getStorageKey()); + contactDeletes.add(ByteBuffer.wrap(key)); + completeKeys.remove(ByteBuffer.wrap(key)); + } + + for (RecipientSettings update : updates) { + byte[] oldKey = Objects.requireNonNull(update.getStorageKey()); + byte[] newKey = generateKey(); + + contactInserts.add(localToRemoteContact(update, newKey)); + contactDeletes.add(ByteBuffer.wrap(oldKey)); + completeKeys.remove(ByteBuffer.wrap(oldKey)); + completeKeys.add(ByteBuffer.wrap(newKey)); + storageKeyUpdates.put(update.getId(), newKey); + } + + if (contactInserts.isEmpty() && contactDeletes.isEmpty()) { + return Optional.absent(); + } else { + List storageInserts = Stream.of(contactInserts).map(c -> SignalStorageRecord.forContact(c.getKey(), c)).toList(); + List contactDeleteBytes = Stream.of(contactDeletes).map(ByteBuffer::array).toList(); + List completeKeysBytes = Stream.of(completeKeys).map(ByteBuffer::array).toList(); + SignalStorageManifest manifest = new SignalStorageManifest(currentManifestVersion + 1, completeKeysBytes); + WriteOperationResult writeOperationResult = new WriteOperationResult(manifest, storageInserts, contactDeleteBytes); + + return Optional.of(new LocalWriteResult(writeOperationResult, storageKeyUpdates)); + } + } + + /** + * Given a list of all the local and remote keys you know about, this will return a result telling + * you which keys are exclusively remote and which are exclusively local. + * + * @param remoteKeys All remote keys available. + * @param localKeys All local keys available. + * + * @return An object describing which keys are exclusive to the remote data set and which keys are + * exclusive to the local data set. + */ + public static @NonNull KeyDifferenceResult findKeyDifference(@NonNull List remoteKeys, + @NonNull List localKeys) + { + Set allRemoteKeys = Stream.of(remoteKeys).map(ByteBuffer::wrap).collect(LinkedHashSet::new, HashSet::add); + Set allLocalKeys = Stream.of(localKeys).map(ByteBuffer::wrap).collect(LinkedHashSet::new, HashSet::add); + + Set remoteOnlyKeys = SetUtil.difference(allRemoteKeys, allLocalKeys); + Set localOnlyKeys = SetUtil.difference(allLocalKeys, allRemoteKeys); + + return new KeyDifferenceResult(Stream.of(remoteOnlyKeys).map(ByteBuffer::array).toList(), + Stream.of(localOnlyKeys).map(ByteBuffer::array).toList()); + } + + /** + * Given two sets of storage records, this will resolve the data into a set of actions that need + * to be applied to resolve the differences. This will handle discovering which records between + * the two collections refer to the same contacts and are actually updates, which are brand new, + * etc. + * + * @param remoteOnlyRecords Records that are only present remotely. + * @param localOnlyRecords Records that are only present locally. + * + * @return A set of actions that should be applied to resolve the conflict. + */ + public static @NonNull MergeResult resolveConflict(@NonNull Collection remoteOnlyRecords, + @NonNull Collection localOnlyRecords) + { + List remoteOnlyContacts = Stream.of(remoteOnlyRecords).filter(r -> r.getContact().isPresent()).map(r -> r.getContact().get()).toList(); + List localOnlyContacts = Stream.of(localOnlyRecords).filter(r -> r.getContact().isPresent()).map(r -> r.getContact().get()).toList(); + + List remoteOnlyUnknowns = Stream.of(remoteOnlyRecords).filter(SignalStorageRecord::isUnknown).toList(); + List localOnlyUnknowns = Stream.of(localOnlyRecords).filter(SignalStorageRecord::isUnknown).toList(); + + ContactRecordMergeResult contactMergeResult = resolveContactConflict(remoteOnlyContacts, localOnlyContacts); + + return new MergeResult(contactMergeResult.localInserts, + contactMergeResult.localUpdates, + contactMergeResult.remoteInserts, + contactMergeResult.remoteUpdates, + new LinkedHashSet<>(remoteOnlyUnknowns), + new LinkedHashSet<>(localOnlyUnknowns)); + } + + /** + * Assumes that the merge result has *not* yet been applied to the local data. That means that + * this method will handle generating the correct final key set based on the merge result. + */ + public static @NonNull WriteOperationResult createWriteOperation(long currentManifestVersion, + @NonNull List currentLocalStorageKeys, + @NonNull MergeResult mergeResult) + { + Set completeKeys = new LinkedHashSet<>(Stream.of(currentLocalStorageKeys).map(ByteBuffer::wrap).toList()); + + for (SignalContactRecord insert : mergeResult.getLocalContactInserts()) { + completeKeys.add(ByteBuffer.wrap(insert.getKey())); + } + + for (SignalContactRecord insert : mergeResult.getRemoteContactInserts()) { + completeKeys.add(ByteBuffer.wrap(insert.getKey())); + } + + for (SignalStorageRecord insert : mergeResult.getLocalUnknownInserts()) { + completeKeys.add(ByteBuffer.wrap(insert.getKey())); + } + + for (ContactUpdate update : mergeResult.getLocalContactUpdates()) { + completeKeys.remove(ByteBuffer.wrap(update.getOldContact().getKey())); + completeKeys.add(ByteBuffer.wrap(update.getNewContact().getKey())); + } + + for (ContactUpdate update : mergeResult.getRemoteContactUpdates()) { + completeKeys.remove(ByteBuffer.wrap(update.getOldContact().getKey())); + completeKeys.add(ByteBuffer.wrap(update.getNewContact().getKey())); + } + + SignalStorageManifest manifest = new SignalStorageManifest(currentManifestVersion + 1, Stream.of(completeKeys).map(ByteBuffer::array).toList()); + + List contactInserts = new ArrayList<>(); + contactInserts.addAll(mergeResult.getRemoteContactInserts()); + contactInserts.addAll(Stream.of(mergeResult.getRemoteContactUpdates()).map(ContactUpdate::getNewContact).toList()); + + List inserts = Stream.of(contactInserts).map(c -> SignalStorageRecord.forContact(c.getKey(), c)).toList(); + + List deletes = Stream.of(mergeResult.getRemoteContactUpdates()).map(ContactUpdate::getOldContact).map(SignalContactRecord::getKey).toList(); + + return new WriteOperationResult(manifest, inserts, deletes); + } + + public static @NonNull SignalContactRecord localToRemoteContact(@NonNull RecipientSettings recipient) { + if (recipient.getStorageKey() == null) { + throw new AssertionError("Must have a storage key!"); + } + + return localToRemoteContact(recipient, recipient.getStorageKey()); + } + + private static @NonNull SignalContactRecord localToRemoteContact(@NonNull RecipientSettings recipient, byte[] storageKey) { + if (recipient.getUuid() == null && recipient.getE164() == null) { + throw new AssertionError("Must have either a UUID or a phone number!"); + } + + return new SignalContactRecord.Builder(storageKey, new SignalServiceAddress(recipient.getUuid(), recipient.getE164())) + .setProfileKey(recipient.getProfileKey()) + .setProfileName(recipient.getProfileName()) + .setBlocked(recipient.isBlocked()) + .setProfileSharingEnabled(recipient.isProfileSharing()) + .setIdentityKey(recipient.getIdentityKey()) + .setIdentityState(localToRemoteIdentityState(recipient.getIdentityStatus())) + .build(); + } + + public static @NonNull IdentityDatabase.VerifiedStatus remoteToLocalIdentityStatus(@NonNull IdentityState identityState) { + switch (identityState) { + case VERIFIED: return IdentityDatabase.VerifiedStatus.VERIFIED; + case UNVERIFIED: return IdentityDatabase.VerifiedStatus.UNVERIFIED; + default: return IdentityDatabase.VerifiedStatus.DEFAULT; + } + } + + public static @NonNull byte[] generateKey() { + if (testKeyGenerator != null) { + return testKeyGenerator.generate(); + } else { + return KEY_GENERATOR.generate(); + } + } + + @VisibleForTesting + static @NonNull SignalContactRecord mergeContacts(@NonNull SignalContactRecord remote, + @NonNull SignalContactRecord local) + { + UUID uuid = remote.getAddress().getUuid().or(local.getAddress().getUuid()).orNull(); + String e164 = remote.getAddress().getNumber().or(local.getAddress().getNumber()).orNull(); + SignalServiceAddress address = new SignalServiceAddress(uuid, e164); + String profileName = remote.getProfileName().or(local.getProfileName()).orNull(); + byte[] profileKey = remote.getProfileKey().or(local.getProfileKey()).orNull(); + String username = remote.getUsername().or(local.getUsername()).orNull(); + IdentityState identityState = remote.getIdentityState(); + byte[] identityKey = remote.getIdentityKey().or(local.getIdentityKey()).orNull(); + String nickname = local.getNickname().orNull(); // TODO [greyson] Update this when we add real nickname support + boolean blocked = remote.isBlocked(); + boolean profileSharing = remote.isProfileSharingEnabled() | local.isProfileSharingEnabled(); + boolean matchesRemote = doParamsMatchContact(remote, address, profileName, profileKey, username, identityState, identityKey, blocked, profileSharing, nickname); + boolean matchesLocal = doParamsMatchContact(local, address, profileName, profileKey, username, identityState, identityKey, blocked, profileSharing, nickname); + + if (remote.getProtoVersion() > 0) { + Log.w(TAG, "Inbound model has version " + remote.getProtoVersion() + ", but our version is 0."); + } + + if (matchesRemote) { + return remote; + } else if (matchesLocal) { + return local; + } else { + return new SignalContactRecord.Builder(generateKey(), address) + .setProfileName(profileName) + .setProfileKey(profileKey) + .setUsername(username) + .setIdentityState(identityState) + .setIdentityKey(identityKey) + .setBlocked(blocked) + .setProfileSharingEnabled(profileSharing) + .setNickname(nickname) + .build(); + } + } + + @VisibleForTesting + static void setTestKeyGenerator(@Nullable KeyGenerator keyGenerator) { + testKeyGenerator = keyGenerator; + } + + private static IdentityState localToRemoteIdentityState(@NonNull IdentityDatabase.VerifiedStatus local) { + switch (local) { + case VERIFIED: return IdentityState.VERIFIED; + case UNVERIFIED: return IdentityState.UNVERIFIED; + default: return IdentityState.DEFAULT; + } + } + + private static boolean doParamsMatchContact(@NonNull SignalContactRecord contact, + @NonNull SignalServiceAddress address, + @Nullable String profileName, + @Nullable byte[] profileKey, + @Nullable String username, + @Nullable IdentityState identityState, + @Nullable byte[] identityKey, + boolean blocked, + boolean profileSharing, + @Nullable String nickname) + { + return Objects.equals(contact.getAddress(), address) && + Objects.equals(contact.getProfileName().orNull(), profileName) && + Arrays.equals(contact.getProfileKey().orNull(), profileKey) && + Objects.equals(contact.getUsername().orNull(), username) && + Objects.equals(contact.getIdentityState(), identityState) && + Arrays.equals(contact.getIdentityKey().orNull(), identityKey) && + contact.isBlocked() == blocked && + contact.isProfileSharingEnabled() == profileSharing && + Objects.equals(contact.getNickname().orNull(), nickname); + } + + private static @NonNull ContactRecordMergeResult resolveContactConflict(@NonNull Collection remoteOnlyRecords, + @NonNull Collection localOnlyRecords) + { + Map localByUuid = new HashMap<>(); + Map localByE164 = new HashMap<>(); + + for (SignalContactRecord contact : localOnlyRecords) { + if (contact.getAddress().getUuid().isPresent()) { + localByUuid.put(contact.getAddress().getUuid().get(), contact); + } + if (contact.getAddress().getNumber().isPresent()) { + localByE164.put(contact.getAddress().getNumber().get(), contact); + } + } + + Set localInserts = new LinkedHashSet<>(remoteOnlyRecords); + Set remoteInserts = new LinkedHashSet<>(localOnlyRecords); + Set localUpdates = new LinkedHashSet<>(); + Set remoteUpdates = new LinkedHashSet<>(); + + for (SignalContactRecord remote : remoteOnlyRecords) { + SignalContactRecord localUuid = remote.getAddress().getUuid().isPresent() ? localByUuid.get(remote.getAddress().getUuid().get()) : null; + SignalContactRecord localE164 = remote.getAddress().getNumber().isPresent() ? localByE164.get(remote.getAddress().getNumber().get()) : null; + + Optional local = Optional.fromNullable(localUuid).or(Optional.fromNullable(localE164)); + + if (local.isPresent()) { + SignalContactRecord merged = mergeContacts(remote, local.get()); + + if (!merged.equals(remote)) { + remoteUpdates.add(new ContactUpdate(remote, merged)); + } + + if (!merged.equals(local.get())) { + localUpdates.add(new ContactUpdate(local.get(), merged)); + } + + localInserts.remove(remote); + remoteInserts.remove(local.get()); + } + } + + return new ContactRecordMergeResult(localInserts, localUpdates, remoteInserts, remoteUpdates); + } + + public static final class ContactUpdate { + private final SignalContactRecord oldContact; + private final SignalContactRecord newContact; + + public ContactUpdate(@NonNull SignalContactRecord oldContact, @NonNull SignalContactRecord newContact) { + this.oldContact = oldContact; + this.newContact = newContact; + } + + public @NonNull + SignalContactRecord getOldContact() { + return oldContact; + } + + public @NonNull + SignalContactRecord getNewContact() { + return newContact; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContactUpdate that = (ContactUpdate) o; + return oldContact.equals(that.oldContact) && + newContact.equals(that.newContact); + } + + @Override + public int hashCode() { + return Objects.hash(oldContact, newContact); + } + } + + public static final class KeyDifferenceResult { + private final List remoteOnlyKeys; + private final List localOnlyKeys; + + private KeyDifferenceResult(@NonNull List remoteOnlyKeys, @NonNull List localOnlyKeys) { + this.remoteOnlyKeys = remoteOnlyKeys; + this.localOnlyKeys = localOnlyKeys; + } + + public @NonNull List getRemoteOnlyKeys() { + return remoteOnlyKeys; + } + + public @NonNull List getLocalOnlyKeys() { + return localOnlyKeys; + } + + public boolean isEmpty() { + return remoteOnlyKeys.isEmpty() && localOnlyKeys.isEmpty(); + } + } + + public static final class MergeResult { + private final Set localContactInserts; + private final Set localContactUpdates; + private final Set remoteContactInserts; + private final Set remoteContactUpdates; + private final Set localUnknownInserts; + private final Set localUnknownDeletes; + + @VisibleForTesting + MergeResult(@NonNull Set localContactInserts, + @NonNull Set localContactUpdates, + @NonNull Set remoteContactInserts, + @NonNull Set remoteContactUpdates, + @NonNull Set localUnknownInserts, + @NonNull Set localUnknownDeletes) + { + this.localContactInserts = localContactInserts; + this.localContactUpdates = localContactUpdates; + this.remoteContactInserts = remoteContactInserts; + this.remoteContactUpdates = remoteContactUpdates; + this.localUnknownInserts = localUnknownInserts; + this.localUnknownDeletes = localUnknownDeletes; + } + + public @NonNull Set getLocalContactInserts() { + return localContactInserts; + } + + public @NonNull Set getLocalContactUpdates() { + return localContactUpdates; + } + + public @NonNull Set getRemoteContactInserts() { + return remoteContactInserts; + } + + public @NonNull Set getRemoteContactUpdates() { + return remoteContactUpdates; + } + + public @NonNull Set getLocalUnknownInserts() { + return localUnknownInserts; + } + + public @NonNull Set getLocalUnknownDeletes() { + return localUnknownDeletes; + } + } + + public static final class WriteOperationResult { + private final SignalStorageManifest manifest; + private final List inserts; + private final List deletes; + + private WriteOperationResult(@NonNull SignalStorageManifest manifest, + @NonNull List inserts, + @NonNull List deletes) + { + this.manifest = manifest; + this.inserts = inserts; + this.deletes = deletes; + } + + public @NonNull SignalStorageManifest getManifest() { + return manifest; + } + + public @NonNull List getInserts() { + return inserts; + } + + public @NonNull List getDeletes() { + return deletes; + } + } + + public static class LocalWriteResult { + private final WriteOperationResult writeResult; + private final Map storageKeyUpdates; + + public LocalWriteResult(WriteOperationResult writeResult, Map storageKeyUpdates) { + this.writeResult = writeResult; + this.storageKeyUpdates = storageKeyUpdates; + } + + public @NonNull WriteOperationResult getWriteResult() { + return writeResult; + } + + public @NonNull Map getStorageKeyUpdates() { + return storageKeyUpdates; + } + } + + private static final class ContactRecordMergeResult { + final Set localInserts; + final Set localUpdates; + final Set remoteInserts; + final Set remoteUpdates; + + ContactRecordMergeResult(@NonNull Set localInserts, + @NonNull Set localUpdates, + @NonNull Set remoteInserts, + @NonNull Set remoteUpdates) + { + this.localInserts = localInserts; + this.localUpdates = localUpdates; + this.remoteInserts = remoteInserts; + this.remoteUpdates = remoteUpdates; + } + } + + interface KeyGenerator { + @NonNull byte[] generate(); + } +} diff --git a/src/org/thoughtcrime/securesms/database/DatabaseFactory.java b/src/org/thoughtcrime/securesms/database/DatabaseFactory.java index c459e1dfea..ec0b075fad 100644 --- a/src/org/thoughtcrime/securesms/database/DatabaseFactory.java +++ b/src/org/thoughtcrime/securesms/database/DatabaseFactory.java @@ -59,6 +59,7 @@ public class DatabaseFactory { private final SearchDatabase searchDatabase; private final JobDatabase jobDatabase; private final StickerDatabase stickerDatabase; + private final StorageKeyDatabase storageKeyDatabase; public static DatabaseFactory getInstance(Context context) { synchronized (lock) { @@ -145,6 +146,10 @@ public class DatabaseFactory { return getInstance(context).stickerDatabase; } + public static StorageKeyDatabase getStorageKeyDatabase(Context context) { + return getInstance(context).storageKeyDatabase; + } + public static SQLiteDatabase getBackupDatabase(Context context) { return getInstance(context).databaseHelper.getReadableDatabase(); } @@ -181,6 +186,7 @@ public class DatabaseFactory { this.searchDatabase = new SearchDatabase(context, databaseHelper); this.jobDatabase = new JobDatabase(context, databaseHelper); this.stickerDatabase = new StickerDatabase(context, databaseHelper, attachmentSecret); + this.storageKeyDatabase = new StorageKeyDatabase(context, databaseHelper); } public void onApplicationLevelUpgrade(@NonNull Context context, @NonNull MasterSecret masterSecret, diff --git a/src/org/thoughtcrime/securesms/database/IdentityDatabase.java b/src/org/thoughtcrime/securesms/database/IdentityDatabase.java index ce313c7fd3..acc57d1b53 100644 --- a/src/org/thoughtcrime/securesms/database/IdentityDatabase.java +++ b/src/org/thoughtcrime/securesms/database/IdentityDatabase.java @@ -26,6 +26,7 @@ import net.sqlcipher.database.SQLiteDatabase; import org.greenrobot.eventbus.EventBus; import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper; +import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.recipients.RecipientId; import org.thoughtcrime.securesms.util.Base64; import org.whispersystems.libsignal.IdentityKey; @@ -39,14 +40,14 @@ public class IdentityDatabase extends Database { @SuppressWarnings("unused") private static final String TAG = IdentityDatabase.class.getSimpleName(); - private static final String TABLE_NAME = "identities"; + static final String TABLE_NAME = "identities"; private static final String ID = "_id"; - private static final String RECIPIENT_ID = "address"; - private static final String IDENTITY_KEY = "key"; + static final String RECIPIENT_ID = "address"; + static final String IDENTITY_KEY = "key"; private static final String TIMESTAMP = "timestamp"; private static final String FIRST_USE = "first_use"; private static final String NONBLOCKING_APPROVAL = "nonblocking_approval"; - private static final String VERIFIED = "verified"; + static final String VERIFIED = "verified"; public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ID + " INTEGER PRIMARY KEY, " + @@ -112,21 +113,8 @@ public class IdentityDatabase extends Database { public void saveIdentity(@NonNull RecipientId recipientId, IdentityKey identityKey, VerifiedStatus verifiedStatus, boolean firstUse, long timestamp, boolean nonBlockingApproval) { - SQLiteDatabase database = databaseHelper.getWritableDatabase(); - String identityKeyString = Base64.encodeBytes(identityKey.serialize()); - - ContentValues contentValues = new ContentValues(); - contentValues.put(RECIPIENT_ID, recipientId.serialize()); - contentValues.put(IDENTITY_KEY, identityKeyString); - contentValues.put(TIMESTAMP, timestamp); - contentValues.put(VERIFIED, verifiedStatus.toInt()); - contentValues.put(NONBLOCKING_APPROVAL, nonBlockingApproval ? 1 : 0); - contentValues.put(FIRST_USE, firstUse ? 1 : 0); - - database.replace(TABLE_NAME, null, contentValues); - - EventBus.getDefault().post(new IdentityRecord(recipientId, identityKey, verifiedStatus, - firstUse, timestamp, nonBlockingApproval)); + saveIdentityInternal(recipientId, identityKey, verifiedStatus, firstUse, timestamp, nonBlockingApproval); + DatabaseFactory.getRecipientDatabase(context).markDirty(recipientId, RecipientDatabase.DirtyState.UPDATE); } public void setApproval(@NonNull RecipientId recipientId, boolean nonBlockingApproval) { @@ -136,6 +124,8 @@ public class IdentityDatabase extends Database { contentValues.put(NONBLOCKING_APPROVAL, nonBlockingApproval); database.update(TABLE_NAME, contentValues, RECIPIENT_ID + " = ?", new String[] {recipientId.serialize()}); + + DatabaseFactory.getRecipientDatabase(context).markDirty(recipientId, RecipientDatabase.DirtyState.UPDATE); } public void setVerified(@NonNull RecipientId recipientId, IdentityKey identityKey, VerifiedStatus verifiedStatus) { @@ -150,6 +140,25 @@ public class IdentityDatabase extends Database { if (updated > 0) { Optional record = getIdentity(recipientId); if (record.isPresent()) EventBus.getDefault().post(record.get()); + DatabaseFactory.getRecipientDatabase(context).markDirty(recipientId, RecipientDatabase.DirtyState.UPDATE); + } + } + + public void updateIdentityAfterSync(@NonNull RecipientId id, IdentityKey identityKey, VerifiedStatus verifiedStatus) { + if (!hasMatchingKey(id, identityKey, verifiedStatus)) { + saveIdentityInternal(id, identityKey, verifiedStatus, false, System.currentTimeMillis(), true); + Optional record = getIdentity(id); + if (record.isPresent()) EventBus.getDefault().post(record.get()); + } + } + + private boolean hasMatchingKey(@NonNull RecipientId id, IdentityKey identityKey, VerifiedStatus verifiedStatus) { + SQLiteDatabase db = databaseHelper.getReadableDatabase(); + String query = RECIPIENT_ID + " = ? AND " + IDENTITY_KEY + " = ? AND " + VERIFIED + " = ?"; + String[] args = new String[]{id.serialize(), Base64.encodeBytes(identityKey.serialize()), String.valueOf(verifiedStatus.toInt())}; + + try (Cursor cursor = db.query(TABLE_NAME, null, query, args, null, null, null)) { + return cursor != null && cursor.moveToFirst(); } } @@ -165,6 +174,26 @@ public class IdentityDatabase extends Database { return new IdentityRecord(RecipientId.from(recipientId), identity, VerifiedStatus.forState(verifiedStatus), firstUse, timestamp, nonblockingApproval); } + private void saveIdentityInternal(@NonNull RecipientId recipientId, IdentityKey identityKey, VerifiedStatus verifiedStatus, + boolean firstUse, long timestamp, boolean nonBlockingApproval) + { + SQLiteDatabase database = databaseHelper.getWritableDatabase(); + String identityKeyString = Base64.encodeBytes(identityKey.serialize()); + + ContentValues contentValues = new ContentValues(); + contentValues.put(RECIPIENT_ID, recipientId.serialize()); + contentValues.put(IDENTITY_KEY, identityKeyString); + contentValues.put(TIMESTAMP, timestamp); + contentValues.put(VERIFIED, verifiedStatus.toInt()); + contentValues.put(NONBLOCKING_APPROVAL, nonBlockingApproval ? 1 : 0); + contentValues.put(FIRST_USE, firstUse ? 1 : 0); + + database.replace(TABLE_NAME, null, contentValues); + + EventBus.getDefault().post(new IdentityRecord(recipientId, identityKey, verifiedStatus, + firstUse, timestamp, nonBlockingApproval)); + } + public static class IdentityRecord { private final RecipientId recipientId; diff --git a/src/org/thoughtcrime/securesms/database/RecipientDatabase.java b/src/org/thoughtcrime/securesms/database/RecipientDatabase.java index 4780c455e8..2791d6230c 100644 --- a/src/org/thoughtcrime/securesms/database/RecipientDatabase.java +++ b/src/org/thoughtcrime/securesms/database/RecipientDatabase.java @@ -10,24 +10,34 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.annimon.stream.Stream; +import com.google.android.gms.common.util.ArrayUtils; import net.sqlcipher.database.SQLiteDatabase; import org.thoughtcrime.securesms.color.MaterialColor; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper; +import org.thoughtcrime.securesms.database.IdentityDatabase.IdentityRecord; import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper; import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.recipients.RecipientId; import org.thoughtcrime.securesms.util.Base64; +import org.thoughtcrime.securesms.util.FeatureFlags; import org.thoughtcrime.securesms.util.GroupUtil; +import org.thoughtcrime.securesms.util.IdentityUtil; import org.thoughtcrime.securesms.util.Util; +import org.whispersystems.libsignal.IdentityKey; +import org.whispersystems.libsignal.InvalidKeyException; import org.whispersystems.libsignal.util.guava.Optional; import org.whispersystems.signalservice.api.push.SignalServiceAddress; import org.whispersystems.signalservice.api.util.UuidUtil; +import org.whispersystems.signalservice.api.storage.SignalContactRecord; import java.io.Closeable; import java.io.IOException; +import java.lang.reflect.Array; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -66,6 +76,7 @@ public class RecipientDatabase extends Database { public static final String SYSTEM_PHONE_TYPE = "system_phone_type"; public static final String SYSTEM_PHONE_LABEL = "system_phone_label"; private static final String SYSTEM_CONTACT_URI = "system_contact_uri"; + private static final String SYSTEM_INFO_PENDING = "system_info_pending"; private static final String PROFILE_KEY = "profile_key"; public static final String SIGNAL_PROFILE_NAME = "signal_profile_name"; private static final String SIGNAL_PROFILE_AVATAR = "signal_profile_avatar"; @@ -73,8 +84,13 @@ public class RecipientDatabase extends Database { private static final String UNIDENTIFIED_ACCESS_MODE = "unidentified_access_mode"; private static final String FORCE_SMS_SELECTION = "force_sms_selection"; private static final String UUID_SUPPORTED = "uuid_supported"; + private static final String STORAGE_SERVICE_KEY = "storage_service_key"; + private static final String DIRTY = "dirty"; private static final String SORT_NAME = "sort_name"; + private static final String IDENTITY_STATUS = "identity_status"; + private static final String IDENTITY_KEY = "identity_key"; + private static final String[] RECIPIENT_PROJECTION = new String[] { UUID, USERNAME, PHONE, EMAIL, GROUP_ID, @@ -82,7 +98,20 @@ public class RecipientDatabase extends Database { PROFILE_KEY, SYSTEM_DISPLAY_NAME, SYSTEM_PHOTO_URI, SYSTEM_PHONE_LABEL, SYSTEM_PHONE_TYPE, SYSTEM_CONTACT_URI, SIGNAL_PROFILE_NAME, SIGNAL_PROFILE_AVATAR, PROFILE_SHARING, NOTIFICATION_CHANNEL, UNIDENTIFIED_ACCESS_MODE, - FORCE_SMS_SELECTION, UUID_SUPPORTED + FORCE_SMS_SELECTION, UUID_SUPPORTED, STORAGE_SERVICE_KEY, DIRTY + }; + + private static final String[] RECIPIENT_FULL_PROJECTION = ArrayUtils.concat( + new String[] { TABLE_NAME + "." + ID }, + RECIPIENT_PROJECTION, + new String[] { + IdentityDatabase.TABLE_NAME + "." + IdentityDatabase.VERIFIED + " AS " + IDENTITY_STATUS, + IdentityDatabase.TABLE_NAME + "." + IdentityDatabase.IDENTITY_KEY + " AS " + IDENTITY_KEY + }); + + + public static final String[] CREATE_INDEXS = new String[] { + "CREATE INDEX IF NOT EXISTS recipient_dirty_index ON " + TABLE_NAME + " (" + DIRTY + ");", }; private static final String[] ID_PROJECTION = new String[]{ID}; @@ -167,6 +196,20 @@ public class RecipientDatabase extends Database { } } + enum DirtyState { + CLEAN(0), UPDATE(1), INSERT(2), DELETE(3); + + private final int id; + + DirtyState(int id) { + this.id = id; + } + + int getId() { + return id; + } + } + public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ID + " INTEGER PRIMARY KEY AUTOINCREMENT, " + UUID + " TEXT UNIQUE DEFAULT NULL, " + @@ -191,13 +234,16 @@ public class RecipientDatabase extends Database { SYSTEM_PHONE_LABEL + " TEXT DEFAULT NULL, " + SYSTEM_PHONE_TYPE + " INTEGER DEFAULT -1, " + SYSTEM_CONTACT_URI + " TEXT DEFAULT NULL, " + + SYSTEM_INFO_PENDING + " INTEGER DEFAULT 0, " + PROFILE_KEY + " TEXT DEFAULT NULL, " + SIGNAL_PROFILE_NAME + " TEXT DEFAULT NULL, " + SIGNAL_PROFILE_AVATAR + " TEXT DEFAULT NULL, " + PROFILE_SHARING + " INTEGER DEFAULT 0, " + UNIDENTIFIED_ACCESS_MODE + " INTEGER DEFAULT 0, " + FORCE_SMS_SELECTION + " INTEGER DEFAULT 0, " + - UUID_SUPPORTED + " INTEGER DEFAULT 0);"; + UUID_SUPPORTED + " INTEGER DEFAULT 0, " + + STORAGE_SERVICE_KEY + " TEXT UNIQUE DEFAULT NULL, " + + DIRTY + " INTEGER DEFAULT 0);"; private static final String INSIGHTS_INVITEE_LIST = "SELECT " + TABLE_NAME + "." + ID + " FROM " + TABLE_NAME + @@ -270,7 +316,7 @@ public class RecipientDatabase extends Database { } public RecipientReader readerForBlocked(Cursor cursor) { - return new RecipientReader(context, cursor); + return new RecipientReader(cursor); } public RecipientReader getRecipientsWithNotificationChannels() { @@ -278,15 +324,16 @@ public class RecipientDatabase extends Database { Cursor cursor = database.query(TABLE_NAME, ID_PROJECTION, NOTIFICATION_CHANNEL + " NOT NULL", null, null, null, null, null); - return new RecipientReader(context, cursor); + return new RecipientReader(cursor); } public @NonNull RecipientSettings getRecipientSettings(@NonNull RecipientId id) { SQLiteDatabase database = databaseHelper.getReadableDatabase(); - String query = ID + " = ?"; + String table = TABLE_NAME + " LEFT OUTER JOIN " + IdentityDatabase.TABLE_NAME + " ON " + TABLE_NAME + "." + ID + " = " + IdentityDatabase.TABLE_NAME + "." + IdentityDatabase.RECIPIENT_ID; + String query = TABLE_NAME + "." + ID + " = ?"; String[] args = new String[] { id.serialize() }; - try (Cursor cursor = database.query(TABLE_NAME, null, query, args, null, null, null)) { + try (Cursor cursor = database.query(table, RECIPIENT_FULL_PROJECTION, query, args, null, null, null)) { if (cursor != null && cursor.moveToNext()) { return getRecipientSettings(cursor); } else { @@ -295,6 +342,191 @@ public class RecipientDatabase extends Database { } } + public @NonNull List getPendingRecipientSyncUpdates() { + return getRecipientSettings(DIRTY + " = ?", new String[] { String.valueOf(DirtyState.UPDATE.getId()) }); + } + + public @NonNull List getPendingRecipientSyncInsertions() { + return getRecipientSettings(DIRTY + " = ?", new String[] { String.valueOf(DirtyState.INSERT.getId()) }); + } + + public @NonNull List getPendingRecipientSyncDeletions() { + return getRecipientSettings(DIRTY + " = ?", new String[] { String.valueOf(DirtyState.DELETE.getId()) }); + } + + public @Nullable RecipientSettings getByStorageSyncKey(@NonNull byte[] key) { + List result = getRecipientSettings(STORAGE_SERVICE_KEY + " = ?", new String[] { Base64.encodeBytes(key) }); + + if (result.size() > 0) { + return result.get(0); + } + + return null; + } + + public void applyStorageSyncKeyUpdates(@NonNull Map keys) { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + + db.beginTransaction(); + try { + String query = ID + " = ?"; + + for (Map.Entry entry : keys.entrySet()) { + ContentValues values = new ContentValues(); + values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(entry.getValue())); + values.put(DIRTY, DirtyState.CLEAN.getId()); + + db.update(TABLE_NAME, values, query, new String[] { entry.getKey().serialize() }); + } + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + } + + public void applyStorageSyncUpdates(@NonNull Collection inserts, + @NonNull Collection updates) + { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + IdentityDatabase identityDatabase = DatabaseFactory.getIdentityDatabase(context); + + db.beginTransaction(); + + try { + for (SignalContactRecord insert : inserts) { + ContentValues values = getValuesForStorageContact(insert); + long id = db.insertOrThrow(TABLE_NAME, null, values); + RecipientId recipientId = RecipientId.from(id); + + if (insert.getIdentityKey().isPresent()) { + try { + IdentityKey identityKey = new IdentityKey(insert.getIdentityKey().get(), 0); + + DatabaseFactory.getIdentityDatabase(context).updateIdentityAfterSync(recipientId, identityKey, StorageSyncHelper.remoteToLocalIdentityStatus(insert.getIdentityState())); + IdentityUtil.markIdentityVerified(context, Recipient.resolved(recipientId), true, true); + } catch (InvalidKeyException e) { + Log.w(TAG, "Failed to process identity key during insert! Skipping.", e); + } + } + } + + for (StorageSyncHelper.ContactUpdate update : updates) { + ContentValues values = getValuesForStorageContact(update.getNewContact()); + int updateCount = db.update(TABLE_NAME, values, STORAGE_SERVICE_KEY + " = ?", new String[]{Base64.encodeBytes(update.getOldContact().getKey())}); + + if (updateCount < 1) { + throw new AssertionError("Had an update, but it didn't match any rows!"); + } + + RecipientId recipientId = getByStorageKeyOrThrow(update.getNewContact().getKey()); + + try { + Optional oldIdentityRecord = identityDatabase.getIdentity(recipientId); + IdentityKey identityKey = update.getNewContact().getIdentityKey().isPresent() ? new IdentityKey(update.getNewContact().getIdentityKey().get(), 0) : null; + + DatabaseFactory.getIdentityDatabase(context).updateIdentityAfterSync(recipientId, identityKey, StorageSyncHelper.remoteToLocalIdentityStatus(update.getNewContact().getIdentityState())); + + Optional newIdentityRecord = identityDatabase.getIdentity(recipientId); + + if ((newIdentityRecord.isPresent() && newIdentityRecord.get().getVerifiedStatus() == IdentityDatabase.VerifiedStatus.VERIFIED) && + (!oldIdentityRecord.isPresent() || oldIdentityRecord.get().getVerifiedStatus() != IdentityDatabase.VerifiedStatus.VERIFIED)) + { + IdentityUtil.markIdentityVerified(context, Recipient.resolved(recipientId), true, true); + } else if ((newIdentityRecord.isPresent() && newIdentityRecord.get().getVerifiedStatus() != IdentityDatabase.VerifiedStatus.VERIFIED) && + (oldIdentityRecord.isPresent() && oldIdentityRecord.get().getVerifiedStatus() == IdentityDatabase.VerifiedStatus.VERIFIED)) + { + IdentityUtil.markIdentityVerified(context, Recipient.resolved(recipientId), false, true); + } + } catch (InvalidKeyException e) { + Log.w(TAG, "Failed to process identity key during update! Skipping.", e); + } + } + + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + } + + private @NonNull RecipientId getByStorageKeyOrThrow(byte[] storageKey) { + SQLiteDatabase db = databaseHelper.getReadableDatabase(); + String query = STORAGE_SERVICE_KEY + " = ?"; + String[] args = new String[]{Base64.encodeBytes(storageKey)}; + + try (Cursor cursor = db.query(TABLE_NAME, ID_PROJECTION, query, args, null, null, null)) { + if (cursor != null && cursor.moveToFirst()) { + long id = cursor.getLong(cursor.getColumnIndexOrThrow(ID)); + return RecipientId.from(id); + } else { + throw new AssertionError("No recipient with that storage key!"); + } + } + } + + private static @NonNull ContentValues getValuesForStorageContact(@NonNull SignalContactRecord contact) { + ContentValues values = new ContentValues(); + + if (contact.getAddress().getUuid().isPresent()) { + values.put(UUID, contact.getAddress().getUuid().get().toString()); + } + + values.put(PHONE, contact.getAddress().getNumber().orNull()); + values.put(SIGNAL_PROFILE_NAME, contact.getProfileName().orNull()); + values.put(PROFILE_KEY, contact.getProfileKey().orNull()); + // TODO [greyson] Username + values.put(PROFILE_SHARING, contact.isProfileSharingEnabled() ? "1" : "0"); + values.put(BLOCKED, contact.isBlocked() ? "1" : "0"); + values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(contact.getKey())); + values.put(DIRTY, DirtyState.CLEAN.getId()); + return values; + } + + private List getRecipientSettings(@Nullable String query, @Nullable String[] args) { + SQLiteDatabase db = databaseHelper.getReadableDatabase(); + String table = TABLE_NAME + " LEFT OUTER JOIN " + IdentityDatabase.TABLE_NAME + " ON " + TABLE_NAME + "." + ID + " = " + IdentityDatabase.TABLE_NAME + "." + IdentityDatabase.RECIPIENT_ID; + List out = new ArrayList<>(); + + try (Cursor cursor = db.query(table, RECIPIENT_FULL_PROJECTION, query, args, null, null, null)) { + while (cursor != null && cursor.moveToNext()) { + out.add(getRecipientSettings(cursor)); + } + } + + return out; + } + + /** + * @return All storage keys, excluding the ones that need to be deleted. + */ + public List getAllStorageSyncKeys() { + return new ArrayList<>(getAllStorageSyncKeysMap().values()); + } + + /** + * @return All storage keys, excluding the ones that need to be deleted. + */ + public Map getAllStorageSyncKeysMap() { + SQLiteDatabase db = databaseHelper.getReadableDatabase(); + String query = STORAGE_SERVICE_KEY + " NOT NULL AND " + DIRTY + " != ?"; + String[] args = new String[]{String.valueOf(DirtyState.DELETE)}; + Map out = new HashMap<>(); + + try (Cursor cursor = db.query(TABLE_NAME, new String[] { ID, STORAGE_SERVICE_KEY }, query, args, null, null, null)) { + while (cursor != null && cursor.moveToNext()) { + RecipientId id = RecipientId.from(cursor.getLong(cursor.getColumnIndexOrThrow(ID))); + String encodedKey = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_SERVICE_KEY)); + + try { + out.put(id, Base64.decode(encodedKey)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + return out; + } + @NonNull RecipientSettings getRecipientSettings(@NonNull Cursor cursor) { long id = cursor.getLong(cursor.getColumnIndexOrThrow(ID)); UUID uuid = UuidUtil.parseOrNull(cursor.getString(cursor.getColumnIndexOrThrow(UUID))); @@ -325,6 +557,9 @@ public class RecipientDatabase extends Database { int unidentifiedAccessMode = cursor.getInt(cursor.getColumnIndexOrThrow(UNIDENTIFIED_ACCESS_MODE)); boolean forceSmsSelection = cursor.getInt(cursor.getColumnIndexOrThrow(FORCE_SMS_SELECTION)) == 1; boolean uuidSupported = cursor.getInt(cursor.getColumnIndexOrThrow(UUID_SUPPORTED)) == 1; + String storageKeyRaw = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_SERVICE_KEY)); + String identityKeyRaw = cursor.getString(cursor.getColumnIndexOrThrow(IDENTITY_KEY)); + int identityStatusRaw = cursor.getInt(cursor.getColumnIndexOrThrow(IDENTITY_STATUS)); MaterialColor color; byte[] profileKey = null; @@ -345,6 +580,22 @@ public class RecipientDatabase extends Database { } } + byte[] storageKey = null; + try { + storageKey = storageKeyRaw != null ? Base64.decode(storageKeyRaw) : null; + } catch (IOException e) { + throw new AssertionError(e); + } + + byte[] identityKey = null; + try { + identityKey = identityKeyRaw != null ? Base64.decode(identityKeyRaw) : null; + } catch (IOException e) { + throw new AssertionError(e); + } + + IdentityDatabase.VerifiedStatus identityStatus = IdentityDatabase.VerifiedStatus.forState(identityStatusRaw); + return new RecipientSettings(RecipientId.from(id), uuid, username, e164, email, groupId, blocked, muteUntil, VibrateState.fromId(messageVibrateState), VibrateState.fromId(callVibrateState), @@ -355,20 +606,18 @@ public class RecipientDatabase extends Database { systemPhoneLabel, systemContactUri, signalProfileName, signalProfileAvatar, profileSharing, notificationChannel, UnidentifiedAccessMode.fromMode(unidentifiedAccessMode), - forceSmsSelection, uuidSupported, InsightsBannerTier.fromId(insightsBannerTier)); + forceSmsSelection, uuidSupported, InsightsBannerTier.fromId(insightsBannerTier), + storageKey, identityKey, identityStatus); } - public BulkOperationsHandle resetAllSystemContactInfo() { + public BulkOperationsHandle beginBulkSystemContactUpdate() { SQLiteDatabase database = databaseHelper.getWritableDatabase(); database.beginTransaction(); ContentValues contentValues = new ContentValues(1); - contentValues.put(SYSTEM_DISPLAY_NAME, (String)null); - contentValues.put(SYSTEM_PHOTO_URI, (String)null); - contentValues.put(SYSTEM_PHONE_LABEL, (String)null); - contentValues.put(SYSTEM_CONTACT_URI, (String)null); + contentValues.put(SYSTEM_INFO_PENDING, 1); - database.update(TABLE_NAME, contentValues, null, null); + database.update(TABLE_NAME, contentValues, SYSTEM_CONTACT_URI + " NOT NULL", null); return new BulkOperationsHandle(database); } @@ -376,29 +625,34 @@ public class RecipientDatabase extends Database { public void setColor(@NonNull RecipientId id, @NonNull MaterialColor color) { ContentValues values = new ContentValues(); values.put(COLOR, color.serialize()); - update(id, values); - Recipient.live(id).refresh(); + if (update(id, values)) { + Recipient.live(id).refresh(); + } } public void setDefaultSubscriptionId(@NonNull RecipientId id, int defaultSubscriptionId) { ContentValues values = new ContentValues(); values.put(DEFAULT_SUBSCRIPTION_ID, defaultSubscriptionId); - update(id, values); - Recipient.live(id).refresh(); + if (update(id, values)) { + Recipient.live(id).refresh(); + } } public void setForceSmsSelection(@NonNull RecipientId id, boolean forceSmsSelection) { ContentValues contentValues = new ContentValues(1); contentValues.put(FORCE_SMS_SELECTION, forceSmsSelection ? 1 : 0); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + Recipient.live(id).refresh(); + } } public void setBlocked(@NonNull RecipientId id, boolean blocked) { ContentValues values = new ContentValues(); values.put(BLOCKED, blocked ? 1 : 0); - update(id, values); - Recipient.live(id).refresh(); + if (update(id, values)) { + markDirty(id, DirtyState.UPDATE); + Recipient.live(id).refresh(); + } } public void setMessageRingtone(@NonNull RecipientId id, @Nullable Uri notification) { @@ -469,7 +723,9 @@ public class RecipientDatabase extends Database { public void setUnidentifiedAccessMode(@NonNull RecipientId id, @NonNull UnidentifiedAccessMode unidentifiedAccessMode) { ContentValues values = new ContentValues(1); values.put(UNIDENTIFIED_ACCESS_MODE, unidentifiedAccessMode.getMode()); - update(id, values); + if (update(id, values)) { + markDirty(id, DirtyState.UPDATE); + } Recipient.live(id).refresh(); } @@ -483,43 +739,53 @@ public class RecipientDatabase extends Database { public void setProfileKey(@NonNull RecipientId id, @Nullable byte[] profileKey) { ContentValues values = new ContentValues(1); values.put(PROFILE_KEY, profileKey == null ? null : Base64.encodeBytes(profileKey)); - update(id, values); - Recipient.live(id).refresh(); + if (update(id, values)) { + markDirty(id, DirtyState.UPDATE); + Recipient.live(id).refresh(); + } } public void setProfileName(@NonNull RecipientId id, @Nullable String profileName) { ContentValues contentValues = new ContentValues(1); contentValues.put(SIGNAL_PROFILE_NAME, profileName); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + markDirty(id, DirtyState.UPDATE); + Recipient.live(id).refresh(); + } } public void setProfileAvatar(@NonNull RecipientId id, @Nullable String profileAvatar) { ContentValues contentValues = new ContentValues(1); contentValues.put(SIGNAL_PROFILE_AVATAR, profileAvatar); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + Recipient.live(id).refresh(); + } } public void setProfileSharing(@NonNull RecipientId id, @SuppressWarnings("SameParameterValue") boolean enabled) { ContentValues contentValues = new ContentValues(1); contentValues.put(PROFILE_SHARING, enabled ? 1 : 0); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + markDirty(id, DirtyState.UPDATE); + Recipient.live(id).refresh(); + } } public void setNotificationChannel(@NonNull RecipientId id, @Nullable String notificationChannel) { ContentValues contentValues = new ContentValues(1); contentValues.put(NOTIFICATION_CHANNEL, notificationChannel); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + Recipient.live(id).refresh(); + } } public void setPhoneNumber(@NonNull RecipientId id, @NonNull String e164) { ContentValues contentValues = new ContentValues(1); contentValues.put(PHONE, e164); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + markDirty(id, DirtyState.UPDATE); + Recipient.live(id).refresh(); + } } public void setUsername(@NonNull RecipientId id, @Nullable String username) { @@ -564,11 +830,14 @@ public class RecipientDatabase extends Database { } public void markRegistered(@NonNull RecipientId id, @NonNull UUID uuid) { - ContentValues contentValues = new ContentValues(2); + ContentValues contentValues = new ContentValues(3); contentValues.put(REGISTERED, RegisteredState.REGISTERED.getId()); contentValues.put(UUID, uuid.toString().toLowerCase()); - update(id, contentValues); - Recipient.live(id).refresh(); + contentValues.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(StorageSyncHelper.generateKey())); + if (update(id, contentValues)) { + markDirty(id, DirtyState.INSERT); + Recipient.live(id).refresh(); + } } /** @@ -579,16 +848,21 @@ public class RecipientDatabase extends Database { public void markRegistered(@NonNull RecipientId id) { ContentValues contentValues = new ContentValues(2); contentValues.put(REGISTERED, RegisteredState.REGISTERED.getId()); - update(id, contentValues); - Recipient.live(id).refresh(); + contentValues.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(StorageSyncHelper.generateKey())); + if (update(id, contentValues)) { + markDirty(id, DirtyState.INSERT); + Recipient.live(id).refresh(); + } } public void markUnregistered(@NonNull RecipientId id) { ContentValues contentValues = new ContentValues(2); contentValues.put(REGISTERED, RegisteredState.NOT_REGISTERED.getId()); contentValues.put(UUID, (String) null); - update(id, contentValues); - Recipient.live(id).refresh(); + if (update(id, contentValues)) { + markDirty(id, DirtyState.DELETE); + Recipient.live(id).refresh(); + } } public void bulkUpdatedRegisteredStatus(@NonNull Map registered, Collection unregistered) { @@ -600,14 +874,18 @@ public class RecipientDatabase extends Database { ContentValues values = new ContentValues(2); values.put(REGISTERED, RegisteredState.REGISTERED.getId()); values.put(UUID, entry.getValue().toLowerCase()); - db.update(TABLE_NAME, values, ID_WHERE, new String[] { entry.getKey().serialize() }); + if (update(entry.getKey(), values)) { + markDirty(entry.getKey(), DirtyState.INSERT); + } } for (RecipientId id : unregistered) { ContentValues values = new ContentValues(1); values.put(REGISTERED, RegisteredState.NOT_REGISTERED.getId()); values.put(UUID, (String) null); - db.update(TABLE_NAME, values, ID_WHERE, new String[] { id.serialize() }); + if (update(id, values)) { + markDirty(id, DirtyState.DELETE); + } } db.setTransactionSuccessful(); @@ -632,7 +910,7 @@ public class RecipientDatabase extends Database { ContentValues contentValues = new ContentValues(1); contentValues.put(REGISTERED, RegisteredState.REGISTERED.getId()); - if (update(activeId, contentValues) > 0) { + if (update(activeId, contentValues)) { Recipient.live(activeId).refresh(); } } @@ -641,7 +919,7 @@ public class RecipientDatabase extends Database { ContentValues contentValues = new ContentValues(1); contentValues.put(REGISTERED, RegisteredState.NOT_REGISTERED.getId()); - if (update(inactiveId, contentValues) > 0) { + if (update(inactiveId, contentValues)) { Recipient.live(inactiveId).refresh(); } } @@ -834,10 +1112,81 @@ public class RecipientDatabase extends Database { ApplicationDependencies.getRecipientCache().clear(); } + public void updateStorageKeys(@NonNull Map keys) { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + db.beginTransaction(); - private int update(@NonNull RecipientId id, ContentValues contentValues) { - SQLiteDatabase database = databaseHelper.getWritableDatabase(); - return database.update(TABLE_NAME, contentValues, ID + " = ?", new String[] { id.serialize() }); + try { + for (Map.Entry entry : keys.entrySet()) { + ContentValues values = new ContentValues(); + values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(entry.getValue())); + db.update(TABLE_NAME, values, ID_WHERE, new String[] { entry.getKey().serialize() }); + } + + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + } + + public void clearDirtyState(@NonNull List recipients) { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + db.beginTransaction(); + + try { + ContentValues values = new ContentValues(); + values.put(DIRTY, DirtyState.CLEAN.getId()); + + for (RecipientId id : recipients) { + db.update(TABLE_NAME, values, ID_WHERE, new String[]{ id.serialize() }); + } + + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + } + + void markDirty(@NonNull RecipientId recipientId, @NonNull DirtyState dirtyState) { + if (!FeatureFlags.STORAGE_SERVICE) return; + + ContentValues contentValues = new ContentValues(1); + contentValues.put(DIRTY, dirtyState.getId()); + + String query = ID + " = ? AND (" + UUID + " NOT NULL OR " + PHONE + " NOT NULL) AND " + DIRTY + " < ?"; + String[] args = new String[] { recipientId.serialize(), String.valueOf(dirtyState.id) }; + + databaseHelper.getWritableDatabase().update(TABLE_NAME, contentValues, query, args); + } + + /** + * Will update the database with the content values you specified. It will make an intelligent + * query such that this will only return true if a row was *actually* updated. + */ + private boolean update(@NonNull RecipientId id, ContentValues contentValues) { + SQLiteDatabase database = databaseHelper.getWritableDatabase(); + StringBuilder qualifier = new StringBuilder(); + Set> valueSet = contentValues.valueSet(); + String[] args = new String[valueSet.size() + 1]; + + args[0] = id.serialize(); + + int i = 0; + + for (Map.Entry entry : valueSet) { + qualifier.append(entry.getKey()).append(" != ?"); + + if (i != valueSet.size() - 1) { + qualifier.append(" OR "); + } + + args[i + 1] = String.valueOf(entry.getValue()); + + i++; + } + + + return database.update(TABLE_NAME, contentValues, ID + " = ? AND (" + qualifier + ")", args) > 0; } private @NonNull Optional getByColumn(@NonNull String column, String value) { @@ -900,23 +1249,62 @@ public class RecipientDatabase extends Database { int systemPhoneType, @Nullable String systemContactUri) { - ContentValues contentValues = new ContentValues(1); - contentValues.put(SYSTEM_DISPLAY_NAME, displayName); - contentValues.put(SYSTEM_PHOTO_URI, photoUri); - contentValues.put(SYSTEM_PHONE_LABEL, systemPhoneLabel); - contentValues.put(SYSTEM_PHONE_TYPE, systemPhoneType); - contentValues.put(SYSTEM_CONTACT_URI, systemContactUri); + ContentValues dirtyQualifyingValues = new ContentValues(); + dirtyQualifyingValues.put(SYSTEM_DISPLAY_NAME, displayName); - update(id, contentValues); - pendingContactInfoMap.put(id, new PendingContactInfo(displayName, photoUri, systemPhoneLabel, systemContactUri)); + if (update(id, dirtyQualifyingValues)) { + markDirty(id, DirtyState.UPDATE); + } + + ContentValues refreshQualifyingValues = new ContentValues(); + refreshQualifyingValues.put(SYSTEM_PHOTO_URI, photoUri); + refreshQualifyingValues.put(SYSTEM_PHONE_LABEL, systemPhoneLabel); + refreshQualifyingValues.put(SYSTEM_PHONE_TYPE, systemPhoneType); + refreshQualifyingValues.put(SYSTEM_CONTACT_URI, systemContactUri); + + if (update(id, refreshQualifyingValues)) { + pendingContactInfoMap.put(id, new PendingContactInfo(displayName, photoUri, systemPhoneLabel, systemContactUri)); + } + + ContentValues otherValues = new ContentValues(); + otherValues.put(SYSTEM_INFO_PENDING, 0); + update(id, otherValues); } public void finish() { + markAllRelevantEntriesDirty(); + clearSystemDataForPendingInfo(); + database.setTransactionSuccessful(); database.endTransaction(); Stream.of(pendingContactInfoMap.entrySet()).forEach(entry -> Recipient.live(entry.getKey()).refresh()); } + + private void markAllRelevantEntriesDirty() { + String query = SYSTEM_INFO_PENDING + " = ? AND " + STORAGE_SERVICE_KEY + " NOT NULL AND " + DIRTY + " < ?"; + String[] args = new String[] { "1", String.valueOf(DirtyState.UPDATE.getId()) }; + + ContentValues values = new ContentValues(1); + values.put(DIRTY, DirtyState.UPDATE.getId()); + + database.update(TABLE_NAME, values, query, args); + } + + private void clearSystemDataForPendingInfo() { + String query = SYSTEM_INFO_PENDING + " = ?"; + String[] args = new String[] { "1" }; + + ContentValues values = new ContentValues(5); + + values.put(SYSTEM_INFO_PENDING, 0); + values.put(SYSTEM_DISPLAY_NAME, (String) null); + values.put(SYSTEM_PHOTO_URI, (String) null); + values.put(SYSTEM_PHONE_LABEL, (String) null); + values.put(SYSTEM_CONTACT_URI, (String) null); + + database.update(TABLE_NAME, values, query, args); + } } public interface ColorUpdater { @@ -924,35 +1312,38 @@ public class RecipientDatabase extends Database { } public static class RecipientSettings { - private final RecipientId id; - private final UUID uuid; - private final String username; - private final String e164; - private final String email; - private final String groupId; - private final boolean blocked; - private final long muteUntil; - private final VibrateState messageVibrateState; - private final VibrateState callVibrateState; - private final Uri messageRingtone; - private final Uri callRingtone; - private final MaterialColor color; - private final int defaultSubscriptionId; - private final int expireMessages; - private final RegisteredState registered; - private final byte[] profileKey; - private final String systemDisplayName; - private final String systemContactPhoto; - private final String systemPhoneLabel; - private final String systemContactUri; - private final String signalProfileName; - private final String signalProfileAvatar; - private final boolean profileSharing; - private final String notificationChannel; - private final UnidentifiedAccessMode unidentifiedAccessMode; - private final boolean forceSmsSelection; - private final boolean uuidSupported; - private final InsightsBannerTier insightsBannerTier; + private final RecipientId id; + private final UUID uuid; + private final String username; + private final String e164; + private final String email; + private final String groupId; + private final boolean blocked; + private final long muteUntil; + private final VibrateState messageVibrateState; + private final VibrateState callVibrateState; + private final Uri messageRingtone; + private final Uri callRingtone; + private final MaterialColor color; + private final int defaultSubscriptionId; + private final int expireMessages; + private final RegisteredState registered; + private final byte[] profileKey; + private final String systemDisplayName; + private final String systemContactPhoto; + private final String systemPhoneLabel; + private final String systemContactUri; + private final String signalProfileName; + private final String signalProfileAvatar; + private final boolean profileSharing; + private final String notificationChannel; + private final UnidentifiedAccessMode unidentifiedAccessMode; + private final boolean forceSmsSelection; + private final boolean uuidSupported; + private final InsightsBannerTier insightsBannerTier; + private final byte[] storageKey; + private final byte[] identityKey; + private final IdentityDatabase.VerifiedStatus identityStatus; RecipientSettings(@NonNull RecipientId id, @Nullable UUID uuid, @@ -981,7 +1372,10 @@ public class RecipientDatabase extends Database { @NonNull UnidentifiedAccessMode unidentifiedAccessMode, boolean forceSmsSelection, boolean uuidSupported, - @NonNull InsightsBannerTier insightsBannerTier) + @NonNull InsightsBannerTier insightsBannerTier, + @Nullable byte[] storageKey, + @Nullable byte[] identityKey, + @NonNull IdentityDatabase.VerifiedStatus identityStatus) { this.id = id; this.uuid = uuid; @@ -1011,7 +1405,10 @@ public class RecipientDatabase extends Database { this.unidentifiedAccessMode = unidentifiedAccessMode; this.forceSmsSelection = forceSmsSelection; this.uuidSupported = uuidSupported; - this.insightsBannerTier = insightsBannerTier; + this.insightsBannerTier = insightsBannerTier; + this.storageKey = storageKey; + this.identityKey = identityKey; + this.identityStatus = identityStatus; } public RecipientId getId() { @@ -1129,15 +1526,25 @@ public class RecipientDatabase extends Database { public boolean isUuidSupported() { return uuidSupported; } + + public @Nullable byte[] getStorageKey() { + return storageKey; + } + + public @Nullable byte[] getIdentityKey() { + return identityKey; + } + + public @NonNull IdentityDatabase.VerifiedStatus getIdentityStatus() { + return identityStatus; + } } public static class RecipientReader implements Closeable { - private final Context context; private final Cursor cursor; - RecipientReader(Context context, Cursor cursor) { - this.context = context; + RecipientReader(Cursor cursor) { this.cursor = cursor; } diff --git a/src/org/thoughtcrime/securesms/database/StorageKeyDatabase.java b/src/org/thoughtcrime/securesms/database/StorageKeyDatabase.java new file mode 100644 index 0000000000..f80b3a86b3 --- /dev/null +++ b/src/org/thoughtcrime/securesms/database/StorageKeyDatabase.java @@ -0,0 +1,115 @@ +package org.thoughtcrime.securesms.database; + +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import net.sqlcipher.database.SQLiteDatabase; + +import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper; +import org.thoughtcrime.securesms.util.Base64; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * A list of storage keys whose types we do not currently have syncing logic for. We need to + * remember that these keys exist so that we don't blast any data away. + */ +public class StorageKeyDatabase extends Database { + + private static final String TABLE_NAME = "storage_key"; + private static final String ID = "_id"; + private static final String TYPE = "type"; + private static final String KEY = "key"; + + public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ID + " INTEGER PRIMARY KEY AUTOINCREMENT, " + + TYPE + " INTEGER, " + + KEY + " TEXT UNIQUE)"; + + public static final String[] CREATE_INDEXES = new String[] { + "CREATE INDEX IF NOT EXISTS storage_key_type_index ON " + TABLE_NAME + " (" + TYPE + ");" + }; + + public StorageKeyDatabase(Context context, SQLCipherOpenHelper databaseHelper) { + super(context, databaseHelper); + } + + public List getAllKeys() { + List keys = new ArrayList<>(); + + try (Cursor cursor = databaseHelper.getReadableDatabase().query(TABLE_NAME, null, null, null, null, null, null)) { + while (cursor != null && cursor.moveToNext()) { + String keyEncoded = cursor.getString(cursor.getColumnIndexOrThrow(KEY)); + try { + keys.add(Base64.decode(keyEncoded)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + return keys; + } + + public @Nullable SignalStorageRecord getByKey(@NonNull byte[] key) { + String query = KEY + " = ?"; + String[] args = new String[] { Base64.encodeBytes(key) }; + + try (Cursor cursor = databaseHelper.getReadableDatabase().query(TABLE_NAME, null, query, args, null, null, null)) { + if (cursor != null && cursor.moveToFirst()) { + int type = cursor.getInt(cursor.getColumnIndexOrThrow(TYPE)); + return SignalStorageRecord.forUnknown(key, type); + } else { + return null; + } + } + } + + public void applyStorageSyncUpdates(@NonNull Collection inserts, + @NonNull Collection deletes) + { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + + db.beginTransaction(); + try { + for (SignalStorageRecord insert : inserts) { + ContentValues values = new ContentValues(); + values.put(TYPE, insert.getType()); + values.put(KEY, Base64.encodeBytes(insert.getKey())); + + db.insert(TABLE_NAME, null, values); + } + + String deleteQuery = KEY + " = ?"; + + for (SignalStorageRecord delete : deletes) { + String[] args = new String[] { Base64.encodeBytes(delete.getKey()) }; + db.delete(TABLE_NAME, deleteQuery, args); + } + + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + + } + + public void deleteByType(int type) { + String query = TYPE + " = ?"; + String[] args = new String[]{String.valueOf(type)}; + + databaseHelper.getWritableDatabase().delete(TABLE_NAME, query, args); + } + + public void deleteAll() { + databaseHelper.getWritableDatabase().delete(TABLE_NAME, null, null); + } +} diff --git a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 94b3a75d13..1daad093cd 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -20,6 +20,7 @@ import net.sqlcipher.database.SQLiteDatabase; import net.sqlcipher.database.SQLiteDatabaseHook; import net.sqlcipher.database.SQLiteOpenHelper; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper; import org.thoughtcrime.securesms.crypto.DatabaseSecret; import org.thoughtcrime.securesms.crypto.MasterSecret; import org.thoughtcrime.securesms.database.AttachmentDatabase; @@ -37,6 +38,7 @@ import org.thoughtcrime.securesms.database.SessionDatabase; import org.thoughtcrime.securesms.database.SignedPreKeyDatabase; import org.thoughtcrime.securesms.database.SmsDatabase; import org.thoughtcrime.securesms.database.StickerDatabase; +import org.thoughtcrime.securesms.database.StorageKeyDatabase; import org.thoughtcrime.securesms.database.ThreadDatabase; import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; import org.thoughtcrime.securesms.jobs.RefreshPreKeysJob; @@ -44,6 +46,7 @@ import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.notifications.NotificationChannels; import org.thoughtcrime.securesms.phonenumbers.PhoneNumberFormatter; import org.thoughtcrime.securesms.service.KeyCachingService; +import org.thoughtcrime.securesms.util.Base64; import org.thoughtcrime.securesms.util.GroupUtil; import org.thoughtcrime.securesms.util.ServiceUtil; import org.thoughtcrime.securesms.util.SqlUtil; @@ -93,9 +96,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int UUIDS = 35; private static final int USERNAMES = 36; private static final int REACTIONS = 37; + private static final int STORAGE_SERVICE = 38; - private static final int DATABASE_VERSION = 37; - + private static final int DATABASE_VERSION = 38; private static final String DATABASE_NAME = "signal.db"; private final Context context; @@ -136,9 +139,11 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL(SignedPreKeyDatabase.CREATE_TABLE); db.execSQL(SessionDatabase.CREATE_TABLE); db.execSQL(StickerDatabase.CREATE_TABLE); + db.execSQL(StorageKeyDatabase.CREATE_TABLE); executeStatements(db, SearchDatabase.CREATE_TABLE); executeStatements(db, JobDatabase.CREATE_TABLE); + executeStatements(db, RecipientDatabase.CREATE_INDEXS); executeStatements(db, SmsDatabase.CREATE_INDEXS); executeStatements(db, MmsDatabase.CREATE_INDEXS); executeStatements(db, AttachmentDatabase.CREATE_INDEXS); @@ -147,6 +152,7 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { executeStatements(db, GroupDatabase.CREATE_INDEXS); executeStatements(db, GroupReceiptDatabase.CREATE_INDEXES); executeStatements(db, StickerDatabase.CREATE_INDEXES); + executeStatements(db, StorageKeyDatabase.CREATE_INDEXES); if (context.getDatabasePath(ClassicOpenHelper.NAME).exists()) { ClassicOpenHelper legacyHelper = new ClassicOpenHelper(context); @@ -171,6 +177,7 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { @Override public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { Log.i(TAG, "Upgrading database: " + oldVersion + ", " + newVersion); + long startTime = System.currentTimeMillis(); db.beginTransaction(); @@ -638,6 +645,34 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL("ALTER TABLE mms ADD COLUMN reactions_last_seen INTEGER DEFAULT -1"); } + if (oldVersion < STORAGE_SERVICE) { + db.execSQL("CREATE TABLE storage_key (_id INTEGER PRIMARY KEY AUTOINCREMENT, " + + "type INTEGER, " + + "key TEXT UNIQUE)"); + db.execSQL("CREATE INDEX IF NOT EXISTS storage_key_type_index ON storage_key (type)"); + + db.execSQL("ALTER TABLE recipient ADD COLUMN system_info_pending INTEGER DEFAULT 0"); + db.execSQL("ALTER TABLE recipient ADD COLUMN storage_service_key TEXT DEFAULT NULL"); + db.execSQL("ALTER TABLE recipient ADD COLUMN dirty INTEGER DEFAULT 0"); + + db.execSQL("CREATE UNIQUE INDEX recipient_storage_service_key ON recipient (storage_service_key)"); + db.execSQL("CREATE INDEX recipient_dirty_index ON recipient (dirty)"); + + // TODO [greyson] Do this in a future DB migration +// db.execSQL("UPDATE recipient SET dirty = 2 WHERE registered = 1"); +// +// try (Cursor cursor = db.rawQuery("SELECT _id FROM recipient WHERE registered = 1", null)) { +// while (cursor != null && cursor.moveToNext()) { +// String id = cursor.getString(cursor.getColumnIndexOrThrow("_id")); +// ContentValues values = new ContentValues(1); +// +// values.put("storage_service_key", Base64.encodeBytes(StorageSyncHelper.generateKey())); +// +// db.update("recipient", values, "_id = ?", new String[] { id }); +// } +// } + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); @@ -646,6 +681,8 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { if (oldVersion < MIGRATE_PREKEYS_VERSION) { PreKeyMigrationHelper.cleanUpPreKeys(context); } + + Log.i(TAG, "Upgrade complete. Took " + (System.currentTimeMillis() - startTime) + " ms."); } public SQLiteDatabase getReadableDatabase() { diff --git a/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index f28203300d..60b3a3ba55 100644 --- a/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -53,11 +53,13 @@ public final class JobManagerFactories { put(MultiDeviceConfigurationUpdateJob.KEY, new MultiDeviceConfigurationUpdateJob.Factory()); put(MultiDeviceContactUpdateJob.KEY, new MultiDeviceContactUpdateJob.Factory()); put(MultiDeviceGroupUpdateJob.KEY, new MultiDeviceGroupUpdateJob.Factory()); + put(MultiDeviceKeysUpdateJob.KEY, new MultiDeviceKeysUpdateJob.Factory()); put(MultiDeviceProfileContentUpdateJob.KEY, new MultiDeviceProfileContentUpdateJob.Factory()); put(MultiDeviceProfileKeyUpdateJob.KEY, new MultiDeviceProfileKeyUpdateJob.Factory()); put(MultiDeviceReadUpdateJob.KEY, new MultiDeviceReadUpdateJob.Factory()); put(MultiDeviceStickerPackOperationJob.KEY, new MultiDeviceStickerPackOperationJob.Factory()); put(MultiDeviceStickerPackSyncJob.KEY, new MultiDeviceStickerPackSyncJob.Factory()); + put(MultiDeviceStorageSyncRequestJob.KEY, new MultiDeviceStorageSyncRequestJob.Factory()); put(MultiDeviceVerifiedUpdateJob.KEY, new MultiDeviceVerifiedUpdateJob.Factory()); put(MultiDeviceViewOnceOpenJob.KEY, new MultiDeviceViewOnceOpenJob.Factory()); put(PushDecryptJob.KEY, new PushDecryptJob.Factory()); @@ -84,6 +86,8 @@ public final class JobManagerFactories { put(SmsSentJob.KEY, new SmsSentJob.Factory()); put(StickerDownloadJob.KEY, new StickerDownloadJob.Factory()); put(StickerPackDownloadJob.KEY, new StickerPackDownloadJob.Factory()); + put(StorageForcePushJob.KEY, new StorageForcePushJob.Factory()); + put(StorageSyncJob.KEY, new StorageSyncJob.Factory()); put(TrimThreadJob.KEY, new TrimThreadJob.Factory()); put(TypingSendJob.KEY, new TypingSendJob.Factory()); put(UpdateApkJob.KEY, new UpdateApkJob.Factory()); diff --git a/src/org/thoughtcrime/securesms/jobs/MultiDeviceKeysUpdateJob.java b/src/org/thoughtcrime/securesms/jobs/MultiDeviceKeysUpdateJob.java new file mode 100644 index 0000000000..d6c8306d3a --- /dev/null +++ b/src/org/thoughtcrime/securesms/jobs/MultiDeviceKeysUpdateJob.java @@ -0,0 +1,91 @@ +package org.thoughtcrime.securesms.jobs; + + +import androidx.annotation.NonNull; + +import org.thoughtcrime.securesms.crypto.UnidentifiedAccessUtil; +import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; +import org.thoughtcrime.securesms.jobmanager.Data; +import org.thoughtcrime.securesms.jobmanager.Job; +import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.util.TextSecurePreferences; +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.SignalServiceMessageSender; +import org.whispersystems.signalservice.api.crypto.UntrustedIdentityException; +import org.whispersystems.signalservice.api.messages.multidevice.ConfigurationMessage; +import org.whispersystems.signalservice.api.messages.multidevice.KeysMessage; +import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSyncMessage; +import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException; +import org.whispersystems.signalservice.api.storage.SignalStorageUtil; +import org.whispersystems.signalservice.internal.configuration.SignalStorageUrl; + +import java.io.IOException; + +public class MultiDeviceKeysUpdateJob extends BaseJob { + + public static final String KEY = "MultiDeviceKeysUpdateJob"; + + private static final String TAG = MultiDeviceKeysUpdateJob.class.getSimpleName(); + + public MultiDeviceKeysUpdateJob() { + this(new Parameters.Builder() + .setQueue("MultiDeviceKeysUpdateJob") + .setMaxInstances(2) + .addConstraint(NetworkConstraint.KEY) + .setMaxAttempts(10) + .build()); + + } + + private MultiDeviceKeysUpdateJob(@NonNull Parameters parameters) { + super(parameters); + } + + @Override + public @NonNull Data serialize() { + return Data.EMPTY; + } + + @Override + public @NonNull String getFactoryKey() { + return KEY; + } + + @Override + public void onRun() throws IOException, UntrustedIdentityException { + if (!TextSecurePreferences.isMultiDevice(context)) { + Log.i(TAG, "Not multi device, aborting..."); + return; + } + + SignalServiceMessageSender messageSender = ApplicationDependencies.getSignalServiceMessageSender(); + + byte[] masterKey = TextSecurePreferences.getMasterKey(context); + byte[] storageServiceKey = masterKey != null ? SignalStorageUtil.computeStorageServiceKey(masterKey) + : null; + + if (storageServiceKey == null) { + Log.w(TAG, "Syncing a null storage service key."); + } + + messageSender.sendMessage(SignalServiceSyncMessage.forKeys(new KeysMessage(Optional.fromNullable(storageServiceKey))), + UnidentifiedAccessUtil.getAccessForSync(context)); + } + + @Override + public boolean onShouldRetry(@NonNull Exception e) { + return e instanceof PushNetworkException; + } + + @Override + public void onCanceled() { + } + + public static final class Factory implements Job.Factory { + @Override + public @NonNull MultiDeviceKeysUpdateJob create(@NonNull Parameters parameters, @NonNull Data data) { + return new MultiDeviceKeysUpdateJob(parameters); + } + } +} diff --git a/src/org/thoughtcrime/securesms/jobs/MultiDeviceProfileContentUpdateJob.java b/src/org/thoughtcrime/securesms/jobs/MultiDeviceProfileContentUpdateJob.java index 188cdd64c6..e1d197e9be 100644 --- a/src/org/thoughtcrime/securesms/jobs/MultiDeviceProfileContentUpdateJob.java +++ b/src/org/thoughtcrime/securesms/jobs/MultiDeviceProfileContentUpdateJob.java @@ -22,6 +22,7 @@ public class MultiDeviceProfileContentUpdateJob extends BaseJob { public MultiDeviceProfileContentUpdateJob() { this(new Parameters.Builder() .setQueue("MultiDeviceProfileUpdateJob") + .setMaxInstances(2) .addConstraint(NetworkConstraint.KEY) .setMaxAttempts(10) .build()); @@ -59,7 +60,6 @@ public class MultiDeviceProfileContentUpdateJob extends BaseJob { return e instanceof PushNetworkException; } - @Override public void onCanceled() { Log.w(TAG, "Did not succeed!"); diff --git a/src/org/thoughtcrime/securesms/jobs/MultiDeviceStorageSyncRequestJob.java b/src/org/thoughtcrime/securesms/jobs/MultiDeviceStorageSyncRequestJob.java new file mode 100644 index 0000000000..07550ba148 --- /dev/null +++ b/src/org/thoughtcrime/securesms/jobs/MultiDeviceStorageSyncRequestJob.java @@ -0,0 +1,74 @@ +package org.thoughtcrime.securesms.jobs; + +import androidx.annotation.NonNull; + +import org.thoughtcrime.securesms.crypto.UnidentifiedAccessUtil; +import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; +import org.thoughtcrime.securesms.jobmanager.Data; +import org.thoughtcrime.securesms.jobmanager.Job; +import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.util.TextSecurePreferences; +import org.whispersystems.signalservice.api.SignalServiceMessageSender; +import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSyncMessage; +import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException; + +public class MultiDeviceStorageSyncRequestJob extends BaseJob { + + public static final String KEY = "MultiDeviceStorageSyncRequestJob"; + + private static final String TAG = Log.tag(MultiDeviceStorageSyncRequestJob.class); + + public MultiDeviceStorageSyncRequestJob() { + this(new Parameters.Builder() + .setQueue("MultiDeviceStorageSyncRequestJob") + .setMaxInstances(2) + .addConstraint(NetworkConstraint.KEY) + .setMaxAttempts(10) + .build()); + } + + private MultiDeviceStorageSyncRequestJob(@NonNull Parameters parameters) { + super(parameters); + } + + @Override + public @NonNull Data serialize() { + return Data.EMPTY; + } + + @Override + public @NonNull String getFactoryKey() { + return KEY; + } + + @Override + protected void onRun() throws Exception { + if (!TextSecurePreferences.isMultiDevice(context)) { + Log.i(TAG, "Not multi device, aborting..."); + return; + } + + SignalServiceMessageSender messageSender = ApplicationDependencies.getSignalServiceMessageSender(); + + messageSender.sendMessage(SignalServiceSyncMessage.forFetchLatest(SignalServiceSyncMessage.FetchType.STORAGE_MANIFEST), + UnidentifiedAccessUtil.getAccessForSync(context)); + } + + @Override + protected boolean onShouldRetry(@NonNull Exception e) { + return e instanceof PushNetworkException; + } + + @Override + public void onCanceled() { + Log.w(TAG, "Did not succeed!"); + } + + public static final class Factory implements Job.Factory { + @Override + public @NonNull MultiDeviceStorageSyncRequestJob create(@NonNull Parameters parameters, @NonNull Data data) { + return new MultiDeviceStorageSyncRequestJob(parameters); + } + } +} diff --git a/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java b/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java index 34153eef06..ba8c65a8f6 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java @@ -679,6 +679,10 @@ public class PushDecryptJob extends BaseJob { TextSecurePreferences.isLinkPreviewsEnabled(context))); ApplicationDependencies.getJobManager().add(new MultiDeviceStickerPackSyncJob()); } + + if (message.isKeysRequest()) { +// ApplicationDependencies.getJobManager().add(new ); + } } private void handleSynchronizeReadMessage(@NonNull List readMessages, long envelopeTimestamp) diff --git a/src/org/thoughtcrime/securesms/jobs/StorageForcePushJob.java b/src/org/thoughtcrime/securesms/jobs/StorageForcePushJob.java new file mode 100644 index 0000000000..147b9798ab --- /dev/null +++ b/src/org/thoughtcrime/securesms/jobs/StorageForcePushJob.java @@ -0,0 +1,142 @@ +package org.thoughtcrime.securesms.jobs; + +import androidx.annotation.NonNull; + +import com.annimon.stream.Stream; + +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper; +import org.thoughtcrime.securesms.database.DatabaseFactory; +import org.thoughtcrime.securesms.database.RecipientDatabase; +import org.thoughtcrime.securesms.database.StorageKeyDatabase; +import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; +import org.thoughtcrime.securesms.jobmanager.Data; +import org.thoughtcrime.securesms.jobmanager.Job; +import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.recipients.Recipient; +import org.thoughtcrime.securesms.recipients.RecipientId; +import org.thoughtcrime.securesms.transport.RetryLaterException; +import org.thoughtcrime.securesms.util.FeatureFlags; +import org.thoughtcrime.securesms.util.TextSecurePreferences; +import org.thoughtcrime.securesms.util.Util; +import org.whispersystems.libsignal.InvalidKeyException; +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.SignalServiceAccountManager; +import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException; +import org.whispersystems.signalservice.api.storage.SignalContactRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageManifest; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageUtil; +import org.whispersystems.signalservice.internal.storage.protos.StorageRecord; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Forces remote storage to match our local state. This should only be done after a key change or + * when we detect that the remote data is badly-encrypted. + */ +public class StorageForcePushJob extends BaseJob { + + public static final String KEY = "StorageForcePushJob"; + + private static final String TAG = Log.tag(StorageForcePushJob.class); + + public StorageForcePushJob() { + this(new Parameters.Builder().addConstraint(NetworkConstraint.KEY) + .setQueue(StorageSyncJob.QUEUE_KEY) + .setMaxInstances(1) + .setLifespan(TimeUnit.DAYS.toMillis(1)) + .build()); + } + + private StorageForcePushJob(@NonNull Parameters parameters) { + super(parameters); + } + + @Override + public @NonNull Data serialize() { + return Data.EMPTY; + } + + @Override + public @NonNull String getFactoryKey() { + return KEY; + } + + @Override + protected void onRun() throws IOException, RetryLaterException { + if (!FeatureFlags.STORAGE_SERVICE) throw new AssertionError(); + + byte[] kbsMasterKey = TextSecurePreferences.getMasterKey(context); + + if (kbsMasterKey == null) { + Log.w(TAG, "No KBS master key is set! Must abort."); + return; + } + + byte[] storageServiceKey = SignalStorageUtil.computeStorageServiceKey(kbsMasterKey); + SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager(); + RecipientDatabase recipientDatabase = DatabaseFactory.getRecipientDatabase(context); + StorageKeyDatabase storageKeyDatabase = DatabaseFactory.getStorageKeyDatabase(context); + + long currentVersion = accountManager.getStorageManifestVersion(); + Map oldContactKeys = recipientDatabase.getAllStorageSyncKeysMap(); + List oldUnknownKeys = storageKeyDatabase.getAllKeys(); + + long newVersion = currentVersion + 1; + Map newContactKeys = generateNewKeys(oldContactKeys); + List keysToDelete = Util.concatenatedList(new ArrayList<>(oldContactKeys.values()), oldUnknownKeys); + List inserts = Stream.of(oldContactKeys.keySet()) + .map(recipientDatabase::getRecipientSettings) + .withoutNulls() + .map(StorageSyncHelper::localToRemoteContact) + .map(r -> SignalStorageRecord.forContact(r.getKey(), r)) + .toList(); + + SignalStorageManifest manifest = new SignalStorageManifest(newVersion, new ArrayList<>(newContactKeys.values())); + + try { + accountManager.writeStorageRecords(storageServiceKey, manifest, inserts, keysToDelete); + } catch (InvalidKeyException e) { + Log.w(TAG, "Hit an invalid key exception, which likely indicates a conflict."); + throw new RetryLaterException(); + } + + TextSecurePreferences.setStorageManifestVersion(context, newVersion); + recipientDatabase.applyStorageSyncKeyUpdates(newContactKeys); + storageKeyDatabase.deleteAll(); + } + + @Override + protected boolean onShouldRetry(@NonNull Exception e) { + return e instanceof PushNetworkException || e instanceof RetryLaterException; + } + + @Override + public void onCanceled() { + } + + private static @NonNull Map generateNewKeys(@NonNull Map oldKeys) { + Map out = new HashMap<>(); + + for (Map.Entry entry : oldKeys.entrySet()) { + out.put(entry.getKey(), StorageSyncHelper.generateKey()); + } + + return out; + } + + public static final class Factory implements Job.Factory { + + @Override + public @NonNull + StorageForcePushJob create(@NonNull Parameters parameters, @NonNull Data data) { + return new StorageForcePushJob(parameters); + } + } +} diff --git a/src/org/thoughtcrime/securesms/jobs/StorageSyncJob.java b/src/org/thoughtcrime/securesms/jobs/StorageSyncJob.java new file mode 100644 index 0000000000..7a8f977d08 --- /dev/null +++ b/src/org/thoughtcrime/securesms/jobs/StorageSyncJob.java @@ -0,0 +1,228 @@ +package org.thoughtcrime.securesms.jobs; + +import android.content.Context; + +import androidx.annotation.NonNull; + +import com.annimon.stream.Stream; + +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.KeyDifferenceResult; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.LocalWriteResult; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.MergeResult; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.WriteOperationResult; +import org.thoughtcrime.securesms.database.DatabaseFactory; +import org.thoughtcrime.securesms.database.RecipientDatabase; +import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings; +import org.thoughtcrime.securesms.database.StorageKeyDatabase; +import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; +import org.thoughtcrime.securesms.jobmanager.Data; +import org.thoughtcrime.securesms.jobmanager.Job; +import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.recipients.RecipientId; +import org.thoughtcrime.securesms.transport.RetryLaterException; +import org.thoughtcrime.securesms.util.FeatureFlags; +import org.thoughtcrime.securesms.util.TextSecurePreferences; +import org.thoughtcrime.securesms.util.Util; +import org.whispersystems.libsignal.InvalidKeyException; +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.SignalServiceAccountManager; +import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException; +import org.whispersystems.signalservice.api.storage.SignalContactRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageManifest; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageUtil; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Does a full sync of our local storage state with the remote storage state. Will write any pending + * local changes and resolve any conflicts with remote storage. + * + * This should be performed whenever a change is made locally, or whenever we want to retrieve + * changes that have been made remotely. + */ +public class StorageSyncJob extends BaseJob { + + public static final String KEY = "StorageSyncJob"; + public static final String QUEUE_KEY = "StorageSyncingJobs"; + + private static final String TAG = Log.tag(StorageSyncJob.class); + + public StorageSyncJob() { + this(new Job.Parameters.Builder().addConstraint(NetworkConstraint.KEY) + .setQueue(QUEUE_KEY) + .setMaxInstances(1) + .setLifespan(TimeUnit.DAYS.toMillis(1)) + .build()); + } + + private StorageSyncJob(@NonNull Parameters parameters) { + super(parameters); + } + + @Override + public @NonNull Data serialize() { + return Data.EMPTY; + } + + @Override + public @NonNull String getFactoryKey() { + return KEY; + } + + @Override + protected void onRun() throws IOException, RetryLaterException { + if (!FeatureFlags.STORAGE_SERVICE) throw new AssertionError(); + + try { + boolean needsMultiDeviceSync = performSync(); + + if (TextSecurePreferences.isMultiDevice(context) && needsMultiDeviceSync) { + ApplicationDependencies.getJobManager().add(new MultiDeviceStorageSyncRequestJob()); + } + } catch (InvalidKeyException e) { + Log.w(TAG, "Failed to decrypt remote storage! Force-pushing and syncing the storage key to linked devices.", e); + + ApplicationDependencies.getJobManager().startChain(new MultiDeviceKeysUpdateJob()) + .then(new StorageForcePushJob()) + .then(new MultiDeviceStorageSyncRequestJob()) + .enqueue(); + } + } + + @Override + protected boolean onShouldRetry(@NonNull Exception e) { + return e instanceof PushNetworkException || e instanceof RetryLaterException; + } + + @Override + public void onCanceled() { + } + + private boolean performSync() throws IOException, RetryLaterException, InvalidKeyException { + SignalServiceAccountManager accountManager = ApplicationDependencies.getSignalServiceAccountManager(); + RecipientDatabase recipientDatabase = DatabaseFactory.getRecipientDatabase(context); + StorageKeyDatabase storageKeyDatabase = DatabaseFactory.getStorageKeyDatabase(context); + byte[] kbsMasterKey = TextSecurePreferences.getMasterKey(context); + + if (kbsMasterKey == null) { + Log.w(TAG, "No KBS master key is set! Must abort."); + return false; + } + + byte[] storageServiceKey = SignalStorageUtil.computeStorageServiceKey(kbsMasterKey); + boolean needsMultiDeviceSync = false; + long localManifestVersion = TextSecurePreferences.getStorageManifestVersion(context); + SignalStorageManifest remoteManifest = accountManager.getStorageManifest(storageServiceKey).or(new SignalStorageManifest(0, Collections.emptyList())); + + if (remoteManifest.getVersion() > localManifestVersion) { + Log.i(TAG, "Newer manifest version found! Our version: " + localManifestVersion + ", their version: " + remoteManifest.getVersion()); + + List allLocalStorageKeys = getAllLocalStorageKeys(context); + KeyDifferenceResult keyDifference = StorageSyncHelper.findKeyDifference(remoteManifest.getStorageKeys(), allLocalStorageKeys); + + if (!keyDifference.isEmpty()) { + List localOnly = buildLocalStorageRecords(context, keyDifference.getLocalOnlyKeys()); + List remoteOnly = accountManager.readStorageRecords(storageServiceKey, keyDifference.getRemoteOnlyKeys()); + MergeResult mergeResult = StorageSyncHelper.resolveConflict(remoteOnly, localOnly); + WriteOperationResult writeOperationResult = StorageSyncHelper.createWriteOperation(remoteManifest.getVersion(), allLocalStorageKeys, mergeResult); + + Optional conflict = accountManager.writeStorageRecords(storageServiceKey, writeOperationResult.getManifest(), writeOperationResult.getInserts(), writeOperationResult.getDeletes()); + + if (conflict.isPresent()) { + Log.w(TAG, "Hit a conflict when trying to resolve the conflict! Retrying."); + throw new RetryLaterException(); + } + + recipientDatabase.applyStorageSyncUpdates(mergeResult.getLocalContactInserts(), mergeResult.getLocalContactUpdates()); + storageKeyDatabase.applyStorageSyncUpdates(mergeResult.getLocalUnknownInserts(), mergeResult.getLocalUnknownDeletes()); + needsMultiDeviceSync = true; + + Log.i(TAG, "[Post-Conflict] Updating local manifest version to: " + writeOperationResult.getManifest().getVersion()); + TextSecurePreferences.setStorageManifestVersion(context, writeOperationResult.getManifest().getVersion()); + } else { + Log.i(TAG, "Remote version was newer, but our local data matched."); + Log.i(TAG, "[Post-Empty-Conflict] Updating local manifest version to: " + remoteManifest.getVersion()); + TextSecurePreferences.setStorageManifestVersion(context, remoteManifest.getVersion()); + } + } + + localManifestVersion = TextSecurePreferences.getStorageManifestVersion(context); + + List allLocalStorageKeys = recipientDatabase.getAllStorageSyncKeys(); + List pendingUpdates = recipientDatabase.getPendingRecipientSyncUpdates(); + List pendingInsertions = recipientDatabase.getPendingRecipientSyncInsertions(); + List pendingDeletions = recipientDatabase.getPendingRecipientSyncDeletions(); + Optional localWriteResult = StorageSyncHelper.buildStorageUpdatesForLocal(localManifestVersion, + allLocalStorageKeys, + pendingUpdates, + pendingInsertions, + pendingDeletions); + + if (localWriteResult.isPresent()) { + WriteOperationResult localWrite = localWriteResult.get().getWriteResult(); + Optional conflict = accountManager.writeStorageRecords(storageServiceKey, localWrite.getManifest(), localWrite.getInserts(), localWrite.getDeletes()); + + if (conflict.isPresent()) { + Log.w(TAG, "Hit a conflict when trying to upload our local writes! Retrying."); + throw new RetryLaterException(); + } + + List clearIds = new ArrayList<>(pendingUpdates.size() + pendingInsertions.size() + pendingDeletions.size()); + + clearIds.addAll(Stream.of(pendingUpdates).map(RecipientSettings::getId).toList()); + clearIds.addAll(Stream.of(pendingInsertions).map(RecipientSettings::getId).toList()); + clearIds.addAll(Stream.of(pendingDeletions).map(RecipientSettings::getId).toList()); + + recipientDatabase.clearDirtyState(clearIds); + recipientDatabase.updateStorageKeys(localWriteResult.get().getStorageKeyUpdates()); + + needsMultiDeviceSync = true; + + Log.i(TAG, "[Post Write] Updating local manifest version to: " + localWriteResult.get().getWriteResult().getManifest().getVersion()); + TextSecurePreferences.setStorageManifestVersion(context, localWriteResult.get().getWriteResult().getManifest().getVersion()); + } else { + Log.i(TAG, "Nothing locally to write."); + } + + return needsMultiDeviceSync; + } + + public static @NonNull List getAllLocalStorageKeys(@NonNull Context context) { + return Util.concatenatedList(DatabaseFactory.getRecipientDatabase(context).getAllStorageSyncKeys(), + DatabaseFactory.getStorageKeyDatabase(context).getAllKeys()); + } + + public static @NonNull List buildLocalStorageRecords(@NonNull Context context, @NonNull List keys) { + RecipientDatabase recipientDatabase = DatabaseFactory.getRecipientDatabase(context); + StorageKeyDatabase storageKeyDatabase = DatabaseFactory.getStorageKeyDatabase(context); + + List records = new ArrayList<>(keys.size()); + + for (byte[] key : keys) { + SignalStorageRecord record = Optional.fromNullable(recipientDatabase.getByStorageSyncKey(key)) + .transform(recipient -> { + SignalContactRecord contact = StorageSyncHelper.localToRemoteContact(recipient); + return SignalStorageRecord.forContact(key, contact); + }) + .or(() -> storageKeyDatabase.getByKey(key)); + records.add(record); + } + + return records; + } + + public static final class Factory implements Job.Factory { + + @Override + public @NonNull StorageSyncJob create(@NonNull Parameters parameters, @NonNull Data data) { + return new StorageSyncJob(parameters); + } + } +} diff --git a/src/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java b/src/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java index b58fab0ce6..9139213ed9 100644 --- a/src/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java +++ b/src/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java @@ -13,6 +13,7 @@ import org.whispersystems.signalservice.internal.configuration.SignalContactDisc import org.whispersystems.signalservice.internal.configuration.SignalKeyBackupServiceUrl; import org.whispersystems.signalservice.internal.configuration.SignalServiceConfiguration; import org.whispersystems.signalservice.internal.configuration.SignalServiceUrl; +import org.whispersystems.signalservice.internal.configuration.SignalStorageUrl; import java.util.HashMap; import java.util.Map; @@ -120,34 +121,49 @@ public class SignalServiceNetworkAccess { final SignalKeyBackupServiceUrl signalContactDiscoveryUrl = new SignalKeyBackupServiceUrl(BuildConfig.SIGNAL_KEY_BACKUP_URL, SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + final SignalStorageUrl baseGoogleStorage = new SignalStorageUrl("https://www.google.com/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + final SignalStorageUrl baseAndroidStorage = new SignalStorageUrl("https://android.clients.google.com/directory", SERVICE_REFLECTOR_HOST, trustStore, PLAY_CONNECTION_SPEC); + final SignalStorageUrl mapsOneAndroidStorage = new SignalStorageUrl("https://clients3.google.com/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAPS_CONNECTION_SPEC); + final SignalStorageUrl mapsTwoAndroidStorage = new SignalStorageUrl("https://clients4.google.com/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAPS_CONNECTION_SPEC); + final SignalStorageUrl mailAndroidStorage = new SignalStorageUrl("https://inbox.google.com/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + final SignalStorageUrl egyptGoogleStorage = new SignalStorageUrl("https://www.google.com.eg/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + final SignalStorageUrl uaeGoogleStorage = new SignalStorageUrl("https://www.google.ae/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + final SignalStorageUrl omanGoogleStorage = new SignalStorageUrl("https://www.google.com.om/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + final SignalStorageUrl qatarGoogleStorage = new SignalStorageUrl("https://www.google.com.qa/directory", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); + this.censorshipConfiguration = new HashMap() {{ put(COUNTRY_CODE_EGYPT, new SignalServiceConfiguration(new SignalServiceUrl[] {egyptGoogleService, baseGoogleService, baseAndroidService, mapsOneAndroidService, mapsTwoAndroidService, mailAndroidService}, new SignalCdnUrl[] {egyptGoogleCdn, baseAndroidCdn, baseGoogleCdn, mapsOneAndroidCdn, mapsTwoAndroidCdn, mailAndroidCdn, mailAndroidCdn}, new SignalContactDiscoveryUrl[] {egyptGoogleDiscovery, baseGoogleDiscovery, baseAndroidDiscovery, mapsOneAndroidDiscovery, mapsTwoAndroidDiscovery, mailAndroidDiscovery}, - new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl })); + new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl }, + new SignalStorageUrl[] {egyptGoogleStorage, baseGoogleStorage, baseAndroidStorage, mapsOneAndroidStorage, mapsTwoAndroidStorage, mailAndroidStorage})); put(COUNTRY_CODE_UAE, new SignalServiceConfiguration(new SignalServiceUrl[] {uaeGoogleService, baseAndroidService, baseGoogleService, mapsOneAndroidService, mapsTwoAndroidService, mailAndroidService}, new SignalCdnUrl[] {uaeGoogleCdn, baseAndroidCdn, baseGoogleCdn, mapsOneAndroidCdn, mapsTwoAndroidCdn, mailAndroidCdn}, new SignalContactDiscoveryUrl[] {uaeGoogleDiscovery, baseGoogleDiscovery, baseAndroidDiscovery, mapsOneAndroidDiscovery, mapsTwoAndroidDiscovery, mailAndroidDiscovery}, - new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl })); + new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl }, + new SignalStorageUrl[] {uaeGoogleStorage, baseGoogleStorage, baseAndroidStorage, mapsOneAndroidStorage, mapsTwoAndroidStorage, mailAndroidStorage})); put(COUNTRY_CODE_OMAN, new SignalServiceConfiguration(new SignalServiceUrl[] {omanGoogleService, baseAndroidService, baseGoogleService, mapsOneAndroidService, mapsTwoAndroidService, mailAndroidService}, new SignalCdnUrl[] {omanGoogleCdn, baseAndroidCdn, baseGoogleCdn, mapsOneAndroidCdn, mapsTwoAndroidCdn, mailAndroidCdn}, new SignalContactDiscoveryUrl[] {omanGoogleDiscovery, baseGoogleDiscovery, baseAndroidDiscovery, mapsOneAndroidDiscovery, mapsTwoAndroidDiscovery, mailAndroidDiscovery}, - new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl })); + new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl }, + new SignalStorageUrl[] {omanGoogleStorage, baseGoogleStorage, baseAndroidStorage, mapsOneAndroidStorage, mapsTwoAndroidStorage, mailAndroidStorage})); put(COUNTRY_CODE_QATAR, new SignalServiceConfiguration(new SignalServiceUrl[] {qatarGoogleService, baseAndroidService, baseGoogleService, mapsOneAndroidService, mapsTwoAndroidService, mailAndroidService}, new SignalCdnUrl[] {qatarGoogleCdn, baseAndroidCdn, baseGoogleCdn, mapsOneAndroidCdn, mapsTwoAndroidCdn, mailAndroidCdn}, new SignalContactDiscoveryUrl[] {qatarGoogleDiscovery, baseGoogleDiscovery, baseAndroidDiscovery, mapsOneAndroidDiscovery, mapsTwoAndroidDiscovery, mailAndroidDiscovery}, - new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl })); + new SignalKeyBackupServiceUrl[] { signalContactDiscoveryUrl }, + new SignalStorageUrl[] {qatarGoogleStorage, baseGoogleStorage, baseAndroidStorage, mapsOneAndroidStorage, mapsTwoAndroidStorage, mailAndroidStorage})); }}; this.uncensoredConfiguration = new SignalServiceConfiguration(new SignalServiceUrl[] {new SignalServiceUrl(BuildConfig.SIGNAL_URL, new SignalServiceTrustStore(context))}, new SignalCdnUrl[] {new SignalCdnUrl(BuildConfig.SIGNAL_CDN_URL, new SignalServiceTrustStore(context))}, new SignalContactDiscoveryUrl[] {new SignalContactDiscoveryUrl(BuildConfig.SIGNAL_CONTACT_DISCOVERY_URL, new SignalServiceTrustStore(context))}, - new SignalKeyBackupServiceUrl[] { new SignalKeyBackupServiceUrl(BuildConfig.SIGNAL_KEY_BACKUP_URL, new SignalServiceTrustStore(context)) }); + new SignalKeyBackupServiceUrl[] { new SignalKeyBackupServiceUrl(BuildConfig.SIGNAL_KEY_BACKUP_URL, new SignalServiceTrustStore(context)) }, + new SignalStorageUrl[] {new SignalStorageUrl(BuildConfig.STORAGE_URL, new SignalServiceTrustStore(context))}); this.censoredCountries = this.censorshipConfiguration.keySet().toArray(new String[0]); } diff --git a/src/org/thoughtcrime/securesms/recipients/Recipient.java b/src/org/thoughtcrime/securesms/recipients/Recipient.java index 73fdeac7d7..0766844c0d 100644 --- a/src/org/thoughtcrime/securesms/recipients/Recipient.java +++ b/src/org/thoughtcrime/securesms/recipients/Recipient.java @@ -22,6 +22,8 @@ import org.thoughtcrime.securesms.contacts.avatars.ResourceContactPhoto; import org.thoughtcrime.securesms.contacts.avatars.SystemContactPhoto; import org.thoughtcrime.securesms.contacts.avatars.TransparentContactPhoto; import org.thoughtcrime.securesms.database.DatabaseFactory; +import org.thoughtcrime.securesms.database.IdentityDatabase; +import org.thoughtcrime.securesms.database.IdentityDatabase.VerifiedStatus; import org.thoughtcrime.securesms.database.RecipientDatabase; import org.thoughtcrime.securesms.database.RecipientDatabase.RegisteredState; import org.thoughtcrime.securesms.database.RecipientDatabase.UnidentifiedAccessMode; @@ -89,6 +91,9 @@ public class Recipient { private final boolean forceSmsSelection; private final boolean uuidSupported; private final InsightsBannerTier insightsBannerTier; + private final byte[] storageKey; + private final byte[] identityKey; + private final VerifiedStatus identityStatus; /** @@ -293,6 +298,9 @@ public class Recipient { this.unidentifiedAccessMode = UnidentifiedAccessMode.DISABLED; this.forceSmsSelection = false; this.uuidSupported = false; + this.storageKey = null; + this.identityKey = null; + this.identityStatus = VerifiedStatus.DEFAULT; } Recipient(@NonNull RecipientId id, @NonNull RecipientDetails details) { @@ -329,6 +337,9 @@ public class Recipient { this.unidentifiedAccessMode = details.unidentifiedAccessMode; this.forceSmsSelection = details.forceSmsSelection; this.uuidSupported = details.uuidSuported; + this.storageKey = details.storageKey; + this.identityKey = details.identityKey; + this.identityStatus = details.identityStatus; } public @NonNull RecipientId getId() { @@ -645,6 +656,18 @@ public class Recipient { return profileKey; } + public @Nullable byte[] getStorageServiceKey() { + return storageKey; + } + + public @NonNull VerifiedStatus getIdentityVerifiedStatus() { + return identityStatus; + } + + public @Nullable byte[] getIdentityKey() { + return identityKey; + } + public @NonNull UnidentifiedAccessMode getUnidentifiedAccessMode() { return unidentifiedAccessMode; } diff --git a/src/org/thoughtcrime/securesms/recipients/RecipientDetails.java b/src/org/thoughtcrime/securesms/recipients/RecipientDetails.java index bbb1fbbca0..be36185a08 100644 --- a/src/org/thoughtcrime/securesms/recipients/RecipientDetails.java +++ b/src/org/thoughtcrime/securesms/recipients/RecipientDetails.java @@ -9,6 +9,7 @@ import androidx.annotation.Nullable; import org.thoughtcrime.securesms.color.MaterialColor; import org.thoughtcrime.securesms.database.RecipientDatabase.InsightsBannerTier; import org.thoughtcrime.securesms.crypto.ProfileKeyUtil; +import org.thoughtcrime.securesms.database.IdentityDatabase.VerifiedStatus; import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings; import org.thoughtcrime.securesms.database.RecipientDatabase.RegisteredState; import org.thoughtcrime.securesms.database.RecipientDatabase.UnidentifiedAccessMode; @@ -55,6 +56,9 @@ public class RecipientDetails { final boolean forceSmsSelection; final boolean uuidSuported; final InsightsBannerTier insightsBannerTier; + final byte[] storageKey; + final byte[] identityKey; + final VerifiedStatus identityStatus; RecipientDetails(@NonNull Context context, @Nullable String name, @@ -95,12 +99,18 @@ public class RecipientDetails { this.forceSmsSelection = settings.isForceSmsSelection(); this.uuidSuported = settings.isUuidSupported(); this.insightsBannerTier = settings.getInsightsBannerTier(); + this.storageKey = settings.getStorageKey(); + this.identityKey = settings.getIdentityKey(); + this.identityStatus = settings.getIdentityStatus(); if (name == null) this.name = settings.getSystemDisplayName(); else this.name = name; } - public RecipientDetails() { + /** + * Only used for {@link Recipient#UNKNOWN}. + */ + RecipientDetails() { this.groupAvatarId = null; this.systemContactPhoto = null; this.customLabel = null; @@ -133,5 +143,8 @@ public class RecipientDetails { this.forceSmsSelection = false; this.name = null; this.uuidSuported = false; + this.storageKey = null; + this.identityKey = null; + this.identityStatus = VerifiedStatus.DEFAULT; } } diff --git a/src/org/thoughtcrime/securesms/util/FeatureFlags.java b/src/org/thoughtcrime/securesms/util/FeatureFlags.java index 89253dcd18..ea767c8801 100644 --- a/src/org/thoughtcrime/securesms/util/FeatureFlags.java +++ b/src/org/thoughtcrime/securesms/util/FeatureFlags.java @@ -23,6 +23,9 @@ public class FeatureFlags { /** Set or migrate PIN to KBS */ public static final boolean KBS = false; + /** Storage service. Requires {@link #KBS}. */ + public static final boolean STORAGE_SERVICE = false; + /** Send support for reactions. */ public static final boolean REACTION_SENDING = false; -} \ No newline at end of file +} diff --git a/src/org/thoughtcrime/securesms/util/SetUtil.java b/src/org/thoughtcrime/securesms/util/SetUtil.java new file mode 100644 index 0000000000..d030d673dc --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/SetUtil.java @@ -0,0 +1,31 @@ +package org.thoughtcrime.securesms.util; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +public final class SetUtil { + private SetUtil() {} + + public static Set intersection(Set a, Set b) { + Set intersection = new LinkedHashSet<>(a); + intersection.retainAll(b); + return intersection; + } + + public static Set difference(Set a, Set b) { + Set difference = new LinkedHashSet<>(a); + difference.removeAll(b); + return difference; + } + + public static Set union(Set... sets) { + Set result = new LinkedHashSet<>(); + + for (Set set : sets) { + result.addAll(set); + } + + return result; + } +} diff --git a/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java b/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java index 982855cf82..bc210882c7 100644 --- a/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java +++ b/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java @@ -213,6 +213,8 @@ public class TextSecurePreferences { private static final String HAS_SEEN_VIDEO_RECORDING_TOOLTIP = "camerax.fragment.has.dismissed.video.recording.tooltip"; + private static final String STORAGE_MANIFEST_VERSION = "pref_storage_manifest_version"; + public static boolean isScreenLockEnabled(@NonNull Context context) { return getBooleanPreference(context, SCREEN_LOCK, false); } @@ -1328,6 +1330,14 @@ public class TextSecurePreferences { setBooleanPreference(context, HAS_SEEN_VIDEO_RECORDING_TOOLTIP, value); } + public static long getStorageManifestVersion(Context context) { + return getLongPreference(context, STORAGE_MANIFEST_VERSION, 0); + } + + public static void setStorageManifestVersion(Context context, long version) { + setLongPreference(context, STORAGE_MANIFEST_VERSION, version); + } + public static void setBooleanPreference(Context context, String key, boolean value) { PreferenceManager.getDefaultSharedPreferences(context).edit().putBoolean(key, value).apply(); } diff --git a/test/unitTest/java/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelperTest.java b/test/unitTest/java/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelperTest.java new file mode 100644 index 0000000000..45bc2e3a2c --- /dev/null +++ b/test/unitTest/java/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelperTest.java @@ -0,0 +1,323 @@ +package org.thoughtcrime.securesms.contacts.sync; + +import androidx.annotation.NonNull; + +import com.annimon.stream.Stream; + +import org.junit.Before; +import org.junit.Test; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.KeyDifferenceResult; +import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.MergeResult; +import org.thoughtcrime.securesms.util.Conversions; +import org.whispersystems.signalservice.api.push.SignalServiceAddress; +import org.whispersystems.signalservice.api.storage.SignalContactRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; +import org.whispersystems.signalservice.api.util.UuidUtil; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import edu.emory.mathcs.backport.java.util.Arrays; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class StorageSyncHelperTest { + + private static final UUID UUID_A = UuidUtil.parseOrThrow("ebef429e-695e-4f51-bcc4-526a60ac68c7"); + private static final UUID UUID_B = UuidUtil.parseOrThrow("32119989-77fb-4e18-af70-81d55185c6b1"); + private static final UUID UUID_C = UuidUtil.parseOrThrow("b5552203-2bca-44aa-b6f5-9f5d87a335b6"); + private static final UUID UUID_D = UuidUtil.parseOrThrow("94829a32-7199-4a7b-8fb4-7e978509ab84"); + + private static final String E164_A = "+16108675309"; + private static final String E164_B = "+16101234567"; + private static final String E164_C = "+16101112222"; + private static final String E164_D = "+16103334444"; + + private static final int UNKNOWN_TYPE = Integer.MAX_VALUE; + + @Before + public void setup() { + StorageSyncHelper.setTestKeyGenerator(null); + } + + + @Test + public void findKeyDifference_allOverlap() { + KeyDifferenceResult result = StorageSyncHelper.findKeyDifference(byteListOf(1, 2, 3), byteListOf(1, 2, 3)); + assertTrue(result.getLocalOnlyKeys().isEmpty()); + assertTrue(result.getRemoteOnlyKeys().isEmpty()); + } + + @Test + public void findKeyDifference_noOverlap() { + KeyDifferenceResult result = StorageSyncHelper.findKeyDifference(byteListOf(1, 2, 3), byteListOf(4, 5, 6)); + assertByteListEquals(byteListOf(1, 2, 3), result.getRemoteOnlyKeys()); + assertByteListEquals(byteListOf(4, 5, 6), result.getLocalOnlyKeys()); + } + + @Test + public void findKeyDifference_someOverlap() { + KeyDifferenceResult result = StorageSyncHelper.findKeyDifference(byteListOf(1, 2, 3), byteListOf(2, 3, 4)); + assertByteListEquals(byteListOf(1), result.getRemoteOnlyKeys()); + assertByteListEquals(byteListOf(4), result.getLocalOnlyKeys()); + } + + @Test + public void resolveConflict_noOverlap() { + SignalContactRecord remote1 = contact(1, UUID_A, E164_A, "a"); + SignalContactRecord local1 = contact(2, UUID_B, E164_B, "b"); + + MergeResult result = StorageSyncHelper.resolveConflict(recordSetOf(remote1), recordSetOf(local1)); + + assertEquals(setOf(remote1), result.getLocalContactInserts()); + assertTrue(result.getLocalContactUpdates().isEmpty()); + assertEquals(setOf(local1), result.getRemoteContactInserts()); + assertTrue(result.getRemoteContactUpdates().isEmpty()); + } + + @Test + public void resolveConflict_sameAsRemote() { + SignalContactRecord remote1 = contact(1, UUID_A, E164_A, "a"); + SignalContactRecord local1 = contact(2, UUID_A, E164_A, "a"); + + MergeResult result = StorageSyncHelper.resolveConflict(recordSetOf(remote1), recordSetOf(local1)); + + SignalContactRecord expectedMerge = contact(1, UUID_A, E164_A, "a"); + + assertTrue(result.getLocalContactInserts().isEmpty()); + assertEquals(setOf(contactUpdate(local1, expectedMerge)), result.getLocalContactUpdates()); + assertTrue(result.getRemoteContactInserts().isEmpty()); + assertTrue(result.getRemoteContactUpdates().isEmpty()); + } + + @Test + public void resolveConflict_sameAsLocal() { + SignalContactRecord remote1 = contact(1, UUID_A, E164_A, null); + SignalContactRecord local1 = contact(2, UUID_A, E164_A, "a"); + + MergeResult result = StorageSyncHelper.resolveConflict(recordSetOf(remote1), recordSetOf(local1)); + + SignalContactRecord expectedMerge = contact(2, UUID_A, E164_A, "a"); + + assertTrue(result.getLocalContactInserts().isEmpty()); + assertTrue(result.getLocalContactUpdates().isEmpty()); + assertTrue(result.getRemoteContactInserts().isEmpty()); + assertEquals(setOf(contactUpdate(remote1, expectedMerge)), result.getRemoteContactUpdates()); + } + + @Test + public void resolveConflict_unknowns() { + SignalStorageRecord remote1 = unknown(3); + SignalStorageRecord remote2 = unknown(4); + SignalStorageRecord local1 = unknown(1); + SignalStorageRecord local2 = unknown(2); + + MergeResult result = StorageSyncHelper.resolveConflict(setOf(remote1, remote2), setOf(local1, local2)); + + assertTrue(result.getLocalContactInserts().isEmpty()); + assertTrue(result.getLocalContactUpdates().isEmpty()); + assertEquals(setOf(remote1, remote2), result.getLocalUnknownInserts()); + assertEquals(setOf(local1, local2), result.getLocalUnknownDeletes()); + } + + @Test + public void resolveConflict_complex() { + SignalContactRecord remote1 = contact(1, UUID_A, null, "a"); + SignalContactRecord local1 = contact(2, UUID_A, E164_A, "a"); + + SignalContactRecord remote2 = contact(3, UUID_B, E164_B, null); + SignalContactRecord local2 = contact(4, UUID_B, null, "b"); + + SignalContactRecord remote3 = contact(5, UUID_C, E164_C, "c"); + SignalContactRecord local3 = contact(6, UUID_D, E164_D, "d"); + + SignalStorageRecord unknownRemote = unknown(7); + SignalStorageRecord unknownLocal = unknown(8); + + StorageSyncHelper.setTestKeyGenerator(new TestGenerator(999)); + + Set remoteOnly = recordSetOf(remote1, remote2, remote3); + Set localOnly = recordSetOf(local1, local2, local3); + + remoteOnly.add(unknownRemote); + localOnly.add(unknownLocal); + + MergeResult result = StorageSyncHelper.resolveConflict(remoteOnly, localOnly); + + SignalContactRecord merge1 = contact(2, UUID_A, E164_A, "a"); + SignalContactRecord merge2 = contact(999, UUID_B, E164_B, "b"); + + assertEquals(setOf(remote3), result.getLocalContactInserts()); + assertEquals(setOf(contactUpdate(local2, merge2)), result.getLocalContactUpdates()); + assertEquals(setOf(local3), result.getRemoteContactInserts()); + assertEquals(setOf(contactUpdate(remote1, merge1), contactUpdate(remote2, merge2)), result.getRemoteContactUpdates()); + assertEquals(setOf(unknownRemote), result.getLocalUnknownInserts()); + assertEquals(setOf(unknownLocal), result.getLocalUnknownDeletes()); + } + + @Test + public void mergeContacts_alwaysPreferRemoteExceptNickname() { + SignalContactRecord remote = new SignalContactRecord.Builder(byteArray(1), new SignalServiceAddress(UUID_A, E164_A)) + .setBlocked(true) + .setIdentityKey(byteArray(2)) + .setIdentityState(SignalContactRecord.IdentityState.VERIFIED) + .setProfileKey(byteArray(3)) + .setProfileName("profile name A") + .setUsername("username A") + .setNickname("nickname A") + .setProfileSharingEnabled(true) + .build(); + SignalContactRecord local = new SignalContactRecord.Builder(byteArray(2), new SignalServiceAddress(UUID_B, E164_B)) + .setBlocked(false) + .setIdentityKey(byteArray(99)) + .setIdentityState(SignalContactRecord.IdentityState.DEFAULT) + .setProfileKey(byteArray(999)) + .setProfileName("profile name B") + .setUsername("username B") + .setNickname("nickname B") + .setProfileSharingEnabled(false) + .build(); + SignalContactRecord merged = StorageSyncHelper.mergeContacts(remote, local); + + assertEquals(UUID_A, merged.getAddress().getUuid().get()); + assertEquals(E164_A, merged.getAddress().getNumber().get()); + assertTrue(merged.isBlocked()); + assertArrayEquals(byteArray(2), merged.getIdentityKey().get()); + assertEquals(SignalContactRecord.IdentityState.VERIFIED, merged.getIdentityState()); + assertArrayEquals(byteArray(3), merged.getProfileKey().get()); + assertEquals("profile name A", merged.getProfileName().get()); + assertEquals("username A", merged.getUsername().get()); + assertEquals("nickname B", merged.getNickname().get()); + assertTrue(merged.isProfileSharingEnabled()); + } + + @Test + public void mergeContacts_fillInGaps() { + SignalContactRecord remote = new SignalContactRecord.Builder(byteArray(1), new SignalServiceAddress(UUID_A, null)) + .setBlocked(true) + .setProfileName("profile name A") + .setProfileSharingEnabled(true) + .build(); + SignalContactRecord local = new SignalContactRecord.Builder(byteArray(2), new SignalServiceAddress(UUID_B, E164_B)) + .setBlocked(false) + .setIdentityKey(byteArray(2)) + .setProfileKey(byteArray(3)) + .setProfileName("profile name B") + .setUsername("username B") + .setProfileSharingEnabled(false) + .build(); + SignalContactRecord merged = StorageSyncHelper.mergeContacts(remote, local); + + assertEquals(UUID_A, merged.getAddress().getUuid().get()); + assertEquals(E164_B, merged.getAddress().getNumber().get()); + assertTrue(merged.isBlocked()); + assertArrayEquals(byteArray(2), merged.getIdentityKey().get()); + assertEquals(SignalContactRecord.IdentityState.DEFAULT, merged.getIdentityState()); + assertArrayEquals(byteArray(3), merged.getProfileKey().get()); + assertEquals("profile name A", merged.getProfileName().get()); + assertEquals("username B", merged.getUsername().get()); + assertTrue(merged.isProfileSharingEnabled()); + } + + @Test + public void createWriteOperation_generic() { + List localKeys = byteListOf(1, 2, 3, 4); + SignalContactRecord insert1 = contact(6, UUID_A, E164_A, "a" ); + SignalContactRecord old1 = contact(1, UUID_B, E164_B, "b" ); + SignalContactRecord new1 = contact(5, UUID_B, E164_B, "z" ); + SignalContactRecord insert2 = contact(7, UUID_C, E164_C, "c" ); + SignalContactRecord old2 = contact(2, UUID_D, E164_D, "d" ); + SignalContactRecord new2 = contact(8, UUID_D, E164_D, "z2"); + SignalStorageRecord unknownInsert = unknown(9); + SignalStorageRecord unknownDelete = unknown(10); + + StorageSyncHelper.WriteOperationResult result = StorageSyncHelper.createWriteOperation(1, + localKeys, + new MergeResult(setOf(insert2), + setOf(contactUpdate(old2, new2)), + setOf(insert1), + setOf(contactUpdate(old1, new1)), + setOf(unknownInsert), + setOf(unknownDelete))); + + assertEquals(2, result.getManifest().getVersion()); + assertByteListEquals(byteListOf(3, 4, 5, 6, 7, 8, 9), result.getManifest().getStorageKeys()); + assertTrue(recordSetOf(insert1, new1).containsAll(result.getInserts())); + assertEquals(2, result.getInserts().size()); + assertByteListEquals(byteListOf(1), result.getDeletes()); + } + + private static Set setOf(E... vals) { + return new LinkedHashSet(Arrays.asList(vals)); + } + + private static Set recordSetOf(SignalContactRecord... contactRecords) { + LinkedHashSet storageRecords = new LinkedHashSet<>(); + + for (SignalContactRecord contactRecord : contactRecords) { + storageRecords.add(SignalStorageRecord.forContact(contactRecord.getKey(), contactRecord)); + } + + return storageRecords; + } + + private static SignalContactRecord contact(int key, + UUID uuid, + String e164, + String profileName) + { + return new SignalContactRecord.Builder(byteArray(key), new SignalServiceAddress(uuid, e164)) + .setProfileName(profileName) + .build(); + } + + private static StorageSyncHelper.ContactUpdate contactUpdate(SignalContactRecord oldContact, SignalContactRecord newContact) { + return new StorageSyncHelper.ContactUpdate(oldContact, newContact); + } + + private static SignalStorageRecord unknown(int key) { + return SignalStorageRecord.forUnknown(byteArray(key), UNKNOWN_TYPE); + } + + private static List byteListOf(int... vals) { + List list = new ArrayList<>(vals.length); + + for (int i = 0; i < vals.length; i++) { + list.add(Conversions.intToByteArray(vals[i])); + + } + return list; + } + + private static byte[] byteArray(int a) { + return Conversions.intToByteArray(a); + } + + private static void assertByteListEquals(List a, List b) { + assertEquals(a.size(), b.size()); + + List aBuffer = Stream.of(a).map(ByteBuffer::wrap).toList(); + List bBuffer = Stream.of(b).map(ByteBuffer::wrap).toList(); + + assertTrue(aBuffer.containsAll(bBuffer)); + } + + private static class TestGenerator implements StorageSyncHelper.KeyGenerator { + private final byte[] key; + + private TestGenerator(int key) { + this.key = byteArray(key); + } + + @Override + public @NonNull byte[] generate() { + return key; + } + } +}