TheArchive/session-android
1
0
Fork 0
mirror of https://github.com/oxen-io/session-android.git synced 2025-12-30 13:26:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update and refactor storage service syncing.

Browse Source 
Switched to proto3, updated protos, and generally refactored things to
make it easier to add new storage record types.
This commit is contained in:
Greyson Parrelli
2020-02-28 13:03:06 -05:00
parent 40d9d663ec
commit 5f7075d39a
25 changed files with 1484 additions and 1387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

767
app/src/main/java/org/thoughtcrime/securesms/contacts/sync/StorageSyncHelper.java
View File

@@ -1,767 +0,0 @@
package org.thoughtcrime.securesms.contacts.sync;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import com.annimon.stream.Collectors;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.database.IdentityDatabase;
import org.thoughtcrime.securesms.database.RecipientDatabase;
import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.recipients.RecipientId;
import org.thoughtcrime.securesms.util.GroupUtil;
import org.thoughtcrime.securesms.util.SetUtil;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.util.guava.Optional;
import org.whispersystems.signalservice.api.push.SignalServiceAddress;
import org.whispersystems.signalservice.api.storage.SignalContactRecord;
import org.whispersystems.signalservice.api.storage.SignalContactRecord.IdentityState;
import org.whispersystems.signalservice.api.storage.SignalGroupV1Record;
import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
import org.whispersystems.signalservice.api.util.OptionalUtil;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import javax.crypto.KeyGenerator;
public final class StorageSyncHelper {
private static final String TAG = Log.tag(StorageSyncHelper.class);
private static final KeyGenerator KEY_GENERATOR = () -> Util.getSecretBytes(16);
private static KeyGenerator testKeyGenerator = null;
/**
* Given the local state of pending storage mutations, this will generate a result that will
* include that data that needs to be written to the storage service, as well as any changes you
* need to write back to local storage (like storage keys that might have changed for updated
* contacts).
*
* @param currentManifestVersion What you think the version is locally.
* @param currentLocalKeys All local keys you have. This assumes that 'inserts' were given keys
* already, and that deletes still have keys.
* @param updates Contacts that have been altered.
* @param inserts Contacts that have been inserted (or newly marked as registered).
* @param deletes Contacts that are no longer registered.
*
* @return If changes need to be written, then it will return those changes. If no changes need
* to be written, this will return {@link Optional#absent()}.
*/
public static @NonNull Optional<LocalWriteResult> buildStorageUpdatesForLocal(long currentManifestVersion,
@NonNull List<byte[]> currentLocalKeys,
@NonNull List<RecipientSettings> updates,
@NonNull List<RecipientSettings> inserts,
@NonNull List<RecipientSettings> deletes)
{
Set<ByteBuffer> completeKeys = new LinkedHashSet<>(Stream.of(currentLocalKeys).map(ByteBuffer::wrap).toList());
Set<SignalStorageRecord> storageInserts = new LinkedHashSet<>();
Set<ByteBuffer> storageDeletes = new LinkedHashSet<>();
Map<RecipientId, byte[]> storageKeyUpdates = new HashMap<>();
for (RecipientSettings insert : inserts) {
storageInserts.add(localToRemoteRecord(insert));
}
for (RecipientSettings delete : deletes) {
byte[] key = Objects.requireNonNull(delete.getStorageKey());
storageDeletes.add(ByteBuffer.wrap(key));
completeKeys.remove(ByteBuffer.wrap(key));
}
for (RecipientSettings update : updates) {
byte[] oldKey = Objects.requireNonNull(update.getStorageKey());
byte[] newKey = generateKey();
storageInserts.add(localToRemoteRecord(update, newKey));
storageDeletes.add(ByteBuffer.wrap(oldKey));
completeKeys.remove(ByteBuffer.wrap(oldKey));
completeKeys.add(ByteBuffer.wrap(newKey));
storageKeyUpdates.put(update.getId(), newKey);
}
if (storageInserts.isEmpty() && storageDeletes.isEmpty()) {
return Optional.absent();
} else {
List<byte[]> contactDeleteBytes = Stream.of(storageDeletes).map(ByteBuffer::array).toList();
List<byte[]> completeKeysBytes = Stream.of(completeKeys).map(ByteBuffer::array).toList();
SignalStorageManifest manifest = new SignalStorageManifest(currentManifestVersion + 1, completeKeysBytes);
WriteOperationResult writeOperationResult = new WriteOperationResult(manifest, new ArrayList<>(storageInserts), contactDeleteBytes);
return Optional.of(new LocalWriteResult(writeOperationResult, storageKeyUpdates));
}
}
/**
* Given a list of all the local and remote keys you know about, this will return a result telling
* you which keys are exclusively remote and which are exclusively local.
*
* @param remoteKeys All remote keys available.
* @param localKeys All local keys available.
*
* @return An object describing which keys are exclusive to the remote data set and which keys are
* exclusive to the local data set.
*/
public static @NonNull KeyDifferenceResult findKeyDifference(@NonNull List<byte[]> remoteKeys,
@NonNull List<byte[]> localKeys)
{
Set<ByteBuffer> allRemoteKeys = Stream.of(remoteKeys).map(ByteBuffer::wrap).collect(LinkedHashSet::new, HashSet::add);
Set<ByteBuffer> allLocalKeys = Stream.of(localKeys).map(ByteBuffer::wrap).collect(LinkedHashSet::new, HashSet::add);
Set<ByteBuffer> remoteOnlyKeys = SetUtil.difference(allRemoteKeys, allLocalKeys);
Set<ByteBuffer> localOnlyKeys = SetUtil.difference(allLocalKeys, allRemoteKeys);
return new KeyDifferenceResult(Stream.of(remoteOnlyKeys).map(ByteBuffer::array).toList(),
Stream.of(localOnlyKeys).map(ByteBuffer::array).toList());
}
/**
* Given two sets of storage records, this will resolve the data into a set of actions that need
* to be applied to resolve the differences. This will handle discovering which records between
* the two collections refer to the same contacts and are actually updates, which are brand new,
* etc.
*
* @param remoteOnlyRecords Records that are only present remotely.
* @param localOnlyRecords Records that are only present locally.
*
* @return A set of actions that should be applied to resolve the conflict.
*/
public static @NonNull MergeResult resolveConflict(@NonNull Collection<SignalStorageRecord> remoteOnlyRecords,
@NonNull Collection<SignalStorageRecord> localOnlyRecords)
{
List<SignalContactRecord> remoteOnlyContacts = Stream.of(remoteOnlyRecords).filter(r -> r.getContact().isPresent()).map(r -> r.getContact().get()).toList();
List<SignalContactRecord> localOnlyContacts = Stream.of(localOnlyRecords).filter(r -> r.getContact().isPresent()).map(r -> r.getContact().get()).toList();
List<SignalGroupV1Record> remoteOnlyGroupV1 = Stream.of(remoteOnlyRecords).filter(r -> r.getGroupV1().isPresent()).map(r -> r.getGroupV1().get()).toList();
List<SignalGroupV1Record> localOnlyGroupV1 = Stream.of(localOnlyRecords).filter(r -> r.getGroupV1().isPresent()).map(r -> r.getGroupV1().get()).toList();
List<SignalStorageRecord> remoteOnlyUnknowns = Stream.of(remoteOnlyRecords).filter(SignalStorageRecord::isUnknown).toList();
List<SignalStorageRecord> localOnlyUnknowns = Stream.of(localOnlyRecords).filter(SignalStorageRecord::isUnknown).toList();
ContactRecordMergeResult contactMergeResult = resolveContactConflict(remoteOnlyContacts, localOnlyContacts);
GroupV1RecordMergeResult groupV1MergeResult = resolveGroupV1Conflict(remoteOnlyGroupV1, localOnlyGroupV1);
Set<SignalStorageRecord> remoteInserts = new HashSet<>();
remoteInserts.addAll(Stream.of(contactMergeResult.remoteInserts).map(SignalStorageRecord::forContact).toList());
remoteInserts.addAll(Stream.of(groupV1MergeResult.remoteInserts).map(SignalStorageRecord::forGroupV1).toList());
Set<RecordUpdate> remoteUpdates = new HashSet<>();
remoteUpdates.addAll(Stream.of(contactMergeResult.remoteUpdates)
.map(c -> new RecordUpdate(SignalStorageRecord.forContact(c.getOld()), SignalStorageRecord.forContact(c.getNew())))
.toList());
remoteUpdates.addAll(Stream.of(groupV1MergeResult.remoteUpdates)
.map(c -> new RecordUpdate(SignalStorageRecord.forGroupV1(c.getOld()), SignalStorageRecord.forGroupV1(c.getNew())))
.toList());
return new MergeResult(contactMergeResult.localInserts,
contactMergeResult.localUpdates,
groupV1MergeResult.localInserts,
groupV1MergeResult.localUpdates,
new LinkedHashSet<>(remoteOnlyUnknowns),
new LinkedHashSet<>(localOnlyUnknowns),
remoteInserts,
remoteUpdates);
}
/**
* Assumes that the merge result has *not* yet been applied to the local data. That means that
* this method will handle generating the correct final key set based on the merge result.
*/
public static @NonNull WriteOperationResult createWriteOperation(long currentManifestVersion,
@NonNull List<byte[]> currentLocalStorageKeys,
@NonNull MergeResult mergeResult)
{
Set<ByteBuffer> completeKeys = new LinkedHashSet<>(Stream.of(currentLocalStorageKeys).map(ByteBuffer::wrap).toList());
for (SignalContactRecord insert : mergeResult.getLocalContactInserts()) {
completeKeys.add(ByteBuffer.wrap(insert.getKey()));
}
for (SignalGroupV1Record insert : mergeResult.getLocalGroupV1Inserts()) {
completeKeys.add(ByteBuffer.wrap(insert.getKey()));
}
for (SignalStorageRecord insert : mergeResult.getRemoteInserts()) {
completeKeys.add(ByteBuffer.wrap(insert.getKey()));
}
for (SignalStorageRecord insert : mergeResult.getLocalUnknownInserts()) {
completeKeys.add(ByteBuffer.wrap(insert.getKey()));
}
for (ContactUpdate update : mergeResult.getLocalContactUpdates()) {
completeKeys.remove(ByteBuffer.wrap(update.getOld().getKey()));
completeKeys.add(ByteBuffer.wrap(update.getNew().getKey()));
}
for (GroupV1Update update : mergeResult.getLocalGroupV1Updates()) {
completeKeys.remove(ByteBuffer.wrap(update.getOld().getKey()));
completeKeys.add(ByteBuffer.wrap(update.getNew().getKey()));
}
for (RecordUpdate update : mergeResult.getRemoteUpdates()) {
completeKeys.remove(ByteBuffer.wrap(update.getOld().getKey()));
completeKeys.add(ByteBuffer.wrap(update.getNew().getKey()));
}
SignalStorageManifest manifest = new SignalStorageManifest(currentManifestVersion + 1, Stream.of(completeKeys).map(ByteBuffer::array).toList());
List<SignalStorageRecord> inserts = new ArrayList<>();
inserts.addAll(mergeResult.getRemoteInserts());
inserts.addAll(Stream.of(mergeResult.getRemoteUpdates()).map(RecordUpdate::getNew).toList());
List<byte[]> deletes = Stream.of(mergeResult.getRemoteUpdates()).map(RecordUpdate::getOld).map(SignalStorageRecord::getKey).toList();
return new WriteOperationResult(manifest, inserts, deletes);
}
public static @NonNull SignalStorageRecord localToRemoteRecord(@NonNull RecipientSettings settings) {
if (settings.getStorageKey() == null) {
throw new AssertionError("Must have a storage key!");
}
return localToRemoteRecord(settings, settings.getStorageKey());
}
public static @NonNull SignalStorageRecord localToRemoteRecord(@NonNull RecipientSettings settings, @NonNull byte[] key) {
if (settings.getGroupType() == RecipientDatabase.GroupType.NONE) {
return SignalStorageRecord.forContact(localToRemoteContact(settings, key));
} else if (settings.getGroupType() == RecipientDatabase.GroupType.SIGNAL_V1) {
return SignalStorageRecord.forGroupV1(localToRemoteGroupV1(settings, key));
} else {
throw new AssertionError("Unsupported type!");
}
}
private static @NonNull SignalContactRecord localToRemoteContact(@NonNull RecipientSettings recipient, byte[] storageKey) {
if (recipient.getUuid() == null && recipient.getE164() == null) {
throw new AssertionError("Must have either a UUID or a phone number!");
}
return new SignalContactRecord.Builder(storageKey, new SignalServiceAddress(recipient.getUuid(), recipient.getE164()))
.setProfileKey(recipient.getProfileKey())
.setGivenName(recipient.getProfileName().getGivenName())
.setFamilyName(recipient.getProfileName().getFamilyName())
.setBlocked(recipient.isBlocked())
.setProfileSharingEnabled(recipient.isProfileSharing())
.setIdentityKey(recipient.getIdentityKey())
.setIdentityState(localToRemoteIdentityState(recipient.getIdentityStatus()))
.build();
}
private static @NonNull SignalGroupV1Record localToRemoteGroupV1(@NonNull RecipientSettings recipient, byte[] storageKey) {
if (recipient.getGroupId() == null) {
throw new AssertionError("Must have a groupId!");
}
return new SignalGroupV1Record.Builder(storageKey, GroupUtil.getDecodedIdOrThrow(recipient.getGroupId()))
.setBlocked(recipient.isBlocked())
.setProfileSharingEnabled(recipient.isProfileSharing())
.build();
}
public static @NonNull IdentityDatabase.VerifiedStatus remoteToLocalIdentityStatus(@NonNull IdentityState identityState) {
switch (identityState) {
case VERIFIED: return IdentityDatabase.VerifiedStatus.VERIFIED;
case UNVERIFIED: return IdentityDatabase.VerifiedStatus.UNVERIFIED;
default: return IdentityDatabase.VerifiedStatus.DEFAULT;
}
}
public static @NonNull byte[] generateKey() {
if (testKeyGenerator != null) {
return testKeyGenerator.generate();
} else {
return KEY_GENERATOR.generate();
}
}
@VisibleForTesting
static @NonNull SignalContactRecord mergeContacts(@NonNull SignalContactRecord remote,
@NonNull SignalContactRecord local)
{
UUID uuid = remote.getAddress().getUuid().or(local.getAddress().getUuid()).orNull();
String e164 = remote.getAddress().getNumber().or(local.getAddress().getNumber()).orNull();
SignalServiceAddress address = new SignalServiceAddress(uuid, e164);
String givenName = remote.getGivenName().or(local.getGivenName()).or("");
String familyName = remote.getFamilyName().or(local.getFamilyName()).or("");
byte[] profileKey = remote.getProfileKey().or(local.getProfileKey()).orNull();
String username = remote.getUsername().or(local.getUsername()).or("");
IdentityState identityState = remote.getIdentityState();
byte[] identityKey = remote.getIdentityKey().or(local.getIdentityKey()).orNull();
String nickname = local.getNickname().or(""); // TODO [greyson] Update this when we add real nickname support
boolean blocked = remote.isBlocked();
boolean profileSharing = remote.isProfileSharingEnabled() || local.isProfileSharingEnabled();
boolean matchesRemote = doParamsMatchContact(remote, address, givenName, familyName, profileKey, username, identityState, identityKey, blocked, profileSharing, nickname);
boolean matchesLocal = doParamsMatchContact(local, address, givenName, familyName, profileKey, username, identityState, identityKey, blocked, profileSharing, nickname);
if (remote.getProtoVersion() > 0) {
Log.w(TAG, "Inbound model has version " + remote.getProtoVersion() + ", but our version is 0.");
}
if (matchesRemote) {
return remote;
} else if (matchesLocal) {
return local;
} else {
return new SignalContactRecord.Builder(generateKey(), address)
.setGivenName(givenName)
.setFamilyName(familyName)
.setProfileKey(profileKey)
.setUsername(username)
.setIdentityState(identityState)
.setIdentityKey(identityKey)
.setBlocked(blocked)
.setProfileSharingEnabled(profileSharing)
.setNickname(nickname)
.build();
}
}
@VisibleForTesting
static @NonNull SignalGroupV1Record mergeGroupV1(@NonNull SignalGroupV1Record remote,
@NonNull SignalGroupV1Record local)
{
boolean blocked = remote.isBlocked();
boolean profileSharing = remote.isProfileSharingEnabled() || local.isProfileSharingEnabled();
boolean matchesRemote = blocked == remote.isBlocked() && profileSharing == remote.isProfileSharingEnabled();
boolean matchesLocal = blocked == local.isBlocked() && profileSharing == local.isProfileSharingEnabled();
if (matchesRemote) {
return remote;
} else if (matchesLocal) {
return local;
} else {
return new SignalGroupV1Record.Builder(generateKey(), remote.getGroupId())
.setBlocked(blocked)
.setProfileSharingEnabled(blocked)
.build();
}
}
@VisibleForTesting
static void setTestKeyGenerator(@Nullable KeyGenerator keyGenerator) {
testKeyGenerator = keyGenerator;
}
private static IdentityState localToRemoteIdentityState(@NonNull IdentityDatabase.VerifiedStatus local) {
switch (local) {
case VERIFIED: return IdentityState.VERIFIED;
case UNVERIFIED: return IdentityState.UNVERIFIED;
default: return IdentityState.DEFAULT;
}
}
private static boolean doParamsMatchContact(@NonNull SignalContactRecord contact,
@NonNull SignalServiceAddress address,
@Nullable String givenName,
@Nullable String familyName,
@Nullable byte[] profileKey,
@Nullable String username,
@Nullable IdentityState identityState,
@Nullable byte[] identityKey,
boolean blocked,
boolean profileSharing,
@Nullable String nickname)
{
return Objects.equals(contact.getAddress(), address) &&
Objects.equals(contact.getGivenName().or(""), givenName) &&
Objects.equals(contact.getFamilyName().or(""), familyName) &&
Arrays.equals(contact.getProfileKey().orNull(), profileKey) &&
Objects.equals(contact.getUsername().or(""), username) &&
Objects.equals(contact.getIdentityState(), identityState) &&
Arrays.equals(contact.getIdentityKey().orNull(), identityKey) &&
contact.isBlocked() == blocked &&
contact.isProfileSharingEnabled() == profileSharing &&
Objects.equals(contact.getNickname().or(""), nickname);
}
private static @NonNull ContactRecordMergeResult resolveContactConflict(@NonNull Collection<SignalContactRecord> remoteOnlyRecords,
@NonNull Collection<SignalContactRecord> localOnlyRecords)
{
Map<UUID, SignalContactRecord> localByUuid = new HashMap<>();
Map<String, SignalContactRecord> localByE164 = new HashMap<>();
for (SignalContactRecord contact : localOnlyRecords) {
if (contact.getAddress().getUuid().isPresent()) {
localByUuid.put(contact.getAddress().getUuid().get(), contact);
}
if (contact.getAddress().getNumber().isPresent()) {
localByE164.put(contact.getAddress().getNumber().get(), contact);
}
}
Set<SignalContactRecord> localInserts = new LinkedHashSet<>(remoteOnlyRecords);
Set<SignalContactRecord> remoteInserts = new LinkedHashSet<>(localOnlyRecords);
Set<ContactUpdate> localUpdates = new LinkedHashSet<>();
Set<ContactUpdate> remoteUpdates = new LinkedHashSet<>();
for (SignalContactRecord remote : remoteOnlyRecords) {
SignalContactRecord localUuid = remote.getAddress().getUuid().isPresent() ? localByUuid.get(remote.getAddress().getUuid().get()) : null;
SignalContactRecord localE164 = remote.getAddress().getNumber().isPresent() ? localByE164.get(remote.getAddress().getNumber().get()) : null;
Optional<SignalContactRecord> local = Optional.fromNullable(localUuid).or(Optional.fromNullable(localE164));
if (local.isPresent()) {
SignalContactRecord merged = mergeContacts(remote, local.get());
if (!merged.equals(remote)) {
remoteUpdates.add(new ContactUpdate(remote, merged));
}
if (!merged.equals(local.get())) {
localUpdates.add(new ContactUpdate(local.get(), merged));
}
localInserts.remove(remote);
remoteInserts.remove(local.get());
}
}
return new ContactRecordMergeResult(localInserts, localUpdates, remoteInserts, remoteUpdates);
}
private static @NonNull GroupV1RecordMergeResult resolveGroupV1Conflict(@NonNull Collection<SignalGroupV1Record> remoteOnlyRecords,
@NonNull Collection<SignalGroupV1Record> localOnlyRecords)
{
Map<String, SignalGroupV1Record> remoteByGroupId = Stream.of(remoteOnlyRecords).collect(Collectors.toMap(g -> GroupUtil.getEncodedId(g.getGroupId(), false), g -> g));
Map<String, SignalGroupV1Record> localByGroupId = Stream.of(localOnlyRecords).collect(Collectors.toMap(g -> GroupUtil.getEncodedId(g.getGroupId(), false), g -> g));
Set<SignalGroupV1Record> localInserts = new LinkedHashSet<>(remoteOnlyRecords);
Set<SignalGroupV1Record> remoteInserts = new LinkedHashSet<>(localOnlyRecords);
Set<GroupV1Update> localUpdates = new LinkedHashSet<>();
Set<GroupV1Update> remoteUpdates = new LinkedHashSet<>();
for (Map.Entry<String, SignalGroupV1Record> entry : remoteByGroupId.entrySet()) {
SignalGroupV1Record remote = entry.getValue();
SignalGroupV1Record local = localByGroupId.get(entry.getKey());
if (local != null) {
SignalGroupV1Record merged = mergeGroupV1(remote, local);
if (!merged.equals(remote)) {
remoteUpdates.add(new GroupV1Update(remote, merged));
}
if (!merged.equals(local)) {
localUpdates.add(new GroupV1Update(local, merged));
}
localInserts.remove(remote);
remoteInserts.remove(local);
}
}
return new GroupV1RecordMergeResult(localInserts, localUpdates, remoteInserts, remoteUpdates);
}
public static final class ContactUpdate {
private final SignalContactRecord oldContact;
private final SignalContactRecord newContact;
ContactUpdate(@NonNull SignalContactRecord oldContact, @NonNull SignalContactRecord newContact) {
this.oldContact = oldContact;
this.newContact = newContact;
}
public @NonNull SignalContactRecord getOld() {
return oldContact;
}
public @NonNull SignalContactRecord getNew() {
return newContact;
}
public boolean profileKeyChanged() {
return !OptionalUtil.byteArrayEquals(oldContact.getProfileKey(), newContact.getProfileKey());
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ContactUpdate that = (ContactUpdate) o;
return oldContact.equals(that.oldContact) &&
newContact.equals(that.newContact);
}
@Override
public int hashCode() {
return Objects.hash(oldContact, newContact);
}
}
public static final class GroupV1Update {
private final SignalGroupV1Record oldGroup;
private final SignalGroupV1Record newGroup;
public GroupV1Update(@NonNull SignalGroupV1Record oldGroup, @NonNull SignalGroupV1Record newGroup) {
this.oldGroup = oldGroup;
this.newGroup = newGroup;
}
public @NonNull SignalGroupV1Record getOld() {
return oldGroup;
}
public @NonNull SignalGroupV1Record getNew() {
return newGroup;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GroupV1Update that = (GroupV1Update) o;
return oldGroup.equals(that.oldGroup) &&
newGroup.equals(that.newGroup);
}
@Override
public int hashCode() {
return Objects.hash(oldGroup, newGroup);
}
}
@VisibleForTesting
static class RecordUpdate {
private final SignalStorageRecord oldRecord;
private final SignalStorageRecord newRecord;
RecordUpdate(@NonNull SignalStorageRecord oldRecord, @NonNull SignalStorageRecord newRecord) {
this.oldRecord = oldRecord;
this.newRecord = newRecord;
}
public @NonNull SignalStorageRecord getOld() {
return oldRecord;
}
public @NonNull SignalStorageRecord getNew() {
return newRecord;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RecordUpdate that = (RecordUpdate) o;
return oldRecord.equals(that.oldRecord) &&
newRecord.equals(that.newRecord);
}
@Override
public int hashCode() {
return Objects.hash(oldRecord, newRecord);
}
}
public static final class KeyDifferenceResult {
private final List<byte[]> remoteOnlyKeys;
private final List<byte[]> localOnlyKeys;
private KeyDifferenceResult(@NonNull List<byte[]> remoteOnlyKeys, @NonNull List<byte[]> localOnlyKeys) {
this.remoteOnlyKeys = remoteOnlyKeys;
this.localOnlyKeys = localOnlyKeys;
}
public @NonNull List<byte[]> getRemoteOnlyKeys() {
return remoteOnlyKeys;
}
public @NonNull List<byte[]> getLocalOnlyKeys() {
return localOnlyKeys;
}
public boolean isEmpty() {
return remoteOnlyKeys.isEmpty() && localOnlyKeys.isEmpty();
}
}
public static final class MergeResult {
private final Set<SignalContactRecord> localContactInserts;
private final Set<ContactUpdate> localContactUpdates;
private final Set<SignalGroupV1Record> localGroupV1Inserts;
private final Set<GroupV1Update> localGroupV1Updates;
private final Set<SignalStorageRecord> localUnknownInserts;
private final Set<SignalStorageRecord> localUnknownDeletes;
private final Set<SignalStorageRecord> remoteInserts;
private final Set<RecordUpdate> remoteUpdates;
@VisibleForTesting
MergeResult(@NonNull Set<SignalContactRecord> localContactInserts,
@NonNull Set<ContactUpdate> localContactUpdates,
@NonNull Set<SignalGroupV1Record> localGroupV1Inserts,
@NonNull Set<GroupV1Update> localGroupV1Updates,
@NonNull Set<SignalStorageRecord> localUnknownInserts,
@NonNull Set<SignalStorageRecord> localUnknownDeletes,
@NonNull Set<SignalStorageRecord> remoteInserts,
@NonNull Set<RecordUpdate> remoteUpdates)
{
this.localContactInserts = localContactInserts;
this.localContactUpdates = localContactUpdates;
this.localGroupV1Inserts = localGroupV1Inserts;
this.localGroupV1Updates = localGroupV1Updates;
this.localUnknownInserts = localUnknownInserts;
this.localUnknownDeletes = localUnknownDeletes;
this.remoteInserts = remoteInserts;
this.remoteUpdates = remoteUpdates;
}
public @NonNull Set<SignalContactRecord> getLocalContactInserts() {
return localContactInserts;
}
public @NonNull Set<ContactUpdate> getLocalContactUpdates() {
return localContactUpdates;
}
public @NonNull Set<SignalGroupV1Record> getLocalGroupV1Inserts() {
return localGroupV1Inserts;
}
public @NonNull Set<GroupV1Update> getLocalGroupV1Updates() {
return localGroupV1Updates;
}
public @NonNull Set<SignalStorageRecord> getLocalUnknownInserts() {
return localUnknownInserts;
}
public @NonNull Set<SignalStorageRecord> getLocalUnknownDeletes() {
return localUnknownDeletes;
}
public @NonNull Set<SignalStorageRecord> getRemoteInserts() {
return remoteInserts;
}
public @NonNull Set<RecordUpdate> getRemoteUpdates() {
return remoteUpdates;
}
@Override
public @NonNull String toString() {
return String.format(Locale.ENGLISH,
"localContactInserts: %d, localContactUpdates: %d, localGroupInserts: %d, localGroupUpdates: %d, localUnknownInserts: %d, localUnknownDeletes: %d, remoteInserts: %d, remoteUpdates: %d",
localContactInserts.size(), localContactUpdates.size(), localGroupV1Inserts.size(), localGroupV1Updates.size(), localUnknownInserts.size(), localUnknownDeletes.size(), remoteInserts.size(), remoteUpdates.size());
}
}
public static final class WriteOperationResult {
private final SignalStorageManifest manifest;
private final List<SignalStorageRecord> inserts;
private final List<byte[]> deletes;
private WriteOperationResult(@NonNull SignalStorageManifest manifest,
@NonNull List<SignalStorageRecord> inserts,
@NonNull List<byte[]> deletes)
{
this.manifest = manifest;
this.inserts = inserts;
this.deletes = deletes;
}
public @NonNull SignalStorageManifest getManifest() {
return manifest;
}
public @NonNull List<SignalStorageRecord> getInserts() {
return inserts;
}
public @NonNull List<byte[]> getDeletes() {
return deletes;
}
public boolean isEmpty() {
return inserts.isEmpty() && deletes.isEmpty();
}
@Override
public @NonNull String toString() {
return String.format(Locale.ENGLISH,
"ManifestVersion: %d, Total Keys: %d, Inserts: %d, Deletes: %d",
manifest.getVersion(),
manifest.getStorageKeys().size(),
inserts.size(),
deletes.size());
}
}
public static class LocalWriteResult {
private final WriteOperationResult writeResult;
private final Map<RecipientId, byte[]> storageKeyUpdates;
private LocalWriteResult(WriteOperationResult writeResult, Map<RecipientId, byte[]> storageKeyUpdates) {
this.writeResult = writeResult;
this.storageKeyUpdates = storageKeyUpdates;
}
public @NonNull WriteOperationResult getWriteResult() {
return writeResult;
}
public @NonNull Map<RecipientId, byte[]> getStorageKeyUpdates() {
return storageKeyUpdates;
}
}
private static final class ContactRecordMergeResult {
final Set<SignalContactRecord> localInserts;
final Set<ContactUpdate> localUpdates;
final Set<SignalContactRecord> remoteInserts;
final Set<ContactUpdate> remoteUpdates;
ContactRecordMergeResult(@NonNull Set<SignalContactRecord> localInserts,
@NonNull Set<ContactUpdate> localUpdates,
@NonNull Set<SignalContactRecord> remoteInserts,
@NonNull Set<ContactUpdate> remoteUpdates)
{
this.localInserts = localInserts;
this.localUpdates = localUpdates;
this.remoteInserts = remoteInserts;
this.remoteUpdates = remoteUpdates;
}
}
private static final class GroupV1RecordMergeResult {
final Set<SignalGroupV1Record> localInserts;
final Set<GroupV1Update> localUpdates;
final Set<SignalGroupV1Record> remoteInserts;
final Set<GroupV1Update> remoteUpdates;
GroupV1RecordMergeResult(@NonNull Set<SignalGroupV1Record> localInserts,
@NonNull Set<GroupV1Update> localUpdates,
@NonNull Set<SignalGroupV1Record> remoteInserts,
@NonNull Set<GroupV1Update> remoteUpdates)
{
this.localInserts = localInserts;
this.localUpdates = localUpdates;
this.remoteInserts = remoteInserts;
this.remoteUpdates = remoteUpdates;
}
}
interface KeyGenerator {
@NonNull byte[] generate();
}
}

93
app/src/main/java/org/thoughtcrime/securesms/database/RecipientDatabase.java
View File

@@ -17,7 +17,9 @@ import net.sqlcipher.database.SQLiteDatabase;
import org.signal.zkgroup.profiles.ProfileKey;
import org.signal.zkgroup.profiles.ProfileKeyCredential;
import org.thoughtcrime.securesms.color.MaterialColor;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper;
import org.thoughtcrime.securesms.storage.StorageSyncHelper;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.RecordUpdate;
import org.thoughtcrime.securesms.storage.StorageSyncModels;
import org.thoughtcrime.securesms.database.IdentityDatabase.IdentityRecord;
import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
@@ -39,6 +41,7 @@ import org.whispersystems.signalservice.api.profiles.SignalServiceProfile;
import org.whispersystems.signalservice.api.push.SignalServiceAddress;
import org.whispersystems.signalservice.api.storage.SignalContactRecord;
import org.whispersystems.signalservice.api.storage.SignalGroupV1Record;
import org.whispersystems.signalservice.api.storage.StorageId;
import org.whispersystems.signalservice.api.util.UuidUtil;
import java.io.Closeable;
@@ -92,7 +95,7 @@ public class RecipientDatabase extends Database {
private static final String FORCE_SMS_SELECTION = "force_sms_selection";
private static final String UUID_CAPABILITY = "uuid_supported";
private static final String GROUPS_V2_CAPABILITY = "gv2_capability";
private static final String STORAGE_SERVICE_KEY = "storage_service_key";
private static final String STORAGE_SERVICE_ID = "storage_service_key";
private static final String DIRTY = "dirty";
private static final String PROFILE_GIVEN_NAME = "signal_profile_name";
private static final String PROFILE_FAMILY_NAME = "profile_family_name";
@@ -113,7 +116,7 @@ public class RecipientDatabase extends Database {
UNIDENTIFIED_ACCESS_MODE,
FORCE_SMS_SELECTION,
UUID_CAPABILITY, GROUPS_V2_CAPABILITY,
STORAGE_SERVICE_KEY, DIRTY
STORAGE_SERVICE_ID, DIRTY
};
private static final String[] RECIPIENT_FULL_PROJECTION = ArrayUtils.concat(
@@ -282,7 +285,7 @@ public class RecipientDatabase extends Database {
FORCE_SMS_SELECTION + " INTEGER DEFAULT 0, " +
UUID_CAPABILITY + " INTEGER DEFAULT " + Recipient.Capability.UNKNOWN.serialize() + ", " +
GROUPS_V2_CAPABILITY + " INTEGER DEFAULT " + Recipient.Capability.UNKNOWN.serialize() + ", " +
STORAGE_SERVICE_KEY + " TEXT UNIQUE DEFAULT NULL, " +
STORAGE_SERVICE_ID + " TEXT UNIQUE DEFAULT NULL, " +
DIRTY + " INTEGER DEFAULT " + DirtyState.CLEAN.getId() + ");";
private static final String INSIGHTS_INVITEE_LIST = "SELECT " + TABLE_NAME + "." + ID +
@@ -355,7 +358,7 @@ public class RecipientDatabase extends Database {
} else {
values.put(GROUP_TYPE, GroupType.SIGNAL_V1.getId());
values.put(DIRTY, DirtyState.INSERT.getId());
values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(StorageSyncHelper.generateKey()));
values.put(STORAGE_SERVICE_ID, Base64.encodeBytes(StorageSyncHelper.generateKey()));
}
update(result.recipientId, values);
@@ -399,28 +402,28 @@ public class RecipientDatabase extends Database {
}
public @NonNull List<RecipientSettings> getPendingRecipientSyncUpdates() {
String query = DIRTY + " = ? AND " + STORAGE_SERVICE_KEY + " NOT NULL";
String query = DIRTY + " = ? AND " + STORAGE_SERVICE_ID + " NOT NULL";
String[] args = new String[] { String.valueOf(DirtyState.UPDATE.getId()) };
return getRecipientSettings(query, args);
}
public @NonNull List<RecipientSettings> getPendingRecipientSyncInsertions() {
String query = DIRTY + " = ? AND " + STORAGE_SERVICE_KEY + " NOT NULL";
String query = DIRTY + " = ? AND " + STORAGE_SERVICE_ID + " NOT NULL";
String[] args = new String[] { String.valueOf(DirtyState.INSERT.getId()) };
return getRecipientSettings(query, args);
}
public @NonNull List<RecipientSettings> getPendingRecipientSyncDeletions() {
String query = DIRTY + " = ? AND " + STORAGE_SERVICE_KEY + " NOT NULL";
String query = DIRTY + " = ? AND " + STORAGE_SERVICE_ID + " NOT NULL";
String[] args = new String[] { String.valueOf(DirtyState.DELETE.getId()) };
return getRecipientSettings(query, args);
}
public @Nullable RecipientSettings getByStorageSyncKey(@NonNull byte[] key) {
List<RecipientSettings> result = getRecipientSettings(STORAGE_SERVICE_KEY + " = ?", new String[] { Base64.encodeBytes(key) });
public @Nullable RecipientSettings getByStorageId(@NonNull byte[] storageId) {
List<RecipientSettings> result = getRecipientSettings(STORAGE_SERVICE_ID + " = ?", new String[] { Base64.encodeBytes(storageId) });
if (result.size() > 0) {
return result.get(0);
@@ -429,16 +432,16 @@ public class RecipientDatabase extends Database {
return null;
}
public void applyStorageSyncKeyUpdates(@NonNull Map<RecipientId, byte[]> keys) {
public void applyStorageIdUpdates(@NonNull Map<RecipientId, StorageId> storageIds) {
SQLiteDatabase db = databaseHelper.getWritableDatabase();
db.beginTransaction();
try {
String query = ID + " = ?";
for (Map.Entry<RecipientId, byte[]> entry : keys.entrySet()) {
for (Map.Entry<RecipientId, StorageId> entry : storageIds.entrySet()) {
ContentValues values = new ContentValues();
values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(entry.getValue()));
values.put(STORAGE_SERVICE_ID, Base64.encodeBytes(entry.getValue().getRaw()));
values.put(DIRTY, DirtyState.CLEAN.getId());
db.update(TABLE_NAME, values, query, new String[] { entry.getKey().serialize() });
@@ -449,10 +452,10 @@ public class RecipientDatabase extends Database {
}
}
public void applyStorageSyncUpdates(@NonNull Collection<SignalContactRecord> contactInserts,
@NonNull Collection<StorageSyncHelper.ContactUpdate> contactUpdates,
@NonNull Collection<SignalGroupV1Record> groupV1Inserts,
@NonNull Collection<StorageSyncHelper.GroupV1Update> groupV1Updates)
public void applyStorageSyncUpdates(@NonNull Collection<SignalContactRecord> contactInserts,
@NonNull Collection<RecordUpdate<SignalContactRecord>> contactUpdates,
@NonNull Collection<SignalGroupV1Record> groupV1Inserts,
@NonNull Collection<RecordUpdate<SignalGroupV1Record>> groupV1Updates)
{
SQLiteDatabase db = databaseHelper.getWritableDatabase();
IdentityDatabase identityDatabase = DatabaseFactory.getIdentityDatabase(context);
@@ -482,7 +485,7 @@ public class RecipientDatabase extends Database {
try {
IdentityKey identityKey = new IdentityKey(insert.getIdentityKey().get(), 0);
DatabaseFactory.getIdentityDatabase(context).updateIdentityAfterSync(recipientId, identityKey, StorageSyncHelper.remoteToLocalIdentityStatus(insert.getIdentityState()));
DatabaseFactory.getIdentityDatabase(context).updateIdentityAfterSync(recipientId, identityKey, StorageSyncModels.remoteToLocalIdentityStatus(insert.getIdentityState()));
IdentityUtil.markIdentityVerified(context, Recipient.resolved(recipientId), true, true);
} catch (InvalidKeyException e) {
Log.w(TAG, "Failed to process identity key during insert! Skipping.", e);
@@ -495,17 +498,17 @@ public class RecipientDatabase extends Database {
}
}
for (StorageSyncHelper.ContactUpdate update : contactUpdates) {
for (RecordUpdate<SignalContactRecord> update : contactUpdates) {
ContentValues values = getValuesForStorageContact(update.getNew());
int updateCount = db.update(TABLE_NAME, values, STORAGE_SERVICE_KEY + " = ?", new String[]{Base64.encodeBytes(update.getOld().getKey())});
int updateCount = db.update(TABLE_NAME, values, STORAGE_SERVICE_ID + " = ?", new String[]{Base64.encodeBytes(update.getOld().getId().getRaw())});
if (updateCount < 1) {
throw new AssertionError("Had an update, but it didn't match any rows!");
}
RecipientId recipientId = getByStorageKeyOrThrow(update.getNew().getKey());
RecipientId recipientId = getByStorageKeyOrThrow(update.getNew().getId().getRaw());
if (update.profileKeyChanged()) {
if (StorageSyncHelper.profileKeyChanged(update)) {
clearProfileKeyCredential(recipientId);
}
@@ -514,7 +517,7 @@ public class RecipientDatabase extends Database {
if (update.getNew().getIdentityKey().isPresent()) {
IdentityKey identityKey = new IdentityKey(update.getNew().getIdentityKey().get(), 0);
DatabaseFactory.getIdentityDatabase(context).updateIdentityAfterSync(recipientId, identityKey, StorageSyncHelper.remoteToLocalIdentityStatus(update.getNew().getIdentityState()));
DatabaseFactory.getIdentityDatabase(context).updateIdentityAfterSync(recipientId, identityKey, StorageSyncModels.remoteToLocalIdentityStatus(update.getNew().getIdentityState()));
}
Optional<IdentityRecord> newIdentityRecord = identityDatabase.getIdentity(recipientId);
@@ -537,9 +540,9 @@ public class RecipientDatabase extends Database {
db.insertOrThrow(TABLE_NAME, null, getValuesForStorageGroupV1(insert));
}
for (StorageSyncHelper.GroupV1Update update : groupV1Updates) {
for (RecordUpdate<SignalGroupV1Record> update : groupV1Updates) {
ContentValues values = getValuesForStorageGroupV1(update.getNew());
int updateCount = db.update(TABLE_NAME, values, STORAGE_SERVICE_KEY + " = ?", new String[]{Base64.encodeBytes(update.getOld().getKey())});
int updateCount = db.update(TABLE_NAME, values, STORAGE_SERVICE_ID + " = ?", new String[]{Base64.encodeBytes(update.getOld().getId().getRaw())});
if (updateCount < 1) {
throw new AssertionError("Had an update, but it didn't match any rows!");
@@ -576,7 +579,7 @@ public class RecipientDatabase extends Database {
private @NonNull RecipientId getByStorageKeyOrThrow(byte[] storageKey) {
SQLiteDatabase db = databaseHelper.getReadableDatabase();
String query = STORAGE_SERVICE_KEY + " = ?";
String query = STORAGE_SERVICE_ID + " = ?";
String[] args = new String[]{Base64.encodeBytes(storageKey)};
try (Cursor cursor = db.query(TABLE_NAME, ID_PROJECTION, query, args, null, null, null)) {
@@ -607,7 +610,7 @@ public class RecipientDatabase extends Database {
values.put(USERNAME, TextUtils.isEmpty(username) ? null : username);
values.put(PROFILE_SHARING, contact.isProfileSharingEnabled() ? "1" : "0");
values.put(BLOCKED, contact.isBlocked() ? "1" : "0");
values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(contact.getKey()));
values.put(STORAGE_SERVICE_ID, Base64.encodeBytes(contact.getId().getRaw()));
values.put(DIRTY, DirtyState.CLEAN.getId());
return values;
}
@@ -618,7 +621,7 @@ public class RecipientDatabase extends Database {
values.put(GROUP_TYPE, GroupType.SIGNAL_V1.getId());
values.put(PROFILE_SHARING, groupV1.isProfileSharingEnabled() ? "1" : "0");
values.put(BLOCKED, groupV1.isBlocked() ? "1" : "0");
values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(groupV1.getKey()));
values.put(STORAGE_SERVICE_ID, Base64.encodeBytes(groupV1.getId().getRaw()));
values.put(DIRTY, DirtyState.CLEAN.getId());
return values;
}
@@ -640,25 +643,31 @@ public class RecipientDatabase extends Database {
/**
* @return All storage keys, excluding the ones that need to be deleted.
*/
public List<byte[]> getAllStorageSyncKeys() {
public List<StorageId> getAllStorageSyncKeys() {
return new ArrayList<>(getAllStorageSyncKeysMap().values());
}
/**
* @return All storage keys, excluding the ones that need to be deleted.
*/
public Map<RecipientId, byte[]> getAllStorageSyncKeysMap() {
SQLiteDatabase db = databaseHelper.getReadableDatabase();
String query = STORAGE_SERVICE_KEY + " NOT NULL AND " + DIRTY + " != ?";
String[] args = new String[]{String.valueOf(DirtyState.DELETE)};
Map<RecipientId, byte[]> out = new HashMap<>();
public @NonNull Map<RecipientId, StorageId> getAllStorageSyncKeysMap() {
SQLiteDatabase db = databaseHelper.getReadableDatabase();
String query = STORAGE_SERVICE_ID + " NOT NULL AND " + DIRTY + " != ?";
String[] args = new String[]{String.valueOf(DirtyState.DELETE)};
Map<RecipientId, StorageId> out = new HashMap<>();
try (Cursor cursor = db.query(TABLE_NAME, new String[] { ID, STORAGE_SERVICE_KEY }, query, args, null, null, null)) {
try (Cursor cursor = db.query(TABLE_NAME, new String[] { ID, STORAGE_SERVICE_ID, GROUP_TYPE }, query, args, null, null, null)) {
while (cursor != null && cursor.moveToNext()) {
RecipientId id = RecipientId.from(cursor.getLong(cursor.getColumnIndexOrThrow(ID)));
String encodedKey = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_SERVICE_KEY));
String encodedKey = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_SERVICE_ID));
GroupType groupType = GroupType.fromId(cursor.getInt(cursor.getColumnIndexOrThrow(GROUP_TYPE)));
byte[] key = Base64.decodeOrThrow(encodedKey);
out.put(id, Base64.decodeOrThrow(encodedKey));
if (groupType == GroupType.NONE) {
out.put(id, StorageId.forContact(key));
} else {
out.put(id, StorageId.forGroupV1(key));
}
}
}
@@ -699,7 +708,7 @@ public class RecipientDatabase extends Database {
boolean forceSmsSelection = cursor.getInt(cursor.getColumnIndexOrThrow(FORCE_SMS_SELECTION)) == 1;
int uuidCapabilityValue = cursor.getInt(cursor.getColumnIndexOrThrow(UUID_CAPABILITY));
int groupsV2CapabilityValue = cursor.getInt(cursor.getColumnIndexOrThrow(GROUPS_V2_CAPABILITY));
String storageKeyRaw = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_SERVICE_KEY));
String storageKeyRaw = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_SERVICE_ID));
String identityKeyRaw = cursor.getString(cursor.getColumnIndexOrThrow(IDENTITY_KEY));
int identityStatusRaw = cursor.getInt(cursor.getColumnIndexOrThrow(IDENTITY_STATUS));
@@ -1106,7 +1115,7 @@ public class RecipientDatabase extends Database {
contentValues.put(REGISTERED, registeredState.getId());
if (registeredState == RegisteredState.REGISTERED) {
contentValues.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(StorageSyncHelper.generateKey()));
contentValues.put(STORAGE_SERVICE_ID, Base64.encodeBytes(StorageSyncHelper.generateKey()));
}
if (update(id, contentValues)) {
@@ -1357,7 +1366,7 @@ public class RecipientDatabase extends Database {
try {
for (Map.Entry<RecipientId, byte[]> entry : keys.entrySet()) {
ContentValues values = new ContentValues();
values.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(entry.getValue()));
values.put(STORAGE_SERVICE_ID, Base64.encodeBytes(entry.getValue()));
db.update(TABLE_NAME, values, ID_WHERE, new String[] { entry.getKey().serialize() });
}
@@ -1397,7 +1406,7 @@ public class RecipientDatabase extends Database {
query += "(" + DIRTY + " < ? OR " + DIRTY + " = ?)";
args = SqlUtil.appendArg(args, String.valueOf(DirtyState.DELETE.getId()));
contentValues.put(STORAGE_SERVICE_KEY, Base64.encodeBytes(StorageSyncHelper.generateKey()));
contentValues.put(STORAGE_SERVICE_ID, Base64.encodeBytes(StorageSyncHelper.generateKey()));
break;
case DELETE:
query += "(" + DIRTY + " < ? OR " + DIRTY + " = ?)";
@@ -1526,7 +1535,7 @@ public class RecipientDatabase extends Database {
}
private void markAllRelevantEntriesDirty() {
String query = SYSTEM_INFO_PENDING + " = ? AND " + STORAGE_SERVICE_KEY + " NOT NULL AND " + DIRTY + " < ?";
String query = SYSTEM_INFO_PENDING + " = ? AND " + STORAGE_SERVICE_ID + " NOT NULL AND " + DIRTY + " < ?";
String[] args = new String[] { "1", String.valueOf(DirtyState.UPDATE.getId()) };
ContentValues values = new ContentValues(1);

33
app/src/main/java/org/thoughtcrime/securesms/database/StorageKeyDatabase.java
View File

@@ -12,11 +12,11 @@ import net.sqlcipher.database.SQLiteDatabase;
import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
import org.thoughtcrime.securesms.util.Base64;
import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
import org.whispersystems.signalservice.api.storage.StorageId;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
@@ -28,11 +28,11 @@ public class StorageKeyDatabase extends Database {
private static final String TABLE_NAME = "storage_key";
private static final String ID = "_id";
private static final String TYPE = "type";
private static final String KEY = "key";
private static final String STORAGE_ID = "key";
public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ID + " INTEGER PRIMARY KEY AUTOINCREMENT, " +
TYPE + " INTEGER, " +
KEY + " TEXT UNIQUE)";
public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ID + " INTEGER PRIMARY KEY AUTOINCREMENT, " +
TYPE + " INTEGER, " +
STORAGE_ID + " TEXT UNIQUE)";
public static final String[] CREATE_INDEXES = new String[] {
"CREATE INDEX IF NOT EXISTS storage_key_type_index ON " + TABLE_NAME + " (" + TYPE + ");"
@@ -42,14 +42,15 @@ public class StorageKeyDatabase extends Database {
super(context, databaseHelper);
}
public List<byte[]> getAllKeys() {
List<byte[]> keys = new ArrayList<>();
public List<StorageId> getAllKeys() {
List<StorageId> keys = new ArrayList<>();
try (Cursor cursor = databaseHelper.getReadableDatabase().query(TABLE_NAME, null, null, null, null, null, null)) {
while (cursor != null && cursor.moveToNext()) {
String keyEncoded = cursor.getString(cursor.getColumnIndexOrThrow(KEY));
String keyEncoded = cursor.getString(cursor.getColumnIndexOrThrow(STORAGE_ID));
int type = cursor.getInt(cursor.getColumnIndexOrThrow(TYPE));
try {
keys.add(Base64.decode(keyEncoded));
keys.add(StorageId.forType(Base64.decode(keyEncoded), type));
} catch (IOException e) {
throw new AssertionError(e);
}
@@ -59,14 +60,14 @@ public class StorageKeyDatabase extends Database {
return keys;
}
public @Nullable SignalStorageRecord getByKey(@NonNull byte[] key) {
String query = KEY + " = ?";
String[] args = new String[] { Base64.encodeBytes(key) };
public @Nullable SignalStorageRecord getById(@NonNull byte[] rawId) {
String query = STORAGE_ID + " = ?";
String[] args = new String[] { Base64.encodeBytes(rawId) };
try (Cursor cursor = databaseHelper.getReadableDatabase().query(TABLE_NAME, null, query, args, null, null, null)) {
if (cursor != null && cursor.moveToFirst()) {
int type = cursor.getInt(cursor.getColumnIndexOrThrow(TYPE));
return SignalStorageRecord.forUnknown(key, type);
return SignalStorageRecord.forUnknown(StorageId.forType(rawId, type));
} else {
return null;
}
@@ -83,15 +84,15 @@ public class StorageKeyDatabase extends Database {
for (SignalStorageRecord insert : inserts) {
ContentValues values = new ContentValues();
values.put(TYPE, insert.getType());
values.put(KEY, Base64.encodeBytes(insert.getKey()));
values.put(STORAGE_ID, Base64.encodeBytes(insert.getId().getRaw()));
db.insert(TABLE_NAME, null, values);
}
String deleteQuery = KEY + " = ?";
String deleteQuery = STORAGE_ID + " = ?";
for (SignalStorageRecord delete : deletes) {
String[] args = new String[] { Base64.encodeBytes(delete.getKey()) };
String[] args = new String[] { Base64.encodeBytes(delete.getId().getRaw()) };
db.delete(TABLE_NAME, deleteQuery, args);
}

3
app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java
View File

@@ -21,7 +21,7 @@ import net.sqlcipher.database.SQLiteDatabase;
import net.sqlcipher.database.SQLiteDatabaseHook;
import net.sqlcipher.database.SQLiteOpenHelper;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper;
import org.thoughtcrime.securesms.storage.StorageSyncHelper;
import org.thoughtcrime.securesms.crypto.DatabaseSecret;
import org.thoughtcrime.securesms.crypto.MasterSecret;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
@@ -57,7 +57,6 @@ import org.thoughtcrime.securesms.util.TextSecurePreferences;
import org.thoughtcrime.securesms.util.Util;
import java.io.File;
import java.io.FilenameFilter;
import java.util.List;
public class SQLCipherOpenHelper extends SQLiteOpenHelper {

29
app/src/main/java/org/thoughtcrime/securesms/jobs/StorageForcePushJob.java
View File

@@ -4,7 +4,8 @@ import androidx.annotation.NonNull;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper;
import org.thoughtcrime.securesms.storage.StorageSyncHelper;
import org.thoughtcrime.securesms.storage.StorageSyncModels;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.database.RecipientDatabase;
import org.thoughtcrime.securesms.database.StorageKeyDatabase;
@@ -16,12 +17,10 @@ import org.thoughtcrime.securesms.keyvalue.SignalStore;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.recipients.RecipientId;
import org.thoughtcrime.securesms.transport.RetryLaterException;
import org.thoughtcrime.securesms.util.FeatureFlags;
import org.thoughtcrime.securesms.util.TextSecurePreferences;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.InvalidKeyException;
import org.whispersystems.signalservice.api.SignalServiceAccountManager;
import org.whispersystems.signalservice.api.kbs.MasterKey;
import org.whispersystems.signalservice.api.storage.StorageId;
import org.whispersystems.signalservice.api.storage.StorageKey;
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException;
import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
@@ -76,15 +75,15 @@ public class StorageForcePushJob extends BaseJob {
RecipientDatabase recipientDatabase = DatabaseFactory.getRecipientDatabase(context);
StorageKeyDatabase storageKeyDatabase = DatabaseFactory.getStorageKeyDatabase(context);
long currentVersion = accountManager.getStorageManifestVersion();
Map<RecipientId, byte[]> oldStorageKeys = recipientDatabase.getAllStorageSyncKeysMap();
long currentVersion = accountManager.getStorageManifestVersion();
Map<RecipientId, StorageId> oldStorageKeys = recipientDatabase.getAllStorageSyncKeysMap();
long newVersion = currentVersion + 1;
Map<RecipientId, byte[]> newStorageKeys = generateNewKeys(oldStorageKeys);
List<SignalStorageRecord> inserts = Stream.of(oldStorageKeys.keySet())
long newVersion = currentVersion + 1;
Map<RecipientId, StorageId> newStorageKeys = generateNewKeys(oldStorageKeys);
List<SignalStorageRecord> inserts = Stream.of(oldStorageKeys.keySet())
.map(recipientDatabase::getRecipientSettings)
.withoutNulls()
.map(s -> StorageSyncHelper.localToRemoteRecord(s, Objects.requireNonNull(newStorageKeys.get(s.getId()))))
.map(s -> StorageSyncModels.localToRemoteRecord(s, Objects.requireNonNull(newStorageKeys.get(s.getId())).getRaw()))
.toList();
SignalStorageManifest manifest = new SignalStorageManifest(newVersion, new ArrayList<>(newStorageKeys.values()));
@@ -110,7 +109,7 @@ public class StorageForcePushJob extends BaseJob {
Log.i(TAG, "Force push succeeded. Updating local manifest version to: " + newVersion);
TextSecurePreferences.setStorageManifestVersion(context, newVersion);
recipientDatabase.applyStorageSyncKeyUpdates(newStorageKeys);
recipientDatabase.applyStorageIdUpdates(newStorageKeys);
storageKeyDatabase.deleteAll();
}
@@ -123,11 +122,11 @@ public class StorageForcePushJob extends BaseJob {
public void onFailure() {
}
private static @NonNull Map<RecipientId, byte[]> generateNewKeys(@NonNull Map<RecipientId, byte[]> oldKeys) {
Map<RecipientId, byte[]> out = new HashMap<>();
private static @NonNull Map<RecipientId, StorageId> generateNewKeys(@NonNull Map<RecipientId, StorageId> oldKeys) {
Map<RecipientId, StorageId> out = new HashMap<>();
for (Map.Entry<RecipientId, byte[]> entry : oldKeys.entrySet()) {
out.put(entry.getKey(), StorageSyncHelper.generateKey());
for (Map.Entry<RecipientId, StorageId> entry : oldKeys.entrySet()) {
out.put(entry.getKey(), entry.getValue().withNewBytes(StorageSyncHelper.generateKey()));
}
return out;

37
app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.java
View File

@@ -6,11 +6,12 @@ import androidx.annotation.NonNull;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.KeyDifferenceResult;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.LocalWriteResult;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.MergeResult;
import org.thoughtcrime.securesms.contacts.sync.StorageSyncHelper.WriteOperationResult;
import org.thoughtcrime.securesms.storage.StorageSyncHelper;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.KeyDifferenceResult;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.LocalWriteResult;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.MergeResult;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.WriteOperationResult;
import org.thoughtcrime.securesms.storage.StorageSyncModels;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.database.RecipientDatabase;
import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings;
@@ -29,6 +30,7 @@ import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.InvalidKeyException;
import org.whispersystems.libsignal.util.guava.Optional;
import org.whispersystems.signalservice.api.SignalServiceAccountManager;
import org.whispersystems.signalservice.api.storage.StorageId;
import org.whispersystems.signalservice.api.storage.StorageKey;
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException;
import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
@@ -36,7 +38,6 @@ import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
@@ -145,8 +146,8 @@ public class StorageSyncJob extends BaseJob {
if (remoteManifest.isPresent() && remoteManifestVersion > localManifestVersion) {
Log.i(TAG, "[Remote Newer] Newer manifest version found!");
List<byte[]> allLocalStorageKeys = getAllLocalStorageKeys(context);
KeyDifferenceResult keyDifference = StorageSyncHelper.findKeyDifference(remoteManifest.get().getStorageKeys(), allLocalStorageKeys);
List<StorageId> allLocalStorageKeys = getAllLocalStorageKeys(context);
KeyDifferenceResult keyDifference = StorageSyncHelper.findKeyDifference(remoteManifest.get().getStorageIds(), allLocalStorageKeys);
if (!keyDifference.isEmpty()) {
Log.i(TAG, "[Remote Newer] There's a difference in keys. Local-only: " + keyDifference.getLocalOnlyKeys().size() + ", Remote-only: " + keyDifference.getRemoteOnlyKeys().size());
@@ -162,9 +163,9 @@ public class StorageSyncJob extends BaseJob {
Log.i(TAG, "[Remote Newer] WriteOperationResult :: " + writeOperationResult);
Log.i(TAG, "[Remote Newer] We have something to write remotely.");
if (writeOperationResult.getManifest().getStorageKeys().size() != remoteManifest.get().getStorageKeys().size() + writeOperationResult.getInserts().size() - writeOperationResult.getDeletes().size()) {
if (writeOperationResult.getManifest().getStorageIds().size() != remoteManifest.get().getStorageIds().size() + writeOperationResult.getInserts().size() - writeOperationResult.getDeletes().size()) {
Log.w(TAG, String.format(Locale.ENGLISH, "Bad storage key management! originalRemoteKeys: %d, newRemoteKeys: %d, insertedKeys: %d, deletedKeys: %d",
remoteManifest.get().getStorageKeys().size(), writeOperationResult.getManifest().getStorageKeys().size(), writeOperationResult.getInserts().size(), writeOperationResult.getDeletes().size()));
remoteManifest.get().getStorageIds().size(), writeOperationResult.getManifest().getStorageIds().size(), writeOperationResult.getInserts().size(), writeOperationResult.getDeletes().size()));
}
Optional<SignalStorageManifest> conflict = accountManager.writeStorageRecords(storageServiceKey, writeOperationResult.getManifest(), writeOperationResult.getInserts(), writeOperationResult.getDeletes());
@@ -194,7 +195,7 @@ public class StorageSyncJob extends BaseJob {
localManifestVersion = TextSecurePreferences.getStorageManifestVersion(context);
List<byte[]> allLocalStorageKeys = recipientDatabase.getAllStorageSyncKeys();
List<StorageId> allLocalStorageKeys = recipientDatabase.getAllStorageSyncKeys();
List<RecipientSettings> pendingUpdates = recipientDatabase.getPendingRecipientSyncUpdates();
List<RecipientSettings> pendingInsertions = recipientDatabase.getPendingRecipientSyncInsertions();
List<RecipientSettings> pendingDeletions = recipientDatabase.getPendingRecipientSyncDeletions();
@@ -242,21 +243,21 @@ public class StorageSyncJob extends BaseJob {
return needsMultiDeviceSync;
}
private static @NonNull List<byte[]> getAllLocalStorageKeys(@NonNull Context context) {
private static @NonNull List<StorageId> getAllLocalStorageKeys(@NonNull Context context) {
return Util.concatenatedList(DatabaseFactory.getRecipientDatabase(context).getAllStorageSyncKeys(),
DatabaseFactory.getStorageKeyDatabase(context).getAllKeys());
}
private static @NonNull List<SignalStorageRecord> buildLocalStorageRecords(@NonNull Context context, @NonNull List<byte[]> keys) {
private static @NonNull List<SignalStorageRecord> buildLocalStorageRecords(@NonNull Context context, @NonNull List<StorageId> ids) {
RecipientDatabase recipientDatabase = DatabaseFactory.getRecipientDatabase(context);
StorageKeyDatabase storageKeyDatabase = DatabaseFactory.getStorageKeyDatabase(context);
List<SignalStorageRecord> records = new ArrayList<>(keys.size());
List<SignalStorageRecord> records = new ArrayList<>(ids.size());
for (byte[] key : keys) {
SignalStorageRecord record = Optional.fromNullable(recipientDatabase.getByStorageSyncKey(key))
.transform(StorageSyncHelper::localToRemoteRecord)
.or(() -> storageKeyDatabase.getByKey(key));
for (StorageId id : ids) {
SignalStorageRecord record = Optional.fromNullable(recipientDatabase.getByStorageId(id.getRaw()))
.transform(StorageSyncModels::localToRemoteRecord)
.or(() -> storageKeyDatabase.getById(id.getRaw()));
records.add(record);
}

109
app/src/main/java/org/thoughtcrime/securesms/storage/ContactConflictMerger.java Normal file
View File

@@ -0,0 +1,109 @@
package org.thoughtcrime.securesms.storage;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.whispersystems.libsignal.util.guava.Optional;
import org.whispersystems.signalservice.api.push.SignalServiceAddress;
import org.whispersystems.signalservice.api.storage.SignalContactRecord;
import org.whispersystems.signalservice.internal.storage.protos.ContactRecord.IdentityState;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
class ContactConflictMerger implements StorageSyncHelper.ConflictMerger<SignalContactRecord> {
private final Map<UUID, SignalContactRecord> localByUuid = new HashMap<>();
private final Map<String, SignalContactRecord> localByE164 = new HashMap<>();
ContactConflictMerger(@NonNull Collection<SignalContactRecord> localOnly) {
for (SignalContactRecord contact : localOnly) {
if (contact.getAddress().getUuid().isPresent()) {
localByUuid.put(contact.getAddress().getUuid().get(), contact);
}
if (contact.getAddress().getNumber().isPresent()) {
localByE164.put(contact.getAddress().getNumber().get(), contact);
}
}
}
@Override
public @NonNull Optional<SignalContactRecord> getMatching(@NonNull SignalContactRecord record) {
SignalContactRecord localUuid = record.getAddress().getUuid().isPresent() ? localByUuid.get(record.getAddress().getUuid().get()) : null;
SignalContactRecord localE164 = record.getAddress().getNumber().isPresent() ? localByE164.get(record.getAddress().getNumber().get()) : null;
return Optional.fromNullable(localUuid).or(Optional.fromNullable(localE164));
}
@Override
public @NonNull SignalContactRecord merge(@NonNull SignalContactRecord remote, @NonNull SignalContactRecord local, @NonNull StorageSyncHelper.KeyGenerator keyGenerator) {
String givenName;
String familyName;
if (remote.getGivenName().isPresent() || remote.getFamilyName().isPresent()) {
givenName = remote.getGivenName().or("");
familyName = remote.getFamilyName().or("");
} else {
givenName = local.getGivenName().or("");
familyName = local.getFamilyName().or("");
}
UUID uuid = remote.getAddress().getUuid().or(local.getAddress().getUuid()).orNull();
String e164 = remote.getAddress().getNumber().or(local.getAddress().getNumber()).orNull();
SignalServiceAddress address = new SignalServiceAddress(uuid, e164);
byte[] profileKey = remote.getProfileKey().or(local.getProfileKey()).orNull();
String username = remote.getUsername().or(local.getUsername()).or("");
IdentityState identityState = remote.getIdentityState();
byte[] identityKey = remote.getIdentityKey().or(local.getIdentityKey()).orNull();
boolean blocked = remote.isBlocked();
boolean profileSharing = remote.isProfileSharingEnabled() || local.isProfileSharingEnabled();
boolean archived = remote.isArchived();
boolean matchesRemote = doParamsMatch(remote, address, givenName, familyName, profileKey, username, identityState, identityKey, blocked, profileSharing, archived);
boolean matchesLocal = doParamsMatch(local, address, givenName, familyName, profileKey, username, identityState, identityKey, blocked, profileSharing, archived);
if (matchesRemote) {
return remote;
} else if (matchesLocal) {
return local;
} else {
return new SignalContactRecord.Builder(keyGenerator.generate(), address)
.setGivenName(givenName)
.setFamilyName(familyName)
.setProfileKey(profileKey)
.setUsername(username)
.setIdentityState(identityState)
.setIdentityKey(identityKey)
.setBlocked(blocked)
.setProfileSharingEnabled(profileSharing)
.build();
}
}
private static boolean doParamsMatch(@NonNull SignalContactRecord contact,
@NonNull SignalServiceAddress address,
@NonNull String givenName,
@NonNull String familyName,
@Nullable byte[] profileKey,
@NonNull String username,
@Nullable IdentityState identityState,
@Nullable byte[] identityKey,
boolean blocked,
boolean profileSharing,
boolean archived)
{
return Objects.equals(contact.getAddress(), address) &&
Objects.equals(contact.getGivenName().or(""), givenName) &&
Objects.equals(contact.getFamilyName().or(""), familyName) &&
Arrays.equals(contact.getProfileKey().orNull(), profileKey) &&
Objects.equals(contact.getUsername().or(""), username) &&
Objects.equals(contact.getIdentityState(), identityState) &&
Arrays.equals(contact.getIdentityKey().orNull(), identityKey) &&
contact.isBlocked() == blocked &&
contact.isProfileSharingEnabled() == profileSharing &&
contact.isArchived() == archived;
}
}

48
app/src/main/java/org/thoughtcrime/securesms/storage/GroupV1ConflictMerger.java Normal file
View File

@@ -0,0 +1,48 @@
package org.thoughtcrime.securesms.storage;
import androidx.annotation.NonNull;
import com.annimon.stream.Collectors;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.util.GroupUtil;
import org.whispersystems.libsignal.util.guava.Optional;
import org.whispersystems.signalservice.api.storage.SignalGroupV1Record;
import java.util.Collection;
import java.util.Map;
class GroupV1ConflictMerger implements StorageSyncHelper.ConflictMerger<SignalGroupV1Record> {
private final Map<String, SignalGroupV1Record> localByGroupId;
GroupV1ConflictMerger(@NonNull Collection<SignalGroupV1Record> localOnly) {
localByGroupId = Stream.of(localOnly).collect(Collectors.toMap(g -> GroupUtil.getEncodedId(g.getGroupId(), false), g -> g));
}
@Override
public @NonNull Optional<SignalGroupV1Record> getMatching(@NonNull SignalGroupV1Record record) {
return Optional.fromNullable(localByGroupId.get(GroupUtil.getEncodedId(record.getGroupId(), false)));
}
@Override
public @NonNull SignalGroupV1Record merge(@NonNull SignalGroupV1Record remote, @NonNull SignalGroupV1Record local, @NonNull StorageSyncHelper.KeyGenerator keyGenerator) {
boolean blocked = remote.isBlocked();
boolean profileSharing = remote.isProfileSharingEnabled() || local.isProfileSharingEnabled();
boolean archived = remote.isArchived();
boolean matchesRemote = blocked == remote.isBlocked() && profileSharing == remote.isProfileSharingEnabled() && archived == remote.isArchived();
boolean matchesLocal = blocked == local.isBlocked() && profileSharing == local.isProfileSharingEnabled() && archived == local.isArchived();
if (matchesRemote) {
return remote;
} else if (matchesLocal) {
return local;
} else {
return new SignalGroupV1Record.Builder(keyGenerator.generate(), remote.getGroupId())
.setBlocked(blocked)
.setProfileSharingEnabled(blocked)
.build();
}
}
}

474
app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.java Normal file
View File

@@ -0,0 +1,474 @@
package org.thoughtcrime.securesms.storage;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.recipients.RecipientId;
import org.thoughtcrime.securesms.util.SetUtil;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.util.guava.Optional;
import org.whispersystems.signalservice.api.storage.SignalContactRecord;
import org.whispersystems.signalservice.api.storage.SignalGroupV1Record;
import org.whispersystems.signalservice.api.storage.SignalRecord;
import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
import org.whispersystems.signalservice.api.storage.StorageId;
import org.whispersystems.signalservice.api.util.OptionalUtil;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public final class StorageSyncHelper {
private static final String TAG = Log.tag(StorageSyncHelper.class);
private static final KeyGenerator KEY_GENERATOR = () -> Util.getSecretBytes(16);
private static KeyGenerator keyGenerator = KEY_GENERATOR;
/**
* Given the local state of pending storage mutations, this will generate a result that will
* include that data that needs to be written to the storage service, as well as any changes you
* need to write back to local storage (like storage keys that might have changed for updated
* contacts).
*
* @param currentManifestVersion What you think the version is locally.
* @param currentLocalKeys All local keys you have. This assumes that 'inserts' were given keys
* already, and that deletes still have keys.
* @param updates Contacts that have been altered.
* @param inserts Contacts that have been inserted (or newly marked as registered).
* @param deletes Contacts that are no longer registered.
*
* @return If changes need to be written, then it will return those changes. If no changes need
* to be written, this will return {@link Optional#absent()}.
*/
public static @NonNull Optional<LocalWriteResult> buildStorageUpdatesForLocal(long currentManifestVersion,
@NonNull List<StorageId> currentLocalKeys,
@NonNull List<RecipientSettings> updates,
@NonNull List<RecipientSettings> inserts,
@NonNull List<RecipientSettings> deletes)
{
Set<StorageId> completeKeys = new LinkedHashSet<>(currentLocalKeys);
Set<SignalStorageRecord> storageInserts = new LinkedHashSet<>();
Set<ByteBuffer> storageDeletes = new LinkedHashSet<>();
Map<RecipientId, byte[]> storageKeyUpdates = new HashMap<>();
for (RecipientSettings insert : inserts) {
storageInserts.add(StorageSyncModels.localToRemoteRecord(insert));
}
for (RecipientSettings delete : deletes) {
byte[] key = Objects.requireNonNull(delete.getStorageKey());
storageDeletes.add(ByteBuffer.wrap(key));
completeKeys.remove(StorageId.forContact(key));
}
for (RecipientSettings update : updates) {
byte[] oldKey = Objects.requireNonNull(update.getStorageKey());
byte[] newKey = generateKey();
storageInserts.add(StorageSyncModels.localToRemoteRecord(update, newKey));
storageDeletes.add(ByteBuffer.wrap(oldKey));
completeKeys.remove(StorageId.forContact(oldKey));
completeKeys.add(StorageId.forContact(newKey));
storageKeyUpdates.put(update.getId(), newKey);
}
if (storageInserts.isEmpty() && storageDeletes.isEmpty()) {
return Optional.absent();
} else {
List<byte[]> contactDeleteBytes = Stream.of(storageDeletes).map(ByteBuffer::array).toList();
List<StorageId> completeKeysBytes = new ArrayList<>(completeKeys);
SignalStorageManifest manifest = new SignalStorageManifest(currentManifestVersion + 1, completeKeysBytes);
WriteOperationResult writeOperationResult = new WriteOperationResult(manifest, new ArrayList<>(storageInserts), contactDeleteBytes);
return Optional.of(new LocalWriteResult(writeOperationResult, storageKeyUpdates));
}
}
/**
* Given a list of all the local and remote keys you know about, this will return a result telling
* you which keys are exclusively remote and which are exclusively local.
*
* @param remoteKeys All remote keys available.
* @param localKeys All local keys available.
*
* @return An object describing which keys are exclusive to the remote data set and which keys are
* exclusive to the local data set.
*/
public static @NonNull KeyDifferenceResult findKeyDifference(@NonNull List<StorageId> remoteKeys,
@NonNull List<StorageId> localKeys)
{
Set<StorageId> remoteOnlyKeys = SetUtil.difference(remoteKeys, localKeys);
Set<StorageId> localOnlyKeys = SetUtil.difference(localKeys, remoteKeys);
return new KeyDifferenceResult(new ArrayList<>(remoteOnlyKeys), new ArrayList<>(localOnlyKeys));
}
/**
* Given two sets of storage records, this will resolve the data into a set of actions that need
* to be applied to resolve the differences. This will handle discovering which records between
* the two collections refer to the same contacts and are actually updates, which are brand new,
* etc.
*
* @param remoteOnlyRecords Records that are only present remotely.
* @param localOnlyRecords Records that are only present locally.
*
* @return A set of actions that should be applied to resolve the conflict.
*/
public static @NonNull MergeResult resolveConflict(@NonNull Collection<SignalStorageRecord> remoteOnlyRecords,
@NonNull Collection<SignalStorageRecord> localOnlyRecords)
{
List<SignalContactRecord> remoteOnlyContacts = Stream.of(remoteOnlyRecords).filter(r -> r.getContact().isPresent()).map(r -> r.getContact().get()).toList();
List<SignalContactRecord> localOnlyContacts = Stream.of(localOnlyRecords).filter(r -> r.getContact().isPresent()).map(r -> r.getContact().get()).toList();
List<SignalGroupV1Record> remoteOnlyGroupV1 = Stream.of(remoteOnlyRecords).filter(r -> r.getGroupV1().isPresent()).map(r -> r.getGroupV1().get()).toList();
List<SignalGroupV1Record> localOnlyGroupV1 = Stream.of(localOnlyRecords).filter(r -> r.getGroupV1().isPresent()).map(r -> r.getGroupV1().get()).toList();
List<SignalStorageRecord> remoteOnlyUnknowns = Stream.of(remoteOnlyRecords).filter(SignalStorageRecord::isUnknown).toList();
List<SignalStorageRecord> localOnlyUnknowns = Stream.of(localOnlyRecords).filter(SignalStorageRecord::isUnknown).toList();
RecordMergeResult<SignalContactRecord, RecordUpdate<SignalContactRecord>> contactMergeResult = resolveRecordConflict(remoteOnlyContacts, localOnlyContacts, new ContactConflictMerger(localOnlyContacts));
RecordMergeResult<SignalGroupV1Record, RecordUpdate<SignalGroupV1Record>> groupV1MergeResult = resolveRecordConflict(remoteOnlyGroupV1, localOnlyGroupV1, new GroupV1ConflictMerger(localOnlyGroupV1));
Set<SignalStorageRecord> remoteInserts = new HashSet<>();
remoteInserts.addAll(Stream.of(contactMergeResult.remoteInserts).map(SignalStorageRecord::forContact).toList());
remoteInserts.addAll(Stream.of(groupV1MergeResult.remoteInserts).map(SignalStorageRecord::forGroupV1).toList());
Set<RecordUpdate<SignalStorageRecord>> remoteUpdates = new HashSet<>();
remoteUpdates.addAll(Stream.of(contactMergeResult.remoteUpdates)
.map(c -> new RecordUpdate<>(SignalStorageRecord.forContact(c.getOld()), SignalStorageRecord.forContact(c.getNew())))
.toList());
remoteUpdates.addAll(Stream.of(groupV1MergeResult.remoteUpdates)
.map(c -> new RecordUpdate<>(SignalStorageRecord.forGroupV1(c.getOld()), SignalStorageRecord.forGroupV1(c.getNew())))
.toList());
return new MergeResult(contactMergeResult.localInserts,
contactMergeResult.localUpdates,
groupV1MergeResult.localInserts,
groupV1MergeResult.localUpdates,
new LinkedHashSet<>(remoteOnlyUnknowns),
new LinkedHashSet<>(localOnlyUnknowns),
remoteInserts,
remoteUpdates);
}
/**
* Assumes that the merge result has *not* yet been applied to the local data. That means that
* this method will handle generating the correct final key set based on the merge result.
*/
public static @NonNull WriteOperationResult createWriteOperation(long currentManifestVersion,
@NonNull List<StorageId> currentLocalStorageKeys,
@NonNull MergeResult mergeResult)
{
Set<StorageId> completeKeys = new HashSet<>(currentLocalStorageKeys);
completeKeys.addAll(Stream.of(mergeResult.getAllNewRecords()).map(SignalRecord::getId).toList());
completeKeys.removeAll(Stream.of(mergeResult.getAllRemovedRecords()).map(SignalRecord::getId).toList());
SignalStorageManifest manifest = new SignalStorageManifest(currentManifestVersion + 1, new ArrayList<>(completeKeys));
List<SignalStorageRecord> inserts = new ArrayList<>();
inserts.addAll(mergeResult.getRemoteInserts());
inserts.addAll(Stream.of(mergeResult.getRemoteUpdates()).map(RecordUpdate::getNew).toList());
List<byte[]> deletes = Stream.of(mergeResult.getRemoteUpdates()).map(RecordUpdate::getOld).map(SignalStorageRecord::getId).map(StorageId::getRaw).toList();
return new WriteOperationResult(manifest, inserts, deletes);
}
public static @NonNull byte[] generateKey() {
return keyGenerator.generate();
}
@VisibleForTesting
static void setTestKeyGenerator(@Nullable KeyGenerator testKeyGenerator) {
keyGenerator = testKeyGenerator;
}
private static @NonNull <E extends SignalRecord> RecordMergeResult<E, RecordUpdate<E>> resolveRecordConflict(@NonNull Collection<E> remoteOnlyRecords,
@NonNull Collection<E> localOnlyRecords,
@NonNull ConflictMerger<E> merger)
{
Set<E> localInserts = new LinkedHashSet<>(remoteOnlyRecords);
Set<E> remoteInserts = new LinkedHashSet<>(localOnlyRecords);
Set<RecordUpdate<E>> localUpdates = new LinkedHashSet<>();
Set<RecordUpdate<E>> remoteUpdates = new LinkedHashSet<>();
for (E remote : remoteOnlyRecords) {
Optional<E> local = merger.getMatching(remote);
if (local.isPresent()) {
E merged = merger.merge(remote, local.get(), keyGenerator);
if (!merged.equals(remote)) {
remoteUpdates.add(new RecordUpdate<>(remote, merged));
}
if (!merged.equals(local.get())) {
localUpdates.add(new RecordUpdate<>(local.get(), merged));
}
localInserts.remove(remote);
remoteInserts.remove(local.get());
}
}
return new RecordMergeResult<>(localInserts, localUpdates, remoteInserts, remoteUpdates);
}
public static boolean profileKeyChanged(RecordUpdate<SignalContactRecord> update) {
return !OptionalUtil.byteArrayEquals(update.getOld().getProfileKey(), update.getNew().getProfileKey());
}
public static final class KeyDifferenceResult {
private final List<StorageId> remoteOnlyKeys;
private final List<StorageId> localOnlyKeys;
private KeyDifferenceResult(@NonNull List<StorageId> remoteOnlyKeys,
@NonNull List<StorageId> localOnlyKeys)
{
this.remoteOnlyKeys = remoteOnlyKeys;
this.localOnlyKeys = localOnlyKeys;
}
public @NonNull List<StorageId> getRemoteOnlyKeys() {
return remoteOnlyKeys;
}
public @NonNull List<StorageId> getLocalOnlyKeys() {
return localOnlyKeys;
}
public boolean isEmpty() {
return remoteOnlyKeys.isEmpty() && localOnlyKeys.isEmpty();
}
}
public static final class MergeResult {
private final Set<SignalContactRecord> localContactInserts;
private final Set<RecordUpdate<SignalContactRecord>> localContactUpdates;
private final Set<SignalGroupV1Record> localGroupV1Inserts;
private final Set<RecordUpdate<SignalGroupV1Record>> localGroupV1Updates;
private final Set<SignalStorageRecord> localUnknownInserts;
private final Set<SignalStorageRecord> localUnknownDeletes;
private final Set<SignalStorageRecord> remoteInserts;
private final Set<RecordUpdate<SignalStorageRecord>> remoteUpdates;
@VisibleForTesting
MergeResult(@NonNull Set<SignalContactRecord> localContactInserts,
@NonNull Set<RecordUpdate<SignalContactRecord>> localContactUpdates,
@NonNull Set<SignalGroupV1Record> localGroupV1Inserts,
@NonNull Set<RecordUpdate<SignalGroupV1Record>> localGroupV1Updates,
@NonNull Set<SignalStorageRecord> localUnknownInserts,
@NonNull Set<SignalStorageRecord> localUnknownDeletes,
@NonNull Set<SignalStorageRecord> remoteInserts,
@NonNull Set<RecordUpdate<SignalStorageRecord>> remoteUpdates)
{
this.localContactInserts = localContactInserts;
this.localContactUpdates = localContactUpdates;
this.localGroupV1Inserts = localGroupV1Inserts;
this.localGroupV1Updates = localGroupV1Updates;
this.localUnknownInserts = localUnknownInserts;
this.localUnknownDeletes = localUnknownDeletes;
this.remoteInserts = remoteInserts;
this.remoteUpdates = remoteUpdates;
}
public @NonNull Set<SignalContactRecord> getLocalContactInserts() {
return localContactInserts;
}
public @NonNull Set<RecordUpdate<SignalContactRecord>> getLocalContactUpdates() {
return localContactUpdates;
}
public @NonNull Set<SignalGroupV1Record> getLocalGroupV1Inserts() {
return localGroupV1Inserts;
}
public @NonNull Set<RecordUpdate<SignalGroupV1Record>> getLocalGroupV1Updates() {
return localGroupV1Updates;
}
public @NonNull Set<SignalStorageRecord> getLocalUnknownInserts() {
return localUnknownInserts;
}
public @NonNull Set<SignalStorageRecord> getLocalUnknownDeletes() {
return localUnknownDeletes;
}
public @NonNull Set<SignalStorageRecord> getRemoteInserts() {
return remoteInserts;
}
public @NonNull Set<RecordUpdate<SignalStorageRecord>> getRemoteUpdates() {
return remoteUpdates;
}
@NonNull Set<SignalRecord> getAllNewRecords() {
Set<SignalRecord> records = new HashSet<>();
records.addAll(localContactInserts);
records.addAll(localGroupV1Inserts);
records.addAll(remoteInserts);
records.addAll(localUnknownInserts);
records.addAll(Stream.of(localContactUpdates).map(RecordUpdate::getNew).toList());
records.addAll(Stream.of(localGroupV1Updates).map(RecordUpdate::getNew).toList());
records.addAll(Stream.of(remoteUpdates).map(RecordUpdate::getNew).toList());
return records;
}
@NonNull Set<SignalRecord> getAllRemovedRecords() {
Set<SignalRecord> records = new HashSet<>();
records.addAll(localUnknownDeletes);
records.addAll(Stream.of(localContactUpdates).map(RecordUpdate::getOld).toList());
records.addAll(Stream.of(localGroupV1Updates).map(RecordUpdate::getOld).toList());
records.addAll(Stream.of(remoteUpdates).map(RecordUpdate::getOld).toList());
return records;
}
@Override
public @NonNull String toString() {
return String.format(Locale.ENGLISH,
"localContactInserts: %d, localContactUpdates: %d, localGroupInserts: %d, localGroupUpdates: %d, localUnknownInserts: %d, localUnknownDeletes: %d, remoteInserts: %d, remoteUpdates: %d",
localContactInserts.size(), localContactUpdates.size(), localGroupV1Inserts.size(), localGroupV1Updates.size(), localUnknownInserts.size(), localUnknownDeletes.size(), remoteInserts.size(), remoteUpdates.size());
}
}
public static final class WriteOperationResult {
private final SignalStorageManifest manifest;
private final List<SignalStorageRecord> inserts;
private final List<byte[]> deletes;
private WriteOperationResult(@NonNull SignalStorageManifest manifest,
@NonNull List<SignalStorageRecord> inserts,
@NonNull List<byte[]> deletes)
{
this.manifest = manifest;
this.inserts = inserts;
this.deletes = deletes;
}
public @NonNull SignalStorageManifest getManifest() {
return manifest;
}
public @NonNull List<SignalStorageRecord> getInserts() {
return inserts;
}
public @NonNull List<byte[]> getDeletes() {
return deletes;
}
public boolean isEmpty() {
return inserts.isEmpty() && deletes.isEmpty();
}
@Override
public @NonNull String toString() {
return String.format(Locale.ENGLISH,
"ManifestVersion: %d, Total Keys: %d, Inserts: %d, Deletes: %d",
manifest.getVersion(),
manifest.getStorageIds().size(),
inserts.size(),
deletes.size());
}
}
public static class LocalWriteResult {
private final WriteOperationResult writeResult;
private final Map<RecipientId, byte[]> storageKeyUpdates;
private LocalWriteResult(WriteOperationResult writeResult, Map<RecipientId, byte[]> storageKeyUpdates) {
this.writeResult = writeResult;
this.storageKeyUpdates = storageKeyUpdates;
}
public @NonNull WriteOperationResult getWriteResult() {
return writeResult;
}
public @NonNull Map<RecipientId, byte[]> getStorageKeyUpdates() {
return storageKeyUpdates;
}
}
public static class RecordUpdate<E extends SignalRecord> {
private final E oldRecord;
private final E newRecord;
RecordUpdate(@NonNull E oldRecord, @NonNull E newRecord) {
this.oldRecord = oldRecord;
this.newRecord = newRecord;
}
public @NonNull E getOld() {
return oldRecord;
}
public @NonNull E getNew() {
return newRecord;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RecordUpdate that = (RecordUpdate) o;
return oldRecord.equals(that.oldRecord) &&
newRecord.equals(that.newRecord);
}
@Override
public int hashCode() {
return Objects.hash(oldRecord, newRecord);
}
}
private static class RecordMergeResult<Record, Update> {
final Set<Record> localInserts;
final Set<Update> localUpdates;
final Set<Record> remoteInserts;
final Set<Update> remoteUpdates;
RecordMergeResult(@NonNull Set<Record> localInserts,
@NonNull Set<Update> localUpdates,
@NonNull Set<Record> remoteInserts,
@NonNull Set<Update> remoteUpdates)
{
this.localInserts = localInserts;
this.localUpdates = localUpdates;
this.remoteInserts = remoteInserts;
this.remoteUpdates = remoteUpdates;
}
}
interface ConflictMerger<E extends SignalRecord> {
@NonNull Optional<E> getMatching(@NonNull E record);
@NonNull E merge(@NonNull E remote, @NonNull E local, @NonNull KeyGenerator keyGenerator);
}
interface KeyGenerator {
@NonNull byte[] generate();
}
}

78
app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncModels.java Normal file
View File

@@ -0,0 +1,78 @@
package org.thoughtcrime.securesms.storage;
import androidx.annotation.NonNull;
import org.thoughtcrime.securesms.database.IdentityDatabase;
import org.thoughtcrime.securesms.database.RecipientDatabase;
import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings;
import org.thoughtcrime.securesms.util.GroupUtil;
import org.whispersystems.signalservice.api.push.SignalServiceAddress;
import org.whispersystems.signalservice.api.storage.SignalContactRecord;
import org.whispersystems.signalservice.api.storage.SignalGroupV1Record;
import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
import org.whispersystems.signalservice.internal.storage.protos.ContactRecord.IdentityState;
public final class StorageSyncModels {
private StorageSyncModels() {}
public static @NonNull SignalStorageRecord localToRemoteRecord(@NonNull RecipientSettings settings) {
if (settings.getStorageKey() == null) {
throw new AssertionError("Must have a storage key!");
}
return localToRemoteRecord(settings, settings.getStorageKey());
}
public static @NonNull SignalStorageRecord localToRemoteRecord(@NonNull RecipientSettings settings, @NonNull byte[] rawStorageId) {
switch (settings.getGroupType()) {
case NONE: return SignalStorageRecord.forContact(localToRemoteContact(settings, rawStorageId));
case SIGNAL_V1: return SignalStorageRecord.forGroupV1(localToRemoteGroupV1(settings, rawStorageId));
default: throw new AssertionError("Unsupported type!");
}
}
private static @NonNull SignalContactRecord localToRemoteContact(@NonNull RecipientSettings recipient, byte[] rawStorageId) {
if (recipient.getUuid() == null && recipient.getE164() == null) {
throw new AssertionError("Must have either a UUID or a phone number!");
}
return new SignalContactRecord.Builder(rawStorageId, new SignalServiceAddress(recipient.getUuid(), recipient.getE164()))
.setProfileKey(recipient.getProfileKey())
.setGivenName(recipient.getProfileName().getGivenName())
.setFamilyName(recipient.getProfileName().getFamilyName())
.setBlocked(recipient.isBlocked())
.setProfileSharingEnabled(recipient.isProfileSharing())
.setIdentityKey(recipient.getIdentityKey())
.setIdentityState(localToRemoteIdentityState(recipient.getIdentityStatus()))
.build();
}
private static @NonNull SignalGroupV1Record localToRemoteGroupV1(@NonNull RecipientSettings recipient, byte[] rawStorageId) {
if (recipient.getGroupId() == null) {
throw new AssertionError("Must have a groupId!");
}
return new SignalGroupV1Record.Builder(rawStorageId, GroupUtil.getDecodedIdOrThrow(recipient.getGroupId()))
.setBlocked(recipient.isBlocked())
.setProfileSharingEnabled(recipient.isProfileSharing())
.build();
}
public static @NonNull IdentityDatabase.VerifiedStatus remoteToLocalIdentityStatus(@NonNull IdentityState identityState) {
switch (identityState) {
case VERIFIED: return IdentityDatabase.VerifiedStatus.VERIFIED;
case UNVERIFIED: return IdentityDatabase.VerifiedStatus.UNVERIFIED;
default: return IdentityDatabase.VerifiedStatus.DEFAULT;
}
}
private static IdentityState localToRemoteIdentityState(@NonNull IdentityDatabase.VerifiedStatus local) {
switch (local) {
case VERIFIED: return IdentityState.VERIFIED;
case UNVERIFIED: return IdentityState.UNVERIFIED;
default: return IdentityState.DEFAULT;
}
}
}