TheArchive/session-android
1
0
Fork 0
mirror of https://github.com/oxen-io/session-android.git synced 2024-11-27 20:15:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Full backup importer refactoring.

Browse Source
This commit is contained in:
Anton Chekulaev 2020-11-05 21:08:00 +11:00
parent 7b46849d07
commit bf3c35f750
10 changed files with 407 additions and 518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/org/thoughtcrime/securesms/RegistrationActivity.java
View File

@ -35,7 +35,7 @@ import org.greenrobot.eventbus.EventBus;
import org.greenrobot.eventbus.Subscribe;
import org.greenrobot.eventbus.ThreadMode;
import org.thoughtcrime.securesms.animation.AnimationCompleteListener;
import org.thoughtcrime.securesms.backup.FullBackupBase;
import org.thoughtcrime.securesms.backup.BackupEvent;
import org.thoughtcrime.securesms.backup.FullBackupImporter;
import org.thoughtcrime.securesms.components.LabeledEditText;
import org.thoughtcrime.securesms.components.registration.CallMeCountDownView;
@ -327,7 +327,7 @@ public class RegistrationActivity extends BaseActionBarActivity implements Verif
Context context = RegistrationActivity.this;
SQLiteDatabase database = DatabaseFactory.getBackupDatabase(context);
FullBackupImporter.importFile(context,
FullBackupImporter.importFromUri(context,
AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
database, Uri.fromFile(backup.getFile()), passphrase);
@ -872,7 +872,7 @@ public class RegistrationActivity extends BaseActionBarActivity implements Verif
}
@Subscribe(threadMode = ThreadMode.MAIN)
public void onEvent(FullBackupBase.BackupEvent event) {
public void onEvent(BackupEvent event) {
if (event.getCount() == 0) restoreBackupProgress.setText(R.string.RegistrationActivity_checking);
else restoreBackupProgress.setText(getString(R.string.RegistrationActivity_d_messages_so_far, event.getCount()));
}

14
src/org/thoughtcrime/securesms/backup/BackupEvent.kt Normal file
View File

@ -0,0 +1,14 @@
package org.thoughtcrime.securesms.backup
data class BackupEvent constructor(val type: Type, val count: Int, val exception: Exception?) {
enum class Type {
PROGRESS, FINISHED
}
companion object {
@JvmStatic fun createProgress(count: Int) = BackupEvent(Type.PROGRESS, count, null)
@JvmStatic fun createFinished() = BackupEvent(Type.FINISHED, 0, null)
@JvmStatic fun createFinished(e: Exception?) = BackupEvent(Type.FINISHED, 0, e)
}
}

83
src/org/thoughtcrime/securesms/backup/FullBackupBase.java
View File

@ -1,83 +0,0 @@
package org.thoughtcrime.securesms.backup;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.greenrobot.eventbus.EventBus;
import org.whispersystems.libsignal.util.ByteUtil;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public abstract class FullBackupBase {
@SuppressWarnings("unused")
private static final String TAG = FullBackupBase.class.getSimpleName();
static class BackupStream {
static @NonNull byte[] getBackupKey(@NonNull String passphrase, @Nullable byte[] salt) {
try {
EventBus.getDefault().post(BackupEvent.createProgress(0));
MessageDigest digest = MessageDigest.getInstance("SHA-512");
byte[] input = passphrase.replace(" ", "").getBytes();
byte[] hash = input;
if (salt != null) digest.update(salt);
for (int i=0;i<250000;i++) {
if (i % 1000 == 0) EventBus.getDefault().post(BackupEvent.createProgress(0));
digest.update(hash);
hash = digest.digest(input);
}
return ByteUtil.trim(hash, 32);
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(e);
}
}
}
public static class BackupEvent {
public enum Type {
PROGRESS,
FINISHED
}
private final Type type;
private final int count;
private final @Nullable Exception exception;
private BackupEvent(Type type, int count, @Nullable Exception exception) {
this.type = type;
this.count = count;
this.exception = exception;
}
public Type getType() {
return type;
}
public int getCount() {
return count;
}
@Nullable
public Exception getException() {
return exception;
}
public static BackupEvent createProgress(int count) {
return new BackupEvent(Type.PROGRESS, count, null);
}
public static BackupEvent createFinished() {
return new BackupEvent(Type.FINISHED, 0, null);
}
public static BackupEvent createFinished(@Nullable Exception e) {
return new BackupEvent(Type.FINISHED, 0, e);
}
}
}

53
src/org/thoughtcrime/securesms/backup/FullBackupExporter.kt
View File

@ -20,6 +20,7 @@ import org.thoughtcrime.securesms.logging.Log
import org.thoughtcrime.securesms.loki.database.LokiAPIDatabase
import org.thoughtcrime.securesms.loki.database.LokiBackupFilesDatabase
import org.thoughtcrime.securesms.profiles.AvatarHelper
import org.thoughtcrime.securesms.util.BackupUtil
import org.thoughtcrime.securesms.util.Conversions
import org.thoughtcrime.securesms.util.TextSecurePreferences
import org.thoughtcrime.securesms.util.Util
@ -35,7 +36,7 @@ import javax.crypto.*
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
object FullBackupExporter : FullBackupBase() {
object FullBackupExporter {
private val TAG = FullBackupExporter::class.java.simpleName
@JvmStatic
@ -120,7 +121,6 @@ object FullBackupExporter : FullBackupBase() {
table != OneTimePreKeyDatabase.TABLE_NAME &&
table != SessionDatabase.TABLE_NAME &&
table != PushDatabase.TABLE_NAME &&
// table != DraftDatabase.TABLE_NAME &&
table != LokiBackupFilesDatabase.TABLE_NAME &&
table != LokiAPIDatabase.openGroupProfilePictureTable &&
@ -278,7 +278,7 @@ object FullBackupExporter : FullBackupBase() {
return false
}
private class BackupFrameOutputStream : BackupStream, Closeable, Flushable {
private class BackupFrameOutputStream : Closeable, Flushable {
private val outputStream: OutputStream
private var cipher: Cipher
@ -292,7 +292,7 @@ object FullBackupExporter : FullBackupBase() {
constructor(outputStream: OutputStream, passphrase: String) : super() {
try {
val salt = Util.getSecretBytes(32)
val key = getBackupKey(passphrase, salt)
val key = BackupUtil.computeBackupKey(passphrase, salt)
val derived = HKDFv3().deriveSecrets(key, "Backup Export".toByteArray(), 64)
val split = ByteUtil.split(derived, 32, 32)
cipherKey = split[0]
@ -309,13 +309,16 @@ object FullBackupExporter : FullBackupBase() {
.build().toByteArray()
outputStream.write(Conversions.intToByteArray(header.size))
outputStream.write(header)
} catch (e: NoSuchAlgorithmException) {
throw AssertionError(e)
} catch (e: NoSuchPaddingException) {
throw AssertionError(e)
} catch (e: InvalidKeyException) {
} catch (e: Exception) {
when (e) {
is NoSuchAlgorithmException,
is NoSuchPaddingException,
is InvalidKeyException -> {
throw AssertionError(e)
}
else -> throw e
}
}
}
@Throws(IOException::class)
@ -394,15 +397,17 @@ object FullBackupExporter : FullBackupBase() {
mac.update(remainder)
val attachmentDigest = mac.doFinal()
outputStream.write(attachmentDigest, 0, 10)
} catch (e: InvalidKeyException) {
throw AssertionError(e)
} catch (e: InvalidAlgorithmParameterException) {
throw AssertionError(e)
} catch (e: IllegalBlockSizeException) {
throw AssertionError(e)
} catch (e: BadPaddingException) {
} catch (e: Exception) {
when (e) {
is InvalidKeyException,
is InvalidAlgorithmParameterException,
is IllegalBlockSizeException,
is BadPaddingException -> {
throw AssertionError(e)
}
else -> throw e
}
}
}
@Throws(IOException::class)
@ -416,15 +421,17 @@ object FullBackupExporter : FullBackupBase() {
out.write(length)
out.write(frameCiphertext)
out.write(frameMac, 0, 10)
} catch (e: InvalidKeyException) {
throw AssertionError(e)
} catch (e: InvalidAlgorithmParameterException) {
throw AssertionError(e)
} catch (e: IllegalBlockSizeException) {
throw AssertionError(e)
} catch (e: BadPaddingException) {
} catch (e: Exception) {
when (e) {
is InvalidKeyException,
is InvalidAlgorithmParameterException,
is IllegalBlockSizeException,
is BadPaddingException -> {
throw AssertionError(e)
}
else -> throw e
}
}
}
@Throws(IOException::class)

394
src/org/thoughtcrime/securesms/backup/FullBackupImporter.java
View File

@ -1,394 +0,0 @@
package org.thoughtcrime.securesms.backup;
import android.annotation.SuppressLint;
import android.content.ContentValues;
import android.content.Context;
import android.content.SharedPreferences;
import android.database.Cursor;
import android.net.Uri;
import android.util.Pair;
import androidx.annotation.NonNull;
import net.sqlcipher.database.SQLiteDatabase;
import org.greenrobot.eventbus.EventBus;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.backup.BackupProtos.Attachment;
import org.thoughtcrime.securesms.backup.BackupProtos.BackupFrame;
import org.thoughtcrime.securesms.backup.BackupProtos.DatabaseVersion;
import org.thoughtcrime.securesms.backup.BackupProtos.SharedPreference;
import org.thoughtcrime.securesms.backup.BackupProtos.SqlStatement;
import org.thoughtcrime.securesms.backup.BackupProtos.Sticker;
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream;
import org.thoughtcrime.securesms.database.Address;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.database.GroupReceiptDatabase;
import org.thoughtcrime.securesms.database.MmsDatabase;
import org.thoughtcrime.securesms.database.SearchDatabase;
import org.thoughtcrime.securesms.database.StickerDatabase;
import org.thoughtcrime.securesms.database.ThreadDatabase;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.profiles.AvatarHelper;
import org.thoughtcrime.securesms.util.Conversions;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.kdf.HKDFv3;
import org.whispersystems.libsignal.util.ByteUtil;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.LinkedList;
import java.util.List;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.Mac;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
public class FullBackupImporter extends FullBackupBase {
/**
* Because BackupProtos.SharedPreference was made only to serialize string values,
* we use these 3-char prefixes to explicitly cast the values before inserting to a preference file.
*/
public static final String PREF_PREFIX_TYPE_INT = "i__";
public static final String PREF_PREFIX_TYPE_BOOLEAN = "b__";
private static final String TAG = FullBackupImporter.class.getSimpleName();
public static void importFile(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret,
@NonNull SQLiteDatabase db, @NonNull Uri fileUri, @NonNull String passphrase)
throws IOException
{
InputStream baseInputStream = context.getContentResolver().openInputStream(fileUri);
if (baseInputStream == null) {
throw new IOException("Cannot open an input stream for the file URI: " + fileUri.toString());
}
int count = 0;
try (BackupRecordInputStream inputStream = new BackupRecordInputStream(baseInputStream, passphrase)) {
db.beginTransaction();
dropAllTables(db);
BackupFrame frame;
while (!(frame = inputStream.readFrame()).getEnd()) {
if (count++ % 100 == 0) EventBus.getDefault().post(BackupEvent.createProgress(count));
if (frame.hasVersion()) processVersion(db, frame.getVersion());
else if (frame.hasStatement()) processStatement(db, frame.getStatement());
else if (frame.hasPreference()) processPreference(context, frame.getPreference());
else if (frame.hasAttachment()) processAttachment(context, attachmentSecret, db, frame.getAttachment(), inputStream);
else if (frame.hasSticker()) processSticker(context, attachmentSecret, db, frame.getSticker(), inputStream);
else if (frame.hasAvatar()) processAvatar(context, frame.getAvatar(), inputStream);
}
trimEntriesForExpiredMessages(context, db);
db.setTransactionSuccessful();
} finally {
if (db.inTransaction()) {
db.endTransaction();
}
}
EventBus.getDefault().post(BackupEvent.createFinished());
}
private static void processVersion(@NonNull SQLiteDatabase db, DatabaseVersion version) throws IOException {
if (version.getVersion() > db.getVersion()) {
throw new DatabaseDowngradeException(db.getVersion(), version.getVersion());
}
db.setVersion(version.getVersion());
}
private static void processStatement(@NonNull SQLiteDatabase db, SqlStatement statement) {
boolean isForSmsFtsSecretTable = statement.getStatement().contains(SearchDatabase.SMS_FTS_TABLE_NAME + "_");
boolean isForMmsFtsSecretTable = statement.getStatement().contains(SearchDatabase.MMS_FTS_TABLE_NAME + "_");
boolean isForSqliteSecretTable = statement.getStatement().toLowerCase().startsWith("create table sqlite_");
if (isForSmsFtsSecretTable || isForMmsFtsSecretTable || isForSqliteSecretTable) {
Log.i(TAG, "Ignoring import for statement: " + statement.getStatement());
return;
}
List<Object> parameters = new LinkedList<>();
for (SqlStatement.SqlParameter parameter : statement.getParametersList()) {
if (parameter.hasStringParamter()) parameters.add(parameter.getStringParamter());
else if (parameter.hasDoubleParameter()) parameters.add(parameter.getDoubleParameter());
else if (parameter.hasIntegerParameter()) parameters.add(parameter.getIntegerParameter());
else if (parameter.hasBlobParameter()) parameters.add(parameter.getBlobParameter().toByteArray());
else if (parameter.hasNullparameter()) parameters.add(null);
}
if (parameters.size() > 0) db.execSQL(statement.getStatement(), parameters.toArray());
else db.execSQL(statement.getStatement());
}
private static void processAttachment(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret, @NonNull SQLiteDatabase db, @NonNull Attachment attachment, BackupRecordInputStream inputStream)
throws IOException
{
File partsDirectory = context.getDir(AttachmentDatabase.DIRECTORY, Context.MODE_PRIVATE);
File dataFile = File.createTempFile("part", ".mms", partsDirectory);
Pair<byte[], OutputStream> output = ModernEncryptingPartOutputStream.createFor(attachmentSecret, dataFile, false);
inputStream.readAttachmentTo(output.second, attachment.getLength());
ContentValues contentValues = new ContentValues();
contentValues.put(AttachmentDatabase.DATA, dataFile.getAbsolutePath());
contentValues.put(AttachmentDatabase.THUMBNAIL, (String)null);
contentValues.put(AttachmentDatabase.DATA_RANDOM, output.first);
db.update(AttachmentDatabase.TABLE_NAME, contentValues,
AttachmentDatabase.ROW_ID + " = ? AND " + AttachmentDatabase.UNIQUE_ID + " = ?",
new String[] {String.valueOf(attachment.getRowId()), String.valueOf(attachment.getAttachmentId())});
}
private static void processSticker(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret, @NonNull SQLiteDatabase db, @NonNull Sticker sticker, BackupRecordInputStream inputStream)
throws IOException
{
File stickerDirectory = context.getDir(AttachmentDatabase.DIRECTORY, Context.MODE_PRIVATE);
File dataFile = File.createTempFile("sticker", ".mms", stickerDirectory);
Pair<byte[], OutputStream> output = ModernEncryptingPartOutputStream.createFor(attachmentSecret, dataFile, false);
inputStream.readAttachmentTo(output.second, sticker.getLength());
ContentValues contentValues = new ContentValues();
contentValues.put(StickerDatabase.FILE_PATH, dataFile.getAbsolutePath());
contentValues.put(StickerDatabase.FILE_RANDOM, output.first);
db.update(StickerDatabase.TABLE_NAME, contentValues,
StickerDatabase._ID + " = ?",
new String[] {String.valueOf(sticker.getRowId())});
}
private static void processAvatar(@NonNull Context context, @NonNull BackupProtos.Avatar avatar, @NonNull BackupRecordInputStream inputStream) throws IOException {
inputStream.readAttachmentTo(new FileOutputStream(AvatarHelper.getAvatarFile(context, Address.fromExternal(context, avatar.getName()))), avatar.getLength());
}
@SuppressLint("ApplySharedPref")
private static void processPreference(@NonNull Context context, SharedPreference preference) {
SharedPreferences preferences = context.getSharedPreferences(preference.getFile(), 0);
String key = preference.getKey();
String value = preference.getValue();
// See comments next to PREF_PREFIX_TYPE_* constants.
if (key.startsWith(PREF_PREFIX_TYPE_INT)) {
preferences.edit().putInt(
key.substring(3),
Integer.parseInt(value)
).commit();
} else if (key.startsWith(PREF_PREFIX_TYPE_BOOLEAN)) {
preferences.edit().putBoolean(
key.substring(3),
Boolean.parseBoolean(value)
).commit();
} else {
preferences.edit().putString(key, value).commit();
}
}
private static void dropAllTables(@NonNull SQLiteDatabase db) {
try (Cursor cursor = db.rawQuery("SELECT name, type FROM sqlite_master", null)) {
while (cursor != null && cursor.moveToNext()) {
String name = cursor.getString(0);
String type = cursor.getString(1);
if ("table".equals(type) && !name.startsWith("sqlite_")) {
db.execSQL("DROP TABLE IF EXISTS " + name);
}
}
}
}
private static void trimEntriesForExpiredMessages(@NonNull Context context, @NonNull SQLiteDatabase db) {
String trimmedCondition = " NOT IN (SELECT " + MmsDatabase.ID + " FROM " + MmsDatabase.TABLE_NAME + ")";
db.delete(GroupReceiptDatabase.TABLE_NAME, GroupReceiptDatabase.MMS_ID + trimmedCondition, null);
String[] columns = new String[] { AttachmentDatabase.ROW_ID, AttachmentDatabase.UNIQUE_ID };
String where = AttachmentDatabase.MMS_ID + trimmedCondition;
try (Cursor cursor = db.query(AttachmentDatabase.TABLE_NAME, columns, where, null, null, null, null)) {
while (cursor != null && cursor.moveToNext()) {
DatabaseFactory.getAttachmentDatabase(context).deleteAttachment(new AttachmentId(cursor.getLong(0), cursor.getLong(1)));
}
}
try (Cursor cursor = db.query(ThreadDatabase.TABLE_NAME, new String[] { ThreadDatabase.ID }, ThreadDatabase.EXPIRES_IN + " > 0", null, null, null, null)) {
while (cursor != null && cursor.moveToNext()) {
DatabaseFactory.getThreadDatabase(context).update(cursor.getLong(0), false);
}
}
}
private static class BackupRecordInputStream extends BackupStream implements Closeable {
private final InputStream in;
private final Cipher cipher;
private final Mac mac;
private final byte[] cipherKey;
private final byte[] macKey;
private byte[] iv;
private int counter;
private BackupRecordInputStream(@NonNull InputStream inputStream, @NonNull String passphrase) throws IOException {
try {
this.in = inputStream;
byte[] headerLengthBytes = new byte[4];
Util.readFully(in, headerLengthBytes);
int headerLength = Conversions.byteArrayToInt(headerLengthBytes);
byte[] headerFrame = new byte[headerLength];
Util.readFully(in, headerFrame);
BackupFrame frame = BackupFrame.parseFrom(headerFrame);
if (!frame.hasHeader()) {
throw new IOException("Backup stream does not start with header!");
}
BackupProtos.Header header = frame.getHeader();
this.iv = header.getIv().toByteArray();
if (iv.length != 16) {
throw new IOException("Invalid IV length!");
}
byte[] key = getBackupKey(passphrase, header.hasSalt() ? header.getSalt().toByteArray() : null);
byte[] derived = new HKDFv3().deriveSecrets(key, "Backup Export".getBytes(), 64);
byte[][] split = ByteUtil.split(derived, 32, 32);
this.cipherKey = split[0];
this.macKey = split[1];
this.cipher = Cipher.getInstance("AES/CTR/NoPadding");
this.mac = Mac.getInstance("HmacSHA256");
this.mac.init(new SecretKeySpec(macKey, "HmacSHA256"));
this.counter = Conversions.byteArrayToInt(iv);
} catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException e) {
throw new AssertionError(e);
}
}
BackupFrame readFrame() throws IOException {
return readFrame(in);
}
void readAttachmentTo(OutputStream out, int length) throws IOException {
try {
Conversions.intToByteArray(iv, 0, counter++);
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
mac.update(iv);
byte[] buffer = new byte[8192];
while (length > 0) {
int read = in.read(buffer, 0, Math.min(buffer.length, length));
if (read == -1) throw new IOException("File ended early!");
mac.update(buffer, 0, read);
byte[] plaintext = cipher.update(buffer, 0, read);
if (plaintext != null) {
out.write(plaintext, 0, plaintext.length);
}
length -= read;
}
byte[] plaintext = cipher.doFinal();
if (plaintext != null) {
out.write(plaintext, 0, plaintext.length);
}
out.close();
byte[] ourMac = ByteUtil.trim(mac.doFinal(), 10);
byte[] theirMac = new byte[10];
try {
Util.readFully(in, theirMac);
} catch (IOException e) {
//destination.delete();
throw new IOException(e);
}
if (!MessageDigest.isEqual(ourMac, theirMac)) {
//destination.delete();
throw new IOException("Bad MAC");
}
} catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
throw new AssertionError(e);
}
}
private BackupFrame readFrame(InputStream in) throws IOException {
try {
byte[] length = new byte[4];
Util.readFully(in, length);
byte[] frame = new byte[Conversions.byteArrayToInt(length)];
Util.readFully(in, frame);
byte[] theirMac = new byte[10];
System.arraycopy(frame, frame.length - 10, theirMac, 0, theirMac.length);
mac.update(frame, 0, frame.length - 10);
byte[] ourMac = ByteUtil.trim(mac.doFinal(), 10);
if (!MessageDigest.isEqual(ourMac, theirMac)) {
throw new IOException("Bad MAC");
}
Conversions.intToByteArray(iv, 0, counter++);
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(cipherKey, "AES"), new IvParameterSpec(iv));
byte[] plaintext = cipher.doFinal(frame, 0, frame.length - 10);
return BackupFrame.parseFrom(plaintext);
} catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
throw new AssertionError(e);
}
}
@Override
public void close() throws IOException {
in.close();
}
}
public static class DatabaseDowngradeException extends IOException {
DatabaseDowngradeException(int currentVersion, int backupVersion) {
super("Tried to import a backup with version " + backupVersion + " into a database with version " + currentVersion);
}
}
}

319
src/org/thoughtcrime/securesms/backup/FullBackupImporter.kt Normal file
View File

@ -0,0 +1,319 @@
package org.thoughtcrime.securesms.backup
import android.annotation.SuppressLint
import android.content.ContentValues
import android.content.Context
import android.net.Uri
import net.sqlcipher.database.SQLiteDatabase
import org.greenrobot.eventbus.EventBus
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.backup.BackupProtos.*
import org.thoughtcrime.securesms.crypto.AttachmentSecret
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream
import org.thoughtcrime.securesms.database.*
import org.thoughtcrime.securesms.logging.Log
import org.thoughtcrime.securesms.profiles.AvatarHelper
import org.thoughtcrime.securesms.util.BackupUtil
import org.thoughtcrime.securesms.util.Conversions
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.libsignal.kdf.HKDFv3
import org.whispersystems.libsignal.util.ByteUtil
import java.io.*
import java.security.InvalidAlgorithmParameterException
import java.security.InvalidKeyException
import java.security.MessageDigest
import java.security.NoSuchAlgorithmException
import java.util.*
import javax.crypto.*
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
object FullBackupImporter {
/**
* Because BackupProtos.SharedPreference was made only to serialize string values,
* we use these 3-char prefixes to explicitly cast the values before inserting to a preference file.
*/
const val PREF_PREFIX_TYPE_INT = "i__"
const val PREF_PREFIX_TYPE_BOOLEAN = "b__"
private val TAG = FullBackupImporter::class.java.simpleName
@Throws(IOException::class)
@JvmStatic
fun importFromUri(context: Context,
attachmentSecret: AttachmentSecret,
db: SQLiteDatabase,
fileUri: Uri,
passphrase: String) {
val baseInputStream = context.contentResolver.openInputStream(fileUri)
?: throw IOException("Cannot open an input stream for the file URI: $fileUri")
var count = 0
try {
BackupRecordInputStream(baseInputStream, passphrase).use { inputStream ->
db.beginTransaction()
dropAllTables(db)
var frame: BackupFrame
while (!inputStream.readFrame().also { frame = it }.end) {
if (count++ % 100 == 0) EventBus.getDefault().post(BackupEvent.createProgress(count))
if (frame.hasVersion()) processVersion(db, frame.version) else if (frame.hasStatement()) processStatement(db, frame.statement) else if (frame.hasPreference()) processPreference(context, frame.preference) else if (frame.hasAttachment()) processAttachment(context, attachmentSecret, db, frame.attachment, inputStream) else if (frame.hasSticker()) processSticker(context, attachmentSecret, db, frame.sticker, inputStream) else if (frame.hasAvatar()) processAvatar(context, frame.avatar, inputStream)
}
trimEntriesForExpiredMessages(context, db)
db.setTransactionSuccessful()
}
} finally {
if (db.inTransaction()) {
db.endTransaction()
}
}
EventBus.getDefault().post(BackupEvent.createFinished())
}
@Throws(IOException::class)
private fun processVersion(db: SQLiteDatabase, version: DatabaseVersion) {
if (version.version > db.version) {
throw DatabaseDowngradeException(db.version, version.version)
}
db.version = version.version
}
private fun processStatement(db: SQLiteDatabase, statement: SqlStatement) {
val isForSmsFtsSecretTable = statement.statement.contains(SearchDatabase.SMS_FTS_TABLE_NAME + "_")
val isForMmsFtsSecretTable = statement.statement.contains(SearchDatabase.MMS_FTS_TABLE_NAME + "_")
val isForSqliteSecretTable = statement.statement.toLowerCase(Locale.ENGLISH).startsWith("create table sqlite_")
if (isForSmsFtsSecretTable || isForMmsFtsSecretTable || isForSqliteSecretTable) {
Log.i(TAG, "Ignoring import for statement: " + statement.statement)
return
}
val parameters: MutableList<Any?> = LinkedList()
for (parameter in statement.parametersList) {
if (parameter.hasStringParamter()) parameters.add(parameter.stringParamter) else if (parameter.hasDoubleParameter()) parameters.add(parameter.doubleParameter) else if (parameter.hasIntegerParameter()) parameters.add(parameter.integerParameter) else if (parameter.hasBlobParameter()) parameters.add(parameter.blobParameter.toByteArray()) else if (parameter.hasNullparameter()) parameters.add(null)
}
if (parameters.size > 0) db.execSQL(statement.statement, parameters.toTypedArray()) else db.execSQL(statement.statement)
}
@Throws(IOException::class)
private fun processAttachment(context: Context, attachmentSecret: AttachmentSecret, db: SQLiteDatabase, attachment: Attachment, inputStream: BackupRecordInputStream) {
val partsDirectory = context.getDir(AttachmentDatabase.DIRECTORY, Context.MODE_PRIVATE)
val dataFile = File.createTempFile("part", ".mms", partsDirectory)
val output = ModernEncryptingPartOutputStream.createFor(attachmentSecret, dataFile, false)
inputStream.readAttachmentTo(output.second, attachment.length)
val contentValues = ContentValues()
contentValues.put(AttachmentDatabase.DATA, dataFile.absolutePath)
contentValues.put(AttachmentDatabase.THUMBNAIL, null as String?)
contentValues.put(AttachmentDatabase.DATA_RANDOM, output.first)
db.update(AttachmentDatabase.TABLE_NAME, contentValues,
AttachmentDatabase.ROW_ID + " = ? AND " + AttachmentDatabase.UNIQUE_ID + " = ?", arrayOf(attachment.rowId.toString(), attachment.attachmentId.toString()))
}
@Throws(IOException::class)
private fun processSticker(context: Context, attachmentSecret: AttachmentSecret, db: SQLiteDatabase, sticker: Sticker, inputStream: BackupRecordInputStream) {
val stickerDirectory = context.getDir(AttachmentDatabase.DIRECTORY, Context.MODE_PRIVATE)
val dataFile = File.createTempFile("sticker", ".mms", stickerDirectory)
val output = ModernEncryptingPartOutputStream.createFor(attachmentSecret, dataFile, false)
inputStream.readAttachmentTo(output.second, sticker.length)
val contentValues = ContentValues()
contentValues.put(StickerDatabase.FILE_PATH, dataFile.absolutePath)
contentValues.put(StickerDatabase.FILE_RANDOM, output.first)
db.update(StickerDatabase.TABLE_NAME, contentValues,
StickerDatabase._ID + " = ?", arrayOf(sticker.rowId.toString()))
}
@Throws(IOException::class)
private fun processAvatar(context: Context, avatar: Avatar, inputStream: BackupRecordInputStream) {
inputStream.readAttachmentTo(FileOutputStream(AvatarHelper.getAvatarFile(context, Address.fromExternal(context, avatar.name))), avatar.length)
}
@SuppressLint("ApplySharedPref")
private fun processPreference(context: Context, preference: SharedPreference) {
val preferences = context.getSharedPreferences(preference.file, 0)
val key = preference.key
val value = preference.value
// See the comment next to PREF_PREFIX_TYPE_* constants.
when {
key.startsWith(PREF_PREFIX_TYPE_INT) ->
preferences.edit().putInt(
key.substring(PREF_PREFIX_TYPE_INT.length),
value.toInt()
).commit()
key.startsWith(PREF_PREFIX_TYPE_BOOLEAN) ->
preferences.edit().putBoolean(
key.substring(PREF_PREFIX_TYPE_BOOLEAN.length),
value.toBoolean()
).commit()
else ->
preferences.edit().putString(key, value).commit()
}
}
private fun dropAllTables(db: SQLiteDatabase) {
db.rawQuery("SELECT name, type FROM sqlite_master", null).use { cursor ->
while (cursor != null && cursor.moveToNext()) {
val name = cursor.getString(0)
val type = cursor.getString(1)
if ("table" == type && !name.startsWith("sqlite_")) {
db.execSQL("DROP TABLE IF EXISTS $name")
}
}
}
}
private fun trimEntriesForExpiredMessages(context: Context, db: SQLiteDatabase) {
val trimmedCondition = " NOT IN (SELECT ${MmsDatabase.ID} FROM ${MmsDatabase.TABLE_NAME})"
db.delete(GroupReceiptDatabase.TABLE_NAME, GroupReceiptDatabase.MMS_ID + trimmedCondition, null)
val columns = arrayOf(AttachmentDatabase.ROW_ID, AttachmentDatabase.UNIQUE_ID)
val where = AttachmentDatabase.MMS_ID + trimmedCondition
db.query(AttachmentDatabase.TABLE_NAME, columns, where, null, null, null, null).use { cursor ->
while (cursor != null && cursor.moveToNext()) {
DatabaseFactory.getAttachmentDatabase(context).deleteAttachment(AttachmentId(cursor.getLong(0), cursor.getLong(1)))
}
}
db.query(ThreadDatabase.TABLE_NAME, arrayOf(ThreadDatabase.ID), ThreadDatabase.EXPIRES_IN + " > 0", null, null, null, null).use { cursor ->
while (cursor != null && cursor.moveToNext()) {
DatabaseFactory.getThreadDatabase(context).update(cursor.getLong(0), false)
}
}
}
private class BackupRecordInputStream : Closeable {
private val inputStream: InputStream
private val cipher: Cipher
private val mac: Mac
private val cipherKey: ByteArray
private val macKey: ByteArray
private val iv: ByteArray
private var counter = 0
@Throws(IOException::class)
constructor(inputStream: InputStream, passphrase: String) : super() {
try {
this.inputStream = inputStream
val headerLengthBytes = ByteArray(4)
Util.readFully(this.inputStream, headerLengthBytes)
val headerLength = Conversions.byteArrayToInt(headerLengthBytes)
val headerFrame = ByteArray(headerLength)
Util.readFully(this.inputStream, headerFrame)
val frame = BackupFrame.parseFrom(headerFrame)
if (!frame.hasHeader()) {
throw IOException("Backup stream does not start with header!")
}
val header = frame.header
iv = header.iv.toByteArray()
if (iv.size != 16) {
throw IOException("Invalid IV length!")
}
val key = BackupUtil.computeBackupKey(passphrase, if (header.hasSalt()) header.salt.toByteArray() else null)
val derived = HKDFv3().deriveSecrets(key, "Backup Export".toByteArray(), 64)
val split = ByteUtil.split(derived, 32, 32)
cipherKey = split[0]
macKey = split[1]
cipher = Cipher.getInstance("AES/CTR/NoPadding")
mac = Mac.getInstance("HmacSHA256")
mac.init(SecretKeySpec(macKey, "HmacSHA256"))
counter = Conversions.byteArrayToInt(iv)
} catch (e: Exception) {
when (e) {
is NoSuchAlgorithmException,
is NoSuchPaddingException,
is InvalidKeyException -> {
throw AssertionError(e)
}
else -> throw e
}
}
}
@Throws(IOException::class)
fun readFrame(): BackupFrame {
return readFrame(inputStream)
}
@Throws(IOException::class)
fun readAttachmentTo(out: OutputStream, length: Int) {
var length = length
try {
Conversions.intToByteArray(iv, 0, counter++)
cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(cipherKey, "AES"), IvParameterSpec(iv))
mac.update(iv)
val buffer = ByteArray(8192)
while (length > 0) {
val read = inputStream.read(buffer, 0, Math.min(buffer.size, length))
if (read == -1) throw IOException("File ended early!")
mac.update(buffer, 0, read)
val plaintext = cipher.update(buffer, 0, read)
if (plaintext != null) {
out.write(plaintext, 0, plaintext.size)
}
length -= read
}
val plaintext = cipher.doFinal()
if (plaintext != null) {
out.write(plaintext, 0, plaintext.size)
}
out.close()
val ourMac = ByteUtil.trim(mac.doFinal(), 10)
val theirMac = ByteArray(10)
try {
Util.readFully(inputStream, theirMac)
} catch (e: IOException) {
throw IOException(e)
}
if (!MessageDigest.isEqual(ourMac, theirMac)) {
throw IOException("Bad MAC")
}
} catch (e: Exception) {
when (e) {
is InvalidKeyException,
is InvalidAlgorithmParameterException,
is IllegalBlockSizeException,
is BadPaddingException -> {
throw AssertionError(e)
}
else -> throw e
}
}
}
@Throws(IOException::class)
private fun readFrame(`in`: InputStream?): BackupFrame {
return try {
val length = ByteArray(4)
Util.readFully(`in`, length)
val frame = ByteArray(Conversions.byteArrayToInt(length))
Util.readFully(`in`, frame)
val theirMac = ByteArray(10)
System.arraycopy(frame, frame.size - 10, theirMac, 0, theirMac.size)
mac.update(frame, 0, frame.size - 10)
val ourMac = ByteUtil.trim(mac.doFinal(), 10)
if (!MessageDigest.isEqual(ourMac, theirMac)) {
throw IOException("Bad MAC")
}
Conversions.intToByteArray(iv, 0, counter++)
cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(cipherKey, "AES"), IvParameterSpec(iv))
val plaintext = cipher.doFinal(frame, 0, frame.size - 10)
BackupFrame.parseFrom(plaintext)
} catch (e: Exception) {
when (e) {
is InvalidKeyException,
is InvalidAlgorithmParameterException,
is IllegalBlockSizeException,
is BadPaddingException -> {
throw AssertionError(e)
}
else -> throw e
}
}
}
@Throws(IOException::class)
override fun close() {
inputStream.close()
}
}
class DatabaseDowngradeException internal constructor(currentVersion: Int, backupVersion: Int) :
IOException("Tried to import a backup with version $backupVersion into a database with version $currentVersion")
}

4
src/org/thoughtcrime/securesms/loki/activities/BackupRestoreActivity.kt
View File

@ -29,10 +29,8 @@ import org.thoughtcrime.securesms.BaseActionBarActivity
import org.thoughtcrime.securesms.backup.FullBackupImporter
import org.thoughtcrime.securesms.backup.FullBackupImporter.DatabaseDowngradeException
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
import org.thoughtcrime.securesms.database.Address
import org.thoughtcrime.securesms.database.DatabaseFactory
import org.thoughtcrime.securesms.logging.Log
import org.thoughtcrime.securesms.loki.api.ResetThreadSessionJob
import org.thoughtcrime.securesms.loki.utilities.setUpActionBarSessionLogo
import org.thoughtcrime.securesms.loki.utilities.show
import org.thoughtcrime.securesms.notifications.NotificationChannels
@ -122,7 +120,7 @@ class BackupRestoreActivity : BaseActionBarActivity() {
return try {
val context: Context = this@BackupRestoreActivity
val database = DatabaseFactory.getBackupDatabase(context)
FullBackupImporter.importFile(
FullBackupImporter.importFromUri(
context,
AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
DatabaseFactory.getBackupDatabase(context),

2
src/org/thoughtcrime/securesms/preferences/ChatsPreferenceFragment.java
View File

@ -19,7 +19,7 @@ import org.greenrobot.eventbus.Subscribe;
import org.greenrobot.eventbus.ThreadMode;
import org.thoughtcrime.securesms.ApplicationContext;
import org.thoughtcrime.securesms.backup.BackupDialog;
import org.thoughtcrime.securesms.backup.FullBackupBase.BackupEvent;
import org.thoughtcrime.securesms.backup.BackupEvent;
import org.thoughtcrime.securesms.components.SwitchPreferenceCompat;
import org.thoughtcrime.securesms.jobs.LocalBackupJob;
import org.thoughtcrime.securesms.logging.Log;

29
src/org/thoughtcrime/securesms/util/BackupUtil.kt
View File

@ -13,6 +13,8 @@ import androidx.annotation.WorkerThread
import androidx.documentfile.provider.DocumentFile
import androidx.fragment.app.Fragment
import network.loki.messenger.R
import org.greenrobot.eventbus.EventBus
import org.thoughtcrime.securesms.backup.BackupEvent
import org.thoughtcrime.securesms.backup.BackupPassphrase
import org.thoughtcrime.securesms.backup.FullBackupExporter
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
@ -21,6 +23,8 @@ import org.thoughtcrime.securesms.loki.database.BackupFileRecord
import org.thoughtcrime.securesms.service.LocalBackupListener
import org.whispersystems.libsignal.util.ByteUtil
import java.io.IOException
import java.security.MessageDigest
import java.security.NoSuchAlgorithmException
import java.security.SecureRandom
import java.text.SimpleDateFormat
import java.util.*
@ -160,11 +164,17 @@ object BackupUtil {
throw IOException("Cannot create writable file in the dir $dirUri")
}
try {
FullBackupExporter.export(context,
AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
DatabaseFactory.getBackupDatabase(context),
fileUri,
backupPassword)
} catch (e: Exception) {
// Delete the backup file on any error.
DocumentsContract.deleteDocument(context.contentResolver, fileUri)
throw e
}
//TODO Use real file size.
val record = DatabaseFactory.getLokiBackupFilesDatabase(context)
@ -198,6 +208,25 @@ object BackupUtil {
Log.v(TAG, "Backup file was deleted: ${record.uri}")
}
}
@JvmStatic
fun computeBackupKey(passphrase: String, salt: ByteArray?): ByteArray {
return try {
EventBus.getDefault().post(BackupEvent.createProgress(0))
val digest = MessageDigest.getInstance("SHA-512")
val input = passphrase.replace(" ", "").toByteArray()
var hash: ByteArray = input
if (salt != null) digest.update(salt)
for (i in 0..249999) {
if (i % 1000 == 0) EventBus.getDefault().post(BackupEvent.createProgress(0))
digest.update(hash)
hash = digest.digest(input)
}
ByteUtil.trim(hash, 32)
} catch (e: NoSuchAlgorithmException) {
throw AssertionError(e)
}
}
}
/**

1
src/org/thoughtcrime/securesms/util/TextSecurePreferences.java
View File

@ -14,7 +14,6 @@ import androidx.core.app.NotificationCompat;
import org.greenrobot.eventbus.EventBus;
import org.thoughtcrime.securesms.backup.BackupProtos;
import org.thoughtcrime.securesms.crypto.MasterSecretUtil;
import org.thoughtcrime.securesms.jobmanager.impl.SqlCipherMigrationConstraintObserver;
import org.thoughtcrime.securesms.lock.RegistrationLockReminders;
import org.thoughtcrime.securesms.logging.Log;