Skip to content

Commit

Permalink
chore: improve google drive sync. (#1200)
Browse files Browse the repository at this point in the history 
improve google drive sync, removes the lock, change to protobuf, and potentially fix deviceId not being unique, since it wasn't appState...
  • Loading branch information
@kaiserbh
kaiserbh authored Jun 26, 2024
1 parent d29a4ff commit c2eece0
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 129 deletions.
2 changes: 1 addition & 1 deletion app/src/main/java/eu/kanade/domain/sync/SyncPreferences.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class SyncPreferences(
)

fun uniqueDeviceID(): String {
val uniqueIDPreference = preferenceStore.getString("unique_device_id", "")
val uniqueIDPreference = preferenceStore.getString(Preference.appStateKey("unique_device_id"), "")

// Retrieve the current value of the preference
var uniqueID = uniqueIDPreference.get()
Expand Down
159 changes: 31 additions & 128 deletions app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeToken
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse
import com.google.api.client.http.ByteArrayContent
import com.google.api.client.http.InputStreamContent
import com.google.api.client.http.javanet.NetHttpTransport
import com.google.api.client.json.JsonFactory
Expand All @@ -20,11 +19,10 @@ import com.google.api.services.drive.DriveScopes
import com.google.api.services.drive.model.File
import eu.kanade.domain.sync.SyncPreferences
import eu.kanade.tachiyomi.data.backup.models.Backup
import kotlinx.coroutines.delay
import eu.kanade.tachiyomi.data.backup.models.BackupSerializer
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import kotlinx.serialization.json.encodeToStream
import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority
import logcat.logcat
import tachiyomi.core.common.i18n.stringResource
Expand All @@ -37,7 +35,6 @@ import uy.kohesive.injekt.api.get
import java.io.IOException
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.time.Instant
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream

Expand All @@ -64,12 +61,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync

private val appName = context.stringResource(MR.strings.app_name)

private val remoteFileName = "${appName}_sync_data.gz"

private val lockFileName = "${appName}_sync.lock"
private val remoteFileName = "${appName}_sync.proto.gz"

private val googleDriveService = GoogleDriveService(context)

private val protoBuf: ProtoBuf = Injekt.get()

override suspend fun doSync(syncData: SyncData): Backup? {
beforeSync()

Expand Down Expand Up @@ -107,64 +104,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
}

private suspend fun beforeSync() {
try {
googleDriveService.refreshToken()
val drive = googleDriveService.driveService
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))

var backoff = 1000L
var retries = 0 // Retry counter
val maxRetries = 10 // Maximum number of retries

while (retries < maxRetries) {
val lockFiles = findLockFile(drive)
logcat(LogPriority.DEBUG) { "Found ${lockFiles.size} lock file(s)" }

when {
lockFiles.isEmpty() -> {
logcat(LogPriority.DEBUG) { "No lock file found, creating a new one" }
createLockFile(drive)
break
}
lockFiles.size == 1 -> {
val lockFile = lockFiles.first()
val createdTime = Instant.parse(lockFile.createdTime.toString())
val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes()
logcat(LogPriority.DEBUG) { "Lock file age: $ageMinutes minutes" }
if (ageMinutes <= 3) {
logcat(LogPriority.DEBUG) { "Lock file is new, proceeding with sync" }
break
} else {
logcat(LogPriority.DEBUG) { "Lock file is old, deleting and creating a new one" }
deleteLockFile(drive)
createLockFile(drive)
break
}
}
else -> {
logcat(LogPriority.DEBUG) { "Multiple lock files found, applying backoff" }
delay(backoff) // Apply backoff strategy
backoff = (backoff * 2).coerceAtMost(16000L)
logcat(LogPriority.DEBUG) { "Backoff increased to $backoff milliseconds" }
}
}
retries++ // Increment retry counter
logcat(LogPriority.DEBUG) { "Loop iteration complete, retry count: $retries, backoff time: $backoff" }
}

if (retries >= maxRetries) {
logcat(LogPriority.ERROR) { "Max retries reached, exiting sync process" }
throw Exception(context.stringResource(SYMR.strings.error_before_sync_gdrive) + ": Max retries reached.")
}
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error in GoogleDrive beforeSync" }
throw Exception(context.stringResource(SYMR.strings.error_before_sync_gdrive) + ": ${e.message}", e)
}
googleDriveService.refreshToken()
}

private fun pullSyncData(): SyncData? {
val drive = googleDriveService.driveService ?:
throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
val drive = googleDriveService.driveService
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))

val fileList = getAppDataFileList(drive)
if (fileList.isEmpty()) {
Expand All @@ -178,7 +123,10 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
try {
drive.files().get(gdriveFileId).executeMediaAsInputStream().use { inputStream ->
GZIPInputStream(inputStream).use { gzipInputStream ->
return Json.decodeFromStream(SyncData.serializer(), gzipInputStream)
val byteArray = gzipInputStream.readBytes()
val backup = protoBuf.decodeFromByteArray(BackupSerializer, byteArray)
val deviceId = fileList[0].appProperties["deviceId"] ?: ""
return SyncData(deviceId = deviceId, backup = backup)
}
}
} catch (e: Exception) {
Expand All @@ -192,29 +140,40 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))

val fileList = getAppDataFileList(drive)
val backup = syncData.backup ?: return

val byteArray = protoBuf.encodeToByteArray(BackupSerializer, backup)
if (byteArray.isEmpty()) {
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
}

PipedOutputStream().use { pos ->
PipedInputStream(pos).use { pis ->
withIOContext {
// Start a coroutine or a background thread to write JSON to the PipedOutputStream
launch {
GZIPOutputStream(pos).use { gzipOutputStream ->
Json.encodeToStream(SyncData.serializer(), syncData, gzipOutputStream)
gzipOutputStream.write(byteArray)
}
}

val mediaContent = InputStreamContent("application/octet-stream", pis)

if (fileList.isNotEmpty()) {
val fileId = fileList[0].id
val mediaContent = InputStreamContent("application/gzip", pis)
drive.files().update(fileId, null, mediaContent).execute()
val fileMetadata = File().apply {
name = remoteFileName
mimeType = "application/octet-stream"
appProperties = mapOf("deviceId" to syncData.deviceId)
}
drive.files().update(fileId, fileMetadata, mediaContent).execute()
logcat(LogPriority.DEBUG) { "Updated existing sync data file in Google Drive with file ID: $fileId" }
} else {
val fileMetadata = File().apply {
name = remoteFileName
mimeType = "application/gzip"
mimeType = "application/octet-stream"
parents = listOf("appDataFolder")
appProperties = mapOf("deviceId" to syncData.deviceId)
}
val mediaContent = InputStreamContent("application/gzip", pis)
val uploadedFile = drive.files().create(fileMetadata, mediaContent)
.setFields("id")
.execute()
Expand All @@ -228,12 +187,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
private fun getAppDataFileList(drive: Drive): MutableList<File> {
try {
// Search for the existing file by name in the appData folder
val query = "mimeType='application/gzip' and name = '$remoteFileName'"
val query = "mimeType='application/x-gzip' and name = '$remoteFileName'"
val fileList = drive.files()
.list()
.setSpaces("appDataFolder")
.setQ(query)
.setFields("files(id, name, createdTime)")
.setFields("files(id, name, createdTime, appProperties)")
.execute()
.files
logcat { "AppData folder file list: $fileList" }
Expand All @@ -245,62 +204,6 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
}
}

private fun createLockFile(drive: Drive) {
try {
val fileMetadata = File().apply {
name = lockFileName
mimeType = "text/plain"
parents = listOf("appDataFolder")
}

// Create an empty content to upload as the lock file
val emptyContent = ByteArrayContent.fromString("text/plain", "")

val file = drive.files().create(fileMetadata, emptyContent)
.setFields("id, name, createdTime")
.execute()

logcat { "Created lock file with ID: ${file.id}" }
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error creating lock file" }
throw Exception(e.message, e)
}
}

private fun findLockFile(drive: Drive): MutableList<File> {
return try {
val query = "mimeType='text/plain' and name = '$lockFileName'"
val fileList = drive.files()
.list()
.setSpaces("appDataFolder")
.setQ(query)
.setFields("files(id, name, createdTime)")
.execute().files
logcat { "Lock file search result: $fileList" }
fileList
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error finding lock file" }
mutableListOf()
}
}

private fun deleteLockFile(drive: Drive) {
try {
val lockFiles = findLockFile(drive)

if (lockFiles.isNotEmpty()) {
for (file in lockFiles) {
drive.files().delete(file.id).execute()
logcat { "Deleted lock file with ID: ${file.id}" }
}
} else {
logcat { "No lock file found to delete." }
}
} catch (e: Exception) {
logcat(LogPriority.ERROR, throwable = e) { "Error deleting lock file" }
throw Exception(context.stringResource(SYMR.strings.error_deleting_google_drive_lock_file), e)
}
}

suspend fun deleteSyncDataFromGoogleDrive(): DeleteSyncDataStatus {
val drive = googleDriveService.driveService
Expand Down

0 comments on commit c2eece0

Please sign in to comment.