Streaming backup file import

app/src/main/java/org/tasks/backup/BackupContainer.kt

@@ -3,44 +3,16 @@ package org.tasks.backup
 import kotlinx.serialization.Serializable
 import org.tasks.backup.TasksJsonImporter.LegacyLocation
 import org.tasks.data.GoogleTask
-import org.tasks.data.GoogleTaskAccount
-import org.tasks.data.GoogleTaskList
 import org.tasks.data.entity.Alarm
 import org.tasks.data.entity.Attachment
-import org.tasks.data.entity.CaldavAccount
-import org.tasks.data.entity.CaldavCalendar
 import org.tasks.data.entity.CaldavTask
-import org.tasks.data.entity.Filter
 import org.tasks.data.entity.Geofence
-import org.tasks.data.entity.Place
 import org.tasks.data.entity.Tag
-import org.tasks.data.entity.TagData
 import org.tasks.data.entity.Task
-import org.tasks.data.entity.TaskAttachment
-import org.tasks.data.entity.TaskListMetadata
 import org.tasks.data.entity.UserActivity
 
-@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
 @Serializable
-class BackupContainer(
+class TaskBackup(
-    val tasks: List<TaskBackup>? = null,
-    val places: List<Place>? = null,
-    val tags: List<TagData>? = null,
-    val filters: List<Filter>? = null,
-    val caldavAccounts: List<CaldavAccount>? = null,
-    val caldavCalendars: List<CaldavCalendar>? = null,
-    val taskListMetadata: List<TaskListMetadata>? = null,
-    val taskAttachments: List<TaskAttachment>? = null,
-    val intPrefs: Map<String, Integer>? = null,
-    val longPrefs: Map<String, java.lang.Long>? = null,
-    val stringPrefs: Map<String, String>? = null,
-    val boolPrefs: Map<String, java.lang.Boolean>? = null,
-    val setPrefs: Map<String, java.util.Set<String>>? = null,
-    val googleTaskAccounts: List<GoogleTaskAccount>? = null,
-    val googleTaskLists: List<GoogleTaskList>? = null,
-) {
-    @Serializable
-    class TaskBackup(
     val task: Task,
     val alarms: List<Alarm>? = null,
     val geofences: List<Geofence>? = null,
@@ -51,5 +23,4 @@ class BackupContainer(
     val vtodo: String? = null,
     val google: List<GoogleTask>? = null,
     val locations: List<LegacyLocation>? = null,
-    )
+)
-}

app/src/main/java/org/tasks/backup/TasksJsonExporter.kt

@@ -37,7 +37,6 @@ import java.io.File
 import java.io.IOException
 import java.io.OutputStream
 import java.io.Writer
-import java.nio.charset.Charset
 import java.util.Set
 import javax.inject.Inject
 
@@ -182,7 +181,6 @@ class TasksJsonExporter @Inject constructor(
     }
 
     companion object {
-        val UTF_8: Charset = Charset.forName("UTF-8")
         private const val MIME = "application/json"
         private const val EXTENSION = ".json"
         private val dateForExport: String

app/src/main/java/org/tasks/backup/TasksJsonImporter.kt

@@ -4,6 +4,7 @@ import android.app.ProgressDialog
 import android.content.Context
 import android.net.Uri
 import android.os.Handler
+import android.util.JsonReader
 import com.todoroo.astrid.dao.TaskDao
 import com.todoroo.astrid.service.TaskCreator.Companion.getDefaultAlarms
 import com.todoroo.astrid.service.TaskMover
@@ -16,13 +17,11 @@ import com.todoroo.astrid.service.Upgrader.Companion.V6_4
 import com.todoroo.astrid.service.Upgrader.Companion.getAndroidColor
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.decodeFromJsonElement
-import kotlinx.serialization.json.int
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
 import org.tasks.LocalBroadcastManager
 import org.tasks.R
 import org.tasks.caldav.VtodoCache
+import org.tasks.data.GoogleTaskAccount
+import org.tasks.data.GoogleTaskList
 import org.tasks.data.convertPictureUri
 import org.tasks.data.dao.AlarmDao
 import org.tasks.data.dao.CaldavDao
@@ -38,22 +37,27 @@ import org.tasks.data.entity.CaldavAccount
 import org.tasks.data.entity.CaldavAccount.Companion.TYPE_GOOGLE_TASKS
 import org.tasks.data.entity.CaldavCalendar
 import org.tasks.data.entity.CaldavTask
+import org.tasks.data.entity.Filter
 import org.tasks.data.entity.Geofence
 import org.tasks.data.entity.Place
 import org.tasks.data.entity.Tag
 import org.tasks.data.entity.TagData
 import org.tasks.data.entity.Task
+import org.tasks.data.entity.TaskAttachment
+import org.tasks.data.entity.TaskListMetadata
 import org.tasks.db.Migrations.repeatFrom
 import org.tasks.db.Migrations.withoutFrom
+import org.tasks.extensions.forEach
+import org.tasks.extensions.jsonString
 import org.tasks.filters.FilterCriteriaProvider
 import org.tasks.preferences.Preferences
 import timber.log.Timber
 import java.io.FileNotFoundException
 import java.io.IOException
 import java.io.InputStream
-import java.io.InputStreamReader
 import javax.inject.Inject
 
+@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
 class TasksJsonImporter @Inject constructor(
     private val tagDataDao: TagDataDao,
     private val userActivityDao: UserActivityDao,
@@ -70,8 +74,7 @@ class TasksJsonImporter @Inject constructor(
     private val taskListMetadataDao: TaskListMetadataDao,
     private val vtodoCache: VtodoCache,
     private val filterCriteriaProvider: FilterCriteriaProvider,
-    ) {
+) {
-
     private val result = ImportResult()
 
     private fun setProgressMessage(
@@ -83,67 +86,65 @@ class TasksJsonImporter @Inject constructor(
     }
 
     suspend fun importTasks(context: Context, backupFile: Uri?, progressDialog: ProgressDialog?): ImportResult {
+        Timber.d("Importing backup file $backupFile")
         val handler = Handler(context.mainLooper)
         val `is`: InputStream? = try {
             context.contentResolver.openInputStream(backupFile!!)
         } catch (e: FileNotFoundException) {
             throw IllegalStateException(e)
         }
-        val reader = InputStreamReader(`is`, TasksJsonExporter.UTF_8)
+        val bufferedReader = `is`!!.bufferedReader()
-        val input = Json.parseToJsonElement(reader.readText())
+        val reader = JsonReader(bufferedReader)
+        reader.isLenient = true
+        val ignoreKeys = ignorePrefs.map { context.getString(it) }
         try {
-            val data = input.jsonObject["data"]!!
+            reader.beginObject()
-            val version = input.jsonObject["version"]!!.jsonPrimitive.int
+            var version = 0
-            val backupContainer = json.decodeFromJsonElement<BackupContainer>(data)
+            while (reader.hasNext()) {
-            backupContainer.tags?.forEach { tagData ->
+                when (val name = reader.nextName()) {
-                findTagData(tagData)?.let {
+                    "version" -> version = reader.nextInt().also { Timber.d("Backup version: $it") }
-                    return@forEach
+                    "timestamp" -> reader.nextLong().let { Timber.d("Backup timestamp: $it") }
-                }
+                    "data" -> {
-                tagDataDao.insert(
+                        reader.beginObject()
-                    tagData.copy(
+                        while (reader.hasNext()) {
-                        color = themeToColor(context, version, tagData.color ?: 0),
+                            when (val element = reader.nextName()) {
-                        icon = tagData.icon.migrateLegacyIcon(),
+                                "tasks" -> {
-                    )
+                                    reader.forEach<TaskBackup> { backup ->
-                )
+                                        result.taskCount++
-            }
+                                        setProgressMessage(
-            backupContainer.googleTaskAccounts?.forEach { googleTaskAccount ->
+                                            handler,
-                if (caldavDao.getAccount(TYPE_GOOGLE_TASKS, googleTaskAccount.account!!) == null) {
+                                            progressDialog,
-                    caldavDao.insert(
+                                            context.getString(R.string.import_progress_read, result.taskCount))
-                        CaldavAccount(
+                                        importTask(backup, version)
-                            accountType = TYPE_GOOGLE_TASKS,
-                            uuid = googleTaskAccount.account,
-                            name = googleTaskAccount.account,
-                            username = googleTaskAccount.account,
-                        )
-                    )
                                     }
                                 }
-            backupContainer.places?.forEach { place ->
+                                "places" -> reader.forEach<Place> { place ->
                                     if (locationDao.getByUid(place.uid!!) == null) {
                                         locationDao.insert(
-                        place.copy(
+                                            place.copy(icon = place.icon.migrateLegacyIcon())
-                            icon = place.icon.migrateLegacyIcon(),
-                        )
                                         )
                                     }
                                 }
-            backupContainer.googleTaskLists?.forEach { googleTaskList ->
+                                "tags" -> reader.forEach<TagData> { tagData ->
-                if (caldavDao.getCalendar(googleTaskList.remoteId!!) == null) {
+                                    findTagData(tagData)?.let {
-                    caldavDao.insert(
+                                        return@forEach
-                        CaldavCalendar(
+                                    }
-                            account = googleTaskList.account,
+                                    tagDataDao.insert(
-                            uuid = googleTaskList.remoteId,
+                                        tagData.copy(
-                            color = themeToColor(context, version, googleTaskList.color ?: 0),
+                                            color = themeToColor(context, version, tagData.color ?: 0),
-                            icon = googleTaskList.icon?.toString().migrateLegacyIcon(),
+                                            icon = tagData.icon.migrateLegacyIcon(),
                                         )
                                     )
                                 }
+                                "filters" -> reader.forEach<Filter> {
+                                    it
+                                        .let {
+                                            if (version < Upgrade_13_2.VERSION)
+                                                filterCriteriaProvider.rebuildFilter(it)
+                                            else
+                                                it
                                         }
-            backupContainer.filters
+                                        .let { filter ->
-                ?.map {
-                    if (version < Upgrade_13_2.VERSION) filterCriteriaProvider.rebuildFilter(it)
-                    else it
-                }?.forEach { filter ->
                                             if (filterDao.getByName(filter.title!!) == null) {
                                                 filterDao.insert(
                                                     filter.copy(
@@ -153,12 +154,13 @@ class TasksJsonImporter @Inject constructor(
                                                 )
                                             }
                                         }
-            backupContainer.caldavAccounts?.forEach { account ->
+                                }
+                                "caldavAccounts" -> reader.forEach<CaldavAccount> { account ->
                                     if (caldavDao.getAccountByUuid(account.uuid!!) == null) {
                                         caldavDao.insert(account)
                                     }
                                 }
-            backupContainer.caldavCalendars?.forEach { calendar ->
+                                "caldavCalendars" -> reader.forEach<CaldavCalendar> { calendar ->
                                     if (caldavDao.getCalendarByUuid(calendar.uuid!!) == null) {
                                         caldavDao.insert(
                                             calendar.copy(
@@ -168,28 +170,102 @@ class TasksJsonImporter @Inject constructor(
                                         )
                                     }
                                 }
-            backupContainer.taskListMetadata?.forEach { tlm ->
+                                "taskListMetadata" -> reader.forEach<TaskListMetadata> { tlm ->
                                     val id = tlm.filter.takeIf { it?.isNotBlank() == true } ?: tlm.tagUuid!!
                                     if (taskListMetadataDao.fetchByTagOrFilter(id) == null) {
                                         taskListMetadataDao.insert(tlm)
                                     }
                                 }
-            backupContainer.taskAttachments?.forEach { attachment ->
+                                "taskAttachments" -> reader.forEach<TaskAttachment> { attachment ->
                                     if (taskAttachmentDao.getAttachment(attachment.remoteId) == null) {
                                         taskAttachmentDao.insert(attachment)
                                     }
                                 }
-            backupContainer.tasks?.forEach { backup ->
+                                "intPrefs" ->
-                result.taskCount++
+                                    Json.decodeFromString<Map<String, Integer>>(reader.jsonString())
-                setProgressMessage(
+                                        .filterNot { (key, _) -> ignoreKeys.contains(key) }
-                        handler,
+                                        .forEach { (k, v) -> preferences.setInt(k, v as Int) }
-                        progressDialog,
+                                "longPrefs" ->
-                        context.getString(R.string.import_progress_read, result.taskCount))
+                                    Json.decodeFromString<Map<String, java.lang.Long>>(reader.jsonString())
+                                        .filterNot { (key, _) -> ignoreKeys.contains(key) }
+                                        .forEach { (k, v) -> preferences.setLong(k, v as Long)}
+                                "stringPrefs" ->
+                                    Json.decodeFromString<Map<String, String>>(reader.jsonString())
+                                        .filterNot { (k, _) -> ignoreKeys.contains(k) }
+                                        .forEach { (k, v) -> preferences.setString(k, v)}
+                                "boolPrefs" ->
+                                    Json.decodeFromString<Map<String, java.lang.Boolean>>(reader.jsonString())
+                                        .filterNot { (k, _) -> ignoreKeys.contains(k) }
+                                        .forEach { (k, v) -> preferences.setBoolean(k, v as Boolean) }
+                                "setPrefs" ->
+                                    Json.decodeFromString<Map<String, Set<String>>>(reader.jsonString())
+                                        .filterNot { (k, _) -> ignoreKeys.contains(k) }
+                                        .forEach { (k, v) -> preferences.setStringSet(k, v as HashSet<String>)}
+                                "googleTaskAccounts" -> reader.forEach<GoogleTaskAccount> { googleTaskAccount ->
+                                    if (caldavDao.getAccount(TYPE_GOOGLE_TASKS, googleTaskAccount.account!!) == null) {
+                                        caldavDao.insert(
+                                            CaldavAccount(
+                                                accountType = TYPE_GOOGLE_TASKS,
+                                                uuid = googleTaskAccount.account,
+                                                name = googleTaskAccount.account,
+                                                username = googleTaskAccount.account,
+                                            )
+                                        )
+                                    }
+                                }
+                                "googleTaskLists" -> reader.forEach<GoogleTaskList> { googleTaskList ->
+                                    if (caldavDao.getCalendar(googleTaskList.remoteId!!) == null) {
+                                        caldavDao.insert(
+                                            CaldavCalendar(
+                                                account = googleTaskList.account,
+                                                uuid = googleTaskList.remoteId,
+                                                color = themeToColor(context, version, googleTaskList.color ?: 0),
+                                                icon = googleTaskList.icon?.toString().migrateLegacyIcon(),
+                                            )
+                                        )
+                                    }
+                                }
+                                else -> {
+                                    Timber.w("Skipping $element")
+                                    reader.skipValue()
+                                }
+                            }
+                        }
+                        reader.endObject()
+                    }
+                    else -> {
+                        Timber.w("Skipping $name")
+                        reader.skipValue()
+                    }
+                }
+            }
+            if (version < Upgrader.V8_2) {
+                val themeIndex = preferences.getInt(R.string.p_theme_color, 7)
+                preferences.setInt(
+                        R.string.p_theme_color,
+                        getAndroidColor(context, themeIndex))
+            }
+            if (version < Upgrader.V9_6) {
+                taskMover.migrateLocalTasks()
+            }
+            Timber.d("Updating parents")
+            caldavDao.updateParents()
+            reader.close()
+            bufferedReader.close()
+            `is`!!.close()
+        } catch (e: IOException) {
+            Timber.e(e)
+        }
+        localBroadcastManager.broadcastRefresh()
+        return result
+    }
+
+    private suspend fun importTask(backup: TaskBackup, version: Int) {
         val task = backup.task
         taskDao.fetch(task.uuid)
             ?.let {
                 result.skipCount++
-                            return@forEach
+                return
             }
         if (
             backup.caldavTasks
@@ -209,7 +285,7 @@ class TasksJsonImporter @Inject constructor(
                 } == true
         ) {
             result.skipCount++
-                    return@forEach
+            return
         }
         task.suppressRefresh()
         task.suppressSync()
@@ -305,46 +381,6 @@ class TasksJsonImporter @Inject constructor(
         }
         result.importCount++
     }
-            Timber.d("Updating parents")
-            caldavDao.updateParents()
-            val ignoreKeys = ignorePrefs.map { context.getString(it) }
-            backupContainer
-                    .intPrefs
-                    ?.filterNot { (key, _) -> ignoreKeys.contains(key) }
-                    ?.forEach { (key, value) -> preferences.setInt(key, value as Int) }
-            backupContainer
-                    .longPrefs
-                    ?.filterNot { (key, _) -> ignoreKeys.contains(key) }
-                    ?.forEach { (key, value) -> preferences.setLong(key, value as Long) }
-            backupContainer
-                    .stringPrefs
-                    ?.filterNot { (key, _) -> ignoreKeys.contains(key) }
-                    ?.forEach { (key, value) -> preferences.setString(key, value) }
-            backupContainer
-                    .boolPrefs
-                    ?.filterNot { (key, _) -> ignoreKeys.contains(key) }
-                    ?.forEach { (key, value) -> preferences.setBoolean(key, value as Boolean) }
-            backupContainer
-                    .setPrefs
-                    ?.filterNot { (key, _) -> ignoreKeys.contains(key) }
-                    ?.forEach { (key, value) -> preferences.setStringSet(key, value as HashSet<String>)}
-            if (version < Upgrader.V8_2) {
-                val themeIndex = preferences.getInt(R.string.p_theme_color, 7)
-                preferences.setInt(
-                        R.string.p_theme_color,
-                        getAndroidColor(context, themeIndex))
-            }
-            if (version < Upgrader.V9_6) {
-                taskMover.migrateLocalTasks()
-            }
-            reader.close()
-            `is`!!.close()
-        } catch (e: IOException) {
-            Timber.e(e)
-        }
-        localBroadcastManager.broadcastRefresh()
-        return result
-    }
 
     private suspend fun findTagData(tagData: TagData) =
             findTagData(tagData.remoteId!!, tagData.name!!)

app/src/main/java/org/tasks/extensions/JsonReaderExtensions.kt

@@ -0,0 +1,66 @@
+package org.tasks.extensions
+
+import android.util.JsonReader
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import java.io.StringWriter
+
+inline fun <reified T>JsonReader.forEach(callback: (@Serializable T) -> Unit) where T : Any {
+    beginArray()
+    while (hasNext()) callback(Json.decodeFromString(jsonString()))
+    endArray()
+}
+
+fun JsonReader.jsonString(): String {
+    val stringWriter = StringWriter()
+    val jsonWriter = android.util.JsonWriter(stringWriter)
+    jsonWriter.isLenient = true
+
+    copyJsonToken(this, jsonWriter)
+    jsonWriter.close()
+
+    return stringWriter.toString()
+}
+
+// Helper function to copy JSON tokens
+private fun copyJsonToken(reader: JsonReader, writer: android.util.JsonWriter) {
+    when (reader.peek()) {
+        android.util.JsonToken.BEGIN_ARRAY -> {
+            reader.beginArray()
+            writer.beginArray()
+            while (reader.hasNext()) {
+                copyJsonToken(reader, writer)
+            }
+            reader.endArray()
+            writer.endArray()
+        }
+        android.util.JsonToken.BEGIN_OBJECT -> {
+            reader.beginObject()
+            writer.beginObject()
+            while (reader.hasNext()) {
+                val name = reader.nextName()
+                writer.name(name)
+                copyJsonToken(reader, writer)
+            }
+            reader.endObject()
+            writer.endObject()
+        }
+        android.util.JsonToken.BOOLEAN -> {
+            val value = reader.nextBoolean()
+            writer.value(value)
+        }
+        android.util.JsonToken.NULL -> {
+            reader.nextNull()
+            writer.nullValue()
+        }
+        android.util.JsonToken.NUMBER -> {
+            val value = reader.nextString()
+            writer.value(value)
+        }
+        android.util.JsonToken.STRING -> {
+            val value = reader.nextString()
+            writer.value(value)
+        }
+        else -> throw IllegalStateException("Unexpected token: ${reader.peek()}")
+    }
+}