Streaming backup file import

pull/3436/head
Alex Baker 9 months ago
parent 4c4d5cdc14
commit 62f8e7fcdb

@ -3,53 +3,24 @@ package org.tasks.backup
import kotlinx.serialization.Serializable
import org.tasks.backup.TasksJsonImporter.LegacyLocation
import org.tasks.data.GoogleTask
import org.tasks.data.GoogleTaskAccount
import org.tasks.data.GoogleTaskList
import org.tasks.data.entity.Alarm
import org.tasks.data.entity.Attachment
import org.tasks.data.entity.CaldavAccount
import org.tasks.data.entity.CaldavCalendar
import org.tasks.data.entity.CaldavTask
import org.tasks.data.entity.Filter
import org.tasks.data.entity.Geofence
import org.tasks.data.entity.Place
import org.tasks.data.entity.Tag
import org.tasks.data.entity.TagData
import org.tasks.data.entity.Task
import org.tasks.data.entity.TaskAttachment
import org.tasks.data.entity.TaskListMetadata
import org.tasks.data.entity.UserActivity
@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
@Serializable
class BackupContainer(
val tasks: List<TaskBackup>? = null,
val places: List<Place>? = null,
val tags: List<TagData>? = null,
val filters: List<Filter>? = null,
val caldavAccounts: List<CaldavAccount>? = null,
val caldavCalendars: List<CaldavCalendar>? = null,
val taskListMetadata: List<TaskListMetadata>? = null,
val taskAttachments: List<TaskAttachment>? = null,
val intPrefs: Map<String, Integer>? = null,
val longPrefs: Map<String, java.lang.Long>? = null,
val stringPrefs: Map<String, String>? = null,
val boolPrefs: Map<String, java.lang.Boolean>? = null,
val setPrefs: Map<String, java.util.Set<String>>? = null,
val googleTaskAccounts: List<GoogleTaskAccount>? = null,
val googleTaskLists: List<GoogleTaskList>? = null,
) {
@Serializable
class TaskBackup(
val task: Task,
val alarms: List<Alarm>? = null,
val geofences: List<Geofence>? = null,
val tags: List<Tag>? = null,
val comments: List<UserActivity>? = null,
val attachments: List<Attachment>? = null,
val caldavTasks: List<CaldavTask>? = null,
val vtodo: String? = null,
val google: List<GoogleTask>? = null,
val locations: List<LegacyLocation>? = null,
)
}
class TaskBackup(
val task: Task,
val alarms: List<Alarm>? = null,
val geofences: List<Geofence>? = null,
val tags: List<Tag>? = null,
val comments: List<UserActivity>? = null,
val attachments: List<Attachment>? = null,
val caldavTasks: List<CaldavTask>? = null,
val vtodo: String? = null,
val google: List<GoogleTask>? = null,
val locations: List<LegacyLocation>? = null,
)

@ -37,7 +37,6 @@ import java.io.File
import java.io.IOException
import java.io.OutputStream
import java.io.Writer
import java.nio.charset.Charset
import java.util.Set
import javax.inject.Inject
@ -182,7 +181,6 @@ class TasksJsonExporter @Inject constructor(
}
companion object {
val UTF_8: Charset = Charset.forName("UTF-8")
private const val MIME = "application/json"
private const val EXTENSION = ".json"
private val dateForExport: String

@ -4,6 +4,7 @@ import android.app.ProgressDialog
import android.content.Context
import android.net.Uri
import android.os.Handler
import android.util.JsonReader
import com.todoroo.astrid.dao.TaskDao
import com.todoroo.astrid.service.TaskCreator.Companion.getDefaultAlarms
import com.todoroo.astrid.service.TaskMover
@ -16,13 +17,11 @@ import com.todoroo.astrid.service.Upgrader.Companion.V6_4
import com.todoroo.astrid.service.Upgrader.Companion.getAndroidColor
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromJsonElement
import kotlinx.serialization.json.int
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import org.tasks.LocalBroadcastManager
import org.tasks.R
import org.tasks.caldav.VtodoCache
import org.tasks.data.GoogleTaskAccount
import org.tasks.data.GoogleTaskList
import org.tasks.data.convertPictureUri
import org.tasks.data.dao.AlarmDao
import org.tasks.data.dao.CaldavDao
@ -38,22 +37,27 @@ import org.tasks.data.entity.CaldavAccount
import org.tasks.data.entity.CaldavAccount.Companion.TYPE_GOOGLE_TASKS
import org.tasks.data.entity.CaldavCalendar
import org.tasks.data.entity.CaldavTask
import org.tasks.data.entity.Filter
import org.tasks.data.entity.Geofence
import org.tasks.data.entity.Place
import org.tasks.data.entity.Tag
import org.tasks.data.entity.TagData
import org.tasks.data.entity.Task
import org.tasks.data.entity.TaskAttachment
import org.tasks.data.entity.TaskListMetadata
import org.tasks.db.Migrations.repeatFrom
import org.tasks.db.Migrations.withoutFrom
import org.tasks.extensions.forEach
import org.tasks.extensions.jsonString
import org.tasks.filters.FilterCriteriaProvider
import org.tasks.preferences.Preferences
import timber.log.Timber
import java.io.FileNotFoundException
import java.io.IOException
import java.io.InputStream
import java.io.InputStreamReader
import javax.inject.Inject
@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
class TasksJsonImporter @Inject constructor(
private val tagDataDao: TagDataDao,
private val userActivityDao: UserActivityDao,
@ -70,8 +74,7 @@ class TasksJsonImporter @Inject constructor(
private val taskListMetadataDao: TaskListMetadataDao,
private val vtodoCache: VtodoCache,
private val filterCriteriaProvider: FilterCriteriaProvider,
) {
) {
private val result = ImportResult()
private fun setProgressMessage(
@ -83,251 +86,159 @@ class TasksJsonImporter @Inject constructor(
}
suspend fun importTasks(context: Context, backupFile: Uri?, progressDialog: ProgressDialog?): ImportResult {
Timber.d("Importing backup file $backupFile")
val handler = Handler(context.mainLooper)
val `is`: InputStream? = try {
context.contentResolver.openInputStream(backupFile!!)
} catch (e: FileNotFoundException) {
throw IllegalStateException(e)
}
val reader = InputStreamReader(`is`, TasksJsonExporter.UTF_8)
val input = Json.parseToJsonElement(reader.readText())
val bufferedReader = `is`!!.bufferedReader()
val reader = JsonReader(bufferedReader)
reader.isLenient = true
val ignoreKeys = ignorePrefs.map { context.getString(it) }
try {
val data = input.jsonObject["data"]!!
val version = input.jsonObject["version"]!!.jsonPrimitive.int
val backupContainer = json.decodeFromJsonElement<BackupContainer>(data)
backupContainer.tags?.forEach { tagData ->
findTagData(tagData)?.let {
return@forEach
}
tagDataDao.insert(
tagData.copy(
color = themeToColor(context, version, tagData.color ?: 0),
icon = tagData.icon.migrateLegacyIcon(),
)
)
}
backupContainer.googleTaskAccounts?.forEach { googleTaskAccount ->
if (caldavDao.getAccount(TYPE_GOOGLE_TASKS, googleTaskAccount.account!!) == null) {
caldavDao.insert(
CaldavAccount(
accountType = TYPE_GOOGLE_TASKS,
uuid = googleTaskAccount.account,
name = googleTaskAccount.account,
username = googleTaskAccount.account,
)
)
}
}
backupContainer.places?.forEach { place ->
if (locationDao.getByUid(place.uid!!) == null) {
locationDao.insert(
place.copy(
icon = place.icon.migrateLegacyIcon(),
)
)
}
}
backupContainer.googleTaskLists?.forEach { googleTaskList ->
if (caldavDao.getCalendar(googleTaskList.remoteId!!) == null) {
caldavDao.insert(
CaldavCalendar(
account = googleTaskList.account,
uuid = googleTaskList.remoteId,
color = themeToColor(context, version, googleTaskList.color ?: 0),
icon = googleTaskList.icon?.toString().migrateLegacyIcon(),
)
)
}
}
backupContainer.filters
?.map {
if (version < Upgrade_13_2.VERSION) filterCriteriaProvider.rebuildFilter(it)
else it
}?.forEach { filter ->
if (filterDao.getByName(filter.title!!) == null) {
filterDao.insert(
filter.copy(
color = themeToColor(context, version, filter.color ?: 0),
icon = filter.icon.migrateLegacyIcon(),
)
)
}
}
backupContainer.caldavAccounts?.forEach { account ->
if (caldavDao.getAccountByUuid(account.uuid!!) == null) {
caldavDao.insert(account)
}
}
backupContainer.caldavCalendars?.forEach { calendar ->
if (caldavDao.getCalendarByUuid(calendar.uuid!!) == null) {
caldavDao.insert(
calendar.copy(
color = themeToColor(context, version, calendar.color),
icon = calendar.icon.migrateLegacyIcon(),
)
)
}
}
backupContainer.taskListMetadata?.forEach { tlm ->
val id = tlm.filter.takeIf { it?.isNotBlank() == true } ?: tlm.tagUuid!!
if (taskListMetadataDao.fetchByTagOrFilter(id) == null) {
taskListMetadataDao.insert(tlm)
}
}
backupContainer.taskAttachments?.forEach { attachment ->
if (taskAttachmentDao.getAttachment(attachment.remoteId) == null) {
taskAttachmentDao.insert(attachment)
}
}
backupContainer.tasks?.forEach { backup ->
result.taskCount++
setProgressMessage(
handler,
progressDialog,
context.getString(R.string.import_progress_read, result.taskCount))
val task = backup.task
taskDao.fetch(task.uuid)
?.let {
result.skipCount++
return@forEach
}
if (
backup.caldavTasks
?.filter { it.deleted == 0L }
?.any {
val existing = if (
it.obj.isNullOrBlank() ||
it.obj == "null.ics" // caused by an old bug
) {
it.remoteId?.let { remoteId ->
caldavDao.getTaskByRemoteId(it.calendar!!, remoteId)
reader.beginObject()
var version = 0
while (reader.hasNext()) {
when (val name = reader.nextName()) {
"version" -> version = reader.nextInt().also { Timber.d("Backup version: $it") }
"timestamp" -> reader.nextLong().let { Timber.d("Backup timestamp: $it") }
"data" -> {
reader.beginObject()
while (reader.hasNext()) {
when (val element = reader.nextName()) {
"tasks" -> {
reader.forEach<TaskBackup> { backup ->
result.taskCount++
setProgressMessage(
handler,
progressDialog,
context.getString(R.string.import_progress_read, result.taskCount))
importTask(backup, version)
}
}
"places" -> reader.forEach<Place> { place ->
if (locationDao.getByUid(place.uid!!) == null) {
locationDao.insert(
place.copy(icon = place.icon.migrateLegacyIcon())
)
}
}
"tags" -> reader.forEach<TagData> { tagData ->
findTagData(tagData)?.let {
return@forEach
}
tagDataDao.insert(
tagData.copy(
color = themeToColor(context, version, tagData.color ?: 0),
icon = tagData.icon.migrateLegacyIcon(),
)
)
}
"filters" -> reader.forEach<Filter> {
it
.let {
if (version < Upgrade_13_2.VERSION)
filterCriteriaProvider.rebuildFilter(it)
else
it
}
.let { filter ->
if (filterDao.getByName(filter.title!!) == null) {
filterDao.insert(
filter.copy(
color = themeToColor(context, version, filter.color ?: 0),
icon = filter.icon.migrateLegacyIcon(),
)
)
}
}
}
"caldavAccounts" -> reader.forEach<CaldavAccount> { account ->
if (caldavDao.getAccountByUuid(account.uuid!!) == null) {
caldavDao.insert(account)
}
}
"caldavCalendars" -> reader.forEach<CaldavCalendar> { calendar ->
if (caldavDao.getCalendarByUuid(calendar.uuid!!) == null) {
caldavDao.insert(
calendar.copy(
color = themeToColor(context, version, calendar.color),
icon = calendar.icon.migrateLegacyIcon(),
)
)
}
}
"taskListMetadata" -> reader.forEach<TaskListMetadata> { tlm ->
val id = tlm.filter.takeIf { it?.isNotBlank() == true } ?: tlm.tagUuid!!
if (taskListMetadataDao.fetchByTagOrFilter(id) == null) {
taskListMetadataDao.insert(tlm)
}
}
"taskAttachments" -> reader.forEach<TaskAttachment> { attachment ->
if (taskAttachmentDao.getAttachment(attachment.remoteId) == null) {
taskAttachmentDao.insert(attachment)
}
}
"intPrefs" ->
Json.decodeFromString<Map<String, Integer>>(reader.jsonString())
.filterNot { (key, _) -> ignoreKeys.contains(key) }
.forEach { (k, v) -> preferences.setInt(k, v as Int) }
"longPrefs" ->
Json.decodeFromString<Map<String, java.lang.Long>>(reader.jsonString())
.filterNot { (key, _) -> ignoreKeys.contains(key) }
.forEach { (k, v) -> preferences.setLong(k, v as Long)}
"stringPrefs" ->
Json.decodeFromString<Map<String, String>>(reader.jsonString())
.filterNot { (k, _) -> ignoreKeys.contains(k) }
.forEach { (k, v) -> preferences.setString(k, v)}
"boolPrefs" ->
Json.decodeFromString<Map<String, java.lang.Boolean>>(reader.jsonString())
.filterNot { (k, _) -> ignoreKeys.contains(k) }
.forEach { (k, v) -> preferences.setBoolean(k, v as Boolean) }
"setPrefs" ->
Json.decodeFromString<Map<String, Set<String>>>(reader.jsonString())
.filterNot { (k, _) -> ignoreKeys.contains(k) }
.forEach { (k, v) -> preferences.setStringSet(k, v as HashSet<String>)}
"googleTaskAccounts" -> reader.forEach<GoogleTaskAccount> { googleTaskAccount ->
if (caldavDao.getAccount(TYPE_GOOGLE_TASKS, googleTaskAccount.account!!) == null) {
caldavDao.insert(
CaldavAccount(
accountType = TYPE_GOOGLE_TASKS,
uuid = googleTaskAccount.account,
name = googleTaskAccount.account,
username = googleTaskAccount.account,
)
)
}
}
"googleTaskLists" -> reader.forEach<GoogleTaskList> { googleTaskList ->
if (caldavDao.getCalendar(googleTaskList.remoteId!!) == null) {
caldavDao.insert(
CaldavCalendar(
account = googleTaskList.account,
uuid = googleTaskList.remoteId,
color = themeToColor(context, version, googleTaskList.color ?: 0),
icon = googleTaskList.icon?.toString().migrateLegacyIcon(),
)
)
}
}
else -> {
Timber.w("Skipping $element")
reader.skipValue()
}
} else {
caldavDao.getTask(it.calendar!!, it.obj!!)
}
existing != null
} == true
) {
result.skipCount++
return@forEach
}
task.suppressRefresh()
task.suppressSync()
taskDao.createNew(task)
val taskId = task.id
val taskUuid = task.uuid
backup.alarms?.map { it.copy(task = taskId) }?.let { alarmDao.insert(it) }
if (version < V12_4) {
task.defaultReminders(task.ringFlags)
alarmDao.insert(task.getDefaultAlarms())
task.ringFlags = when {
task.isNotifyModeFive -> Task.NOTIFY_MODE_FIVE
task.isNotifyModeNonstop -> Task.NOTIFY_MODE_NONSTOP
else -> 0
}
taskDao.save(task)
}
if (version < V12_8) {
task.repeatFrom = task.recurrence.repeatFrom()
task.recurrence = task.recurrence.withoutFrom()
}
backup.comments?.forEach { comment ->
comment.targetId = taskUuid
if (version < V6_4) {
comment.convertPictureUri()
}
reader.endObject()
}
userActivityDao.createNew(comment)
}
backup.google?.forEach { googleTask ->
caldavDao.insert(
CaldavTask(
task = taskId,
calendar = googleTask.listId,
remoteId = googleTask.remoteId,
remoteOrder = googleTask.remoteOrder,
remoteParent = googleTask.remoteParent,
lastSync = googleTask.lastSync,
)
)
}
backup.locations?.forEach { location ->
val place = Place(
longitude = location.longitude,
latitude = location.latitude,
name = location.name,
address = location.address,
url = location.url,
phone = location.phone,
)
locationDao.insert(place)
locationDao.insert(
Geofence(
task = taskId,
place = place.uid,
isArrival = location.arrival,
isDeparture = location.departure,
)
)
}
backup.tags?.forEach tags@ { tag ->
val tagData = findTagData(tag) ?: return@tags
tagDao.insert(
tag.copy(
task = taskId,
taskUid = task.remoteId,
tagUid = tagData.remoteId
)
)
}
backup.geofences?.forEach { geofence ->
locationDao.insert(
geofence.copy(task = taskId)
)
}
backup.attachments
?.mapNotNull { taskAttachmentDao.getAttachment(it.attachmentUid) }
?.map {
Attachment(
task = taskId,
fileId = it.id!!,
attachmentUid = it.remoteId,
)
else -> {
Timber.w("Skipping $name")
reader.skipValue()
}
?.let { taskAttachmentDao.insert(it) }
backup.caldavTasks?.forEach { caldavTask ->
caldavDao.insert(caldavTask.copy(task = taskId))
}
backup.vtodo?.let {
val caldavTask =
backup.caldavTasks?.firstOrNull { t -> !t.isDeleted() } ?: return@let
val caldavCalendar = caldavDao.getCalendar(caldavTask.calendar!!) ?: return@let
vtodoCache.putVtodo(caldavCalendar, caldavTask, it)
}
result.importCount++
}
Timber.d("Updating parents")
caldavDao.updateParents()
val ignoreKeys = ignorePrefs.map { context.getString(it) }
backupContainer
.intPrefs
?.filterNot { (key, _) -> ignoreKeys.contains(key) }
?.forEach { (key, value) -> preferences.setInt(key, value as Int) }
backupContainer
.longPrefs
?.filterNot { (key, _) -> ignoreKeys.contains(key) }
?.forEach { (key, value) -> preferences.setLong(key, value as Long) }
backupContainer
.stringPrefs
?.filterNot { (key, _) -> ignoreKeys.contains(key) }
?.forEach { (key, value) -> preferences.setString(key, value) }
backupContainer
.boolPrefs
?.filterNot { (key, _) -> ignoreKeys.contains(key) }
?.forEach { (key, value) -> preferences.setBoolean(key, value as Boolean) }
backupContainer
.setPrefs
?.filterNot { (key, _) -> ignoreKeys.contains(key) }
?.forEach { (key, value) -> preferences.setStringSet(key, value as HashSet<String>)}
if (version < Upgrader.V8_2) {
val themeIndex = preferences.getInt(R.string.p_theme_color, 7)
preferences.setInt(
@ -337,7 +248,10 @@ class TasksJsonImporter @Inject constructor(
if (version < Upgrader.V9_6) {
taskMover.migrateLocalTasks()
}
Timber.d("Updating parents")
caldavDao.updateParents()
reader.close()
bufferedReader.close()
`is`!!.close()
} catch (e: IOException) {
Timber.e(e)
@ -346,6 +260,128 @@ class TasksJsonImporter @Inject constructor(
return result
}
private suspend fun importTask(backup: TaskBackup, version: Int) {
val task = backup.task
taskDao.fetch(task.uuid)
?.let {
result.skipCount++
return
}
if (
backup.caldavTasks
?.filter { it.deleted == 0L }
?.any {
val existing = if (
it.obj.isNullOrBlank() ||
it.obj == "null.ics" // caused by an old bug
) {
it.remoteId?.let { remoteId ->
caldavDao.getTaskByRemoteId(it.calendar!!, remoteId)
}
} else {
caldavDao.getTask(it.calendar!!, it.obj!!)
}
existing != null
} == true
) {
result.skipCount++
return
}
task.suppressRefresh()
task.suppressSync()
taskDao.createNew(task)
val taskId = task.id
val taskUuid = task.uuid
backup.alarms?.map { it.copy(task = taskId) }?.let { alarmDao.insert(it) }
if (version < V12_4) {
task.defaultReminders(task.ringFlags)
alarmDao.insert(task.getDefaultAlarms())
task.ringFlags = when {
task.isNotifyModeFive -> Task.NOTIFY_MODE_FIVE
task.isNotifyModeNonstop -> Task.NOTIFY_MODE_NONSTOP
else -> 0
}
taskDao.save(task)
}
if (version < V12_8) {
task.repeatFrom = task.recurrence.repeatFrom()
task.recurrence = task.recurrence.withoutFrom()
}
backup.comments?.forEach { comment ->
comment.targetId = taskUuid
if (version < V6_4) {
comment.convertPictureUri()
}
userActivityDao.createNew(comment)
}
backup.google?.forEach { googleTask ->
caldavDao.insert(
CaldavTask(
task = taskId,
calendar = googleTask.listId,
remoteId = googleTask.remoteId,
remoteOrder = googleTask.remoteOrder,
remoteParent = googleTask.remoteParent,
lastSync = googleTask.lastSync,
)
)
}
backup.locations?.forEach { location ->
val place = Place(
longitude = location.longitude,
latitude = location.latitude,
name = location.name,
address = location.address,
url = location.url,
phone = location.phone,
)
locationDao.insert(place)
locationDao.insert(
Geofence(
task = taskId,
place = place.uid,
isArrival = location.arrival,
isDeparture = location.departure,
)
)
}
backup.tags?.forEach tags@ { tag ->
val tagData = findTagData(tag) ?: return@tags
tagDao.insert(
tag.copy(
task = taskId,
taskUid = task.remoteId,
tagUid = tagData.remoteId
)
)
}
backup.geofences?.forEach { geofence ->
locationDao.insert(
geofence.copy(task = taskId)
)
}
backup.attachments
?.mapNotNull { taskAttachmentDao.getAttachment(it.attachmentUid) }
?.map {
Attachment(
task = taskId,
fileId = it.id!!,
attachmentUid = it.remoteId,
)
}
?.let { taskAttachmentDao.insert(it) }
backup.caldavTasks?.forEach { caldavTask ->
caldavDao.insert(caldavTask.copy(task = taskId))
}
backup.vtodo?.let {
val caldavTask =
backup.caldavTasks?.firstOrNull { t -> !t.isDeleted() } ?: return@let
val caldavCalendar = caldavDao.getCalendar(caldavTask.calendar!!) ?: return@let
vtodoCache.putVtodo(caldavCalendar, caldavTask, it)
}
result.importCount++
}
private suspend fun findTagData(tagData: TagData) =
findTagData(tagData.remoteId!!, tagData.name!!)

@ -0,0 +1,66 @@
package org.tasks.extensions
import android.util.JsonReader
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import java.io.StringWriter
inline fun <reified T>JsonReader.forEach(callback: (@Serializable T) -> Unit) where T : Any {
beginArray()
while (hasNext()) callback(Json.decodeFromString(jsonString()))
endArray()
}
fun JsonReader.jsonString(): String {
val stringWriter = StringWriter()
val jsonWriter = android.util.JsonWriter(stringWriter)
jsonWriter.isLenient = true
copyJsonToken(this, jsonWriter)
jsonWriter.close()
return stringWriter.toString()
}
// Helper function to copy JSON tokens
private fun copyJsonToken(reader: JsonReader, writer: android.util.JsonWriter) {
when (reader.peek()) {
android.util.JsonToken.BEGIN_ARRAY -> {
reader.beginArray()
writer.beginArray()
while (reader.hasNext()) {
copyJsonToken(reader, writer)
}
reader.endArray()
writer.endArray()
}
android.util.JsonToken.BEGIN_OBJECT -> {
reader.beginObject()
writer.beginObject()
while (reader.hasNext()) {
val name = reader.nextName()
writer.name(name)
copyJsonToken(reader, writer)
}
reader.endObject()
writer.endObject()
}
android.util.JsonToken.BOOLEAN -> {
val value = reader.nextBoolean()
writer.value(value)
}
android.util.JsonToken.NULL -> {
reader.nextNull()
writer.nullValue()
}
android.util.JsonToken.NUMBER -> {
val value = reader.nextString()
writer.value(value)
}
android.util.JsonToken.STRING -> {
val value = reader.nextString()
writer.value(value)
}
else -> throw IllegalStateException("Unexpected token: ${reader.peek()}")
}
}
Loading…
Cancel
Save