Improved recursive query

app/src/main/java/com/todoroo/astrid/dao/TaskDao.kt

@@ -144,8 +144,7 @@ class TaskDao @Inject constructor(
 
     internal suspend fun insert(task: Task): Long = taskDao.insert(task)
 
-    internal suspend fun fetchTasks(queries: List<String>): List<TaskContainer> =
+    internal suspend fun fetchTasks(query: String): List<TaskContainer> = taskDao.fetchTasks(query)
-            taskDao.fetchTasks(queries)
 
     internal suspend fun getAll(): List<Task> = taskDao.getAll()
 

app/src/main/java/org/tasks/data/TaskDaoExtensions.kt

@@ -8,8 +8,6 @@ import org.tasks.data.sql.Field
 import org.tasks.data.sql.Query
 import org.tasks.filters.Filter
 import org.tasks.preferences.QueryPreferences
-import org.tasks.time.DateTimeUtils2.currentTimeMillis
-import timber.log.Timber
 
 suspend fun TaskDao.fetchTasks(preferences: QueryPreferences, filter: Filter): List<TaskContainer> =
     fetchTasks(TaskListQuery.getQuery(preferences, filter))
@@ -25,9 +23,7 @@ suspend fun TaskDao.fetchFiltered(filter: Filter): List<Task> = fetchFiltered(fi
 
 suspend fun TaskDao.fetchFiltered(queryTemplate: String): List<Task> {
     val query = getQuery(queryTemplate, Task.FIELDS)
-    val start = if (BuildConfig.DEBUG) currentTimeMillis() else 0
     val tasks = fetchTasks(query)
-    Timber.v("%sms: %s", currentTimeMillis() - start, query)
     return tasks.map(TaskContainer::task)
 }
 

app/src/main/java/org/tasks/widget/TasksWidgetViewFactory.kt

@@ -271,7 +271,7 @@ internal class TasksWidgetViewFactory(
         }
     }
 
-    private suspend fun getQuery(filter: Filter): List<String> {
+    private suspend fun getQuery(filter: Filter): String {
         subtasksHelper.applySubtasksToWidgetFilter(filter, widgetPreferences)
         return getQuery(widgetPreferences, filter)
     }

data/src/commonMain/kotlin/org/tasks/data/dao/TaskDao.kt

@@ -6,7 +6,6 @@ import androidx.room.Query
 import androidx.room.RawQuery
 import androidx.room.RoomRawQuery
 import androidx.room.Update
-import androidx.room.execSQL
 import co.touchlab.kermit.Logger
 import org.tasks.IS_DEBUG
 import org.tasks.data.TaskContainer
@@ -18,7 +17,6 @@ import org.tasks.data.entity.Alarm
 import org.tasks.data.entity.Task
 import org.tasks.data.sql.Criterion
 import org.tasks.data.sql.Functions
-import org.tasks.data.withTransaction
 import org.tasks.time.DateTimeUtils2
 
 private const val MAX_TIME = 9999999999999
@@ -110,21 +108,13 @@ FROM (
             + "WHERE completed > 0 AND calendarUri IS NOT NULL AND calendarUri != ''")
     abstract suspend fun clearCompletedCalendarEvents(): Int
 
-    open suspend fun fetchTasks(queries: List<String>): List<TaskContainer> =
+    suspend fun fetchTasks(query: String): List<TaskContainer> {
-        database.withTransaction {
+        val start = DateTimeUtils2.currentTimeMillis()
-            val start = if (IS_DEBUG) DateTimeUtils2.currentTimeMillis() else 0
+        val result = fetchRaw(RoomRawQuery(query))
-            val last = queries.size - 1
+        val end = DateTimeUtils2.currentTimeMillis()
-            for (i in 0 until last) {
+        Logger.v("TaskDao") { "${end - start}ms: ${query.replace(Regex("\\s+"), " ").trim()}" }
-                execSQL(queries[i])
+        return result
-            }
+    }
-            val result = fetchTasks(queries[last])
-            Logger.v("TaskDao") {
-                "${DateTimeUtils2.currentTimeMillis() - start}ms: ${queries.joinToString(";\n")}"
-            }
-            result
-        }
-
-    suspend fun fetchTasks(query: String): List<TaskContainer> = fetchRaw(RoomRawQuery(query))
 
     @RawQuery
     internal abstract suspend fun fetchRaw(query: RoomRawQuery): List<TaskContainer>

kmp/src/commonMain/kotlin/org/tasks/data/TaskListQuery.kt

@@ -24,10 +24,7 @@ object TaskListQuery {
             field("$CALDAV_METADATA_JOIN.cd_deleted").eq(0))
     val JOINS = """
         ${Join.left(CaldavTask.TABLE.`as`(CALDAV_METADATA_JOIN), JOIN_CALDAV)}
-        ${
+        ${Join.left(CaldavCalendar.TABLE, field("$CALDAV_METADATA_JOIN.cd_calendar").eq(CaldavCalendar.UUID))}
-        Join.left(
-        CaldavCalendar.TABLE, field("$CALDAV_METADATA_JOIN.cd_calendar").eq(
-            CaldavCalendar.UUID))}
         ${Join.left(CaldavAccount.TABLE, CaldavCalendar.ACCOUNT.eq(CaldavAccount.UUID))}
         ${Join.left(Geofence.TABLE, Geofence.TASK.eq(Task.ID))}
         ${Join.left(Place.TABLE, Place.UID.eq(Geofence.PLACE))}
@@ -43,7 +40,7 @@ object TaskListQuery {
     fun getQuery(
         preferences: QueryPreferences,
         filter: Filter,
-    ): MutableList<String> {
+    ): String {
         val start = currentTimeMillis()
         return when {
             filter.supportsManualSort() && preferences.isManualSort ->

kmp/src/commonMain/kotlin/org/tasks/data/TaskListQueryNonRecursive.kt

@@ -29,7 +29,7 @@ internal object TaskListQueryNonRecursive {
             field("tasks.completed > 0").`as`("parentComplete")
         )).toTypedArray()
 
-    fun getNonRecursiveQuery(filter: Filter, preferences: QueryPreferences): MutableList<String> {
+    fun getNonRecursiveQuery(filter: Filter, preferences: QueryPreferences): String {
         val joinedQuery = JOINS + if (filter is AstridOrderingFilter) filter.getSqlQuery() else filter.sql!!
         val sortMode = preferences.sortMode
         val groupMode = preferences.groupMode
@@ -52,10 +52,9 @@ internal object TaskListQueryNonRecursive {
             else ->
                 "$query GROUP BY ${Task.ID}"
         }
-        return mutableListOf(
+        return Query.select(*FIELDS.plus(sortGroup))
-                Query.select(*FIELDS.plus(sortGroup))
+            .withQueryTemplate(PermaSql.replacePlaceholdersForQuery(groupedQuery))
-                        .withQueryTemplate(PermaSql.replacePlaceholdersForQuery(groupedQuery))
+            .from(Task.TABLE)
-                        .from(Task.TABLE)
+            .toString()
-                        .toString())
     }
 }

kmp/src/commonMain/kotlin/org/tasks/data/TaskListQueryRecursive.kt

@@ -5,12 +5,10 @@ import com.todoroo.astrid.core.SortHelper
 import org.tasks.data.dao.TaskDao.TaskCriteria.activeAndVisible
 import org.tasks.data.db.Table
 import org.tasks.data.entity.CaldavTask
-import org.tasks.data.entity.Tag
 import org.tasks.data.entity.Task
 import org.tasks.data.sql.Criterion
 import org.tasks.data.sql.Field.Companion.field
 import org.tasks.data.sql.Join
-import org.tasks.data.sql.Query
 import org.tasks.data.sql.QueryTemplate
 import org.tasks.filters.CaldavFilter
 import org.tasks.filters.Filter
@@ -19,32 +17,16 @@ import org.tasks.preferences.QueryPreferences
 internal object TaskListQueryRecursive {
     private val RECURSIVE = Table("recursive_tasks")
     private val RECURSIVE_TASK = field("$RECURSIVE.task")
-    private val FIELDS =
-            TaskListQuery.FIELDS.plus(listOf(
-                    field("(${
-                        Query.select(field("group_concat(distinct(tag_uid))")).from(Tag.TABLE).where(
-                        Task.ID.eq(Tag.TASK))} GROUP BY ${Tag.TASK})").`as`("tags"),
-                    field("indent"),
-                    field("sort_group"),
-                    field("children"),
-                    field("primary_sort"),
-                    field("secondary_sort"),
-                    field("parent_complete"),
-            )).toTypedArray()
-    private val JOINS = """
-        ${Join.inner(RECURSIVE, Task.ID.eq(RECURSIVE_TASK))}
-        LEFT JOIN (SELECT parent, count(distinct recursive_tasks.task) AS children FROM recursive_tasks GROUP BY parent) AS recursive_children ON recursive_children.parent = tasks._id
-        ${TaskListQuery.JOINS}
-    """.trimIndent()
     private val SUBTASK_QUERY =
             QueryTemplate()
-                    .join(Join.inner(RECURSIVE, Task.PARENT.eq(RECURSIVE_TASK)))
+                .join(Join.inner(RECURSIVE, Task.PARENT.eq(RECURSIVE_TASK)))
-                    .where(activeAndVisible())
+                .where(activeAndVisible())
+                .toString()
 
     fun getRecursiveQuery(
         filter: Filter,
         preferences: QueryPreferences,
-    ): MutableList<String> {
+    ): String {
         val parentQuery = when (filter) {
             is CaldavFilter -> newCaldavQuery(filter.uuid)
             else -> PermaSql.replacePlaceholdersForQuery(filter.sql!!)
@@ -74,48 +56,104 @@ internal object TaskListQueryRecursive {
             preferences.sortAscending && sortMode != SortHelper.SORT_GTASKS && sortMode != SortHelper.SORT_CALDAV
         val subtaskAscending =
             preferences.subtaskAscending && subtaskMode != SortHelper.SORT_GTASKS && subtaskMode != SortHelper.SORT_CALDAV
-        val primaryGroupSelector = SortHelper.orderSelectForSortTypeRecursive(groupMode, true)
+        val completedAtBottom = preferences.completedTasksAtBottom
-        val primarySortSelect = SortHelper.orderSelectForSortTypeRecursive(sortMode, false)
+        val parentCompleted = if (completedAtBottom) "tasks.completed > 0" else "0"
-        val subtaskSort = SortHelper.orderSelectForSortTypeRecursive(subtaskMode, false)
+        val completionSort = if (completedAtBottom) {
-        val parentCompleted = if (preferences.completedTasksAtBottom) "tasks.completed > 0" else "0"
-        val completionSort = if (preferences.completedTasksAtBottom) {
             "(CASE WHEN tasks.completed > 0 THEN ${SortHelper.orderSelectForSortTypeRecursive(completedMode, false)} ELSE 0 END)"
         } else {
             "0"
         }
-        val withClause = """
+        val query = """
-            CREATE TEMPORARY TABLE `recursive_tasks` AS
+            WITH RECURSIVE recursive_tasks AS (
-            WITH RECURSIVE recursive_tasks (task, parent_complete, subtask_complete, completion_sort, parent, collapsed, hidden, indent, title, primary_group, primary_sort, secondary_sort, sort_group) AS (
+                SELECT 
-                SELECT tasks._id, $parentCompleted as parent_complete, 0 as subtask_complete, $completionSort as completion_sort, 0 as parent, tasks.collapsed as collapsed, 0 as hidden, 0 AS sort_indent, UPPER(tasks.title) AS sort_title, $primaryGroupSelector as primary_group, $primarySortSelect as primary_sort, NULL as secondarySort, ${SortHelper.getSortGroup(groupMode)}
+                    tasks._id AS task,
+                    $parentCompleted AS parent_complete,
+                    0 AS subtask_complete,
+                    $completionSort AS completion_sort,
+                    0 AS parent,
+                    tasks.collapsed AS collapsed,
+                    0 AS hidden,
+                    0 AS indent,
+                    UPPER(tasks.title) AS sort_title,
+                    ${SortHelper.orderSelectForSortTypeRecursive(groupMode, true)} AS primary_group,
+                    ${SortHelper.orderSelectForSortTypeRecursive(sortMode, false)} AS primary_sort,
+                    NULL as secondary_sort,
+                    ${SortHelper.getSortGroup(groupMode)} AS sort_group
                 FROM tasks
                 ${
                     if (groupMode == SortHelper.SORT_LIST) {
                         """
-                            INNER JOIN caldav_tasks on cd_task = tasks._id AND cd_deleted = 0
+                            INNER JOIN caldav_tasks ON cd_task = tasks._id AND cd_deleted = 0
-                            INNER JOIN caldav_lists on cd_calendar = cdl_uuid
+                            INNER JOIN caldav_lists ON cd_calendar = cdl_uuid
                         """.trimIndent()
                     } else {
                         ""
                     }
                 }
                 $parentQuery
-                UNION ALL SELECT tasks._id, recursive_tasks.parent_complete, $parentCompleted as subtask_complete, $completionSort as completion_sort, recursive_tasks.task as parent, tasks.collapsed as collapsed, CASE WHEN recursive_tasks.collapsed > 0 OR recursive_tasks.hidden > 0 THEN 1 ELSE 0 END as hidden, recursive_tasks.indent+1 AS sort_indent, UPPER(tasks.title) AS sort_title, recursive_tasks.primary_group as primary_group, recursive_tasks.primary_sort as primary_sort, $subtaskSort as secondary_sort, recursive_tasks.sort_group FROM tasks
+                UNION ALL SELECT
+                    tasks._id AS task,
+                    recursive_tasks.parent_complete AS parent_complete,
+                    $parentCompleted AS subtask_complete,
+                    $completionSort AS completion_sort,
+                    recursive_tasks.task AS parent,
+                    tasks.collapsed AS collapsed,
+                    CASE WHEN recursive_tasks.collapsed > 0 OR recursive_tasks.hidden > 0 THEN 1 ELSE 0 END AS hidden,
+                    recursive_tasks.indent+1 AS indent,
+                    UPPER(tasks.title) AS sort_title,
+                    recursive_tasks.primary_group AS primary_group,
+                    recursive_tasks.primary_sort AS primary_sort,
+                    ${SortHelper.orderSelectForSortTypeRecursive(subtaskMode, false)} AS secondary_sort,
+                    recursive_tasks.sort_group AS sort_group
+                FROM tasks
                 $SUBTASK_QUERY
-                ORDER BY parent_complete ASC, sort_indent DESC, subtask_complete ASC, completion_sort ${if (preferences.completedAscending) "ASC" else "DESC"}, ${SortHelper.orderForGroupTypeRecursive(groupMode, groupAscending)}, ${SortHelper.orderForSortTypeRecursive(sortMode, sortAscending, subtaskMode, subtaskAscending)}
+                ORDER BY
-            ) SELECT * FROM recursive_tasks
+                    parent_complete,
-            WHERE indent = (SELECT MAX(indent) FROM recursive_tasks as r WHERE r.task = recursive_tasks.task)
+                    indent DESC,
+                    subtask_complete,
+                    completion_sort ${if (preferences.completedAscending) "" else "DESC"},
+                    ${SortHelper.orderForGroupTypeRecursive(groupMode, groupAscending)},
+                    ${SortHelper.orderForSortTypeRecursive(sortMode, sortAscending, subtaskMode, subtaskAscending)}
+            ),
+            numbered_tasks AS (
+                SELECT task, ROW_NUMBER() OVER () AS sequence
+                FROM recursive_tasks
+            ),
+            max_indent AS (
+                SELECT task,
+                MAX(recursive_tasks.indent) OVER (PARTITION BY task) AS max_indent
+                FROM recursive_tasks
+            ),
+            child_counts AS (
+                SELECT DISTINCT(parent),
+                COUNT(*) OVER (PARTITION BY parent) AS children
+                FROM recursive_tasks
+                WHERE parent > 0
+            )
+            SELECT
+                ${TaskListQuery.FIELDS.joinToString(",\n") { it.toStringInSelect() }},
+                group_concat(distinct(tag_uid)) AS tags,
+                indent,
+                sort_group,
+                children,
+                primary_sort,
+                secondary_sort,
+                parent_complete
+            FROM tasks
+                INNER JOIN numbered_tasks ON tasks._id = numbered_tasks.task
+                INNER JOIN max_indent ON tasks._id = max_indent.task
+                INNER JOIN recursive_tasks ON recursive_tasks.task = tasks._id
+                LEFT JOIN child_counts ON child_counts.parent = tasks._id
+                LEFT JOIN tags ON tags.task = tasks._id
+                ${TaskListQuery.JOINS}
+            WHERE
+                recursive_tasks.hidden = 0
+                AND recursive_tasks.indent = max_indent
+                GROUP BY tasks._id
+                ORDER BY sequence
         """.trimIndent()
 
-        return mutableListOf(
+        return SortHelper.adjustQueryForFlags(preferences, query)
-            "DROP TABLE IF EXISTS `recursive_tasks`",
-            SortHelper.adjustQueryForFlags(preferences, withClause),
-            "CREATE INDEX `r_tasks` ON `recursive_tasks` (`task`)",
-            "CREATE INDEX `r_parents` ON `recursive_tasks` (`parent`)",
-            Query.select(*FIELDS)
-                .withQueryTemplate(PermaSql.replacePlaceholdersForQuery("$JOINS WHERE recursive_tasks.hidden = 0"))
-                .from(Task.TABLE)
-                .toString(),
-        )
     }
 
     private fun newCaldavQuery(list: String) =