From e91df38e0c45ff7b8bf7c00ae0948cfc8b77eb14 Mon Sep 17 00:00:00 2001 From: Sam Bosley Date: Mon, 19 Mar 2012 16:17:19 -0700 Subject: [PATCH 1/5] Bug fixes to last modified logic in push task on save --- .../com/todoroo/astrid/actfm/sync/ActFmSyncService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java index 1b9049a7b..410feacb2 100644 --- a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java +++ b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java @@ -430,9 +430,10 @@ public final class ActFmSyncService { } catch (IOException e) { if (notPermanentError(e)) addFailedPush(new FailedPush(PUSH_TYPE_TASK, task.getId())); - else - handleException("task-save-io", e); - task.setValue(Task.LAST_SYNC, DateUtilities.now() + 1000L); + else { + handleException("task-save-io", e); + task.setValue(Task.LAST_SYNC, DateUtilities.now() + 1000L); + } } task.putTransitory(SyncFlags.ACTFM_SUPPRESS_SYNC, true); @@ -445,7 +446,7 @@ public final class ActFmSyncService { */ public void pushTask(long taskId) { Task task = taskService.fetchById(taskId, Task.PROPERTIES); - if (task != null) + if (task != null && task.getValue(Task.MODIFICATION_DATE) > task.getValue(Task.LAST_SYNC)) pushTaskOnSave(task, task.getMergedValues()); } From 9b6689a90db5f0de36b4c4a06e671a7ad1436915 Mon Sep 17 00:00:00 2001 From: Sam Bosley Date: Mon, 19 Mar 2012 16:15:30 -0700 Subject: [PATCH 2/5] Sync conflict merge, not totally working --- .../astrid/actfm/sync/ActFmSyncService.java | 183 +++++++++++++----- 1 file changed, 130 insertions(+), 53 deletions(-) diff --git a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java index 410feacb2..364ea0102 100644 --- a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java +++ b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java @@ -428,9 +428,9 @@ public final class ActFmSyncService { } catch (JSONException e) { handleException("task-save-json", e); } catch (IOException e) { - if (notPermanentError(e)) + if (notPermanentError(e)) { addFailedPush(new FailedPush(PUSH_TYPE_TASK, task.getId())); - else { + } else { handleException("task-save-io", e); task.setValue(Task.LAST_SYNC, DateUtilities.now() + 1000L); } @@ -717,57 +717,58 @@ public final class ActFmSyncService { * @param done */ public void fetchActiveTasks(final boolean manual, SyncExceptionHandler handler, Runnable done) { - invokeFetchList("task", manual, handler, new ListItemProcessor() { - @Override - protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { - Task remote = new Task(); - - ArrayList metadata = new ArrayList(); - HashSet ids = new HashSet(list.length()); - for(int i = 0; i < list.length(); i++) { - JSONObject item = list.getJSONObject(i); - readIds(locals, item, remote); - JsonHelper.taskFromJson(item, remote, metadata); - - if(remote.getValue(Task.USER_ID) == 0) { - if(!remote.isSaved()) - StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_CREATED); - else if(remote.isCompleted()) - StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_COMPLETED); - } - - if(!remote.isSaved() && remote.hasDueDate() && - remote.getValue(Task.DUE_DATE) < DateUtilities.now()) - remote.setFlag(Task.REMINDER_FLAGS, Task.NOTIFY_AFTER_DEADLINE, false); - - remote.putTransitory(SyncFlags.ACTFM_SUPPRESS_SYNC, true); - taskService.save(remote); - ids.add(remote.getId()); - metadataService.synchronizeMetadata(remote.getId(), metadata, MetadataCriteria.withKey(TagService.KEY)); - remote.clear(); - } - - if(manual) { - Long[] localIds = ids.toArray(new Long[ids.size()]); - taskService.deleteWhere(Criterion.and(TaskCriteria.activeAndVisible(), - Task.REMOTE_ID.isNotNull(), - Criterion.not(Task.ID.in(localIds)))); - } - } - - @Override - protected HashMap getLocalModels() { - TodorooCursor cursor = taskService.query(Query.select(Task.ID, - Task.REMOTE_ID).where(Task.REMOTE_ID.in(remoteIds)).orderBy( - Order.asc(Task.REMOTE_ID))); - return cursorToMap(cursor, taskDao, Task.REMOTE_ID, Task.ID); - } - - @Override - protected Class typeClass() { - return Task.class; - } - }, done, "active_tasks"); + invokeFetchList("task", manual, handler, new TaskListItemProcessor(manual), done, "active_tasks"); +// invokeFetchList("task", manual, handler, new ListItemProcessor() { +// @Override +// protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { +// Task remote = new Task(); +// +// ArrayList metadata = new ArrayList(); +// HashSet ids = new HashSet(list.length()); +// for(int i = 0; i < list.length(); i++) { +// JSONObject item = list.getJSONObject(i); +// readIds(locals, item, remote); +// JsonHelper.taskFromJson(item, remote, metadata); +// +// if(remote.getValue(Task.USER_ID) == 0) { +// if(!remote.isSaved()) +// StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_CREATED); +// else if(remote.isCompleted()) +// StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_COMPLETED); +// } +// +// if(!remote.isSaved() && remote.hasDueDate() && +// remote.getValue(Task.DUE_DATE) < DateUtilities.now()) +// remote.setFlag(Task.REMINDER_FLAGS, Task.NOTIFY_AFTER_DEADLINE, false); +// +// remote.putTransitory(SyncFlags.ACTFM_SUPPRESS_SYNC, true); +// taskService.save(remote); +// ids.add(remote.getId()); +// metadataService.synchronizeMetadata(remote.getId(), metadata, MetadataCriteria.withKey(TagService.KEY)); +// remote.clear(); +// } +// +// if(manual) { +// Long[] localIds = ids.toArray(new Long[ids.size()]); +// taskService.deleteWhere(Criterion.and(TaskCriteria.activeAndVisible(), +// Task.REMOTE_ID.isNotNull(), +// Criterion.not(Task.ID.in(localIds)))); +// } +// } +// +// @Override +// protected HashMap getLocalModels() { +// TodorooCursor cursor = taskService.query(Query.select(Task.ID, +// Task.REMOTE_ID).where(Task.REMOTE_ID.in(remoteIds)).orderBy( +// Order.asc(Task.REMOTE_ID))); +// return cursorToMap(cursor, taskDao, Task.REMOTE_ID, Task.ID); +// } +// +// @Override +// protected Class typeClass() { +// return Task.class; +// } +// }, done, "active_tasks"); } /** @@ -1063,6 +1064,82 @@ public final class ActFmSyncService { } + private class TaskListItemProcessor extends ListItemProcessor { + + private final boolean deleteExtras; + private final HashMap modificationDates; + + public TaskListItemProcessor(boolean deleteExtras) { + this.deleteExtras = deleteExtras; + this.modificationDates = new HashMap(); + } + + @Override + protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { + Task remote = new Task(); + + ArrayList metadata = new ArrayList(); + HashSet ids = new HashSet(list.length()); + for(int i = 0; i < list.length(); i++) { + JSONObject item = list.getJSONObject(i); + readIds(locals, item, remote); + + long serverModificationDate = item.optLong("updated_at") * 1000; + System.err.println("Server mod: " + serverModificationDate + ", local mod: " + modificationDates.get(remote.getId())); + if (serverModificationDate > 0 && modificationDates.containsKey(remote.getId()) && serverModificationDate < modificationDates.get(remote.getId())) { + ids.add(remote.getId()); + continue; // Modified locally more recently than remotely -- don't overwrite changes + } + + JsonHelper.taskFromJson(item, remote, metadata); + + if(remote.getValue(Task.USER_ID) == 0) { + if(!remote.isSaved()) + StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_CREATED); + else if(remote.isCompleted()) + StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_COMPLETED); + } + + if(!remote.isSaved() && remote.hasDueDate() && + remote.getValue(Task.DUE_DATE) < DateUtilities.now()) + remote.setFlag(Task.REMINDER_FLAGS, Task.NOTIFY_AFTER_DEADLINE, false); + + remote.putTransitory(SyncFlags.ACTFM_SUPPRESS_SYNC, true); + taskService.save(remote); + ids.add(remote.getId()); + metadataService.synchronizeMetadata(remote.getId(), metadata, MetadataCriteria.withKey(TagService.KEY)); + remote.clear(); + } + + if(deleteExtras) { + Long[] localIds = ids.toArray(new Long[ids.size()]); + taskService.deleteWhere(Criterion.and(TaskCriteria.activeAndVisible(), + Task.REMOTE_ID.isNotNull(), + Criterion.not(Task.ID.in(localIds)))); + } + } + + @Override + protected HashMap getLocalModels() { + TodorooCursor cursor = taskService.query(Query.select(Task.ID, Task.MODIFICATION_DATE, + Task.REMOTE_ID).where(Task.REMOTE_ID.in(remoteIds)).orderBy( + Order.asc(Task.REMOTE_ID))); + Task task = new Task(); + for (cursor.moveToFirst(); !cursor.isAfterLast(); cursor.moveToNext()) { + task.readFromCursor(cursor); + modificationDates.put(task.getId(), task.getValue(Task.MODIFICATION_DATE)); + } + + return cursorToMap(cursor, taskDao, Task.REMOTE_ID, Task.ID); + } + + @Override + protected Class typeClass() { + return Task.class; + } + + } + /** Call sync method */ private void invokeFetchList(final String model, final boolean manual, final SyncExceptionHandler handler, final ListItemProcessor processor, final Runnable done, final String lastSyncKey, From 0ed4e6a72a6dc7bc6c9fe512972d501e9a3c7652 Mon Sep 17 00:00:00 2001 From: Sam Bosley Date: Mon, 19 Mar 2012 16:39:18 -0700 Subject: [PATCH 3/5] Apply last modified check to syncing tags --- .../astrid/actfm/sync/ActFmSyncService.java | 116 +++--------------- 1 file changed, 14 insertions(+), 102 deletions(-) diff --git a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java index 364ea0102..467ba8f4f 100644 --- a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java +++ b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java @@ -718,57 +718,6 @@ public final class ActFmSyncService { */ public void fetchActiveTasks(final boolean manual, SyncExceptionHandler handler, Runnable done) { invokeFetchList("task", manual, handler, new TaskListItemProcessor(manual), done, "active_tasks"); -// invokeFetchList("task", manual, handler, new ListItemProcessor() { -// @Override -// protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { -// Task remote = new Task(); -// -// ArrayList metadata = new ArrayList(); -// HashSet ids = new HashSet(list.length()); -// for(int i = 0; i < list.length(); i++) { -// JSONObject item = list.getJSONObject(i); -// readIds(locals, item, remote); -// JsonHelper.taskFromJson(item, remote, metadata); -// -// if(remote.getValue(Task.USER_ID) == 0) { -// if(!remote.isSaved()) -// StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_CREATED); -// else if(remote.isCompleted()) -// StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_COMPLETED); -// } -// -// if(!remote.isSaved() && remote.hasDueDate() && -// remote.getValue(Task.DUE_DATE) < DateUtilities.now()) -// remote.setFlag(Task.REMINDER_FLAGS, Task.NOTIFY_AFTER_DEADLINE, false); -// -// remote.putTransitory(SyncFlags.ACTFM_SUPPRESS_SYNC, true); -// taskService.save(remote); -// ids.add(remote.getId()); -// metadataService.synchronizeMetadata(remote.getId(), metadata, MetadataCriteria.withKey(TagService.KEY)); -// remote.clear(); -// } -// -// if(manual) { -// Long[] localIds = ids.toArray(new Long[ids.size()]); -// taskService.deleteWhere(Criterion.and(TaskCriteria.activeAndVisible(), -// Task.REMOTE_ID.isNotNull(), -// Criterion.not(Task.ID.in(localIds)))); -// } -// } -// -// @Override -// protected HashMap getLocalModels() { -// TodorooCursor cursor = taskService.query(Query.select(Task.ID, -// Task.REMOTE_ID).where(Task.REMOTE_ID.in(remoteIds)).orderBy( -// Order.asc(Task.REMOTE_ID))); -// return cursorToMap(cursor, taskDao, Task.REMOTE_ID, Task.ID); -// } -// -// @Override -// protected Class typeClass() { -// return Task.class; -// } -// }, done, "active_tasks"); } /** @@ -778,54 +727,14 @@ public final class ActFmSyncService { * @param done */ public void fetchTasksForTag(final TagData tagData, final boolean manual, Runnable done) { - invokeFetchList("task", manual, null, new ListItemProcessor() { + invokeFetchList("task", manual, null, new TaskListItemProcessor(manual) { @Override - protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { - Task remote = new Task(); - - ArrayList metadata = new ArrayList(); - HashSet ids = new HashSet(list.length()); - for(int i = 0; i < list.length(); i++) { - - JSONObject item = list.getJSONObject(i); - readIds(locals, item, remote); - JsonHelper.taskFromJson(item, remote, metadata); - - if(remote.getValue(Task.USER_ID) == 0) { - if(!remote.isSaved()) - StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_CREATED); - else if(remote.isCompleted()) - StatisticsService.reportEvent(StatisticsConstants.ACTFM_TASK_COMPLETED); - } - - remote.putTransitory(SyncFlags.ACTFM_SUPPRESS_SYNC, true); - taskService.save(remote); - ids.add(remote.getId()); - metadataService.synchronizeMetadata(remote.getId(), metadata, MetadataCriteria.withKey(TagService.KEY)); - remote.clear(); - } - - if(manual) { - Long[] localIds = ids.toArray(new Long[ids.size()]); - taskService.deleteWhere(Criterion.and( - TagService.memberOfTagData(tagData.getValue(TagData.REMOTE_ID)), - TaskCriteria.activeAndVisible(), - Task.REMOTE_ID.isNotNull(), - Criterion.not(Task.ID.in(localIds)))); - } - } - - @Override - protected HashMap getLocalModels() { - TodorooCursor cursor = taskService.query(Query.select(Task.ID, - Task.REMOTE_ID).where(Task.REMOTE_ID.in(remoteIds)).orderBy( - Order.asc(Task.REMOTE_ID))); - return cursorToMap(cursor, taskDao, Task.REMOTE_ID, Task.ID); - } - - @Override - protected Class typeClass() { - return Task.class; + protected void deleteExtras(Long[] localIds) { + taskService.deleteWhere(Criterion.and( + TagService.memberOfTagData(tagData.getValue(TagData.REMOTE_ID)), + TaskCriteria.activeAndVisible(), + Task.REMOTE_ID.isNotNull(), + Criterion.not(Task.ID.in(localIds)))); } }, done, "tasks:" + tagData.getId(), "tag_id", tagData.getValue(TagData.REMOTE_ID)); } @@ -1085,7 +994,6 @@ public final class ActFmSyncService { readIds(locals, item, remote); long serverModificationDate = item.optLong("updated_at") * 1000; - System.err.println("Server mod: " + serverModificationDate + ", local mod: " + modificationDates.get(remote.getId())); if (serverModificationDate > 0 && modificationDates.containsKey(remote.getId()) && serverModificationDate < modificationDates.get(remote.getId())) { ids.add(remote.getId()); continue; // Modified locally more recently than remotely -- don't overwrite changes @@ -1113,12 +1021,16 @@ public final class ActFmSyncService { if(deleteExtras) { Long[] localIds = ids.toArray(new Long[ids.size()]); - taskService.deleteWhere(Criterion.and(TaskCriteria.activeAndVisible(), - Task.REMOTE_ID.isNotNull(), - Criterion.not(Task.ID.in(localIds)))); + deleteExtras(localIds); } } + protected void deleteExtras(Long[] localIds) { + taskService.deleteWhere(Criterion.and(TaskCriteria.activeAndVisible(), + Task.REMOTE_ID.isNotNull(), + Criterion.not(Task.ID.in(localIds)))); + } + @Override protected HashMap getLocalModels() { TodorooCursor cursor = taskService.query(Query.select(Task.ID, Task.MODIFICATION_DATE, From e99a2f1d8ba5b372dc82984cc3af3d18c34ffac3 Mon Sep 17 00:00:00 2001 From: Sam Bosley Date: Mon, 19 Mar 2012 17:13:38 -0700 Subject: [PATCH 4/5] Push unsynced tasks in tags --- .../astrid/actfm/sync/ActFmSyncV2Provider.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncV2Provider.java b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncV2Provider.java index c5f3dc0a2..d3fd33499 100644 --- a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncV2Provider.java +++ b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncV2Provider.java @@ -17,9 +17,11 @@ import com.todoroo.andlib.data.TodorooCursor; import com.todoroo.andlib.service.Autowired; import com.todoroo.andlib.service.ContextManager; import com.todoroo.andlib.sql.Criterion; +import com.todoroo.andlib.sql.Join; import com.todoroo.andlib.sql.Query; import com.todoroo.andlib.utility.Preferences; import com.todoroo.astrid.dao.TaskDao.TaskCriteria; +import com.todoroo.astrid.data.Metadata; import com.todoroo.astrid.data.RemoteModel; import com.todoroo.astrid.data.TagData; import com.todoroo.astrid.data.Task; @@ -306,6 +308,8 @@ public class ActFmSyncV2Provider extends SyncV2Provider { actFmSyncService.fetchTasksForTag(tagData, manual, new Runnable() { @Override public void run() { + pushQueuedTasksByTag(tagData, callback, finisher); + callback.incrementProgress(30); if(finisher.decrementAndGet() == 0) callback.finished(); @@ -313,4 +317,15 @@ public class ActFmSyncV2Provider extends SyncV2Provider { }); } + private void pushQueuedTasksByTag(TagData tagData, SyncResultCallback callback, AtomicInteger finisher) { + TodorooCursor taskCursor = taskService.query(Query.select(Task.PROPERTIES) + .join(Join.inner(Metadata.TABLE, Criterion.and(Metadata.KEY.eq(TagService.KEY), Metadata.TASK.eq(Task.ID), TagService.TAG.eq(tagData.getId())))) + .where(Criterion.or( + Criterion.and(TaskCriteria.isActive(), + Task.REMOTE_ID.isNull()), + Criterion.and(Task.REMOTE_ID.isNotNull(), + Task.MODIFICATION_DATE.gt(Task.LAST_SYNC))))); + pushQueued(callback, finisher, taskCursor, false, taskPusher); + } + } From 59dead0ff96e2bcc505dfb3418ea499e7f36ffae Mon Sep 17 00:00:00 2001 From: Sam Bosley Date: Mon, 19 Mar 2012 17:28:30 -0700 Subject: [PATCH 5/5] Account for any discrepancies in server time and client time --- .../astrid/actfm/sync/ActFmSyncService.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java index 467ba8f4f..691985617 100644 --- a/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java +++ b/astrid/plugin-src/com/todoroo/astrid/actfm/sync/ActFmSyncService.java @@ -609,7 +609,7 @@ public final class ActFmSyncService { public void fetchTagDataDashboard(boolean manual, final Runnable done) { invokeFetchList("goal", manual, null, new ListItemProcessor() { @Override - protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { + protected void mergeAndSave(JSONArray list, HashMap locals, long serverTime) throws JSONException { TagData remote = new TagData(); for(int i = 0; i < list.length(); i++) { JSONObject item = list.getJSONObject(i); @@ -843,7 +843,7 @@ public final class ActFmSyncService { private class UpdateListItemProcessor extends ListItemProcessor { @Override - protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { + protected void mergeAndSave(JSONArray list, HashMap locals, long serverTime) throws JSONException { Update remote = new Update(); for(int i = 0; i < list.length(); i++) { JSONObject item = list.getJSONObject(i); @@ -927,13 +927,13 @@ public final class ActFmSyncService { abstract protected Class typeClass(); abstract protected void mergeAndSave(JSONArray list, - HashMap locals) throws JSONException; + HashMap locals, long serverTime) throws JSONException; - public void process(JSONArray list) throws JSONException { + public void process(JSONArray list, long serverTime) throws JSONException { readRemoteIds(list); synchronized (typeClass()) { HashMap locals = getLocalModels(); - mergeAndSave(list, locals); + mergeAndSave(list, locals, serverTime); } } @@ -984,17 +984,21 @@ public final class ActFmSyncService { } @Override - protected void mergeAndSave(JSONArray list, HashMap locals) throws JSONException { + protected void mergeAndSave(JSONArray list, HashMap locals, long serverTime) throws JSONException { Task remote = new Task(); ArrayList metadata = new ArrayList(); HashSet ids = new HashSet(list.length()); + + long timeDelta = serverTime == 0 ? 0 : DateUtilities.now() - serverTime * 1000; + for(int i = 0; i < list.length(); i++) { JSONObject item = list.getJSONObject(i); readIds(locals, item, remote); long serverModificationDate = item.optLong("updated_at") * 1000; - if (serverModificationDate > 0 && modificationDates.containsKey(remote.getId()) && serverModificationDate < modificationDates.get(remote.getId())) { + if (serverModificationDate > 0 && modificationDates.containsKey(remote.getId()) + && serverModificationDate < (modificationDates.get(remote.getId()) - timeDelta)) { ids.add(remote.getId()); continue; // Modified locally more recently than remotely -- don't overwrite changes } @@ -1069,9 +1073,10 @@ public final class ActFmSyncService { JSONObject result = null; try { result = actFmInvoker.invoke(model + "_list", getParams); + long serverTime = result.optLong("time", 0); JSONArray list = result.getJSONArray("list"); - processor.process(list); - Preferences.setLong("actfm_time_" + lastSyncKey, result.optLong("time", 0)); + processor.process(list, serverTime); + Preferences.setLong("actfm_time_" + lastSyncKey, serverTime); Preferences.setLong("actfm_last_" + lastSyncKey, DateUtilities.now()); if(done != null)