diff --git a/app/src/main/java/com/todoroo/astrid/adapter/TaskAdapter.java b/app/src/main/java/com/todoroo/astrid/adapter/TaskAdapter.java
index a72db400e..33a3397a4 100644
--- a/app/src/main/java/com/todoroo/astrid/adapter/TaskAdapter.java
+++ b/app/src/main/java/com/todoroo/astrid/adapter/TaskAdapter.java
@@ -66,6 +66,10 @@ public class TaskAdapter {
     this.helper = helper;
   }
 
+  public int getNumSelected() {
+    return selected.size();
+  }
+
   public List<Long> getSelected() {
     return newArrayList(selected);
   }
diff --git a/app/src/main/java/org/tasks/tasklist/ActionModeProvider.java b/app/src/main/java/org/tasks/tasklist/ActionModeProvider.java
index e8d142afb..b71fe8eec 100644
--- a/app/src/main/java/org/tasks/tasklist/ActionModeProvider.java
+++ b/app/src/main/java/org/tasks/tasklist/ActionModeProvider.java
@@ -113,8 +113,11 @@ public class ActionModeProvider {
 
               @Override
               public void onDestroyActionMode(ActionMode actionMode) {
-                adapter.clearSelections();
                 taskListRecyclerAdapter.onDestroyActionMode();
+                if (adapter.getNumSelected() > 0) {
+                  adapter.clearSelections();
+                  taskListRecyclerAdapter.notifyDataSetChanged();
+                }
               }
 
               private void deleteSelectedItems() {
diff --git a/app/src/main/java/org/tasks/tasklist/ItemTouchHelperCallback.java b/app/src/main/java/org/tasks/tasklist/ItemTouchHelperCallback.java
index 9ab35211e..1b27fe508 100644
--- a/app/src/main/java/org/tasks/tasklist/ItemTouchHelperCallback.java
+++ b/app/src/main/java/org/tasks/tasklist/ItemTouchHelperCallback.java
@@ -1,5 +1,8 @@
 package org.tasks.tasklist;
 
+import static android.support.v7.widget.helper.ItemTouchHelper.DOWN;
+import static android.support.v7.widget.helper.ItemTouchHelper.UP;
+
 import android.graphics.Canvas;
 import android.support.v7.widget.RecyclerView;
 import android.support.v7.widget.helper.ItemTouchHelper;
@@ -13,7 +16,6 @@ public class ItemTouchHelperCallback extends ItemTouchHelper.Callback {
   private final TaskListFragment taskList;
   private int from = -1;
   private int to = -1;
-  private boolean dragging;
 
   ItemTouchHelperCallback(
       TaskAdapter adapter, TaskListRecyclerAdapter recyclerAdapter, TaskListFragment taskList) {
@@ -22,43 +24,32 @@ public class ItemTouchHelperCallback extends ItemTouchHelper.Callback {
     this.taskList = taskList;
   }
 
-  public boolean isDragging() {
-    return dragging;
-  }
-
-  public void setDragging(boolean dragging) {
-    this.dragging = dragging;
-  }
-
   @Override
   public void onSelectedChanged(RecyclerView.ViewHolder viewHolder, int actionState) {
     super.onSelectedChanged(viewHolder, actionState);
     if (actionState == ItemTouchHelper.ACTION_STATE_DRAG) {
+      recyclerAdapter.startActionMode();
       ((ViewHolder) viewHolder).setMoving(true);
     }
   }
 
   @Override
   public int getMovementFlags(RecyclerView recyclerView, RecyclerView.ViewHolder viewHolder) {
-    if (!adapter.isManuallySorted()) {
-      return makeMovementFlags(0, 0);
-    }
-    ViewHolder vh = (ViewHolder) viewHolder;
-    if (!recyclerAdapter.isActionModeActive()) {
-      int indentFlags = 0;
-      if (vh.isIndented()) {
-        indentFlags |= ItemTouchHelper.LEFT;
-      }
-      int position = vh.getAdapterPosition();
-      if (position > 0 && adapter.canIndent(position, vh.task)) {
-        indentFlags |= ItemTouchHelper.RIGHT;
-      }
-      return makeMovementFlags(0, indentFlags);
+    return adapter.isManuallySorted() && adapter.getNumSelected() == 0
+        ? makeMovementFlags(UP | DOWN, getSwipeFlags((ViewHolder) viewHolder))
+        : makeMovementFlags(0, 0);
+  }
+
+  private int getSwipeFlags(ViewHolder vh) {
+    int indentFlags = 0;
+    if (vh.isIndented()) {
+      indentFlags |= ItemTouchHelper.LEFT;
     }
-    if (dragging) {
-      return makeMovementFlags(ItemTouchHelper.UP | ItemTouchHelper.DOWN, 0);
+    int position = vh.getAdapterPosition();
+    if (position > 0 && adapter.canIndent(position, vh.task)) {
+      indentFlags |= ItemTouchHelper.RIGHT;
     }
-    return makeMovementFlags(0, 0);
+    return indentFlags;
   }
 
   @Override
@@ -101,17 +92,16 @@ public class ItemTouchHelperCallback extends ItemTouchHelper.Callback {
   public void clearView(RecyclerView recyclerView, RecyclerView.ViewHolder viewHolder) {
     super.clearView(recyclerView, viewHolder);
     ViewHolder vh = (ViewHolder) viewHolder;
-    if (dragging) {
-      vh.setMoving(false);
-      dragging = false;
-      if (from != -1) {
-        if (from >= 0 && from != to) {
-          if (from < to) {
-            to++;
-          }
-          adapter.moved(from, to);
-          taskList.loadTaskListContent(false);
+    vh.setMoving(false);
+    if (recyclerAdapter.isActionModeActive()) {
+      recyclerAdapter.toggle(vh);
+    } else {
+      if (from >= 0 && from != to) {
+        if (from < to) {
+          to++;
         }
+        adapter.moved(from, to);
+        taskList.loadTaskListContent(false);
       }
     }
     from = -1;
diff --git a/app/src/main/java/org/tasks/tasklist/TaskListRecyclerAdapter.java b/app/src/main/java/org/tasks/tasklist/TaskListRecyclerAdapter.java
index 6af16c239..d80d22c70 100644
--- a/app/src/main/java/org/tasks/tasklist/TaskListRecyclerAdapter.java
+++ b/app/src/main/java/org/tasks/tasklist/TaskListRecyclerAdapter.java
@@ -110,35 +110,39 @@ public class TaskListRecyclerAdapter extends RecyclerView.Adapter<ViewHolder>
 
   @Override
   public boolean onLongPress(ViewHolder viewHolder) {
-    toggle(viewHolder);
+    if (!adapter.isManuallySorted()) {
+      startActionMode();
+    }
+    if (mode != null) {
+      toggle(viewHolder);
+    }
     return true;
   }
 
-  private void toggle(ViewHolder viewHolder) {
+  void startActionMode() {
+    if (mode == null) {
+      mode = actionModeProvider.startActionMode(adapter, taskList, this);
+      updateModeTitle();
+      if (adapter.isManuallySorted()) {
+        Flags.set(Flags.TLFP_NO_INTERCEPT_TOUCH);
+      }
+    }
+  }
+
+  void toggle(ViewHolder viewHolder) {
     adapter.toggleSelection(viewHolder.task);
     notifyItemChanged(viewHolder.getAdapterPosition());
     if (adapter.getSelected().isEmpty()) {
-      itemTouchHelperCallback.setDragging(false);
       finishActionMode();
     } else {
-      if (mode == null) {
-        mode = actionModeProvider.startActionMode(adapter, taskList, this);
-        if (adapter.isManuallySorted()) {
-          itemTouchHelperCallback.setDragging(true);
-          Flags.set(Flags.TLFP_NO_INTERCEPT_TOUCH);
-        } else {
-          itemTouchHelperCallback.setDragging(false);
-        }
-      } else {
-        itemTouchHelperCallback.setDragging(false);
-      }
       updateModeTitle();
     }
   }
 
   private void updateModeTitle() {
     if (mode != null) {
-      mode.setTitle(Integer.toString(adapter.getSelected().size()));
+      int count = Math.max(1, adapter.getNumSelected());
+      mode.setTitle(Integer.toString(count));
     }
   }
 
@@ -197,14 +201,11 @@ public class TaskListRecyclerAdapter extends RecyclerView.Adapter<ViewHolder>
     return asyncPagedListDiffer;
   }
 
-  public boolean isActionModeActive() {
+  boolean isActionModeActive() {
     return mode != null;
   }
 
   void onDestroyActionMode() {
     mode = null;
-    if (!itemTouchHelperCallback.isDragging()) {
-      notifyDataSetChanged();
-    }
   }
 }