From d016dcc6f6a3daf8c19e2ececd3c676cd274381a Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 9 Oct 2013 12:02:52 +0200 Subject: [PATCH 1/5] Refactor multi-threaded autocomple contact searching to make it available for other purposes, too --- plugins/acl/acl.php | 4 +- program/js/app.js | 200 ++++++++++++++++++++-------- program/steps/mail/autocomplete.inc | 4 +- 3 files changed, 147 insertions(+), 61 deletions(-) diff --git a/plugins/acl/acl.php b/plugins/acl/acl.php index a840bcd58..203340bdb 100644 --- a/plugins/acl/acl.php +++ b/plugins/acl/acl.php @@ -86,7 +86,7 @@ class acl extends rcube_plugin $this->load_config(); $search = rcube_utils::get_input_value('_search', rcube_utils::INPUT_GPC, true); - $sid = rcube_utils::get_input_value('_id', rcube_utils::INPUT_GPC); + $reqid = rcube_utils::get_input_value('_reqid', rcube_utils::INPUT_GPC); $users = array(); if ($this->init_ldap()) { @@ -115,7 +115,7 @@ class acl extends rcube_plugin sort($users, SORT_LOCALE_STRING); - $this->rc->output->command('ksearch_query_results', $users, $search, $sid); + $this->rc->output->command('ksearch_query_results', $users, $search, $reqid); $this->rc->output->send(); } diff --git a/program/js/app.js b/program/js/app.js index 5ea01e750..43ab7be17 100644 --- a/program/js/app.js +++ b/program/js/app.js @@ -31,6 +31,7 @@ function rcube_webmail() this.onloads = []; this.messages = {}; this.group2expand = {}; + this.http_request_jobs = {}; // webmail client settings this.dblclick_time = 500; @@ -3963,7 +3964,7 @@ function rcube_webmail() p = inp_value.lastIndexOf(this.env.recipients_separator, cpos-1), q = inp_value.substring(p+1, cpos), min = this.env.autocomplete_min_length, - ac = this.ksearch_data; + data = this.ksearch_data; // trim query string q = $.trim(q); @@ -3990,34 +3991,26 @@ function rcube_webmail() return; // ...new search value contains old one and previous search was not finished or its result was empty - if (old_value && old_value.length && q.indexOf(old_value) == 0 && (!ac || ac.num <= 0) && this.env.contacts && !this.env.contacts.length) + if (old_value && old_value.length && q.indexOf(old_value) == 0 && (!data || data.num <= 0) && this.env.contacts && !this.env.contacts.length) return; - var i, lock, source, xhr, reqid = new Date().getTime(), - post_data = {_search: q, _id: reqid}, - threads = props && props.threads ? props.threads : 1, - sources = props && props.sources ? props.sources : [], - action = props && props.action ? props.action : 'mail/autocomplete'; - - this.ksearch_data = {id: reqid, sources: sources.slice(), action: action, - locks: [], requests: [], num: sources.length}; - - for (i=0; i 1 && source === undefined) - break; - - post_data._source = source ? source : ''; - lock = this.display_message(this.get_label('searching'), 'loading'); - xhr = this.http_post(action, post_data, lock); + var sources = props && props.sources ? props.sources : ['']; + var reqid = this.multi_thread_http_request({ + items: sources, + threads: props && props.threads ? props.threads : 1, + action: props && props.action ? props.action : 'mail/autocomplete', + postdata: { _search:q, _source:'%s' }, + lock: this.display_message(this.get_label('searching'), 'loading') + }); - this.ksearch_data.locks.push(lock); - this.ksearch_data.requests.push(xhr); - } + this.ksearch_data = { id:reqid, sources:sources.slice(), num:sources.length }; }; this.ksearch_query_results = function(results, search, reqid) { + // trigger multi-thread http response callback + this.multi_thread_http_response(results, reqid); + // search stopped in meantime? if (!this.ksearch_value) return; @@ -4029,7 +4022,6 @@ function rcube_webmail() // display search results var i, len, ul, li, text, init, value = this.ksearch_value, - data = this.ksearch_data, maxlen = this.env.autocomplete_max ? this.env.autocomplete_max : 15; // create results pane if not present @@ -4083,27 +4075,8 @@ function rcube_webmail() if (len) this.env.contacts = this.env.contacts.concat(results); - // run next parallel search - if (data.id == reqid) { - data.num--; - if (maxlen > 0 && data.sources.length) { - var lock, xhr, source = data.sources.shift(), post_data; - if (source) { - post_data = {_search: value, _id: reqid, _source: source}; - lock = this.display_message(this.get_label('searching'), 'loading'); - xhr = this.http_post(data.action, post_data, lock); - - this.ksearch_data.locks.push(lock); - this.ksearch_data.requests.push(xhr); - } - } - else if (!maxlen) { - if (!this.ksearch_msg) - this.ksearch_msg = this.display_message(this.get_label('autocompletemore')); - // abort pending searches - this.ksearch_abort(); - } - } + if (this.ksearch_data.id == reqid) + this.ksearch_data.num--; }; this.ksearch_click = function(node) @@ -4138,7 +4111,8 @@ function rcube_webmail() // Clears autocomplete data/requests this.ksearch_destroy = function() { - this.ksearch_abort(); + if (this.ksearch_data) + this.multi_thread_request_abort(this.ksearch_data.id); if (this.ksearch_info) this.hide_message(this.ksearch_info); @@ -4149,18 +4123,6 @@ function rcube_webmail() this.ksearch_data = null; this.ksearch_info = null; this.ksearch_msg = null; - } - - // Aborts pending autocomplete requests - this.ksearch_abort = function() - { - var i, len, ac = this.ksearch_data; - - if (!ac) - return; - - for (i=0, len=ac.locks.length; i 0 && i < prop.requests.length; i++) { + if (prop.requests[i].abort) + prop.requests[i].abort(); + } + + prop.running = 0; + prop.cancelled = true; + this.set_busy(false, '', prop.lock); + } + }; + // post the given form to a hidden iframe this.async_upload_form = function(form, action, onload) { diff --git a/program/steps/mail/autocomplete.inc b/program/steps/mail/autocomplete.inc index f9e8d71a4..c34133a28 100644 --- a/program/steps/mail/autocomplete.inc +++ b/program/steps/mail/autocomplete.inc @@ -47,7 +47,7 @@ $mode = (int) $RCMAIL->config->get('addressbook_search_mode'); $single = (bool) $RCMAIL->config->get('autocomplete_single'); $search = get_input_value('_search', RCUBE_INPUT_GPC, true); $source = get_input_value('_source', RCUBE_INPUT_GPC); -$sid = get_input_value('_id', RCUBE_INPUT_GPC); +$reqid = get_input_value('_reqid', RCUBE_INPUT_GPC); if (strlen($source)) $book_types = array($source); @@ -141,5 +141,5 @@ if (!empty($book_types) && strlen($search)) { } } -$OUTPUT->command('ksearch_query_results', $contacts, $search, $sid); +$OUTPUT->command('ksearch_query_results', $contacts, $search, $reqid); $OUTPUT->send(); From f540f86d19ae1d374ba3c78fe11eaf3f771af150 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 9 Oct 2013 12:04:17 +0200 Subject: [PATCH 2/5] Prepare message list to display IMAP folder --- program/js/app.js | 4 +++- program/js/list.js | 14 ++++++++++++++ program/lib/Roundcube/rcube_imap.php | 1 + program/lib/Roundcube/rcube_message_header.php | 9 +++++++++ program/steps/mail/func.inc | 4 +++- skins/larry/mail.css | 8 ++++++++ 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/program/js/app.js b/program/js/app.js index 43ab7be17..1d59794d7 100644 --- a/program/js/app.js +++ b/program/js/app.js @@ -5998,8 +5998,10 @@ function rcube_webmail() if ((n = $.inArray('status', this.env.coltypes)) >= 0) this.env.status_col = n; - if (list) + if (list) { + list.hide_column('folder', !(this.env.search_request || this.env.search_id)); list.init_header(); + } }; // replace content of row count display diff --git a/program/js/list.js b/program/js/list.js index 0f8d56c19..32e34e080 100644 --- a/program/js/list.js +++ b/program/js/list.js @@ -372,6 +372,20 @@ blur: function() }, +/** + * Set/unset the given column as hidden + */ +hide_column: function(col, hide) +{ + var method = hide ? 'addClass' : 'removeClass'; + + if (this.fixed_header) + $(this.row_tagname()+' '+this.col_tagname()+'.'+col, this.fixed_header)[method]('hidden'); + + $(this.row_tagname()+' '+this.col_tagname()+'.'+col, this.list)[method]('hidden'); +}, + + /** * onmousedown-handler of message list column */ diff --git a/program/lib/Roundcube/rcube_imap.php b/program/lib/Roundcube/rcube_imap.php index 9faf1bbc6..e0dce6f79 100644 --- a/program/lib/Roundcube/rcube_imap.php +++ b/program/lib/Roundcube/rcube_imap.php @@ -1076,6 +1076,7 @@ class rcube_imap extends rcube_storage } foreach ($headers as $h) { + $h->folder = $folder; $a_msg_headers[$h->uid] = $h; } diff --git a/program/lib/Roundcube/rcube_message_header.php b/program/lib/Roundcube/rcube_message_header.php index 2c5e2b6c8..2bda930eb 100644 --- a/program/lib/Roundcube/rcube_message_header.php +++ b/program/lib/Roundcube/rcube_message_header.php @@ -166,6 +166,13 @@ class rcube_message_header */ public $mdn_to; + /** + * IMAP folder this message is stored in + * + * @var string + */ + public $folder; + /** * Other message headers * @@ -189,6 +196,8 @@ class rcube_message_header 'reply-to' => 'replyto', 'cc' => 'cc', 'bcc' => 'bcc', + 'mbox' => 'folder', + 'folder' => 'folder', 'content-transfer-encoding' => 'encoding', 'in-reply-to' => 'in_reply_to', 'content-type' => 'ctype', diff --git a/program/steps/mail/func.inc b/program/steps/mail/func.inc index 48afecb60..b677c4675 100644 --- a/program/steps/mail/func.inc +++ b/program/steps/mail/func.inc @@ -363,6 +363,8 @@ function rcmail_js_message_list($a_headers, $insert_top=FALSE, $a_show_cols=null $cont = show_bytes($header->$col); else if ($col == 'date') $cont = format_date($header->date); + else if ($col == 'folder') + $cont = Q(rcube_charset::convert($header->folder, 'UTF7-IMAP')); else $cont = Q($header->$col); @@ -386,7 +388,7 @@ function rcmail_js_message_list($a_headers, $insert_top=FALSE, $a_show_cols=null $a_msg_flags['prio'] = (int) $header->priority; $a_msg_flags['ctype'] = Q($header->ctype); - $a_msg_flags['mbox'] = $mbox; + $a_msg_flags['mbox'] = $header->folder; // merge with plugin result (Deprecated, use $header->flags) if (!empty($header->list_flags) && is_array($header->list_flags)) diff --git a/skins/larry/mail.css b/skins/larry/mail.css index b65b08112..3c7242b4f 100644 --- a/skins/larry/mail.css +++ b/skins/larry/mail.css @@ -521,6 +521,14 @@ table.messagelist.fixedcopy { width: 135px; } +.messagelist tr td.folder { + width: 135px; +} + +.messagelist tr td.hidden { + display: none; +} + .messagelist tr.message { /* background-color: #fff; */ } From 9e63cd5f24defa521724dfe3dcbbaa4385761836 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 9 Oct 2013 17:12:30 +0200 Subject: [PATCH 3/5] Support globally unique message UIDs with IMAP folder name appended --- program/js/app.js | 27 +++++++++++++------- program/steps/mail/copy.inc | 6 ++--- program/steps/mail/func.inc | 44 +++++++++++++++++++++++++++++++++ program/steps/mail/mark.inc | 10 +++++--- program/steps/mail/move_del.inc | 21 ++++++++-------- program/steps/mail/search.inc | 16 +++++++++++- 6 files changed, 97 insertions(+), 27 deletions(-) diff --git a/program/js/app.js b/program/js/app.js index 1d59794d7..94f343f9e 100644 --- a/program/js/app.js +++ b/program/js/app.js @@ -641,7 +641,7 @@ function rcube_webmail() case 'open': if (uid = this.get_single_uid()) { - obj.href = this.url('show', {_mbox: this.env.mailbox, _uid: uid}); + obj.href = this.url('show', {_mbox: this.get_message_mailbox(uid), _uid: uid}); return true; } break; @@ -737,9 +737,9 @@ function rcube_webmail() this.load_contact(cid, 'edit'); else if (this.task == 'settings' && props) this.load_identity(props, 'edit-identity'); - else if (this.task == 'mail' && (cid = this.get_single_uid())) { - url = { _mbox: this.env.mailbox }; - url[this.env.mailbox == this.env.drafts_mailbox && props != 'new' ? '_draft_uid' : '_uid'] = cid; + else if (this.task == 'mail' && (uid = this.get_single_uid())) { + url = { _mbox: this.get_message_mailbox(uid) }; + url[this.env.mailbox == this.env.drafts_mailbox && props != 'new' ? '_draft_uid' : '_uid'] = uid; this.open_compose_step(url); } break; @@ -1029,7 +1029,7 @@ function rcube_webmail() case 'reply-list': case 'reply': if (uid = this.get_single_uid()) { - url = {_reply_uid: uid, _mbox: this.env.mailbox}; + url = {_reply_uid: uid, _mbox: this.get_message_mailbox(uid)}; if (command == 'reply-all') // do reply-list, when list is detected and popup menu wasn't used url._all = (!props && this.commands['reply-list'] ? 'list' : 'all'); @@ -1057,7 +1057,7 @@ function rcube_webmail() this.gui_objects.messagepartframe.contentWindow.print(); } else if (uid = this.get_single_uid()) { - ref.printwin = this.open_window(this.env.comm_path+'&_action=print&_uid='+uid+'&_mbox='+urlencode(this.env.mailbox)+(this.env.safemode ? '&_safe=1' : ''), true, true); + ref.printwin = this.open_window(this.env.comm_path+'&_action=print&_uid='+uid+'&_mbox='+urlencode(this.get_message_mailbox(uid))+(this.env.safemode ? '&_safe=1' : ''), true, true); if (this.printwin) { if (this.env.action != 'show') this.mark_message('read', uid); @@ -1074,8 +1074,9 @@ function rcube_webmail() if (this.env.action == 'get') { location.href = location.href.replace(/_frame=/, '_download='); } - else if (uid = this.get_single_uid()) - this.goto_url('viewsource', { _uid: uid, _mbox: this.env.mailbox, _save: 1 }); + else if (uid = this.get_single_uid()) { + this.goto_url('viewsource', { _uid: uid, _mbox: this.get_message_mailbox(uid), _save: 1 }); + } break; // quicksearch @@ -1772,6 +1773,7 @@ function rcube_webmail() selected: this.select_all_mode || this.message_list.in_selection(uid), ml: flags.ml?1:0, ctype: flags.ctype, + mbox: flags.mbox, // flags from plugins flags: flags.extra_flags }); @@ -1974,7 +1976,7 @@ function rcube_webmail() var win, target = window, action = preview ? 'preview': 'show', - url = '&_action='+action+'&_uid='+id+'&_mbox='+urlencode(this.env.mailbox); + url = '&_action='+action+'&_uid='+id+'&_mbox='+urlencode(this.get_message_mailbox(id)); if (preview && (win = this.get_frame_window(this.env.contentframe))) { target = win; @@ -6882,6 +6884,13 @@ function rcube_webmail() return this.env.cid ? this.env.cid : (this.contact_list ? this.contact_list.get_single_selection() : null); }; + // get the IMP mailbox of the message with the given UID + this.get_message_mailbox = function(uid) + { + var msg = this.env.messages ? this.env.messages[uid] : {}; + return msg.mbox || this.env.mailbox; + } + // gets cursor position this.get_caret_pos = function(obj) { diff --git a/program/steps/mail/copy.inc b/program/steps/mail/copy.inc index 876657485..0ed0d05a1 100644 --- a/program/steps/mail/copy.inc +++ b/program/steps/mail/copy.inc @@ -25,11 +25,11 @@ if (!$OUTPUT->ajax_call) // move messages if (!empty($_POST['_uid']) && strlen($_POST['_target_mbox'])) { - $uids = get_input_value('_uid', RCUBE_INPUT_POST); $target = get_input_value('_target_mbox', RCUBE_INPUT_POST, true); - $mbox = get_input_value('_mbox', RCUBE_INPUT_POST, true); - $copied = $RCMAIL->storage->copy_message($uids, $target, $mbox); + foreach (rcmail_get_uids() as $mbox => $uids) { + $copied += (int)$RCMAIL->storage->copy_message($uids, $target, $mbox); + } if (!$copied) { // send error message diff --git a/program/steps/mail/func.inc b/program/steps/mail/func.inc index b677c4675..807ec3ce9 100644 --- a/program/steps/mail/func.inc +++ b/program/steps/mail/func.inc @@ -66,6 +66,21 @@ if (!empty($_REQUEST['_search']) && isset($_SESSION['search']) $OUTPUT->set_env('search_text', $_SESSION['last_text_search']); } +// remove mbox part from _uid +if (($_uid = get_input_value('_uid', RCUBE_INPUT_GPC)) && preg_match('/^\d+-[^,]+$/', $_uid)) { + list($_uid, $mbox) = explode('-', $_uid); + if (isset($_GET['_uid'])) $_GET['_uid'] = $_uid; + if (isset($_POST['_uid'])) $_POST['_uid'] = $_uid; + $_REQUEST['_uid'] = $_uid; + unset($_uid); + + if (empty($_REQUEST['_mbox']) && !empty($mbox)) { + $_GET['_mbox'] = $mbox; + $_POST['_mbox'] = $mbox; + } +} + + // set main env variables, labels and page title if (empty($RCMAIL->action) || $RCMAIL->action == 'list') { // connect to storage server and trigger error on failure @@ -161,6 +176,35 @@ $RCMAIL->register_action_map(array( )); +/** + * Returns message UID(s) and IMAP folder(s) from GET/POST data + * + * @return array List of message UIDs per folder + */ +function rcmail_get_uids() +{ + // message UID (or comma-separated list of IDs) is provided in + // the form of -[,-]* + + $_uid = get_input_value('_uid', RCUBE_INPUT_GPC); + $_mbox = (string)get_input_value('_mbox', RCUBE_INPUT_GPC); + + if (is_array($uid)) { + return $uid; + } + + // create a per-folder UIDs array + $result = array(); + foreach (explode(',', $_uid) as $uid) { + list($uid, $mbox) = explode('-', $uid, 2); + if (empty($mbox)) + $mbox = $_mbox; + $result[$mbox][] = $uid; + } + + return $result; +} + /** * Returns 'to' if current folder is configured Sent or Drafts diff --git a/program/steps/mail/mark.inc b/program/steps/mail/mark.inc index dfc892ea1..fad11d0fc 100644 --- a/program/steps/mail/mark.inc +++ b/program/steps/mail/mark.inc @@ -32,7 +32,7 @@ $a_flags_map = array( $threading = (bool) $RCMAIL->storage->get_threading(); -if (($uids = get_input_value('_uid', RCUBE_INPUT_POST)) && ($flag = get_input_value('_flag', RCUBE_INPUT_POST))) +if (($_uids = get_input_value('_uid', RCUBE_INPUT_POST)) && ($flag = get_input_value('_flag', RCUBE_INPUT_POST))) { $flag = $a_flags_map[$flag] ? $a_flags_map[$flag] : strtoupper($flag); @@ -40,10 +40,12 @@ if (($uids = get_input_value('_uid', RCUBE_INPUT_POST)) && ($flag = get_input_va // count messages before changing anything $old_count = $RCMAIL->storage->count(NULL, $threading ? 'THREADS' : 'ALL'); $old_pages = ceil($old_count / $RCMAIL->storage->get_pagesize()); - $count = sizeof(explode(',', $uids)); } - $marked = $RCMAIL->storage->set_flag($uids, $flag); + foreach (rcmail_get_uids() as $mbox => $uids) { + $marked += (int)$RCMAIL->storage->set_flag($uids, $flag, $mbox); + $count += count($uids); + } if (!$marked) { // send error message @@ -117,7 +119,7 @@ if (($uids = get_input_value('_uid', RCUBE_INPUT_POST)) && ($flag = get_input_va } // add new rows from next page (if any) - if ($count && $uids != '*' && ($jump_back || $nextpage_count > 0)) { + if ($old_count && $_uids != '*' && ($jump_back || $nextpage_count > 0)) { $a_headers = $RCMAIL->storage->list_messages($mbox, NULL, rcmail_sort_column(), rcmail_sort_order(), $jump_back ? NULL : $count); diff --git a/program/steps/mail/move_del.inc b/program/steps/mail/move_del.inc index f15cd2460..e172d7188 100644 --- a/program/steps/mail/move_del.inc +++ b/program/steps/mail/move_del.inc @@ -5,7 +5,7 @@ | program/steps/mail/move_del.inc | | | | This file is part of the Roundcube Webmail client | - | Copyright (C) 2005-2009, The Roundcube Dev Team | + | Copyright (C) 2005-2013, The Roundcube Dev Team | | | | Licensed under the GNU General Public License version 3 or | | any later version with exceptions for skins & plugins. | @@ -30,12 +30,13 @@ $old_pages = ceil($old_count / $RCMAIL->storage->get_pagesize()); // move messages if ($RCMAIL->action == 'move' && !empty($_POST['_uid']) && strlen($_POST['_target_mbox'])) { - $count = sizeof(explode(',', ($uids = get_input_value('_uid', RCUBE_INPUT_POST)))); $target = get_input_value('_target_mbox', RCUBE_INPUT_POST, true); - $mbox = get_input_value('_mbox', RCUBE_INPUT_POST, true); $trash = $RCMAIL->config->get('trash_mbox'); - $moved = $RCMAIL->storage->move_message($uids, $target, $mbox); + foreach (rcmail_get_uids() as $mbox => $uids) { + $moved += (int)$RCMAIL->storage->move_message($uids, $target, $mbox); + $count += count($uids); + } if (!$moved) { // send error message @@ -46,17 +47,17 @@ if ($RCMAIL->action == 'move' && !empty($_POST['_uid']) && strlen($_POST['_targe exit; } else { - $OUTPUT->show_message('messagemoved', 'confirmation'); + $OUTPUT->show_message('messagemoved', 'confirmation'); } $addrows = true; } // delete messages else if ($RCMAIL->action=='delete' && !empty($_POST['_uid'])) { - $count = sizeof(explode(',', ($uids = get_input_value('_uid', RCUBE_INPUT_POST)))); - $mbox = get_input_value('_mbox', RCUBE_INPUT_POST, true); - - $del = $RCMAIL->storage->delete_message($uids, $mbox); + foreach (rcmail_get_uids() as $mbox => $uids) { + $del += (int)$RCMAIL->storage->delete_message($uids, $mbox); + $count += count($uids); + } if (!$del) { // send error message @@ -67,7 +68,7 @@ else if ($RCMAIL->action=='delete' && !empty($_POST['_uid'])) { exit; } else { - $OUTPUT->show_message('messagedeleted', 'confirmation'); + $OUTPUT->show_message('messagedeleted', 'confirmation'); } $addrows = true; diff --git a/program/steps/mail/search.inc b/program/steps/mail/search.inc index fb1b48797..0632b042a 100644 --- a/program/steps/mail/search.inc +++ b/program/steps/mail/search.inc @@ -126,9 +126,23 @@ $_SESSION['search_request'] = $search_request; $result_h = $RCMAIL->storage->list_messages($mbox, 1, $sort_column, rcmail_sort_order()); $count = $RCMAIL->storage->count($mbox, $RCMAIL->storage->get_threading() ? 'THREADS' : 'ALL'); +// Add 'folder' column to list +if ($multi_folder_search) { + $a_show_cols = $_SESSION['list_attrib']['columns'] ? $_SESSION['list_attrib']['columns'] : (array)$CONFIG['list_cols']; + if (!in_array($a_show_cols)) + $a_show_cols[] = 'folder'; + + // make message UIDs unique by appending the folder name + foreach ($result_h as $i => $header) { + $header->uid .= '-'.$header->folder; + if ($header->parent_uid) + $header->parent_uid .= '-'.$header->folder; + } +} + // Make sure we got the headers if (!empty($result_h)) { - rcmail_js_message_list($result_h); + rcmail_js_message_list($result_h, false, $a_show_cols); if ($search_str) $OUTPUT->show_message('searchsuccessful', 'confirmation', array('nr' => $RCMAIL->storage->count(NULL, 'ALL'))); } From 7e3e3ef81ad48f161d01044dcdc2b8cf51811a4f Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 14 Oct 2013 21:57:53 +0200 Subject: [PATCH 4/5] First attempt to search in multiple folders; do it multi-threaded using pthreads if available --- program/lib/Roundcube/rcube_imap.php | 80 ++++- program/lib/Roundcube/rcube_imap_search.php | 327 ++++++++++++++++++ .../Roundcube/rcube_result_multifolder.php | 211 +++++++++++ program/steps/mail/search.inc | 12 +- 4 files changed, 620 insertions(+), 10 deletions(-) create mode 100644 program/lib/Roundcube/rcube_imap_search.php create mode 100644 program/lib/Roundcube/rcube_result_multifolder.php diff --git a/program/lib/Roundcube/rcube_imap.php b/program/lib/Roundcube/rcube_imap.php index e0dce6f79..98c8f7a4b 100644 --- a/program/lib/Roundcube/rcube_imap.php +++ b/program/lib/Roundcube/rcube_imap.php @@ -910,6 +910,50 @@ class rcube_imap extends rcube_storage return array(); } + // gather messages from a multi-folder search + if ($this->search_set->multi) { + $page_size = $this->page_size; + $sort_field = $this->sort_field; + $search_set = $this->search_set; + + $this->sort_field = null; + $this->page_size = 100; // limit to 100 messages per folder + + $a_msg_headers = array(); + foreach ($search_set->sets as $resultset) { + if (!$resultset->is_empty()) { + $this->search_set = $resultset; + $this->search_threads = $resultset instanceof rcube_result_thread; + $a_msg_headers = array_merge($a_msg_headers, $this->list_search_messages($resultset->get_parameters('MAILBOX'), 1)); + } + } + + // do sorting and paging + $cnt = $search_set->count(); + $from = ($page-1) * $page_size; + $to = $from + $page_size; + + // sort headers + if (!$this->threading) { + $a_msg_headers = $this->conn->sortHeaders($a_msg_headers, $this->sort_field, $this->sort_order); + } + + // only return the requested part of the set + $slice_length = min($page_size, $cnt - ($to > $cnt ? $from : $to)); + $a_msg_headers = array_slice(array_values($a_msg_headers), $from, $slice_length); + + if ($slice) { + $a_msg_headers = array_slice($a_msg_headers, -$slice, $slice); + } + + // restore members + $this->sort_field = $sort_field; + $this->page_size = $page_size; + $this->search_set = $search_set; + + return $a_msg_headers; + } + // use saved messages from searching if ($this->threading) { return $this->list_search_thread_messages($folder, $page, $slice); @@ -1388,11 +1432,33 @@ class rcube_imap extends rcube_storage $str = 'ALL'; } - if (!strlen($folder)) { + if (empty($folder)) { $folder = $this->folder; } - $results = $this->search_index($folder, $str, $charset, $sort_field); + // multi-folder search + if (is_array($folder) && count($folder) > 1 && $str != 'ALL') { + new rcube_result_index; // trigger autoloader and make these classes available for threaded context + new rcube_result_thread; + + // connect IMAP + if (!defined('PTHREADS_INHERIT_ALL')) { + $this->check_connection(); + } + + $searcher = new rcube_imap_search($this->options, $this->conn); + $results = $searcher->exec( + $folder, + $str, + $charset ? $charset : $this->default_charset, + $sort_field && $this->get_capability('SORT') ? $sort_field : null, + $this->threading + ); + } + else { + $folder = is_array($folder) ? $folder[0] : $folder; + $results = $this->search_index($folder, $str, $charset, $sort_field); + } $this->set_search_set(array($str, $results, $charset, $sort_field, $this->threading || $this->search_sorted ? true : false)); @@ -1466,7 +1532,7 @@ class rcube_imap extends rcube_storage // but I've seen that Courier doesn't support UTF-8) if ($threads->is_error() && $charset && $charset != 'US-ASCII') { $threads = $this->conn->thread($folder, $this->threading, - $this->convert_criteria($criteria, $charset), true, 'US-ASCII'); + self::convert_criteria($criteria, $charset), true, 'US-ASCII'); } return $threads; @@ -1480,7 +1546,7 @@ class rcube_imap extends rcube_storage // but I've seen Courier with disabled UTF-8 support) if ($messages->is_error() && $charset && $charset != 'US-ASCII') { $messages = $this->conn->sort($folder, $sort_field, - $this->convert_criteria($criteria, $charset), true, 'US-ASCII'); + self::convert_criteria($criteria, $charset), true, 'US-ASCII'); } if (!$messages->is_error()) { @@ -1495,7 +1561,7 @@ class rcube_imap extends rcube_storage // Error, try with US-ASCII (some servers may support only US-ASCII) if ($messages->is_error() && $charset && $charset != 'US-ASCII') { $messages = $this->conn->search($folder, - $this->convert_criteria($criteria, $charset), true); + self::convert_criteria($criteria, $charset), true); } $this->search_sorted = false; @@ -1513,7 +1579,7 @@ class rcube_imap extends rcube_storage * * @return string Search string */ - protected function convert_criteria($str, $charset, $dest_charset='US-ASCII') + public static function convert_criteria($str, $charset, $dest_charset='US-ASCII') { // convert strings to US_ASCII if (preg_match_all('/\{([0-9]+)\}\r\n/', $str, $matches, PREG_OFFSET_CAPTURE)) { @@ -2375,7 +2441,7 @@ class rcube_imap extends rcube_storage $this->refresh_search(); } else { - $this->search_set->filter(explode(',', $uids)); + $this->search_set->filter(explode(',', $uids), $this->folder); } } diff --git a/program/lib/Roundcube/rcube_imap_search.php b/program/lib/Roundcube/rcube_imap_search.php new file mode 100644 index 000000000..ed4face98 --- /dev/null +++ b/program/lib/Roundcube/rcube_imap_search.php @@ -0,0 +1,327 @@ + | + +-----------------------------------------------------------------------+ +*/ + +// create classes defined by the pthreads module if that isn't installed +if (!defined('PTHREADS_INHERIT_ALL')) { + class Worker { } + class Stackable { } +} + +/** + * Class to control search jobs on multiple IMAP folders. + * This implement a simple threads pool using the pthreads extension. + * + * @package Framework + * @subpackage Storage + * @author Thomas Bruederli + */ +class rcube_imap_search +{ + public $options = array(); + + private $size = 10; + private $next = 0; + private $workers = array(); + private $states = array(); + private $jobs = array(); + private $conn; + + /** + * Default constructor + */ + public function __construct($options, $conn) + { + $this->options = $options; + $this->conn = $conn; + } + + /** + * Invoke search request to IMAP server + * + * @param array $folders List of IMAP folders to search in + * @param string $str Search criteria + * @param string $charset Search charset + * @param string $sort_field Header field to sort by + * @param boolean $threading True if threaded listing is active + */ + public function exec($folders, $str, $charset = null, $sort_field = null, $threading=null) + { + $pthreads = defined('PTHREADS_INHERIT_ALL'); + + // start a search job for every folder to search in + foreach ($folders as $folder) { + $job = new rcube_imap_search_job($folder, $str, $charset, $sort_field, $threading); + if ($pthreads && $this->submit($job)) { + $this->jobs[] = $job; + } + else { + $job->worker = $this; + $job->run(); + $this->jobs[] = $job; + } + } + + // wait for all workers to be done + $this->shutdown(); + + // gather results + $results = new rcube_result_multifolder; + foreach ($this->jobs as $job) { + $results->add($job->get_result()); + } + + return $results; + } + + /** + * Assign the given job object to one of the worker threads for execution + */ + public function submit(Stackable $job) + { + if (count($this->workers) < $this->size) { + $id = count($this->workers); + $this->workers[$id] = new rcube_imap_search_worker($id, $this->options); + $this->workers[$id]->start(PTHREADS_INHERIT_ALL); + + if ($this->workers[$id]->stack($job)) { + return $job; + } + else { + // trigger_error(sprintf("Failed to push Stackable onto %s", $id), E_USER_WARNING); + } + } + if (($worker = $this->workers[$this->next])) { + $this->next = ($this->next+1) % $this->size; + if ($worker->stack($job)) { + return $job; + } + else { + // trigger_error(sprintf("Failed to stack onto selected worker %s", $worker->id), E_USER_WARNING); + } + } + else { + // trigger_error(sprintf("Failed to select a worker for Stackable"), E_USER_WARNING); + } + + return false; + } + + /** + * Shutdown the pool of threads cleanly, retaining exit status locally + */ + public function shutdown() + { + foreach ($this->workers as $worker) { + $this->states[$worker->getThreadId()] = $worker->shutdown(); + $worker->close(); + } + + # console('shutdown', $this->states); + } + + /** + * Get connection to the IMAP server + * (used for single-thread mode) + */ + public function get_imap() + { + return $this->conn; + } +} + + +/** + * Stackable item to run the search on a specific IMAP folder + */ +class rcube_imap_search_job extends Stackable +{ + private $folder; + private $search; + private $charset; + private $sort_field; + private $threading; + private $searchset; + private $result; + private $pagesize = 100; + + public function __construct($folder, $str, $charset = null, $sort_field = null, $threading=false) + { + $this->folder = $folder; + $this->search = $str; + $this->charset = $charset; + $this->sort_field = $sort_field; + $this->threading = $threading; + } + + public function run() + { + #trigger_error("Start search $this->folder", E_USER_NOTICE); + $this->result = $this->search_index(); + #trigger_error("End search $this->folder: " . $this->result->count(), E_USER_NOTICE); + } + + /** + * Copy of rcube_imap::search_index() + */ + protected function search_index() + { + $criteria = $this->search; + $charset = $this->charset; + + $imap = $this->worker->get_imap(); + + if (!$imap->connected()) { + if ($this->threading) { + return new rcube_result_thread(); + } + else { + return new rcube_result_index(); + } + } + + if ($this->worker->options['skip_deleted'] && !preg_match('/UNDELETED/', $criteria)) { + $criteria = 'UNDELETED '.$criteria; + } + + // unset CHARSET if criteria string is ASCII, this way + // SEARCH won't be re-sent after "unsupported charset" response + if ($charset && $charset != 'US-ASCII' && is_ascii($criteria)) { + $charset = 'US-ASCII'; + } + + if ($this->threading) { + $threads = $imap->thread($this->folder, $this->threading, $criteria, true, $charset); + + // Error, try with US-ASCII (RFC5256: SORT/THREAD must support US-ASCII and UTF-8, + // but I've seen that Courier doesn't support UTF-8) + if ($threads->is_error() && $charset && $charset != 'US-ASCII') { + $threads = $imap->thread($this->folder, $this->threading, + rcube_imap::convert_criteria($criteria, $charset), true, 'US-ASCII'); + } + + return $threads; + } + + if ($this->sort_field) { + $messages = $imap->sort($this->folder, $this->sort_field, $criteria, true, $charset); + + // Error, try with US-ASCII (RFC5256: SORT/THREAD must support US-ASCII and UTF-8, + // but I've seen Courier with disabled UTF-8 support) + if ($messages->is_error() && $charset && $charset != 'US-ASCII') { + $messages = $imap->sort($this->folder, $this->sort_field, + rcube_imap::convert_criteria($criteria, $charset), true, 'US-ASCII'); + } + + if (!$messages->is_error()) { + return $messages; + } + } + + $messages = $imap->search($this->folder, + ($charset && $charset != 'US-ASCII' ? "CHARSET $charset " : '') . $criteria, true); + + // Error, try with US-ASCII (some servers may support only US-ASCII) + if ($messages->is_error() && $charset && $charset != 'US-ASCII') { + $messages = $imap->search($this->folder, + rcube_imap::convert_criteria($criteria, $charset), true); + } + + return $messages; + } + + public function get_search_set() + { + return array( + $this->search, + $this->result, + $this->charset, + $this->sort_field, + $this->threading, + ); + } + + public function get_result() + { + return $this->result; + } +} + + +/** + * Wrker thread to run search jobs while maintaining a common context + */ +class rcube_imap_search_worker extends Worker +{ + public $id; + public $options; + + private $conn; + + /** + * Default constructor + */ + public function __construct($id, $options) + { + $this->id = $id; + $this->options = $options; + } + + /** + * Get a dedicated connection to the IMAP server + */ + public function get_imap() + { + // TODO: make this connection persistent for several jobs + #if ($this->conn) + # return $this->conn; + + $conn = new rcube_imap_generic(); + # $conn->setDebug(true, function($conn, $message){ trigger_error($message, E_USER_NOTICE); }); + + if ($this->options['user'] && $this->options['password']) { + $conn->connect($this->options['host'], $this->options['user'], $this->options['password'], $this->options); + } + + if ($conn->error) + trigger_error($this->conn->error, E_USER_WARNING); + + #$this->conn = $conn; + return $conn; + } + + /** + * @override + */ + public function run() + { + + } + + /** + * Close IMAP connection + */ + public function close() + { + if ($this->conn) { + $this->conn->close(); + } + } +} + diff --git a/program/lib/Roundcube/rcube_result_multifolder.php b/program/lib/Roundcube/rcube_result_multifolder.php new file mode 100644 index 000000000..8d7ae5de8 --- /dev/null +++ b/program/lib/Roundcube/rcube_result_multifolder.php @@ -0,0 +1,211 @@ + | + +-----------------------------------------------------------------------+ +*/ + +/** + * Class holding a set of rcube_result_index instances that together form a + * result set of a multi-folder search + * + * @package Framework + * @subpackage Storage + */ +class rcube_result_multifolder +{ + public $multi = true; + public $sets = array(); + + protected $meta = array(); + protected $order = 'ASC'; + + + /** + * Object constructor. + */ + public function __construct() + { + $this->meta = array('count' => 0); + } + + + /** + * Initializes object with SORT command response + * + * @param string $data IMAP response string + */ + public function add($result) + { + $this->sets[] = $result; + $this->meta['count'] += $result->count(); + } + + + /** + * Checks the result from IMAP command + * + * @return bool True if the result is an error, False otherwise + */ + public function is_error() + { + return false; + } + + + /** + * Checks if the result is empty + * + * @return bool True if the result is empty, False otherwise + */ + public function is_empty() + { + return empty($this->sets) || $this->meta['count'] == 0; + } + + + /** + * Returns number of elements in the result + * + * @return int Number of elements + */ + public function count() + { + return $this->meta['count']; + } + + + /** + * Returns number of elements in the result. + * Alias for count() for compatibility with rcube_result_thread + * + * @return int Number of elements + */ + public function count_messages() + { + return $this->count(); + } + + + /** + * Reverts order of elements in the result + */ + public function revert() + { + $this->order = $this->order == 'ASC' ? 'DESC' : 'ASC'; + } + + + /** + * Check if the given message ID exists in the object + * + * @param int $msgid Message ID + * @param bool $get_index When enabled element's index will be returned. + * Elements are indexed starting with 0 + * @return mixed False if message ID doesn't exist, True if exists or + * index of the element if $get_index=true + */ + public function exists($msgid, $get_index = false) + { + return false; + } + + + /** + * Filters data set. Removes elements listed in $ids list. + * + * @param array $ids List of IDs to remove. + * @param string $folder IMAP folder + */ + public function filter($ids = array(), $folder = null) + { + $this->meta['count'] = 0; + foreach ($this->sets as $set) { + if ($set->get_parameters('MAILBOX') == $folder) { + $set->filter($ids); + } + $this->meta['count'] += $set->count(); + } + } + + /** + * Filters data set. Removes elements not listed in $ids list. + * + * @param array $ids List of IDs to keep. + */ + public function intersect($ids = array()) + { + // not implemented + } + + /** + * Return all messages in the result. + * + * @return array List of message IDs + */ + public function get() + { + return array(); + } + + + /** + * Return all messages in the result. + * + * @return array List of message IDs + */ + public function get_compressed() + { + return ''; + } + + + /** + * Return result element at specified index + * + * @param int|string $index Element's index or "FIRST" or "LAST" + * + * @return int Element value + */ + public function get_element($index) + { + return null; + } + + + /** + * Returns response parameters, e.g. ESEARCH's MIN/MAX/COUNT/ALL/MODSEQ + * or internal data e.g. MAILBOX, ORDER + * + * @param string $param Parameter name + * + * @return array|string Response parameters or parameter value + */ + public function get_parameters($param=null) + { + return $params; + } + + + /** + * Returns length of internal data representation + * + * @return int Data length + */ + protected function length() + { + return $this->count(); + } +} diff --git a/program/steps/mail/search.inc b/program/steps/mail/search.inc index 0632b042a..7d128c73c 100644 --- a/program/steps/mail/search.inc +++ b/program/steps/mail/search.inc @@ -107,9 +107,12 @@ if (!empty($subject)) { $search_str = trim($search_str); $sort_column = rcmail_sort_column(); +// TEMPORARY: search all folders +$mboxes = $RCMAIL->storage->list_folders_subscribed('', '*', 'mail'); + // execute IMAP search if ($search_str) - $RCMAIL->storage->search($mbox, $search_str, $imap_charset, $sort_column); + $RCMAIL->storage->search($mboxes, $search_str, $imap_charset, $sort_column); // save search results in session if (!is_array($_SESSION['search'])) @@ -127,17 +130,20 @@ $result_h = $RCMAIL->storage->list_messages($mbox, 1, $sort_column, rcmail_sort_ $count = $RCMAIL->storage->count($mbox, $RCMAIL->storage->get_threading() ? 'THREADS' : 'ALL'); // Add 'folder' column to list -if ($multi_folder_search) { +if ($_SESSION['search'][1]->multi) { $a_show_cols = $_SESSION['list_attrib']['columns'] ? $_SESSION['list_attrib']['columns'] : (array)$CONFIG['list_cols']; - if (!in_array($a_show_cols)) + if (!in_array('folder', $a_show_cols)) $a_show_cols[] = 'folder'; // make message UIDs unique by appending the folder name foreach ($result_h as $i => $header) { $header->uid .= '-'.$header->folder; + $header->flags['skip_mbox_check'] = true; if ($header->parent_uid) $header->parent_uid .= '-'.$header->folder; } + + $OUTPUT->command('select_folder', ''); } // Make sure we got the headers From 010a350715f1a36eab666fe26d3118ed025133c1 Mon Sep 17 00:00:00 2001 From: Thomas Bruederli Date: Tue, 15 Oct 2013 11:44:34 +0200 Subject: [PATCH 5/5] Minor improvements to threaded searching --- program/lib/Roundcube/rcube_imap.php | 8 ++--- program/lib/Roundcube/rcube_imap_search.php | 36 +++++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/program/lib/Roundcube/rcube_imap.php b/program/lib/Roundcube/rcube_imap.php index 98c8f7a4b..ff88bdccb 100644 --- a/program/lib/Roundcube/rcube_imap.php +++ b/program/lib/Roundcube/rcube_imap.php @@ -934,7 +934,7 @@ class rcube_imap extends rcube_storage $to = $from + $page_size; // sort headers - if (!$this->threading) { + if (!$this->threading && !empty($a_msg_headers)) { $a_msg_headers = $this->conn->sortHeaders($a_msg_headers, $this->sort_field, $this->sort_order); } @@ -1441,10 +1441,8 @@ class rcube_imap extends rcube_storage new rcube_result_index; // trigger autoloader and make these classes available for threaded context new rcube_result_thread; - // connect IMAP - if (!defined('PTHREADS_INHERIT_ALL')) { - $this->check_connection(); - } + // connect IMAP to have all the required classes and settings loaded + $this->check_connection(); $searcher = new rcube_imap_search($this->options, $this->conn); $results = $searcher->exec( diff --git a/program/lib/Roundcube/rcube_imap_search.php b/program/lib/Roundcube/rcube_imap_search.php index ed4face98..d82ec8a24 100644 --- a/program/lib/Roundcube/rcube_imap_search.php +++ b/program/lib/Roundcube/rcube_imap_search.php @@ -172,9 +172,9 @@ class rcube_imap_search_job extends Stackable public function run() { - #trigger_error("Start search $this->folder", E_USER_NOTICE); + // trigger_error("Start search $this->folder", E_USER_NOTICE); $this->result = $this->search_index(); - #trigger_error("End search $this->folder: " . $this->result->count(), E_USER_NOTICE); + // trigger_error("End search $this->folder: " . $this->result->count(), E_USER_NOTICE); } /** @@ -182,6 +182,7 @@ class rcube_imap_search_job extends Stackable */ protected function search_index() { + $pthreads = defined('PTHREADS_INHERIT_ALL'); $criteria = $this->search; $charset = $this->charset; @@ -216,6 +217,10 @@ class rcube_imap_search_job extends Stackable rcube_imap::convert_criteria($criteria, $charset), true, 'US-ASCII'); } + // close IMAP connection again + if ($pthreads) + $imap->closeConnection(); + return $threads; } @@ -228,21 +233,23 @@ class rcube_imap_search_job extends Stackable $messages = $imap->sort($this->folder, $this->sort_field, rcube_imap::convert_criteria($criteria, $charset), true, 'US-ASCII'); } - - if (!$messages->is_error()) { - return $messages; - } } - $messages = $imap->search($this->folder, - ($charset && $charset != 'US-ASCII' ? "CHARSET $charset " : '') . $criteria, true); - - // Error, try with US-ASCII (some servers may support only US-ASCII) - if ($messages->is_error() && $charset && $charset != 'US-ASCII') { + if (!$messages || !$messages->is_error()) { $messages = $imap->search($this->folder, - rcube_imap::convert_criteria($criteria, $charset), true); + ($charset && $charset != 'US-ASCII' ? "CHARSET $charset " : '') . $criteria, true); + + // Error, try with US-ASCII (some servers may support only US-ASCII) + if ($messages->is_error() && $charset && $charset != 'US-ASCII') { + $messages = $imap->search($this->folder, + rcube_imap::convert_criteria($criteria, $charset), true); + } } + // close IMAP connection again + if ($pthreads) + $imap->closeConnection(); + return $messages; } @@ -279,6 +286,8 @@ class rcube_imap_search_worker extends Worker */ public function __construct($id, $options) { + $options['ident']['command'] = 'search-'.$id; + $this->id = $id; $this->options = $options; } @@ -296,11 +305,12 @@ class rcube_imap_search_worker extends Worker # $conn->setDebug(true, function($conn, $message){ trigger_error($message, E_USER_NOTICE); }); if ($this->options['user'] && $this->options['password']) { + // TODO: do this synchronized to avoid warnings like "Only one Id allowed in non-authenticated state" $conn->connect($this->options['host'], $this->options['user'], $this->options['password'], $this->options); } if ($conn->error) - trigger_error($this->conn->error, E_USER_WARNING); + trigger_error($conn->error, E_USER_WARNING); #$this->conn = $conn; return $conn;