First attempt to search in multiple folders; do it multi-threaded using pthreads if available

pull/193/head
Thomas 11 years ago committed by Thomas Bruederli
parent 9684dc018f
commit 566747af00

@ -945,6 +945,50 @@ class rcube_imap extends rcube_storage
return array();
}
// gather messages from a multi-folder search
if ($this->search_set->multi) {
$page_size = $this->page_size;
$sort_field = $this->sort_field;
$search_set = $this->search_set;
$this->sort_field = null;
$this->page_size = 100; // limit to 100 messages per folder
$a_msg_headers = array();
foreach ($search_set->sets as $resultset) {
if (!$resultset->is_empty()) {
$this->search_set = $resultset;
$this->search_threads = $resultset instanceof rcube_result_thread;
$a_msg_headers = array_merge($a_msg_headers, $this->list_search_messages($resultset->get_parameters('MAILBOX'), 1));
}
}
// do sorting and paging
$cnt = $search_set->count();
$from = ($page-1) * $page_size;
$to = $from + $page_size;
// sort headers
if (!$this->threading) {
$a_msg_headers = $this->conn->sortHeaders($a_msg_headers, $this->sort_field, $this->sort_order);
}
// only return the requested part of the set
$slice_length = min($page_size, $cnt - ($to > $cnt ? $from : $to));
$a_msg_headers = array_slice(array_values($a_msg_headers), $from, $slice_length);
if ($slice) {
$a_msg_headers = array_slice($a_msg_headers, -$slice, $slice);
}
// restore members
$this->sort_field = $sort_field;
$this->page_size = $page_size;
$this->search_set = $search_set;
return $a_msg_headers;
}
// use saved messages from searching
if ($this->threading) {
return $this->list_search_thread_messages($folder, $page, $slice);
@ -1423,11 +1467,33 @@ class rcube_imap extends rcube_storage
$str = 'ALL';
}
if (!strlen($folder)) {
if (empty($folder)) {
$folder = $this->folder;
}
$results = $this->search_index($folder, $str, $charset, $sort_field);
// multi-folder search
if (is_array($folder) && count($folder) > 1 && $str != 'ALL') {
new rcube_result_index; // trigger autoloader and make these classes available for threaded context
new rcube_result_thread;
// connect IMAP
if (!defined('PTHREADS_INHERIT_ALL')) {
$this->check_connection();
}
$searcher = new rcube_imap_search($this->options, $this->conn);
$results = $searcher->exec(
$folder,
$str,
$charset ? $charset : $this->default_charset,
$sort_field && $this->get_capability('SORT') ? $sort_field : null,
$this->threading
);
}
else {
$folder = is_array($folder) ? $folder[0] : $folder;
$results = $this->search_index($folder, $str, $charset, $sort_field);
}
$this->set_search_set(array($str, $results, $charset, $sort_field,
$this->threading || $this->search_sorted ? true : false));
@ -1501,7 +1567,7 @@ class rcube_imap extends rcube_storage
// but I've seen that Courier doesn't support UTF-8)
if ($threads->is_error() && $charset && $charset != 'US-ASCII') {
$threads = $this->conn->thread($folder, $this->threading,
$this->convert_criteria($criteria, $charset), true, 'US-ASCII');
self::convert_criteria($criteria, $charset), true, 'US-ASCII');
}
return $threads;
@ -1515,7 +1581,7 @@ class rcube_imap extends rcube_storage
// but I've seen Courier with disabled UTF-8 support)
if ($messages->is_error() && $charset && $charset != 'US-ASCII') {
$messages = $this->conn->sort($folder, $sort_field,
$this->convert_criteria($criteria, $charset), true, 'US-ASCII');
self::convert_criteria($criteria, $charset), true, 'US-ASCII');
}
if (!$messages->is_error()) {
@ -1530,7 +1596,7 @@ class rcube_imap extends rcube_storage
// Error, try with US-ASCII (some servers may support only US-ASCII)
if ($messages->is_error() && $charset && $charset != 'US-ASCII') {
$messages = $this->conn->search($folder,
$this->convert_criteria($criteria, $charset), true);
self::convert_criteria($criteria, $charset), true);
}
$this->search_sorted = false;
@ -1548,7 +1614,7 @@ class rcube_imap extends rcube_storage
*
* @return string Search string
*/
protected function convert_criteria($str, $charset, $dest_charset='US-ASCII')
public static function convert_criteria($str, $charset, $dest_charset='US-ASCII')
{
// convert strings to US_ASCII
if (preg_match_all('/\{([0-9]+)\}\r\n/', $str, $matches, PREG_OFFSET_CAPTURE)) {
@ -2410,7 +2476,7 @@ class rcube_imap extends rcube_storage
$this->refresh_search();
}
else {
$this->search_set->filter(explode(',', $uids));
$this->search_set->filter(explode(',', $uids), $this->folder);
}
}

@ -0,0 +1,327 @@
<?php
/*
+-----------------------------------------------------------------------+
| This file is part of the Roundcube Webmail client |
| |
| Copyright (C) 2013, The Roundcube Dev Team |
| Copyright (C) 2013, Kolab Systems AG |
| |
| Licensed under the GNU General Public License version 3 or |
| any later version with exceptions for skins & plugins. |
| See the README file for a full license statement. |
| |
| PURPOSE: |
| Execute (multi-threaded) searches in multiple IMAP folders |
+-----------------------------------------------------------------------+
| Author: Thomas Bruederli <roundcube@gmail.com> |
+-----------------------------------------------------------------------+
*/
// create classes defined by the pthreads module if that isn't installed
if (!defined('PTHREADS_INHERIT_ALL')) {
class Worker { }
class Stackable { }
}
/**
* Class to control search jobs on multiple IMAP folders.
* This implement a simple threads pool using the pthreads extension.
*
* @package Framework
* @subpackage Storage
* @author Thomas Bruederli <roundcube@gmail.com>
*/
class rcube_imap_search
{
public $options = array();
private $size = 10;
private $next = 0;
private $workers = array();
private $states = array();
private $jobs = array();
private $conn;
/**
* Default constructor
*/
public function __construct($options, $conn)
{
$this->options = $options;
$this->conn = $conn;
}
/**
* Invoke search request to IMAP server
*
* @param array $folders List of IMAP folders to search in
* @param string $str Search criteria
* @param string $charset Search charset
* @param string $sort_field Header field to sort by
* @param boolean $threading True if threaded listing is active
*/
public function exec($folders, $str, $charset = null, $sort_field = null, $threading=null)
{
$pthreads = defined('PTHREADS_INHERIT_ALL');
// start a search job for every folder to search in
foreach ($folders as $folder) {
$job = new rcube_imap_search_job($folder, $str, $charset, $sort_field, $threading);
if ($pthreads && $this->submit($job)) {
$this->jobs[] = $job;
}
else {
$job->worker = $this;
$job->run();
$this->jobs[] = $job;
}
}
// wait for all workers to be done
$this->shutdown();
// gather results
$results = new rcube_result_multifolder;
foreach ($this->jobs as $job) {
$results->add($job->get_result());
}
return $results;
}
/**
* Assign the given job object to one of the worker threads for execution
*/
public function submit(Stackable $job)
{
if (count($this->workers) < $this->size) {
$id = count($this->workers);
$this->workers[$id] = new rcube_imap_search_worker($id, $this->options);
$this->workers[$id]->start(PTHREADS_INHERIT_ALL);
if ($this->workers[$id]->stack($job)) {
return $job;
}
else {
// trigger_error(sprintf("Failed to push Stackable onto %s", $id), E_USER_WARNING);
}
}
if (($worker = $this->workers[$this->next])) {
$this->next = ($this->next+1) % $this->size;
if ($worker->stack($job)) {
return $job;
}
else {
// trigger_error(sprintf("Failed to stack onto selected worker %s", $worker->id), E_USER_WARNING);
}
}
else {
// trigger_error(sprintf("Failed to select a worker for Stackable"), E_USER_WARNING);
}
return false;
}
/**
* Shutdown the pool of threads cleanly, retaining exit status locally
*/
public function shutdown()
{
foreach ($this->workers as $worker) {
$this->states[$worker->getThreadId()] = $worker->shutdown();
$worker->close();
}
# console('shutdown', $this->states);
}
/**
* Get connection to the IMAP server
* (used for single-thread mode)
*/
public function get_imap()
{
return $this->conn;
}
}
/**
* Stackable item to run the search on a specific IMAP folder
*/
class rcube_imap_search_job extends Stackable
{
private $folder;
private $search;
private $charset;
private $sort_field;
private $threading;
private $searchset;
private $result;
private $pagesize = 100;
public function __construct($folder, $str, $charset = null, $sort_field = null, $threading=false)
{
$this->folder = $folder;
$this->search = $str;
$this->charset = $charset;
$this->sort_field = $sort_field;
$this->threading = $threading;
}
public function run()
{
#trigger_error("Start search $this->folder", E_USER_NOTICE);
$this->result = $this->search_index();
#trigger_error("End search $this->folder: " . $this->result->count(), E_USER_NOTICE);
}
/**
* Copy of rcube_imap::search_index()
*/
protected function search_index()
{
$criteria = $this->search;
$charset = $this->charset;
$imap = $this->worker->get_imap();
if (!$imap->connected()) {
if ($this->threading) {
return new rcube_result_thread();
}
else {
return new rcube_result_index();
}
}
if ($this->worker->options['skip_deleted'] && !preg_match('/UNDELETED/', $criteria)) {
$criteria = 'UNDELETED '.$criteria;
}
// unset CHARSET if criteria string is ASCII, this way
// SEARCH won't be re-sent after "unsupported charset" response
if ($charset && $charset != 'US-ASCII' && is_ascii($criteria)) {
$charset = 'US-ASCII';
}
if ($this->threading) {
$threads = $imap->thread($this->folder, $this->threading, $criteria, true, $charset);
// Error, try with US-ASCII (RFC5256: SORT/THREAD must support US-ASCII and UTF-8,
// but I've seen that Courier doesn't support UTF-8)
if ($threads->is_error() && $charset && $charset != 'US-ASCII') {
$threads = $imap->thread($this->folder, $this->threading,
rcube_imap::convert_criteria($criteria, $charset), true, 'US-ASCII');
}
return $threads;
}
if ($this->sort_field) {
$messages = $imap->sort($this->folder, $this->sort_field, $criteria, true, $charset);
// Error, try with US-ASCII (RFC5256: SORT/THREAD must support US-ASCII and UTF-8,
// but I've seen Courier with disabled UTF-8 support)
if ($messages->is_error() && $charset && $charset != 'US-ASCII') {
$messages = $imap->sort($this->folder, $this->sort_field,
rcube_imap::convert_criteria($criteria, $charset), true, 'US-ASCII');
}
if (!$messages->is_error()) {
return $messages;
}
}
$messages = $imap->search($this->folder,
($charset && $charset != 'US-ASCII' ? "CHARSET $charset " : '') . $criteria, true);
// Error, try with US-ASCII (some servers may support only US-ASCII)
if ($messages->is_error() && $charset && $charset != 'US-ASCII') {
$messages = $imap->search($this->folder,
rcube_imap::convert_criteria($criteria, $charset), true);
}
return $messages;
}
public function get_search_set()
{
return array(
$this->search,
$this->result,
$this->charset,
$this->sort_field,
$this->threading,
);
}
public function get_result()
{
return $this->result;
}
}
/**
* Wrker thread to run search jobs while maintaining a common context
*/
class rcube_imap_search_worker extends Worker
{
public $id;
public $options;
private $conn;
/**
* Default constructor
*/
public function __construct($id, $options)
{
$this->id = $id;
$this->options = $options;
}
/**
* Get a dedicated connection to the IMAP server
*/
public function get_imap()
{
// TODO: make this connection persistent for several jobs
#if ($this->conn)
# return $this->conn;
$conn = new rcube_imap_generic();
# $conn->setDebug(true, function($conn, $message){ trigger_error($message, E_USER_NOTICE); });
if ($this->options['user'] && $this->options['password']) {
$conn->connect($this->options['host'], $this->options['user'], $this->options['password'], $this->options);
}
if ($conn->error)
trigger_error($this->conn->error, E_USER_WARNING);
#$this->conn = $conn;
return $conn;
}
/**
* @override
*/
public function run()
{
}
/**
* Close IMAP connection
*/
public function close()
{
if ($this->conn) {
$this->conn->close();
}
}
}

@ -0,0 +1,211 @@
<?php
/*
+-----------------------------------------------------------------------+
| This file is part of the Roundcube Webmail client |
| Copyright (C) 2005-2011, The Roundcube Dev Team |
| Copyright (C) 2011, Kolab Systems AG |
| |
| Licensed under the GNU General Public License version 3 or |
| any later version with exceptions for skins & plugins. |
| See the README file for a full license statement. |
| |
| PURPOSE: |
| SORT/SEARCH/ESEARCH response handler |
+-----------------------------------------------------------------------+
| Author: Thomas Bruederli <roundcube@gmail.com> |
+-----------------------------------------------------------------------+
*/
/**
* Class holding a set of rcube_result_index instances that together form a
* result set of a multi-folder search
*
* @package Framework
* @subpackage Storage
*/
class rcube_result_multifolder
{
public $multi = true;
public $sets = array();
protected $meta = array();
protected $order = 'ASC';
/**
* Object constructor.
*/
public function __construct()
{
$this->meta = array('count' => 0);
}
/**
* Initializes object with SORT command response
*
* @param string $data IMAP response string
*/
public function add($result)
{
$this->sets[] = $result;
$this->meta['count'] += $result->count();
}
/**
* Checks the result from IMAP command
*
* @return bool True if the result is an error, False otherwise
*/
public function is_error()
{
return false;
}
/**
* Checks if the result is empty
*
* @return bool True if the result is empty, False otherwise
*/
public function is_empty()
{
return empty($this->sets) || $this->meta['count'] == 0;
}
/**
* Returns number of elements in the result
*
* @return int Number of elements
*/
public function count()
{
return $this->meta['count'];
}
/**
* Returns number of elements in the result.
* Alias for count() for compatibility with rcube_result_thread
*
* @return int Number of elements
*/
public function count_messages()
{
return $this->count();
}
/**
* Reverts order of elements in the result
*/
public function revert()
{
$this->order = $this->order == 'ASC' ? 'DESC' : 'ASC';
}
/**
* Check if the given message ID exists in the object
*
* @param int $msgid Message ID
* @param bool $get_index When enabled element's index will be returned.
* Elements are indexed starting with 0
* @return mixed False if message ID doesn't exist, True if exists or
* index of the element if $get_index=true
*/
public function exists($msgid, $get_index = false)
{
return false;
}
/**
* Filters data set. Removes elements listed in $ids list.
*
* @param array $ids List of IDs to remove.
* @param string $folder IMAP folder
*/
public function filter($ids = array(), $folder = null)
{
$this->meta['count'] = 0;
foreach ($this->sets as $set) {
if ($set->get_parameters('MAILBOX') == $folder) {
$set->filter($ids);
}
$this->meta['count'] += $set->count();
}
}
/**
* Filters data set. Removes elements not listed in $ids list.
*
* @param array $ids List of IDs to keep.
*/
public function intersect($ids = array())
{
// not implemented
}
/**
* Return all messages in the result.
*
* @return array List of message IDs
*/
public function get()
{
return array();
}
/**
* Return all messages in the result.
*
* @return array List of message IDs
*/
public function get_compressed()
{
return '';
}
/**
* Return result element at specified index
*
* @param int|string $index Element's index or "FIRST" or "LAST"
*
* @return int Element value
*/
public function get_element($index)
{
return null;
}
/**
* Returns response parameters, e.g. ESEARCH's MIN/MAX/COUNT/ALL/MODSEQ
* or internal data e.g. MAILBOX, ORDER
*
* @param string $param Parameter name
*
* @return array|string Response parameters or parameter value
*/
public function get_parameters($param=null)
{
return $params;
}
/**
* Returns length of internal data representation
*
* @return int Data length
*/
protected function length()
{
return $this->count();
}
}

@ -106,6 +106,9 @@ if (!empty($subject)) {
$search_str = trim($search_str);
$sort_column = rcmail_sort_column();
// TEMPORARY: search all folders
$mboxes = $RCMAIL->storage->list_folders_subscribed('', '*', 'mail');
// execute IMAP search
if ($search_str) {
$RCMAIL->storage->search($mbox, $search_str, $imap_charset, $sort_column);
@ -128,17 +131,20 @@ $result_h = $RCMAIL->storage->list_messages($mbox, 1, $sort_column, rcmail_sort_
$count = $RCMAIL->storage->count($mbox, $RCMAIL->storage->get_threading() ? 'THREADS' : 'ALL');
// Add 'folder' column to list
if ($multi_folder_search) {
if ($_SESSION['search'][1]->multi) {
$a_show_cols = $_SESSION['list_attrib']['columns'] ? $_SESSION['list_attrib']['columns'] : (array)$CONFIG['list_cols'];
if (!in_array($a_show_cols))
if (!in_array('folder', $a_show_cols))
$a_show_cols[] = 'folder';
// make message UIDs unique by appending the folder name
foreach ($result_h as $i => $header) {
$header->uid .= '-'.$header->folder;
$header->flags['skip_mbox_check'] = true;
if ($header->parent_uid)
$header->parent_uid .= '-'.$header->folder;
}
$OUTPUT->command('select_folder', '');
}
// Make sure we got the headers

Loading…
Cancel
Save