Branch 'dev/kolab-cache-refactoring' - 5 commits - plugins/kolab_auth plugins/libkolab
Thomas Brüderli
bruederli at kolabsys.com
Mon Oct 7 17:34:55 CEST 2013
plugins/kolab_auth/kolab_auth.php | 41 +++++++++++++++
plugins/libkolab/config.inc.php.dist | 6 ++
plugins/libkolab/lib/kolab_storage_cache.php | 69 +++++++++++++++++++++++++-
plugins/libkolab/lib/kolab_storage_folder.php | 51 ++++++++++++-------
4 files changed, 149 insertions(+), 18 deletions(-)
New commits:
commit 6b89e36c4e00c134efa6c552428936e1787beacd
Merge: 66e33b9 7161951
Author: Thomas Bruederli <bruederli at kolabsys.com>
Date: Mon Oct 7 17:34:17 2013 +0200
Merge remote-tracking branch 'origin/libkolab-cache-bypass' into dev/kolab-cache-refactoring
diff --cc plugins/libkolab/lib/kolab_storage_cache.php
index 6a3e0d6,1165631..3600e74
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@@ -128,61 -96,57 +128,66 @@@ class kolab_storage_cach
// increase time limit
@set_time_limit($this->max_sync_lock_time);
- // lock synchronization for this folder or wait if locked
- $this->_sync_lock();
+ // read cached folder metadata
+ $this->_read_folder_data();
- // disable messages cache if configured to do so
- $this->bypass(true);
+ // check cache status hash first ($this->metadata is set in _read_folder_data())
+ if ($this->metadata['ctag'] != $this->folder->get_ctag()) {
- // synchronize IMAP mailbox cache
- $this->imap->folder_sync($this->folder->name);
+ // lock synchronization for this folder or wait if locked
+ $this->_sync_lock();
- // compare IMAP index with object cache index
- $imap_index = $this->imap->index($this->folder->name);
- $this->index = $imap_index->get();
++ // disable messages cache if configured to do so
++ $this->bypass(true);
+
- // determine objects to fetch or to invalidate
- if ($this->ready) {
- // read cache index
- $sql_result = $this->db->query(
- "SELECT msguid, uid FROM kolab_cache WHERE resource=? AND type<>?",
- $this->resource_uri,
- 'lock'
- );
+ // synchronize IMAP mailbox cache
+ $this->imap->folder_sync($this->folder->name);
- $old_index = array();
- while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
- $old_index[] = $sql_arr['msguid'];
- $this->uid2msg[$sql_arr['uid']] = $sql_arr['msguid'];
- }
+ // compare IMAP index with object cache index
+ $imap_index = $this->imap->index($this->folder->name);
+ $this->index = $imap_index->get();
- // fetch new objects from imap
- foreach (array_diff($this->index, $old_index) as $msguid) {
- if ($object = $this->folder->read_object($msguid, '*')) {
- $this->_extended_insert($msguid, $object);
- }
- }
- $this->_extended_insert(0, null);
-
- // delete invalid entries from local DB
- $del_index = array_diff($old_index, $this->index);
- if (!empty($del_index)) {
- $quoted_ids = join(',', array_map(array($this->db, 'quote'), $del_index));
- $this->db->query(
- "DELETE FROM kolab_cache WHERE resource=? AND msguid IN ($quoted_ids)",
- $this->resource_uri
+ // determine objects to fetch or to invalidate
+ if ($this->ready) {
+ // read cache index
+ $sql_result = $this->db->query(
+ "SELECT msguid, uid FROM $this->cache_table WHERE folder_id=?",
+ $this->folder_id
);
+
+ $old_index = array();
+ while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+ $old_index[] = $sql_arr['msguid'];
+ $this->uid2msg[$sql_arr['uid']] = $sql_arr['msguid'];
+ }
+
+ // fetch new objects from imap
+ foreach (array_diff($this->index, $old_index) as $msguid) {
+ if ($object = $this->folder->read_object($msguid, '*')) {
+ $this->_extended_insert($msguid, $object);
+ }
+ }
+ $this->_extended_insert(0, null);
+
+ // delete invalid entries from local DB
+ $del_index = array_diff($old_index, $this->index);
+ if (!empty($del_index)) {
+ $quoted_ids = join(',', array_map(array($this->db, 'quote'), $del_index));
+ $this->db->query(
+ "DELETE FROM $this->cache_table WHERE folder_id=? AND msguid IN ($quoted_ids)",
+ $this->folder_id
+ );
+ }
+
+ // update ctag value (will be written to database in _sync_unlock())
+ $this->metadata['ctag'] = $this->folder->get_ctag();
}
- }
- $this->bypass(false);
++ $this->bypass(false);
+
- // remove lock
- $this->_sync_unlock();
+ // remove lock
+ $this->_sync_unlock();
+ }
$this->synched = time();
}
@@@ -814,15 -794,58 +825,71 @@@
}
/**
+ * Getter for protected member variables
+ */
+ public function __get($name)
+ {
+ if ($name == 'folder_id') {
+ $this->_read_folder_data();
+ }
+
+ return $this->$name;
+ }
+
++ /**
+ * Bypass Roundcube messages cache.
+ * Roundcube cache duplicates information already stored in kolab_cache.
+ *
+ * @param bool $disable True disables, False enables messages cache
+ */
+ public function bypass($disable = false)
+ {
+ // if kolab cache is disabled do nothing
+ if (!$this->enabled) {
+ return;
+ }
+
+ static $messages_cache, $cache_bypass;
+
+ if ($messages_cache === null) {
+ $rcmail = rcube::get_instance();
+ $messages_cache = (bool) $rcmail->config->get('messages_cache');
+ $cache_bypass = (int) $rcmail->config->get('kolab_messages_cache_bypass');
+ }
+
+ if ($messages_cache) {
+ // handle recurrent (multilevel) bypass() calls
+ if ($disable) {
+ $this->cache_bypassed += 1;
+ if ($this->cache_bypassed > 1) {
+ return;
+ }
+ }
+ else {
+ $this->cache_bypassed -= 1;
+ if ($this->cache_bypassed > 0) {
+ return;
+ }
+ }
+
+ switch ($cache_bypass) {
+ case 2:
+ // Disable messages cache completely
+ $this->imap->set_messages_caching(!$disable);
+ break;
+
+ case 1:
+ // We'll disable messages cache, but keep index cache.
+ // Default mode is both (MODE_INDEX | MODE_MESSAGE)
+ $mode = rcube_imap_cache::MODE_INDEX;
+
+ if (!$disable) {
+ $mode |= rcube_imap_cache::MODE_MESSAGE;
+ }
+
+ $this->imap->set_messages_caching(true, $mode);
+ }
+ }
+ }
++
}
commit 71619510c49633527c66de8ca2d98c06a71c4c35
Author: Aleksander Machniak <machniak at kolabsys.com>
Date: Mon Oct 7 15:57:55 2013 +0200
Improve bypass() method so it works "recursively"
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 0e30dfa..1165631 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -99,16 +99,16 @@ class kolab_storage_cache
// lock synchronization for this folder or wait if locked
$this->_sync_lock();
- // synchronize IMAP mailbox cache
+ // disable messages cache if configured to do so
$this->bypass(true);
+
+ // synchronize IMAP mailbox cache
$this->imap->folder_sync($this->folder->name);
// compare IMAP index with object cache index
$imap_index = $this->imap->index($this->folder->name);
$this->index = $imap_index->get();
- $this->bypass(false);
-
// determine objects to fetch or to invalidate
if ($this->ready) {
// read cache index
@@ -143,6 +143,8 @@ class kolab_storage_cache
}
}
+ $this->bypass(false);
+
// remove lock
$this->_sync_unlock();
@@ -813,11 +815,25 @@ class kolab_storage_cache
}
if ($messages_cache) {
+ // handle recurrent (multilevel) bypass() calls
+ if ($disable) {
+ $this->cache_bypassed += 1;
+ if ($this->cache_bypassed > 1) {
+ return;
+ }
+ }
+ else {
+ $this->cache_bypassed -= 1;
+ if ($this->cache_bypassed > 0) {
+ return;
+ }
+ }
+
switch ($cache_bypass) {
case 2:
// Disable messages cache completely
$this->imap->set_messages_caching(!$disable);
- return;
+ break;
case 1:
// We'll disable messages cache, but keep index cache.
commit 9d174daf9f98e8e60cddf77cf1416e42470ba77b
Author: Aleksander Machniak <machniak at kolabsys.com>
Date: Mon Oct 7 15:05:34 2013 +0200
Add option kolab_messages_cache_bypass
diff --git a/plugins/libkolab/config.inc.php.dist b/plugins/libkolab/config.inc.php.dist
index 6260f52..0c612a3 100644
--- a/plugins/libkolab/config.inc.php.dist
+++ b/plugins/libkolab/config.inc.php.dist
@@ -24,3 +24,9 @@ $rcmail_config['kolab_custom_display_names'] = false;
// See http://pear.php.net/manual/en/package.http.http-request2.config.php
// for list of supported configuration options (array keys)
$rcmail_config['kolab_http_request'] = array();
+
+// When kolab_cache is enabled Roundcube's messages cache will be redundant
+// when working on kolab folders. Here we can:
+// 2 - bypass messages/indexes cache completely
+// 1 - bypass only messages, but use index cache
+$rcmail_config['kolab_messages_cache_bypass'] = 0;
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 3b1d857..0e30dfa 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -804,21 +804,32 @@ class kolab_storage_cache
return;
}
- if ($this->messages_cache === null) {
+ static $messages_cache, $cache_bypass;
+
+ if ($messages_cache === null) {
$rcmail = rcube::get_instance();
- $this->messages_cache = (bool) $rcmail->config->get('messages_cache');
+ $messages_cache = (bool) $rcmail->config->get('messages_cache');
+ $cache_bypass = (int) $rcmail->config->get('kolab_messages_cache_bypass');
}
- if ($this->messages_cache) {
- // we'll disable messages cache, but keep index cache
- // default mode is both (MODE_INDEX | MODE_MESSAGE)
- $mode = rcube_imap_cache::MODE_INDEX;
+ if ($messages_cache) {
+ switch ($cache_bypass) {
+ case 2:
+ // Disable messages cache completely
+ $this->imap->set_messages_caching(!$disable);
+ return;
- if (!$disable) {
- $mode |= rcube_imap_cache::MODE_MESSAGE;
- }
+ case 1:
+ // We'll disable messages cache, but keep index cache.
+ // Default mode is both (MODE_INDEX | MODE_MESSAGE)
+ $mode = rcube_imap_cache::MODE_INDEX;
- $this->imap->set_messages_caching(true, $mode);
+ if (!$disable) {
+ $mode |= rcube_imap_cache::MODE_MESSAGE;
+ }
+
+ $this->imap->set_messages_caching(true, $mode);
+ }
}
}
}
commit 16d9509a5d3795dce8aa2b04b22b69ec1e03879b
Author: Aleksander Machniak <machniak at kolabsys.com>
Date: Mon Oct 7 09:56:06 2013 +0200
Improved performance of kolab cache by bypassing Roundcube messages cache (Request #1740)
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index a23fbaa..3b1d857 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -100,12 +100,15 @@ class kolab_storage_cache
$this->_sync_lock();
// synchronize IMAP mailbox cache
+ $this->bypass(true);
$this->imap->folder_sync($this->folder->name);
// compare IMAP index with object cache index
$imap_index = $this->imap->index($this->folder->name);
$this->index = $imap_index->get();
+ $this->bypass(false);
+
// determine objects to fetch or to invalidate
if ($this->ready) {
// read cache index
@@ -523,8 +526,14 @@ class kolab_storage_cache
if (!$type)
$type = $this->folder->type;
+ $this->bypass(true);
+
$results = array();
- foreach ((array)$this->imap->fetch_headers($this->folder->name, $index, false) as $msguid => $headers) {
+ $headers = $this->imap->fetch_headers($this->folder->name, $index, false);
+
+ $this->bypass(false);
+
+ foreach ((array)$headers as $msguid => $headers) {
$object_type = kolab_format::mime2object_type($headers->others['x-kolab-type']);
// check object type header and abort on mismatch
@@ -782,4 +791,34 @@ class kolab_storage_cache
return $this->uid2msg[$uid];
}
+ /**
+ * Bypass Roundcube messages cache.
+ * Roundcube cache duplicates information already stored in kolab_cache.
+ *
+ * @param bool $disable True disables, False enables messages cache
+ */
+ public function bypass($disable = false)
+ {
+ // if kolab cache is disabled do nothing
+ if (!$this->enabled) {
+ return;
+ }
+
+ if ($this->messages_cache === null) {
+ $rcmail = rcube::get_instance();
+ $this->messages_cache = (bool) $rcmail->config->get('messages_cache');
+ }
+
+ if ($this->messages_cache) {
+ // we'll disable messages cache, but keep index cache
+ // default mode is both (MODE_INDEX | MODE_MESSAGE)
+ $mode = rcube_imap_cache::MODE_INDEX;
+
+ if (!$disable) {
+ $mode |= rcube_imap_cache::MODE_MESSAGE;
+ }
+
+ $this->imap->set_messages_caching(true, $mode);
+ }
+ }
}
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index e81153d..5be0f5f 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -340,7 +340,6 @@ class kolab_storage_folder
return $subscribed ? kolab_storage::folder_subscribe($this->name) : kolab_storage::folder_unsubscribe($this->name);
}
-
/**
* Get number of objects stored in this folder
*
@@ -502,6 +501,7 @@ class kolab_storage_folder
* @param string The IMAP message UID to fetch
* @param string The object type expected (use wildcard '*' to accept all types)
* @param string The folder name where the message is stored
+ *
* @return mixed Hash array representing the Kolab object, a kolab_format instance or false if not found
*/
public function read_object($msguid, $type = null, $folder = null)
@@ -511,31 +511,31 @@ class kolab_storage_folder
$this->imap->set_folder($folder);
- $headers = $this->imap->get_message_headers($msguid);
- $message = null;
+ $this->cache->bypass(true);
+ $message = new rcube_message($msguid);
+ $this->cache->bypass(false);
// Message doesn't exist?
- if (empty($headers)) {
+ if (empty($message->headers)) {
return false;
}
// extract the X-Kolab-Type header from the XML attachment part if missing
- if (empty($headers->others['x-kolab-type'])) {
- $message = new rcube_message($msguid);
+ if (empty($message->headers->others['x-kolab-type'])) {
foreach ((array)$message->attachments as $part) {
if (strpos($part->mimetype, kolab_format::KTYPE_PREFIX) === 0) {
- $headers->others['x-kolab-type'] = $part->mimetype;
+ $message->headers->others['x-kolab-type'] = $part->mimetype;
break;
}
}
}
// fix buggy messages stating the X-Kolab-Type header twice
- else if (is_array($headers->others['x-kolab-type'])) {
- $headers->others['x-kolab-type'] = reset($headers->others['x-kolab-type']);
+ else if (is_array($message->headers->others['x-kolab-type'])) {
+ $message->headers->others['x-kolab-type'] = reset($message->headers->others['x-kolab-type']);
}
// no object type header found: abort
- if (empty($headers->others['x-kolab-type'])) {
+ if (empty($message->headers->others['x-kolab-type'])) {
rcube::raise_error(array(
'code' => 600,
'type' => 'php',
@@ -546,14 +546,13 @@ class kolab_storage_folder
return false;
}
- $object_type = kolab_format::mime2object_type($headers->others['x-kolab-type']);
- $content_type = kolab_format::KTYPE_PREFIX . $object_type;
+ $object_type = kolab_format::mime2object_type($message->headers->others['x-kolab-type']);
+ $content_type = kolab_format::KTYPE_PREFIX . $object_type;
// check object type header and abort on mismatch
if ($type != '*' && $object_type != $type)
return false;
- if (!$message) $message = new rcube_message($msguid);
$attachments = array();
// get XML part
@@ -595,7 +594,7 @@ class kolab_storage_folder
}
// check kolab format version
- $format_version = $headers->others['x-kolab-mime-version'];
+ $format_version = $message->headers->others['x-kolab-mime-version'];
if (empty($format_version)) {
list($xmltype, $subtype) = explode('.', $object_type);
$xmlhead = substr($xml, 0, 512);
@@ -749,7 +748,9 @@ class kolab_storage_folder
// delete old message
if ($result && !empty($object['_msguid']) && !empty($object['_mailbox'])) {
+ $this->cache->bypass(true);
$this->imap->delete_message($object['_msguid'], $object['_mailbox']);
+ $this->cache->bypass(false);
$this->cache->set($object['_msguid'], false, $object['_mailbox']);
}
@@ -844,6 +845,8 @@ class kolab_storage_folder
$msguid = is_array($object) ? $object['_msguid'] : $this->cache->uid2msguid($object);
$success = false;
+ $this->cache->bypass(true);
+
if ($msguid && $expunge) {
$success = $this->imap->delete_message($msguid, $this->name);
}
@@ -851,6 +854,8 @@ class kolab_storage_folder
$success = $this->imap->set_flag($msguid, 'DELETED', $this->name);
}
+ $this->cache->bypass(false);
+
if ($success) {
$this->cache->set($msguid, false);
}
@@ -865,7 +870,11 @@ class kolab_storage_folder
public function delete_all()
{
$this->cache->purge();
- return $this->imap->clear_folder($this->name);
+ $this->cache->bypass(true);
+ $result = $this->imap->clear_folder($this->name);
+ $this->cache->bypass(false);
+
+ return $result;
}
@@ -878,7 +887,11 @@ class kolab_storage_folder
public function undelete($uid)
{
if ($msguid = $this->cache->uid2msguid($uid, true)) {
- if ($this->imap->set_flag($msguid, 'UNDELETED', $this->name)) {
+ $this->cache->bypass(true);
+ $result = $this->imap->set_flag($msguid, 'UNDELETED', $this->name);
+ $this->cache->bypass(false);
+
+ if ($result) {
return $msguid;
}
}
@@ -897,7 +910,11 @@ class kolab_storage_folder
public function move($uid, $target_folder)
{
if ($msguid = $this->cache->uid2msguid($uid)) {
- if ($this->imap->move_message($msguid, $target_folder, $this->name)) {
+ $this->cache->bypass(true);
+ $result = $this->imap->move_message($msguid, $target_folder, $this->name);
+ $this->cache->bypass(false);
+
+ if ($result) {
$this->cache->move($msguid, $uid, $target_folder);
return true;
}
commit c97615aeefeaed5feaa68034583949df7f9a96e2
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Fri Oct 4 12:59:38 2013 +0200
Log failed logins (always)
diff --git a/plugins/kolab_auth/kolab_auth.php b/plugins/kolab_auth/kolab_auth.php
index e440218..cf5818f 100644
--- a/plugins/kolab_auth/kolab_auth.php
+++ b/plugins/kolab_auth/kolab_auth.php
@@ -339,6 +339,16 @@ class kolab_auth extends rcube_plugin
$ldap = self::ldap();
if (!$ldap || !$ldap->ready) {
$args['abort'] = true;
+ $message = sprintf(
+ 'Login failure for user %s from %s in session %s (error %s)',
+ $user,
+ rcube_utils::remote_ip(),
+ session_id(),
+ "LDAP not ready"
+ );
+
+ rcube::write_log('userlogins', $message);
+
return $args;
}
@@ -347,6 +357,16 @@ class kolab_auth extends rcube_plugin
if (empty($record)) {
$args['abort'] = true;
+ $message = sprintf(
+ 'Login failure for user %s from %s in session %s (error %s)',
+ $user,
+ rcube_utils::remote_ip(),
+ session_id(),
+ "No user record found"
+ );
+
+ rcube::write_log('userlogins', $message);
+
return $args;
}
@@ -380,6 +400,16 @@ class kolab_auth extends rcube_plugin
if (!$result) {
$args['abort'] = true;
+ $message = sprintf(
+ 'Login failure for user %s from %s in session %s (error %s)',
+ $user,
+ rcube_utils::remote_ip(),
+ session_id(),
+ "Unable to bind with '" . $record['dn'] . "'"
+ );
+
+ rcube::write_log('userlogins', $message);
+
return $args;
}
@@ -421,6 +451,17 @@ class kolab_auth extends rcube_plugin
if (empty($record)) {
$args['abort'] = true;
+ $message = sprintf(
+ 'Login failure for user %s (as user %s) from %s in session %s (error %s)',
+ $user,
+ $loginas,
+ rcube_utils::remote_ip(),
+ session_id(),
+ "No user record found for '" . $loginas . "'"
+ );
+
+ rcube::write_log('userlogins', $message);
+
return $args;
}
More information about the commits
mailing list