6 commits - kolab-cli.py pykolab/auth pykolab/cli pykolab/imap
Jeroen van Meeuwen
vanmeeuwen at kolabsys.com
Mon Nov 17 09:52:25 CET 2014
kolab-cli.py | 2 -
pykolab/auth/__init__.py | 4 +-
pykolab/auth/ldap/__init__.py | 7 ++-
pykolab/cli/cmd_delete_mailbox.py | 16 +++++---
pykolab/cli/cmd_sync.py | 76 ++++++++++++++++++++++++++++++++++----
pykolab/imap/__init__.py | 72 +++++++++++++++++++++++++-----------
6 files changed, 138 insertions(+), 39 deletions(-)
New commits:
commit 3bdd15ce70a1054e66746230e7b6613a447747b1
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Mon Nov 17 09:51:51 2014 +0100
Multi-thread the creation of mailboxes for existing LDAP entries
diff --git a/pykolab/cli/cmd_sync.py b/pykolab/cli/cmd_sync.py
index e4f4bdf..3e2191a 100644
--- a/pykolab/cli/cmd_sync.py
+++ b/pykolab/cli/cmd_sync.py
@@ -19,27 +19,50 @@
import commands
+from distutils import version
+import multiprocessing
+
+import sys
import time
+
import pykolab
+from pykolab import utils
from pykolab.auth import Auth
+from pykolab.imap import IMAP
from pykolab.translate import _
log = pykolab.getLogger('pykolab.cli')
conf = pykolab.getConf()
+imap = None
+pool = None
+
def __init__():
commands.register('sync', execute, description="Synchronize Kolab Users with IMAP.")
def cli_options():
my_option_group = conf.add_cli_parser_option_group(_("CLI Options"))
- my_option_group.add_option( '--resync',
- dest = "resync",
- action = "store_true",
- default = False,
- help = _("Resync from the beginning"))
+ my_option_group.add_option(
+ '--threads',
+ dest = "threads",
+ action = "store",
+ default = 20,
+ type = int,
+ help = _("Synchronize LDAP and IMAP")
+ )
+
+ my_option_group.add_option(
+ '--resync',
+ dest = "resync",
+ action = "store_true",
+ default = False,
+ help = _("Resync from the beginning")
+ )
def execute(*args, **kw):
+ global imap, pool
+
auth = Auth()
log.debug(_("Listing domains..."), level=5)
start_time = time.time()
@@ -53,17 +76,56 @@ def execute(*args, **kw):
level=8
)
- all_folders = []
+ if version.StrictVersion(sys.version[:3]) >= version.StrictVersion("2.7"):
+ pool = multiprocessing.Pool(conf.threads, worker_process, (), 1)
+ else:
+ pool = multiprocessing.Pool(conf.threads, worker_process, ())
for primary_domain in list(set(domains.values())):
log.debug(_("Running for domain %s") % (primary_domain), level=8)
auth = Auth(primary_domain)
auth.connect(primary_domain)
start_time = time.time()
- auth.synchronize(mode='_paged_search')
+ auth.synchronize(mode='_paged_search', callback=queue_add)
end_time = time.time()
log.info(_("Synchronizing users for %s took %d seconds")
% (primary_domain, (end_time-start_time))
)
+ while not pool._taskqueue.empty():
+ time.sleep(1)
+
+def queue_add(*args, **kw):
+ global pool
+ for dn, entry in kw['entry']:
+ entry['dn'] = dn
+ pool.apply_async(_synchronize, (), dict(**entry))
+
+def worker_process(*args, **kw):
+ pass
+
+def _synchronize(*args, **kw):
+ log.info(_("Worker process %s handling %s") % (multiprocessing.current_process().name, kw['dn']))
+
+ entry = utils.normalize(entry)
+
+ if not entry.has_key('mail'):
+ return
+
+ if not 'kolabinetorgperson' in entry['objectclass']:
+ return
+
+ imap = IMAP()
+ imap.connect()
+
+ if not imap.user_mailbox_exists(entry['mail']):
+ if entry.has_key('mailhost'):
+ server = entry['mailhost']
+ else:
+ server = None
+
+ imap.user_mailbox_create(entry['mail'], server=server)
+
+ imap.disconnect()
+
commit 26bfe007f6bffe506a4d8b9e28cb6410e957abe8
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Mon Nov 17 09:48:43 2014 +0100
Ensure the correct autocreate_folders settings are pulled if they are available
Suppress logging too many "Waiting for Cyrus IMAP Murder to settle" messages
If the backend server is known, connect to it directly to create additional folders
diff --git a/pykolab/imap/__init__.py b/pykolab/imap/__init__.py
index 8bd74a9..343ff10 100644
--- a/pykolab/imap/__init__.py
+++ b/pykolab/imap/__init__.py
@@ -110,6 +110,8 @@ class IMAP(object):
if uri == None:
if conf.has_section(domain) and conf.has_option(domain, 'imap_uri'):
uri = conf.get(domain, 'imap_uri')
+ else:
+ self.domain = None
scheme = None
hostname = None
@@ -477,12 +479,21 @@ class IMAP(object):
self.connect()
created = False
+ last_log = time.time()
while not created:
created = self.has_folder(folder_name)
if not created:
- log.info(_("Waiting for the Cyrus IMAP Murder to settle..."))
+ if time.time() - last_log > 5:
+ log.info(_("Waiting for the Cyrus IMAP Murder to settle..."))
+ last_log = time.time()
+
time.sleep(0.5)
+ _additional_folders = None
+
+ if not hasattr(self, 'domain'):
+ self.domain == None
+
if not self.domain == None:
if conf.has_option(self.domain, "autocreate_folders"):
_additional_folders = conf.get_raw(
@@ -490,28 +501,28 @@ class IMAP(object):
"autocreate_folders"
)
- elif conf.has_option('kolab', "autocreate_folders"):
+ if _additional_folders == None:
+ if conf.has_option('kolab', "autocreate_folders"):
_additional_folders = conf.get_raw(
'kolab',
"autocreate_folders"
)
- else:
- _additional_folders = {}
-
- additional_folders = conf.plugins.exec_hook(
- "create_user_folders",
- kw={
- 'folder': folder_name,
- 'additional_folders': _additional_folders
- }
- )
- if not additional_folders == None:
- self.user_mailbox_create_additional_folders(
- mailbox_base_name,
- additional_folders
- )
+ additional_folders = conf.plugins.exec_hook(
+ "create_user_folders",
+ kw={
+ 'folder': folder_name,
+ 'additional_folders': _additional_folders
+ }
+ )
+
+ if not additional_folders == None:
+ self.user_mailbox_create_additional_folders(
+ mailbox_base_name,
+ additional_folders
+ )
+ if not self.domain == None:
if conf.has_option(self.domain, "sieve_mgmt"):
sieve_mgmt_enabled = conf.get(self.domain, 'sieve_mgmt')
if utils.true_or_false(sieve_mgmt_enabled):
@@ -535,17 +546,26 @@ class IMAP(object):
admin_login = conf.get(backend, 'admin_login')
admin_password = conf.get(backend, 'admin_password')
+ if backend == "cyrus-imap" and hasattr(self.imap, 'murder') and self.imap.murder:
+ server = self.user_mailbox_server(folder)
+ else:
+ server = None
+
success = False
+ last_log = time.time()
while not success:
try:
self.disconnect()
- self.connect(login=False)
+ self.connect(login=False, server=server)
self.login_plain(admin_login, admin_password, folder)
(personal, other, shared) = self.namespaces()
success = True
except Exception, errmsg:
- log.debug(_("Waiting for the Cyrus murder to settle... %r") % (errmsg))
+ if time.time() - last_log > 5:
+ log.debug(_("Waiting for the Cyrus murder to settle... %r") % (errmsg))
+ last_log = time.time()
+
if conf.debuglevel > 8:
import traceback
traceback.print_exc()
@@ -561,7 +581,16 @@ class IMAP(object):
folder_name = "%s%s" % (personal, folder_name)
try:
- self.create_folder(folder_name)
+ created = False
+ last_log = time.time()
+ while not created:
+ created = self.has_folder(folder_name)
+ if not created:
+ if time.time() - last_log > 5:
+ log.info(_("Waiting for the Cyrus IMAP Murder to settle..."))
+ last_log = time.time()
+
+ time.sleep(0.5)
except:
log.warning(_("Mailbox already exists: %s") % (folder_name))
if conf.debuglevel > 8:
commit b5483a0653ecbdbcbd29a06deb75b47700a09fdd
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Mon Nov 17 09:46:18 2014 +0100
Allow multiple arguments to be specified
diff --git a/pykolab/cli/cmd_delete_mailbox.py b/pykolab/cli/cmd_delete_mailbox.py
index ea64abc..9a7e1b1 100644
--- a/pykolab/cli/cmd_delete_mailbox.py
+++ b/pykolab/cli/cmd_delete_mailbox.py
@@ -40,9 +40,7 @@ def execute(*args, **kw):
Delete mailbox
"""
- try:
- delete_folder = conf.cli_args.pop(0)
- except IndexError, e:
+ if len(conf.cli_args) < 1:
print >> sys.stderr, _("No mailbox specified")
sys.exit(1)
@@ -50,10 +48,18 @@ def execute(*args, **kw):
imap.connect()
- delete_folders = imap.list_folders(delete_folder)
+ delete_folders = []
+ while len(conf.cli_args) > 0:
+ folder = conf.cli_args.pop(0)
+ folders = imap.list_folders(folder)
+
+ if len(folders) < 1:
+ print >> sys.stderr, _("No such folder(s): %s") % (folder)
+
+ delete_folders.extend(folders)
if len(delete_folders) == 0:
- print >> sys.stderr, _("No such folder(s)")
+ print >> sys.stderr, _("No folders to delete.")
sys.exit(1)
for delete_folder in delete_folders:
commit 33789dbe2b3c95d291218953e88e9d7c82ba05c3
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Mon Nov 17 09:45:56 2014 +0100
Prepend the local search path rather than appending it
diff --git a/kolab-cli.py b/kolab-cli.py
index b4a98e3..760c902 100755
--- a/kolab-cli.py
+++ b/kolab-cli.py
@@ -23,7 +23,7 @@ import os
import sys
# For development purposes
-sys.path.extend(['.', '..'])
+sys.path = [ '.' ] + sys.path
from pykolab.translate import _
from pykolab.cli import Cli
commit 61b00207a248079c22215fa1fcaac4ddb37ab751
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Mon Nov 17 09:45:28 2014 +0100
Allow a callback to be specified for issuing a synchronize() job
diff --git a/pykolab/auth/__init__.py b/pykolab/auth/__init__.py
index ac75118..86479ce 100644
--- a/pykolab/auth/__init__.py
+++ b/pykolab/auth/__init__.py
@@ -256,8 +256,8 @@ class Auth(pykolab.base.Base):
return self.domains
- def synchronize(self, mode=0):
- self._auth.synchronize(mode=mode)
+ def synchronize(self, mode=0, callback=None):
+ self._auth.synchronize(mode=mode, callback=callback)
def domain_default_quota(self, domain):
return self._auth._domain_default_quota(domain)
diff --git a/pykolab/auth/ldap/__init__.py b/pykolab/auth/ldap/__init__.py
index c240405..c71c397 100644
--- a/pykolab/auth/ldap/__init__.py
+++ b/pykolab/auth/ldap/__init__.py
@@ -976,7 +976,7 @@ class LDAP(pykolab.base.Base):
except:
log.error(_("Could not update dn %r:\n%r") % (dn, modlist))
- def synchronize(self, mode=0):
+ def synchronize(self, mode=0, callback=None):
"""
Synchronize with LDAP
"""
@@ -1017,6 +1017,9 @@ class LDAP(pykolab.base.Base):
log.debug(_("Synchronization is searching against base DN: %s") % (base_dn), level=8)
+ if callback == None:
+ callback = self._synchronize_callback
+
try:
self._search(
base_dn,
@@ -1028,7 +1031,7 @@ class LDAP(pykolab.base.Base):
'modifytimestamp'
],
override_search=override_search,
- callback=self._synchronize_callback,
+ callback=callback,
)
except Exception, errmsg:
log.error("Exception occurred: %r" % (errmsg))
commit 05393f0c9c697363769e5223e6c6bbddd294ded6
Author: Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen at kolabsys.com>
Date: Sun Nov 16 15:38:48 2014 +0100
Avoid splitting a log message over two lines
diff --git a/pykolab/imap/__init__.py b/pykolab/imap/__init__.py
index 7ee19a6..8bd74a9 100644
--- a/pykolab/imap/__init__.py
+++ b/pykolab/imap/__init__.py
@@ -218,8 +218,7 @@ class IMAP(object):
return True
except:
log.error(
- _("Could not create folder %r") + \
- _(" on server %r") % (
+ _("Could not create folder %r on server %r") % (
folder_path,
server
)
More information about the commits
mailing list