Branch 'kolab/integration/4.13.0' - 4 commits - akonadi/collectionsync.cpp akonadi/monitor_p.cpp akonadi/tests
Christian Mollekopf
mollekopf at kolabsys.com
Thu Oct 9 20:20:10 CEST 2014
akonadi/collectionsync.cpp | 981 ++++++++++++++++-------------------
akonadi/monitor_p.cpp | 7
akonadi/tests/collectionsynctest.cpp | 115 ++++
3 files changed, 580 insertions(+), 523 deletions(-)
New commits:
commit 41af4bd1dcbd1751b2a361d7b5a00ba0a82b6628
Author: Dan Vrátil <dvratil at redhat.com>
Date: Thu Sep 25 19:52:35 2014 +0200
CollectionSync: Optimized algorithm
This is combination of commits by Daniel Vratil.
* The weakest point of CollectionSync is in findMatchingLocalNode() method
that needs to traverse the LocalNode tree every time from up down to find the
LocalNode* that matches the given remote Collection, which is very inefficient,
especially in the case of initial synchronization.
* Calling parentCollection() is expensive (even though Collection is implicitly shared),
because of Entity::Entity(const Entity &other) which is very ineffective (but probably
not possible to fix without API changes), so we try to reduce the number of calls to
minimum by calling it once and then storing the parent collection in a variable.
* The new algorithm can sync 50 000 collections in about 2 minutes, which is 4 times
faster then what I ever was able to optimize the original algorithm.
The algorithm stores the \"tree\" structure of collections differently than the old
one: it stores list of collections for each parent in a QHash, and then starts
recursively querying Akonadi for collections, easilly comparing list of descendats
it got from Akonadi with the list in the QHash for the same parent RID chain.
* With non-hieararchical RIDs, we don't have the full parent chain available -
each collection only refers to RID of its parent, but we know nothing of the
parent's parent. To correctly match those, we need to compare only parts of
RemoteID chains.
* Instead of listing children for each Collection individually, we request all Collections
at once and then process them in one go. This means that the algorithm will require more
memory, but it will be much faster when dealing with large amounts of Collections.
* This adds two banchmarks for CollectionSync: one that does an initial sync
of 60 000 Collections, and one that does incremental sync against 60 000
Collections.
On my computer, the results are around 160 seconds for benchmarkInitialSync()
and around 8 seconds for benchmarkIncrementalSync().
The benchmarks are disabled by default, because the cleanup takes insane
amount of time (I blame poor implementation of DELETE command on the server).
diff --git a/akonadi/collectionsync.cpp b/akonadi/collectionsync.cpp
index 4e043dc..17aa83d 100644
--- a/akonadi/collectionsync.cpp
+++ b/akonadi/collectionsync.cpp
@@ -32,58 +32,101 @@
#include <kdebug.h>
#include <KLocalizedString>
#include <QtCore/QVariant>
+#include <QTime>
+#include <QHash>
using namespace Akonadi;
-struct RemoteNode;
+static const char CONTENTMIMETYPES[] = "CONTENTMIMETYPES";
-/**
- LocalNode is used to build a tree structure of all our locally existing collections.
-*/
-struct LocalNode
+static const char ROOTPARENTRID[] = "AKONADI_ROOT_COLLECTION";
+
+static const char PARENTCOLLECTIONRID[] = "ParentCollectionRid";
+static const char PARENTCOLLECTION[] = "ParentCollection";
+
+class RemoteId
{
- LocalNode(const Collection &col)
- : collection(col)
- , processed(false)
- {}
+public:
+ explicit RemoteId()
+ {
+ }
- ~LocalNode()
+ explicit inline RemoteId(const QStringList &ridChain):
+ ridChain(ridChain)
{
- qDeleteAll(childNodes);
- qDeleteAll(pendingRemoteNodes);
- }
-
- Collection collection;
- QList<LocalNode *> childNodes;
- QHash<QString, LocalNode *> childRidMap;
- QHash<QString, LocalNode *> childNameMap;
- /** When using hierarchical RIDs we attach a list of not yet processable remote nodes to
- the closest already existing local ancestor node. They will be re-evaluated once a new
- child node is added. */
- QList<RemoteNode *> pendingRemoteNodes;
- bool processed;
-};
+ }
-Q_DECLARE_METATYPE(LocalNode *)
-static const char LOCAL_NODE[] = "LocalNode";
+ explicit inline RemoteId(const QString &rid)
+ {
+ ridChain.append(rid);
+ }
-/**
- RemoteNode is used as a container for remote collections which typically don't have a UID set
- and thus cannot easily be compared or put into maps etc.
-*/
-struct RemoteNode
-{
- RemoteNode(const Collection &col)
- : collection(col)
- {}
+ inline ~RemoteId()
+ {
+ }
- Collection collection;
+ inline bool isAbsolute() const
+ {
+ return ridChain.last() == QString::fromAscii(ROOTPARENTRID);
+ }
+
+ inline bool isEmpty() const
+ {
+ return ridChain.isEmpty();
+ }
+
+ inline bool operator==(const RemoteId &other) const
+ {
+ return ridChain == other.ridChain;
+ }
+
+ QStringList ridChain;
+
+ static RemoteId rootRid;
};
-Q_DECLARE_METATYPE(RemoteNode *)
-static const char REMOTE_NODE[] = "RemoteNode";
+RemoteId RemoteId::rootRid = RemoteId(QStringList() << QString::fromAscii(ROOTPARENTRID));
-static const char CONTENTMIMETYPES[] = "CONTENTMIMETYPES";
+Q_DECLARE_METATYPE(RemoteId);
+
+
+uint qHash(const RemoteId &rid)
+{
+ uint hash = 0;
+ for (QStringList::ConstIterator iter = rid.ridChain.constBegin(),
+ end = rid.ridChain.constEnd();
+ iter != end;
+ ++iter)
+ {
+ hash += qHash(*iter);
+ }
+ return hash;
+}
+
+inline bool operator<(const RemoteId &r1, const RemoteId &r2)
+{
+ if (r1.ridChain.length() == r2.ridChain.length()) {
+ QStringList::ConstIterator it1 = r1.ridChain.constBegin(),
+ end1 = r1.ridChain.constEnd(),
+ it2 = r2.ridChain.constBegin();
+ while (it1 != end1) {
+ if ((*it1) == (*it2)) {
+ ++it1;
+ ++it2;
+ continue;
+ }
+ return (*it1) < (*it2);
+ }
+ } else {
+ return r1.ridChain.length() < r2.ridChain.length();
+ }
+ return false;
+}
+
+QDebug operator<<(QDebug s, const RemoteId &rid) {
+ s.nospace() << "RemoteId(" << rid.ridChain << ")";
+ return s;
+}
/**
* @internal
@@ -95,297 +138,232 @@ public:
: q(parent)
, pendingJobs(0)
, progress(0)
- , localRoot(0)
, currentTransaction(0)
- , knownLocalCollections(0)
, incremental(false)
, streaming(false)
, hierarchicalRIDs(false)
, localListDone(false)
, deliveryDone(false)
+ , akonadiRootCollection(Collection::root())
{
}
~Private()
{
- delete localRoot;
- qDeleteAll(rootRemoteNodes);
}
- /** Utility method to reset the node tree. */
- void resetNodeTree()
+ RemoteId remoteIdForCollection(const Collection &collection) const
{
- delete localRoot;
- localRoot = new LocalNode(Collection::root());
- localRoot->processed = true; // never try to delete that one
- if (currentTransaction) {
- // we are running the update transaction, initialize pending remote nodes
- localRoot->pendingRemoteNodes.swap(rootRemoteNodes);
+ if (collection == Collection::root()) {
+ return RemoteId::rootRid;
}
- localUidMap.clear();
- localRidMap.clear();
- localUidMap.insert(localRoot->collection.id(), localRoot);
if (!hierarchicalRIDs) {
- localRidMap.insert(QString(), localRoot);
+ return RemoteId(collection.remoteId());
}
- }
- /** Create a local node from the given local collection and integrate it into the local tree structure. */
- LocalNode *createLocalNode(const Collection &col)
- {
- LocalNode *node = new LocalNode(col);
- Q_ASSERT(!localUidMap.contains(col.id()));
- localUidMap.insert(node->collection.id(), node);
- if (!hierarchicalRIDs && !col.remoteId().isEmpty()) {
- localRidMap.insert(node->collection.remoteId(), node);
- }
-
- // add already existing children
- if (localPendingCollections.contains(col.id())) {
- QVector<Collection::Id> childIds = localPendingCollections.take(col.id());
- foreach (Collection::Id childId, childIds) {
- Q_ASSERT(localUidMap.contains(childId));
- LocalNode *childNode = localUidMap.value(childId);
- node->childNodes.append(childNode);
- if (!childNode->collection.remoteId().isEmpty()) {
- node->childRidMap.insert(childNode->collection.remoteId(), childNode);
- node->childNameMap.insert(childNode->collection.name(), childNode);
- }
- }
- }
- // set our parent and add ourselves as child
- if (localUidMap.contains(col.parentCollection().id())) {
- LocalNode *parentNode = localUidMap.value(col.parentCollection().id());
- parentNode->childNodes.append(node);
- if (!node->collection.remoteId().isEmpty()) {
- parentNode->childRidMap.insert(node->collection.remoteId(), node);
- parentNode->childNameMap.insert(node->collection.name(), node);
+ RemoteId rid;
+ Collection parent = collection;
+ while (parent.isValid() || !parent.remoteId().isEmpty()) {
+ QString prid = parent.remoteId();
+ if (prid.isEmpty() && parent.isValid()) {
+ prid = uidRidMap.value(parent.id());
+ }
+ if (prid.isEmpty()) {
+ break;
+ }
+ rid.ridChain.append(prid);
+ parent = parent.parentCollection();
+ if (parent == akonadiRootCollection) {
+ rid.ridChain.append(QString::fromAscii(ROOTPARENTRID));
+ break;
}
- } else {
- localPendingCollections[col.parentCollection().id()].append(col.id());
- }
-
- return node;
- }
-
- /** Same as createLocalNode() for remote collections. */
- void createRemoteNode(const Collection &col)
- {
- if (col.remoteId().isEmpty()) {
- kWarning() << "Collection '" << col.name() << "' does not have a remote identifier - skipping";
- return;
}
- RemoteNode *node = new RemoteNode(col);
- rootRemoteNodes.append(node);
+ return rid;
}
- /** Create local nodes as we receive the local listing from the Akonadi server. */
- void localCollectionsReceived(const Akonadi::Collection::List &localCols)
+ void addRemoteColection(const Collection &collection, bool removed = false)
{
- foreach (const Collection &c, localCols) {
- createLocalNode(c);
- knownLocalCollections++;
+ QHash<RemoteId, QList<Collection> > &map = (removed ? removedRemoteCollections : remoteCollections);
+ const Collection parentCollection = collection.parentCollection();
+ if (parentCollection.remoteId() == akonadiRootCollection.remoteId() || parentCollection.id() == akonadiRootCollection.id()) {
+ Collection c2(collection);
+ c2.setParentCollection(akonadiRootCollection);
+ map[RemoteId::rootRid].append(c2);
+ } else {
+ Q_ASSERT(!parentCollection.remoteId().isEmpty());
+ map[remoteIdForCollection(parentCollection)].append(collection);
}
}
- /** Once the local collection listing finished we can continue with the interesting stuff. */
- void localCollectionFetchResult(KJob *job)
+ /* Compares collections by remoteId and falls back to name comparision in case
+ * local collection does not have remoteId (which can happen in some cases)
+ */
+ bool matchLocalAndRemoteCollection(const Collection &local, const Collection &remote)
{
- if (job->error()) {
- return; // handled by the base class
- }
-
- // safety check: the local tree has to be connected
- if (!localPendingCollections.isEmpty()) {
- q->setError(Unknown);
- q->setErrorText(i18n("Inconsistent local collection tree detected."));
- q->emitResult();
- return;
+ if (!local.remoteId().isEmpty()) {
+ return local.remoteId() == remote.remoteId();
+ } else {
+ return local.name() == remote.name();
}
-
- localListDone = true;
- execute();
}
- /**
- * Find a child node with matching collection name.
- * @note This is used as a fallback if the resource lost the RID update somehow.
- * This can be used because the Akonadi server enforces unique child collection names inside the hierarchy
- */
- LocalNode *findLocalChildNodeByName(LocalNode *localParentNode, const QString &name) const
+ void localCollectionsReceived(const Akonadi::Collection::List &localCols)
{
- if (name.isEmpty()) { // shouldn't happen...
- return 0;
- }
-
- if (localParentNode == localRoot) { // possibly non-unique names on top-level
- return 0;
+ Q_FOREACH (const Akonadi::Collection &collection, localCols) {
+ const RemoteId parentRid = remoteIdForCollection(collection.parentCollection());
+ localCollections[parentRid] += collection;
}
- LocalNode *childNode = localParentNode->childNameMap.value(name, 0);
- // the restriction on empty RIDs can possibly removed, but for now I only understand the implication for this case
- if (childNode && childNode->collection.remoteId().isEmpty()) {
- return childNode;
- }
-
- return 0;
}
- /**
- Find the local node that matches the given remote collection, returns 0
- if that doesn't exist (yet).
- */
- LocalNode *findMatchingLocalNode(const Collection &collection) const
+ void processCollections(const RemoteId &parentRid)
{
- if (!hierarchicalRIDs) {
- if (localRidMap.contains(collection.remoteId())) {
- return localRidMap.value(collection.remoteId());
- }
- return 0;
- } else {
- if (collection.id() == Collection::root().id() || collection.remoteId() == Collection::root().remoteId()) {
- return localRoot;
- }
- LocalNode *localParent = 0;
- if (collection.parentCollection().id() < 0 && collection.parentCollection().remoteId().isEmpty()) {
- kWarning() << "Remote collection without valid parent found: " << collection;
- return 0;
+ QList<Collection> remoteChildren = remoteCollections.value(parentRid);
+ QList<Collection> removedChildren = removedRemoteCollections.value(parentRid);
+ QList<Collection> localChildren = localCollections.value(parentRid);
+
+ // Iterate over the list of local children of localParent
+ QList<Collection>::Iterator localIter, localEnd,
+ removedIter, removedEnd,
+ remoteIter, remoteEnd;
+
+ for (localIter = localChildren.begin(), localEnd = localChildren.end(); localIter != localEnd;)
+ {
+ const Collection localCollection = *localIter;
+ bool matched = false;
+ uidRidMap.insert(localIter->id(), localIter->remoteId());
+
+ // Try to map removed remote collections (from incremental sync) to local collections
+ for (removedIter = removedChildren.begin(), removedEnd = removedChildren.end(); removedIter != removedEnd;)
+ {
+ Collection removedCollection = *removedIter;
+
+ if (matchLocalAndRemoteCollection(localCollection, removedCollection)) {
+ matched = true;
+ localCollectionsToRemove.append(localCollection);
+ // Remove the matched removed collection from the list so that
+ // we don't have to iterate over it again next time.
+ removedIter = removedChildren.erase(removedIter);
+ removedEnd = removedChildren.end();
+ break;
+ } else {
+ // Keep looking
+ ++removedIter;
+ }
}
- if (collection.parentCollection().id() == Collection::root().id() || collection.parentCollection().remoteId() == Collection::root().remoteId()) {
- localParent = localRoot;
- } else {
- localParent = findMatchingLocalNode(collection.parentCollection());
+
+ if (matched) {
+ // Remove the matched local collection from the list, because we
+ // have already put it into localCollectionsToRemove
+ localIter = localChildren.erase(localIter);
+ localEnd = localChildren.end();
+ continue;
}
- if (localParent) {
- if (localParent->childRidMap.contains(collection.remoteId())) {
- return localParent->childRidMap.value(collection.remoteId());
- }
- // check if we have a local folder with a matching name and no RID, if so let's use that one
- // we would get an error if we don't do this anyway, as we'd try to create two sibling nodes with the same name
- if (LocalNode *recoveredLocalNode = findLocalChildNodeByName(localParent, collection.name())) {
- kDebug() << "Recovering collection with lost RID:" << collection << recoveredLocalNode->collection;
- return recoveredLocalNode;
+ // Try to find a matching collection in the list of remote children
+ for (remoteIter = remoteChildren.begin(), remoteEnd = remoteChildren.end(); !matched && remoteIter != remoteEnd;)
+ {
+ Collection remoteCollection = *remoteIter;
+
+ // Yay, we found a match!
+ if (matchLocalAndRemoteCollection(localCollection, remoteCollection)) {
+ matched = true;
+
+ // Check if the local and remote collections differ and thus if
+ // we need to update it
+ if (collectionNeedsUpdate(localCollection, remoteCollection)) {
+ // We need to store both local and remote collections, so that
+ // we can copy over attributes to be preserved
+ remoteCollectionsToUpdate.append(qMakePair(localCollection, remoteCollection));
+ } else {
+ // Collections are the same, no need to update anything
+ }
+
+ // Remove the matched remote collection from the list so that
+ // in the end we are left with list of collections that don't
+ // exist locally (i.e. new collections)
+ remoteIter = remoteChildren.erase(remoteIter);
+ remoteEnd = remoteChildren.end();
+ break;
+ } else {
+ // Keep looking
+ ++remoteIter;
}
}
- return 0;
- }
- }
- /**
- Find the local node that is the nearest ancestor of the given remote collection
- (when using hierarchical RIDs only, otherwise it's always the local root node).
- Never returns 0.
- */
- LocalNode *findBestLocalAncestor(const Collection &collection, bool *exactMatch = 0)
- {
- if (!hierarchicalRIDs) {
- return localRoot;
- }
- if (collection == Collection::root()) {
- if (exactMatch) {
- *exactMatch = true;
+ if (matched) {
+ // Remove the matched local collection from the list so that
+ // in the end we are left with list of collections that don't
+ // exist remotely (i.e. removed collections)
+ localIter = localChildren.erase(localIter);
+ localEnd = localChildren.end();
+ } else {
+ ++localIter;
}
- return localRoot;
- }
- if (collection.parentCollection().id() < 0 && collection.parentCollection().remoteId().isEmpty()) {
- kWarning() << "Remote collection without valid parent found: " << collection;
- return 0;
}
- bool parentIsExact = false;
- LocalNode *localParent = findBestLocalAncestor(collection.parentCollection(), &parentIsExact);
- if (!parentIsExact) {
- if (exactMatch) {
- *exactMatch = false;
- }
- return localParent;
+
+ if (!removedChildren.isEmpty()) {
+ removedRemoteCollections[parentRid] = removedChildren;
+ } else {
+ removedRemoteCollections.remove(parentRid);
}
- if (localParent->childRidMap.contains(collection.remoteId())) {
- if (exactMatch) {
- *exactMatch = true;
- }
- return localParent->childRidMap.value(collection.remoteId());
+
+ if (!remoteChildren.isEmpty()) {
+ remoteCollections[parentRid] = remoteChildren;
+ } else {
+ remoteCollections.remove(parentRid);
}
- if (exactMatch) {
- *exactMatch = false;
+
+ if (!localChildren.isEmpty()) {
+ localCollections[parentRid] = localChildren;
+ } else {
+ localCollections.remove(parentRid);
}
- return localParent;
}
- /**
- Checks if any of the remote nodes is not equal to the current local one. If so return true.
- */
- bool checkPendingRemoteNodes() const
+ void processLocalCollections(const RemoteId &parentRid, const Collection &parentCollection)
{
- if (rootRemoteNodes.size() != knownLocalCollections) {
- return true;
- }
+ const QList<Collection> originalChildren = localCollections.value(parentRid);
+ processCollections(parentRid);
- foreach (RemoteNode *remoteNode, rootRemoteNodes) {
- // every remote note should have a local node already
- LocalNode *localNode = findMatchingLocalNode(remoteNode->collection);
- if (localNode) {
- if (checkLocalCollection(localNode, remoteNode)) {
- return true;
- }
- } else {
- return true;
+ const QList<Collection> remoteChildren = remoteCollections.take(parentRid);
+ const QList<Collection> localChildren = localCollections.take(parentRid);
+
+ // At this point remoteChildren contains collections that don't exist locally yet
+ if (!remoteChildren.isEmpty()) {
+ Q_FOREACH (Collection c, remoteChildren) {
+ c.setParentCollection(parentCollection);
+ remoteCollectionsToCreate.append(c);
}
}
- return false;
+ // At this point localChildren contains collections that don't exist remotely anymore
+ if (!localChildren.isEmpty() && !incremental) {
+ localCollectionsToRemove += localChildren;
+ }
+
+ // Recurse into children
+ Q_FOREACH (const Collection &c, originalChildren) {
+ processLocalCollections(remoteIdForCollection(c), c);
+ }
}
- /**
- Checks the pending remote nodes attached to the given local root node
- to see if any of them can be processed by now. If not, they are moved to
- the closest ancestor available.
- */
- void processPendingRemoteNodes(LocalNode *_localRoot)
+ void localCollectionFetchResult(KJob *job)
{
- QList<RemoteNode *> pendingRemoteNodes(_localRoot->pendingRemoteNodes);
- _localRoot->pendingRemoteNodes.clear();
- QHash<LocalNode *, QList<RemoteNode *> > pendingCreations;
- foreach (RemoteNode *remoteNode, pendingRemoteNodes) {
- // step 1: see if we have a matching local node already
- LocalNode *localNode = findMatchingLocalNode(remoteNode->collection);
- if (localNode) {
- Q_ASSERT(!localNode->processed);
- updateLocalCollection(localNode, remoteNode);
- continue;
- }
- // step 2: check if we have the parent at least, then we can create it
- localNode = findMatchingLocalNode(remoteNode->collection.parentCollection());
- if (localNode) {
- pendingCreations[localNode].append(remoteNode);
- continue;
- }
- // step 3: find the best matching ancestor and enqueue it for later processing
- localNode = findBestLocalAncestor(remoteNode->collection);
- if (!localNode) {
- q->setError(Unknown);
- q->setErrorText(i18n("Remote collection without root-terminated ancestor chain provided, resource is broken."));
- q->emitResult();
- return;
- }
- localNode->pendingRemoteNodes.append(remoteNode);
+ if (job->error()) {
+ return; // handled by the base class
}
- // process the now possible collection creations
- for (QHash<LocalNode *, QList<RemoteNode *> >::const_iterator it = pendingCreations.constBegin();
- it != pendingCreations.constEnd(); ++it) {
- createLocalCollections(it.key(), it.value());
- }
+ processLocalCollections(RemoteId::rootRid, akonadiRootCollection);
+ localListDone = true;
+ execute();
}
/**
- Checks if the given localNode and remoteNode are different
+ Checks if the given localCollection and remoteCollection are different
*/
- bool checkLocalCollection(LocalNode *localNode, RemoteNode *remoteNode) const
+ bool collectionNeedsUpdate(const Collection &localCollection, const Collection &remoteCollection) const
{
- const Collection &localCollection = localNode->collection;
- const Collection &remoteCollection = remoteNode->collection;
-
if (!keepLocalChanges.contains(CONTENTMIMETYPES) && !remoteCollection.keepLocalChanges().contains(CONTENTMIMETYPES)) {
if (localCollection.contentMimeTypes().size() != remoteCollection.contentMimeTypes().size()) {
return true;
@@ -419,7 +397,7 @@ public:
}
// CollectionModifyJob adds the remote attributes to the local collection
- foreach (const Attribute *attr, remoteCollection.attributes()) {
+ Q_FOREACH (const Attribute *attr, remoteCollection.attributes()) {
const Attribute *localAttr = localCollection.attribute(attr->type());
if (localAttr && (keepLocalChanges.contains(attr->type()) || remoteCollection.keepLocalChanges().contains(CONTENTMIMETYPES))) {
continue;
@@ -433,180 +411,193 @@ public:
return false;
}
- /**
- Performs a local update for the given node pair.
- */
- void updateLocalCollection(LocalNode *localNode, RemoteNode *remoteNode)
- {
- Collection upd(remoteNode->collection);
- Q_ASSERT(!upd.remoteId().isEmpty());
- Q_ASSERT(currentTransaction);
- upd.setId(localNode->collection.id());
- if (keepLocalChanges.contains(CONTENTMIMETYPES) || remoteNode->collection.keepLocalChanges().contains(CONTENTMIMETYPES)) {
- upd.setContentMimeTypes(localNode->collection.contentMimeTypes());
- }
- foreach (Attribute *remoteAttr, upd.attributes()) {
- if ((keepLocalChanges.contains(remoteAttr->type()) || remoteNode->collection.keepLocalChanges().contains(remoteAttr->type()))&& localNode->collection.hasAttribute(remoteAttr->type())) {
- //We don't want to overwrite the attribute changes with the defaults provided by the resource.
- Attribute *localAttr = localNode->collection.attribute(remoteAttr->type());
- upd.removeAttribute(localAttr->type());
- upd.addAttribute(localAttr->clone());
- }
- }
- if (checkLocalCollection(localNode, remoteNode)) {
- // ### HACK to work around the implicit move attempts of CollectionModifyJob
- // which we do explicitly below
- Collection c(upd);
- c.setParentCollection(localNode->collection.parentCollection());
- ++pendingJobs;
- CollectionModifyJob *mod = new CollectionModifyJob(c, currentTransaction);
- connect(mod, SIGNAL(result(KJob*)), q, SLOT(updateLocalCollectionResult(KJob*)));
+ void createLocalCollections()
+ {
+ if (remoteCollectionsToCreate.isEmpty()) {
+ updateLocalCollections();
+ return;
}
- // detecting moves is only possible with global RIDs
- if (!hierarchicalRIDs) {
- LocalNode *oldParent = localUidMap.value(localNode->collection.parentCollection().id());
- LocalNode *newParent = findMatchingLocalNode(remoteNode->collection.parentCollection());
- // TODO: handle the newParent == 0 case correctly, ie. defer the move until the new
- // local parent has been created
- if (newParent && oldParent != newParent) {
+ Collection::List::Iterator iter, end;
+ for (iter = remoteCollectionsToCreate.begin(), end = remoteCollectionsToCreate.end(); iter != end;) {
+ const Collection col = *iter;
+ const Collection parentCollection = col.parentCollection();
+ // The parent already exists locally
+ if (parentCollection == akonadiRootCollection || parentCollection.id() > 0) {
++pendingJobs;
- CollectionMoveJob *move = new CollectionMoveJob(upd, newParent->collection, currentTransaction);
- connect(move, SIGNAL(result(KJob*)), q, SLOT(updateLocalCollectionResult(KJob*)));
+ CollectionCreateJob *create = new CollectionCreateJob(col, currentTransaction);
+ connect(create, SIGNAL(result(KJob*)),
+ q, SLOT(createLocalCollectionResult(KJob*)));
+
+ // Commit transaction after every 100 collections are created,
+ // otherwise it overlads database journal and things get veeery slow
+ if (pendingJobs % 100 == 0) {
+ currentTransaction->commit();
+ createTransaction();
+ }
+
+ iter = remoteCollectionsToCreate.erase(iter);
+ end = remoteCollectionsToCreate.end();
+ } else {
+ // Skip the collection, we'll try again once we create all the other
+ // collection we already have a parent for
+ ++iter;
}
}
-
- localNode->processed = true;
- delete remoteNode;
}
- void updateLocalCollectionResult(KJob *job)
+ void createLocalCollectionResult(KJob *job)
{
--pendingJobs;
if (job->error()) {
return; // handled by the base class
}
- if (qobject_cast<CollectionModifyJob *>(job)) {
- ++progress;
- }
- checkDone();
- }
- /**
- Creates local folders for the given local parent and remote nodes.
- @todo group CollectionCreateJobs into a single one once it supports that
- */
- void createLocalCollections(LocalNode *localParent, QList<RemoteNode *> remoteNodes)
- {
- foreach (RemoteNode *remoteNode, remoteNodes) {
- ++pendingJobs;
- Collection col(remoteNode->collection);
- Q_ASSERT(!col.remoteId().isEmpty());
- col.setParentCollection(localParent->collection);
- CollectionCreateJob *create = new CollectionCreateJob(col, currentTransaction);
- create->setProperty(LOCAL_NODE, QVariant::fromValue(localParent));
- create->setProperty(REMOTE_NODE, QVariant::fromValue(remoteNode));
- connect(create, SIGNAL(result(KJob*)), q, SLOT(createLocalCollectionResult(KJob*)));
-
- // Commit transaction after every 100 collections are created,
- // otherwise it overlads database journal and things get veeery slow
- if (pendingJobs % 50 == 0) {
- currentTransaction->commit();
- createTransaction();
+ q->setProcessedAmount(KJob::Bytes, ++progress);
+
+ const Collection newLocal = static_cast<CollectionCreateJob *>(job)->collection();
+ uidRidMap.insert(newLocal.id(), newLocal.remoteId());
+ const RemoteId newLocalRID = remoteIdForCollection(newLocal);
+
+ // See if there are any pending collections that this collection is parent of and
+ // update them if so
+ Collection::List::Iterator iter, end;
+ for (iter = remoteCollectionsToCreate.begin(), end = remoteCollectionsToCreate.end(); iter != end; ++iter) {
+ const Collection parentCollection = iter->parentCollection();
+ if (parentCollection != akonadiRootCollection && parentCollection.id() <= 0) {
+ const RemoteId remoteRID = remoteIdForCollection(*iter);
+ if (remoteRID.isAbsolute()) {
+ if (newLocalRID == remoteIdForCollection(*iter)) {
+ iter->setParentCollection(newLocal);
+ }
+ } else if (!hierarchicalRIDs) {
+ if (remoteRID.ridChain.startsWith(parentCollection.remoteId())) {
+ iter->setParentCollection(newLocal);
+ }
+ }
}
}
- }
- void createLocalCollectionResult(KJob *job)
- {
- --pendingJobs;
- if (job->error()) {
- return; // handled by the base class
+ // Enqueue all pending remote collections that are children of the just-created
+ // collection
+ QList<Collection> collectionsToCreate = remoteCollections.take(newLocalRID);
+ if (collectionsToCreate.isEmpty() && !hierarchicalRIDs) {
+ collectionsToCreate = remoteCollections.take(RemoteId(newLocal.remoteId()));
+ }
+ Q_FOREACH (Collection col, collectionsToCreate) {
+ col.setParentCollection(newLocal);
+ remoteCollectionsToCreate.append(col);
}
- const Collection newLocal = static_cast<CollectionCreateJob *>(job)->collection();
- LocalNode *localNode = createLocalNode(newLocal);
- localNode->processed = true;
+ // If there are still any collections to create left, try if we just created
+ // a parent for any of them
+ if (!remoteCollectionsToCreate.isEmpty()) {
+ createLocalCollections();
+ } else if (pendingJobs == 0) {
+ Q_ASSERT(remoteCollectionsToCreate.isEmpty());
+ if (!remoteCollections.isEmpty()) {
+ currentTransaction->rollback();
+ q->setError(Unknown);
+ q->setErrorText(i18n("Found unresolved orphan collections"));
+ q->emitResult();
+ return;
+ }
- LocalNode *localParent = job->property(LOCAL_NODE).value<LocalNode *>();
- Q_ASSERT(localParent->childNodes.contains(localNode));
- RemoteNode *remoteNode = job->property(REMOTE_NODE).value<RemoteNode *>();
- delete remoteNode;
- ++progress;
+ currentTransaction->commit();
+ createTransaction();
- processPendingRemoteNodes(localParent);
- if (!hierarchicalRIDs) {
- processPendingRemoteNodes(localRoot);
+ // Otherwise move to next task: updating existing collections
+ updateLocalCollections();
}
-
- checkDone();
+ /*
+ * else if (!remoteCollections.isEmpty()) {
+ currentTransaction->rollback();
+ q->setError(Unknown);
+ q->setErrorText(i18n("Incomplete collection tree"));
+ q->emitResult();
+ return;
+ }
+ */
}
/**
- Checks if the given local node has processed child nodes.
+ Performs a local update for the given node pair.
*/
- bool hasProcessedChildren(LocalNode *localNode) const
+ void updateLocalCollections()
{
- if (localNode->processed) {
- return true;
- }
- foreach (LocalNode *child, localNode->childNodes) {
- if (hasProcessedChildren(child)) {
- return true;
- }
+ if (remoteCollectionsToUpdate.isEmpty()) {
+ deleteLocalCollections();
+ return;
}
- return false;
- }
- /**
- Find all local nodes that are not marked as processed and have no children that
- are marked as processed.
- */
- Collection::List findUnprocessedLocalCollections(LocalNode *localNode) const
- {
- Collection::List rv;
- if (!localNode->processed) {
- if (hasProcessedChildren(localNode)) {
- kWarning() << "Found unprocessed local node with processed children, excluding from deletion";
- kWarning() << localNode->collection;
- return rv;
+ typedef QPair<Collection, Collection> CollectionPair;
+ Q_FOREACH (const CollectionPair &pair, remoteCollectionsToUpdate) {
+ const Collection local = pair.first;
+ const Collection remote = pair.second;
+ Collection upd(remote);
+
+ Q_ASSERT(!upd.remoteId().isEmpty());
+ Q_ASSERT(currentTransaction);
+ upd.setId(local.id());
+ if (keepLocalChanges.contains(CONTENTMIMETYPES) || remote.keepLocalChanges().contains(CONTENTMIMETYPES)) {
+ upd.setContentMimeTypes(local.contentMimeTypes());
}
- if (localNode->collection.remoteId().isEmpty()) {
- kWarning() << "Found unprocessed local node without remoteId, excluding from deletion";
- kWarning() << localNode->collection;
- return rv;
+ Q_FOREACH (Attribute *remoteAttr, upd.attributes()) {
+ if ((keepLocalChanges.contains(remoteAttr->type()) || remote.keepLocalChanges().contains(remoteAttr->type()))&& local.hasAttribute(remoteAttr->type())) {
+ //We don't want to overwrite the attribute changes with the defaults provided by the resource.
+ Attribute *localAttr = local.attribute(remoteAttr->type());
+ upd.removeAttribute(localAttr->type());
+ upd.addAttribute(localAttr->clone());
+ }
}
- rv.append(localNode->collection);
- return rv;
- }
- foreach (LocalNode *child, localNode->childNodes) {
- rv.append(findUnprocessedLocalCollections(child));
+ // ### HACK to work around the implicit move attempts of CollectionModifyJob
+ // which we do explicitly below
+ Collection c(upd);
+ c.setParentCollection(local.parentCollection());
+ ++pendingJobs;
+ CollectionModifyJob *mod = new CollectionModifyJob(c, currentTransaction);
+ connect(mod, SIGNAL(result(KJob*)), q, SLOT(updateLocalCollectionResult(KJob*)));
+
+ // detecting moves is only possible with global RIDs
+ if (!hierarchicalRIDs) {
+ if (remote.parentCollection().isValid() && remote.parentCollection().id() != local.parentCollection().id()) {
+ ++pendingJobs;
+ CollectionMoveJob *move = new CollectionMoveJob(upd, remote.parentCollection(), currentTransaction);
+ connect(move, SIGNAL(result(KJob*)), q, SLOT(updateLocalCollectionResult(KJob*)));
+ }
+ }
}
- return rv;
}
- /**
- Deletes unprocessed local nodes, in non-incremental mode.
- */
- void deleteUnprocessedLocalNodes()
+ void updateLocalCollectionResult(KJob *job)
{
- if (incremental) {
- return;
+ --pendingJobs;
+ if (job->error()) {
+ return; // handled by the base class
+ }
+ if (qobject_cast<CollectionModifyJob *>(job)) {
+ q->setProcessedAmount(KJob::Bytes, ++progress);
+ }
+
+ // All updates are done, time to move on to next task: deletion
+ if (pendingJobs == 0) {
+ currentTransaction->commit();
+ createTransaction();
+
+ deleteLocalCollections();
}
- const Collection::List cols = findUnprocessedLocalCollections(localRoot);
- deleteLocalCollections(cols);
}
- /**
- Deletes the given collection list.
- @todo optimize delete job to support batch operations
- */
- void deleteLocalCollections(const Collection::List &cols)
+ void deleteLocalCollections()
{
- q->setTotalAmount(KJob::Bytes, q->totalAmount(KJob::Bytes) + cols.size());
- foreach (const Collection &col, cols) {
+ if (localCollectionsToRemove.isEmpty()) {
+ done();
+ return;
+ }
+
+ Q_FOREACH (const Collection &col, localCollectionsToRemove) {
Q_ASSERT(!col.remoteId().isEmpty()); // empty RID -> stuff we haven't even written to the remote side yet
++pendingJobs;
@@ -625,9 +616,28 @@ public:
void deleteLocalCollectionsResult(KJob *)
{
--pendingJobs;
+ q->setProcessedAmount(KJob::Bytes, ++progress);
- ++progress;
- checkDone();
+ if (pendingJobs == 0) {
+ currentTransaction->commit();
+ currentTransaction = 0;
+
+ done();
+ }
+ }
+
+ void done()
+ {
+ if (currentTransaction) {
+ currentTransaction->commit();
+ currentTransaction = 0;
+ }
+
+ if (!remoteCollections.isEmpty()) {
+ q->setError(Unknown);
+ q->setErrorText(i18n("Found unresolved orphan collections"));
+ }
+ q->emitResult();
}
void createTransaction()
@@ -638,26 +648,6 @@ public:
q, SLOT(transactionSequenceResult(KJob*)));
}
- /**
- Check update necessity.
- */
- void checkUpdateNecessity()
- {
- bool updateNeeded = checkPendingRemoteNodes();
- if (!updateNeeded) {
- // We can end right now
- q->emitResult();
- return;
- }
-
- // Since there are differences with the remote collections we need to sync. Start a transaction here.
- Q_ASSERT(!currentTransaction);
- createTransaction();
-
- // Now that a transaction is started we need to fetch local collections again and do the update
- q->doStart();
- }
-
/** After the transaction has finished report we're done as well. */
void transactionSequenceResult(KJob *job)
{
@@ -668,6 +658,7 @@ public:
// If this was the last transaction, then finish, otherwise there's
// a new transaction in the queue already
if (job == currentTransaction) {
+ currentTransaction = 0;
q->emitResult();
}
}
@@ -677,84 +668,39 @@ public:
*/
void execute()
{
- kDebug() << Q_FUNC_INFO << "localListDone: " << localListDone << " deliveryDone: " << deliveryDone;
- if (!localListDone || !deliveryDone) {
+ kDebug() << "localListDone: " << localListDone << " deliveryDone: " << deliveryDone;
+ if (!localListDone && !deliveryDone) {
return;
}
- // If a transaction is not started yet we are still checking if the update is
- // actually needed.
- if (!currentTransaction) {
- checkUpdateNecessity();
+ if (!localListDone && deliveryDone) {
+ Job *parent = (currentTransaction ? static_cast<Job *>(currentTransaction) : static_cast<Job *>(q));
+ CollectionFetchJob *job = new CollectionFetchJob(akonadiRootCollection, CollectionFetchJob::Recursive, parent);
+ job->fetchScope().setResource(resourceId);
+ job->fetchScope().setIncludeUnsubscribed(true);
+ job->fetchScope().setAncestorRetrieval(CollectionFetchScope::All);
+ q->connect(job, SIGNAL(collectionsReceived(Akonadi::Collection::List)),
+ q, SLOT(localCollectionsReceived(Akonadi::Collection::List)));
+ q->connect(job, SIGNAL(result(KJob*)),
+ q, SLOT(localCollectionFetchResult(KJob*)));
return;
}
- // Since the transaction is already running we need to execute the update.
- processPendingRemoteNodes(localRoot);
-
- if (!incremental && deliveryDone) {
- deleteUnprocessedLocalNodes();
- }
-
- if (!hierarchicalRIDs) {
- deleteLocalCollections(removedRemoteCollections);
- } else {
- Collection::List localCols;
- foreach (const Collection &c, removedRemoteCollections) {
- LocalNode *node = findMatchingLocalNode(c);
- if (node) {
- localCols.append(node->collection);
- }
+ // If a transaction is not started yet, it means we just finished local listing
+ if (!currentTransaction) {
+ // There's nothing to do after local listing -> we are done!
+ if (remoteCollectionsToCreate.isEmpty() && remoteCollectionsToUpdate.isEmpty() && localCollectionsToRemove.isEmpty()) {
+ kDebug() << "Nothing to do";
+ q->emitResult();
+ return;
}
- deleteLocalCollections(localCols);
+ // Ok, there's some work to do, so create a transaction we can use
+ createTransaction();
}
- removedRemoteCollections.clear();
- checkDone();
- }
-
- /**
- Finds pending remote nodes, which at the end of the day should be an empty set.
- */
- QList<RemoteNode *> findPendingRemoteNodes(LocalNode *localNode)
- {
- QList<RemoteNode *> rv;
- rv.append(localNode->pendingRemoteNodes);
- foreach (LocalNode *child, localNode->childNodes) {
- rv.append(findPendingRemoteNodes(child));
- }
- return rv;
+ createLocalCollections();
}
- /**
- Are we there yet??
- @todo progress reporting
- */
- void checkDone()
- {
- q->setProcessedAmount(KJob::Bytes, progress);
-
- // still running jobs or not fully delivered local/remote state
- if (!deliveryDone || pendingJobs > 0 || !localListDone) {
- return;
- }
-
- // safety check: there must be no pending remote nodes anymore
- QList<RemoteNode *> orphans = findPendingRemoteNodes(localRoot);
- if (!orphans.isEmpty()) {
- q->setError(Unknown);
- q->setErrorText(i18n("Found unresolved orphan collections"));
- foreach (RemoteNode *orphan, orphans) {
- kDebug() << "found orphan collection:" << orphan->collection;
- }
- q->emitResult();
- return;
- }
-
- kDebug() << Q_FUNC_INFO << "q->commit()";
- Q_ASSERT(currentTransaction);
- currentTransaction->commit();
- }
CollectionSync *q;
@@ -763,23 +709,7 @@ public:
int pendingJobs;
int progress;
- LocalNode *localRoot;
TransactionSequence *currentTransaction;
- QHash<Collection::Id, LocalNode *> localUidMap;
- QHash<QString, LocalNode *> localRidMap;
-
- // temporary during build-up of the local node tree, must be empty afterwards
- QHash<Collection::Id, QVector<Collection::Id> > localPendingCollections;
-
- // removed remote collections in incremental mode
- Collection::List removedRemoteCollections;
-
- // used to store the list of remote nodes passed by the user
- QList<RemoteNode *> rootRemoteNodes;
-
- // keep track of the total number of local collections that are known
- // only used during the preliminary check to see if updating is needed
- int knownLocalCollections;
bool incremental;
bool streaming;
@@ -790,6 +720,22 @@ public:
// List of parts where local changes should not be overwritten
QSet<QByteArray> keepLocalChanges;
+
+ QHash<RemoteId /* parent */, QList<Collection> /* children */ > removedRemoteCollections;
+ QHash<RemoteId /* parent */, QList<Collection> /* children */ > remoteCollections;
+ QHash<RemoteId /* parent */, QList<Collection> /* children */ > localCollections;
+
+ Collection::List localCollectionsToRemove;
+ Collection::List remoteCollectionsToCreate;
+ QList<QPair<Collection /* local */, Collection /* remote */> > remoteCollectionsToUpdate;
+ QHash<Collection::Id, QString> uidRidMap;
+
+ // HACK: To workaround Collection copy constructor being very expensive, we
+ // store the Collection::root() collection in a variable here for faster
+ // access
+ Collection akonadiRootCollection;
+
+
};
CollectionSync::CollectionSync(const QString &resourceId, QObject *parent)
@@ -808,8 +754,8 @@ CollectionSync::~CollectionSync()
void CollectionSync::setRemoteCollections(const Collection::List &remoteCollections)
{
setTotalAmount(KJob::Bytes, totalAmount(KJob::Bytes) + remoteCollections.count());
- foreach (const Collection &c, remoteCollections) {
- d->createRemoteNode(c);
+ Q_FOREACH (const Collection &c, remoteCollections) {
+ d->addRemoteColection(c);
}
if (!d->streaming) {
@@ -822,10 +768,12 @@ void CollectionSync::setRemoteCollections(const Collection::List &changedCollect
{
setTotalAmount(KJob::Bytes, totalAmount(KJob::Bytes) + changedCollections.count());
d->incremental = true;
- foreach (const Collection &c, changedCollections) {
- d->createRemoteNode(c);
+ Q_FOREACH (const Collection &c, changedCollections) {
+ d->addRemoteColection(c);
+ }
+ Q_FOREACH (const Collection &c, removedCollections) {
+ d->addRemoteColection(c, true);
}
- d->removedRemoteCollections += removedCollections;
if (!d->streaming) {
d->deliveryDone = true;
@@ -835,16 +783,6 @@ void CollectionSync::setRemoteCollections(const Collection::List &changedCollect
void CollectionSync::doStart()
{
- d->resetNodeTree();
- d->knownLocalCollections = 0;
- Job *parent = (d->currentTransaction ? static_cast<Job *>(d->currentTransaction) : static_cast<Job *>(this));
- CollectionFetchJob *job = new CollectionFetchJob(Collection::root(), CollectionFetchJob::Recursive, parent);
- job->fetchScope().setResource(d->resourceId);
- job->fetchScope().setIncludeUnsubscribed(true);
- job->fetchScope().setAncestorRetrieval(CollectionFetchScope::Parent);
- connect(job, SIGNAL(collectionsReceived(Akonadi::Collection::List)),
- SLOT(localCollectionsReceived(Akonadi::Collection::List)));
- connect(job, SIGNAL(result(KJob*)), SLOT(localCollectionFetchResult(KJob*)));
}
void CollectionSync::setStreamingEnabled(bool streaming)
@@ -876,3 +814,4 @@ void CollectionSync::setKeepLocalChanges(const QSet<QByteArray> &parts)
}
#include "moc_collectionsync_p.cpp"
+
diff --git a/akonadi/tests/collectionsynctest.cpp b/akonadi/tests/collectionsynctest.cpp
index 9da0b28..d389a3b 100644
--- a/akonadi/tests/collectionsynctest.cpp
+++ b/akonadi/tests/collectionsynctest.cpp
@@ -33,8 +33,10 @@
#include <QtCore/QObject>
#include <QSignalSpy>
+#include <QEventLoop>
#include <qtest_akonadi.h>
+#include <resourceselectjob_p.h>
using namespace Akonadi;
@@ -68,6 +70,85 @@ class CollectionSyncTest : public QObject
QTest::newRow( "akonadi_knut_resource_2 hierarchical RID" ) << true << "akonadi_knut_resource_2";
}
+ Collection createCollection(const QString &name, const QString &remoteId, const Collection &parent)
+ {
+ Collection c;
+ c.setName(name);
+ c.setRemoteId(remoteId);
+ c.setParentCollection(parent);
+ c.setResource("akonadi_knut_resource_0");
+ c.setContentMimeTypes(QStringList() << Collection::mimeType());
+ return c;
+ }
+
+ Collection::List prepareBenchmark()
+ {
+ Collection::List collections = fetchCollections(QLatin1String("akonadi_knut_resource_0"));
+
+ ResourceSelectJob *resJob = new ResourceSelectJob("akonadi_knut_resource_0");
+ Q_ASSERT(resJob->exec());
+
+ Collection root;
+ Q_FOREACH (const Collection &col, collections) {
+ if (col.parentCollection() == Collection::root()) {
+ root = col;
+ break;
+ }
+ }
+ Q_ASSERT(root.isValid());
+
+ // we must build on top of existing collections, because only resource is
+ // allowed to create top-level collection
+ Collection::List baseCollections;
+ for (int i = 0; i < 20; ++i) {
+ baseCollections << createCollection(QString::fromLatin1("Base Col %1").arg(i), QString::fromLatin1("/baseCol%1").arg(i), root);
+ }
+ collections += baseCollections;
+
+ const Collection shared = createCollection(QLatin1String("Shared collections"), QLatin1String("/shared"), root);
+ baseCollections << shared;
+ collections << shared;
+ for (int i = 0; i < 10000; ++i) {
+ const Collection col = createCollection(QString::fromLatin1("Shared Col %1").arg(i), QString::fromLatin1("/shared%1").arg(i), shared);
+ collections << col;
+ for (int j = 0; j < 6; ++j) {
+ collections << createCollection(QString::fromLatin1("Shared Subcol %1-%2").arg(i).arg(j), QString::fromLatin1("/shared%1-%2").arg(i).arg(j), col);
+ }
+ }
+ return collections;
+ }
+
+ CollectionSync *prepareBenchmarkSyncer(const Collection::List &collections)
+ {
+ CollectionSync *syncer = new CollectionSync("akonadi_knut_resource_0");
+ connect(syncer, SIGNAL(percent(KJob*,ulong)), this, SLOT(syncBenchmarkProgress(KJob*,ulong)));
+ syncer->setHierarchicalRemoteIds(false);
+ syncer->setRemoteCollections(collections);
+ return syncer;
+ }
+
+ void cleanupBenchmark(const Collection::List &collections)
+ {
+ Collection::List baseCols;
+ Q_FOREACH(const Collection &col, collections) {
+ if (col.remoteId().startsWith(QLatin1String("/baseCol")) || col.remoteId() == QLatin1String("/shared")) {
+ baseCols << col;
+ }
+ }
+ Q_FOREACH (const Collection &col, baseCols) {
+ CollectionDeleteJob *del = new CollectionDeleteJob(col);
+ AKVERIFYEXEC(del);
+ }
+ }
+
+
+ public Q_SLOTS:
+ void syncBenchmarkProgress(KJob *job, ulong percent)
+ {
+ Q_UNUSED(job);
+ qDebug() << "CollectionSync progress:" << percent << "%";
+ }
+
private Q_SLOTS:
void initTestCase()
{
@@ -299,6 +380,40 @@ class CollectionSyncTest : public QObject
}
}
}
+
+// Disabled by default, because they take ~15 minutes to complete
+#if 0
+ void benchmarkInitialSync()
+ {
+ const Collection::List collections = prepareBenchmark();
+
+ CollectionSync *syncer = prepareBenchmarkSyncer(collections);
+
+ QBENCHMARK_ONCE {
+ AKVERIFYEXEC(syncer);
+ }
+
+ cleanupBenchmark(collections);
+ }
+
+ void benchmarkIncrementalSync()
+ {
+ const Collection::List collections = prepareBenchmark();
+
+ // First populate Akonadi with Collections
+ CollectionSync *syncer = prepareBenchmarkSyncer(collections);
+ AKVERIFYEXEC(syncer);
+
+ // Now create a new syncer to benchmark the incremental sync
+ syncer = prepareBenchmarkSyncer(collections);
+
+ QBENCHMARK_ONCE {
+ AKVERIFYEXEC(syncer);
+ }
+
+ cleanupBenchmark(collections);
+ }
+#endif
};
QTEST_AKONADIMAIN( CollectionSyncTest, NoGUI )
commit d58504c979eac1a10a944771f7ca4ee040cbd0d6
Author: Dan Vrátil <dvratil at redhat.com>
Date: Thu Sep 25 20:01:11 2014 +0200
Decrease the amount of CollectionCreateJob queued within one TransactionSequence
Ony my reasonably-avarage setup, comitting each 50 INSERTs yields slightly better
performance than every 100 INSERTs, most notably on SQLITE backend.
diff --git a/akonadi/collectionsync.cpp b/akonadi/collectionsync.cpp
index ea70fc4..4e043dc 100644
--- a/akonadi/collectionsync.cpp
+++ b/akonadi/collectionsync.cpp
@@ -511,7 +511,7 @@ public:
// Commit transaction after every 100 collections are created,
// otherwise it overlads database journal and things get veeery slow
- if (pendingJobs % 100 == 0) {
+ if (pendingJobs % 50 == 0) {
currentTransaction->commit();
createTransaction();
}
commit bbef85a25dd9f66a733bdb9f47f93845692aaca0
Author: Christian Mollekopf <chrigi_1 at fastmail.fm>
Date: Thu Oct 9 12:07:03 2014 +0200
Monitor: Avoid fetching already removed collections.
This avoids a shitload of fetchjobs that fail anyways (If a bunch of collections is removed).
diff --git a/akonadi/monitor_p.cpp b/akonadi/monitor_p.cpp
index 7fa898c..58500db 100644
--- a/akonadi/monitor_p.cpp
+++ b/akonadi/monitor_p.cpp
@@ -432,6 +432,11 @@ bool MonitorPrivate::ensureDataAvailable(const NotificationMessageV3 &msg)
return true;
}
+ if (msg.operation() == NotificationMessageV2::Remove && msg.type() == NotificationMessageV2::Collections) {
+ //For collection removals the collection is gone anyways, so we can't fetch it. Rid will be set later on instead.
+ return true;
+ }
+
bool allCached = true;
if (fetchCollection) {
if (!collectionCache->ensureCached(msg.parentCollection(), mCollectionFetchScope)) {
commit 6a36631760b4149458a8d14265449d5491c29a13
Author: Christian Mollekopf <chrigi_1 at fastmail.fm>
Date: Thu Oct 9 12:02:36 2014 +0200
Revert "Monitor: Avoid fetching removed collections."
This reverts commit ec5e53286a8394b1b81e5d38b49ba95d54539137.
diff --git a/akonadi/monitor_p.cpp b/akonadi/monitor_p.cpp
index 0554734..7fa898c 100644
--- a/akonadi/monitor_p.cpp
+++ b/akonadi/monitor_p.cpp
@@ -432,11 +432,6 @@ bool MonitorPrivate::ensureDataAvailable(const NotificationMessageV3 &msg)
return true;
}
- if (msg.operation() == NotificationMessageV2::Remove) {
- // For both collection and item removals we don't require the parent collection, that may be gone already anyways. So let's avoid a failing fetchjob
- return true;
- }
-
bool allCached = true;
if (fetchCollection) {
if (!collectionCache->ensureCached(msg.parentCollection(), mCollectionFetchScope)) {
@@ -446,6 +441,9 @@ bool MonitorPrivate::ensureDataAvailable(const NotificationMessageV3 &msg)
allCached = false;
}
}
+ if (msg.operation() == NotificationMessageV2::Remove) {
+ return allCached; // the actual object is gone already, nothing to fetch there
+ }
if (msg.type() == NotificationMessageV2::Items && !mItemFetchScope.isEmpty()) {
ItemFetchScope scope(mItemFetchScope);
More information about the commits
mailing list