diff --git a/src/common/preparedsqlquerymanager.h b/src/common/preparedsqlquerymanager.h index a5afa0fe66d93..17ba2c186f69f 100644 --- a/src/common/preparedsqlquerymanager.h +++ b/src/common/preparedsqlquerymanager.h @@ -108,6 +108,8 @@ class OCSYNC_EXPORT PreparedSqlQueryManager GetE2EeLockedFoldersQuery, DeleteE2EeLockedFolderQuery, ListAllTopLevelE2eeFoldersStatusLessThanQuery, + UpdateTagListQuery, + GetTagListQuery, PreparedQueryCount }; diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 895ad115c4d81..66f67487a1838 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -49,7 +49,7 @@ Q_LOGGING_CATEGORY(lcDb, "nextcloud.sync.database", QtInfoMsg) #define GET_FILE_RECORD_QUERY \ "SELECT path, inode, modtime, type, md5, fileid, remotePerm, filesize," \ " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum, e2eMangledName, isE2eEncrypted, " \ - " lock, lockOwnerDisplayName, lockOwnerId, lockType, lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe" \ + " lock, lockOwnerDisplayName, lockOwnerId, lockType, lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe, tagList" \ " FROM metadata" \ " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" @@ -77,6 +77,7 @@ static void fillFileRecordFromGetQuery(SyncJournalFileRecord &rec, SqlQuery &que rec._isShared = query.intValue(19) > 0; rec._lastShareStateFetchedTimestamp = query.int64Value(20); rec._sharedByMe = query.intValue(21) > 0; + rec._tagList = query.baValue(22); } static QByteArray defaultJournalMode(const QString &dbPath) @@ -832,6 +833,8 @@ bool SyncJournalDb::updateMetadataTableStructure() } commitInternal(QStringLiteral("update database structure: add basePath index")); + addColumn(QStringLiteral("tagList"), QStringLiteral("TEXT")); + return re; } @@ -958,7 +961,8 @@ Result SyncJournalDb::setFileRecord(const SyncJournalFileRecord & << "lock editor:" << record._lockstate._lockEditorApp << "sharedByMe:" << record._sharedByMe << "isShared:" << record._isShared - << "lastShareStateFetchedTimestamp:" << record._lastShareStateFetchedTimestamp; + << "lastShareStateFetchedTimestamp:" << record._lastShareStateFetchedTimestamp + << "tagList:"< SyncJournalDb::setFileRecord(const SyncJournalFileRecord & const auto query = _queryManager.get(PreparedSqlQueryManager::SetFileRecordQuery, QByteArrayLiteral("INSERT OR REPLACE INTO metadata " "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, " "contentChecksum, contentChecksumTypeId, e2eMangledName, isE2eEncrypted, lock, lockType, lockOwnerDisplayName, lockOwnerId, " - "lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe) " - "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28);"), + "lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe, tagList) " + "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29);"), _db); if (!query) { return query->error(); @@ -1019,6 +1023,7 @@ Result SyncJournalDb::setFileRecord(const SyncJournalFileRecord & query->bindValue(26, record._isShared); query->bindValue(27, record._lastShareStateFetchedTimestamp); query->bindValue(28, record._sharedByMe); + query->bindValue(29, record._tagList); if (!query->exec()) { return query->error(); @@ -1497,6 +1502,58 @@ int SyncJournalDb::getFileRecordCount() return -1; } +bool SyncJournalDb::updateMetadataTagList(const QString &filename,const QByteArray* tagList) +{ + QMutexLocker locker(&_mutex); + + qCInfo(lcDb) << "Updating file tag list " << filename; + + if (!checkConnect()) { + qCWarning(lcDb) << "Failed to connect database."; + return false; + } + + const qint64 phash = getPHash(filename.toUtf8()); + + const auto query = _queryManager.get(PreparedSqlQueryManager::UpdateTagListQuery, + QByteArrayLiteral("UPDATE metadata" + " SET tagList = ?2" + " WHERE phash = ?1;"), + _db); + if (!query) { + return false; + } + query->bindValue(1, phash); + query->bindValue(2, *tagList); + return query->exec(); +} + +QByteArray SyncJournalDb::tagList(const QString &file) +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) + return QByteArray(); + + const qint64 phash = getPHash(file.toUtf8()); + + const auto query = _queryManager.get(PreparedSqlQueryManager::GetTagListQuery, + QByteArrayLiteral("SELECT tagList FROM metadata" + " WHERE phash = ?1;"), + _db); + if (!query) { + return QByteArray(); + } + + query->bindValue(1, phash); + if (!query->exec())return QByteArray(); + + auto next = query->next(); + if (!next.ok || !next.hasData)return QByteArray(); + + QByteArray result = query->baValue(0); + return result; +} + bool SyncJournalDb::updateFileRecordChecksum(const QString &filename, const QByteArray &contentChecksum, const QByteArray &contentChecksumType) diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index ae4824a41766c..c24cd8d777f5c 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -246,6 +246,16 @@ class OCSYNC_EXPORT SyncJournalDb : public QObject void setDataFingerprint(const QByteArray &dataFingerprint); QByteArray dataFingerprint(); + /*! + * \brief Update the taglist in the metadata table. + */ + bool updateMetadataTagList(const QString &file, + const QByteArray* tagList); + + /*! + * \brief Returns the taglist of the queried file. + */ + QByteArray tagList(const QString &file); // Conflict record functions diff --git a/src/common/syncjournalfilerecord.cpp b/src/common/syncjournalfilerecord.cpp index 6ca9f43af4397..566312f27dec7 100644 --- a/src/common/syncjournalfilerecord.cpp +++ b/src/common/syncjournalfilerecord.cpp @@ -51,6 +51,7 @@ bool operator==(const SyncJournalFileRecord &lhs, && lhs._fileSize == rhs._fileSize && lhs._remotePerm == rhs._remotePerm && lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles - && lhs._checksumHeader == rhs._checksumHeader; + && lhs._checksumHeader == rhs._checksumHeader + && lhs._tagList == rhs._tagList; } } diff --git a/src/common/syncjournalfilerecord.h b/src/common/syncjournalfilerecord.h index 7270fac137962..cf8f4b8967921 100644 --- a/src/common/syncjournalfilerecord.h +++ b/src/common/syncjournalfilerecord.h @@ -87,6 +87,7 @@ class OCSYNC_EXPORT SyncJournalFileRecord bool _isShared = false; qint64 _lastShareStateFetchedTimestamp = 0; bool _sharedByMe = false; + QByteArray _tagList; }; QDebug& operator<<(QDebug &stream, const SyncJournalFileRecord::EncryptionStatus status); diff --git a/src/csync/csync.h b/src/csync/csync.h index 235f0cd729afb..2680f40319492 100644 --- a/src/csync/csync.h +++ b/src/csync/csync.h @@ -221,6 +221,8 @@ struct OCSYNC_EXPORT csync_file_stat_s { QByteArray directDownloadCookies; QByteArray original_path; // only set if locale conversion fails + QByteArray tagList; // \n-separated List of tags + // In the local tree, this can hold a checksum and its type if it is // computed during discovery for some reason. // In the remote tree, this will have the server checksum, if available. diff --git a/src/csync/vio/csync_vio_local_unix.cpp b/src/csync/vio/csync_vio_local_unix.cpp index c5e22abb3c8b0..2ed52cbb92df0 100644 --- a/src/csync/vio/csync_vio_local_unix.cpp +++ b/src/csync/vio/csync_vio_local_unix.cpp @@ -28,6 +28,15 @@ #include +#ifdef __APPLE__ +#include +#include +#endif + +#ifdef __linux__ +#include +#endif + #include "c_private.h" #include "c_lib.h" #include "csync.h" @@ -120,6 +129,65 @@ std::unique_ptr csync_vio_local_readdir(csync_vio_handle_t *h file_stat->type = ItemTypeSkip; } + +#ifdef __APPLE__ + + // Create necessary system related objects + CFStringRef cfstr = CFStringCreateWithCString(kCFAllocatorDefault, + fullPath.constData(), + kCFStringEncodingUTF8); + CFURLRef urlref = CFURLCreateWithFileSystemPath(kCFAllocatorDefault, + cfstr, + kCFURLPOSIXPathStyle, + dirent->d_type == DT_DIR); + + // Query tags + CFArrayRef labels=NULL; + Boolean result = CFURLCopyResourcePropertyForKey(urlref, + kCFURLTagNamesKey, + &labels, + NULL); + + if(result==true && labels != NULL){ + // Extract the labels to our array + int count = (int) CFArrayGetCount(labels); + + if(count>0){ + QStringList tagarray; + + for(int index=0;index0)tagarray << QString::fromCFString(str); + } + tagarray.sort(Qt::CaseInsensitive); + QString tagList = tagarray.join(QChar(0x000A)); + file_stat->tagList=tagList.toUtf8(); + } + } + + // Clean up + CFRelease(cfstr); + CFRelease(urlref); + if(labels!=NULL)CFRelease(labels); + +#endif + +#ifdef __linux__ + const int sz=4096; + char buffer[sz]; + int result = getxattr(fullPath.constData(),"user.xdg.tags",buffer,sz); + + if(result>0) + { + // Data is store usually as comma(,) separated list. + // So we just need to replace ',' by '\n'. + QByteArray tagList = QByteArray(buffer,result); + tagList.replace(',','\n'); + printf("%s\n",buffer); + file_stat->tagList= tagList; + } +#endif + // Override type for virtual files if desired if (vfs) { // Directly modifies file_stat->type. diff --git a/src/gui/editlocallyjob.cpp b/src/gui/editlocallyjob.cpp index 167cf99c33cb9..65bb65f764f32 100644 --- a/src/gui/editlocallyjob.cpp +++ b/src/gui/editlocallyjob.cpp @@ -213,7 +213,9 @@ void EditLocallyJob::fetchRemoteFileParentInfo() QByteArrayLiteral("http://owncloud.org/ns:size"), QByteArrayLiteral("http://owncloud.org/ns:id"), QByteArrayLiteral("http://owncloud.org/ns:permissions"), - QByteArrayLiteral("http://owncloud.org/ns:checksums")}; + QByteArrayLiteral("http://owncloud.org/ns:checksums"), + QByteArrayLiteral("http://owncloud.org/ns:tags"), + QByteArrayLiteral("http://nextcloud.org/ns:system-tags")}; job->setProperties(props); connect(job, &LsColJob::directoryListingIterated, this, &EditLocallyJob::slotDirectoryListingIterated); diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index ace5524cb3d91..fb3062da87f1b 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -558,7 +558,6 @@ void Folder::slotWatchedPathChanged(const QString &path, ChangeReason reason) qCDebug(lcFolder) << "Changed path is not contained in folder, ignoring:" << path; return; } - auto relativePath = path.midRef(this->path().size()); if (pathIsIgnored(path)) { @@ -610,7 +609,7 @@ void Folder::slotWatchedPathChanged(const QString &path, ChangeReason reason) // an attribute change (pin state) that caused the notification bool spurious = false; if (record.isValid() - && !FileSystem::fileChanged(path, record._fileSize, record._modtime)) { + && !FileSystem::fileChanged(path, record._fileSize, record._modtime,record._tagList)) { spurious = true; if (auto pinState = _vfs->pinState(relativePath.toString())) { @@ -628,7 +627,6 @@ void Folder::slotWatchedPathChanged(const QString &path, ChangeReason reason) } } warnOnNewExcludedItem(record, relativePath); - emit watchedFileChangedExternally(path); // Also schedule this folder for a sync, but only after some delay: diff --git a/src/gui/folderwatcher_mac.cpp b/src/gui/folderwatcher_mac.cpp index 588b94a27b561..be81315c52618 100644 --- a/src/gui/folderwatcher_mac.cpp +++ b/src/gui/folderwatcher_mac.cpp @@ -71,8 +71,9 @@ static void callback( | kFSEventStreamEventFlagItemInodeMetaMod // for mtime change | kFSEventStreamEventFlagItemRenamed // also coming for moves to trash in finder | kFSEventStreamEventFlagItemModified // for content change - | kFSEventStreamEventFlagItemCloned; // for cloned items (since 10.13) - //We ignore other flags, e.g. for owner change, xattr change, Finder label change etc + | kFSEventStreamEventFlagItemCloned // for cloned items (since 10.13) + | kFSEventStreamEventFlagItemXattrMod; // for tags change (which are stored as xattr) + //We ignore other flags, e.g. for owner change etc. QStringList paths; CFArrayRef eventPaths = (CFArrayRef)eventPathsVoid; @@ -111,7 +112,7 @@ void FolderWatcherPrivate::startWatching() pathsToWatch, kFSEventStreamEventIdSinceNow, 0, // latency - kFSEventStreamCreateFlagUseCFTypes | kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagIgnoreSelf); + kFSEventStreamCreateFlagUseCFTypes | kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagIgnoreSelf | kFSEventStreamEventFlagItemXattrMod); CFRelease(pathsToWatch); CFRelease(folderCF); diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 6749ff4adc5e2..f01631ea88dbd 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -46,6 +46,8 @@ set(libsync_SRCS encryptedfoldermetadatahandler.cpp filesystem.h filesystem.cpp + filetags.h + filetags.cpp helpers.cpp httplogger.h httplogger.cpp diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 2112df8765d0b..b9f2404b316bc 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -17,6 +17,7 @@ #include "common/filesystembase.h" #include "common/syncjournaldb.h" #include "filesystem.h" +#include "filetags.h" #include "syncfileitem.h" #include "progressdispatcher.h" #include @@ -482,7 +483,8 @@ void ProcessDirectoryJob::processFile(PathTuple path, << " | e2ee: " << dbEntry.isE2eEncrypted() << "/" << serverEntry.isE2eEncrypted() << " | e2eeMangledName: " << dbEntry.e2eMangledName() << "/" << serverEntry.e2eMangledName << " | file lock: " << localFileIsLocked << "//" << serverFileIsLocked - << " | metadata missing: /" << localEntry.isMetadataMissing << '/'; + << " | metadata missing: /" << localEntry.isMetadataMissing << '/' + << " | tagList: " << dbEntry._tagList << "//" << localEntry.tagList << "//" << serverEntry.tagList; if (localEntry.isValid() && !serverEntry.isValid() @@ -491,6 +493,19 @@ void ProcessDirectoryJob::processFile(PathTuple path, qCWarning(lcDisco) << "File" << path._original << "was modified before the last sync run and is not in the sync journal and server"; } + if(localEntry.isValid() + && dbEntry.isValid()) { + + // Tag synchronization + FileTagManager::syncTags(_discoveryData->_account, + _discoveryData->_statedb, + _discoveryData->_localDir, + path._original, + localEntry, + serverEntry, + dbEntry); + } + if (_discoveryData->isRenamed(path._original)) { qCDebug(lcDisco) << "Ignoring renamed"; return; // Ignore this. @@ -656,6 +671,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(const SyncFileItemPtr &it item->_lockEditorApp = serverEntry.lockEditorApp; item->_lockTime = serverEntry.lockTime; item->_lockTimeout = serverEntry.lockTimeout; + item->_tagList = serverEntry.tagList; qCDebug(lcDisco()) << "item lock for:" << item->_file << item->_locked diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 7ae2022841803..ce89d2cffae1a 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -16,6 +16,7 @@ #include "common/utility.h" #include "configfile.h" #include "discovery.h" +#include "filetags.h" #include "helpers.h" #include "progressdispatcher.h" @@ -344,6 +345,7 @@ void DiscoverySingleLocalDirectoryJob::run() { i.isVirtualFile = dirent->type == ItemTypeVirtualFile || dirent->type == ItemTypeVirtualFileDownload; i.isMetadataMissing = dirent->is_metadata_missing; i.type = dirent->type; + i.tagList=dirent->tagList; results.push_back(i); } if (errno != 0) { @@ -391,7 +393,10 @@ void DiscoverySingleDirectoryJob::start() << "http://owncloud.org/ns:downloadURL" << "http://owncloud.org/ns:dDC" << "http://owncloud.org/ns:permissions" - << "http://owncloud.org/ns:checksums"; + << "http://owncloud.org/ns:checksums" + << "http://owncloud.org/ns:tags" + << "http://nextcloud.org/ns:system-tags" + << "http://nextcloud.org/ns:metadata_etag"; if (_isRootPath) props << "http://owncloud.org/ns:data-fingerprint"; @@ -538,7 +543,12 @@ static void propertyMapToRemoteInfo(const QMap &map, RemoteInf result.lockTimeout = 0; } } - + if(property == "system-tags" || property == "tags"){ + FileTagManager::fromPropertiesToTagList(result.tagList,value); + } + if(property == "metadata_etag"){ + //TODO: RMD Handle metadata etag for tag synchronization + } } if (result.isDirectory && map.contains("size")) { diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index ccad6bb92a361..379702f4cf276 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -86,6 +86,8 @@ struct RemoteInfo QString lockEditorApp; qint64 lockTime = 0; qint64 lockTimeout = 0; + + QByteArray tagList; // DAV property system-tags }; struct LocalInfo @@ -102,6 +104,9 @@ struct LocalInfo bool isVirtualFile = false; bool isSymLink = false; bool isMetadataMissing = false; + + QByteArray tagList; + [[nodiscard]] bool isValid() const { return !name.isNull(); } }; diff --git a/src/libsync/filesystem.cpp b/src/libsync/filesystem.cpp index 7c58c7b2bde45..4b2c2c174b896 100644 --- a/src/libsync/filesystem.cpp +++ b/src/libsync/filesystem.cpp @@ -13,6 +13,7 @@ */ #include "filesystem.h" +#include "filetags.h" #include "common/utility.h" #include @@ -85,10 +86,12 @@ bool FileSystem::setModTime(const QString &filename, time_t modTime) bool FileSystem::fileChanged(const QString &fileName, qint64 previousSize, - time_t previousMtime) + time_t previousMtime, + const QByteArray &taglist) { return getSize(fileName) != previousSize - || getModTime(fileName) != previousMtime; + || getModTime(fileName) != previousMtime + || FileTagManager::readTagListFromLocalFile(fileName) != taglist; } bool FileSystem::verifyFileUnchanged(const QString &fileName, diff --git a/src/libsync/filesystem.h b/src/libsync/filesystem.h index 602f74bb7f2bc..5a488b165d4bf 100644 --- a/src/libsync/filesystem.h +++ b/src/libsync/filesystem.h @@ -77,7 +77,8 @@ namespace FileSystem { */ bool OWNCLOUDSYNC_EXPORT fileChanged(const QString &fileName, qint64 previousSize, - time_t previousMtime); + time_t previousMtime, + const QByteArray &taglist); /** * @brief Like !fileChanged() but with verbose logging if the file *did* change. diff --git a/src/libsync/filetags.cpp b/src/libsync/filetags.cpp new file mode 100644 index 0000000000000..26fdb4e1ebd42 --- /dev/null +++ b/src/libsync/filetags.cpp @@ -0,0 +1,488 @@ +/* + * Copyright (C) 2024 by Uwe Runtemund + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#ifdef __APPLE__ +#include +#endif + +#ifdef __linux__ +#include +#endif + +#include +#include + +#include "common/syncjournalfilerecord.h" +#include "discoveryphase.h" +#include "filetags.h" + +namespace OCC{ + +Q_LOGGING_CATEGORY(lcFTM, "nextcloud.sync.filetags", QtInfoMsg) + +FileTagManager* gFileTagManger=NULL; + +FileTagManager::FileTagManager():QObject(NULL) +{} + +FileTagManager* FileTagManager::GetInstance() +{ + if(gFileTagManger==NULL)gFileTagManger=new FileTagManager(); + return gFileTagManger; +} + +void FileTagManager::fromPropertiesToTagList(QByteArray &list,const QString &properties) +{ + if(properties==NULL || properties.isEmpty()) return; + + QStringList tags; + QString token; + QXmlStreamReader reader(""%properties%""); + bool insideTag = false; + + while (!reader.atEnd()){ + QXmlStreamReader::TokenType type = reader.readNext(); + QString name = reader.name().toString(); + + // Start elements with DAV: + if (type == QXmlStreamReader::StartElement && (name=="system-tag" || name=="tag")){ + insideTag=true; + } + else if(type== QXmlStreamReader::Characters && insideTag){ + token.append(reader.text()); + } + else if (type == QXmlStreamReader::EndElement && (name == "system-tag" || name=="tag")){ + if(token.size()>0)tags << token; + token.clear(); + insideTag=false; + } + } + + if(tags.size()>0){ + if(list.size()>0)tags.append(QString(list).split(QChar(0x000A))); + tags.removeDuplicates(); + tags.sort(Qt::CaseInsensitive); + list=tags.join(QChar(0x000A)).toUtf8(); + } +} + +QByteArray FileTagManager::readTagListFromLocalFile(const QString &path) +{ +#ifdef __APPLE__ + // Create necessary system related objects + CFStringRef cfstr = path.toCFString(); + CFURLRef urlref = CFURLCreateWithFileSystemPath(kCFAllocatorDefault, + cfstr, + kCFURLPOSIXPathStyle, + false); + + // Query tags + CFArrayRef labels=NULL; + Boolean result = CFURLCopyResourcePropertyForKey(urlref, + kCFURLTagNamesKey, + &labels, + NULL); + + QStringList tagarray; + + if(result==true && labels != NULL){ + // Extract the labels to our array + int count = (int) CFArrayGetCount(labels); + + if(count>0){ + for(int index=0;index0)tagarray << QString::fromCFString(str); + } + } + } + + tagarray.sort(Qt::CaseInsensitive); + + // Clean up + CFRelease(cfstr); + CFRelease(urlref); + if(labels!=NULL)CFRelease(labels); + + return tagarray.join(QChar(0x000A)).toUtf8(); +#endif +#ifdef __linux__ + const int sz=4096; + char buffer[sz]; + int result = getxattr(path.toUtf8().constData(),"user.xdg.tags",buffer,sz); + + if(result>0){ + // Data is stored usually as comma(,) separated list. + // So we just need to replace ',' by '\n'. + QByteArray tagList = QByteArray(buffer,result); + tagList.replace(',','\n'); + return tagList; + } + else return QByteArray(); +#endif +} + +bool FileTagManager::writeTagListToLocalFile(const QString &localpath,const QByteArray &taglist) +{ + qCInfo(lcFTM) << "Write file tag list to file system: " << localpath; + +#ifdef __APPLE__ + + QStringList strlist=QString(taglist).split(QChar(0x000A)); + + //Create new array and append tag + int newsize=strlist.size(); + CFStringRef* strs = new CFStringRef[newsize]; + + for(int index=0;indextagList(file)); +} + +void FileTagManager::syncTags(AccountPtr account, + SyncJournalDb* journal, + const QString &localDir, + const QString &filePath, + const LocalInfo &localEntry, + const RemoteInfo &serverEntry, + const SyncJournalFileRecord &dbEntry) +{ + QString fullpath=localDir%filePath; + + if(localEntry.isValid() + && dbEntry.isValid() + && serverEntry.isValid()){ + + if(localEntry.tagList!= dbEntry._tagList + || localEntry.tagList!=serverEntry.tagList + || dbEntry._tagList!=serverEntry.tagList){ + + // For syncing tags, the following matrix applies: + // + // LC: Local, DB: SyncDatabase, RM: Remote + // X: Tag is present, -: Tag is not present + //----------------------------------------- + // LC DB RM What happend ACTION + //----------------------------------------- + // 1 - - - no tags NOP + // 2 X X X tag present NOP + // 3 X - X lc/rm added ADD (DB) + // 4 X - - lc added ADD (RM+DB) + // 5 - - X rm added ADD (LC+DB) + // 6 - X - lc/rm deleted DEL (DB) + // 7 - X X lc deleted DEL (RM+DB) + // 8 X X - rm deleted DEL (LC+DB) + //------------------------------------------ + + // Note: The lists from the entries are always alphabetically sorted. + QStringList lcTags; + QStringList dbTags; + QStringList rmTags; + QStringList syncedTagList; + + // Avoid adding empty entries to the list + if(localEntry.tagList.size()>0)lcTags=QString(localEntry.tagList).split(QChar(0x000A)); + if(serverEntry.tagList.size()>0)rmTags=QString(serverEntry.tagList).split(QChar(0x000A)); + if(dbEntry._tagList.size()>0)dbTags=QString(dbEntry._tagList).split(QChar(0x000A)); + + // The "new" synced list ist a joined list of all present tags, and then deleting + // Tags according to row 6-8 of above table + syncedTagList << lcTags; + syncedTagList << rmTags; + syncedTagList << dbTags; + syncedTagList.removeDuplicates(); + syncedTagList.sort(Qt::CaseInsensitive); + + for(int index=0;indexsyncTagsStep1(account, + journal, + fullpath, + syncedTagList, + dbEntry, + syncLocal, + syncDb); + } + else{ + GetInstance()->syncTagsStep2(journal, + fullpath, + newTagList, + dbEntry, + syncLocal, + syncDb); + } + } + } + else{ + localSync(account, + journal, + fullpath, + filePath, + localEntry, + dbEntry); + } +} + +void FileTagManager::syncTagsStep1(AccountPtr account, + SyncJournalDb* journal, + const QString &fullpath, + QStringList list, + const SyncJournalFileRecord &dbEntry, + bool syncLocal,bool syncDb) +{ + // SYNCING REMOTE CAN ONLY SET oc:tags, not nc:system-tags + // oc:tags can double nc:system-tags, but we try to prevent that. + // We do following: + // 1. propfind oc:tags list and od:system-tags of file + // 3. remove system-tags from list (we cannot delete them acually) + // 3. deside wich tag to set or remove + // 4. proppatch oc:tags + // TODO: Update this procedure, as soon as Nextcloud Server supports PROPPATCH of nc:system-tags + // Check: https://github.com/nextcloud/server/blob/master/apps/dav/lib/SystemTag/SystemTagPlugin.php + + qCInfo(lcFTM) << "Start remote sync: " + << fullpath <<' ' + << (syncLocal?'L':'-') + << (syncDb?'D':'-'); + + // Note: We must not delete tags explicitly. The server removes non present tags automatically. + QString tags; + for(int index=0;index"); + tags.append(list[index]); + tags.append(""); + } + + QMap setproperties; + setproperties.insert(QString("http://owncloud.org/ns:tags").toUtf8(),tags.toUtf8()); + + ProppatchJob* job = new ProppatchJob(account,dbEntry._path); + if(!setproperties.isEmpty())job->setProperties(setproperties); + + QByteArray newTagList = list.join(QChar(0x000A)).toUtf8(); + + QObject::connect(job, &ProppatchJob::success, GetInstance(), [=](){ + qCInfo(lcFTM) << "Remote sync successfull: " << fullpath; + syncTagsStep2(journal,fullpath,newTagList,dbEntry,syncLocal,syncDb); + }); + + QObject::connect(job, &ProppatchJob::finishedWithError, GetInstance(),[=]() { + // Do nothing, ignore. + qCInfo(lcFTM) << "Remote sync with errors: " << fullpath; + }); + + job->start(); +} + +void FileTagManager::syncTagsStep2(SyncJournalDb* journal, + const QString &fullpath, + const QByteArray &newTagList, + const SyncJournalFileRecord &dbEntry, + bool syncLocal,bool syncDb) +{ + bool localSuccess=true; + if(syncLocal) + { + localSuccess=writeTagListToLocalFile(fullpath,newTagList); + } + + if(localSuccess && syncDb) + { + journal->updateMetadataTagList(dbEntry._path,&newTagList); + } +}; + +void FileTagManager::localSync(AccountPtr account, + SyncJournalDb* journal, + const QString &localDir, + const QString &filePath, + const LocalInfo &localEntry, + const SyncJournalFileRecord &dbEntry) +{ + QString fullpath=localDir%filePath; + + if(localEntry.isValid() && dbEntry.isValid()){ + + if(localEntry.tagList!= dbEntry._tagList){ + + // For syncing tags, the following matrix applies: + // + // LC DB What happened Action + // 1 - - no tags NOP + // 2 X X tag present NOP + // 3 - X rm add or lc del FULL SYNC + // 4 X - lc add or rm del FULL SYNC + // + // Note: The lists from the entries are always alphabetically sorted. + QStringList lcTags; + QStringList dbTags; + QStringList syncedTagList; + + // Avoid adding empty entries to the list + if(localEntry.tagList.size()>0)lcTags=QString(localEntry.tagList).split(QChar(0x000A)); + if(dbEntry._tagList.size()>0)dbTags=QString(dbEntry._tagList).split(QChar(0x000A)); + + // The "new" synced list ist just a joined list of all present tags. + syncedTagList << lcTags; + syncedTagList << dbTags; + syncedTagList.removeDuplicates(); + syncedTagList.sort(Qt::CaseInsensitive); + + QByteArray newTagList = syncedTagList.join(QChar(0x000A)).toUtf8(); + + bool fullSync = newTagList != localEntry.tagList || newTagList != dbEntry._tagList; + + qCInfo(lcFTM) << "Local sync: " + << filePath + << '('<< syncedTagList.join(QChar((int)',')).toUtf8().data() << ')' + << (fullSync?'F':'N'); + + if(fullSync){ + const auto propfindJob = new PropfindJob(account, filePath, GetInstance()); + propfindJob->setProperties({ QByteArrayLiteral("http://owncloud.org/ns:tags"), + QByteArrayLiteral("http://nextcloud.org/ns:system-tags") }); + + connect(propfindJob, &PropfindJob::result, GetInstance(), [=](const QVariantMap &result){ + qCInfo(lcFTM) << "Fetching remote info ok: " << fullpath; + + RemoteInfo remoteInfo; + int slash = filePath.lastIndexOf('/'); + remoteInfo.name = filePath.mid(slash + 1); + + //TODO: SNIPPTE TAKEN FROM fieldtagmodel.h, but I quess doubling it is not good, + // also I prefer to remove the "special" code from PropfindJob and handle it + const auto normalTags = result.value(QStringLiteral("tags")).toStringList(); + const auto systemTags = result.value(QStringLiteral("system-tags")).toList(); + + QStringList systemTagStringList = QStringList(); + for (const auto &systemTagMapVariant : systemTags) { + const auto systemTagMap = systemTagMapVariant.toMap(); + const auto systemTag = systemTagMap.value(QStringLiteral("tag")).toString(); + systemTagStringList << systemTag; + } + + QStringList tags=QStringList(); + tags << normalTags << systemTagStringList; + tags.removeDuplicates(); + tags.removeAll(QString("")); + tags.sort(Qt::CaseInsensitive); + + remoteInfo.tagList=tags.join(QChar(0x000A)).toUtf8().constData(); + + QByteArray rl =remoteInfo.tagList; + rl.replace('\n',','); + + syncTags(account, + journal, + localDir, + filePath, + localEntry, + remoteInfo, + dbEntry); + }); + + connect(propfindJob, &PropfindJob::finishedWithError, GetInstance(), [=](QNetworkReply*){ + //Do nothing here. + qCInfo(lcFTM) << "Fetching remote info with errors: " << fullpath; + }); + + propfindJob->start(); + } + } + } +} + +} diff --git a/src/libsync/filetags.h b/src/libsync/filetags.h new file mode 100644 index 0000000000000..075d397512b35 --- /dev/null +++ b/src/libsync/filetags.h @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2024 by Uwe Runtemund + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + + +#ifndef filetags_h +#define filetags_h + +#include "account.h" +#include "discoveryphase.h" +#include "networkjobs.h" +#include "owncloudlib.h" +#include "owncloudpropagator.h" +#include "syncfileitem.h" + +#include +#include + +namespace OCC +{ +/*! + * THE FILE TAG SYNCING PROCESS + * + * Tags are special keywords for files or directories to give them an additional meaning. This + * feature is provided by Nextcloud as well as by many operating systems. So it is a straight + * forward assumption to expect, that also Nextcloud will sync tags with the local file system. + * + * Tags are stored on Nextcloud and can be fetched by a PROPFIND query. Locally the properties + * are stored in the operating file system as well as in the syncronization database. Because the + * tags can only be queried by a PROPFIND and our goal is not to heavily grow the number of slow + * network queries, we go the following way for synchronizing the tags: + * + * In theory for synchronizing correctly, three locations are relevant: + * 1. The local file system (LC) + * 2. The local sync database (DB) + * 3. The remote nextcloud server (RM) + * + * Changes of the tags are done on the RM or LC. Also we add the implementation on already existing + * Nextcloud instances with tags existing propably local or remote, we need a gracefully strategy + * to synchronizes all these tags without loosing any of them. + * + *The following matrix show all possible situations and solutions for syncing the tags: + * + * LC: Local, DB: SyncDatabase, RM: Remote + * X: Tag is present, -: Tag is not present + *----------------------------------------- + * LC DB RM What happend ACTION + *----------------------------------------- + * 1 - - - no tags NOP + * 2 X X X tag present NOP + * 3 X - X lc/rm added ADD (DB) + * 4 X - - lc added ADD (RM+DB) + * 5 - - X rm added ADD (LC+DB) + * 6 - X - lc/rm deleted DEL (DB) + * 7 - X X lc deleted DEL (RM+DB) + * 8 X X - rm deleted DEL (LC+DB) + *------------------------------------------ + * + * IF WE HAVE ALL THREE POINTS, WE SYNC THAT WAY! + * + * In fact, most of the time, we have not all three data points available. Most of the time we have + * LC/DB or RM/DB, so we have to deal a little bit different, to reach the same approach. + * To avoid an massive increasment of the network polls, we use following approach: + * + * - DB is frequently overridden by the client with the RM value, so we cannot say for sure the + * correct sync status + * LC DB What happened Action + * 1 - - no tags NOP + * 2 X X tag present NOP + * 3 - X rm add or lc del FULL SYNC + * 4 X - lc add or rm del FULL SYNC + * + */ +class OCSYNC_EXPORT FileTagManager : public QObject +{ + Q_OBJECT + public: + + /*! + * \brief Converts the given XML fragment of system-tag entries to a sorted list of tags. + * + * The list is a QByteArray with the content. The tags are delimited by 0x0A ('\\n') + */ + static void fromPropertiesToTagList(QByteArray &list,const QString &properties); + + /*! + * \brief Reads the tags from the local file system. + */ + static QByteArray readTagListFromLocalFile(const QString &localpath); + + /*! Restore the tags in the local file system. + * \note: is needed after file download. + */ + static bool restoreLocalFileTags(SyncJournalDb* journal, + const QString &localdir, + const QString &file); + + /*! + * \brief Makes a "full" synchronization. If all data is valid, full synchronization is directly + * done. If not, local sync is checked, if data is not synchronized, a full sync is started. + */ + static void syncTags(AccountPtr account, + SyncJournalDb* journal, + const QString &localDir, + const QString &filePath, + const LocalInfo &localEntry, + const RemoteInfo &serverEntry, + const SyncJournalFileRecord &dbEntry); + +private: + + //! Private constructor. No need for an instance actually. + FileTagManager(); + + //! Needed for QT connect + static FileTagManager* GetInstance(); + + //! Helper method for syncTags + void syncTagsStep1(AccountPtr account, + SyncJournalDb* journal, + const QString &fullpath, + QStringList list, + const SyncJournalFileRecord &dbEntry, + bool syncLocal,bool syncDb); + + //! Helper method for synTags + void syncTagsStep2(SyncJournalDb* journal, + const QString &fullpath, + const QByteArray &newTagList, + const SyncJournalFileRecord &dbEntry, + bool syncLocal,bool syncDb); + + /*! + * \brief Makes a "local" synchronization. Will do a server sync, if initial sync is assumed. + */ + static void localSync(AccountPtr account, + SyncJournalDb* journal, + const QString &localDir, + const QString &filePath, + const LocalInfo &localEntry, + const SyncJournalFileRecord &dbEntry); + + //! Write the tag list to the local file. + static bool writeTagListToLocalFile(const QString &localpath,const QByteArray &taglist); + + }; + +} // namespace OCC + +#endif /* filetags_h */ diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 700afb41741f5..e168dbd985efb 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -861,13 +861,36 @@ ProppatchJob::ProppatchJob(AccountPtr account, const QString &path, QObject *par void ProppatchJob::start() { - if (_properties.isEmpty()) { + if (_setproperties.isEmpty()) { qCWarning(lcProppatchJob) << "Proppatch with no properties!"; } QNetworkRequest req; - QByteArray propStr; - QMapIterator it(_properties); + // Properties which shall be set + QByteArray setPropStr; + QMapIterator it(_setproperties); + while (it.hasNext()) { + it.next(); + QByteArray keyName = it.key(); + QByteArray keyNs; + if (keyName.contains(':')) { + int colIdx = keyName.lastIndexOf(":"); + keyNs = keyName.left(colIdx); + keyName = keyName.mid(colIdx + 1); + } + + setPropStr += " <" + keyName; + if (!keyNs.isEmpty()) { + setPropStr += " xmlns=\"" + keyNs + "\" "; + } + setPropStr += ">"; + setPropStr += it.value(); + setPropStr += "\n"; + } + + // Properties which shall be removed + QByteArray remPropStr; + it = QMapIterator(_removeproperties); while (it.hasNext()) { it.next(); QByteArray keyName = it.key(); @@ -878,19 +901,29 @@ void ProppatchJob::start() keyName = keyName.mid(colIdx + 1); } - propStr += " <" + keyName; + remPropStr += " <" + keyName; if (!keyNs.isEmpty()) { - propStr += " xmlns=\"" + keyNs + "\" "; + remPropStr += " xmlns=\"" + keyNs + "\" "; } - propStr += ">"; - propStr += it.value(); - propStr += "\n"; + remPropStr += ">"; + remPropStr += it.value(); + remPropStr += "\n"; } + QByteArray xml = "\n" - "\n" - " \n" - + propStr + " \n" - "\n"; + "\n"; + + if(!setPropStr.isEmpty()){ + xml += " \n" + + setPropStr + " \n"; + } + + if(!remPropStr.isEmpty()){ + xml += " \n" + + remPropStr + " \n"; + } + + xml += "\n"; auto *buf = new QBuffer(this); buf->setData(xml); @@ -901,12 +934,22 @@ void ProppatchJob::start() void ProppatchJob::setProperties(QMap properties) { - _properties = properties; + _setproperties = properties; } -QMap ProppatchJob::properties() const +QMap ProppatchJob::propertiesToSet() const { - return _properties; + return _setproperties; +} + +void ProppatchJob::removeProperties(QMap properties) +{ + _removeproperties = properties; +} + +QMap ProppatchJob::propertiesToRemove() const +{ + return _removeproperties; } bool ProppatchJob::finished() diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index f623345021238..c050972ace6a8 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -276,7 +276,10 @@ class OWNCLOUDSYNC_EXPORT ProppatchJob : public AbstractNetworkJob * e.g. "ns:with:colons:bar", which is "bar" in the "ns:with:colons" namespace */ void setProperties(QMap properties); - [[nodiscard]] QMap properties() const; + [[nodiscard]] QMap propertiesToSet() const; + + void removeProperties(QMap properties); + [[nodiscard]] QMap propertiesToRemove() const; signals: void success(); @@ -286,7 +289,8 @@ private slots: bool finished() override; private: - QMap _properties; + QMap _setproperties; + QMap _removeproperties; }; /** diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index caa6922dbdb24..a6add91ed851c 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -21,6 +21,7 @@ #include "common/syncjournalfilerecord.h" #include "common/utility.h" #include "filesystem.h" +#include "filetags.h" #include "propagatorjobs.h" #include #include @@ -1364,6 +1365,10 @@ void PropagateDownloadFile::updateMetadata(bool isConflict) if (isLikelyFinishedQuickly() && duration > 5 * 1000) { qCWarning(lcPropagateDownload) << "WARNING: Unexpectedly slow connection, took" << duration << "msec for" << _item->_size - _resumeStart << "bytes for" << _item->_file; } + + FileTagManager::restoreLocalFileTags(propagator()->_journal, + propagator()->localPath(), + _item->_file); } void PropagateDownloadFile::slotDownloadProgress(qint64 received, qint64) diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 72c043fbbafc3..22e8be6a67430 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -24,6 +24,7 @@ #include "common/syncfilestatus.h" #include "csync_exclude.h" #include "filesystem.h" +#include "filetags.h" #include "deletejob.h" #include "propagatedownload.h" #include "common/asserts.h" diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index 4a169d9cbefbb..4cc7c2d500aaa 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -18,6 +18,7 @@ #include "common/utility.h" #include "helpers.h" #include "filesystem.h" +#include "filetags.h" #include #include "csync/vio/csync_vio_local.h" @@ -125,6 +126,7 @@ SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QStri rec._lockstate._lockEditorApp = _lockEditorApp; rec._lockstate._lockTime = _lockTime; rec._lockstate._lockTimeout = _lockTimeout; + rec._tagList = _tagList; // Update the inode if possible rec._inode = _inode; @@ -166,6 +168,7 @@ SyncFileItemPtr SyncFileItem::fromSyncJournalFileRecord(const SyncJournalFileRec item->_sharedByMe = rec._sharedByMe; item->_isShared = rec._isShared; item->_lastShareStateFetchedTimestamp = rec._lastShareStateFetchedTimestamp; + item->_tagList = rec._tagList; return item; } @@ -238,6 +241,14 @@ SyncFileItemPtr SyncFileItem::fromProperties(const QString &filePath, const QMap item->_direction = SyncFileItem::None; item->_instruction = CSYNC_INSTRUCTION_NONE; + // Tags + if (properties.contains(QStringLiteral("tags"))) { + FileTagManager::fromPropertiesToTagList(item->_tagList,properties.value("tags")); + } + if (properties.contains(QStringLiteral("system-tags"))) { + FileTagManager::fromPropertiesToTagList(item->_tagList,properties.value("system-tags")); + } + return item; } diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 89e68ca99b209..2dd7b6a10159e 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -335,6 +335,8 @@ class OWNCLOUDSYNC_EXPORT SyncFileItem bool _isFileDropDetected = false; bool _isEncryptedMetadataNeedUpdate = false; + + QByteArray _tagList; }; inline bool operator<(const SyncFileItemPtr &item1, const SyncFileItemPtr &item2)