diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b72d6af..a47ca35 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,11 +24,11 @@ jobs: shell: bash run: | sudo apt-get update - sudo apt-get install -y notmuch python3-notmuch python3-venv flake8 + sudo apt-get install -y notmuch libnotmuch-dev python3-venv flake8 python3 -m venv env source ./env/bin/activate pip install setuptools setuptools_scm pytest dkimpy - ln -s /usr/lib/python3/dist-packages/notmuch ./env/lib/python*/site-packages + pip install notmuch2 - name: flake8 lint run: | source ./env/bin/activate diff --git a/afew/Database.py b/afew/Database.py index ee9812e..208ed2b 100644 --- a/afew/Database.py +++ b/afew/Database.py @@ -5,7 +5,7 @@ import time import logging -import notmuch +import notmuch2 from afew.NotmuchSettings import notmuch_settings, get_notmuch_new_tags @@ -46,19 +46,18 @@ def __exit__(self, exc_type, exc_value, traceback): """ self.close() - def open(self, rw=False, retry_for=180, retry_delay=1, create=False): + def open(self, rw=False, retry_for=180, retry_delay=1): if rw: - if self.handle and self.handle.mode == notmuch.Database.MODE.READ_WRITE: + if self.handle and self.handle.mode == notmuch2.Database.MODE.READ_WRITE: return self.handle start_time = time.time() while True: try: - self.handle = notmuch.Database(self.db_path, - mode=notmuch.Database.MODE.READ_WRITE, - create=create) + self.handle = notmuch2.Database(self.db_path, + mode=notmuch2.Database.MODE.READ_WRITE) break - except notmuch.NotmuchError: + except notmuch2.NotmuchError: time_left = int(retry_for - (time.time() - start_time)) if time_left <= 0: @@ -71,7 +70,8 @@ def open(self, rw=False, retry_for=180, retry_delay=1, create=False): time.sleep(retry_delay) else: if not self.handle: - self.handle = notmuch.Database(self.db_path, create=create) + self.handle = notmuch2.Database(self.db_path, + mode=notmuch2.Database.MODE.READ_ONLY) return self.handle @@ -93,7 +93,7 @@ def do_query(self, query): :rtype: :class:`notmuch.Query` """ logging.debug('Executing query %r' % query) - return notmuch.Query(self.open(), query) + return notmuch2.Database.messages(self.open(), query) def get_messages(self, query, full_thread=False): """ @@ -106,10 +106,10 @@ def get_messages(self, query, full_thread=False): :returns: an iterator over :class:`notmuch.Message` objects """ if not full_thread: - for message in self.do_query(query).search_messages(): + for message in self.do_query(query): yield message else: - for thread in self.do_query(query).search_threads(): + for thread in self.do_query(query): for message in self.walk_thread(thread): yield message @@ -163,16 +163,13 @@ def add_message(self, path, sync_maildir_flags=False, new_mail_handler=None): """ # TODO: it would be nice to update notmuchs directory index here handle = self.open(rw=True) - if hasattr(notmuch.Database, 'index_file'): - message, status = handle.index_file(path, sync_maildir_flags=sync_maildir_flags) - else: - message, status = handle.add_message(path, sync_maildir_flags=sync_maildir_flags) + message, duplicate = handle.add(path, sync_flags=sync_maildir_flags) - if status != notmuch.STATUS.DUPLICATE_MESSAGE_ID: + if not duplicate: logging.info('Found new mail in {}'.format(path)) for tag in get_notmuch_new_tags(): - message.add_tag(tag) + message.tags.add(tag) if new_mail_handler: new_mail_handler(message) diff --git a/afew/MailMover.py b/afew/MailMover.py index 81caec0..5b11394 100644 --- a/afew/MailMover.py +++ b/afew/MailMover.py @@ -8,8 +8,6 @@ from datetime import date, datetime, timedelta from subprocess import check_call, CalledProcessError, DEVNULL -import notmuch - from afew.Database import Database from afew.utils import get_message_summary @@ -21,7 +19,7 @@ class MailMover(Database): def __init__(self, max_age=0, rename=False, dry_run=False, notmuch_args='', quiet=False): super().__init__() - self.db = notmuch.Database(self.db_path) + self.db = Database() self.query = 'folder:"{folder}" AND {subquery}' if max_age: days = timedelta(int(max_age)) @@ -61,11 +59,11 @@ def move(self, maildir, rules): main_query = self.query.format( folder=maildir.replace("\"", "\\\""), subquery=query) logging.debug("query: {}".format(main_query)) - messages = notmuch.Query(self.db, main_query).search_messages() + messages = self.db.get_messages(main_query) for message in messages: # a single message (identified by Message-ID) can be in several # places; only touch the one(s) that exists in this maildir - all_message_fnames = message.get_filenames() + all_message_fnames = (str(name) for name in message.filenames()) to_move_fnames = [name for name in all_message_fnames if maildir in name] if not to_move_fnames: diff --git a/afew/files.py b/afew/files.py index 8186c09..475b65a 100644 --- a/afew/files.py +++ b/afew/files.py @@ -8,7 +8,7 @@ import platform import queue import threading -import notmuch +import notmuch2 import pyinotify import ctypes import contextlib @@ -43,19 +43,19 @@ def process_IN_MOVED_TO(self, event): def new_mail(message): for filter_ in self.options.enable_filters: try: - filter_.run('id:"{}"'.format(message.get_message_id())) + filter_.run('id:"{}"'.format(message.messageid)) filter_.commit(self.options.dry_run) except Exception as e: logging.warning('Error processing mail with filter {!r}: {}'.format(filter_.message, e)) try: self.database.add_message(event.pathname, - sync_maildir_flags=True, + sync_flags=True, new_mail_handler=new_mail) - except notmuch.FileError as e: + except notmuch2.FileError as e: logging.warning('Error opening mail file: {}'.format(e)) return - except notmuch.FileNotEmailError as e: + except notmuch2.FileNotEmailError as e: logging.warning('File does not look like an email: {}'.format(e)) return else: diff --git a/afew/filters/BaseFilter.py b/afew/filters/BaseFilter.py index b4e52ea..716f8b0 100644 --- a/afew/filters/BaseFilter.py +++ b/afew/filters/BaseFilter.py @@ -55,27 +55,27 @@ def run(self, query): self.handle_message(message) def handle_message(self, message): - if not self._tag_blacklist.intersection(message.get_tags()): + if not self._tag_blacklist.intersection(message.tags): self.remove_tags(message, *self._tags_to_remove) self.add_tags(message, *self._tags_to_add) def add_tags(self, message, *tags): if tags: self.log.debug('Adding tags %s to id:%s' % (', '.join(tags), - message.get_message_id())) - self._add_tags[message.get_message_id()].update(tags) + message.messageid)) + self._add_tags[message.messageid].update(tags) def remove_tags(self, message, *tags): if tags: filtered_tags = list(tags) self.log.debug('Removing tags %s from id:%s' % (', '.join(filtered_tags), - message.get_message_id())) - self._remove_tags[message.get_message_id()].update(filtered_tags) + message.messageid)) + self._remove_tags[message.messageid].update(filtered_tags) def flush_tags(self, message): self.log.debug('Removing all tags from id:%s' % - message.get_message_id()) - self._flush_tags.append(message.get_message_id()) + message.messageid) + self._flush_tags.append(message.messageid) def commit(self, dry_run=True): dirty_messages = set() @@ -93,15 +93,18 @@ def commit(self, dry_run=True): db = self.database.open(rw=True) for message_id in dirty_messages: - message = db.find_message(message_id) + message = db.find(message_id) if message_id in self._flush_tags: message.remove_all_tags() for tag in self._add_tags.get(message_id, []): - message.add_tag(tag) + message.tags.add(tag) for tag in self._remove_tags.get(message_id, []): - message.remove_tag(tag) + try: + message.tags.remove(tag) + except KeyError: + pass self.flush_changes() diff --git a/afew/filters/DKIMValidityFilter.py b/afew/filters/DKIMValidityFilter.py index ad7be1a..46f8138 100644 --- a/afew/filters/DKIMValidityFilter.py +++ b/afew/filters/DKIMValidityFilter.py @@ -50,14 +50,18 @@ def __init__(self, database, ok_tag='dkim-ok', fail_tag='dkim-fail'): self.__module__, self.__class__.__name__)) def handle_message(self, message): - if message.get_header(self.header): + try: + selfhead = message.header(self.header) + except LookupError: + selfhead = '' + if selfhead: try: - dkim_ok = all(map(verify_dkim, message.get_filenames())) + dkim_ok = all(map(verify_dkim, message.filenames())) except DKIMVerifyError as verify_error: self.log.warning( "Failed to verify DKIM of '%s': %s " "(marked as 'dkim-fail')", - message.get_message_id(), + message.messageid, verify_error ) dkim_ok = False diff --git a/afew/filters/DMARCReportInspectionFilter.py b/afew/filters/DMARCReportInspectionFilter.py index f3cbda1..24d423b 100644 --- a/afew/filters/DMARCReportInspectionFilter.py +++ b/afew/filters/DMARCReportInspectionFilter.py @@ -147,7 +147,7 @@ def __init__(self, # pylint: disable=too-many-arguments self.__module__, self.__class__.__name__)) def handle_message(self, message): - if not self.dmarc_subject.match(message.get_header('Subject')): + if not self.dmarc_subject.match(message.header('Subject')): return auth_results = {'dkim': True, 'spf': True} @@ -165,6 +165,6 @@ def handle_message(self, message): except DMARCInspectionError as inspection_error: self.log.error( "Failed to verify DMARC report of '%s': %s (not tagging)", - message.get_message_id(), + message.messageid, inspection_error ) diff --git a/afew/filters/FolderNameFilter.py b/afew/filters/FolderNameFilter.py index af8ed78..6b2a535 100644 --- a/afew/filters/FolderNameFilter.py +++ b/afew/filters/FolderNameFilter.py @@ -24,8 +24,8 @@ def __init__(self, database, folder_blacklist='', folder_transforms='', def handle_message(self, message): # Find all the dirs in the mail directory that this message # belongs to - maildirs = [re.match(self.__filename_pattern, filename) - for filename in message.get_filenames()] + maildirs = [re.match(self.__filename_pattern, str(filename)) + for filename in message.filenames()] maildirs = filter(None, maildirs) if maildirs: # Make the folders relative to mail_root and split them. @@ -34,8 +34,12 @@ def handle_message(self, message): folders = set([folder for folder_group in folder_groups for folder in folder_group]) + try: + subject = message.header('subject') + except LookupError: + subject = '' self.log.debug('found folders {} for message {!r}'.format( - folders, message.get_header('subject'))) + folders, subject)) # remove blacklisted folders clean_folders = folders - self.__folder_blacklist diff --git a/afew/filters/HeaderMatchingFilter.py b/afew/filters/HeaderMatchingFilter.py index 4cbcf86..6bd6bbb 100644 --- a/afew/filters/HeaderMatchingFilter.py +++ b/afew/filters/HeaderMatchingFilter.py @@ -6,7 +6,7 @@ from afew.filters.BaseFilter import Filter -from notmuch.errors import NullPointerError +from notmuch2._errors import NullPointerError import re @@ -23,14 +23,14 @@ def __init__(self, database, **kwargs): def handle_message(self, message): if self.header is not None and self.pattern is not None: - if not self._tag_blacklist.intersection(message.get_tags()): + if not self._tag_blacklist.intersection(message.tags): try: - value = message.get_header(self.header) + value = message.header(self.header) match = self.pattern.search(value) if match: tagdict = {k: v.lower() for k, v in match.groupdict().items()} sub = (lambda tag: tag.format(**tagdict)) self.remove_tags(message, *map(sub, self._tags_to_remove)) self.add_tags(message, *map(sub, self._tags_to_add)) - except NullPointerError: + except (NullPointerError, LookupError): pass diff --git a/afew/filters/KillThreadsFilter.py b/afew/filters/KillThreadsFilter.py index a9187bb..08a5b9f 100644 --- a/afew/filters/KillThreadsFilter.py +++ b/afew/filters/KillThreadsFilter.py @@ -9,7 +9,7 @@ class KillThreadsFilter(Filter): query = 'NOT tag:killed' def handle_message(self, message): - query = self.database.get_messages('thread:"%s" AND tag:killed' % message.get_thread_id()) + query = self.database.get_messages('thread:"%s" AND tag:killed' % message.threadid) if len(list(query)): self.add_tags(message, 'killed') diff --git a/afew/filters/MeFilter.py b/afew/filters/MeFilter.py index 38b1c97..6ddded3 100644 --- a/afew/filters/MeFilter.py +++ b/afew/filters/MeFilter.py @@ -26,5 +26,5 @@ def __init__(self, database, me_tag='to-me', tags_blacklist=[]): self.me_tag = me_tag def handle_message(self, message): - if not self._tag_blacklist.intersection(message.get_tags()): + if not self._tag_blacklist.intersection(message.tags): self.add_tags(message, self.me_tag) diff --git a/afew/filters/PropagateTagsByRegexInThreadFilter.py b/afew/filters/PropagateTagsByRegexInThreadFilter.py index 7587c67..ea59590 100644 --- a/afew/filters/PropagateTagsByRegexInThreadFilter.py +++ b/afew/filters/PropagateTagsByRegexInThreadFilter.py @@ -39,7 +39,7 @@ class PropagateTagsByRegexInThreadFilter(Filter): """ def handle_message(self, message): - thread_query = 'thread:"%s"' % (message.get_thread_id(),) + thread_query = 'thread:"%s"' % (message.threadid,) if self._filter: query = self.database.get_messages("(%s) AND (%s)" % (thread_query, self._filter)) else: @@ -50,7 +50,7 @@ def handle_message(self, message): messages = list(query) # flatten tags - tags_in_thread_t = {m.get_tags() for m in messages} # a set of Tags instances + tags_in_thread_t = {m.tags for m in messages} # a set of Tags instances tags_in_thread = set(_flatten(tags_in_thread_t)) # filter tags diff --git a/afew/filters/PropagateTagsInThreadFilter.py b/afew/filters/PropagateTagsInThreadFilter.py index a87c34e..f25da48 100644 --- a/afew/filters/PropagateTagsInThreadFilter.py +++ b/afew/filters/PropagateTagsInThreadFilter.py @@ -20,7 +20,7 @@ class PropagateTagsInThreadFilter(Filter): def handle_message(self, message): for tag in self._propagate_tags: - tag_query = 'thread:"%s" AND is:"%s"' % (message.get_thread_id(), tag) + tag_query = 'thread:"%s" AND is:"%s"' % (message.threadid, tag) if self._filter: query = self.database.get_messages("(%s) AND (%s)" % (tag_query, self._filter)) else: diff --git a/afew/filters/SentMailsFilter.py b/afew/filters/SentMailsFilter.py index 78ce3bf..a948bc3 100644 --- a/afew/filters/SentMailsFilter.py +++ b/afew/filters/SentMailsFilter.py @@ -38,7 +38,10 @@ def handle_message(self, message): self.add_tags(message, self.sent_tag) if self.to_transforms: for header in ('To', 'Cc', 'Bcc'): - email = self.__get_bare_email(message.get_header(header)) + try: + email = self.__get_bare_email(message.header(header)) + except LookupError: + email = '' for tag in self.__pick_tags(email): self.add_tags(message, tag) else: diff --git a/afew/tests/test_dkimvalidityfilter.py b/afew/tests/test_dkimvalidityfilter.py index 2596728..9e39334 100644 --- a/afew/tests/test_dkimvalidityfilter.py +++ b/afew/tests/test_dkimvalidityfilter.py @@ -39,19 +39,19 @@ def _make_message(): Mocked methods: - - `get_header()` returns non-empty string. When testing with mocked + - `header()` returns non-empty string. When testing with mocked function for verifying DKIM signature, DKIM signature doesn't matter as long as it's non-empty string. - - `get_filenames()` returns list of non-empty string. When testing with + - `filenames()` returns list of non-empty string. When testing with mocked file open, it must just be non-empty string. - - `get_message_id()` returns some generated message ID. + - `messageid` returns some generated message ID. """ message = mock.Mock() - message.get_header.return_value = 'sig' - message.get_filenames.return_value = ['a'] - message.get_message_id.return_value = make_msgid() + message.header.return_value = 'sig' + message.filenames.return_value = ['a'] + message.messageid = make_msgid() return message @@ -65,7 +65,7 @@ def test_no_dkim_header(self): """ dkim_filter, tags = _make_dkim_validity_filter() message = _make_message() - message.get_header.return_value = False + message.header.return_value = False with mock.patch('afew.filters.DKIMValidityFilter.dkim.verify') \ as dkim_verify: @@ -82,7 +82,7 @@ def test_dkim_all_ok(self): """ dkim_filter, tags = _make_dkim_validity_filter() message = _make_message() - message.get_filenames.return_value = ['a', 'b', 'c'] + message.filenames.return_value = ['a', 'b', 'c'] with mock.patch('afew.filters.DKIMValidityFilter.dkim.verify') \ as dkim_verify: @@ -99,7 +99,7 @@ def test_dkim_all_fail(self): """ dkim_filter, tags = _make_dkim_validity_filter() message = _make_message() - message.get_filenames.return_value = ['a', 'b', 'c'] + message.filenames.return_value = ['a', 'b', 'c'] with mock.patch('afew.filters.DKIMValidityFilter.dkim.verify') \ as dkim_verify: @@ -116,7 +116,7 @@ def test_dkim_some_fail(self): """ dkim_filter, tags = _make_dkim_validity_filter() message = _make_message() - message.get_filenames.return_value = ['a', 'b', 'c'] + message.filenames.return_value = ['a', 'b', 'c'] with mock.patch('afew.filters.DKIMValidityFilter.dkim.verify') \ as dkim_verify: diff --git a/afew/tests/test_headermatchingfilter.py b/afew/tests/test_headermatchingfilter.py index 8b09a67..fee1f4b 100644 --- a/afew/tests/test_headermatchingfilter.py +++ b/afew/tests/test_headermatchingfilter.py @@ -7,7 +7,7 @@ from afew.Database import Database from afew.filters.HeaderMatchingFilter import HeaderMatchingFilter -from notmuch.errors import NullPointerError +from notmuch2._errors import NullPointerError class _AddTags: # pylint: disable=too-few-public-methods @@ -38,23 +38,23 @@ def _make_message(should_fail): Mocked methods: - - `get_header()` returns non-empty string. When testing with mocked + - `header()` returns non-empty string. When testing with mocked function for verifying DKIM signature, DKIM signature doesn't matter as long as it's non-empty string. - - `get_filenames()` returns list of non-empty string. When testing with + - `filenames()` returns list of non-empty string. When testing with mocked file open, it must just be non-empty string. - - `get_message_id()` returns some generated message ID. + - `message` returns some generated message ID. """ message = mock.Mock() if should_fail: - message.get_header.side_effect = NullPointerError + message.header.side_effect = NullPointerError else: - message.get_header.return_value = 'header' - message.get_filenames.return_value = ['a'] - message.get_tags.return_value = ['a'] - message.get_message_id.return_value = make_msgid() + message.header.return_value = 'header' + message.filenames.return_value = ['a'] + message.tags = ['a'] + message.messageid = make_msgid() return message diff --git a/afew/tests/test_mailmover.py b/afew/tests/test_mailmover.py index b4f240e..a8ffdc7 100644 --- a/afew/tests/test_mailmover.py +++ b/afew/tests/test_mailmover.py @@ -8,6 +8,7 @@ import shutil import tempfile import unittest +import notmuch2 from afew.Database import Database from afew.NotmuchSettings import notmuch_settings, write_notmuch_settings @@ -35,7 +36,7 @@ def create_mail(msg, maildir, notmuch_db, tags, old=False): fname = os.path.join(maildir._path, maildir._lookup(message_key)) notmuch_msg = notmuch_db.add_message(fname) for tag in tags: - notmuch_msg.add_tag(tag, False) + notmuch_msg.tags.add(tag) # Remove the angle brackets automatically added around the message ID by make_msgid. stripped_msgid = email_message['Message-ID'].strip('<>') @@ -55,7 +56,7 @@ def setUp(self): write_notmuch_settings() # Create notmuch database - Database().open(create=True).close() + notmuch2.Database.create().close() self.root = mailbox.Maildir(self.test_dir) self.inbox = self.root.add_folder('inbox') @@ -88,10 +89,12 @@ def tearDown(self): @staticmethod def get_folder_content(db, folder): - return { - (os.path.basename(msg.get_message_id()), msg.get_part(1).decode()) - for msg in db.do_query('folder:{}'.format(folder)).search_messages() - } + ret = set() + for msg in db.open().messages('folder:{}'.format(folder)): + with open(msg.path) as f: + ret.add((os.path.basename(msg.messageid), + email.message_from_file(f).get_payload())) + return ret def test_all_rule_cases(self): from afew import MailMover diff --git a/afew/utils.py b/afew/utils.py index 8a9f491..04b41e9 100644 --- a/afew/utils.py +++ b/afew/utils.py @@ -6,15 +6,21 @@ def get_message_summary(message): - when = datetime.fromtimestamp(float(message.get_date())) + when = datetime.fromtimestamp(float(message.date)) sender = get_sender(message) - subject = message.get_header('Subject') + try: + subject = message.header('Subject') + except LookupError: + subject = '' return '[{date}] {sender} | {subject}'.format(date=when, sender=sender, subject=subject) def get_sender(message): - sender = message.get_header('From') + try: + sender = message.header('From') + except LookupError: + sender = '' name_match = re.search(r'(.+) <.+@.+\..+>', sender) if name_match: sender = name_match.group(1) diff --git a/docs/extending.rst b/docs/extending.rst index 844f9e6..1d6012a 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -36,7 +36,7 @@ we ensure we don't bother looking at messages we've already looked at. The `handle_message()` method is the key one to implement. This will be called for each message that matches the query. The argument is a `notmuch message object`_ -and the key methods used by the afew filters are `get_header()`, `get_filename()` +and the key methods used by the afew filters are `header()`, `filename()` and `get_thread()`. .. _notmuch message object: http://pythonhosted.org/notmuch/#message-a-single-message diff --git a/setup.py b/setup.py index 2c4b558..78933e8 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def get_requires(): if os.environ.get('TRAVIS') != 'true' and os.environ.get('READTHEDOCS') != 'True': - yield 'notmuch' + yield 'notmuch2' yield 'chardet' yield 'dkimpy'