diff options
author | Sean Whitton <spwhitton@spwhitton.name> | 2020-04-19 16:53:48 -0700 |
---|---|---|
committer | Sean Whitton <spwhitton@spwhitton.name> | 2020-04-19 16:53:48 -0700 |
commit | 73fba2e178b189332ab146b06dbf8bdc16980a31 (patch) | |
tree | 975b3b6faa4896152bae27676b548246cb500a92 | |
parent | 7ed5070972befb67961d5519f719c38b0d4a2222 (diff) | |
parent | 5c1db2b078879ca2f7c492d855b080b0707a6ada (diff) | |
download | mailscripts-73fba2e178b189332ab146b06dbf8bdc16980a31.tar.gz |
Merge tag 'debian/0.19-1' into buster-bpo
mailscripts release 0.19-1 for unstable (sid) [dgit]
[dgit distro=debian no-split --quilt=linear]
# gpg: Signature made Fri 20 Mar 2020 01:16:48 PM MST
# gpg: using RSA key 9B917007AE030E36E4FC248B695B7AE4BF066240
# gpg: Good signature from "Sean Whitton <spwhitton@spwhitton.name>" [ultimate]
# Primary key fingerprint: 8DC2 487E 51AB DD90 B5C4 753F 0F56 D055 3B6D 411B
# Subkey fingerprint: 9B91 7007 AE03 0E36 E4FC 248B 695B 7AE4 BF06 6240
-rw-r--r-- | debian/changelog | 17 | ||||
-rw-r--r-- | debian/control | 2 | ||||
-rw-r--r-- | debian/copyright | 1 | ||||
-rwxr-xr-x | imap-dl | 107 | ||||
-rw-r--r-- | imap-dl.1.pod | 56 | ||||
-rwxr-xr-x | notmuch-slurp-debbug | 106 |
6 files changed, 197 insertions, 92 deletions
diff --git a/debian/changelog b/debian/changelog index a869f8b..d9ac182 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,20 @@ +mailscripts (0.19-1) unstable; urgency=medium + + * notmuch-slurp-debbug: rework to use Mail::Box rather than shelling out + to scripts to move mail around. + - Drop dependency on libmime-tools-perl. + * imap-dl: + - Update documentation to be less oriented towards former users of + getmail (Closes: #953582). + Thanks to Daniel Kahn Gillmor for the patch. + - Allow specifying ssl_ciphers. + Thanks to Robbie Harwood for the patch. + - Add support for GSSAPI authentication. + Thanks to Robbie Harwood for the patch and Daniel Kahn Gillmor for + review. + + -- Sean Whitton <spwhitton@spwhitton.name> Fri, 20 Mar 2020 13:12:58 -0700 + mailscripts (0.18-1~bpo10+1) buster-backports; urgency=medium * Rebuild for buster-backports. diff --git a/debian/control b/debian/control index 260db76..3f089a0 100644 --- a/debian/control +++ b/debian/control @@ -42,7 +42,6 @@ Depends: libipc-system-simple-perl, liblist-moreutils-perl, libmail-box-perl, - libmime-tools-perl, python3, ${misc:Depends}, ${perl:Depends}, @@ -51,6 +50,7 @@ Recommends: git, notmuch, python3-argcomplete, + python3-gssapi, python3-pgpy, Suggests: gnutls-bin, diff --git a/debian/copyright b/debian/copyright index 17997a8..db97f3d 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,6 +3,7 @@ Collection of scripts for manipulating e-mail on Debian Copyright (C)2017-2020 Sean Whitton Copyright (C)2019-2020 Daniel Kahn Gillmor +Copyright (C)2020 Red Hat, Inc. These programs are free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -3,6 +3,7 @@ # -*- coding: utf-8 -*- # Copyright (C) 2019-2020 Daniel Kahn Gillmor +# Copyright (C) 2020 Red Hat, Inc. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,22 +18,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. -DESCRIPTION = '''A simple replacement for a minimalist use of getmail. +DESCRIPTION = '''Fetch messages from an IMAP inbox into a maildir -In particular, if you use getmail to reach an IMAP server as though it -were POP (retrieving from the server and optionally deleting), you can -point this script to the getmail config and it should do the same -thing. - -It tries to ensure that the configuration file is of the expected -type, and will terminate raising an exception, and it should not lose -messages. - -If there's any interest in supporting other similarly simple use cases -for getmail, patches are welcome. - -If you've never used getmail, you can make the simplest possible -config file like so: +Example config file: ---------- [retriever] @@ -46,6 +34,8 @@ path = /home/foo/Maildir [options] delete = True ---------- + +Run "man imap-dl" for more details. ''' import re @@ -61,13 +51,19 @@ import argparse import statistics import configparser -from typing import Dict, List, Union, Tuple +from typing import Dict, List, Optional, Tuple, Union try: import argcomplete #type: ignore except ImportError: argcomplete = None +try: + import gssapi # type: ignore +except ModuleNotFoundError: + gssapi = None + + class Splitter(object): def __init__(self, name:str, match:bytes): self._splitter = re.compile(match) @@ -91,6 +87,60 @@ summary_splitter = Splitter('summary', _summary_re) _fetch_re = rb'^(?P<id>[0-9]+) \(UID (?P<uid>[0-9]+) (FLAGS \([\\A-Za-z ]*\) )?BODY\[\] \{(?P<size>[0-9]+)\}$' fetch_splitter = Splitter('fetch', _fetch_re) +def auth_builtin(username:str, imap:imaplib.IMAP4_SSL, + conf:configparser.ConfigParser, server:str) -> None: + logging.info('Logging in as %s', username) + resp:Tuple[str, List[Union[bytes,Tuple[bytes,bytes]]]] + resp = imap.login(username, conf.get('retriever', 'password')) + if resp[0] != 'OK': + raise Exception(f'login failed with {resp} as user {username} on {server}') + +# imaplib auth methods need to be in the form of callables, and they all +# requre both additional parameters and storage beyond what the function +# interface provides. +class GSSAPI_handler(): + gss_vc:gssapi.SecurityContext + username:str + + def __init__(self, server:str, username:str) -> None: + name = gssapi.Name(f'imap@{server}', gssapi.NameType.hostbased_service) + self.gss_vc = gssapi.SecurityContext(usage="initiate", name=name) + self.username = username + + def __call__(self, token:Optional[bytes]) -> bytes: + if token == b"": + token = None + if not self.gss_vc.complete: + response = self.gss_vc.step(token) + return response if response else b"" # type: ignore + elif token is None: + return b"" + + response = self.gss_vc.unwrap(token) + + # Preserve the "length" of the message we received, and set the first + # byte to one. If username is provided, it's next. + reply:List[int] = [] + reply[0:4] = response.message[0:4] + reply[0] = 1 + if self.username: + reply[5:] = self.username.encode("utf-8") + + response = self.gss_vc.wrap(bytes(reply), response.encrypted) + return response.message if response.message else b"" # type: ignore + +def auth_gssapi(username:str, imap:imaplib.IMAP4_SSL, + conf:configparser.ConfigParser, server:str) -> None: + if not gssapi: + raise Exception('Kerberos requested, but python3-gssapi not found') + + logging.info(f'Logging in as {username} with GSSAPI') + + callback = GSSAPI_handler(server, username) + resp = imap.authenticate("GSSAPI", callback) + if resp[0] != 'OK': + raise Exception(f'GSSAPI login failed with {resp} as user {username} on {server}') + def scan_msgs(configfile:str, verbose:bool) -> None: conf = configparser.ConfigParser() conf.read_file(open(configfile, 'r')) @@ -127,16 +177,31 @@ def scan_msgs(configfile:str, verbose:bool) -> None: '(found "{on_size_mismatch_str}")') ctx = ssl.create_default_context(cafile=ca_certs) + ssl_ciphers = conf.get('retriever', 'ssl_ciphers', fallback=None) + if ssl_ciphers: + ctx.set_ciphers(ssl_ciphers) + server:str = conf.get('retriever', 'server') with imaplib.IMAP4_SSL(host=server, #type: ignore port=int(conf.get('retriever', 'port', fallback=993)), ssl_context=ctx) as imap: username:str = conf.get('retriever', 'username') - logging.info('Logging in as %s', username) - resp:Tuple[str, List[Union[bytes,Tuple[bytes,bytes]]]] - resp = imap.login(username, conf.get('retriever', 'password')) - if resp[0] != 'OK': - raise Exception(f'login failed with {resp} as user {username} on {server}') + authentication:str = conf.get('retriever', 'authentication', + fallback='basic') + # FIXME: have the default automatically choose an opinionated + # best authentication method. e.g., if the gssapi module is + # installed and the user has a reasonable identity in their + # local credential cache, choose kerberos, otherwise, choose + # "basic". + if authentication in ['kerberos', 'gssapi']: + auth_gssapi(username, imap, conf, server) + elif authentication == 'basic': + auth_builtin(username, imap, conf, server) + else: + # FIXME: implement other authentication mechanisms + raise Exception(f'retriever.authentication should be one of:\n' + '"basic" or "gssapi" (or "kerberos"). Got "{authentication}"') + if verbose: # only enable debugging after login to avoid leaking credentials in the log imap.debug = 4 logging.info("capabilities reported: %s", ', '.join(imap.capabilities)) diff --git a/imap-dl.1.pod b/imap-dl.1.pod index 9fb77c3..402a167 100644 --- a/imap-dl.1.pod +++ b/imap-dl.1.pod @@ -2,7 +2,7 @@ =head1 NAME -imap-dl -- a simple replacement for a minimalist user of getmail +imap-dl -- fetch messages from an IMAP inbox into a maildir =head1 SYNOPSIS @@ -10,35 +10,69 @@ B<imap-dl> [B<-v>|B<--verbose>] B<configfile>... =head1 DESCRIPTION +B<imap-dl> tries to retrieve all messages from an IMAP inbox and put +them in a maildir. + If you use getmail to reach an IMAP server as though it were POP (retrieving from the server, storing it in a maildir and optionally -deleting), you can point this script to the getmail config and it -should do the same thing. +deleting), you can point this script to the getmail configfile and it +should do the same thing. While the minimal configuration file +matches the syntax for common getmail configurations, some other +options might be specific to B<imap-dl>. -It tries to ensure that the configuration file is of the expected -type, and otherwise it will terminate with an error. It should not -lose e-mail messages. +B<imap-dl> tries to ensure that the configuration file is of the +expected type, and otherwise it will terminate with an error. It +should never lose e-mail messages. If there's any interest in supporting other similarly simple use cases -for getmail, patches are welcome. +for fetching messages from an IMAP account into a maildir, patches are +welcome. =head1 OPTIONS B<-v> or B<--verbose> causes B<imap-dl> to print more details about what it is doing. +=head1 CONFIGFILE OPTIONS + +B<imap-dl> uses an ini-style configfile, with [Sections] which each +have keyword arguments within them. We use the syntax B<foo.bar> here +to mean keyword B<bar> in section B<foo>. B<imap-dl> inherits some +basic configuration options from B<getmail>, including the following +options: + +B<retriever.server> is the dns name of the mailserver. + +B<retriever.authentication> is either "basic" (the default, using the +IMAP LOGIN verb) or "gssapi" (IMAP AUTHENTICATE with GSSAPI, requires +the python3-gssapi module). "kerberos" is an alias for "gssapi". + +B<retriever.username> is the username of the IMAP account. + +B<retriever.password> is the password for the IMAP account when +B<retriever.authentication> is set to "basic". + +B<retriever.ssl_ciphers> is an OpenSSL cipher string to use instead of the +defaults. (The defaults are good; this should be avoided except to work +around bugs.) + +B<destination.path> is the location of the target maildir. + +B<options.delete> is a boolean, whether to delete the messages that +were successfully retreived (default: false). + In addition to parts of the standard B<getmail> configuration, -B<imap-dl> supports the following keywords in the config file: +B<imap-dl> supports the following keywords in the configfile: B<options.on_size_mismatch> can be set to B<error>, B<none>, or B<warn>. This governs what to do when the remote IMAP server claims a different size in the message summary list than the actual message retrieval (default: B<error>). -=head1 EXAMPLE CONFIG +=head1 EXAMPLE CONFIGFILE -If you've never used getmail, you can make the simplest possible -config file like so: +This configfile fetches all the mail from the given IMAP account's +inbox, and deletes the messages when they are successfully fetched: =over 4 diff --git a/notmuch-slurp-debbug b/notmuch-slurp-debbug index ff5a54f..c187596 100755 --- a/notmuch-slurp-debbug +++ b/notmuch-slurp-debbug @@ -2,7 +2,7 @@ # notmuch-slurp-debbug -- add messages from a Debian bug to notmuch -# Copyright (C) 2018-2019 Sean Whitton +# Copyright (C) 2018-2020 Sean Whitton # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,80 +23,68 @@ use warnings; use Config::Tiny; use File::Spec::Functions qw(catfile); use File::Which; -use File::Temp; use Getopt::Long; use IPC::System::Simple qw(systemx capturex); -use MIME::Head; +use Mail::Box::Manager; -my $Config = Config::Tiny->new; - -my $bts_server = undef; -GetOptions('bts-server=s' => \$bts_server); -die "notmuch-slurp-debbug: usage: notmuch-slurp-debbug [--bts-server=SERVER] BUG" - if (scalar @ARGV != 1); +my $bts = "https://bugs.debian.org"; +GetOptions "bts-server=s" => \$bts; +die "usage: notmuch-slurp-debbug [--bts-server=SERVER] BUG" + unless @ARGV == 1; die "notmuch-slurp-debbug: this script requires notmuch to be installed" - unless defined which "notmuch"; + unless which "notmuch"; die "notmuch-slurp-debbug: this script requires the 'devscripts' apt package" - unless defined which "bts"; - -my $maildir; + unless which "bts"; my $bug = pop @ARGV; -my $mailscripts_conf_dir = defined $ENV{'XDG_CONFIG_HOME'} - ? catfile $ENV{'XDG_CONFIG_HOME'}, "/mailscripts" - : catfile $ENV{'HOME'}, "/.config/mailscripts"; - -my $notmuch_slurp_debbug_conf = "$mailscripts_conf_dir/notmuch-slurp-debbug"; -if (-f $notmuch_slurp_debbug_conf) { - $Config = Config::Tiny->read($notmuch_slurp_debbug_conf); +my $mgr = Mail::Box::Manager->new; +my $maildir; +my $conf_r = $ENV{XDG_CONFIG_HOME} || catfile $ENV{HOME}, ".config"; +my $conf_f = catfile $conf_r, "mailscripts", "notmuch-slurp-debbug"; +if (-f $conf_f) { + my $Config = Config::Tiny::read($conf_f); $maildir = $Config->{_}->{maildir}; } else { # default to where a lot of people have their inbox - my $database_path = `notmuch config get database.path`; - chomp $database_path; + chomp(my $database_path = `notmuch config get database.path`); $maildir = catfile $database_path, "inbox"; } - -die "notmuch-slurp-debbug: $maildir does not look to be a maildir" - unless (-d catfile($maildir, "cur") - && -d catfile($maildir, "new") - && -d catfile($maildir, "tmp")); - -my @bts_server_args = defined $bts_server - ? ("--bts-server", $bts_server) - : undef; - -# see #904182 for why we have to do it like this -my @bts_args = grep defined, @bts_server_args, - qw(--mbox --mailreader), "true %s", "show", $bug; -systemx("bts", @bts_args); - -my $dir = File::Temp->newdir(); -mkdir catfile($dir, "cur"); -mkdir catfile($dir, "new"); -mkdir catfile($dir, "tmp"); - -my $devscripts_cache = defined $ENV{'XDG_CACHE_HOME'} - ? catfile $ENV{'XDG_CACHE_HOME'}, "devscripts", "bts" - : catfile $ENV{'HOME'}, ".cache", "devscripts", "bts"; - -my $mbox = catfile $devscripts_cache, "$bug.mbox"; - -# note that mb2md won't work; it thinks Debian BTS mboxes contain just -# a single message -systemx("mbox2maildir", $mbox, $dir); - -foreach my $message (glob "$dir/*/*") { - my $message_head = MIME::Head->from_file($message); - my $mid = $message_head->get('Message-ID'); +$maildir = $mgr->open( + folder => $maildir, + access => "a", + keep_dups => 1, + type => "maildir" +); + +# we use bts(1) to download the mbox because it has some logic to find +# the right URI and the user might have enabled its caching features. +# see #904182 for why we invoke it like this +systemx( + qw(bts --bts-server), + $bts, qw(--mbox --mailreader), + "true %s", "show", $bug +); + +my $cache_r = $ENV{XDG_CACHE_HOME} || catfile $ENV{HOME}, ".cache"; +my $cache_d = catfile $cache_r, "devscripts", "bts"; +my $mbox = $mgr->open( + folder => catfile($cache_d, "$bug.mbox"), + access => "r", + keep_dups => 1, + type => "mbox" +); + +foreach my $message ($mbox->messages) { + my $mid = $message->messageId; # if this message does not have a message-id, do not import it; - # that is asking for trouble + # that would be asking for trouble next unless defined $mid; $mid =~ s/(<|>)//g; - my $match = capturex(qw(notmuch search), "id:$mid"); - my $match_lines = $match =~ tr/\n//; - systemx("mdmv", $message, $maildir) if ($match_lines == 0); + + chomp(my $match = capturex(qw(notmuch search), "id:$mid")); + + $mgr->copyMessage($maildir, $message) unless $match; } systemx(qw(notmuch new)); |