summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSean Whitton <spwhitton@spwhitton.name>2020-04-19 16:53:48 -0700
committerSean Whitton <spwhitton@spwhitton.name>2020-04-19 16:53:48 -0700
commit73fba2e178b189332ab146b06dbf8bdc16980a31 (patch)
tree975b3b6faa4896152bae27676b548246cb500a92
parent7ed5070972befb67961d5519f719c38b0d4a2222 (diff)
parent5c1db2b078879ca2f7c492d855b080b0707a6ada (diff)
downloadmailscripts-73fba2e178b189332ab146b06dbf8bdc16980a31.tar.gz
Merge tag 'debian/0.19-1' into buster-bpo
mailscripts release 0.19-1 for unstable (sid) [dgit] [dgit distro=debian no-split --quilt=linear] # gpg: Signature made Fri 20 Mar 2020 01:16:48 PM MST # gpg: using RSA key 9B917007AE030E36E4FC248B695B7AE4BF066240 # gpg: Good signature from "Sean Whitton <spwhitton@spwhitton.name>" [ultimate] # Primary key fingerprint: 8DC2 487E 51AB DD90 B5C4 753F 0F56 D055 3B6D 411B # Subkey fingerprint: 9B91 7007 AE03 0E36 E4FC 248B 695B 7AE4 BF06 6240
-rw-r--r--debian/changelog17
-rw-r--r--debian/control2
-rw-r--r--debian/copyright1
-rwxr-xr-ximap-dl107
-rw-r--r--imap-dl.1.pod56
-rwxr-xr-xnotmuch-slurp-debbug106
6 files changed, 197 insertions, 92 deletions
diff --git a/debian/changelog b/debian/changelog
index a869f8b..d9ac182 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,20 @@
+mailscripts (0.19-1) unstable; urgency=medium
+
+ * notmuch-slurp-debbug: rework to use Mail::Box rather than shelling out
+ to scripts to move mail around.
+ - Drop dependency on libmime-tools-perl.
+ * imap-dl:
+ - Update documentation to be less oriented towards former users of
+ getmail (Closes: #953582).
+ Thanks to Daniel Kahn Gillmor for the patch.
+ - Allow specifying ssl_ciphers.
+ Thanks to Robbie Harwood for the patch.
+ - Add support for GSSAPI authentication.
+ Thanks to Robbie Harwood for the patch and Daniel Kahn Gillmor for
+ review.
+
+ -- Sean Whitton <spwhitton@spwhitton.name> Fri, 20 Mar 2020 13:12:58 -0700
+
mailscripts (0.18-1~bpo10+1) buster-backports; urgency=medium
* Rebuild for buster-backports.
diff --git a/debian/control b/debian/control
index 260db76..3f089a0 100644
--- a/debian/control
+++ b/debian/control
@@ -42,7 +42,6 @@ Depends:
libipc-system-simple-perl,
liblist-moreutils-perl,
libmail-box-perl,
- libmime-tools-perl,
python3,
${misc:Depends},
${perl:Depends},
@@ -51,6 +50,7 @@ Recommends:
git,
notmuch,
python3-argcomplete,
+ python3-gssapi,
python3-pgpy,
Suggests:
gnutls-bin,
diff --git a/debian/copyright b/debian/copyright
index 17997a8..db97f3d 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -3,6 +3,7 @@ Collection of scripts for manipulating e-mail on Debian
Copyright (C)2017-2020 Sean Whitton
Copyright (C)2019-2020 Daniel Kahn Gillmor
+Copyright (C)2020 Red Hat, Inc.
These programs are free software: you can redistribute it and/or
modify it under the terms of the GNU General Public License as
diff --git a/imap-dl b/imap-dl
index 469b81d..5a8494c 100755
--- a/imap-dl
+++ b/imap-dl
@@ -3,6 +3,7 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2020 Daniel Kahn Gillmor
+# Copyright (C) 2020 Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -17,22 +18,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-DESCRIPTION = '''A simple replacement for a minimalist use of getmail.
+DESCRIPTION = '''Fetch messages from an IMAP inbox into a maildir
-In particular, if you use getmail to reach an IMAP server as though it
-were POP (retrieving from the server and optionally deleting), you can
-point this script to the getmail config and it should do the same
-thing.
-
-It tries to ensure that the configuration file is of the expected
-type, and will terminate raising an exception, and it should not lose
-messages.
-
-If there's any interest in supporting other similarly simple use cases
-for getmail, patches are welcome.
-
-If you've never used getmail, you can make the simplest possible
-config file like so:
+Example config file:
----------
[retriever]
@@ -46,6 +34,8 @@ path = /home/foo/Maildir
[options]
delete = True
----------
+
+Run "man imap-dl" for more details.
'''
import re
@@ -61,13 +51,19 @@ import argparse
import statistics
import configparser
-from typing import Dict, List, Union, Tuple
+from typing import Dict, List, Optional, Tuple, Union
try:
import argcomplete #type: ignore
except ImportError:
argcomplete = None
+try:
+ import gssapi # type: ignore
+except ModuleNotFoundError:
+ gssapi = None
+
+
class Splitter(object):
def __init__(self, name:str, match:bytes):
self._splitter = re.compile(match)
@@ -91,6 +87,60 @@ summary_splitter = Splitter('summary', _summary_re)
_fetch_re = rb'^(?P<id>[0-9]+) \(UID (?P<uid>[0-9]+) (FLAGS \([\\A-Za-z ]*\) )?BODY\[\] \{(?P<size>[0-9]+)\}$'
fetch_splitter = Splitter('fetch', _fetch_re)
+def auth_builtin(username:str, imap:imaplib.IMAP4_SSL,
+ conf:configparser.ConfigParser, server:str) -> None:
+ logging.info('Logging in as %s', username)
+ resp:Tuple[str, List[Union[bytes,Tuple[bytes,bytes]]]]
+ resp = imap.login(username, conf.get('retriever', 'password'))
+ if resp[0] != 'OK':
+ raise Exception(f'login failed with {resp} as user {username} on {server}')
+
+# imaplib auth methods need to be in the form of callables, and they all
+# requre both additional parameters and storage beyond what the function
+# interface provides.
+class GSSAPI_handler():
+ gss_vc:gssapi.SecurityContext
+ username:str
+
+ def __init__(self, server:str, username:str) -> None:
+ name = gssapi.Name(f'imap@{server}', gssapi.NameType.hostbased_service)
+ self.gss_vc = gssapi.SecurityContext(usage="initiate", name=name)
+ self.username = username
+
+ def __call__(self, token:Optional[bytes]) -> bytes:
+ if token == b"":
+ token = None
+ if not self.gss_vc.complete:
+ response = self.gss_vc.step(token)
+ return response if response else b"" # type: ignore
+ elif token is None:
+ return b""
+
+ response = self.gss_vc.unwrap(token)
+
+ # Preserve the "length" of the message we received, and set the first
+ # byte to one. If username is provided, it's next.
+ reply:List[int] = []
+ reply[0:4] = response.message[0:4]
+ reply[0] = 1
+ if self.username:
+ reply[5:] = self.username.encode("utf-8")
+
+ response = self.gss_vc.wrap(bytes(reply), response.encrypted)
+ return response.message if response.message else b"" # type: ignore
+
+def auth_gssapi(username:str, imap:imaplib.IMAP4_SSL,
+ conf:configparser.ConfigParser, server:str) -> None:
+ if not gssapi:
+ raise Exception('Kerberos requested, but python3-gssapi not found')
+
+ logging.info(f'Logging in as {username} with GSSAPI')
+
+ callback = GSSAPI_handler(server, username)
+ resp = imap.authenticate("GSSAPI", callback)
+ if resp[0] != 'OK':
+ raise Exception(f'GSSAPI login failed with {resp} as user {username} on {server}')
+
def scan_msgs(configfile:str, verbose:bool) -> None:
conf = configparser.ConfigParser()
conf.read_file(open(configfile, 'r'))
@@ -127,16 +177,31 @@ def scan_msgs(configfile:str, verbose:bool) -> None:
'(found "{on_size_mismatch_str}")')
ctx = ssl.create_default_context(cafile=ca_certs)
+ ssl_ciphers = conf.get('retriever', 'ssl_ciphers', fallback=None)
+ if ssl_ciphers:
+ ctx.set_ciphers(ssl_ciphers)
+
server:str = conf.get('retriever', 'server')
with imaplib.IMAP4_SSL(host=server, #type: ignore
port=int(conf.get('retriever', 'port', fallback=993)),
ssl_context=ctx) as imap:
username:str = conf.get('retriever', 'username')
- logging.info('Logging in as %s', username)
- resp:Tuple[str, List[Union[bytes,Tuple[bytes,bytes]]]]
- resp = imap.login(username, conf.get('retriever', 'password'))
- if resp[0] != 'OK':
- raise Exception(f'login failed with {resp} as user {username} on {server}')
+ authentication:str = conf.get('retriever', 'authentication',
+ fallback='basic')
+ # FIXME: have the default automatically choose an opinionated
+ # best authentication method. e.g., if the gssapi module is
+ # installed and the user has a reasonable identity in their
+ # local credential cache, choose kerberos, otherwise, choose
+ # "basic".
+ if authentication in ['kerberos', 'gssapi']:
+ auth_gssapi(username, imap, conf, server)
+ elif authentication == 'basic':
+ auth_builtin(username, imap, conf, server)
+ else:
+ # FIXME: implement other authentication mechanisms
+ raise Exception(f'retriever.authentication should be one of:\n'
+ '"basic" or "gssapi" (or "kerberos"). Got "{authentication}"')
+
if verbose: # only enable debugging after login to avoid leaking credentials in the log
imap.debug = 4
logging.info("capabilities reported: %s", ', '.join(imap.capabilities))
diff --git a/imap-dl.1.pod b/imap-dl.1.pod
index 9fb77c3..402a167 100644
--- a/imap-dl.1.pod
+++ b/imap-dl.1.pod
@@ -2,7 +2,7 @@
=head1 NAME
-imap-dl -- a simple replacement for a minimalist user of getmail
+imap-dl -- fetch messages from an IMAP inbox into a maildir
=head1 SYNOPSIS
@@ -10,35 +10,69 @@ B<imap-dl> [B<-v>|B<--verbose>] B<configfile>...
=head1 DESCRIPTION
+B<imap-dl> tries to retrieve all messages from an IMAP inbox and put
+them in a maildir.
+
If you use getmail to reach an IMAP server as though it were POP
(retrieving from the server, storing it in a maildir and optionally
-deleting), you can point this script to the getmail config and it
-should do the same thing.
+deleting), you can point this script to the getmail configfile and it
+should do the same thing. While the minimal configuration file
+matches the syntax for common getmail configurations, some other
+options might be specific to B<imap-dl>.
-It tries to ensure that the configuration file is of the expected
-type, and otherwise it will terminate with an error. It should not
-lose e-mail messages.
+B<imap-dl> tries to ensure that the configuration file is of the
+expected type, and otherwise it will terminate with an error. It
+should never lose e-mail messages.
If there's any interest in supporting other similarly simple use cases
-for getmail, patches are welcome.
+for fetching messages from an IMAP account into a maildir, patches are
+welcome.
=head1 OPTIONS
B<-v> or B<--verbose> causes B<imap-dl> to print more details
about what it is doing.
+=head1 CONFIGFILE OPTIONS
+
+B<imap-dl> uses an ini-style configfile, with [Sections] which each
+have keyword arguments within them. We use the syntax B<foo.bar> here
+to mean keyword B<bar> in section B<foo>. B<imap-dl> inherits some
+basic configuration options from B<getmail>, including the following
+options:
+
+B<retriever.server> is the dns name of the mailserver.
+
+B<retriever.authentication> is either "basic" (the default, using the
+IMAP LOGIN verb) or "gssapi" (IMAP AUTHENTICATE with GSSAPI, requires
+the python3-gssapi module). "kerberos" is an alias for "gssapi".
+
+B<retriever.username> is the username of the IMAP account.
+
+B<retriever.password> is the password for the IMAP account when
+B<retriever.authentication> is set to "basic".
+
+B<retriever.ssl_ciphers> is an OpenSSL cipher string to use instead of the
+defaults. (The defaults are good; this should be avoided except to work
+around bugs.)
+
+B<destination.path> is the location of the target maildir.
+
+B<options.delete> is a boolean, whether to delete the messages that
+were successfully retreived (default: false).
+
In addition to parts of the standard B<getmail> configuration,
-B<imap-dl> supports the following keywords in the config file:
+B<imap-dl> supports the following keywords in the configfile:
B<options.on_size_mismatch> can be set to B<error>, B<none>, or
B<warn>. This governs what to do when the remote IMAP server claims a
different size in the message summary list than the actual message
retrieval (default: B<error>).
-=head1 EXAMPLE CONFIG
+=head1 EXAMPLE CONFIGFILE
-If you've never used getmail, you can make the simplest possible
-config file like so:
+This configfile fetches all the mail from the given IMAP account's
+inbox, and deletes the messages when they are successfully fetched:
=over 4
diff --git a/notmuch-slurp-debbug b/notmuch-slurp-debbug
index ff5a54f..c187596 100755
--- a/notmuch-slurp-debbug
+++ b/notmuch-slurp-debbug
@@ -2,7 +2,7 @@
# notmuch-slurp-debbug -- add messages from a Debian bug to notmuch
-# Copyright (C) 2018-2019 Sean Whitton
+# Copyright (C) 2018-2020 Sean Whitton
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -23,80 +23,68 @@ use warnings;
use Config::Tiny;
use File::Spec::Functions qw(catfile);
use File::Which;
-use File::Temp;
use Getopt::Long;
use IPC::System::Simple qw(systemx capturex);
-use MIME::Head;
+use Mail::Box::Manager;
-my $Config = Config::Tiny->new;
-
-my $bts_server = undef;
-GetOptions('bts-server=s' => \$bts_server);
-die "notmuch-slurp-debbug: usage: notmuch-slurp-debbug [--bts-server=SERVER] BUG"
- if (scalar @ARGV != 1);
+my $bts = "https://bugs.debian.org";
+GetOptions "bts-server=s" => \$bts;
+die "usage: notmuch-slurp-debbug [--bts-server=SERVER] BUG"
+ unless @ARGV == 1;
die "notmuch-slurp-debbug: this script requires notmuch to be installed"
- unless defined which "notmuch";
+ unless which "notmuch";
die "notmuch-slurp-debbug: this script requires the 'devscripts' apt package"
- unless defined which "bts";
-
-my $maildir;
+ unless which "bts";
my $bug = pop @ARGV;
-my $mailscripts_conf_dir = defined $ENV{'XDG_CONFIG_HOME'}
- ? catfile $ENV{'XDG_CONFIG_HOME'}, "/mailscripts"
- : catfile $ENV{'HOME'}, "/.config/mailscripts";
-
-my $notmuch_slurp_debbug_conf = "$mailscripts_conf_dir/notmuch-slurp-debbug";
-if (-f $notmuch_slurp_debbug_conf) {
- $Config = Config::Tiny->read($notmuch_slurp_debbug_conf);
+my $mgr = Mail::Box::Manager->new;
+my $maildir;
+my $conf_r = $ENV{XDG_CONFIG_HOME} || catfile $ENV{HOME}, ".config";
+my $conf_f = catfile $conf_r, "mailscripts", "notmuch-slurp-debbug";
+if (-f $conf_f) {
+ my $Config = Config::Tiny::read($conf_f);
$maildir = $Config->{_}->{maildir};
} else {
# default to where a lot of people have their inbox
- my $database_path = `notmuch config get database.path`;
- chomp $database_path;
+ chomp(my $database_path = `notmuch config get database.path`);
$maildir = catfile $database_path, "inbox";
}
-
-die "notmuch-slurp-debbug: $maildir does not look to be a maildir"
- unless (-d catfile($maildir, "cur")
- && -d catfile($maildir, "new")
- && -d catfile($maildir, "tmp"));
-
-my @bts_server_args = defined $bts_server
- ? ("--bts-server", $bts_server)
- : undef;
-
-# see #904182 for why we have to do it like this
-my @bts_args = grep defined, @bts_server_args,
- qw(--mbox --mailreader), "true %s", "show", $bug;
-systemx("bts", @bts_args);
-
-my $dir = File::Temp->newdir();
-mkdir catfile($dir, "cur");
-mkdir catfile($dir, "new");
-mkdir catfile($dir, "tmp");
-
-my $devscripts_cache = defined $ENV{'XDG_CACHE_HOME'}
- ? catfile $ENV{'XDG_CACHE_HOME'}, "devscripts", "bts"
- : catfile $ENV{'HOME'}, ".cache", "devscripts", "bts";
-
-my $mbox = catfile $devscripts_cache, "$bug.mbox";
-
-# note that mb2md won't work; it thinks Debian BTS mboxes contain just
-# a single message
-systemx("mbox2maildir", $mbox, $dir);
-
-foreach my $message (glob "$dir/*/*") {
- my $message_head = MIME::Head->from_file($message);
- my $mid = $message_head->get('Message-ID');
+$maildir = $mgr->open(
+ folder => $maildir,
+ access => "a",
+ keep_dups => 1,
+ type => "maildir"
+);
+
+# we use bts(1) to download the mbox because it has some logic to find
+# the right URI and the user might have enabled its caching features.
+# see #904182 for why we invoke it like this
+systemx(
+ qw(bts --bts-server),
+ $bts, qw(--mbox --mailreader),
+ "true %s", "show", $bug
+);
+
+my $cache_r = $ENV{XDG_CACHE_HOME} || catfile $ENV{HOME}, ".cache";
+my $cache_d = catfile $cache_r, "devscripts", "bts";
+my $mbox = $mgr->open(
+ folder => catfile($cache_d, "$bug.mbox"),
+ access => "r",
+ keep_dups => 1,
+ type => "mbox"
+);
+
+foreach my $message ($mbox->messages) {
+ my $mid = $message->messageId;
# if this message does not have a message-id, do not import it;
- # that is asking for trouble
+ # that would be asking for trouble
next unless defined $mid;
$mid =~ s/(<|>)//g;
- my $match = capturex(qw(notmuch search), "id:$mid");
- my $match_lines = $match =~ tr/\n//;
- systemx("mdmv", $message, $maildir) if ($match_lines == 0);
+
+ chomp(my $match = capturex(qw(notmuch search), "id:$mid"));
+
+ $mgr->copyMessage($maildir, $message) unless $match;
}
systemx(qw(notmuch new));