From 8d6ef730fb7cb884c037b660fb81edfcffb4e51c Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Tue, 28 Aug 2007 09:57:41 +0200 Subject: [PATCH 01/16] lower max msg size limit for spamfilter to 512k, the spamc default --- procmail/defines | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmail/defines b/procmail/defines index 516afa6..9b404de 100644 --- a/procmail/defines +++ b/procmail/defines @@ -44,7 +44,7 @@ DEFAULT=$BASE/ ORGMAIL=$HOME/BOUNCED-MAIL # maximum message size for spam checking -SPAMCHECK_MAX_MESSAGE_SIZE=2500000 +SPAMCHECK_MAX_MESSAGE_SIZE=512000 # if crm114 is unsure and SA returns a score less-than-or-equal to this, # autotrain crm114 with ham -- 2.39.2 From 1bf3ee58d7f262b15c2cb9eecf4058ef4844ee06 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Tue, 28 Aug 2007 11:54:20 +0200 Subject: [PATCH 02/16] kill ul ponotices --- procmail/rules/ul | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmail/rules/ul b/procmail/rules/ul index 9c04de4..27af1c5 100644 --- a/procmail/rules/ul +++ b/procmail/rules/ul @@ -6,7 +6,7 @@ *$ 1^0 ^TO_Events@$RE_UL_DOMAIN *$ 1^0 ^TO_Bereavements@$RE_UL_DOMAIN *$ 1^0 ^TO_Timetables@$RE_UL_DOMAIN - *$ 1^0 ^TO_(HR|BE)Notices@$RE_UL_DOMAIN + *$ 1^0 ^TO_(HR|BE|PO)Notices@$RE_UL_DOMAIN *$ 1^0 ^TO_Opinions@$RE_UL_DOMAIN *$ 1^0 ^TO_AllStaff@$RE_UL_DOMAIN *$ 1^0 ^TO_ResearchPostgraduates@$RE_UL_DOMAIN -- 2.39.2 From 97551d10e98e6d47b588f665041085bd548edc97 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Tue, 28 Aug 2007 14:14:24 +0200 Subject: [PATCH 03/16] make tickle processing more robust --- procmail/tickle | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/procmail/tickle b/procmail/tickle index 8ad5ac9..672358d 100644 --- a/procmail/tickle +++ b/procmail/tickle @@ -2,30 +2,38 @@ * ORIGINAL_TO ?? ^.+@tickle\.madduck\.net$ * ORIGINAL_TO ?? ^\/[^@]+ { + BIN_DATE=/bin/date + DELIVERY_TIME=04:20 STAMP + :0 * MATCH ?? ^[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]$ - { STAMP="$MATCH (`/bin/date -d @$MATCH +%c`)" } + { STAMP="$MATCH (`$BIN_DATE -d @$MATCH +%c`)" } + + :0 E + * MATCH ?? ^[0-9]+h$ + * MATCH ?? ^\/[^h]+ + { DATESTR="$DATE + $MATCH hours" } + + JUST_DATE="`$BIN_DATE -d \"$DATE\" +%x`" :0 E * MATCH ?? ^[0-9]+m$ * MATCH ?? ^\/[^m]+ - { DATESTR="$DATE + $MATCH months 00:00" } + { DATESTR="$JUST_DATE + $MATCH months $DELIVERY_TIME" } :0 E * MATCH ?? ^[0-9]+w$ * MATCH ?? ^\/[^w]+ - { DATESTR="$DATE + $MATCH weeks 00:00" } + { DATESTR="$JUST_DATE + $MATCH weeks $DELIVERY_TIME" } :0 E * MATCH ?? ^[0-9]+d?$ * MATCH ?? ^\/[^d]+ - { DATESTR="$DATE + $MATCH days 00:00" } + { DATESTR="$JUST_DATE + $MATCH days $DELIVERY_TIME" } - :0 E - * MATCH ?? ^[0-9]+h$ - * MATCH ?? ^\/[^h]+ - { DATESTR="$DATE + $MATCH hours" } + JUST_DATE + DELIVERY_TIME :0 E { @@ -36,13 +44,23 @@ :0 * ! STAMP ?? . - { STAMP=`/bin/date +'%s (%c)' -d "$DATESTR"` } + { STAMP=`$BIN_DATE +'%s (%c)' -d "$DATESTR"` } + + :0 + * ! STAMP ?? ^[0-9]+ \([A-Za-z ]+ [0-9 :]+\)$ + { + LOG="tickle: error creating timestamp ($STAMP), using \$now$NL" + STAMP="`$BIN_DATE +%s` (ERROR: original stamp '$STAMP' invalid)" + } - LOG="tickle: stamping message $MSGID with $STAMP$NL" + :0 E + { LOG="tickle: stamping message $MSGID with $STAMP$NL" } :0 fw |$FORMAIL -i "X-Tickle: $STAMP" :0 $BASE/.tickler/ + + BIN_DATE } -- 2.39.2 From 84b9aa9ec86b40cfaabb816bccf1511a06e95661 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Tue, 28 Aug 2007 16:49:01 +0200 Subject: [PATCH 04/16] also let netconf-commits pass justme --- config/justme-address-exceptions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/justme-address-exceptions b/config/justme-address-exceptions index 60ec5db..2336aa1 100644 --- a/config/justme-address-exceptions +++ b/config/justme-address-exceptions @@ -11,7 +11,7 @@ ^logcheck-devel=lists\.alioth\.debian\.org@mass\.madduck\.net$ ^pkg-mdadm-devel=lists\.alioth\.debian\.org@mass\.madduck\.net$ ^debconf-(team|press)=lists\.debconf\.org@mass\.madduck\.net$ -^netconf-devel=lists\.alioth\.debian\.org@mass\.madduck\.net$ +^netconf-(devel|commits)=lists\.alioth\.debian\.org@mass\.madduck\.net$ ^packaging-handbook-project=lists\.alioth\.debian\.org@mass\.madduck\.net$ ^music=(yarn\.)?skein\.org@mass\.madduck\.net$ ^slug=sccs\.swarthmore\.edu@mass\.madduck\.net$ -- 2.39.2 From 39c4720c11712d9bf5bb99bc91eb73539b9067c1 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Wed, 29 Aug 2007 16:15:27 +0200 Subject: [PATCH 05/16] replace existing x-tickle-delivered header --- bin/process-tickler | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/process-tickler b/bin/process-tickler index 0c3839f..d7af9b6 100755 --- a/bin/process-tickler +++ b/bin/process-tickler @@ -24,7 +24,10 @@ msgids = dict() def move_to_inbox(key): msg = tmd.get_message(key) print >>sys.stdout, 'I: move message %s to inbox' % key - msg.add_header('X-Tickle-Delivered', time.strftime('%c')) + if msg.has_key('X-Tickle-Delivered'): + msg.replace_header('X-Tickle-Delivered', time.strftime('%c')) + else: + msg.add_header('X-Tickle-Delivered', time.strftime('%c')) imd.add(msg) tmd.discard(key) -- 2.39.2 From fbffc571b617642282d9dea5d8ec308dbfce9d4c Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Wed, 29 Aug 2007 18:25:53 +0200 Subject: [PATCH 06/16] filter riss@ul --- procmail/rules/ul | 1 + 1 file changed, 1 insertion(+) diff --git a/procmail/rules/ul b/procmail/rules/ul index 27af1c5..e88e8f9 100644 --- a/procmail/rules/ul +++ b/procmail/rules/ul @@ -12,6 +12,7 @@ *$ 1^0 ^TO_ResearchPostgraduates@$RE_UL_DOMAIN *$ 1^0 ^TO_NewsandOther@$RE_UL_DOMAIN *$ 1^0 ^TO_ForSaleorWanted@$RE_UL_DOMAIN + *$ 1^0 ^TO_ResearchInformationRISS@$RE_UL_DOMAIN $DISCARD RE_UL_DOMAIN -- 2.39.2 From 8737f8da7a7514d661f6632a5947afee0cbfdd3d Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Wed, 12 Sep 2007 12:41:41 +0200 Subject: [PATCH 07/16] ignore most ailab lists --- config/justme-address-exceptions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/justme-address-exceptions b/config/justme-address-exceptions index 2336aa1..fa2b9b3 100644 --- a/config/justme-address-exceptions +++ b/config/justme-address-exceptions @@ -1,7 +1,7 @@ =lists\.madduck\.net@mass\.madduck\.net$ -=lists\.ailab\.ch@mass\.madduck\.net$ =sudetia\.de@mass\.madduck\.net$ -announce=[^@]+@mass\.madduck\.net$ +^(curiosa|cluster-.+|events|lib.+)=lists\.ailab\.ch@mass\.madduck\.net$ ^debian-private=lists\.debian\.org@mass.madduck.net$ ^jppy=zanu\.org\.uk@mass\.madduck\.net$ ^os-renet=insel\.cs\.tu-berlin\.de@mass\.madduck\.net$ -- 2.39.2 From 8800da281e9f2341b55bc8fab4afe3961079a656 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Wed, 12 Sep 2007 16:14:20 +0200 Subject: [PATCH 08/16] add more keywords for debian book --- config/justme-keyword-exceptions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/justme-keyword-exceptions b/config/justme-keyword-exceptions index 5e66e6c..ed5d2c5 100644 --- a/config/justme-keyword-exceptions +++ b/config/justme-keyword-exceptions @@ -1,6 +1,8 @@ mdadm the debian system -debiansystem +debian-?(system|book) +debian book +book (on|about) debian method diffusion z([üu]|ue)rich logcheck -- 2.39.2 From a0f03dbfefec79510c486c42c79a9e4997e0d740 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Thu, 13 Sep 2007 14:49:00 +0200 Subject: [PATCH 09/16] proper quotes to allow variable substitution --- procmail/spammers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmail/spammers b/procmail/spammers index f1a1fa5..a41e110 100644 --- a/procmail/spammers +++ b/procmail/spammers @@ -10,7 +10,7 @@ SPAMMER :0 * SPAMMER ?? @ { - SPAMTRAPPED='from known spammer ($SPAMMER)' + SPAMTRAPPED="from known spammer ($SPAMMER)" LOG="spammers: $SPAMTRAPPED$NL" } SPAMMER -- 2.39.2 From 9560f41b30a0460ff3d88313eead30039ac9ecdf Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 16 Sep 2007 11:58:40 +0200 Subject: [PATCH 10/16] add sqlite msgid index --- procmail/msgid-index | 27 +++++++++++++++++++++++++++ procmail/procmailrc | 1 + 2 files changed, 28 insertions(+) create mode 100644 procmail/msgid-index diff --git a/procmail/msgid-index b/procmail/msgid-index new file mode 100644 index 0000000..cef6476 --- /dev/null +++ b/procmail/msgid-index @@ -0,0 +1,27 @@ +# create table logfiles (sender text not null, recipient text not null, +# subject text not null, msgid text unique not null, filename text primary key +# not null unique, timestamp timestamp with time zone); +# +# create trigger insert_logfiles_timestamp after insert on logfiles begin +# update logfiles set timestamp = datetime('now') where filename +# = new.filename; end; + +SQLITE=/usr/bin/sqlite3 +MSGID_INDEX_DB=$PMVAR/msgid-index.sqlite + +DO_QUERY="$SQLITE $MSGID_INDEX_DB" + +SQ="'\\''" +SQE="'\\\''" +FIELDS="$FROM$NL$ORIGINAL_TO$NL$SUBJECT$NL$MSGID" +QUERY="insert into logfiles (sender, recipient, subject, msgid, filename) values ( + `echo \"$FIELDS\" | sed -e 's,\\\",\\\\\",g; s,'\'','$SQE$SQE',g; s/^.*$/'\$SQE'&'\$SQE',/'` + $SQ$THISLOGFILE_REL$SQ)" +TRAP="$TRAP; $DO_QUERY '$QUERY'" + +QUERY +FIELDS +SQE +SQ +DO_QUERY +MSGID_INDEX_DB diff --git a/procmail/procmailrc b/procmail/procmailrc index 282b6d8..e1f817d 100755 --- a/procmail/procmailrc +++ b/procmail/procmailrc @@ -17,6 +17,7 @@ INCLUDERC=$PMDIR/smsmail #VERBOSE=yes INCLUDERC=$PMDIR/logging +INCLUDERC=$PMDIR/msgid-index INCLUDERC=$PMDIR/dejavu INCLUDERC=$PMDIR/eqdomains -- 2.39.2 From e60c43899788940d1beab52e038cedc3ad6f8c33 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 16 Sep 2007 12:01:10 +0200 Subject: [PATCH 11/16] no need to quote double quotes now --- procmail/msgid-index | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmail/msgid-index b/procmail/msgid-index index cef6476..b7f2109 100644 --- a/procmail/msgid-index +++ b/procmail/msgid-index @@ -15,7 +15,7 @@ SQ="'\\''" SQE="'\\\''" FIELDS="$FROM$NL$ORIGINAL_TO$NL$SUBJECT$NL$MSGID" QUERY="insert into logfiles (sender, recipient, subject, msgid, filename) values ( - `echo \"$FIELDS\" | sed -e 's,\\\",\\\\\",g; s,'\'','$SQE$SQE',g; s/^.*$/'\$SQE'&'\$SQE',/'` + `echo \"$FIELDS\" | sed -e 's,'\'','$SQE$SQE',g; s/^.*$/'\$SQE'&'\$SQE',/'` $SQ$THISLOGFILE_REL$SQ)" TRAP="$TRAP; $DO_QUERY '$QUERY'" -- 2.39.2 From 45925ed89f43f353b49798d2f4a68f02fb7855b0 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 16 Sep 2007 12:23:45 +0200 Subject: [PATCH 12/16] also log final destination to database --- procmail/logging | 3 ++- procmail/msgid-index | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/procmail/logging b/procmail/logging index d683e49..d812032 100644 --- a/procmail/logging +++ b/procmail/logging @@ -20,7 +20,8 @@ from: $FROM original-to: $ORIGINAL_TO subject: $SUBJECT$NL" -TRAP="echo \"delivered: \${LASTFOLDER#$BASE/}\" >> '$THISLOGFILE'; +TRAP="DEST=\"\${LASTFOLDER#$BASE/}\"; + echo \"delivered: \$DEST\" >> '$THISLOGFILE'; echo >> '$THISLOGFILE'; egrep -v '^XX(procmail:)?[[:space:]\"]' '$THISLOGFILE' >> $MASTERLOGFILE" diff --git a/procmail/msgid-index b/procmail/msgid-index index b7f2109..1d31f0a 100644 --- a/procmail/msgid-index +++ b/procmail/msgid-index @@ -1,6 +1,6 @@ # create table logfiles (sender text not null, recipient text not null, # subject text not null, msgid text unique not null, filename text primary key -# not null unique, timestamp timestamp with time zone); +# not null unique, timestamp timestamp with time zone, dest text not null); # # create trigger insert_logfiles_timestamp after insert on logfiles begin # update logfiles set timestamp = datetime('now') where filename @@ -14,9 +14,9 @@ DO_QUERY="$SQLITE $MSGID_INDEX_DB" SQ="'\\''" SQE="'\\\''" FIELDS="$FROM$NL$ORIGINAL_TO$NL$SUBJECT$NL$MSGID" -QUERY="insert into logfiles (sender, recipient, subject, msgid, filename) values ( +QUERY="insert into logfiles (sender, recipient, subject, msgid, filename, dest) values ( `echo \"$FIELDS\" | sed -e 's,'\'','$SQE$SQE',g; s/^.*$/'\$SQE'&'\$SQE',/'` - $SQ$THISLOGFILE_REL$SQ)" + $SQ$THISLOGFILE_REL$SQ, '\\'\$DEST\\'')" TRAP="$TRAP; $DO_QUERY '$QUERY'" QUERY -- 2.39.2 From e11fbc3da82836b4edb0d8350075c2461ebc2f00 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 23 Sep 2007 12:50:47 +0200 Subject: [PATCH 13/16] add package filters for mdadm,logcheck,netconf --- procmail/rules/debian | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/procmail/rules/debian b/procmail/rules/debian index e11dec0..f5442e9 100644 --- a/procmail/rules/debian +++ b/procmail/rules/debian @@ -3,6 +3,18 @@ * ? grep -q "^${MATCH}$" $CONF/debian-packages $BASE/.debian.$MATCH/ +:0 +* ORIGINAL_TO ?? ^logcheck-(devel|commits)=lists\.alioth\.debian\.org@mass\.madduck\.net$ +$BASE/.debian.logcheck/ + +:0 +* ORIGINAL_TO ?? ^pkg-mdadm-(devel|commits)=lists\.alioth\.debian\.org@mass\.madduck\.net$ +$BASE/.debian.mdadm/ + +:0 +* ORIGINAL_TO ?? ^netconf-(devel|commits)=lists\.alioth\.debian\.org@mass\.madduck\.net$ +$BASE/.debian.netconf/ + :0 * ORIGINAL_TO ?? ^madduck=debian\.org@forward\.madduck\.net$ $BASE/.debian/ -- 2.39.2 From 977bf4ebd9d06d5ee7c493db630f53163e1525a3 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 23 Sep 2007 17:03:01 +0200 Subject: [PATCH 14/16] cleanup headers we add earlier in the chain --- procmail/cleanup | 6 ++++++ procmail/pre-spam-cleanup | 2 +- procmail/procmailrc | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 procmail/cleanup diff --git a/procmail/cleanup b/procmail/cleanup new file mode 100644 index 0000000..646d2ab --- /dev/null +++ b/procmail/cleanup @@ -0,0 +1,6 @@ +# remove all headers our mailfilter adds +:0 fw +|$FORMAIL -IX-EqDomain -IX-EqTo -IX-Justme -IContent-Length + +# now should be idempotent + diff --git a/procmail/pre-spam-cleanup b/procmail/pre-spam-cleanup index e360aad..93fee2a 100755 --- a/procmail/pre-spam-cleanup +++ b/procmail/pre-spam-cleanup @@ -26,6 +26,6 @@ PMDIR=${PMDIR:-$HOME/.etc/mailfilter/procmail} # remove other unwanted headers from all messages :0 fw |$FORMAIL -IX-Virus -IX-UID -IX-OfflineIMAP- -IX-Status -IStatus -IX-Keywords -IX-Greylist \ - -IX-Spam -IX-CRM114 -IX-Auto -IX-Trained-As -IX-EqDomain -IX-EqTo -IX-Justme + -IX-Spam -IX-CRM114 -IX-Auto -IX-Trained-As # now should be idempotent diff --git a/procmail/procmailrc b/procmail/procmailrc index e1f817d..15ef1fb 100755 --- a/procmail/procmailrc +++ b/procmail/procmailrc @@ -20,6 +20,7 @@ INCLUDERC=$PMDIR/logging INCLUDERC=$PMDIR/msgid-index INCLUDERC=$PMDIR/dejavu +INCLUDERC=$PMDIR/cleanup INCLUDERC=$PMDIR/eqdomains INCLUDERC=$PMDIR/duplicates -- 2.39.2 From f133cb12ac933b7a2fe3849938621cc59cdab43b Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 23 Sep 2007 17:03:26 +0200 Subject: [PATCH 15/16] also skip spamfiltering for reinjected messages --- procmail/spamfilter | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/procmail/spamfilter b/procmail/spamfilter index 2ffec51..95a0cbc 100755 --- a/procmail/spamfilter +++ b/procmail/spamfilter @@ -30,15 +30,20 @@ PMDIR=${PMDIR:-$HOME/.etc/mailfilter/procmail} #} # check whether this message is being reinjected -TRAINED_AS :0 *$ $MSG_DEJAVU -* ^X-Trained-As: \/(h|sp)am { - LOG="spamfilter: skipping already trained $MATCH$NL" :0 - * MATCH ?? spam - { IS_SPAM=already-trained } + * ^X-Trained-As: \/(h|sp)am + { + LOG="spamfilter: skipping already trained $MATCH$NL" + :0 + * MATCH ?? spam + { IS_SPAM=already-trained } + } + + :0 E + { LOG="spamfilter: skipping reinjected message$NL" } } # let earlier parts of the mailfilter cause bypassing the checks -- 2.39.2 From ce01c73dbc58af7f829bbed24b3547e7ff7588e4 Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Sun, 23 Sep 2007 17:03:50 +0200 Subject: [PATCH 16/16] prepare tickler for resubmitting, instead of inbox writing --- bin/process-tickler | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/process-tickler b/bin/process-tickler index d7af9b6..71f750a 100755 --- a/bin/process-tickler +++ b/bin/process-tickler @@ -15,32 +15,32 @@ import time HOME = os.getenv('HOME') MAILDIR = os.path.join(HOME, '.maildir') TICKLER_DIR = os.path.join(MAILDIR, '.tickler') -INBOX_DIR = MAILDIR +DEST_DIR = os.path.join(MAILDIR, '.resubmit') tmd = mailbox.Maildir(TICKLER_DIR) -imd = mailbox.Maildir(INBOX_DIR) +dmd = mailbox.Maildir(DEST_DIR) msgids = dict() -def move_to_inbox(key): +def resubmit(key): msg = tmd.get_message(key) - print >>sys.stdout, 'I: move message %s to inbox' % key + print >>sys.stdout, 'I: resubmit message %s' % key if msg.has_key('X-Tickle-Delivered'): msg.replace_header('X-Tickle-Delivered', time.strftime('%c')) else: msg.add_header('X-Tickle-Delivered', time.strftime('%c')) - imd.add(msg) + dmd.add(msg) tmd.discard(key) for key, msg in tmd.iteritems(): msgid = msg.get('Message-Id') if msgid is None: print >>sys.stderr, 'W: message without ID: ' + key - move_to_inbox(key) + resubmit(key) tickle = msg.get('X-Tickle') if tickle is None: print >>sys.stderr, 'W: message without tickle information: ' + key - move_to_inbox(key) + resubmit(key) if msgids.get(msgid, None) is None: msgids[msgid] = list() @@ -58,7 +58,7 @@ for msgid, msgs in msgids.iteritems(): t = time.time() if t >= tickle: - move_to_inbox(key) + resubmit(key) else: print >>sys.stdout, 'I: message %s still has %d seconds' % (key, int(tickle - t)) -- 2.39.2