Gentoo mailfiltering gateway guide

Code Listing 2.1: Installing Amavis, Spamassassin and Clamav

# emerge amavisd-new spamassassin clamav 

Code Listing 2.2: Setting up DNS

(Create a MX record for the gateway server)
                MX      10      mailgateway.mydomain.tld.
(Create an A record for the gateway server)
mailgateway     A       mgw.ip.add.here
(Create an A record for the next hop mail server)
mail            A       ms.ip.add.here

Code Listing 2.3: Changing the master.cf file

smtp-amavis     unix -        -       n     -       2  smtp
  -o smtp_data_done_timeout=1200
  -o smtp_send_xforward_command=yes
#Equivalently when using lmtp:
#lmtp-amavis    unix -        -       n     -       2  lmtp
#   -o lmtp_data_done_timeout=1200
#   -o lmtp_send_xforward_command=yes
     
127.0.0.1:10025 inet n        -       n     -       -  smtpd
  -o content_filter=
  -o local_recipient_maps=
  -o relay_recipient_maps=
  -o smtpd_restriction_classes=
  -o smtpd_client_restrictions=
  -o smtpd_helo_restrictions=
  -o smtpd_sender_restrictions=
  -o smtpd_recipient_restrictions=permit_mynetworks,reject
  -o mynetworks=127.0.0.0/8
  -o strict_rfc821_envelopes=yes
  -o smtpd_error_sleep_time=0
  -o smtpd_soft_error_limit=1001
  -o smtpd_hard_error_limit=1000

#If you want to use proxy filtering instead
#smtp            inet n         -       n      -       8 smtpd
# -o smtpd_proxy_filter=127.0.0.1:10024
# -o smtpd_client_connection_count_limit=4
#If you don't want to scan outgoing mail use this
#10.0.0.2:smtp   inet n         -       n       -      - smtpd
#-o content_filter=

Code Listing 2.4: Modifying /etc/postfix/main.cf

biff = no
empty_address_recipient = MAILER-DAEMON
queue_minfree = 120000000

content_filter = smtp-amavis:[127.0.0.1]:10024
#Equivalently when using lmtp:
#content_filter = lmtp-amavis:[127.0.0.1]:10024

# TRANSPORT MAP
#
# Insert text from sample-transport.cf if you need explicit routing.
transport_maps = hash:/etc/postfix/transport

relay_domains = $transport_maps

Code Listing 2.5: /etc/postfix/transport

mydomain.tld                          smtp:mail.mydomain.tld

Code Listing 2.6: Enabling soft_bounce

# postconf -e "soft_bounce = yes"
# /etc/init.d/postfix reload

Code Listing 2.7: Editing /etc/amavisd.conf

(Insert the domains to be scanned)
$mydomain = 'example.com';     
(Bind only to loopback interface)
$inet_socket_bind = '127.0.0.1';
(Forward to Postfix on port 10025)
$forward_method = 'smtp:127.0.0.1:10025';
$notify_method = $forward_method;       
(Define the account to send virus alert emails)
$virus_admin = "virusalert\@$mydomain";
(Always add spam headers)
$sa_tag_level_deflt  = -100; 
(Add spam detected header aka X-Spam-Status: Yes)
$sa_tag2_level_deflt = 5; 
(Trigger evasive action at this spam level)
$sa_kill_level_deflt = $sa_tag2_level_deflt;
(Do not send delivery status notification to sender.  It does not affect 
delivery of spam to recipient. To do that, use the kill_level)
$sa_dsn_cutoff_level = 10;
Don't bounce messages left and right, quarantine
instead
$final_virus_destiny      = D_DISCARD;  # (defaults to D_DISCARD)
$final_banned_destiny     = D_DISCARD;  # (defaults to D_BOUNCE)
$final_spam_destiny       = D_DISCARD;  # (defaults to D_BOUNCE)

Code Listing 2.8: Create a quarantine directory for the virus mails

# mkdir /var/amavis/virusmails
# chown amavis:amavis /var/amavis/virusmails
# chmod 750 /var/amavis/virusmails

Code Listing 2.9: Edit /etc/clamd.conf

(Verbose logging with syslog)
LogSyslog
LogVerbose
LogFacility LOG_MAIL
(Change pid file location) 
PidFile /var/run/amavis/clamd.pid
(Set the clamav socket) 
LocalSocket /var/amavis/clamd
(Close the connection when this limit is exceeded)
StreamMaxLength 10M
(Don't run clamd as root)
User amavis
(Newer versions require you to uncomment this)
ScanMail
ScanArchive

Code Listing 2.10: Edit /etc/freshclam.conf

(Syslog logging)
LogSyslog
(Verbose logging)
LogVerbose
(Explicitly drop root privileges)
DatabaseOwner clamav
(Check for updates every two hours. That is the official recommendation)
Checks 12
(Use the mirror closest to you. Replace XY with your country code
DatabaseMirror db.XY.clamav.net

Code Listing 2.11: Modifying /etc/conf.d/clamd

START_CLAMD=yes
FRESHCLAM_OPTS="-d"

Code Listing 2.12: Modifying /etc/amavisd.conf

(Uncomment the clamav scanner and modify socket location)
['ClamAV-clamd',
\&ask_daemon, ["CONTSCAN {}\n", "/var/amavis/clamd"],
  qr/\bOK$/, qr/\bFOUND$/,
  qr/^.*?: (?!Infected Archive)(.*) FOUND$/ ],

Code Listing 2.13: Creating the required configuration files

# emerge razor

(Temporarily set amavis' shell to bash)
# usermod -s /bin/bash amavis
# su - amavis
$ razor-admin -create
$ exit

(Reset the shell to /bin/false)
# usermod -s /bin/false amavis

Code Listing 2.14: Installing DCC

# emerge dcc

Code Listing 2.15: Create /etc/mail/spamassassin/local.cf

# Enable the Bayes system
use_bayes               1

# Enable all network checks
skip_rbl_checks         0

# Mail using languages used in these country codes will not be marked
# as being possibly spam in a foreign language.
# - danish english norwegian swedish
ok_languages            da en no sv

# Mail using locales used in these country codes will not be marked
# as being possibly spam in a foreign language.
ok_locales              en

# Use a sensible bayes path
bayes_path              /var/amavis/.spamassassin/bayes

Code Listing 3.1: Modifying amavisd.conf to do sitewide scoring

read_hash("/var/amavis/sender_scores_sitewide"),

Code Listing 3.2: whitelist_sender example

(Whitelist all emails from the specific email address)
postmaster@example.net                -3.0
(Whitelist all emails from the example.net excluding subdomains) 
.example.net                          1.0

Code Listing 3.3: By pass spam filters for all postmaster and abuse mails

map { $bypass_spam_checks{lc($_)}=1 } (qw(
        postmaster@
        abuse@
));

Code Listing 5.1: Testing freshclam

# freshclam
ClamAV update process started at Sun May  2 09:13:41 2004
Reading CVD header (main.cvd): OK
Downloading main.cvd [*]
main.cvd updated (version: 22, sigs: 20229, f-level: 1, builder: tkojm)
Reading CVD header (daily.cvd): OK
Downloading daily.cvd [*]
daily.cvd updated (version: 298, sigs: 1141, f-level: 2, builder: diego)
Database updated (21370 signatures) from database.clamav.net (193.1.219.100).

Code Listing 5.2: Start amavisd and clamd and reload postfix configuration

# /etc/init.d/clamd start
# /etc/init.d/amavisd start
# /etc/init.d/postfix reload

Code Listing 5.3: Checking log files

# tail -f /var/log/mail.log

Code Listing 5.4: Manually checking that amavisd and postfix are listning to the new ports

 
# nc localhost 10024
(Amavis working)
220 [127.0.0.1] ESMTP amavisd-new service ready
nc localhost 10025
(Postfix reinject working)
220 example.com ESMTP Postfix

Code Listing 5.5: Add amavisd and clamd to the default runlevel

# rc-update add clamd default
# rc-update add amavisd default

Code Listing 6.1: Create spamtrap account

# useradd -m spamtrap
# maildirmake /home/spamtrap/.maildir
# chown -R spamtrap:mailusers /home/spamtrap/.maildir
(Give the spamtrap user a sensible password)
# passwd spamtrap

Code Listing 6.2: Creating /home/spamtrap/.procmailrc

#Set some default variables
MAILDIR=$HOME/.maildir

SPAM_FOLDER=$MAILDIR/.spam-found/

LIKELY_SPAM_FOLDER=$MAILDIR/.likely-spam-found/

#Sort mails with a spamscore of 7+ to the spamfolder
:0:
* ^X-Spam-Status: Yes
* ^X-Spam-Level: \*\*\*\*\*\*\*
$SPAM_FOLDER

#Sort mail with a spamscore between 5-7 to the likely spam folder
:0:
* ^X-Spam-Status: Yes
$LIKELY_SPAM_FOLDER

#Sort all other mails to the inbox
:0
*
./

Code Listing 6.3: Modifying /etc/postfix/main.cf

mailbox_command = /usr/bin/procmail -a "DOMAIN"

Code Listing 6.4: Create the necessary mailfolders

# maildirmake /var/amavis/.maildir
# maildirmake -S /var/amavis/.maildir/Bayes
# maildirmake -s write -f spam /var/amavis/.maildir/Bayes
# maildirmake -s write -f ham /var/amavis/.maildir/Bayes
# maildirmake -s write -f redeliver /var/amavis/.maildir/Bayes

Code Listing 6.5: Setting the proper permissions

# groupadd mailusers
# usermod -G mailusers spamtrap
# chown -R amavis:mailusers /var/amavis/.maildir/
# chown amavis:mailusers /var/amavis/
# chmod -R 1733 /var/amavis/.maildir/Bayes/
# chmod g+rx /var/amavis/.maildir/
# chmod g+rx /var/amavis/.maildir/Bayes/

Code Listing 6.6: Adding the shared folders to the users mailfolder

$ maildirmake --add Bayes=/var/amavis/.maildir/Bayes $HOME/.maildir

Code Listing 6.7: amavis crontab

#Auto learn
0 * * * *          /usr/bin/sa-learn --spam /var/amavis/.maildir/Bayes/.spam/{cur,new} \
                    > /dev/null 2>&1 
0 * * * *          /usr/bin/sa-learn --ham /var/amavis/.maildir/Bayes/.ham/{cur,new} > \
                   /dev/null 2>&1 

Code Listing 6.8: Modifying /etc/amavisd.conf

(Define the account to send virus spam emails)
$spam_quarantine_to = "spamtrap\@$myhostname";

Code Listing 6.9: Installing revised smtpclient

(Switch to your portage OVERLAY)
# wget http://home.coming.dk/files/smtpclient.tar.gz
# tar xzf smtpclient.tar.gz
# echo "mail-client/smtpclient    ~x86" >> /etc/portage/package.keywords
# echo "mail-client/smtpclient    fullheaders" >> /etc/portage/package.use
# emerge smtpclient

Code Listing 6.10: amavis crontab

#Redeliver false positives
* * * * *          find /var/amavis/.maildir/Bayes/.redeliver/cur/ -type f -name \
                   "[0-9]*" -exec cp {} /var/amavis/.maildir/Bayes/.ham/cur/ \; \
                   && find /var/amavis/.maildir/Bayes/.redeliver/cur/ -type f \
                   -name "[0-9]*" -exec /usr/local/bin/redeliver.pl {} \;

Code Listing 6.11: redeliver.pl

#!/usr/bin/perl -w

# Redelivers mail using a modified version of smtpclient
# By: Jens Hilligsoe <gentoo@hilli.dk>

use strict;

if(!($#ARGV == 0)) {
        die "Usage:\n$0 maildir_mail\n";
}

my $mail = $ARGV[0];
my $to = "";
my $from = "";

sub prunefile ( $ );

# Retrieve To and From envelope adresses
open (MAIL, $mail) or die "Could not open $mail: $?\n";
while(<MAIL>) {
    if(($to eq "") || ($from eq "")) {
        chop;
        (my $key, my $value) = split (/:/);
        if($key eq "X-Envelope-To") {
            $to = $value;
            $to =~ s/[\<\>,]//g; # Remove "<", ">" and ","
            $to =~ s/^\s+|\s+$//g; #Remove whitespace before and after
        }
        if($key eq "X-Envelope-From") {
            $from = $value;
            $from =~ s/[\<\>,]//g;
            $from =~ s/^\s+|\s+$//g;
            if($from eq "") {
                $from = "postmaster";
            }
        }
    }
}

if($to eq "") {
    prunefile($ARGV[0]); # Just nuke it if to is empty
} else {
    my $redelivercmd = "cat $ARGV[0] | smtpclient -F -S 127.0.0.1 -P 10025 -f $from $to";
    unless (system($redelivercmd) == 0 ) {
        die "Unable to redeliver: $?";
    }
    prunefile($ARGV[0]); # Clean up
}

sub prunefile ( $ ) {
    my ($file) = @_;
    unless (unlink $file) {
        die "Unable to remove mail: $?";
    }
}

Code Listing 6.12: Modifying root crontab

# Clean up
# Keep virusmails for a week (24*7 hours)
15 0 * * *      /usr/sbin/tmpwatch -c -f -d --quiet 168 /var/amavis/virusmails/
# Delete spam and ham learned after a week
15 0 * * *      /usr/sbin/tmpwatch -c -f -d --quiet 168 /var/amavis/.maildir/Bayes/

Code Listing 7.1: Getting greylist.pl

# cp /usr/portage/distfiles/postfix-your-version-here.tar.gz /root/
# tar xzf postfix-your-version-here.tar.gz
# cp postfix-2.1.0/examples/smtpd-policy/greylist.pl /usr/bin/

Code Listing 7.2: Creating directory for the greylisting database

# mkdir /var/mta
# chown nobody /var/mta

Code Listing 7.3: Modifying master.cf to use greylisting

policy-greylist  unix  -       n       n       -       -       spawn
   user=nobody argv=/usr/bin/perl /usr/bin/greylist.pl

Code Listing 7.4: Modifying main.cf to use greylisting

policy-greylist_time_limit = 3600
(Under smtpd_recipient_restrictions add:)
check_sender_access hash:/etc/postfix/sender_access
(Later on add:)
restriction_classes = greylist
greylist = check_policy_service unix:private/policy-greylist

Code Listing 7.5: Format of sender_access

aol.com     greylist
hotmail.com greylist
bigfoot.com greylist

Code Listing 7.6: Adding all domains to sender_access

# wget http://www.monkeys.com/anti-spam/filtering/sender-domain-validate.in
# cat sender-domain-validate.in | sort | awk {'print $1 "\t\t greylist"'} > /etc/postfix/sender_access

Code Listing 7.7: Initialize sender_access

# postmap /etc/postfix/sender_access

Code Listing 7.8: Installing postgrey

# emerge postgrey

Code Listing 7.9: Modifying main.cf to use greylisting

(Under smtpd_recipient_restrictions add:)
check_sender_access hash:/etc/postfix/sender_access
(Later on add:)
smtpd_restriction_classes = greylist
greylist = check_policy_service inet:127.0.0.1:10030

Code Listing 7.10: Starting postgrey

# /etc/init.d/postgrey start
# rc-update add postgrey default

Code Listing 8.1: Example SPF record

example.com.  IN TXT  "v=spf1 a mx ptr -all"

Code Listing 8.2: Installing spf.pl

# cp postfix-<version>/examples/smtpd-policy/spf.pl /usr/local/bin/

Code Listing 8.3: Installing the needed Perl libraries

# emerge Mail-SPF-Query Net-CIDR-Lite Sys-Hostname-Long

Code Listing 8.4: Modifying master.cf to use SPF

policy-spf  unix  -       n       n       -       -       spawn
   user=nobody argv=/usr/bin/perl /usr/local/bin/spf.pl

Code Listing 8.5: Modifying main.cf to use SPF

(Under smtpd_recipient_restrictions add:)
check_policy_service unix:private/policy-spf

Code Listing 9.1: Creating the MySQL database and user

# mysql -u root -p mysql
Enter password:
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 78 to server version: 4.0.18-log

Type 'help;' or '\h' for help. Type '\c' to clear the buffer.
mysql> create database maildb;
mysql> GRANT INSERT,UPDATE,DELETE,SELECT ON maildb.* TO 'mail'@'localhost' IDENTIFIED BY 'very_secret_password';
mysql> use maildb;

Code Listing 9.2: MySQL table layout

CREATE TABLE users (
    id         int unsigned NOT NULL auto_increment,
    priority   int          NOT NULL DEFAULT '7',  -- 0 is low priority
    policy_id  int unsigned NOT NULL DEFAULT '1',
    email      varchar(255) NOT NULL,
    fullname   varchar(255) DEFAULT NULL,    -- not used by amavisd-new
    local      char(1),     -- Y/N  (optional field, see note further down)
    PRIMARY KEY (id),
    KEY email (email)
    );
CREATE UNIQUE INDEX users_idx_email ON users(email);

(any e-mail address, external or local, used as senders in wblist)
CREATE TABLE mailaddr (
   id         int unsigned NOT NULL auto_increment,
   priority   int          NOT NULL DEFAULT '7',  -- 0 is low priority
   email      varchar(255) NOT NULL,
   PRIMARY KEY (id),
   KEY email (email)
   );
CREATE UNIQUE INDEX mailaddr_idx_email ON mailaddr(email);

(-- per-recipient whitelist and/or blacklist,
-- puts sender and recipient in relation wb)
(white or blacklisted sender)
CREATE TABLE wblist (
   rid        int unsigned NOT NULL,     -- recipient: users.id
   sid        int unsigned NOT NULL,     -- sender:    mailaddr.id
   wb         char(1) NOT NULL, -- W or Y / B or N / space=neutral
   PRIMARY KEY (rid,sid)
   );
  
CREATE TABLE policy (
   id               int unsigned NOT NULL auto_increment,
   policy_name      varchar(32),     -- not used by amavisd-new
   virus_lover          char(1),     -- Y/N
   spam_lover           char(1),     -- Y/N  (optional field)
   banned_files_lover   char(1),     -- Y/N  (optional field)
   bad_header_lover     char(1),     -- Y/N  (optional field)
   bypass_virus_checks  char(1),     -- Y/N
   bypass_spam_checks   char(1),     -- Y/N
   bypass_banned_checks char(1),     -- Y/N  (optional field)
   bypass_header_checks char(1),     -- Y/N (optional field)
   spam_modifies_subj   char(1),     -- Y/N (optional field)
   spam_quarantine_to   varchar(64) DEFAULT NULL, -- (optional field)
   spam_tag_level  float,  -- higher score inserts spam info headers
   spam_tag2_level float DEFAULT NULL,  -- higher score inserts
               -- 'declared spam' info header fields 
   spam_kill_level float,  -- higher score activates evasive actions, e.g.
               -- reject/drop, quarantine, ...
               -- (subject to final_spam_destiny setting)
   PRIMARY KEY (id)
  );

Code Listing 9.3: Create test user and test policy

INSERT INTO users
   SET
      id         =1,
      priority   =9,
      policy_id  =1,
      email      ="johndoe@example.com",
      fullname   ="John Doe",
      local      ="Y";

INSERT INTO policy
   SET
      id                     =1,
      policy_name            ="Test policy 1",
      virus_lover            ="N",
      spam_lover             ="N",
      banned_files_lover     ="N",
      bad_header_lover       ="N",
      bypass_virus_checks    ="N",
      bypass_spam_checks     ="N",
      bypass_banned_checks   ="N",
      bypass_header_checks   ="N",
      spam_modifies_subj     ="N",
      spam_quarantine_to     =NULL,
      spam_tag_level         =-50.0,
      spam_tag2_level        =7.0,
      spam_kill_level        =10.0;

Code Listing 9.4: Modifying amavisd.conf

@lookup_sql_dsn =
   ( ['DBI:mysql:maildb:host1', 'mail', 'very_secret_password']  );

(For clarity uncomment the default)
$sql_select_policy = 'SELECT *,users.id FROM users,policy'.
   ' WHERE (users.policy_id=policy.id) AND (users.email IN (%k))'.
   ' ORDER BY users.priority DESC';

(If you want sender white/blacklisting)
   $sql_select_white_black_list = 'SELECT wb FROM wblist,mailaddr'.
     ' WHERE (wblist.rid=?) AND (wblist.sid=mailaddr.id)'.
     '   AND (mailaddr.email IN (%k))'.
     ' ORDER BY mailaddr.priority DESC';

Code Listing 10.1: Creating the new MySQL database and user

# mysql -u root -p mysql
Enter password:
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 78 to server version: 4.0.18-log

Type 'help;' or '\h' for help. Type '\c' to clear the buffer.
mysql> create database dbname;
mysql> GRANT INSERT,UPDATE,DELETE,SELECT ON dbname.* TO 'dbuser'@'localhost' IDENTIFIED BY 'another_very_secret_password';
mysql> use dbname;

Code Listing 10.2: MySQL table layout

      CREATE TABLE bayes_expire (
          id                    int(11)         NOT NULL default '0',
          runtime               int(11)         NOT NULL default '0',
          KEY bayes_expire_idx1 (id)
          ) TYPE=MyISAM;

      CREATE TABLE bayes_global_vars (
          variable              varchar(30)     NOT NULL default '',
          value                 varchar(200)    NOT NULL default '',
          PRIMARY KEY           (variable)
          ) TYPE=MyISAM;

      INSERT INTO bayes_global_vars VALUES ('VERSION','3');

      CREATE TABLE bayes_seen (
          id                    int(11)         NOT NULL default '0',
          msgid                 varchar(200) binary NOT NULL default '',
          flag                  char(1)         NOT NULL default '',
          PRIMARY KEY           (id,msgid)
          ) TYPE=MyISAM;

      CREATE TABLE bayes_token (
          id                    int(11)         NOT NULL default '0',
          token                 char(5)         NOT NULL default '',
          spam_count            int(11)         NOT NULL default '0',
          ham_count             int(11)         NOT NULL default '0',
          atime                 int(11)         NOT NULL default '0',
          PRIMARY KEY           (id, token),
	  INDEX                 (id, atime)
          ) TYPE=MyISAM;

      CREATE TABLE bayes_vars (
          id                    int(11)         NOT NULL AUTO_INCREMENT,
          username              varchar(200)    NOT NULL default '',
          spam_count            int(11)         NOT NULL default '0',
          ham_count             int(11)         NOT NULL default '0',
          token_count           int(11)         NOT NULL default '0',
          last_expire           int(11)         NOT NULL default '0',
          last_atime_delta      int(11)         NOT NULL default '0',
          last_expire_reduce    int(11)         NOT NULL default '0',
          oldest_token_age      int(11)         NOT NULL default '2147483647',
          newest_token_age      int(11)         NOT NULL default '0',
          PRIMARY KEY           (id),
          UNIQUE bayes_vars_idx1 (username)
          ) TYPE=MyISAM;
                        
      CREATE TABLE awl (
          username              varchar(100)    NOT NULL default '',
          email                 varchar(200)    NOT NULL default '',
          ip                    varchar(10)     NOT NULL default '',
          count                 int(11)         default '0',
          totscore              float           default '0',
          PRIMARY KEY           (username,email,ip)
          ) TYPE=MyISAM;

Code Listing 10.3: Converting Bayes data from a DBM Database

su - amavis 
sa-learn --sync 
sa-learn --backup > backup.txt 
sa-learn --restore backup.txt 

Code Listing 10.4: Modifying /etc/mail/spamassassin/secrets.cf

(Tell Spamassassin to use MySQL for bayes data
bayes_store_module              Mail::SpamAssassin::BayesStore::SQL
bayes_sql_dsn                   DBI:mysql:sa_bayes:localhost:3306
bayes_sql_username              db_name
bayes_sql_password              another_very_secret_password

(Tell Spamassassin to use MySQL for AWL data
auto_whitelist_factory          Mail::SpamAssassin::SQLBasedAddrList
user_awl_dsn                    DBI:mysql:sa_bayes:localhost:3306
user_awl_sql_username           db_name
user_awl_sql_password           another_very_secret_password

Code Listing 10.5: Changing permissions

# chmod 400 /etc/mail/spamassassin/secrets.cf