#!/usr/bin/python2.7
# -*- coding: utf-8 -*-
#
# Univention Mail Dovecot - OpenXchange spam/ham bayes feeding
#
# Copyright 2015-2017 Univention GmbH
#
# http://www.univention.de/
#
# All rights reserved.
#
# The source code of this program is made available
# under the terms of the GNU Affero General Public License version 3
# (GNU AGPL V3) as published by the Free Software Foundation.
#
# Binary versions of this program provided by Univention to you as
# well as other copyrighted, protected or trademarked materials like
# Logos, graphics, fonts, specific documentations and configurations,
# cryptographic keys etc. are subject to a license agreement between
# you and Univention and not subject to the GNU AGPL V3.
#
# In the case you use this program under the terms of the GNU AGPL V3,
# the program is provided in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License with the Debian GNU/Linux or Univention distribution in file
# /usr/share/common-licenses/AGPL-3; if not, see
# <http://www.gnu.org/licenses/>.

import MySQLdb
import sys
import fcntl
import os
import os.path
import stat
import pwd
import grp
import subprocess
import traceback
import re
import syslog
import argparse
import univention.config_registry

CFPROP = "/opt/open-xchange/etc/configdb.properties"
USER_RE = "^readProperty.1=user=(.*)$"
PASS_RE = "^readProperty.2=password=(.*)$"
DBPATH = "/var/lib/amavis/.spamassassin"
LOCKFILE = "/var/lib/amavis/.spamassassin/.spamrunner.lock"
SA_USER = "amavis"
SA_GROUP = "amavis"
OXADMIN_TARGET_FOLDER_SPAM = "spamlearn"
OXADMIN_TARGET_FOLDER_HAM = "hamlearn"


syslog.openlog("spamrunner", syslog.LOG_PID, syslog.LOG_MAIL)


parser = argparse.ArgumentParser(description="Collect user-confirmed ham and spam messages and let SpamAssassin learn"
            "them. All output always goes to syslog (/var/log/mail.log).")
parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
args = parser.parse_args()

verbose = args.verbose

ucr = univention.config_registry.ConfigRegistry()
ucr.load()

contextId = ucr.get('ox/context/id', '10')
if contextId.isdigit():
	contextId = int(contextId)
else:
	contextId = 10
contextadmin = 'oxadmin-context%d' % (contextId,)
if contextadmin == 'oxadmin-context10':
	contextadmin = 'oxadmin'

def read_from_ext_proc(cmd, verbose=False, quiet=False):
	try:
		cmd_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		cmd_out, cmd_err = cmd_proc.communicate()
		cmd_exit = cmd_proc.wait()
		if verbose:
			syslog.syslog(str(cmd))
		elif not verbose and not quiet and (cmd_err or cmd_exit != 0):
			syslog.syslog(syslog.LOG_ERR, str(cmd))
		if not quiet:
			if cmd_exit != 0:
				syslog.syslog(syslog.LOG_ERR, "Return code: %d" % cmd_exit)
			if cmd_err:
				syslog.syslog(syslog.LOG_ERR, cmd_err)
		if cmd_out:
			if verbose:
				syslog.syslog(cmd_out.strip())
			return cmd_out.strip()
	except:
		syslog.syslog(syslog.LOG_ERR, "Failed run %r.\n%s" % (str(cmd), traceback.format_exc()))
		raise

def get_user_home(username):
	try:
		cmd = ["/usr/bin/doveadm", 'user', "-f", "home", username]
		return read_from_ext_proc(cmd)
	except:
		syslog.syslog(syslog.LOG_ERR, "Failed to get mail home for user %r." % username)
		raise

def get_mysql_cred():
	un_re = re.compile(USER_RE)
	pw_re = re.compile(PASS_RE)
	username = password = ""
	with open(CFPROP, "r") as oxcfg:
		for line in oxcfg:
			_username = un_re.match(line)
			if _username:
				username = _username.groups()[0]
			_password = pw_re.match(line)
			if _password:
				password = _password.groups()[0]
			if username and password:
				return username, password
	raise Exception("Did not find MySQL credentials in %s." % CFPROP)


# get MySQL credentials
try:
	mysql_username, mysql_password = get_mysql_cred()
except Exception, e:
	syslog.syslog(syslog.LOG_ERR, "Error retrieving MySQL credentials: %s" % str(e))
	sys.exit(1)

# get a lock
try:
	f_lock = open(LOCKFILE, "wb")
	fcntl.lockf(f_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
	syslog.syslog(syslog.LOG_ERR, "Could not acquire lock.")
	sys.exit(1)

# set permissions
try:
	for bfile in ["bayes_journal", "bayes_seen", "bayes_toks"]:
		path = os.path.join(DBPATH, bfile)
		with open(path, "a"):
			os.utime(path, None)
		sa_uid = pwd.getpwnam(SA_USER).pw_uid
		sa_gid = grp.getgrnam(SA_GROUP).gr_gid
		os.chown(path, sa_uid, sa_gid)
		os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
except:
	syslog.syslog(syslog.LOG_ERR, "Could not touch spamassassin files.")
	sys.exit(1)

try:
	con = None
	con = MySQLdb.connect('localhost', mysql_username, mysql_password)
	cur = con.cursor(MySQLdb.cursors.DictCursor)

	# basic connection check
	cur.execute("SELECT VERSION()")
	ver = cur.fetchone()
	if verbose:
		syslog.syslog("Database version: %s." % str(ver))

	# find relevant database
	cur.execute("SHOW DATABASES")
	dbs = cur.fetchall()
	for dbdict in dbs:
		if "Database" in dbdict and dbdict["Database"].startswith("localhost"):
			db = dbdict["Database"]
			if verbose:
				syslog.syslog("DB: %s." % db)
			break
	else:
		syslog.syslog(syslog.LOG_ERR, "No DB found.")
		sys.exit(1)

	# get oxadmin email address
	users_query = """SELECT send_addr
					FROM %(db)s.user_setting_mail, %(db)s.login2user
					WHERE user_setting_mail.user=login2user.id
					AND login2user.cid=%(contextId)d
					AND login2user.uid='%(context-oxadmin)s'""" % {"db": db, 'contextId': contextId, 'context-oxadmin': contextadmin}
	cur.execute(users_query)
	oxadmin = cur.fetchone()
	if verbose:
		syslog.syslog("%s: %s." % (contextadmin, oxadmin))
	oxadmin_email = oxadmin["send_addr"]

	# get users spam folders from mysql
	users_query = """SELECT login2user.uid,user_setting_mail.bits,
						user_setting_mail.confirmed_spam,
						user_setting_mail.confirmed_ham,
						send_addr
					FROM %(db)s.user_setting_mail, %(db)s.login2user
					WHERE user_setting_mail.user=login2user.id
					AND login2user.cid=%(contextId)d""" % {"db": db, 'contextId': contextId}
	cur.execute(users_query)
	users = cur.fetchall()

	# create target mailboxes
	cmd = ["/usr/bin/doveadm", "-v", "mailbox", "create", "-u", oxadmin_email, OXADMIN_TARGET_FOLDER_SPAM, OXADMIN_TARGET_FOLDER_HAM]
	read_from_ext_proc(cmd, False, True)

	# copy ham/spam from each user to oxadmin, delete afterwards
	target_paths = {"spam": set(), "ham": set()}
	for user in users:
		if not os.path.exists('%s/Maildir' % (get_user_home(user["send_addr"],))):
			syslog.syslog('User: %s - missing maildir - skipping user' % (user,))
			continue
		if verbose:
			syslog.syslog("User: %s." % str(user))
		target_paths["spam"].add(user["confirmed_spam"])
		target_paths["ham"].add(user["confirmed_ham"])
		maildir = "maildir:%s/Maildir" % get_user_home(user["send_addr"])
		try:
			cmd = ["/usr/bin/doveadm", "-v", "import", "-u", oxadmin_email, maildir, OXADMIN_TARGET_FOLDER_SPAM, "mailbox", user["confirmed_spam"]]
			read_from_ext_proc(cmd, verbose, True)
			cmd = ["/usr/bin/doveadm", "-v", "expunge", "-u", user["send_addr"], "mailbox", user["confirmed_spam"], "all"]
			read_from_ext_proc(cmd, verbose, True)
			cmd = ["/usr/bin/doveadm", "-v", "import", "-u", oxadmin_email, maildir, OXADMIN_TARGET_FOLDER_HAM, "mailbox", user["confirmed_ham"]]
			read_from_ext_proc(cmd, verbose, True)
			cmd = ["/usr/bin/doveadm", "-v", "expunge", "-u", user["send_addr"], "mailbox", user["confirmed_ham"], "all"]
			read_from_ext_proc(cmd, verbose, True)
		except:
			syslog.syslog(syslog.LOG_ERR, "Failed to copy h/spam mails for user %r.\n%s" % (user["send_addr"], traceback.format_exc()))
			continue

	# move mails from subfolder into target folder
	for target_path in target_paths["spam"]:
		cmd = ["/usr/bin/doveadm", "-v", "move", "-u", oxadmin_email, OXADMIN_TARGET_FOLDER_SPAM, "mailbox", "%s/%s" % (OXADMIN_TARGET_FOLDER_SPAM, target_path), "all"]
		read_from_ext_proc(cmd, verbose, True)
		cmd = ["/usr/bin/doveadm", "-v", "mailbox", 'delete', "-u", oxadmin_email, "%s/%s" % (OXADMIN_TARGET_FOLDER_SPAM, target_path)]
		read_from_ext_proc(cmd, verbose, True)
	for target_path in target_paths["ham"]:
		cmd = ["/usr/bin/doveadm", "-v", "move", "-u", oxadmin_email, OXADMIN_TARGET_FOLDER_HAM, "mailbox", "%s/%s" % (OXADMIN_TARGET_FOLDER_HAM, target_path), "all"]
		read_from_ext_proc(cmd, verbose, True)  # ignore errors
		cmd = ["/usr/bin/doveadm", "-v", "mailbox", 'delete', "-u", oxadmin_email, "%s/%s" % (OXADMIN_TARGET_FOLDER_HAM, target_path)]
		read_from_ext_proc(cmd, verbose, True)

	# learn ham and spam at oxadmin folders
	oxadmin_maildir = "%s/Maildir" % get_user_home(oxadmin_email)
	cmd = ["/usr/bin/sa-learn", "--spam", "--dbpath", DBPATH, "%s/.%s" % (oxadmin_maildir, OXADMIN_TARGET_FOLDER_SPAM)]
	read_from_ext_proc(cmd, verbose)
	cmd = ["/usr/bin/sa-learn", "--ham", "--dbpath", DBPATH, "%s/.%s" % (oxadmin_maildir, OXADMIN_TARGET_FOLDER_HAM)]
	read_from_ext_proc(cmd, verbose)

	# remove ham and spam mails from oxadmin
	cmd = ["/usr/bin/doveadm", "-v", "expunge", "-u", oxadmin_email, "mailbox", OXADMIN_TARGET_FOLDER_SPAM, "all"]
	read_from_ext_proc(cmd, verbose, True)
	cmd = ["/usr/bin/doveadm", "-v", "expunge", "-u", oxadmin_email, "mailbox", OXADMIN_TARGET_FOLDER_HAM, "all"]
	read_from_ext_proc(cmd, verbose, True)

except MySQLdb.Error, e:
	syslog.syslog(syslog.LOG_ERR, "Error: %s" % str(e))
	sys.exit(1)
finally:
	if con:
		con.close()
	fcntl.lockf(f_lock, fcntl.LOCK_UN)
	f_lock.close()

if not f_lock.closed:
	fcntl.lockf(f_lock, fcntl.LOCK_UN)
	f_lock.close()
