From 021fad453fcc1d1c710553b6e4b8423415130ca8 Mon Sep 17 00:00:00 2001 From: Jesko Date: Sat, 13 Apr 2024 22:15:14 +0200 Subject: [PATCH] initial_upload --- docker-compose.override.yml | 39 ++++ independent_aliases.txt | 2 + independent_mailboxes.txt | 5 + independent_maillists.txt | 1 + syncer.py | 400 ++++++++++++++++++++++++++++++++++++ 5 files changed, 447 insertions(+) create mode 100644 docker-compose.override.yml create mode 100644 independent_aliases.txt create mode 100644 independent_mailboxes.txt create mode 100644 independent_maillists.txt create mode 100644 syncer.py diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..1b377af --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,39 @@ +version: "2.1" +services: + linuxmuster-mailcow: + image: ghcr.io/linuxmuster/linuxmuster-mailcow:latest + container_name: mailcowcustomized_linuxmuster-mailcow + volumes: + - ./data/conf/dovecot:/conf/dovecot:rw + - ./data/conf/sogo:/conf/sogo:rw + - ../syncer.py:/syncer.py:rw + - ../independent_mailboxes.txt:/independent_mailboxes.txt:r + - ../independent_maillists.txt:/independent_maillists.txt:r + - ../independent_aliases.txt:/independent_aliases.txt:r + depends_on: + - nginx-mailcow + - dockerapi-mailcow + - php-fpm-mailcow + - sogo-mailcow + - dovecot-mailcow + environment: + - LINUXMUSTER_MAILCOW_LDAP_URI=ldap://10.16.1.1 + - LINUXMUSTER_MAILCOW_LDAP_BASE_DN=DC=morz,DC=de + - LINUXMUSTER_MAILCOW_LDAP_BIND_DN=CN=********,OU=******,OU=******,DC=******,DC=** + - LINUXMUSTER_MAILCOW_LDAP_BIND_DN_PASSWORD=****** + - LINUXMUSTER_MAILCOW_API_KEY=******-******-******-******-****** + - LINUXMUSTER_MAILCOW_SYNC_INTERVAL=300 + - LINUXMUSTER_MAILCOW_DOMAIN_QUOTA=500000 + - LINUXMUSTER_MAILCOW_ENABLE_GAL=1 +# Port muss stimmen... bei mir eben 9443, weil ich den Port umgebogen hab + - LINUXMUSTER_MAILCOW_API_URI=https://nginx-mailcow:9443 + - LINUXMUSTER_MAILCOW_DOCKERAPI_URI=https://dockerapi-mailcow:9443 + networks: + mailcow-network: + aliases: + - linuxmuster + +# ich hab den Look von SOGo noch an die Schulfarben angepasst, deshalb zusätzlich noch das hier... + sogo-mailcow: + volumes: + - ./data/conf/sogo/custom-theme.css:/usr/lib/GNUstep/SOGo/WebServerResources/css/theme-default.css:z diff --git a/independent_aliases.txt b/independent_aliases.txt new file mode 100644 index 0000000..de9ec2d --- /dev/null +++ b/independent_aliases.txt @@ -0,0 +1,2 @@ +beispiel1@schuldomain.de:adresse1@woauchimmer.tld,adresse2@irgendwoandersodernicht.net +beispiel2@schuldomain.de:adresse3@egalwo.tld diff --git a/independent_mailboxes.txt b/independent_mailboxes.txt new file mode 100644 index 0000000..6872b9f --- /dev/null +++ b/independent_mailboxes.txt @@ -0,0 +1,5 @@ +Support:support@schuldomain.tld:10240 +Nextcloud:claudi@schuldomain.tld:1 +Sekretariat:sekretariat@schuldomain.tld:10240 +Hausmeister:hausmeister@schuldomain.tld:10240 +Sozialarbeiter:nachhilfe@schuldomain.tld:10240 diff --git a/independent_maillists.txt b/independent_maillists.txt new file mode 100644 index 0000000..dcbc22a --- /dev/null +++ b/independent_maillists.txt @@ -0,0 +1 @@ +Name für das Verteilerpostfach:beispiel@adresse.de:empfaenger1@domain.de,empfaenger2@woanders.de,empfaenger3@woanders.de diff --git a/syncer.py b/syncer.py new file mode 100644 index 0000000..b521167 --- /dev/null +++ b/syncer.py @@ -0,0 +1,400 @@ +import sys +import os +import string +import time +import datetime +import logging +import coloredlogs +import random +import templateHelper +import json + +from mailcowHelper import MailcowHelper, MailcowException +from ldapHelper import LdapHelper +from objectStorageHelper import DomainListStorage, MailboxListStorage, AliasListStorage, FilterListStorage +from dockerapiHelper import DockerapiHelper +from requests.exceptions import ConnectionError + +coloredlogs.install( + level='INFO', fmt='%(asctime)s - [%(levelname)s] %(message)s') + + +class LinuxmusterMailcowSyncer: + + ldapSogoUserFilter = "(sophomorixRole='student' OR sophomorixRole='teacher')" + ldapUserFilter = "(|(sophomorixRole=student)(sophomorixRole=teacher))" + ldapMailingListFilter = "(|(sophomorixType=adminclass)(sophomorixType=project))" + ldapMailingListMemberFilter = f"(&(memberof:1.2.840.113556.1.4.1941:=@@mailingListDn@@){ldapUserFilter})" + + def __init__(self): + self._config = self._readConfig() + + self._mailcow = MailcowHelper( + self._config['API_URI'], + self._config['API_KEY'] + ) + self._ldap = LdapHelper( + self._config['LDAP_URI'], + self._config['LDAP_BIND_DN'], + self._config['LDAP_BIND_DN_PASSWORD'], + self._config['LDAP_BASE_DN'] + ) + + self._dockerapi = DockerapiHelper(self._config["DOCKERAPI_URI"]) + + templateHelper.applyAllTemplates(self._config, self._dockerapi) + + def sync(self): + while (True): + logging.info("=== Starting sync ===") + logging.info("##### angepasst von Jesko - Datei liegt in /srv/docker/syncer.py #####") + if not self._sync(): + logging.critical("!!! The sync failed, see above errors !!!") + interval = 30 + else: + logging.info("=== Sync finished successfully ==") + interval = int(self._config['SYNC_INTERVAL']) + + logging.info(f"sleeping {interval} seconds before next cycle") + time.sleep(interval) + + def _sync(self): + + logging.info("Step 1: Loading current Data from AD") + + logging.info(" * Binding to ldap") + if not self._ldap.bind(): + return False + + logging.info(" * Loading users from AD") + ret, adUsers = self._ldap.search( + self.ldapUserFilter, + ["mail", "proxyAddresses", "sophomorixStatus", + "sophomorixMailQuotaCalculated", "displayName"] + ) + if not ret: + logging.critical("!!! Error getting users from AD !!!") + return False + + logging.info(" * Loading groups from AD") + ret, adLists = self._ldap.search( + self.ldapMailingListFilter, + ["mail", "proxyAddresses", "distinguishedName", + "sophomorixMailList", "sAMAccountName"] + ) + if not ret: + logging.critical("!!! Error getting lists from AD !!!") + return False + + mailcowDomains = DomainListStorage() + mailcowMailboxes = MailboxListStorage(mailcowDomains) + mailcowAliases = AliasListStorage(mailcowDomains) + mailcowFilters = FilterListStorage(mailcowDomains) + + logging.info("Step 2: Loading current Data from Mailcow") + + try: + rawData = self._mailcow.getAllElementsOfType("domain") + mailcowDomains.loadRawData(rawData) + + rawData = self._mailcow.getAllElementsOfType("mailbox") + mailcowMailboxes.loadRawData(rawData) + + rawData = self._mailcow.getAllElementsOfType("alias") + mailcowAliases.loadRawData(rawData) + + # It is actially "filters" (plural); nobody knows why + rawData = self._mailcow.getAllElementsOfType("filters") + mailcowFilters.loadRawData(rawData) + #with open('independent_maillists.txt', 'w') as file: + # json.dump(rawData, file) + except MailcowException: + return False + except ConnectionError as e: + logging.error(e) + logging.critical( + "!!! A connection error occured, is mailcow still starting up? !!!") + return False + except Exception as e: + logging.exception("An exception occured: ", exc_info=e) + return False + + logging.info("Step 3: Loading current data from independent lists") + logging.info(" * Processing /independent_maillists.txt") + try: + with open("/independent_maillists.txt", "r") as file: + for line in file: + line = line.strip() + if not line: + continue + parts = line.split(":") + if len(parts) != 2: + logging.error(f"invalid line in independent_maillists.txt: {line}") + continue + listAddress = parts[0] + memberAddresses = parts[1].split(",") + self._addMailbox({ + "mail": listAddress, + "sophomorixStatus": "U", + "sophomorixMailQuotaCalculated": 1, + "displayName": listAddress + " (independent list)" + }, mailcowMailboxes) + self._addListFilter(listAddress, memberAddresses, mailcowFilters) + logging.info(f" - adding independent mailinglist {listAddress} with members {memberAddresses}") + except Exception as e: + logging.exception("An exception occured during processing of independent_maillists.txt: ", exc_info=e) + return False + + logging.info(" * processing /independent_mailboxes.txt") + try: + with open("/independent_mailboxes.txt", "r") as file: + for line in file: + line = line.strip() + if not line: + continue + parts = line.split(":") + if len(parts) != 3: + logging.error(f"invalid line in independent_mailboxes.txt: {line}") + continue + mboxName = parts[0] + mboxAddress = parts[1] + mboxQuota = parts[2] + self._addMailbox({ + "mail": mboxAddress, + "sophomorixStatus": "U", + "sophomorixMailQuotaCalculated": mboxQuota, + "displayName": mboxName, + }, mailcowMailboxes) + logging.info(f" - adding independent mbox {mboxAddress} for {mboxName} with quota {mboxQuota}") + except Exception as e: + logging.exception("An exception occured during processing of independent_mailboxes.txt: ", exc_info=e) + return False + logging.info(" * Processing /independent_mailaliases.txt") + try: + with open("/independent_aliases.txt", "r") as file: + for line in file: + line = line.strip() + if not line: + continue + parts = line.split(":") + if len(parts) != 2: + logging.error(f"invalid line in independent_aliases.txt: {line}") + continue + alias = parts[0] + forwardingTo = parts[1] + self._addAlias(alias, forwardingTo, mailcowAliases) + logging.info(f" - adding independent alias {alias} forwarding to {forwardingTo}") + except Exception as e: + logging.exception("An exception occured during processing of independent_aliases.txt: ", exc_info=e) + return False + + + logging.info("Step 4: Calculating deltas between AD and Mailcow") + + for user in adUsers: + mail = user["mail"] + maildomain = mail.split("@")[-1] + + if not self._addDomain(maildomain, mailcowDomains): + continue + + self._addMailbox(user, mailcowMailboxes) + self._addAliasesFromProxyAddresses(user, mail, mailcowAliases) + + for mailingList in adLists: + if not mailingList["sophomorixMailList"] == "TRUE": + continue + + mail = mailingList["mail"] + maildomain = mail.split("@")[-1] + ret, members = self._ldap.search( + self.ldapMailingListMemberFilter.replace( + "@@mailingListDn@@", mailingList["distinguishedName"]), + ["mail"] + ) + #logging.info(f" * Adding mailinglist {mail} with members {members}") + if not ret: + continue + + if not self._addDomain(maildomain, mailcowDomains): + continue + + self._addMailbox({ + "mail": mail, + "sophomorixStatus": "U", + "sophomorixMailQuotaCalculated": 1, + "displayName": mailingList["sAMAccountName"] + " (list)" + }, mailcowMailboxes) + self._addAliasesFromProxyAddresses( + mailingList, mail, mailcowAliases) + + self._addListFilter(mail, list( + map(lambda x: x["mail"], members)), mailcowFilters) + + if mailcowDomains.queuesAreEmpty() and mailcowMailboxes.queuesAreEmpty() and mailcowAliases.queuesAreEmpty() and mailcowFilters.queuesAreEmpty(): + logging.info(" * Everything up-to-date!") + return True + else: + logging.info("* Found deltas:") + logging.info( + f" * {mailcowDomains.getQueueCountsString('domains')}") + logging.info( + f" * {mailcowMailboxes.getQueueCountsString('mailboxes')}") + logging.info( + f" * {mailcowAliases.getQueueCountsString('aliases')}") + logging.info( + f" * {mailcowFilters.getQueueCountsString('filters')}") + + logging.info("Step 4: Syncing deltas to Mailcow") + + try: + self._mailcow.killElementsOfType( + "filter", mailcowFilters.killQueue()) + self._mailcow.killElementsOfType( + "alias", mailcowAliases.killQueue()) + self._mailcow.killElementsOfType( + "mailbox", mailcowMailboxes.killQueue()) + self._mailcow.killElementsOfType( + "domain", mailcowDomains.killQueue()) + + self._mailcow.addElementsOfType( + "domain", mailcowDomains.addQueue()) + self._mailcow.updateElementsOfType( + "domain", mailcowDomains.updateQueue()) + + self._mailcow.addElementsOfType( + "mailbox", mailcowMailboxes.addQueue()) + self._mailcow.updateElementsOfType( + "mailbox", mailcowMailboxes.updateQueue()) + + self._mailcow.addElementsOfType("alias", mailcowAliases.addQueue()) + self._mailcow.updateElementsOfType( + "alias", mailcowAliases.updateQueue()) + + self._mailcow.addElementsOfType( + "filter", mailcowFilters.addQueue()) + self._mailcow.updateElementsOfType( + "filter", mailcowFilters.updateQueue()) + except MailcowException: + return False + + self._ldap.unbind() + return True + + def _addDomain(self, domainName, mailcowDomains): + return mailcowDomains.addElement({ + "domain": domainName, + "defquota": 1, + "maxquota": self._config['DOMAIN_QUOTA'], + "quota": self._config['DOMAIN_QUOTA'], + "description": DomainListStorage.validityCheckDescription, + "active": 1, + "restart_sogo": 1, + "mailboxes": 10000, + "aliases": 10000, + "gal": int(self._config['ENABLE_GAL']) + }, domainName) + + def _addMailbox(self, user, mailcowMailboxes): + mail = user["mail"] + domain = mail.split("@")[-1] + localPart = mail.split("@")[0] + password = ''.join(random.choices( + string.ascii_letters + string.digits, k=20)) + active = 0 if user["sophomorixStatus"] in [ + "L", "D", "R", "K", "F"] else 1 + return mailcowMailboxes.addElement({ + "domain": domain, + "local_part": localPart, + "active": active, + "quota": user["sophomorixMailQuotaCalculated"], + "password": password, + "password2": password, + "name": user["displayName"] + }, mail) + + def _addAliasesFromProxyAddresses(self, user, mail, mailcowAliases): + aliases = [] + + if "proxyAddresses" in user: + if isinstance(user["proxyAddresses"], list): + aliases = user["proxyAddresses"] + else: + aliases = [user["proxyAddresses"]] + + if len(aliases) > 0: + for alias in aliases: + self._addAlias(alias, mail, mailcowAliases) + + def _addAlias(self, alias, goto, mailcowAliases): + mailcowAliases.addElement({ + "address": alias, + "goto": goto, + "active": 1, + "sogo_visible": 1 + }, alias) + pass + + def _addListFilter(self, listAddress, memberAddresses, mailcowFilters): + scriptData = "### Auto-generated mailinglist filter by linuxmuster ###\r\n\r\n" + scriptData += "require \"copy\";\r\n\r\n" + for memberAddress in memberAddresses: + scriptData += f"redirect :copy \"{memberAddress}\";\r\n" + scriptData += "\r\ndiscard;stop;" + mailcowFilters.addElement({ + 'active': 1, + 'username': listAddress, + 'filter_type': 'prefilter', + 'script_data': scriptData, + 'script_desc': f"Auto-generated mailinglist filter for {listAddress}" + }, listAddress) + + def _readConfig(self): + requiredConfigKeys = [ + 'LINUXMUSTER_MAILCOW_LDAP_URI', + 'LINUXMUSTER_MAILCOW_LDAP_BASE_DN', + 'LINUXMUSTER_MAILCOW_LDAP_BIND_DN', + 'LINUXMUSTER_MAILCOW_LDAP_BIND_DN_PASSWORD', + 'LINUXMUSTER_MAILCOW_API_KEY', + 'LINUXMUSTER_MAILCOW_SYNC_INTERVAL', + 'LINUXMUSTER_MAILCOW_DOMAIN_QUOTA', + 'LINUXMUSTER_MAILCOW_ENABLE_GAL' + ] + + allowedConfigKeys = [ + "LINUXMUSTER_MAILCOW_DOCKERAPI_URI", + "LINUXMUSTER_MAILCOW_API_URI" + ] + + config = { + "LDAP_SOGO_USER_FILTER": self.ldapSogoUserFilter, + "LDAP_USER_FILTER": self.ldapUserFilter, + "DOCKERAPI_URI": "https://dockerapi-mailcow", + "API_URI": "https://nginx-mailcow" + } + + for configKey in requiredConfigKeys: + if configKey not in os.environ: + sys.exit(f"Required environment value {configKey} is not set") + config[configKey.replace( + 'LINUXMUSTER_MAILCOW_', '')] = os.environ[configKey] + + for configKey in allowedConfigKeys: + if configKey in os.environ: + config[configKey.replace( + 'LINUXMUSTER_MAILCOW_', '')] = os.environ[configKey] + + logging.info("CONFIG:") + for key, value in config.items(): + logging.info(" * {:25}: {}".format(key, value)) + + return config + + +if __name__ == '__main__': + try: + syncer = LinuxmusterMailcowSyncer() + syncer.sync() + except KeyboardInterrupt: + pass +