Merge remote-tracking branch 'upstream/master' into fix-doc-commands-1

.mergify.yml

@@ -4,6 +4,8 @@ pull_request_rules:
     conditions:
       - -title~=(WIP|wip)
       - -label~=^(status/wip|status/blocked)$
+      - -closed
+      - -merged
     actions:
       comment:
         message: |

CHANGELOG.md

@@ -9,6 +9,9 @@ v1.6.1 - unreleased
 -------------------
 - Enhancement: Make Unbound drop privileges after binding to port
 - Enhancement: Create an Authentication Token with IPv6 address restriction ([#829](https://github.com/Mailu/Mailu/issues/829))
+- Bug: Fix creating new fetched accounts
+- Enhancement: Missing wildcard option in alias flask command ([#869](https://github.com/Mailu/Mailu/issues/869))
+- Bug: Fix poor performance if ANTIVIRUS is configured to none.
 
 v1.6.0 - 2019-01-18
 -------------------
@@ -34,7 +37,6 @@ v1.6.0 - 2019-01-18
 - Feature: Automated Releases ([#487](https://github.com/Mailu/Mailu/issues/487))
 - Feature: Support for ARC ([#495](https://github.com/Mailu/Mailu/issues/495))
 - Feature: Add posibilty to run webmail on root ([#501](https://github.com/Mailu/Mailu/issues/501))
-- Feature: Upgrade docker-compose.yml to version 3 ([#539](https://github.com/Mailu/Mailu/issues/539))
 - Feature: Documentation to deploy mailu on a docker swarm ([#551](https://github.com/Mailu/Mailu/issues/551))
 - Feature: Add optional Maildir-Compression ([#553](https://github.com/Mailu/Mailu/issues/553))
 - Feature: Preserve rspamd history on container restart ([#561](https://github.com/Mailu/Mailu/issues/561))
@@ -85,6 +87,7 @@ v1.6.0 - 2019-01-18
 - Enhancement: Include favicon package ([#801](https://github.com/Mailu/Mailu/issues/801), ([#802](https://github.com/Mailu/Mailu/issues/802))
 - Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588](https://github.com/Mailu/Mailu/issues/588))
 - Enhancement: Mark message as seen when reporting as spam
+- Enhancement: Better support and document IPv6 ([#827](https://github.com/Mailu/Mailu/issues/827))
 - Upstream: Update Roundcube
 - Upstream: Update Rainloop
 - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93))

core/admin/Dockerfile

@@ -1,8 +1,9 @@
 FROM alpine:3.8
 # python3 shared with most images
 RUN apk add --no-cache \
-    python3 py3-pip \
+    python3 py3-pip git \
   && pip3 install --upgrade pip
+RUN pip3 install git+https://github.com/usrpro/MailuStart.git#egg=mailustart
 # Image specific layers under this line
 RUN mkdir -p /app
 WORKDIR /app

core/admin/mailu/configuration.py

@@ -1,5 +1,5 @@
 import os
-
+from mailustart import resolve
 
 DEFAULT_CONFIG = {
     # Specific to the admin UI
@@ -61,7 +61,6 @@
     'POD_ADDRESS_RANGE': None
 }
 
-
 class ConfigManager(dict):
     """ Naive configuration manager that uses environment only
     """
@@ -75,6 +74,12 @@ class ConfigManager(dict):
     def __init__(self):
         self.config = dict()
 
+    def resolve_host(self):
+        self.config['HOST_IMAP'] = resolve(self.config['HOST_IMAP'])
+        self.config['HOST_POP3'] = resolve(self.config['HOST_POP3'])
+        self.config['HOST_AUTHSMTP'] = resolve(self.config['HOST_AUTHSMTP'])
+        self.config['HOST_SMTP'] = resolve(self.config['HOST_SMTP'])
+
     def __coerce_value(self, value):
         if isinstance(value, str) and value.lower() in ('true','yes'):
             return True
@@ -89,6 +94,7 @@ def init_app(self, app):
             key: self.__coerce_value(os.environ.get(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
+        self.resolve_host()
 
         # automatically set the sqlalchemy string
         if self.config['DB_FLAVOR']:

core/admin/mailu/internal/nginx.py

@@ -2,7 +2,6 @@
 from flask import current_app as app
 
 import re
-import socket
 import urllib
 
 
@@ -89,5 +88,4 @@ def get_server(protocol, authenticated=False):
             hostname, port = extract_host_port(app.config['HOST_AUTHSMTP'], 10025)
         else:
             hostname, port = extract_host_port(app.config['HOST_SMTP'], 25)
-    address = socket.gethostbyname(hostname)
-    return address, port
+    return hostname, port

core/admin/mailu/manage.py

@@ -292,8 +292,9 @@ def alias_delete(email):
 @click.argument('localpart')
 @click.argument('domain_name')
 @click.argument('destination')
+@click.option('-w', '--wildcard', is_flag=True)
 @flask_cli.with_appcontext
-def alias(localpart, domain_name, destination):
+def alias(localpart, domain_name, destination, wildcard=False):
     """ Create an alias
     """
     domain = models.Domain.query.get(domain_name)
@@ -303,6 +304,7 @@ def alias(localpart, domain_name, destination):
     alias = models.Alias(
         localpart=localpart,
         domain=domain,
+        wildcard=wildcard,
         destination=destination.split(','),
         email="%s@%s" % (localpart, domain_name)
     )

core/admin/mailu/ui/views/fetches.py

@@ -22,7 +22,7 @@ def fetch_create(user_email):
     user_email = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email) or flask.abort(404)
     form = forms.FetchForm()
-    form.pw.validators = [wtforms.validators.DataRequired()]
+    form.password.validators = [wtforms.validators.DataRequired()]
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)

core/dovecot/Dockerfile

@@ -1,12 +1,10 @@
 FROM alpine:3.8
 # python3 shared with most images
 RUN apk add --no-cache \
-    python3 py3-pip \
+    python3 py3-pip git \
   && pip3 install --upgrade pip
 # Shared layer between rspamd, postfix, dovecot, unbound and nginx
-RUN pip3 install jinja2
-# Shared layer between rspamd, postfix, dovecot
-RUN pip3 install tenacity
+RUN pip3 install git+https://github.com/usrpro/MailuStart.git#egg=mailustart
 # Image specific layers under this line
 RUN apk add --no-cache \
   dovecot dovecot-pigeonhole-plugin rspamd-client bash \

core/dovecot/start.py

@@ -1,47 +1,29 @@
 #!/usr/bin/python3
 
-import jinja2
 import os
-import socket
 import glob
 import multiprocessing
-import tenacity
 import logging as log
 import sys
+from mailustart import resolve, convert
 
-from tenacity import retry
 from podop import run_server
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 def start_podop():
     os.setuid(8)
+    url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/dovecot/§"
     run_server(0, "dovecot", "/tmp/podop.socket", [
-		("quota", "url", "http://admin/internal/dovecot/§"),
-		("auth", "url", "http://admin/internal/dovecot/§"),
-		("sieve", "url", "http://admin/internal/dovecot/§"),
+		("quota", "url", url ),
+		("auth", "url", url),
+		("sieve", "url", url),
     ])
 
-def convert(src, dst):
-    logger = log.getLogger("convert()")
-    logger.debug("Source: %s, Destination: %s", src, dst)
-    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
-
-@retry(
-    stop=tenacity.stop_after_attempt(100),
-    wait=tenacity.wait_random(min=2, max=5),
-    before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG),
-    before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO),
-    after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG)
-    )
-def resolve(hostname):
-    logger = log.getLogger("resolve()")
-    logger.info(hostname)
-    return socket.gethostbyname(hostname)
-
 # Actual startup script
 os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
 os.environ["REDIS_ADDRESS"] = resolve(os.environ.get("REDIS_ADDRESS", "redis"))
+os.environ["ADMIN_ADDRESS"] = resolve(os.environ.get("ADMIN_ADDRESS", "admin"))
 if os.environ["WEBMAIL"] != "none":
     os.environ["WEBMAIL_ADDRESS"] = resolve(os.environ.get("WEBMAIL_ADDRESS", "webmail"))
 

core/nginx/Dockerfile

@@ -1,10 +1,10 @@
 FROM alpine:3.8
 # python3 shared with most images
 RUN apk add --no-cache \
-    python3 py3-pip \
+    python3 py3-pip git \
   && pip3 install --upgrade pip
 # Shared layer between rspamd, postfix, dovecot, unbound and nginx
-RUN pip3 install jinja2
+RUN pip3 install git+https://github.com/usrpro/MailuStart.git#egg=mailustart
 # Image specific layers under this line
 RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl \
  && pip3 install idna requests watchdog

core/nginx/config.py

@@ -1,32 +1,27 @@
 #!/usr/bin/python3
 
-import jinja2
 import os
 import logging as log
 import sys
+from mailustart import resolve, convert
 
 args = os.environ.copy()
 
 log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING"))
 
-def convert(src, dst, args):
-    logger = log.getLogger("convert()")
-    logger.debug("Source: %s, Destination: %s", src, dst)
-    open(dst, "w").write(jinja2.Template(open(src).read()).render(**args))
-
 # Get the first DNS server
 with open("/etc/resolv.conf") as handle:
     content = handle.read().split()
     args["RESOLVER"] = content[content.index("nameserver") + 1]
 
-if "HOST_WEBMAIL" not in args:
-    args["HOST_WEBMAIL"] = "webmail"
-if "HOST_ADMIN" not in args:
-    args["HOST_ADMIN"] = "admin"
-if "HOST_WEBDAV" not in args:
-    args["HOST_WEBDAV"] = "webdav:5232"
-if "HOST_ANTISPAM" not in args:
-    args["HOST_ANTISPAM"] = "antispam:11334"
+args["HOST_ADMIN"] = resolve(args.get("HOST_ADMIN", "admin"))
+args["HOST_ANTISPAM"] = resolve(args.get("HOST_ANTISPAM", "antispam:11334"))
+args["HOST_WEBMAIL"] = args.get("HOST_WEBMAIL", "webmail")
+if args["WEBMAIL"] != "none":
+    args["HOST_WEBMAIL"] = resolve(args.get("HOST_WEBMAIL"))
+args["HOST_WEBDAV"] = args.get("HOST_WEBDAV", "webdav:5232")
+if args["WEBDAV"] != "none":
+    args["HOST_WEBDAV"] = resolve(args.get("HOST_WEBDAV"))
 
 # TLS configuration
 cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")

core/postfix/Dockerfile

@@ -1,12 +1,10 @@
 FROM alpine:3.8
 # python3 shared with most images
 RUN apk add --no-cache \
-    python3 py3-pip \
+    python3 py3-pip git \
   && pip3 install --upgrade pip
 # Shared layer between rspamd, postfix, dovecot, unbound and nginx
-RUN pip3 install jinja2
-# Shared layer between rspamd, postfix, dovecot
-RUN pip3 install tenacity
+RUN pip3 install git+https://github.com/usrpro/MailuStart.git#egg=mailustart
 # Image specific layers under this line
 
 RUN apk add --no-cache postfix postfix-pcre rsyslog \

core/postfix/start.py

@@ -1,53 +1,35 @@
 #!/usr/bin/python3
 
-import jinja2
 import os
-import socket
 import glob
 import shutil
-import tenacity
 import multiprocessing
 import logging as log
 import sys
+from mailustart import resolve, convert
 
-from tenacity import retry
 from podop import run_server
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 def start_podop():
     os.setuid(100)
+    url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/postfix/"
     # TODO: Remove verbosity setting from Podop?
     run_server(0, "postfix", "/tmp/podop.socket", [
-		("transport", "url", "http://admin/internal/postfix/transport/§"),
-		("alias", "url", "http://admin/internal/postfix/alias/§"),
-		("domain", "url", "http://admin/internal/postfix/domain/§"),
-        ("mailbox", "url", "http://admin/internal/postfix/mailbox/§"),
-        ("senderaccess", "url", "http://admin/internal/postfix/sender/access/§"),
-        ("senderlogin", "url", "http://admin/internal/postfix/sender/login/§")
+		("transport", "url", url + "transport/§"),
+		("alias", "url", url + "alias/§"),
+		("domain", "url", url + "domain/§"),
+        ("mailbox", "url", url + "mailbox/§"),
+        ("senderaccess", "url", url + "sender/access/§"),
+        ("senderlogin", "url", url + "sender/login/§")
     ])
 
-def convert(src, dst):
-    logger = log.getLogger("convert()")
-    logger.debug("Source: %s, Destination: %s", src, dst)
-    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
-
-@retry(
-    stop=tenacity.stop_after_attempt(100),
-    wait=tenacity.wait_random(min=2, max=5),
-    before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG),
-    before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO),
-    after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG)
-    )
-def resolve(hostname):
-    logger = log.getLogger("resolve()")
-    logger.info(hostname)
-    return socket.gethostbyname(hostname)
-
 # Actual startup script
 os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
-os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332")
-os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525")
+os.environ["ADMIN_ADDRESS"] = resolve(os.environ.get("ADMIN_ADDRESS", "admin"))
+os.environ["HOST_ANTISPAM"] = resolve(os.environ.get("HOST_ANTISPAM", "antispam:11332"))
+os.environ["HOST_LMTP"] = resolve(os.environ.get("HOST_LMTP", "imap:2525"))
 
 for postfix_file in glob.glob("/conf/*.cf"):
     convert(postfix_file, os.path.join("/etc/postfix", os.path.basename(postfix_file)))

docs/cli.rst

@@ -39,7 +39,7 @@ primary difference with simple `user` command is that password is being imported
 
 .. code-block:: bash
 
-  docker-compose run --rm admin flask mailu user-import --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce'
+  docker-compose run --rm admin flask mailu user-import myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' 'SHA512-CRYPT'
 
 user-delete
 ------------

docs/contributors/environment.rst

@@ -227,12 +227,9 @@ trying to fix. When happy, you can approve the PR. When running into failures, m
 Additional commits
 ``````````````````
 
-Sometimes users add new commits after ``bors try`` was  run automatically.
-In such cases, a reviewer will have to re-issue a ``bors try`` manually in order
-to get the latest changes in the test image. The reviewer will have to be sure the
-build finished successful before pulling the new images.
-
-Any previous reviews get dismissed automatically, whenever a new commit is done afterwards.
+On every new commit ``bors try`` is  run automatically. Past approvals get dismissed automatically.
+When doing a subsequent review on the same PR, be sure to pull the latest image from docker hub
+after Bors confirms a successful build.
 
 When bors try fails
 ```````````````````