[IMP] ir_mail_server: IDNA and SMTPUTF8 capabilities

It has been a recurrent request from customers to be able to send email
messages to email addresses containing non-ascii characters. [IDNA] is a
domain extension to allow unicode characters in domain names. [SMTPUTF8]
is a SMTP extension to allow unicode in any header.

IDNA defines the [punycode] encoding which translates unicode to an
ascii representation. This encoding MUST be used to encode domains.

SMTPUTF8 is an SMTP extension that allow utf-8 in all headers on the
envelope.

[IDNA] https://tools.ietf.org/html/rfc5890
[SMTPUTF8] https://tools.ietf.org/html/rfc6531
[punycode] https://tools.ietf.org/html/rfc3492



Task: 2116928
opw-2229906
opw-2248251

closes odoo/odoo#47709

Signed-off-by: Raphael Collet (rco) <rco@openerp.com>

addons/test_mail/tests/test_mail_mail.py

@@ -25,16 +25,6 @@ class TestMailMail(TestMailCommon):
         self.assertSentEmail(mail.env.user.partner_id, ['test@example.com'])
         self.assertEqual(len(self._mails), 1)
 
-    @mute_logger('odoo.addons.mail.models.mail_mail')
-    def test_mail_message_values_unicode(self):
-        mail = self.env['mail.mail'].sudo().create({
-            'body_html': '<p>Test</p>',
-            'email_to': 'test.😊@example.com',
-            'partner_ids': [(4, self.user_employee.partner_id.id)]
-        })
-
-        self.assertRaises(MailDeliveryException, lambda: mail.send(raise_exception=True))
-
 
 class TestMailMailRace(common.TransactionCase):
 

odoo/addons/base/models/ir_mail_server.py

@@ -16,7 +16,7 @@ import html2text
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import UserError
-from odoo.tools import ustr, pycompat
+from odoo.tools import ustr, pycompat, formataddr
 
 _logger = logging.getLogger(__name__)
 _test_logger = logging.getLogger('odoo.tests')
@@ -52,7 +52,7 @@ def extract_rfc2822_addresses(text):
     if not text:
         return []
     candidates = address_pattern.findall(ustr(text))
-    return [c for c in candidates if is_ascii(c)]
+    return [formataddr(('', c), charset='ascii') for c in candidates]
 
 
 class IrMailServer(models.Model):
@@ -415,13 +415,21 @@ class IrMailServer(models.Model):
                 smtp_server, smtp_port, smtp_user, smtp_password,
                 smtp_encryption, smtp_debug, mail_server_id=mail_server_id)
 
-            message_str = message.as_string()
-            # header folding code is buggy and adds redundant carriage
-            # returns, it got fixed in 3.7.4 thanks to bpo-34424
             if sys.version_info < (3, 7, 4):
-                message_str = re.sub('\r+', '\r', message_str)
+                # header folding code is buggy and adds redundant carriage
+                # returns, it got fixed in 3.7.4 thanks to bpo-34424
+                message_str = message.as_string()
+                message_str = re.sub('\r+(?!\n)', '', message_str)
+
+                mail_options = []
+                if any((not is_ascii(addr) for addr in smtp_to_list + [smtp_from])):
+                    # non ascii email found, require SMTPUTF8 extension,
+                    # the relay may reject it
+                    mail_options.append("SMTPUTF8")
+                smtp.sendmail(smtp_from, smtp_to_list, message_str, mail_options=mail_options)
+            else:
+                smtp.send_message(message, smtp_from, smtp_to_list)
 
-            smtp.sendmail(smtp_from, smtp_to_list, message_str)
             # do not quit() a pre-established smtp_session
             if not smtp_session:
                 smtp.quit()

odoo/addons/base/tests/test_mail.py

@@ -351,6 +351,7 @@ class TestEmailTools(BaseCase):
 
     def test_email_formataddr(self):
         email = 'joe@example.com'
+        email_idna = 'joe@examplé.com'
         cases = [
             # (name, address),          charsets            expected
             (('', email),               ['ascii', 'utf-8'], 'joe@example.com'),
@@ -359,17 +360,17 @@ class TestEmailTools(BaseCase):
             (('joe"doe', email),        ['ascii', 'utf-8'], '"joe\\"doe" <joe@example.com>'),
             (('joé', email),            ['ascii'],          '=?utf-8?b?am/DqQ==?= <joe@example.com>'),
             (('joé', email),            ['utf-8'],          '"joé" <joe@example.com>'),
-            (('', 'joé@example.com'),   ['ascii', 'utf-8'], UnicodeEncodeError),  # need SMTPUTF8 support
-            (('', 'joe@examplé.com'),   ['ascii', 'utf-8'], UnicodeEncodeError),  # need IDNA support
+            (('', email_idna),          ['ascii'],          'joe@xn--exampl-gva.com'),
+            (('', email_idna),          ['utf-8'],          'joe@examplé.com'),
+            (('joé', email_idna),       ['ascii'],          '=?utf-8?b?am/DqQ==?= <joe@xn--exampl-gva.com>'),
+            (('joé', email_idna),       ['utf-8'],          '"joé" <joe@examplé.com>'),
+            (('', 'joé@example.com'),   ['ascii', 'utf-8'], 'joé@example.com'),
         ]
 
         for pair, charsets, expected in cases:
             for charset in charsets:
                 with self.subTest(pair=pair, charset=charset):
-                    if isinstance(expected, str):
-                        self.assertEqual(formataddr(pair, charset), expected)
-                    else:
-                        self.assertRaises(expected, formataddr, pair, charset)
+                    self.assertEqual(formataddr(pair, charset), expected)
 
 
 class EmailConfigCase(SavepointCase):
@@ -403,7 +404,19 @@ class TestEmailMessage(TransactionCase):
             def __init__(this):
                 this.email_sent = False
 
-            def sendmail(this, smtp_from, smtp_to_list, message_str):
+            def sendmail(this, smtp_from, smtp_to_list, message_str,
+                         mail_options=(), rcpt_options=()):
+                this.email_sent = True
+                message_truth = (
+                    r'From: .+? <joe@example\.com>\r\n'
+                    r'To: .+? <joe@example\.com>\r\n'
+                    r'\r\n'
+                )
+                self.assertRegex(message_str, message_truth)
+
+            def send_message(this, message, smtp_from, smtp_to_list,
+                             mail_options=(), rcpt_options=()):
+                message_str = message.as_string()
                 this.email_sent = True
                 message_truth = (
                     r'From: .+? <joe@example\.com>\r\n'

odoo/tools/mail.py

@@ -14,6 +14,7 @@ import time
 from email.utils import getaddresses
 from lxml import etree
 from werkzeug import urls
+import idna
 
 import odoo
 from odoo.loglevels import ustr
@@ -546,13 +547,15 @@ def decode_message_header(message, header, separator=' '):
 def formataddr(pair, charset='utf-8'):
     """Pretty format a 2-tuple of the form (realname, email_address).
 
-    Set the charset to ascii to get a RFC-2822 compliant email.
-
-    The email address is considered valid and is left unmodified.
-
     If the first element of pair is falsy then only the email address
     is returned.
 
+    Set the charset to ascii to get a RFC-2822 compliant email. The
+    realname will be base64 encoded (if necessary) and the domain part
+    of the email will be punycode encoded (if necessary). The local part
+    is left unchanged thus require the SMTPUTF8 extension when there are
+    non-ascii characters.
+
     >>> formataddr(('John Doe', 'johndoe@example.com'))
     '"John Doe" <johndoe@example.com>'
 
@@ -560,21 +563,26 @@ def formataddr(pair, charset='utf-8'):
     'johndoe@example.com'
     """
     name, address = pair
-    address.encode('ascii')
+    local, _, domain = address.rpartition('@')
+
+    try:
+        domain.encode(charset)
+    except UnicodeEncodeError:
+        # rfc5890 - Internationalized Domain Names for Applications (IDNA)
+        domain = idna.encode(domain).decode('ascii')
+
     if name:
         try:
             name.encode(charset)
         except UnicodeEncodeError:
             # charset mismatch, encode as utf-8/base64
             # rfc2047 - MIME Message Header Extensions for Non-ASCII Text
-            return "=?utf-8?b?{name}?= <{addr}>".format(
-                name=base64.b64encode(name.encode('utf-8')).decode('ascii'),
-                addr=address)
+            name = base64.b64encode(name.encode('utf-8')).decode('ascii')
+            return f"=?utf-8?b?{name}?= <{local}@{domain}>"
         else:
             # ascii name, escape it if needed
             # rfc2822 - Internet Message Format
             #   #section-3.4 - Address Specification
-            return '"{name}" <{addr}>'.format(
-                name=email_addr_escapes_re.sub(r'\\\g<0>', name),
-                addr=address)
-    return address
+            name = email_addr_escapes_re.sub(r'\\\g<0>', name)
+            return f'"{name}" <{local}@{domain}>'
+    return f"{local}@{domain}"

requirements.txt

@@ -10,6 +10,7 @@ gevent==1.4.0 ; sys_platform == 'win32'
 greenlet==0.4.10 ; python_version < '3.7'
 greenlet==0.4.15 ; python_version >= '3.7'
 html2text==2018.1.9
+idna==2.6
 Jinja2==2.10.1
 libsass==0.17.0
 lxml==3.7.1 ; sys_platform != 'win32' and python_version < '3.7'

setup.cfg

@@ -20,6 +20,7 @@ requires =
   python3-gevent
   python3-greenlet
   python3-html2text
+  python3-idna
   python3-jinja2
   python3-lxml
   python3-mako

setup.py

@@ -29,6 +29,7 @@ setup(
         'feedparser',
         'gevent',
         'html2text',
+        'idna',
         'Jinja2',
         'lxml',  # windows binary http://www.lfd.uci.edu/~gohlke/pythonlibs/
         'libsass',