Unify website slugify method behavior with or without python-slugify, and add some unit tests

addons/website/models/website.py

@@ -8,6 +8,7 @@ import itertools
 import logging
 import math
 import mimetypes
+import unicodedata
 import os
 import re
 import urlparse
@@ -28,6 +29,7 @@ except ImportError:
 import openerp
 from openerp.osv import orm, osv, fields
 from openerp.tools import html_escape as escape
+from openerp.tools import ustr as ustr
 from openerp.tools.safe_eval import safe_eval
 from openerp.addons.web.http import request
 
@@ -85,15 +87,29 @@ def is_multilang_url(local_url, langs=None):
         return False
 
 def slugify(s, max_length=None):
+    """ Transform a string to a slug that can be used in a url path.
+
+    This method will first try to do the job with python-slugify if present.
+    Otherwise it will process string by stripping leading and ending spaces,
+    converting unicode chars to ascii, lowering all chars and replacing spaces
+    and underscore with hyphen "-".
+
+    :param s: str
+    :param max_length: int
+    :rtype: str
+    """
+    s = ustr(s)
     if slugify_lib:
         # There are 2 different libraries only python-slugify is supported
         try:
             return slugify_lib.slugify(s, max_length=max_length)
         except TypeError:
             pass
-    spaceless = re.sub(r'\s+', '-', s)
-    specialless = re.sub(r'[^-_A-Za-z0-9]', '', spaceless)
-    return specialless[:max_length]
+    uni = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode('ascii')
+    slug = re.sub('[\W_]', ' ', uni).strip().lower()
+    slug = re.sub('[-\s]+', '-', slug)
+
+    return slug[:max_length]
 
 def slug(value):
     if isinstance(value, orm.browse_record):
@@ -147,7 +163,7 @@ class website(osv.osv):
     _defaults = {
         'company_id': lambda self,cr,uid,c: self.pool['ir.model.data'].xmlid_to_res_id(cr, openerp.SUPERUSER_ID, 'base.public_user'),
     }
-    
+
     # cf. Wizard hack in website_views.xml
     def noop(self, *args, **kwargs):
         pass

addons/website/tests/test_converter.py

@@ -9,6 +9,7 @@ from lxml.builder import E
 from openerp.tests import common
 from openerp.addons.base.ir import ir_qweb
 from openerp.addons.website.models.ir_qweb import html_to_text
+from openerp.addons.website.models.website import slugify
 
 impl = getDOMImplementation()
 document = impl.createDocument(None, None, None)
@@ -238,3 +239,56 @@ class TestConvertBack(common.TransactionCase):
             "New content",
             "element edition should have been written directly to the m2o record"
         )
+
+class TestTitleToSlug(unittest2.TestCase):
+    """
+    Those tests should pass with or without python-slugify
+    See website/models/website.py slugify method
+    """
+    def test_spaces(self):
+        self.assertEqual(
+            "spaces",
+            slugify(u"   spaces   ")
+        )
+
+    def test_unicode(self):
+        self.assertEqual(
+            "heterogeneite",
+            slugify(u"hétérogénéité")
+        )
+
+    def test_underscore(self):
+        self.assertEqual(
+            "one-two",
+            slugify(u"one_two")
+        )
+
+    def test_caps(self):
+        self.assertEqual(
+            "camelcase",
+            slugify(u"CamelCase")
+        )
+
+    def test_special_chars(self):
+        self.assertEqual(
+            "o-d-o-o",
+            slugify(u"o!#d{|\o/@~o&%^?")
+        )
+
+    def test_str_to_unicode(self):
+        self.assertEqual(
+            "espana",
+            slugify("España")
+        )
+
+    def test_numbers(self):
+        self.assertEqual(
+            "article-1",
+            slugify(u"Article 1")
+        )
+
+    def test_all(self):
+        self.assertEqual(
+            "do-you-know-martine-a-la-plage",
+            slugify(u"Do YOU know 'Martine à la plage' ?")
+        )