[FIX] cloc: fix count of studio field + exclude sa

This commit fix 3 problem

Don't count studio field
------------------------

the commit https://github.com/odoo/odoo/commit/75b8c4eb1ed6e44f427886eea06ad216995e16c8
was supposed to stop counting count field generate by addins a smart
button with studio.

It works fine when base import module is not installed which is never
the case.

When base_import_module the field is counted as a field that belong
to a imported module installed : studio_customization

studio_customization should not be counted as an imported module
installed

Count Field with standard xml_id
--------------------------------

With https://github.com/odoo/odoo/commit/9afce4805fc8bac45fdba817488aa867fddff69b


Each manual can end up with an xml_id from a standard module:
the original module of the model

A standard module should never create a manual field so we can consider
they should be counted unless they match the criteria of the first
problem

If a field has a standard module xml_id and no other the module
should be odoo/studio and not the name of the standard module

Make possible to exclude some db record from cloc
-------------------------------------------------
It's possible to exclude some file in python module
but it's not possible to exclude some field or SA in
the database from the count

Make it possible if they are link to an xml_id
from the module __cloc_exclude__

The exclude record will be shown in cloc report with
the verbose mode

closes odoo/odoo#94848

X-original-commit: 58c3b45c
Related: odoo/enterprise#28956
Signed-off-by: Christophe Simonis <chs@odoo.com>

addons/base_import_module/tests/test_cloc.py

@@ -20,6 +20,20 @@ VALID_XML = """
 
 class TestClocFields(test_cloc.TestClocCustomization):
 
+    def create_studio_module(self):
+        # Studio module does not exist at this stage, so we simulate it
+        # Check for existing module in case the test run on an existing database
+        if not self.env['ir.module.module'].search([('name', '=', 'studio_customization')]):
+            self.env['ir.module.module'].create({
+                'author': 'Odoo',
+                'imported': True,
+                'latest_version': '13.0.1.0.0',
+                'name': 'studio_customization',
+                'state': 'installed',
+                'summary': 'Studio Customization',
+            })
+
+
     def test_fields_from_import_module(self):
         """
             Check that custom computed fields installed with an imported module
@@ -31,15 +45,44 @@ class TestClocFields(test_cloc.TestClocCustomization):
             'imported': True,
         })
         f1 = self.create_field('x_imported_field')
-        self.create_xml_id('import_field', 'ir.model.fields', f1.id, 'imported_module')
+        self.create_xml_id('imported_module', 'import_field', f1)
         cl = cloc.Cloc()
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('imported_module', 0), 1, 'Count fields with xml_id of imported module')
-        f2 = self.create_field('x_base_field')
-        self.create_xml_id('base_field', 'ir.model.fields', f2.id, 'base')
+
+    def test_fields_from_studio(self):
+        self.create_studio_module()
+        f1 = self.create_field('x_field_count')
+        self.create_xml_id('studio_customization', 'field_count', f1)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('studio_customization', 0), 0, "Don't count field generated by studio")
+        f2 = self.create_field('x_studio_manual_field')
+        self.create_xml_id('studio_customization', 'manual_field', f2)
         cl = cloc.Cloc()
         cl.count_customization(self.env)
-        self.assertEqual(cl.code.get('base', 0), 0, "Don't count fields from standard module")
+        self.assertEqual(cl.code.get('studio_customization', 0), 1, "Count manual field created via studio")
+
+    def test_fields_module_name(self):
+        """
+            Check that custom computed fields installed with an imported module
+            is counted as customization
+        """
+        self.env['ir.module.module'].create({
+            'name': 'imported_module',
+            'state': 'installed',
+            'imported': True,
+        })
+        f1 = self.create_field('x_imported_field')
+        self.create_xml_id('imported_module', 'import_field', f1)
+        self.create_xml_id('__export__', 'import_field', f1)
+
+        sa = self.create_server_action("Test imported double xml_id")
+        self.create_xml_id("imported_module", "first", sa)
+        self.create_xml_id("__export__", "second", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('imported_module', 0), 3)
 
     def test_count_qweb_imported_module(self):
         self.env['ir.module.module'].create({
@@ -50,24 +93,15 @@ class TestClocFields(test_cloc.TestClocCustomization):
             'state': 'installed',
             'summary': 'Test imported module for cloc',
         })
-        # Studio module does not exist at this stage, so we simulate it
-        # Check for existing module in case the test run on an existing database
-        if not self.env['ir.module.module'].search([('name', '=', 'studio_customization')]):
-            self.env['ir.module.module'].create({
-                'author': 'Odoo',
-                'imported': True,
-                'latest_version': '15.0.1.0.0',
-                'name': 'studio_customization',
-                'state': 'installed',
-                'summary': 'Studio Customization',
-            })
+        self.create_studio_module()
+
         qweb_view = self.env['ir.ui.view'].create({
             "name": "Qweb Test",
             "type": "qweb",
             "mode": "primary",
             "arch_base": "<html>\n  <body>\n    <t t-out=\"Hello World\"/>\n  </body>\n</html>",
         })
-        self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id, 'test_imported_module')
+        self.create_xml_id('test_imported_module', "qweb_view_test", qweb_view)
 
         # Add qweb view from non imported module
         qweb_view = self.env['ir.ui.view'].create({
@@ -75,7 +109,7 @@ class TestClocFields(test_cloc.TestClocCustomization):
             "type": "qweb",
             "arch_base": "<html>\n  <body>\n    <t t-out=\"Hello World\"/>\n  </body>\n</html>",
         })
-        self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id)
+        self.create_xml_id("studio_customization", "qweb_view_test", qweb_view)
 
         # Add form view from module
         form_view = self.env['ir.ui.view'].create({
@@ -84,7 +118,7 @@ class TestClocFields(test_cloc.TestClocCustomization):
             "model": "res.partner",
             "arch_base": "<form>\n  <field name=\"name\" \n         invisible=\"1\" />\n</form>",
         })
-        self.create_xml_id("form_view_test", 'ir.ui.view', form_view.id, 'test_imported_module')
+        self.create_xml_id("test_imported_module", "form_view_test", form_view)
 
         cl = cloc.Cloc()
         cl.count_customization(self.env)
@@ -116,3 +150,63 @@ class TestClocFields(test_cloc.TestClocCustomization):
         cl = cloc.Cloc()
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('test_imported_module', 0), 35)
+
+    def test_exclude_qweb(self):
+        self.env['ir.module.module'].create({
+            'author': 'Odoo',
+            'imported': True,
+            'latest_version': '15.0.1.0.0',
+            'name': 'test_imported_module',
+            'state': 'installed',
+            'summary': 'Test imported module for cloc',
+        })
+
+        qweb_view = self.env['ir.ui.view'].create({
+            "name": "Qweb Test",
+            "type": "qweb",
+            "mode": "primary",
+            "arch_base": "<html>\n  <body>\n    <t t-out=\"Hello World\"/>\n  </body>\n</html>",
+        })
+        self.create_xml_id('test_imported_module', "qweb_view_test", qweb_view)
+        self.create_xml_id('__cloc_exclude__', "qweb_view_test", qweb_view)
+
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('test_imported_module', 0), 0, "Do not count view with cloc_exclude")
+
+    def test_exclude_attachment_imported_module(self):
+        manifest_content = json.dumps({
+            'name': 'test_imported_module',
+            'description': 'Test',
+            'assets': {
+                'web.assets_backend': [
+                    'test_imported_module/static/src/js/test.js',
+                    'test_imported_module/static/src/css/test.scss',
+                ]
+            },
+            'license': 'LGPL-3',
+        })
+
+        stream = BytesIO()
+        with ZipFile(stream, 'w') as archive:
+            archive.writestr('test_imported_module/__manifest__.py', manifest_content)
+            archive.writestr('test_imported_module/static/src/js/test.js', test_cloc.JS_TEST)
+            archive.writestr('test_imported_module/static/src/js/test.scss', test_cloc.SCSS_TEST)
+            archive.writestr('test_imported_module/static/src/js/test.xml', VALID_XML)
+
+        id_names = [
+            'attachment_/test_imported_module/static/src/js/test_js',
+            'attachment_/test_imported_module/static/src/js/test_scss',
+            'attachment_/test_imported_module/static/src/js/test_xml',
+        ]
+
+        # Import test module
+        self.env['ir.module.module'].import_zipfile(stream)
+        # Create exclude xml_id
+        for name in id_names:
+            rec = self.env.ref(f'test_imported_module.{name}')
+            self.create_xml_id('__cloc_exclude__', name, rec)
+
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('test_imported_module', 0), 0)

odoo/addons/base/tests/test_cloc.py

@@ -129,16 +129,16 @@ SCSS_TEST = '''
 '''
 
 class TestClocCustomization(TransactionCase):
-    def create_xml_id(self, name, model, res_id, module='studio_customization'):
+    def create_xml_id(self, module, name, rec):
         self.env['ir.model.data'].create({
             'name': name,
-            'model': model,
-            'res_id': res_id,
+            'model': rec._name,
+            'res_id': rec.id,
             'module': module,
         })
 
     def create_field(self, name):
-        return self.env['ir.model.fields'].with_context(studio=True).create({
+        field = self.env['ir.model.fields'].with_context(studio=True).create({
             'name': name,
             'field_description': name,
             'model': 'res.partner',
@@ -147,6 +147,23 @@ class TestClocCustomization(TransactionCase):
             'store': False,
             'compute': "for rec in self: rec['x_invoice_count'] = 10",
         })
+        # Simulate the effect of https://github.com/odoo/odoo/commit/9afce4805fc8bac45fdba817488aa867fddff69b
+        # Updating a module create xml_id of the module even for manual field if it's the original module
+        # of the model
+        self.create_xml_id('base', name, field)
+        return field
+
+    def create_server_action(self, name):
+        return self.env['ir.actions.server'].create({
+            'name': name,
+            'code': """
+for rec in records:
+    rec['name'] = test
+            """,
+            'state': 'code',
+            'type': 'ir.actions.server',
+            'model_id': self.env.ref('base.model_res_partner').id,
+        })
 
     def test_ignore_auto_generated_computed_field(self):
         """
@@ -154,12 +171,12 @@ class TestClocCustomization(TransactionCase):
             Having an xml_id but no existing module is consider as not belonging to a module
         """
         f1 = self.create_field('x_invoice_count')
-        self.create_xml_id('invoice_count', 'ir.model.fields', f1.id)
+        self.create_xml_id('studio_customization', 'invoice_count', f1)
         cl = cloc.Cloc()
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Studio auto generated count field should not be counted in cloc')
         f2 = self.create_field('x_studio_custom_field')
-        self.create_xml_id('studio_custom', 'ir.model.fields', f2.id)
+        self.create_xml_id('studio_customization', 'studio_custom', f2)
         cl = cloc.Cloc()
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Count other studio computed field')
@@ -168,11 +185,52 @@ class TestClocCustomization(TransactionCase):
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count fields without xml_id')
         f4 = self.create_field('x_custom_field_export')
-        self.create_xml_id('studio_custom', 'ir.model.fields', f4.id, '__export__')
+        self.create_xml_id('__export__', 'studio_custom', f4)
         cl = cloc.Cloc()
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('odoo/studio', 0), 3, 'Count fields with xml_id but without module')
 
+    def test_several_xml_id(self):
+        sa = self.create_server_action("Test double xml_id")
+        self.create_xml_id("__export__", "first", sa)
+        self.create_xml_id("base", "second", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count Should count SA with a non standard xml_id')
+        self.create_xml_id("__import__", "third", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'SA with several xml_id should be counted only once')
+
+    def test_cloc_exclude_xml_id(self):
+        sa = self.create_server_action("Test double xml_id")
+        self.create_xml_id("__cloc_exclude__", "sa_first", sa)
+        self.create_xml_id("__upgrade__", "sa_second", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Should not count SA with cloc_exclude xml_id')
+
+        f1 = self.create_field('x_invoice_count')
+        self.create_xml_id("__cloc_exclude__", "field_first", f1)
+        self.create_xml_id("__upgrade__", "field_second", f1)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Should not count Field with cloc_exclude xml_id')
+
+    def test_field_no_xml_id(self):
+        self.env['ir.model.fields'].create({
+            'name': "x_no_xml_id",
+            'field_description': "no_xml_id",
+            'model': 'res.partner',
+            'model_id': self.env.ref('base.model_res_partner').id,
+            'ttype': 'integer',
+            'store': False,
+            'compute': "for rec in self: rec['x_invoice_count'] = 10",
+        })
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Should count field with no xml_id at all')
+
 
 class TestClocParser(TransactionCase):
 

odoo/tools/cloc.py

@@ -30,6 +30,7 @@ class Cloc(object):
         self.code = {}
         self.total = {}
         self.errors = {}
+        self.excluded = {}
         self.max_width = 70
 
     #------------------------------------------------------
@@ -95,10 +96,13 @@ class Cloc(object):
     #------------------------------------------------------
     # Enumeration
     #------------------------------------------------------
-    def book(self, module, item='', count=(0, 0)):
+    def book(self, module, item='', count=(0, 0), exclude=False):
         if count[0] == -1:
             self.errors.setdefault(module, {})
             self.errors[module][item] = count[1]
+        elif exclude and item:
+            self.excluded.setdefault(module, {})
+            self.excluded[module][item] = count
         else:
             self.modules.setdefault(module, {})
             if item:
@@ -168,65 +172,93 @@ class Cloc(object):
                 self.count_path(module_path)
 
     def count_customization(self, env):
-        imported_module = ""
+        imported_module_sa = ""
         if env['ir.module.module']._fields.get('imported'):
-            imported_module = "OR (m.imported = TRUE AND m.state = 'installed')"
+            imported_module_sa = "OR (m.imported = TRUE AND m.state = 'installed')"
         query = """
-            SELECT s.id, m.name FROM ir_act_server AS s
-                LEFT JOIN ir_model_data AS d ON (d.res_id = s.id AND d.model = 'ir.actions.server')
-                LEFT JOIN ir_module_module AS m ON m.name = d.module
-            WHERE s.state = 'code' AND (m.name IS null {})
-        """.format(imported_module)
+                SELECT s.id, min(m.name), array_agg(d.module)
+                  FROM ir_act_server AS s
+             LEFT JOIN ir_model_data AS d
+                    ON (d.res_id = s.id AND d.model = 'ir.actions.server')
+             LEFT JOIN ir_module_module AS m
+                    ON m.name = d.module
+                 WHERE s.state = 'code' AND (m.name IS null {})
+              GROUP BY s.id
+        """.format(imported_module_sa)
         env.cr.execute(query)
-        data = {r[0]: r[1] for r in env.cr.fetchall()}
+        data = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
         for a in env['ir.actions.server'].browse(data.keys()):
-            self.book(data[a.id] or "odoo/studio", "ir.actions.server/%s: %s" % (a.id, a.name), self.parse_py(a.code))
-
+            self.book(
+                data[a.id][0] or "odoo/studio",
+                "ir.actions.server/%s: %s" % (a.id, a.name),
+                self.parse_py(a.code),
+                '__cloc_exclude__' in data[a.id][1]
+            )
+
+        imported_module_field = ("'odoo/studio'", "")
+        if env['ir.module.module']._fields.get('imported'):
+            imported_module_field = ("min(m.name)", "AND m.imported = TRUE AND m.state = 'installed'")
+        # We always want to count manual compute field unless they are generated by studio
+        # the module should be odoo/studio unless it comes from an imported module install
+        # because manual field get an external id from the original module of the model
         query = r"""
-            SELECT f.id, m.name FROM ir_model_fields AS f
-                LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
-                LEFT JOIN ir_module_module AS m ON m.name = d.module
-            WHERE f.compute IS NOT null AND (
-                -- Do not count studio field
-                (m.name IS null AND (d.module != 'studio_customization' or d.module is NULL))
-                -- Unless they start with x_studio
-                OR (d.module = 'studio_customization' AND f.name ilike 'x\_studio%')
-                {})
-        """.format(imported_module)
+                SELECT f.id, f.name, {}, array_agg(d.module)
+                  FROM ir_model_fields AS f
+             LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
+             LEFT JOIN ir_module_module AS m ON m.name = d.module {}
+                 WHERE f.compute IS NOT null AND f.state = 'manual'
+              GROUP BY f.id, f.name
+        """.format(*imported_module_field)
         env.cr.execute(query)
-        data = {r[0]: r[1] for r in env.cr.fetchall()}
+        # Do not count field generated by studio
+        all_data = env.cr.fetchall()
+        data = {r[0]: (r[2], r[3]) for r in all_data if not ("studio_customization" in r[3] and not r[1].startswith('x_studio'))}
         for f in env['ir.model.fields'].browse(data.keys()):
-            self.book(data[f.id] or "odoo/studio", "ir.model.fields/%s: %s" % (f.id, f.name), self.parse_py(f.compute))
-
-        if not imported_module:
+            self.book(
+                data[f.id][0] or "odoo/studio",
+                "ir.model.fields/%s: %s" % (f.id, f.name),
+                self.parse_py(f.compute),
+                '__cloc_exclude__' in data[f.id][1]
+            )
+
+        if not env['ir.module.module']._fields.get('imported'):
             return
 
         # Count qweb view only from imported module and not studio
         query = """
-            SELECT view.id, data.module
+            SELECT view.id, min(mod.name), array_agg(data.module)
               FROM ir_ui_view view
         INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view'
-        INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
-             WHERE view.type = 'qweb' AND mod.name != 'studio_customization'
+         LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
+             WHERE view.type = 'qweb' AND data.module != 'studio_customization'
+          GROUP BY view.id
+            HAVING count(mod.name) > 0
         """
         env.cr.execute(query)
-        custom_views = dict(env.cr.fetchall())
+        custom_views = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
         for view in env['ir.ui.view'].browse(custom_views.keys()):
-            modue_name = custom_views[view.id]
-            self.book(modue_name, "/%s/views/%s.xml" % (modue_name, view.name), self.parse_xml(view.arch_base))
+            module_name = custom_views[view.id][0]
+            self.book(
+                module_name,
+                "/%s/views/%s.xml" % (module_name, view.name),
+                self.parse_xml(view.arch_base),
+                '__cloc_exclude__' in custom_views[view.id][1]
+            )
 
         # Count js, xml, css/scss file from imported module
         query = r"""
-            SELECT attach.id, data.module
+            SELECT attach.id, min(mod.name), array_agg(data.module)
               FROM ir_attachment attach
         INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment'
-        INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
+         LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
              WHERE attach.name ~ '.*\.(js|xml|css|scss)$'
+          GROUP BY attach.id
+            HAVING count(mod.name) > 0
         """
         env.cr.execute(query)
-        uploaded_file = dict(env.cr.fetchall())
+        uploaded_file = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
         for attach in env['ir.attachment'].browse(uploaded_file.keys()):
-            module_name = uploaded_file[attach.id]
+            module_name = uploaded_file[attach.id][0]
             ext = os.path.splitext(attach.url)[1].lower()
             if ext not in VALID_EXTENSION:
                 continue
@@ -238,7 +270,12 @@ class Cloc(object):
             # Decode using latin1 to avoid error that may raise by decoding with utf8
             # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
             content = attach.raw.decode('latin1')
-            self.book(module_name, attach.url, self.parse(content, ext))
+            self.book(
+                module_name,
+                attach.url,
+                self.parse(content, ext),
+                '__cloc_exclude__' in uploaded_file[attach.id][1],
+            )
 
     def count_env(self, env):
         self.count_modules(env)
@@ -254,6 +291,7 @@ class Cloc(object):
     #------------------------------------------------------
     # Report
     #------------------------------------------------------
+    # pylint: disable=W0141
     def report(self, verbose=False, width=None):
         # Prepare format
         if not width:
@@ -276,6 +314,16 @@ class Cloc(object):
         s += fmt.format(k='', lines=total, other=total - code, code=code)
         print(s)
 
+        if self.excluded and verbose:
+            ex = fmt.format(k="Excluded", lines="Line", other="Other", code="Code")
+            ex += hr
+            for m in sorted(self.excluded):
+                for i in sorted(self.excluded[m], key=lambda i: self.excluded[m][i][0], reverse=True):
+                    code, total = self.excluded[m][i]
+                    ex += fmt.format(k='    ' + i, lines=total, other=total - code, code=code)
+            ex += hr
+            print(ex)
+
         if self.errors:
             e = "\nErrors\n\n"
             for m in sorted(self.errors):