[FIX] mail: speed up read_progress_bar

This commit fixes the performance issue in getting statistics for
``activity_state`` (colored clock icon for overdue/today/planned) in
CRM.  The query has been tested for several years on a large database
(Odoo's own production database).

Performance test on 29 K crm.lead records (activity_state):

With a filter for 10 records:

```
| measurement        | before | after |
|--------------------+--------+-------|
| number of queries  |     25 |     5 |
| query time, ms     |     12 |    95 | (*)
| remaining time, ms |     32 |     7 |
```

All records:

```
| measurement        | before | after |
|--------------------+--------+-------|
| number of queries  |   1326 |     5 |
| query time, ms     |   1739 |   129 |
| remaining time, ms |  47934 |    17 |
```

As we can see in the last results, the time went from almost 50 seconds
(not responsive at all) to 150 milliseconds (responsive).  The time
increase in (*) may be caused by imperfect measurements, which are raw
and not averaged measures.

---

opw-2346901
task-1915411

closes odoo/odoo#67004

Signed-off-by: Raphael Collet (rco) <rco@openerp.com>
Co-authored-by: Nicolas Seinlet <nse@odoo.com>

addons/mail/models/mail_activity.py

@@ -795,3 +795,70 @@ class MailActivityMixin(models.AbstractModel):
             domain = ['&'] + domain + [('user_id', '=', user_id)]
         self.env['mail.activity'].search(domain).unlink()
         return True
+
+    def _read_progress_bar(self, domain, group_by, progress_bar):
+        group_by_fname = group_by.partition(':')[0]
+        if not (progress_bar['field'] == 'activity_state' and self._fields[group_by_fname].store):
+            return super()._read_progress_bar(domain, group_by, progress_bar)
+
+        # optimization for 'activity_state'
+
+        # explicitly check access rights, since we bypass the ORM
+        self.check_access_rights('read')
+        query = self._where_calc(domain)
+        self._apply_ir_rules(query, 'read')
+        gb = group_by.partition(':')[0]
+        annotated_groupbys = [
+            self._read_group_process_groupby(gb, query)
+            for gb in [group_by, 'activity_state']
+        ]
+        groupby_dict = {gb['groupby']: gb for gb in annotated_groupbys}
+        for gb in annotated_groupbys:
+            if gb['field'] == 'activity_state':
+                gb['qualified_field'] = '"_last_activity_state"."activity_state"'
+        groupby_terms, orderby_terms = self._read_group_prepare('activity_state', [], annotated_groupbys, query)
+        select_terms = [
+            '%s as "%s"' % (gb['qualified_field'], gb['groupby'])
+            for gb in annotated_groupbys
+        ]
+        from_clause, where_clause, where_params = query.get_sql()
+        tz = self._context.get('tz') or self.env.user.tz or 'UTC'
+        select_query = """
+            SELECT 1 AS id, count(*) AS "__count", {fields}
+            FROM {from_clause}
+            JOIN (
+                SELECT res_id,
+                CASE
+                    WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %s))::date) > 0 THEN 'planned'
+                    WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %s))::date) < 0 THEN 'overdue'
+                    WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %s))::date) = 0 THEN 'today'
+                    ELSE null
+                END AS activity_state
+                FROM mail_activity
+                JOIN res_users ON (res_users.id = mail_activity.user_id)
+                JOIN res_partner ON (res_partner.id = res_users.partner_id)
+                WHERE res_model = '{model}'
+                GROUP BY res_id
+            ) AS "_last_activity_state" ON ("{table}".id = "_last_activity_state".res_id)
+            WHERE {where_clause}
+            GROUP BY {group_by}
+        """.format(
+            fields=', '.join(select_terms),
+            from_clause=from_clause,
+            model=self._name,
+            table=self._table,
+            where_clause=where_clause or '1=1',
+            group_by=', '.join(groupby_terms),
+        )
+        self.env.cr.execute(select_query, [tz] * 3 + where_params)
+        fetched_data = self.env.cr.dictfetchall()
+        self._read_group_resolve_many2one_fields(fetched_data, annotated_groupbys)
+        data = [
+            {key: self._read_group_prepare_data(key, val, groupby_dict)
+             for key, val in row.items()}
+            for row in fetched_data
+        ]
+        return [
+            self._read_group_format_result(vals, annotated_groupbys, [group_by], domain)
+            for vals in data
+        ]

addons/test_mail/models/test_mail_models.py

@@ -37,6 +37,7 @@ class MailTestActivity(models.Model):
     _inherit = ['mail.thread', 'mail.activity.mixin']
 
     name = fields.Char()
+    date = fields.Date()
     email_from = fields.Char()
     active = fields.Boolean(default=True)
 

addons/test_mail/tests/__init__.py

@@ -19,3 +19,4 @@ from . import test_discuss
 from . import test_performance
 from . import test_res_users
 from . import test_odoobot
+from . import test_read_progress_bar

addons/test_mail/tests/test_read_progress_bar.py

+# -*- coding: utf-8 -*-
+from odoo.tests import common
+from odoo import fields
+from datetime import timedelta
+
+
+class TestReadProgressBar(common.TransactionCase):
+    """Test for read_progress_bar"""
+
+    def setUp(self):
+        super(TestReadProgressBar, self).setUp()
+        self.Model = self.env['mail.test.activity']
+
+    def test_week_grouping(self):
+        """The labels associated to each record in read_progress_bar should match
+        the ones from read_group, even in edge cases like en_US locale on sundays
+        """
+        context = {"lang": "en_US"}
+        model = self.Model.with_context(context)
+        groupby = "date:week"
+        sunday1 = '2021-05-02'
+        sunday2 = '2021-05-09'
+        sunday3 = '2021-05-16'
+        # Don't mistake fields date and date_deadline:
+        # * date is just a random value
+        # * date_deadline defines activity_state
+        self.Model.create({'date': sunday1, 'name': "Yesterday, all my troubles seemed so far away"}).activity_schedule(
+            'test_mail.mail_act_test_todo',
+            summary="Make another test super asap (yesterday)",
+            date_deadline=fields.Date.context_today(model) - timedelta(days=7)
+        )
+        self.Model.create({'date': sunday2, 'name': "Things we said today"}).activity_schedule(
+            'test_mail.mail_act_test_todo',
+            summary="Make another test asap",
+            date_deadline=fields.Date.context_today(model)
+        )
+        self.Model.create({'date': sunday3, 'name': "Tomorrow Never Knows"}).activity_schedule(
+            'test_mail.mail_act_test_todo',
+            summary="Make a test tomorrow",
+            date_deadline=fields.Date.context_today(model) + timedelta(days=7)
+        )
+
+        progress_bar = {
+            'field': 'activity_state',
+            'colors': {
+                "overdue": 'danger',
+                "today": 'warning',
+                "planned": 'success',
+            }
+        }
+
+        domain = [('date', "!=", False)]
+        # call read_group to compute group names
+        groups = model.read_group(domain, fields=['date'], groupby=[groupby])
+        progressbars = model.read_progress_bar(domain, group_by=groupby, progress_bar=progress_bar)
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(len(progressbars), 3)
+
+        # format the read_progress_bar result to get a dictionary under this format : {activity_state: group_name}
+        # original format (after read_progress_bar) is : {group_name: {activity_state: count}}
+        pg_groups = {
+            next(activity_state for activity_state, count in data.items() if count): group_name \
+                for group_name, data in progressbars.items()
+        }
+
+        self.assertEqual(groups[0][groupby], pg_groups["overdue"])
+        self.assertEqual(groups[1][groupby], pg_groups["today"])
+        self.assertEqual(groups[2][groupby], pg_groups["planned"])