From d17c20e28879d4abd951d56f367e4c1fd21dfe7f Mon Sep 17 00:00:00 2001
From: Ivan Yelizariev <iel@odoo.com>
Date: Mon, 5 Jul 2021 12:49:19 +0000
Subject: [PATCH] [FIX] mail: speed up read_progress_bar
This commit fixes the performance issue in getting statistics for
``activity_state`` (colored clock icon for overdue/today/planned) in
CRM. The query has been tested for several years on a large database
(Odoo's own production database).
Performance test on 29 K crm.lead records (activity_state):
With a filter for 10 records:
```
| measurement | before | after |
|--------------------+--------+-------|
| number of queries | 25 | 5 |
| query time, ms | 12 | 95 | (*)
| remaining time, ms | 32 | 7 |
```
All records:
```
| measurement | before | after |
|--------------------+--------+-------|
| number of queries | 1326 | 5 |
| query time, ms | 1739 | 129 |
| remaining time, ms | 47934 | 17 |
```
As we can see in the last results, the time went from almost 50 seconds
(not responsive at all) to 150 milliseconds (responsive). The time
increase in (*) may be caused by imperfect measurements, which are raw
and not averaged measures.
---
opw-2346901
task-1915411
closes odoo/odoo#67004
Signed-off-by: Raphael Collet (rco) <rco@openerp.com>
Co-authored-by: Nicolas Seinlet <nse@odoo.com>
---
addons/mail/models/mail_activity.py | 67 ++++++++++++++++++
addons/test_mail/models/test_mail_models.py | 1 +
addons/test_mail/tests/__init__.py | 1 +
.../test_mail/tests/test_read_progress_bar.py | 68 +++++++++++++++++++
4 files changed, 137 insertions(+)
create mode 100644 addons/test_mail/tests/test_read_progress_bar.py
diff --git a/addons/mail/models/mail_activity.py b/addons/mail/models/mail_activity.py
index 47161cb73f97..46481fa0412e 100644
--- a/addons/mail/models/mail_activity.py
+++ b/addons/mail/models/mail_activity.py
@@ -795,3 +795,70 @@ class MailActivityMixin(models.AbstractModel):
domain = ['&'] + domain + [('user_id', '=', user_id)]
self.env['mail.activity'].search(domain).unlink()
return True
+
+ def _read_progress_bar(self, domain, group_by, progress_bar):
+ group_by_fname = group_by.partition(':')[0]
+ if not (progress_bar['field'] == 'activity_state' and self._fields[group_by_fname].store):
+ return super()._read_progress_bar(domain, group_by, progress_bar)
+
+ # optimization for 'activity_state'
+
+ # explicitly check access rights, since we bypass the ORM
+ self.check_access_rights('read')
+ query = self._where_calc(domain)
+ self._apply_ir_rules(query, 'read')
+ gb = group_by.partition(':')[0]
+ annotated_groupbys = [
+ self._read_group_process_groupby(gb, query)
+ for gb in [group_by, 'activity_state']
+ ]
+ groupby_dict = {gb['groupby']: gb for gb in annotated_groupbys}
+ for gb in annotated_groupbys:
+ if gb['field'] == 'activity_state':
+ gb['qualified_field'] = '"_last_activity_state"."activity_state"'
+ groupby_terms, orderby_terms = self._read_group_prepare('activity_state', [], annotated_groupbys, query)
+ select_terms = [
+ '%s as "%s"' % (gb['qualified_field'], gb['groupby'])
+ for gb in annotated_groupbys
+ ]
+ from_clause, where_clause, where_params = query.get_sql()
+ tz = self._context.get('tz') or self.env.user.tz or 'UTC'
+ select_query = """
+ SELECT 1 AS id, count(*) AS "__count", {fields}
+ FROM {from_clause}
+ JOIN (
+ SELECT res_id,
+ CASE
+ WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %s))::date) > 0 THEN 'planned'
+ WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %s))::date) < 0 THEN 'overdue'
+ WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %s))::date) = 0 THEN 'today'
+ ELSE null
+ END AS activity_state
+ FROM mail_activity
+ JOIN res_users ON (res_users.id = mail_activity.user_id)
+ JOIN res_partner ON (res_partner.id = res_users.partner_id)
+ WHERE res_model = '{model}'
+ GROUP BY res_id
+ ) AS "_last_activity_state" ON ("{table}".id = "_last_activity_state".res_id)
+ WHERE {where_clause}
+ GROUP BY {group_by}
+ """.format(
+ fields=', '.join(select_terms),
+ from_clause=from_clause,
+ model=self._name,
+ table=self._table,
+ where_clause=where_clause or '1=1',
+ group_by=', '.join(groupby_terms),
+ )
+ self.env.cr.execute(select_query, [tz] * 3 + where_params)
+ fetched_data = self.env.cr.dictfetchall()
+ self._read_group_resolve_many2one_fields(fetched_data, annotated_groupbys)
+ data = [
+ {key: self._read_group_prepare_data(key, val, groupby_dict)
+ for key, val in row.items()}
+ for row in fetched_data
+ ]
+ return [
+ self._read_group_format_result(vals, annotated_groupbys, [group_by], domain)
+ for vals in data
+ ]
diff --git a/addons/test_mail/models/test_mail_models.py b/addons/test_mail/models/test_mail_models.py
index 3347744987f9..69b5a1c3be2b 100644
--- a/addons/test_mail/models/test_mail_models.py
+++ b/addons/test_mail/models/test_mail_models.py
@@ -37,6 +37,7 @@ class MailTestActivity(models.Model):
_inherit = ['mail.thread', 'mail.activity.mixin']
name = fields.Char()
+ date = fields.Date()
email_from = fields.Char()
active = fields.Boolean(default=True)
diff --git a/addons/test_mail/tests/__init__.py b/addons/test_mail/tests/__init__.py
index 2f63dbc73a23..3ff3f2bb136b 100644
--- a/addons/test_mail/tests/__init__.py
+++ b/addons/test_mail/tests/__init__.py
@@ -19,3 +19,4 @@ from . import test_discuss
from . import test_performance
from . import test_res_users
from . import test_odoobot
+from . import test_read_progress_bar
diff --git a/addons/test_mail/tests/test_read_progress_bar.py b/addons/test_mail/tests/test_read_progress_bar.py
new file mode 100644
index 000000000000..341438507b5d
--- /dev/null
+++ b/addons/test_mail/tests/test_read_progress_bar.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+from odoo.tests import common
+from odoo import fields
+from datetime import timedelta
+
+
+class TestReadProgressBar(common.TransactionCase):
+ """Test for read_progress_bar"""
+
+ def setUp(self):
+ super(TestReadProgressBar, self).setUp()
+ self.Model = self.env['mail.test.activity']
+
+ def test_week_grouping(self):
+ """The labels associated to each record in read_progress_bar should match
+ the ones from read_group, even in edge cases like en_US locale on sundays
+ """
+ context = {"lang": "en_US"}
+ model = self.Model.with_context(context)
+ groupby = "date:week"
+ sunday1 = '2021-05-02'
+ sunday2 = '2021-05-09'
+ sunday3 = '2021-05-16'
+ # Don't mistake fields date and date_deadline:
+ # * date is just a random value
+ # * date_deadline defines activity_state
+ self.Model.create({'date': sunday1, 'name': "Yesterday, all my troubles seemed so far away"}).activity_schedule(
+ 'test_mail.mail_act_test_todo',
+ summary="Make another test super asap (yesterday)",
+ date_deadline=fields.Date.context_today(model) - timedelta(days=7)
+ )
+ self.Model.create({'date': sunday2, 'name': "Things we said today"}).activity_schedule(
+ 'test_mail.mail_act_test_todo',
+ summary="Make another test asap",
+ date_deadline=fields.Date.context_today(model)
+ )
+ self.Model.create({'date': sunday3, 'name': "Tomorrow Never Knows"}).activity_schedule(
+ 'test_mail.mail_act_test_todo',
+ summary="Make a test tomorrow",
+ date_deadline=fields.Date.context_today(model) + timedelta(days=7)
+ )
+
+ progress_bar = {
+ 'field': 'activity_state',
+ 'colors': {
+ "overdue": 'danger',
+ "today": 'warning',
+ "planned": 'success',
+ }
+ }
+
+ domain = [('date', "!=", False)]
+ # call read_group to compute group names
+ groups = model.read_group(domain, fields=['date'], groupby=[groupby])
+ progressbars = model.read_progress_bar(domain, group_by=groupby, progress_bar=progress_bar)
+ self.assertEqual(len(groups), 3)
+ self.assertEqual(len(progressbars), 3)
+
+ # format the read_progress_bar result to get a dictionary under this format : {activity_state: group_name}
+ # original format (after read_progress_bar) is : {group_name: {activity_state: count}}
+ pg_groups = {
+ next(activity_state for activity_state, count in data.items() if count): group_name \
+ for group_name, data in progressbars.items()
+ }
+
+ self.assertEqual(groups[0][groupby], pg_groups["overdue"])
+ self.assertEqual(groups[1][groupby], pg_groups["today"])
+ self.assertEqual(groups[2][groupby], pg_groups["planned"])
--
GitLab