[IMP] inter-process signaling, proof-of-concept.

bzr revid: vmt@openerp.com-20120202173522-2grq11zfm7855i6s

gunicorn.conf.py

@@ -17,7 +17,7 @@ pidfile = '.gunicorn.pid'
 # Gunicorn recommends 2-4 x number_of_cpu_cores, but
 # you'll want to vary this a bit to find the best for your
 # particular work load.
-workers = 4
+workers = 1
 
 # Some application-wide initialization is needed.
 on_starting = openerp.wsgi.on_starting
@@ -31,6 +31,8 @@ timeout = 240
 
 max_requests = 2000
 
+#accesslog = '/tmp/blah.txt'
+
 # Equivalent of --load command-line option
 openerp.conf.server_wide_modules = ['web']
 
@@ -39,7 +41,7 @@ conf = openerp.tools.config
 
 # Path to the OpenERP Addons repository (comma-separated for
 # multiple locations)
-conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons'
+conf['addons_path'] = '/home/thu/repos/addons/trunk,/home/thu/repos/web/trunk/addons'
 
 # Optional database config if not using local socket
 #conf['db_name'] = 'mycompany'

openerp-server

@@ -247,7 +247,7 @@ if __name__ == "__main__":
             # Call any post_load hook.
             info = openerp.modules.module.load_information_from_description_file(m)
             if info['post_load']:
-                getattr(sys.modules[m], info['post_load'])()
+                getattr(sys.modules['openerp.addons.' + m], info['post_load'])()
         except Exception:
             msg = ''
             if m == 'web':

openerp/addons/base/base.sql

@@ -347,6 +347,14 @@ CREATE TABLE ir_model_data (
     res_id integer, primary key(id)
 );
 
+-- Inter-process signaling:
+-- The `base_registry_signaling` sequence indicates the whole registry
+-- must be reloaded.
+-- The `base_cache_signaling sequence` indicates all caches must be
+-- invalidated (i.e. cleared).
+CREATE SEQUENCE base_registry_signaling INCREMENT BY 1 START WITH 1;
+CREATE SEQUENCE base_cache_signaling INCREMENT BY 1 START WITH 1;
+
 ---------------------------------
 -- Users
 ---------------------------------

openerp/osv/osv.py

@@ -34,6 +34,8 @@ from openerp.tools.translate import translate
 from openerp.osv.orm import MetaModel, Model, TransientModel, AbstractModel
 import openerp.exceptions
 
+_logger = logging.getLogger(__name__)
+
 # Deprecated.
 class except_osv(Exception):
     def __init__(self, name, value):
@@ -43,6 +45,14 @@ class except_osv(Exception):
 
 service = None
 
+# Inter-process signaling:
+# The `base_registry_signaling` sequence indicates the whole registry
+# must be reloaded.
+# The `base_cache_signaling sequence` indicates all caches must be
+# invalidated (i.e. cleared).
+base_registry_signaling_sequence = None
+base_cache_signaling_sequence = None
+
 class object_proxy(object):
     def __init__(self):
         self.logger = logging.getLogger('web-services')
@@ -167,6 +177,40 @@ class object_proxy(object):
 
     @check
     def execute(self, db, uid, obj, method, *args, **kw):
+
+        # Check if the model registry must be reloaded (e.g. after the
+        # database has been updated by another process).
+        cr = pooler.get_db(db).cursor()
+        registry_reloaded = False
+        try:
+            cr.execute('select last_value from base_registry_signaling')
+            r = cr.fetchone()[0]
+            global base_registry_signaling_sequence
+            if base_registry_signaling_sequence != r:
+                _logger.info("Reloading the model registry after database signaling.")
+                base_registry_signaling_sequence = r
+                # Don't run the cron in the Gunicorn worker.
+                openerp.modules.registry.RegistryManager.new(db, pooljobs=False)
+                registry_reloaded = True
+        finally:
+            cr.close()
+
+        # Check if the model caches must be invalidated (e.g. after a write
+        # occured on another process). Don't clear right after a registry
+        # has been reload.
+        cr = pooler.get_db(db).cursor()
+        try:
+            cr.execute('select last_value from base_cache_signaling')
+            r = cr.fetchone()[0]
+            global base_cache_signaling_sequence
+            if base_cache_signaling_sequence != r and not registry_reloaded:
+                _logger.info("Invalidating all model caches after database signaling.")
+                base_cache_signaling_sequence = r
+                registry = openerp.modules.registry.RegistryManager.get(db, pooljobs=False)
+                registry.clear_caches()
+        finally:
+            cr.close()
+
         cr = pooler.get_db(db).cursor()
         try:
             try:

openerp/service/__init__.py

@@ -63,7 +63,7 @@ def start_services():
     netrpc_server.init_servers()
 
     # Start the main cron thread.
-    openerp.cron.start_master_thread()
+    #openerp.cron.start_master_thread()
 
     # Start the top-level servers threads (normally HTTP, HTTPS, and NETRPC).
     openerp.netsvc.Server.startAll()

openerp/wsgi.py

@@ -421,7 +421,7 @@ def serve():
     port = config['xmlrpc_port']
     try:
         import werkzeug.serving
-        httpd = werkzeug.serving.make_server(interface, port, application, threaded=True)
+        httpd = werkzeug.serving.make_server(interface, port, application, threaded=False)
         logging.getLogger('wsgi').info('HTTP service (werkzeug) running on %s:%s', interface, port)
     except ImportError:
         import wsgiref.simple_server
@@ -436,7 +436,7 @@ def start_server():
 
     The WSGI server can be shutdown with stop_server() below.
     """
-    threading.Thread(target=openerp.wsgi.serve).start()
+    threading.Thread(name='WSGI server', target=openerp.wsgi.serve).start()
 
 def stop_server():
     """ Initiate the shutdown of the WSGI server.
@@ -462,11 +462,11 @@ def on_starting(server):
     openerp.modules.loading.open_openerp_namespace()
     for m in openerp.conf.server_wide_modules:
         try:
-            __import__(m)
+            __import__('openerp.addons.' + m)
             # Call any post_load hook.
             info = openerp.modules.module.load_information_from_description_file(m)
             if info['post_load']:
-                getattr(sys.modules[m], info['post_load'])()
+                getattr(sys.modules['openerp.addons.' + m], info['post_load'])()
         except Exception:
             msg = ''
             if m == 'web':