From ec9a543014a91be17398ba6e443578fe37c3a885 Mon Sep 17 00:00:00 2001
From: Antony Lesuisse <al@openerp.com>
Date: Mon, 19 Jan 2015 01:41:08 +0100
Subject: [PATCH] [FIX] dbmanager: backup support both zip and pg_dump custom
format
- let the user choose between the pg_dump custom format or the zip format including the filestore
- use file objects to allow dumps larger than memory
- postgres subprocess invocation is now clean and thread-safe, we dont touch the local process environ anymore
- add a manifest to the zip dump format with version information about odoo, postgres (pg_dump doesnt output it) and modules
---
addons/web/controllers/main.py | 26 +++----
addons/web/static/src/xml/base.xml | 9 +++
openerp/service/db.py | 114 ++++++++++++++---------------
openerp/tools/misc.py | 67 ++++++++++-------
4 files changed, 119 insertions(+), 97 deletions(-)
diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py
index 1cea529a86a0..8bf395dc59b3 100644
--- a/addons/web/controllers/main.py
+++ b/addons/web/controllers/main.py
@@ -724,21 +724,21 @@ class Database(http.Controller):
return {'error': _('Could not drop database !'), 'title': _('Drop Database')}
@http.route('/web/database/backup', type='http', auth="none")
- def backup(self, backup_db, backup_pwd, token):
+ def backup(self, backup_db, backup_pwd, token, backup_format='zip'):
try:
- db_dump = base64.b64decode(
- request.session.proxy("db").dump(backup_pwd, backup_db))
- filename = "%(db)s_%(timestamp)s.dump" % {
- 'db': backup_db,
- 'timestamp': datetime.datetime.utcnow().strftime(
- "%Y-%m-%d_%H-%M-%SZ")
- }
- return request.make_response(db_dump,
- [('Content-Type', 'application/octet-stream; charset=binary'),
- ('Content-Disposition', content_disposition(filename))],
- {'fileToken': token}
- )
+ openerp.service.security.check_super(backup_pwd)
+ ts = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S")
+ filename = "%s_%s.%s" % (backup_db, ts, backup_format)
+ headers = [
+ ('Content-Type', 'application/octet-stream; charset=binary'),
+ ('Content-Disposition', content_disposition(filename)),
+ ]
+ dump_stream = openerp.service.db.dump_db_stream(backup_db, backup_format)
+ response = werkzeug.wrappers.Response(dump_stream, headers=headers, direct_passthrough=True)
+ response.set_cookie('fileToken', token)
+ return response
except Exception, e:
+ _logger.exception('Database.backup')
return simplejson.dumps([[],[{'error': openerp.tools.ustr(e), 'title': _('Backup Database')}]])
@http.route('/web/database/restore', type='http', auth="none")
diff --git a/addons/web/static/src/xml/base.xml b/addons/web/static/src/xml/base.xml
index 9b80a868c6dc..864bc8999605 100644
--- a/addons/web/static/src/xml/base.xml
+++ b/addons/web/static/src/xml/base.xml
@@ -225,6 +225,15 @@
<input t-if="!widget.db_list" name="backup_db" class="required" type="text" autofocus="autofocus"/>
</td>
</tr>
+ <tr>
+ <td><label for="backup_format">Backup format:</label></td>
+ <td class="oe_form_field oe_form_field_selection">
+ <select name="backup_format">
+ <option value="zip">zip (includes filestore)</option>
+ <option value="dump">pg_dump custom format (without filestore)</option>
+ </select>
+ </td>
+ </tr>
<tr>
<td><label for="backup_pwd">Master Password:</label></td>
<td><input type="password" name="backup_pwd" class="required" /></td>
diff --git a/openerp/service/db.py b/openerp/service/db.py
index 048c160916f1..a0a4b5e2707b 100644
--- a/openerp/service/db.py
+++ b/openerp/service/db.py
@@ -1,14 +1,17 @@
# -*- coding: utf-8 -*-
-from contextlib import closing
-from functools import wraps
+
+import json
import logging
import os
import shutil
+import tempfile
import threading
import traceback
-import tempfile
import zipfile
+from functools import wraps
+from contextlib import closing
+
import psycopg2
import openerp
@@ -143,70 +146,65 @@ def exp_drop(db_name):
shutil.rmtree(fs)
return True
-def _set_pg_password_in_environment(func):
- """ On systems where pg_restore/pg_dump require an explicit
- password (i.e. when not connecting via unix sockets, and most
- importantly on Windows), it is necessary to pass the PG user
- password in the environment or in a special .pgpass file.
-
- This decorator handles setting
- :envvar:`PGPASSWORD` if it is not already
- set, and removing it afterwards.
-
- See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
-
- .. note:: This is not thread-safe, and should never be enabled for
- SaaS (giving SaaS users the super-admin password is not a good idea
- anyway)
- """
- @wraps(func)
- def wrapper(*args, **kwargs):
- if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
- return func(*args, **kwargs)
- else:
- os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
- try:
- return func(*args, **kwargs)
- finally:
- del os.environ['PGPASSWORD']
- return wrapper
-
def exp_dump(db_name):
with tempfile.TemporaryFile() as t:
dump_db(db_name, t)
t.seek(0)
return t.read().encode('base64')
-@_set_pg_password_in_environment
-def dump_db(db, stream):
+def dump_db_manifest(cr):
+ pg_version = "%d.%d" % divmod(cr._obj.connection.server_version / 100, 100)
+ env = openerp.api.Environment(cr, SUPERUSER_ID, {})
+ modules = dict([(i.name,i.latest_version) for i in env['ir.module.module'].search([('state','=','installed')])])
+ manifest = {
+ 'odoo_dump': '1',
+ 'db_name': cr.dbname,
+ 'version': openerp.release.version,
+ 'version_info': openerp.release.version_info,
+ 'major_version': openerp.release.major_version,
+ 'pg_version': pg_version,
+ 'modules': modules,
+ }
+ return manifest
+
+def dump_db(db_name, stream, backup_format='zip'):
"""Dump database `db` into file-like object `stream`"""
- with openerp.tools.osutil.tempdir() as dump_dir:
- registry = openerp.modules.registry.RegistryManager.get(db)
- with registry.cursor() as cr:
- filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
- if os.path.exists(filestore):
- shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
- dump_file = os.path.join(dump_dir, 'dump.sql')
- cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
- if openerp.tools.config['db_user']:
- cmd.append('--username=' + openerp.tools.config['db_user'])
- if openerp.tools.config['db_host']:
- cmd.append('--host=' + openerp.tools.config['db_host'])
- if openerp.tools.config['db_port']:
- cmd.append('--port=' + str(openerp.tools.config['db_port']))
- cmd.append(db)
-
- if openerp.tools.exec_pg_command(*cmd):
- _logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
- 'password on the server. You may need to create a .pgpass file for '
- 'authentication, or specify `db_password` in the server configuration '
- 'file.', db)
- raise Exception("Couldn't dump database")
+ cmd = ['pg_dump', '--no-owner']
+ if openerp.tools.config['db_user']:
+ cmd.append('--username=' + openerp.tools.config['db_user'])
+ if openerp.tools.config['db_host']:
+ cmd.append('--host=' + openerp.tools.config['db_host'])
+ if openerp.tools.config['db_port']:
+ cmd.append('--port=' + str(openerp.tools.config['db_port']))
+ cmd.append(db_name)
+
+ if backup_format == 'zip':
+ with openerp.tools.osutil.tempdir() as dump_dir:
+ registry = openerp.modules.registry.RegistryManager.get(db_name)
+ with registry.cursor() as cr:
+ filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
+ if os.path.exists(filestore):
+ shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
+ manifest = dump_db_manifest(cr)
+ with open(os.path.join(dump_dir, 'manifest.json'), 'w') as fh:
+ json.dump(manifest, fh, indent=4)
+ cmd.insert(-1, '--file=' + os.path.join(dump_dir, 'dump.sql'))
+ openerp.tools.exec_pg_command(*cmd)
+ openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
+ else:
+ cmd.insert(-1, '--format=c')
+ print cmd
+ stdin, stdout = openerp.tools.exec_pg_command_pipe(*cmd)
+ shutil.copyfileobj(stdout, stream)
- openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
+ _logger.info('DUMP DB successful: %s', db_name)
- _logger.info('DUMP DB successful: %s', db)
+def dump_db_stream(db_name, backup_format='zip'):
+ t=tempfile.TemporaryFile()
+ dump_db(db_name, t, backup_format)
+ t.seek(0)
+ return t
def exp_restore(db_name, data, copy=False):
data_file = tempfile.NamedTemporaryFile(delete=False)
@@ -218,7 +216,6 @@ def exp_restore(db_name, data, copy=False):
os.unlink(data_file.name)
return True
-@_set_pg_password_in_environment
def restore_db(db, dump_file, copy=False):
assert isinstance(db, basestring)
if exp_db_exist(db):
@@ -351,4 +348,3 @@ def exp_migrate_databases(databases):
openerp.modules.registry.RegistryManager.new(db, force_demo=False, update_module=True)
return True
-# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py
index ca835f33b010..6dccab4b11ee 100644
--- a/openerp/tools/misc.py
+++ b/openerp/tools/misc.py
@@ -65,6 +65,10 @@ _logger = logging.getLogger(__name__)
# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase)
+#----------------------------------------------------------
+# Subprocesses
+#----------------------------------------------------------
+
def find_in_path(name):
path = os.environ.get('PATH', os.defpath).split(os.pathsep)
if config.get('bin_path') and config['bin_path'] != 'None':
@@ -74,6 +78,24 @@ def find_in_path(name):
except IOError:
return None
+def _exec_pipe(prog, args, env=None):
+ cmd = (prog,) + args
+ # on win32, passing close_fds=True is not compatible
+ # with redirecting std[in/err/out]
+ close_fds = os.name=="posix"
+ pop = subprocess.Popen(cmd, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=close_fds, env=env)
+ return pop.stdin, pop.stdout
+
+def exec_command_pipe(name, *args):
+ prog = find_in_path(name)
+ if not prog:
+ raise Exception('Command `%s` not found.' % name)
+ _exec_pipe(prog, *args)
+
+#----------------------------------------------------------
+# Postgres subprocesses
+#----------------------------------------------------------
+
def find_pg_tool(name):
path = None
if config['pg_path'] and config['pg_path'] != 'None':
@@ -81,38 +103,33 @@ def find_pg_tool(name):
try:
return which(name, path=path)
except IOError:
- return None
+ raise Exception('Command `%s` not found.' % name)
+
+def exec_pg_environ():
+ """ On systems where pg_restore/pg_dump require an explicit password (i.e.
+ on Windows where TCP sockets are used), it is necessary to pass the
+ postgres user password in the PGPASSWORD environment variable or in a
+ special .pgpass file.
+
+ See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
+ """
+ env = os.environ.copy()
+ if not env.get('PGPASSWORD') and openerp.tools.config['db_password']:
+ env['PGPASSWORD'] = openerp.tools.config['db_password']
+ return env
def exec_pg_command(name, *args):
prog = find_pg_tool(name)
- if not prog:
- raise Exception('Couldn\'t find %s' % name)
- args2 = (prog,) + args
-
+ env = exec_pg_environ()
with open(os.devnull) as dn:
- return subprocess.call(args2, stdout=dn, stderr=subprocess.STDOUT)
+ rc = subprocess.call((prog,) + args, stdout=dn, stderr=subprocess.STDOUT)
+ if rc:
+ raise Exception('Postgres subprocess %s error %s' % (args2, rc))
def exec_pg_command_pipe(name, *args):
prog = find_pg_tool(name)
- if not prog:
- raise Exception('Couldn\'t find %s' % name)
- # on win32, passing close_fds=True is not compatible
- # with redirecting std[in/err/out]
- pop = subprocess.Popen((prog,) + args, bufsize= -1,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- close_fds=(os.name=="posix"))
- return pop.stdin, pop.stdout
-
-def exec_command_pipe(name, *args):
- prog = find_in_path(name)
- if not prog:
- raise Exception('Couldn\'t find %s' % name)
- # on win32, passing close_fds=True is not compatible
- # with redirecting std[in/err/out]
- pop = subprocess.Popen((prog,) + args, bufsize= -1,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- close_fds=(os.name=="posix"))
- return pop.stdin, pop.stdout
+ env = exec_pg_environ()
+ return _exec_pipe(prog, args, env)
#----------------------------------------------------------
# File paths
--
GitLab