[IMP] Minimizes memory consumption

Author: lmignon

report_py3o/models/py3o_report.py

@@ -8,12 +8,15 @@
 import json
 import logging
 import os
+from contextlib import closing
+
 import pkg_resources
 import requests
 import sys
-from tempfile import NamedTemporaryFile
+import tempfile
 from zipfile import ZipFile, ZIP_DEFLATED
 
+from openerp.exceptions import AccessError
 from openerp.exceptions import UserError
 from openerp.report.report_sxw import rml_parse
 from openerp import api, fields, models, _
@@ -149,49 +152,72 @@ def _get_parser_context(self, model_instance, data):
         self._extend_parser_context(context_instance, report_xml)
         return context_instance.localcontext
 
-    @api.multi
-    def _postprocess_report(self, content, res_id, save_in_attachment):
+    @api.model
+    def _get_report_from_name(self, report_name):
+        """Get the first record of ir.actions.report.xml having the
+        ``report_name`` as value for the field report_name.
+        """
+        res = super(Py3oReport, self)._get_report_from_name(report_name)
+        if res:
+            return res
+        # maybe a py3o reprot
+        report_obj = self.env['ir.actions.report.xml']
+        return report_obj.search(
+            [('report_type', '=', 'py3o'),
+             ('report_name', '=', report_name)])
+
+    @api.model
+    def _postprocess_report(self, report_path, res_id, save_in_attachment):
         if save_in_attachment.get(res_id):
-            attachment = {
-                'name': save_in_attachment.get(res_id),
-                'datas': base64.encodestring(content),
-                'datas_fname': save_in_attachment.get(res_id),
-                'res_model': save_in_attachment.get('model'),
-                'res_id': res_id,
-            }
-            return self.env['ir.attachment'].create(attachment)
-        return False
+            with open(report_path, 'rb') as pdfreport:
+                attachment = {
+                    'name': save_in_attachment.get(res_id),
+                    'datas': base64.encodestring(pdfreport.read()),
+                    'datas_fname': save_in_attachment.get(res_id),
+                    'res_model': save_in_attachment.get('model'),
+                    'res_id': res_id,
+                }
+                try:
+                    self.env['ir.attachment'].create(attachment)
+                except AccessError:
+                    logger.info("Cannot save PDF report %r as attachment",
+                                attachment['name'])
+                else:
+                    logger.info(
+                        'The PDF document %s is now saved in the database',
+                        attachment['name'])
 
     @api.multi
     def _create_single_report(self, model_instance, data, save_in_attachment):
         """ This function to generate our py3o report
         """
         self.ensure_one()
         report_xml = self.ir_actions_report_xml_id
-
+        filetype = report_xml.py3o_filetype
+        result_fd, result_path = tempfile.mkstemp(
+            suffix='.' + filetype, prefix='p3o.report.tmp.')
         tmpl_data = self.get_template()
 
         in_stream = StringIO(tmpl_data)
-        out_stream = StringIO()
-        template = Template(in_stream, out_stream, escape_false=True)
-        localcontext = self._get_parser_context(model_instance, data)
-        if report_xml.py3o_is_local_fusion:
-            template.render(localcontext)
-            in_stream = out_stream
-            datadict = {}
-        else:
-            expressions = template.get_all_user_python_expression()
-            py_expression = template.convert_py3o_to_python_ast(expressions)
-            convertor = Py3oConvertor()
-            data_struct = convertor(py_expression)
-            datadict = data_struct.render(localcontext)
-
-        filetype = report_xml.py3o_filetype
-        is_native = Formats().get_format(filetype).native
-        if is_native:
-            res = out_stream.getvalue()
-        else:  # Call py3o.server to render the template in the desired format
-            in_stream.seek(0)
+        with closing(os.fdopen(result_fd, 'w+')) as out_stream:
+            template = Template(in_stream, out_stream, escape_false=True)
+            localcontext = self._get_parser_context(model_instance, data)
+            is_native = Formats().get_format(filetype).native
+            if report_xml.py3o_is_local_fusion:
+                template.render(localcontext)
+                out_stream.seek(0)
+                in_stream = out_stream.read()
+                datadict = {}
+            else:
+                expressions = template.get_all_user_python_expression()
+                py_expression = template.convert_py3o_to_python_ast(
+                    expressions)
+                convertor = Py3oConvertor()
+                data_struct = convertor(py_expression)
+                datadict = data_struct.render(localcontext)
+
+        if not is_native or not report_xml.py3o_is_local_fusion:
+            # Call py3o.server to render the template in the desired format
             files = {
                 'tmpl_file': in_stream,
             }
@@ -210,21 +236,13 @@ def _create_single_report(self, model_instance, data, save_in_attachment):
                     _('Fusion server error %s') % r.text,
                 )
 
-            # Here is a little joke about Odoo
-            # we do nice chunked reading from the network...
             chunk_size = 1024
-            with NamedTemporaryFile(
-                suffix=filetype,
-                prefix='py3o-template-'
-            ) as fd:
+            with open(result_path, 'w+') as fd:
                 for chunk in r.iter_content(chunk_size):
                     fd.write(chunk)
-                fd.seek(0)
-                # ... but odoo wants the whole data in memory anyways :)
-                res = fd.read()
         self._postprocess_report(
-            res, model_instance.id, save_in_attachment)
-        return res, "." + self.ir_actions_report_xml_id.py3o_filetype
+            result_path, model_instance.id, save_in_attachment)
+        return result_path
 
     @api.multi
     def _get_or_create_single_report(self, model_instance, data,
@@ -239,43 +257,42 @@ def _get_or_create_single_report(self, model_instance, data,
             model_instance, data, save_in_attachment)
 
     @api.multi
-    def _zip_results(self, results):
+    def _zip_results(self, reports_path):
         self.ensure_one()
         zfname_prefix = self.ir_actions_report_xml_id.name
-        with NamedTemporaryFile(suffix="zip", prefix='py3o-zip-result') as fd:
-            with ZipFile(fd, 'w', ZIP_DEFLATED) as zf:
-                cpt = 0
-                for r, ext in results:
-                    fname = "%s_%d.%s" % (zfname_prefix, cpt, ext)
-                    zf.writestr(fname, r)
-                    cpt += 1
-            fd.seek(0)
-            return fd.read(), 'zip'
+        result_path = tempfile.mktemp(suffix="zip", prefix='py3o-zip-result')
+        with ZipFile(result_path, 'w', ZIP_DEFLATED) as zf:
+            cpt = 0
+            for report in reports_path:
+                fname = "%s_%d.%s" % (
+                    zfname_prefix, cpt, report.split('.')[-1])
+                zf.write(report, fname)
 
-    @api.multi
-    def _merge_pdfs(self, results):
-        from pyPdf import PdfFileWriter, PdfFileReader
-        output = PdfFileWriter()
-        for r in results:
-            reader = PdfFileReader(StringIO(r[0]))
-            for page in range(reader.getNumPages()):
-                output.addPage(reader.getPage(page))
-        s = StringIO()
-        output.write(s)
-        return s.getvalue(), formats.FORMAT_PDF
+                cpt += 1
+        return result_path
 
     @api.multi
-    def _merge_results(self, results):
+    def _merge_results(self, reports_path):
         self.ensure_one()
-        if not results:
-            return False, False
-        if len(results) == 1:
-            return results[0]
         filetype = self.ir_actions_report_xml_id.py3o_filetype
+        if not reports_path:
+            return False, False
+        if len(reports_path) == 1:
+            return reports_path[0], filetype
         if filetype == formats.FORMAT_PDF:
-            return self._merge_pdfs(results)
+            return self._merge_pdf(reports_path), formats.FORMAT_PDF
         else:
-            return self._zip_results(results)
+            return self._zip_results(reports_path), 'zip'
+
+    @api.model
+    def _cleanup_tempfiles(self, temporary_files):
+        # Manual cleanup of the temporary files
+        for temporary_file in temporary_files:
+            try:
+                os.unlink(temporary_file)
+            except (OSError, IOError):
+                logger.error(
+                    'Error when trying to remove file %s' % temporary_file)
 
     @api.multi
     def create_report(self, res_ids, data):
@@ -285,8 +302,21 @@ def create_report(self, res_ids, data):
             res_ids)
         save_in_attachment = self._check_attachment_use(
             model_instances, self.ir_actions_report_xml_id) or {}
-        results = []
+        reports_path = []
         for model_instance in model_instances:
-            results.append(self._get_or_create_single_report(
-                model_instance, data, save_in_attachment))
-        return self._merge_results(results)
+            reports_path.append(
+                self._get_or_create_single_report(
+                    model_instance, data, save_in_attachment))
+
+        result_path, filetype = self._merge_results(reports_path)
+        reports_path.append(result_path)
+
+        # Here is a little joke about Odoo
+        # we do all the generation process using files to avoid memory
+        # consumption...
+        # ... but odoo wants the whole data in memory anyways :)
+
+        with open(result_path, 'r+b') as fd:
+            res = fd.read()
+        self._cleanup_tempfiles(set(reports_path))
+        return res, filetype

report_py3o/tests/test_report_py3o.py

@@ -5,6 +5,7 @@
 import mock
 import os
 import pkg_resources
+import tempfile
 
 from py3o.formats import Formats
 
@@ -60,11 +61,17 @@ def test_reports(self):
         report = self.env.ref("report_py3o.res_users_report_py3o")
         with mock.patch.object(
                 py3o_report.__class__, '_create_single_report') as patched_pdf:
+            result = tempfile.mktemp('.txt')
+            with open(result, 'w') as fp:
+                fp.write('dummy')
+            patched_pdf.return_value = result
             # test the call the the create method inside our custom parser
             report.render_report(self.env.user.ids,
                                  report.report_name,
                                  {})
             self.assertEqual(1, patched_pdf.call_count)
+            # generated files no more exists
+            self.assertFalse(os.path.exists(result))
         res = report.render_report(
             self.env.user.ids, report.report_name, {})
         self.assertTrue(res)
@@ -79,7 +86,7 @@ def test_reports(self):
             magick_response.iter_content.return_value = "test result"
             res = report.render_report(
                 self.env.user.ids, report.report_name, {})
-            self.assertEqual(('test result', '.pdf'), res)
+            self.assertEqual(('test result', 'pdf'), res)
 
     def test_report_template_configs(self):
         report = self.env.ref("report_py3o.res_users_report_py3o")