fix 'savepoint does not exist' stack trace at statement import

account_ebics/__manifest__.py

@@ -3,7 +3,7 @@
 
 {
     "name": "EBICS banking protocol",
-    "version": "16.0.1.3.0",
+    "version": "16.0.1.3.1",
     "license": "LGPL-3",
     "author": "Noviat",
     "website": "https://www.noviat.com",

account_ebics/models/ebics_file.py

@@ -180,21 +180,7 @@ class EbicsFile(models.Model):
     def _process_download_result(self, res):
         statement_ids = res["statement_ids"]
         notifications = res["notifications"]
-        sts_data = []
+        statements = self.env["account.bank.statement"].sudo().browse(statement_ids)
-        if statement_ids:
-            self.env.flush_all()
-            self.env.cr.execute(
-                """
-            SELECT abs.name, abs.date, abs.company_id, rc.name AS company_name
-            FROM account_bank_statement abs
-            JOIN res_company rc ON rc.id = abs.company_id
-            WHERE abs.id in %s
-            ORDER BY abs.date, rc.id
-                """,
-                (tuple(res["statement_ids"]),),
-            )
-            sts_data = self.env.cr.dictfetchall()
-
         st_cnt = len(statement_ids)
         warning_cnt = error_cnt = 0
         if notifications:
@@ -224,11 +210,11 @@ class EbicsFile(models.Model):
                 sp=st_cnt == 1 and _(" has") or _("s have"),
             )
             self.note_process += "\n"
-        for st_data in sts_data:
+        for statement in statements:
             self.note_process += ("\n%s, %s (%s)") % (
-                st_data["date"],
+                statement.date,
-                st_data["name"],
+                statement.name,
-                st_data["company_name"],
+                statement.company_id.name,
             )
         if statement_ids:
             self.sudo().bank_statement_ids = [(4, x) for x in statement_ids]
@@ -376,7 +362,7 @@ class EbicsFile(models.Model):
         EBICS data file and its related bank statements.
         """
 
-    def _process_camt053(self):  # noqa C901
+    def _process_camt053(self):
         """
         The Odoo standard statement import is based on manual selection
         of a financial journal before importing the electronic statement file.
@@ -385,8 +371,6 @@ class EbicsFile(models.Model):
         Hence we need to split the CAMT file into
         single statement CAMT files before we can call the logic
         implemented by the Odoo OE or Community CAMT parsers.
-
-        TODO: refactor method to enable removal of noqa C901
         """
         modules = [
             ("oca", "account_statement_import_camt"),
@@ -408,9 +392,79 @@ class EbicsFile(models.Model):
                 )
             )
         res = {"statement_ids": [], "notifications": []}
+        st_datas = self._split_camt(res)
+        msg_hdr = _("{} : Import failed for file %(fn)s:\n", fn=self.name)
         try:
-            with self.env.cr.savepoint():
+            if author == "oca":
-                transactions = False
+                self._process_camt053_oca(res, st_datas)
+            else:
+                self._process_camt053_oe(res, st_datas)
+        except UserError as e:
+            message = msg_hdr.format(_("Error"))
+            message += "".join(e.args)
+            res["notifications"].append({"type": "error", "message": message})
+        except Exception:
+            tb = "".join(format_exception(*exc_info()))
+            message = msg_hdr.format(_("Error"))
+            message += tb
+            res["notifications"].append({"type": "error", "message": message})
+
+        return self._process_download_result(res)
+
+    def _process_camt053_oca(self, res, st_datas):
+        raise NotImplementedError
+
+    def _process_camt053_oe(self, res, st_datas):
+        """
+        We execute a cr.commit() after every statement import since we get a
+        'savepoint does not exist' error when using 'with self.env.cr.savepoint()'.
+        """
+        for st_data in st_datas:
+            self._create_statement_camt053_oe(res, st_data)
+            self.env.cr.commit()  # pylint: disable=E8102
+
+    def _create_statement_camt053_oe(self, res, st_data):
+        attachment = (
+            self.env["ir.attachment"]
+            .with_company(st_data["company_id"])
+            .create(
+                {
+                    "name": self.name,
+                    "datas": st_data["data"],
+                    "store_fname": self.name,
+                }
+            )
+        )
+        journal = (
+            self.env["account.journal"]
+            .with_company(st_data["company_id"])
+            .browse(st_data["journal_id"])
+        )
+        act = journal._import_bank_statement(attachment)
+        for entry in act["domain"]:
+            if (
+                isinstance(entry, tuple)
+                and entry[0] == "statement_id"
+                and entry[1] == "in"
+            ):
+                res["statement_ids"].extend(entry[2])
+                break
+        notifications = act["context"]["notifications"]
+        if notifications:
+            res["notifications"].append(act["context"]["notifications"])
+
+    def _unlink_camt053(self):
+        """
+        Placeholder for camt053 specific actions before removing the
+        EBICS data file and its related bank statements.
+        """
+
+    def _split_camt(self, res):
+        """
+        Split CAMT file received via EBICS per statement.
+        Statements without transactions are removed.
+        """
+        datas = []
         msg_hdr = _("{} : Import failed for file %(fn)s:\n", fn=self.name)
         file_data = base64.b64decode(self.data)
         root = etree.fromstring(file_data, parser=etree.XMLParser(recover=True))
@@ -434,9 +488,7 @@ class EbicsFile(models.Model):
             if not acc_number:
                 message = msg_hdr.format(_("Error"))
                 message += _("No bank account number found.")
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
             currency_code = stmt.xpath(
                 "ns:Acct/ns:Ccy/text() | ns:Bal/ns:Amt/@Ccy", namespaces=ns
@@ -447,9 +499,7 @@ class EbicsFile(models.Model):
             if not currency:
                 message = msg_hdr.format(_("Error"))
                 message += _("Currency %(cc)s not found.", cc=currency_code)
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
             journal = self.env["account.journal"].search(
                 [
@@ -469,14 +519,10 @@ class EbicsFile(models.Model):
                     nbr=acc_number,
                     cc=currency_code,
                 )
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
 
-                    journal_currency = (
+            journal_currency = journal.currency_id or journal.company_id.currency_id
-                        journal.currency_id or journal.company_id.currency_id
-                    )
             if journal_currency != currency:
                 message = msg_hdr.format(_("Error"))
                 message += _(
@@ -485,9 +531,7 @@ class EbicsFile(models.Model):
                     nbr=acc_number,
                     cc=currency_code,
                 )
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
 
             root_new = deepcopy(root)
@@ -500,134 +544,16 @@ class EbicsFile(models.Model):
             if not entries:
                 continue
 
-                    transactions = True
+            datas.append(
-                    data = base64.b64encode(etree.tostring(root_new))
-
-                    if author == "oca":
-                        # TODO: implement _process_camt053_oca() once OCA camt is
-                        # released for 16.0
-                        raise NotImplementedError
-                    else:
-                        self.env.company = journal.company_id
-                        attachment = self.env["ir.attachment"].create(
-                            {"name": self.name, "datas": data, "store_fname": self.name}
-                        )
-                        act = journal._import_bank_statement(attachment)
-                        for entry in act["domain"]:
-                            if (
-                                isinstance(entry, tuple)
-                                and entry[0] == "statement_id"
-                                and entry[1] == "in"
-                            ):
-                                res["statement_ids"].extend(entry[2])
-                                break
-                        notifications = act["context"]["notifications"]
-                        if notifications:
-                            res["notifications"].append(act["context"]["notifications"])
-
-            if not transactions:
-                message = _(
-                    "Warning:\nNo transactions found in file %(fn)s.", fn=self.name
-                )
-                res["notifications"].append({"type": "warning", "message": message})
-
-        except UserError as e:
-            message = msg_hdr.format(_("Error"))
-            message += "".join(e.args)
-            res["notifications"].append({"type": "error", "message": message})
-        except Exception:
-            tb = "".join(format_exception(*exc_info()))
-            message = msg_hdr.format(_("Error"))
-            message += tb
-            res["notifications"].append({"type": "error", "message": message})
-
-        if author == "oca":
-            # TODO: implement _process_camt053_oca() once OCA camt is
-            # released for 16.0
-            return self._process_camt053_oca()
-        else:
-            return self._process_download_result(res)
-
-    def _process_camt053_oca(self):
-        """
-        Disable this code while waiting on OCA CAMT parser for 16.0
-        """
-        # pylint: disable=W0101
-        raise NotImplementedError
-
-        wiz_model = "account.statement.import"
-        wiz_vals = {
-            "statement_filename": self.name,
-            "statement_file": self.data,
-        }
-        result_action = self.env["ir.actions.act_window"]._for_xml_id(
-            "account.action_bank_statement_tree"
-        )
-        result_action["context"] = safe_eval(result_action["context"])
-        result = {
-            "statement_ids": [],
-            "notifications": [],
-        }
-        statement_ids = []
-        notifications = []
-        wiz = (
-            self.env[wiz_model].with_context(active_model="ebics.file").create(wiz_vals)
-        )
-        msg_hdr = _(
-            "{} : Import failed for EBICS File %(fn)s:\n",
-            fn=wiz.statement_filename,
-        )
-        try:
-            with self.env.cr.savepoint():
-                file_data = base64.b64decode(self.data)
-                wiz.import_single_file(file_data, result)
-
-                if not result["statement_ids"]:
-                    message = msg_hdr.format(_("Warning"))
-                    message += _(
-                        "You have already imported this file, or this file "
-                        "only contains already imported transactions."
-                    )
-                    notifications += [
                 {
-                            "type": "warning",
+                    "acc_number": acc_number,
-                            "message": message,
+                    "journal_id": journal.id,
+                    "company_id": journal.company_id.id,
+                    "data": base64.b64encode(etree.tostring(root_new)),
                 }
-                    ]
+            )
-                else:
-                    statement_ids.extend(result["statement_ids"])
-                notifications.extend(result["notifications"])
 
-        except UserError as e:
+        return datas
-            message = msg_hdr.format(_("Error"))
-            message += "".join(e.args)
-            notifications += [
-                {
-                    "type": "error",
-                    "message": message,
-                }
-            ]
-
-        except Exception:
-            tb = "".join(format_exception(*exc_info()))
-            message = msg_hdr.format(_("Error"))
-            message += tb
-            notifications += [
-                {
-                    "type": "error",
-                    "message": message,
-                }
-            ]
-
-        result_action["context"]["notifications"] = notifications
-        result_action["domain"] = [("id", "in", statement_ids)]
-        return self._process_result_action(result_action)
-
-    def _unlink_camt053(self):
-        """
-        Placeholder for camt053 specific actions before removing the
-        EBICS data file and its related bank statements.
-        """
 
     def _process_pain002(self):
         """