Merge pull request #101 from Noviat/16-ebics-improvements

[IMP] 16.0 ebics improvements

.pre-commit-config.yaml

@@ -1,7 +1,6 @@
 exclude: |
   (?x)
   # NOT INSTALLABLE ADDONS
-  ^account_ebics_oca_statement_import/|
   ^account_ebics_payment_order/|
   # END NOT INSTALLABLE ADDONS
   # Files and folders generated by bots, to avoid loops

account_ebics/README.rst

@@ -214,6 +214,5 @@ Known Issues / Roadmap
 ======================
 
 - Add support to import externally generated keys & certificates (currently only 3SKey signature certificate).
-- For Odoo 16.0 the interaction with the OCA payment order and bank statement import modules (e.g. french CFONB) is not yet available.
 - Electronic Distributed Signature (EDS) is not supported in the current version of this module.
 

account_ebics/__manifest__.py

@@ -3,7 +3,7 @@
 
 {
     "name": "EBICS banking protocol",
-    "version": "16.0.1.3.0",
+    "version": "16.0.1.3.1",
     "license": "LGPL-3",
     "author": "Noviat",
     "website": "https://www.noviat.com",

account_ebics/models/ebics_file.py

@@ -180,21 +180,7 @@ class EbicsFile(models.Model):
     def _process_download_result(self, res):
         statement_ids = res["statement_ids"]
         notifications = res["notifications"]
-        sts_data = []
+        statements = self.env["account.bank.statement"].sudo().browse(statement_ids)
-        if statement_ids:
-            self.env.flush_all()
-            self.env.cr.execute(
-                """
-            SELECT abs.name, abs.date, abs.company_id, rc.name AS company_name
-            FROM account_bank_statement abs
-            JOIN res_company rc ON rc.id = abs.company_id
-            WHERE abs.id in %s
-            ORDER BY abs.date, rc.id
-                """,
-                (tuple(res["statement_ids"]),),
-            )
-            sts_data = self.env.cr.dictfetchall()
-
         st_cnt = len(statement_ids)
         warning_cnt = error_cnt = 0
         if notifications:
@@ -224,11 +210,11 @@ class EbicsFile(models.Model):
                 sp=st_cnt == 1 and _(" has") or _("s have"),
             )
             self.note_process += "\n"
-        for st_data in sts_data:
+        for statement in statements:
             self.note_process += ("\n%s, %s (%s)") % (
-                st_data["date"],
+                statement.date,
-                st_data["name"],
+                statement.name,
-                st_data["company_name"],
+                statement.company_id.name,
             )
         if statement_ids:
             self.sudo().bank_statement_ids = [(4, x) for x in statement_ids]
@@ -376,7 +362,7 @@ class EbicsFile(models.Model):
         EBICS data file and its related bank statements.
         """
 
-    def _process_camt053(self):  # noqa C901
+    def _process_camt053(self):
         """
         The Odoo standard statement import is based on manual selection
         of a financial journal before importing the electronic statement file.
@@ -385,8 +371,6 @@ class EbicsFile(models.Model):
         Hence we need to split the CAMT file into
         single statement CAMT files before we can call the logic
         implemented by the Odoo OE or Community CAMT parsers.
-
-        TODO: refactor method to enable removal of noqa C901
         """
         modules = [
             ("oca", "account_statement_import_camt"),
@@ -408,9 +392,100 @@ class EbicsFile(models.Model):
                 )
             )
         res = {"statement_ids": [], "notifications": []}
+        st_datas = self._split_camt(res)
+        if author == "oca":
+            self._process_camt053_oca(res, st_datas)
+        else:
+            self._process_camt053_oe(res, st_datas)
+        return self._process_download_result(res)
+
+    def _process_camt053_oca(self, res, st_datas):
+        msg_hdr = _("{} : Import failed for file %(fn)s:\n", fn=self.name)
+        for st_data in st_datas:
             try:
                 with self.env.cr.savepoint():
-                transactions = False
+                    self._create_statement_camt053_oca(res, st_data)
+            except UserError as e:
+                message = msg_hdr.format(_("Error"))
+                message += "".join(e.args)
+                res["notifications"].append({"type": "error", "message": message})
+            except Exception:
+                tb = "".join(format_exception(*exc_info()))
+                message = msg_hdr.format(_("Error"))
+                message += tb
+                res["notifications"].append({"type": "error", "message": message})
+
+    def _create_statement_camt053_oca(self, res, st_data):
+        wiz = (
+            self.env["account.statement.import"]
+            .with_company(st_data["company_id"])
+            .with_context(active_model="ebics.file")
+            .create({"statement_filename": self.name})
+        )
+        wiz.import_single_file(base64.b64decode(st_data["data"]), res)
+
+    def _process_camt053_oe(self, res, st_datas):
+        """
+        We execute a cr.commit() after every statement import since we get a
+        'savepoint does not exist' error when using 'with self.env.cr.savepoint()'.
+        """
+        msg_hdr = _("{} : Import failed for file %(fn)s:\n", fn=self.name)
+        for st_data in st_datas:
+            try:
+                self._create_statement_camt053_oe(res, st_data)
+                self.env.cr.commit()  # pylint: disable=E8102
+            except UserError as e:
+                message = msg_hdr.format(_("Error"))
+                message += "".join(e.args)
+                res["notifications"].append({"type": "error", "message": message})
+            except Exception:
+                tb = "".join(format_exception(*exc_info()))
+                message = msg_hdr.format(_("Error"))
+                message += tb
+                res["notifications"].append({"type": "error", "message": message})
+
+    def _create_statement_camt053_oe(self, res, st_data):
+        attachment = (
+            self.env["ir.attachment"]
+            .with_company(st_data["company_id"])
+            .create(
+                {
+                    "name": self.name,
+                    "datas": st_data["data"],
+                    "store_fname": self.name,
+                }
+            )
+        )
+        journal = (
+            self.env["account.journal"]
+            .with_company(st_data["company_id"])
+            .browse(st_data["journal_id"])
+        )
+        act = journal._import_bank_statement(attachment)
+        for entry in act["domain"]:
+            if (
+                isinstance(entry, tuple)
+                and entry[0] == "statement_id"
+                and entry[1] == "in"
+            ):
+                res["statement_ids"].extend(entry[2])
+                break
+        notifications = act["context"]["notifications"]
+        if notifications:
+            res["notifications"].append(act["context"]["notifications"])
+
+    def _unlink_camt053(self):
+        """
+        Placeholder for camt053 specific actions before removing the
+        EBICS data file and its related bank statements.
+        """
+
+    def _split_camt(self, res):
+        """
+        Split CAMT file received via EBICS per statement.
+        Statements without transactions are removed.
+        """
+        datas = []
         msg_hdr = _("{} : Import failed for file %(fn)s:\n", fn=self.name)
         file_data = base64.b64decode(self.data)
         root = etree.fromstring(file_data, parser=etree.XMLParser(recover=True))
@@ -434,9 +509,7 @@ class EbicsFile(models.Model):
             if not acc_number:
                 message = msg_hdr.format(_("Error"))
                 message += _("No bank account number found.")
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
             currency_code = stmt.xpath(
                 "ns:Acct/ns:Ccy/text() | ns:Bal/ns:Amt/@Ccy", namespaces=ns
@@ -447,9 +520,7 @@ class EbicsFile(models.Model):
             if not currency:
                 message = msg_hdr.format(_("Error"))
                 message += _("Currency %(cc)s not found.", cc=currency_code)
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
             journal = self.env["account.journal"].search(
                 [
@@ -469,14 +540,10 @@ class EbicsFile(models.Model):
                     nbr=acc_number,
                     cc=currency_code,
                 )
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
 
-                    journal_currency = (
+            journal_currency = journal.currency_id or journal.company_id.currency_id
-                        journal.currency_id or journal.company_id.currency_id
-                    )
             if journal_currency != currency:
                 message = msg_hdr.format(_("Error"))
                 message += _(
@@ -485,9 +552,7 @@ class EbicsFile(models.Model):
                     nbr=acc_number,
                     cc=currency_code,
                 )
-                        res["notifications"].append(
+                res["notifications"].append({"type": "error", "message": message})
-                            {"type": "error", "message": message}
-                        )
                 continue
 
             root_new = deepcopy(root)
@@ -500,134 +565,16 @@ class EbicsFile(models.Model):
             if not entries:
                 continue
 
-                    transactions = True
+            datas.append(
-                    data = base64.b64encode(etree.tostring(root_new))
-
-                    if author == "oca":
-                        # TODO: implement _process_camt053_oca() once OCA camt is
-                        # released for 16.0
-                        raise NotImplementedError
-                    else:
-                        self.env.company = journal.company_id
-                        attachment = self.env["ir.attachment"].create(
-                            {"name": self.name, "datas": data, "store_fname": self.name}
-                        )
-                        act = journal._import_bank_statement(attachment)
-                        for entry in act["domain"]:
-                            if (
-                                isinstance(entry, tuple)
-                                and entry[0] == "statement_id"
-                                and entry[1] == "in"
-                            ):
-                                res["statement_ids"].extend(entry[2])
-                                break
-                        notifications = act["context"]["notifications"]
-                        if notifications:
-                            res["notifications"].append(act["context"]["notifications"])
-
-            if not transactions:
-                message = _(
-                    "Warning:\nNo transactions found in file %(fn)s.", fn=self.name
-                )
-                res["notifications"].append({"type": "warning", "message": message})
-
-        except UserError as e:
-            message = msg_hdr.format(_("Error"))
-            message += "".join(e.args)
-            res["notifications"].append({"type": "error", "message": message})
-        except Exception:
-            tb = "".join(format_exception(*exc_info()))
-            message = msg_hdr.format(_("Error"))
-            message += tb
-            res["notifications"].append({"type": "error", "message": message})
-
-        if author == "oca":
-            # TODO: implement _process_camt053_oca() once OCA camt is
-            # released for 16.0
-            return self._process_camt053_oca()
-        else:
-            return self._process_download_result(res)
-
-    def _process_camt053_oca(self):
-        """
-        Disable this code while waiting on OCA CAMT parser for 16.0
-        """
-        # pylint: disable=W0101
-        raise NotImplementedError
-
-        wiz_model = "account.statement.import"
-        wiz_vals = {
-            "statement_filename": self.name,
-            "statement_file": self.data,
-        }
-        result_action = self.env["ir.actions.act_window"]._for_xml_id(
-            "account.action_bank_statement_tree"
-        )
-        result_action["context"] = safe_eval(result_action["context"])
-        result = {
-            "statement_ids": [],
-            "notifications": [],
-        }
-        statement_ids = []
-        notifications = []
-        wiz = (
-            self.env[wiz_model].with_context(active_model="ebics.file").create(wiz_vals)
-        )
-        msg_hdr = _(
-            "{} : Import failed for EBICS File %(fn)s:\n",
-            fn=wiz.statement_filename,
-        )
-        try:
-            with self.env.cr.savepoint():
-                file_data = base64.b64decode(self.data)
-                wiz.import_single_file(file_data, result)
-
-                if not result["statement_ids"]:
-                    message = msg_hdr.format(_("Warning"))
-                    message += _(
-                        "You have already imported this file, or this file "
-                        "only contains already imported transactions."
-                    )
-                    notifications += [
                 {
-                            "type": "warning",
+                    "acc_number": acc_number,
-                            "message": message,
+                    "journal_id": journal.id,
+                    "company_id": journal.company_id.id,
+                    "data": base64.b64encode(etree.tostring(root_new)),
                 }
-                    ]
+            )
-                else:
-                    statement_ids.extend(result["statement_ids"])
-                notifications.extend(result["notifications"])
 
-        except UserError as e:
+        return datas
-            message = msg_hdr.format(_("Error"))
-            message += "".join(e.args)
-            notifications += [
-                {
-                    "type": "error",
-                    "message": message,
-                }
-            ]
-
-        except Exception:
-            tb = "".join(format_exception(*exc_info()))
-            message = msg_hdr.format(_("Error"))
-            message += tb
-            notifications += [
-                {
-                    "type": "error",
-                    "message": message,
-                }
-            ]
-
-        result_action["context"]["notifications"] = notifications
-        result_action["domain"] = [("id", "in", statement_ids)]
-        return self._process_result_action(result_action)
-
-    def _unlink_camt053(self):
-        """
-        Placeholder for camt053 specific actions before removing the
-        EBICS data file and its related bank statements.
-        """
 
     def _process_pain002(self):
         """

account_ebics/static/description/index.html

@@ -563,7 +563,6 @@ You can also find this information in the doc folder of this module (file EBICS_
 <h2>Known Issues / Roadmap</h2>
 <ul class="simple">
 <li>Add support to import externally generated keys &amp; certificates (currently only 3SKey signature certificate).</li>
-<li>For Odoo 16.0 the interaction with the OCA payment order and bank statement import modules (e.g. french CFONB) is not yet available.</li>
 <li>Electronic Distributed Signature (EDS) is not supported in the current version of this module.</li>
 </ul>
 </div>

account_ebics_batch/models/ebics_batch_log.py

@@ -126,7 +126,6 @@ class EbicsBatchLog(models.Model):
                 import_dict["errors"].append(err_msg + tb)
         log.file_ids = [(6, 0, ebics_file_ids)]
         try:
-            with self.env.cr.savepoint():
             log._ebics_process(import_dict)
         except UserError as e:
             import_dict["errors"].append(err_msg + " ".join(e.args))

account_ebics_oca_statement_import/README.rst

@@ -6,12 +6,12 @@
 Deploy account_ebics module with OCA Bank Statement Import
 ==========================================================
 
-This module makes it possible to use OCA account_statement_import
+This module makes it possible to use the OCA account_statement_import wizard
 in combination with 'account_ebics'.
 
 This module will be installed automatically when following modules are activated
 on your odoo database :
 
 - account_ebics
-- account_statement_import
+- account_statement_import_file
 

account_ebics_oca_statement_import/__manifest__.py

@@ -11,11 +11,9 @@
     "license": "LGPL-3",
     "depends": [
         "account_ebics",
-        "account_statement_import",
+        "account_statement_import_file",
     ],
-    # installable False unit OCA statement import becomes
+    "installable": True,
-    # available for 16.0
-    "installable": False,
     "auto_install": True,
     "images": ["static/description/cover.png"],
 }

account_ebics_oca_statement_import/wizards/account_statement_import.py

@@ -1,4 +1,4 @@
-# Copyright 2009-2020 Noviat.
+# Copyright 2009-2023 Noviat.
 # License LGPL-3 or later (http://www.gnu.org/licenses/lgpl).
 
 import logging

setup/account_ebics_oca_statement_import/odoo/addons/account_ebics_oca_statement_import

@@ -0,0 +1 @@
+../../../../account_ebics_oca_statement_import

setup/account_ebics_oca_statement_import/setup.py

@@ -0,0 +1,6 @@
+import setuptools
+
+setuptools.setup(
+    setup_requires=['setuptools-odoo'],
+    odoo_addon=True,
+)