generalize auto expand

Auto Expand V2


clean and rewrite rdi transform


Fix when no expand_colum


fix aep

remove rdi object and support compagny in roe detail identifier

fixup

fixup

fixup

fixup

fixup

fixup

fix upo test

add test case for rdi

fixup

do queries

test replace exprs

fixup

test n

test

fix bc for account_id

n test

do not add dependency on aa

fixup

fixup

fixup

test

better test row details

some more tests

even more

some more debug

debug values

fififix

ffff

test

test

fix

fix test

Remove magic number

black


lint

isort

simplify do queries for linting

black again

autoflake

fix init

next fix

Author: TeoGoddet

mis_builder/models/aep.py

@@ -14,6 +14,8 @@
 
 _DOMAIN_START_RE = re.compile(r"\(|(['\"])[!&|]\1")
 
+UNCLASSIFIED_ROW_DETAIL = "other"
+
 
 def _is_domain(s):
     """Test if a string looks like an Odoo domain"""
@@ -299,25 +301,22 @@ def do_queries(
         date_from,
         date_to,
         additional_move_line_filter=None,
-        aml_model=None,
+        aml_model="account.move.line",
+        auto_expand_col_name=None,
     ):
         """Query sums of debit and credit for all accounts and domains
         used in expressions.
 
         This method must be executed after done_parsing().
         """
-        if not aml_model:
-            aml_model = self.env["account.move.line"]
-        else:
-            aml_model = self.env[aml_model]
-        aml_model = aml_model.with_context(active_test=False)
+        aml_model = self.env[aml_model].with_context(active_test=False)
         company_rates = self._get_company_rates(date_to)
         # {(domain, mode): {account_id: (debit, credit)}}
         self._data = defaultdict(dict)
         domain_by_mode = {}
         ends = []
         for key in self._map_account_ids:
-            domain, mode = key
+            (domain, mode) = key
             if mode == self.MODE_END and self.smart_end:
                 # postpone computation of ending balance
                 ends.append((domain, mode))
@@ -330,13 +329,16 @@ def do_queries(
             domain.append(("account_id", "in", self._map_account_ids[key]))
             if additional_move_line_filter:
                 domain.extend(additional_move_line_filter)
+
+            get_fields = ["debit", "credit", "account_id", "company_id"]
+            group_by_fields = ["account_id", "company_id"]
+            if auto_expand_col_name:
+                get_fields = [auto_expand_col_name] + get_fields
+                group_by_fields = [auto_expand_col_name] + group_by_fields
+
             # fetch sum of debit/credit, grouped by account_id
-            accs = aml_model.read_group(
-                domain,
-                ["debit", "credit", "account_id", "company_id"],
-                ["account_id", "company_id"],
-                lazy=False,
-            )
+            accs = aml_model.read_group(domain, get_fields, group_by_fields, lazy=False)
+
             for acc in accs:
                 rate, dp = company_rates[acc["company_id"][0]]
                 debit = acc["debit"] or 0.0
@@ -346,19 +348,45 @@ def do_queries(
                 ):
                     # in initial mode, ignore accounts with 0 balance
                     continue
-                self._data[key][acc["account_id"][0]] = (debit * rate, credit * rate)
+                if (
+                    auto_expand_col_name
+                    and auto_expand_col_name in acc
+                    and acc[auto_expand_col_name]
+                ):
+                    rdi_id = acc[auto_expand_col_name][0]
+                else:
+                    rdi_id = UNCLASSIFIED_ROW_DETAIL
+                if not self._data[key].get(rdi_id, False):
+                    self._data[key][rdi_id] = defaultdict(dict)
+                self._data[key][rdi_id][acc["account_id"][0]] = (
+                    debit * rate,
+                    credit * rate,
+                )
         # compute ending balances by summing initial and variation
         for key in ends:
             domain, mode = key
             initial_data = self._data[(domain, self.MODE_INITIAL)]
             variation_data = self._data[(domain, self.MODE_VARIATION)]
-            account_ids = set(initial_data.keys()) | set(variation_data.keys())
-            for account_id in account_ids:
-                di, ci = initial_data.get(account_id, (AccountingNone, AccountingNone))
-                dv, cv = variation_data.get(
-                    account_id, (AccountingNone, AccountingNone)
+            rdis = set(initial_data.keys()) | set(variation_data.keys())
+            for rdi in rdis:
+                if not initial_data.get(rdi, False):
+                    initial_data[rdi] = defaultdict(dict)
+                if not variation_data.get(rdi, False):
+                    variation_data[rdi] = defaultdict(dict)
+                if not self._data[key].get(rdi, False):
+                    self._data[key][rdi] = defaultdict(dict)
+
+                account_ids = set(initial_data[rdi].keys()) | set(
+                    variation_data[rdi].keys()
                 )
-                self._data[key][account_id] = (di + dv, ci + cv)
+                for account_id in account_ids:
+                    di, ci = initial_data[rdi].get(
+                        account_id, (AccountingNone, AccountingNone)
+                    )
+                    dv, cv = variation_data[rdi].get(
+                        account_id, (AccountingNone, AccountingNone)
+                    )
+                    self._data[key][rdi][account_id] = (di + dv, ci + cv)
 
     def replace_expr(self, expr):
         """Replace accounting variables in an expression by their amount.
@@ -371,23 +399,25 @@ def replace_expr(self, expr):
         def f(mo):
             field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
             key = (ml_domain, mode)
-            account_ids_data = self._data[key]
+            rdi_ids_data = self._data[key]
             v = AccountingNone
             account_ids = self._account_ids_by_acc_domain[acc_domain]
-            for account_id in account_ids:
-                debit, credit = account_ids_data.get(
-                    account_id, (AccountingNone, AccountingNone)
-                )
-                if field == "bal":
-                    v += debit - credit
-                elif field == "pbal" and debit >= credit:
-                    v += debit - credit
-                elif field == "nbal" and debit < credit:
-                    v += debit - credit
-                elif field == "deb":
-                    v += debit
-                elif field == "crd":
-                    v += credit
+            for rdi in rdi_ids_data:
+                account_ids_data = self._data[key][rdi]
+                for account_id in account_ids:
+                    debit, credit = account_ids_data.get(
+                        account_id, (AccountingNone, AccountingNone)
+                    )
+                    if field == "bal":
+                        v += debit - credit
+                    elif field == "pbal" and debit >= credit:
+                        v += debit - credit
+                    elif field == "nbal" and debit < credit:
+                        v += debit - credit
+                    elif field == "deb":
+                        v += debit
+                    elif field == "crd":
+                        v += credit
             # in initial balance mode, assume 0 is None
             # as it does not make sense to distinguish 0 from "no data"
             if (
@@ -401,11 +431,11 @@ def f(mo):
         return self._ACC_RE.sub(f, expr)
 
     def replace_exprs_by_account_id(self, exprs):
-        """Replace accounting variables in a list of expression
-        by their amount, iterating by accounts involved in the expression.
+        """This method is depreciated and replaced by replace_exprs_by_row_detail.
 
+        Replace accounting variables in a list of expression
+        by their amount, iterating by accounts involved in the expression.
         yields account_id, replaced_expr
-
         This method must be executed after do_queries().
         """
 
@@ -417,7 +447,7 @@ def f(mo):
             if account_id not in self._account_ids_by_acc_domain[acc_domain]:
                 return "(AccountingNone)"
             # here we know account_id is involved in acc_domain
-            account_ids_data = self._data[key]
+            account_ids_data = self._data[key][UNCLASSIFIED_ROW_DETAIL]
             debit, credit = account_ids_data.get(
                 account_id, (AccountingNone, AccountingNone)
             )
@@ -452,14 +482,66 @@ def f(mo):
             for mo in self._ACC_RE.finditer(expr):
                 field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
                 key = (ml_domain, mode)
-                account_ids_data = self._data[key]
+                account_ids_data = self._data[key][UNCLASSIFIED_ROW_DETAIL]
                 for account_id in self._account_ids_by_acc_domain[acc_domain]:
                     if account_id in account_ids_data:
                         account_ids.add(account_id)
 
         for account_id in account_ids:
             yield account_id, [self._ACC_RE.sub(f, expr) for expr in exprs]
 
+    def replace_exprs_by_row_detail(self, exprs):
+        """Replace accounting variables in a list of expression
+        by their amount, iterating by accounts involved in the expression.
+
+        yields account_id, replaced_expr
+
+        This method must be executed after do_queries().
+        """
+
+        def f(mo):
+            field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
+            key = (ml_domain, mode)
+            v = AccountingNone
+            account_ids_data = self._data[key][rdi_id]
+            account_ids = self._account_ids_by_acc_domain[acc_domain]
+
+            for account_id in account_ids:
+                debit, credit = account_ids_data.get(
+                    account_id, (AccountingNone, AccountingNone)
+                )
+                if field == "bal":
+                    v += debit - credit
+                elif field == "pbal" and debit >= credit:
+                    v += debit - credit
+                elif field == "nbal" and debit < credit:
+                    v += debit - credit
+                elif field == "deb":
+                    v += debit
+                elif field == "crd":
+                    v += credit
+            # in initial balance mode, assume 0 is None
+            # as it does not make sense to distinguish 0 from "no data"
+            if (
+                v is not AccountingNone
+                and mode in (self.MODE_INITIAL, self.MODE_UNALLOCATED)
+                and float_is_zero(v, precision_digits=self.dp)
+            ):
+                v = AccountingNone
+            return "(" + repr(v) + ")"
+
+        rdi_ids = set()
+        for expr in exprs:
+            for mo in self._ACC_RE.finditer(expr):
+                field, mode, acc_domain, ml_domain = self._parse_match_object(mo)
+                key = (ml_domain, mode)
+                rdis_data = self._data[key]
+                for rdi_id in rdis_data.keys():
+                    rdi_ids.add(rdi_id)
+
+        for rdi_id in rdi_ids:
+            yield rdi_id, [self._ACC_RE.sub(f, expr) for expr in exprs]
+
     @classmethod
     def _get_balances(cls, mode, companies, date_from, date_to):
         expr = "deb{mode}[], crd{mode}[]".format(mode=mode)
@@ -470,7 +552,10 @@ def _get_balances(cls, mode, companies, date_from, date_to):
         aep.parse_expr(expr)
         aep.done_parsing()
         aep.do_queries(date_from, date_to)
-        return aep._data[((), mode)]
+
+        return aep._data[((), mode)].get(UNCLASSIFIED_ROW_DETAIL, {})
+        # to keep compatibility, we give the UNCLASSIFIED_ROW_DETAIL
+        # (expecting that auto_expand_col_names=None was given to do_queries )
 
     @classmethod
     def get_balances_initial(cls, companies, date):

mis_builder/models/expression_evaluator.py

@@ -6,12 +6,7 @@
 
 class ExpressionEvaluator(object):
     def __init__(
-        self,
-        aep,
-        date_from,
-        date_to,
-        additional_move_line_filter=None,
-        aml_model=None,
+        self, aep, date_from, date_to, additional_move_line_filter=None, aml_model=None
     ):
         self.aep = aep
         self.date_from = date_from
@@ -20,13 +15,14 @@ def __init__(
         self.aml_model = aml_model
         self._aep_queries_done = False
 
-    def aep_do_queries(self):
+    def aep_do_queries(self, auto_expand_col_name=None):
         if self.aep and not self._aep_queries_done:
             self.aep.do_queries(
                 self.date_from,
                 self.date_to,
                 self.additional_move_line_filter,
                 self.aml_model,
+                auto_expand_col_name,
             )
             self._aep_queries_done = True
 
@@ -50,6 +46,7 @@ def eval_expressions(self, expressions, locals_dict):
                 drilldown_args.append(None)
         return vals, drilldown_args, name_error
 
+    # TODO remove or keep for compatibility
     def eval_expressions_by_account(self, expressions, locals_dict):
         if not self.aep:
             return
@@ -66,3 +63,20 @@ def eval_expressions_by_account(self, expressions, locals_dict):
                 else:
                     drilldown_args.append(None)
             yield account_id, vals, drilldown_args, name_error
+
+    def eval_expressions_by_row_detail(self, expressions, locals_dict):
+        if not self.aep:
+            return
+        exprs = [e and e.name or "AccountingNone" for e in expressions]
+        for rdi_id, replaced_exprs in self.aep.replace_exprs_by_row_detail(exprs):
+            vals = []
+            drilldown_args = []
+            name_error = False
+            for expr, replaced_expr in zip(exprs, replaced_exprs):
+                val = mis_safe_eval(replaced_expr, locals_dict)
+                vals.append(val)
+                if replaced_expr != expr:
+                    drilldown_args.append({"expr": expr, "row_detail": rdi_id})
+                else:
+                    drilldown_args.append(None)
+            yield rdi_id, vals, drilldown_args, name_error