fix: is_final is the new ending of the navigator classification

Author: th-e-o

explorations.py

@@ -19,11 +19,12 @@
 
 query = """
 MATCH path = (root)-[*]->(n)
-WHERE n.LEVEL = 5
+WHERE n.FINAL = 1
   AND n.embedding IS NOT NULL
   AND root.LEVEL = 0
 RETURN n.embedding as embedding,
        n.NAME as name,
+       n.CODE as code, 
        [node IN nodes(path) | node.CODE] as path_codes,
        [node IN nodes(path) | node.LEVEL] as path_levels
 """
@@ -33,10 +34,14 @@
 embeddings = []
 names = []
 paths = []
+codes_dict = {}
 
-for record in results:
+for idx, record in enumerate(results):
     embeddings.append(record["embedding"])
     names.append(record["name"])
+    code = record["code"]
+    code_clean = code.replace(".", "").replace(" ", "")
+    codes_dict[code_clean] = idx
 
     path_str = " → ".join([
         name for lvl, name in zip(record["path_levels"], record["path_codes"])
@@ -45,76 +50,29 @@
 
 
 print(f"Nœuds récupérés: {len(names)}")
-print(embeddings)
 
+n_nace_nodes = len(embeddings)
 
-# %%
-# Add a query in the embedding space
-
-queries = ["Je vends des croissants", "Livreur de taxi", "Coiffeur"]
-emb_model = OpenAIEmbeddings(
-        model=os.environ['EMBEDDING_MODEL'],
-        openai_api_base=os.environ['URL_EMBEDDING_API'],
-        openai_api_key="EMPTY",
-        tiktoken_enabled=False,
-    )
-for i, query in iter(queries): 
-    query_emb = emb_model.embed_query(query)
-    embeddings.append(query_emb)
-    names.append(f"Query {i}")
-    paths.append(query)
-
-
-# %% UMAP
-reducer = umap.UMAP(random_state=42, n_neighbors=10, min_dist=0.1)
-embeddings = np.array(embeddings)
-coords = reducer.fit_transform(embeddings)
-X, Y = coords.T
-
-# %% Visualisation interactive
-fig = go.Figure()
-
-fig.add_trace(go.Scatter(
-    x=X, y=Y,
-    mode='markers',
-    marker=dict(
-        size=10,
-        color=np.arange(len(X)),  # Couleur par index
-        colorscale='Viridis',
-        showscale=True,
-        line=dict(width=0.5, color='white')
-    ),
-    text=[f"<b>{name}</b><br><br>{path}" for name, path in zip(names, paths)],
-    hovertemplate='%{text}<extra></extra>'
-))
 
-fig.update_layout(
-    title="Nœuds de niveau 5",
-    xaxis_title="UMAP 1",
-    yaxis_title="UMAP 2",
-    width=1200,
-    height=800,
-    hovermode='closest',
-    plot_bgcolor='white',
-    xaxis=dict(showgrid=True, gridcolor='lightgray'),
-    yaxis=dict(showgrid=True, gridcolor='lightgray')
-)
 
-fig.show()
 # %%
 import os
 import s3fs
 os.environ["AWS_ACCESS_KEY_ID"] = 'UN8E5UMY78E5H4AKC7HF'
 os.environ["AWS_SECRET_ACCESS_KEY"] = 'fSUt5up9uh4qfyHH4LIQ6J0GiQp42eFc+fKWrRS2'
 os.environ["AWS_SESSION_TOKEN"] = 'eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJVTjhFNVVNWTc4RTVINEFLQzdIRiIsImFsbG93ZWQtb3JpZ2lucyI6WyIqIl0sImF1ZCI6WyJtaW5pby1kYXRhbm9kZSIsIm9ueXhpYSIsImFjY291bnQiXSwiYXV0aF90aW1lIjoxNzcwNjI4NzkzLCJhenAiOiJvbnl4aWEiLCJlbWFpbCI6InRoZW8uZmVycnlAaW5zZWUuZnIiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwiZXhwIjoxNzcxNTEyODA1LCJmYW1pbHlfbmFtZSI6IkZlcnJ5IiwiZ2l2ZW5fbmFtZSI6IlRoZW8iLCJncm91cHMiOlsiVVNFUl9PTllYSUEiLCJhcGUiLCJtb2RlbHMtaGYiLCJzc3BsYWIiXSwiaWF0IjoxNzcwOTA4MDA0LCJpc3MiOiJodHRwczovL2F1dGgubGFiLnNzcGNsb3VkLmZyL2F1dGgvcmVhbG1zL3NzcGNsb3VkIiwianRpIjoib25ydHJ0OjllMjk1ZmEzLTliNmMtNjZjYi0yMWE0LTA2NDlhNGVkMWUzYSIsImxvY2FsZSI6ImZyIiwibmFtZSI6IlRoZW8gRmVycnkiLCJwb2xpY3kiOiJzdHNvbmx5IiwicHJlZmVycmVkX3VzZXJuYW1lIjoidGhlb2YiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiIsInZpcCIsImRlZmF1bHQtcm9sZXMtc3NwY2xvdWQiXX0sInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIiwidmlwIiwiZGVmYXVsdC1yb2xlcy1zc3BjbG91ZCJdLCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGdyb3VwcyBlbWFpbCIsInNpZCI6ImRiYTY1NzAxLWE3OTctMDFjZi0yYWE1LTRkYjkzY2Q0ZWM4NiIsInN1YiI6IjNlYTdiY2Q0LWJkMjMtNDA2Yy1hYmE2LWFmMzM3ZjBlMTAzNiIsInR5cCI6IkJlYXJlciJ9.keTVOmqa7NmhFGb5Jp384W0EisDdxox7Sip2f1B4MPdfN5z_tDtU85beJbBqCFl6TJdybu0PHVRX_sDW5q4Fgg'
 os.environ["AWS_DEFAULT_REGION"] = 'us-east-1'
+
+N_CODES = 20
+
 fs = s3fs.S3FileSystem(
     client_kwargs={'endpoint_url': 'https://'+'minio.lab.sspcloud.fr'},
     key = os.environ["AWS_ACCESS_KEY_ID"], 
     secret = os.environ["AWS_SECRET_ACCESS_KEY"], 
     token = os.environ["AWS_SESSION_TOKEN"])
 
 
+
 def sample_codes(fs: s3fs.S3FileSystem, population_path: str, code_column: str, n_codes: int):
     """
     Sample codes using Polars from S3.
@@ -139,10 +97,113 @@ def sample_codes(fs: s3fs.S3FileSystem, population_path: str, code_column: str,
     fs=fs,
     population_path=path,
     code_column=columns, 
-    n_codes=10)
+    n_codes=N_CODES)
+
+labels, target_codes = zip(*codes)
+
+emb_model = OpenAIEmbeddings(
+        model=os.environ['EMBEDDING_MODEL'],
+        openai_api_base=os.environ['URL_EMBEDDING_API'],
+        openai_api_key="EMPTY",
+        tiktoken_enabled=False,
+    )
+
+labels_embeddings = emb_model.embed_documents(list(labels))
 
-print(codes)
-labels, codes = zip(*codes)
+label_to_code_idx = {}
+
+for i, (label, label_emb, target_code) in enumerate(zip(labels, labels_embeddings, target_codes)):
+    embeddings.append(label_emb)
+    names.append(label[:50])
+    paths.append(f"Libellé -> Code cible: {target_code}")
+
+    label_idx = n_nace_nodes + i
+    if target_code in codes_dict: 
+        label_to_code_idx[label_idx] = codes_dict[target_code]
+        
+# %%
+print(label_to_code_idx)
+
+# %% UMAP
+reducer = umap.UMAP(random_state=42, n_neighbors=10, min_dist=0.1)
+embeddings = np.array(embeddings)
+coords = reducer.fit_transform(embeddings)
+X, Y = coords.T
+
+
+
+
+# %% Visualisation interactive
+fig = go.Figure()
+
+
+# 1. Ajouter les lignes de connexion AVANT les points
+for label_idx, code_idx in label_to_code_idx.items():
+    fig.add_trace(go.Scatter(
+        x=[X[label_idx], X[code_idx]],
+        y=[Y[label_idx], Y[code_idx]],
+        mode='lines',
+        line=dict(color='rgba(150, 150, 150, 0.8)', width=3, dash='solid'),
+        showlegend=False,
+        hoverinfo='skip'
+    ))
+
+# 2. Ajouter les nœuds NACE (cercles)
+fig.add_trace(go.Scatter(
+    x=X[:n_nace_nodes], 
+    y=Y[:n_nace_nodes],
+    mode='markers',
+    name='Codes NACE',
+    marker=dict(
+        size=10,
+        color=np.arange(n_nace_nodes),
+        colorscale='Viridis',
+        showscale=True,
+        line=dict(width=0.5, color='white'),
+        symbol='circle'
+    ),
+    text=[f"<b>{name}</b><br><br>{path}" for name, path in zip(names[:n_nace_nodes], paths[:n_nace_nodes])],
+    hovertemplate='%{text}<extra></extra>'
+))
+
+# 3. Ajouter les libellés (étoiles)
+fig.add_trace(go.Scatter(
+    x=X[n_nace_nodes:], 
+    y=Y[n_nace_nodes:],
+    mode='markers',
+    name='Libellés',
+    marker=dict(
+        size=15,
+        color='red',
+        symbol='star',  # ou 'diamond', 'square', 'cross', 'x', 'triangle-up'
+        line=dict(width=1, color='darkred')
+    ),
+    text=[f"<b>{name}</b><br><br>{path}" for name, path in zip(names[n_nace_nodes:], paths[n_nace_nodes:])],
+    hovertemplate='%{text}<extra></extra>'
+))
+
+fig.update_layout(
+    title="Nœuds NACE niveau 5 et libellés échantillonnés",
+    xaxis_title="UMAP 1",
+    yaxis_title="UMAP 2",
+    width=1400,
+    height=900,
+    hovermode='closest',
+    plot_bgcolor='white',
+    xaxis=dict(showgrid=True, gridcolor='lightgray'),
+    yaxis=dict(showgrid=True, gridcolor='lightgray'),
+    legend=dict(
+        yanchor="top",
+        y=0.99,
+        xanchor="right",
+        x=0.99
+    )
+)
+
+fig.show()
+
+# %%
+fig.write_html("umap_visualization.html")
 
 # %%
 result = await classify_navigator(labels[0])

src/agents/Text2Code/classifiers/navigator_classifier.py

@@ -21,7 +21,7 @@ def get_instructions(self) -> str:
         return """
         Vous êtes un expert en classification NACE. Votre mission est de naviguer 
         dans l'arborescence afin d'atteindre le code le plus spécifique caractérisant l'activité indiquée.
-        Après avoir vérifié que vous êtes au niveau 4 de l'arbre, et que votre position actuelle est bien finale, 
+        Après avoir vérifié que votre position actuelle est bien finale (is_final = 1), 
         vous renverrez votre position. 
         Si vous n'avez pas réussi à atteindre une position finale, dites-le. 
         Soyez méthodique et justifiez chaque choix !

src/navigator/navigator.py

@@ -58,6 +58,7 @@ def get_code_information(code: str) -> Dict[str, Any]:
             "code": info.get("code"),
             "name": info.get("name"),
             "level": info.get("level"),
+            "is_final": info.get("is_final"),
             "description": info.get("description", "")[:500],  # Limiter la taille
         }
 
@@ -76,10 +77,9 @@ def get_current_children() -> List[Dict[str, Any]]:
         """
         logger.info(f"Navigator: get_current_children called at the position {navigator.current_code}")
         children_found = _unfreeze_list_of_dicts(navigator._cached_get_children(navigator.current_code))
-        keys_to_keep = ["code", "name"]
-        print(f"Keys_to_keep: {keys_to_keep}")
+        keys_to_keep = ["code", "name", "is_final"]
         filtered_children_found = [
-            {k:d[k] for k in keys_to_keep}
+            {k: d[k] for k in keys_to_keep}
             for d in children_found
         ]
         logger.info(f"Navigator children found: {filtered_children_found}")
@@ -371,26 +371,8 @@ def submit_classification(
         go_to_parent,
         go_to_child,
         get_context_summary,
-        # submit_classification
     ]
 
-    """ return [
-        get_current_information,
-        get_code_information,
-        get_current_parent,
-        get_current_children,
-        get_current_siblings,
-        get_current_descendants,
-        navigate_to,
-        go_to_parent,
-        go_to_child,
-        reset_to_root,
-        get_context_summary,
-        get_navigation_history,
-        submit_classification  
-    ] """
-
-
 class Navigator(Graph):
     """
     Classe de navigation dans la hiérarchie NACE avec état persistant.

src/neo4j_graph/graph.py

@@ -183,6 +183,7 @@ def _cached_get_code_information(self, code: str) -> Tuple[Tuple[str, Any], ...]
         RETURN node.CODE as code,
        node.LEVEL as level,
        node.NAME as name,
+       node.FINAL as is_final,
        node.text as description,
        node.Includes as includes,
        node.IncludesAlso as includes_also,
@@ -211,7 +212,7 @@ def _cached_get_children(self, code: str) -> Tuple[Tuple[Tuple[str, Any], ...],
         MATCH (node {CODE: $code})-[:HAS_CHILD]->(child)
         RETURN child.CODE as code,
                child.LEVEL as level,
-               child.FINAL as final,
+               child.FINAL as is_final,
                child.NAME as name,
                child.text as description,
                child.Includes as includes,
@@ -233,6 +234,7 @@ def _cached_get_descendants(
         MATCH (node {{CODE: $code}})-[:HAS_CHILD*{levels}]->(descendant)
         RETURN descendant.CODE as code,
                descendant.LEVEL as level,
+               descendant.FINAL as is_final,
                descendant.NAME as name,
                descendant.text as description,
                descendant.Includes as includes,
@@ -255,6 +257,7 @@ def _cached_get_siblings(self, code: str) -> Tuple[Tuple[Tuple[str, Any], ...],
         RETURN sibling.CODE as code,
                sibling.LEVEL as level,
                sibling.NAME as name,
+               sibling.FINAL as is_final,
                sibling.text as description,
                sibling.Includes as includes,
                sibling.Excludes as excludes
@@ -293,6 +296,7 @@ def _cached_search_codes(self, search_term: str) -> Tuple[Tuple[Tuple[str, Any],
            OR toLower(node.text) CONTAINS toLower($search_term)
         RETURN node.CODE as code,
                node.LEVEL as level,
+               node.FINAL as is_final, 
                node.NAME as name,
                node.text as description
         ORDER BY node.LEVEL, node.CODE