Refactor DataLab kernel to use xeus-python backend for improved performance and debugging support; update installation instructions and add startup script for namespace injection.

Author: PierreRaybaut

README.md

@@ -20,7 +20,7 @@ It enables scientists and engineers to:
 - share notebooks that can be replayed **with or without DataLab**,
 - combine narrative, code, and results without sacrificing interactive visualization.
 
-DataLab-Kernel is **not** a replacement for DataLab’s GUI.  
+DataLab-Kernel is **not** a replacement for DataLab’s GUI.
 It is a **complementary execution layer** that turns DataLab into a hybrid scientific platform:
 **GUI-driven when needed, notebook-driven when appropriate.**
 
@@ -97,15 +97,43 @@ Two connection methods are supported:
 
 ```bash
 pip install datalab-kernel sigima
+python -m datalab_kernel install
 jupyter lab
 ```
 
 Then select **DataLab Kernel** from the kernel list.
 
+### Dependencies
+
+DataLab-Kernel uses **xeus-python** as its backend, which provides:
+
+- Improved performance compared to ipykernel
+- Native debugger support
+- JupyterLite compatibility
+- Better Qt event loop integration
+
+The kernel requires:
+
+- `xeus-python>=0.17.0` - The xeus-based Python kernel
+- `xeus-python-shell>=0.6.0` - Python shell utilities for xeus-python
+- `sigima>=1.0` - Scientific signal and image processing
+- `numpy>=1.22`, `h5py>=3.0`, `matplotlib>=3.5`
+
 ### With DataLab
 
 When installed alongside DataLab, the kernel is automatically available and can be launched directly from the DataLab interface.
 
+### Installing from conda-forge (recommended)
+
+For best compatibility, especially on Windows:
+
+```bash
+mamba create -n datalab-kernel
+mamba activate datalab-kernel
+mamba install xeus-python datalab-kernel -c conda-forge
+python -m datalab_kernel install
+```
+
 ---
 
 ## Persistence and Sharing

datalab_kernel/__init__.py

@@ -8,6 +8,12 @@
 A standalone Jupyter kernel providing seamless, reproducible access to DataLab
 workspaces, with optional live synchronization to the DataLab GUI.
 
+This kernel uses **xeus-python** as its backend for:
+- Improved performance compared to ipykernel
+- Native debugger support
+- JupyterLite compatibility
+- Better Qt event loop integration
+
 Main components:
 - `workspace`: Data access and persistence API
 - `plotter`: Visualization frontend

datalab_kernel/__main__.py

@@ -7,6 +7,7 @@
 Supports running as:
     python -m datalab_kernel install
     python -m datalab_kernel uninstall
+    python -m datalab_kernel  (launches kernel)
 """
 
 from __future__ import annotations
@@ -23,15 +24,54 @@ def main() -> None:
         install_main()
     else:
         # Launch kernel (default behavior when run by Jupyter)
-        # Delayed import: only needed when launching as kernel
-        from ipykernel.kernelapp import (  # pylint: disable=import-outside-toplevel
-            IPKernelApp,
-        )
+        # With xeus-python, we need to inject our namespace and then
+        # let xeus-python handle the kernel protocol
 
-        # pylint: disable-next=import-outside-toplevel
-        from datalab_kernel.kernel import DataLabKernel
+        # First, set up the environment to indicate we want DataLab startup
+        import os
 
-        IPKernelApp.launch_instance(kernel_class=DataLabKernel)
+        os.environ["DATALAB_KERNEL_STARTUP"] = "1"
+
+        try:
+            # Try to use xeus-python's native launch
+            # xeus-python installs an 'xpython' executable
+            import subprocess
+
+            xpython = subprocess.run(
+                ["xpython", "--version"],
+                capture_output=True,
+                text=True,
+            )
+
+            if xpython.returncode == 0:
+                # xpython is available, use it
+                # Pass through all arguments
+                args = ["xpython"] + sys.argv[1:]
+                subprocess.run(args)
+            else:
+                raise FileNotFoundError("xpython not found")
+
+        except (FileNotFoundError, subprocess.SubprocessError):
+            # xpython not available, try Python module approach
+            try:
+                # Import xeus-python's main entry point
+                import xeus_python
+
+                # Set up our namespace first
+                from datalab_kernel.startup import run_startup
+
+                run_startup()
+
+                # Now launch xeus-python
+                xeus_python.main()
+
+            except ImportError as e:
+                print(
+                    f"Error: xeus-python is required but not installed: {e}\n"
+                    "Install with: pip install xeus-python xeus-python-shell",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
 
 
 if __name__ == "__main__":

datalab_kernel/install.py

@@ -6,35 +6,88 @@
 ==================================
 
 This module provides commands to install and uninstall the DataLab kernel.
+The kernel uses xeus-python as its backend with a custom startup script
+for injecting the DataLab namespace.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
+import os
 import shutil
 import sys
 from pathlib import Path
 
 KERNEL_NAME = "datalab-kernel"
 KERNEL_DISPLAY_NAME = "DataLab"
 
+# Startup script content that will be executed when the kernel starts
+STARTUP_SCRIPT_CONTENT = """\
+# DataLab Kernel Startup Script
+# This file is auto-generated - do not edit manually
+
+# Import and run the DataLab startup
+from datalab_kernel.startup import run_startup
+run_startup()
+"""
+
+
+def get_xpython_executable() -> str:
+    """Find the xpython executable path.
+
+    Returns:
+        Path to xpython executable, or fallback to python -m approach.
+    """
+    # Try to find xpython in PATH
+    xpython_path = shutil.which("xpython")
+    if xpython_path:
+        return xpython_path
+
+    # Fallback: use Python module approach
+    # xeus-python can be run via its Python extension module
+    return sys.executable
+
 
 def get_kernel_spec() -> dict:
-    """Generate the kernel.json specification."""
-    return {
-        "argv": [
+    """Generate the kernel.json specification for xeus-python.
+
+    The kernel spec tells Jupyter how to launch the kernel. For xeus-python,
+    we use the xpython executable if available, otherwise fall back to
+    launching via Python.
+    """
+    xpython = get_xpython_executable()
+
+    if xpython == sys.executable:
+        # Use Python module approach for xeus-python
+        # This works when xeus-python is installed as a wheel
+        argv = [
             sys.executable,
-            "-m",
-            "datalab_kernel.kernel",
+            "-c",
+            "from datalab_kernel.startup import run_startup; run_startup(); "
+            "import xeus_python; xeus_python.main()",
+            "-f",
+            "{connection_file}",
+        ]
+    else:
+        # Use native xpython executable
+        argv = [
+            xpython,
             "-f",
             "{connection_file}",
-        ],
+        ]
+
+    return {
+        "argv": argv,
         "display_name": KERNEL_DISPLAY_NAME,
         "language": "python",
         "metadata": {
             "debugger": True,
         },
+        "env": {
+            # Ensure our startup runs
+            "DATALAB_KERNEL_STARTUP": "1",
+        },
     }
 
 
@@ -65,6 +118,70 @@ def get_kernel_dir(user: bool = True) -> Path:
     return data_dir / KERNEL_NAME
 
 
+def get_ipython_startup_dir() -> Path:
+    """Get the IPython startup directory.
+
+    Returns:
+        Path to IPython startup directory (creates if needed).
+    """
+    # Get IPython directory
+    ipython_dir = Path.home() / ".ipython" / "profile_default" / "startup"
+    ipython_dir.mkdir(parents=True, exist_ok=True)
+    return ipython_dir
+
+
+def install_startup_script() -> Path:
+    """Install the DataLab startup script for IPython.
+
+    This script will be executed automatically when xeus-python starts
+    with IPython mode enabled.
+
+    Returns:
+        Path to installed startup script.
+    """
+    startup_dir = get_ipython_startup_dir()
+    startup_file = startup_dir / "00-datalab-kernel.py"
+
+    # Only install if DATALAB_KERNEL_STARTUP env var is set
+    # This prevents the startup from running in regular IPython sessions
+    script_content = '''\
+# DataLab Kernel Startup Script
+# Auto-generated by datalab-kernel - do not edit manually
+
+import os
+
+# Only run startup if we're in a DataLab kernel
+if os.environ.get("DATALAB_KERNEL_STARTUP") == "1":
+    try:
+        from datalab_kernel.startup import run_startup
+        run_startup()
+    except ImportError:
+        pass  # DataLab kernel not installed
+'''
+
+    with open(startup_file, "w", encoding="utf-8") as f:
+        f.write(script_content)
+
+    print(f"Installed DataLab startup script to: {startup_file}")
+    return startup_file
+
+
+def remove_startup_script() -> bool:
+    """Remove the DataLab startup script from IPython.
+
+    Returns:
+        True if script was found and removed.
+    """
+    startup_dir = get_ipython_startup_dir()
+    startup_file = startup_dir / "00-datalab-kernel.py"
+
+    if startup_file.exists():
+        startup_file.unlink()
+        print(f"Removed DataLab startup script from: {startup_file}")
+        return True
+    return False
+
+
 def install_kernel(user: bool = True, prefix: str | None = None) -> Path:
     """Install the DataLab kernel.
 
@@ -94,7 +211,11 @@ def install_kernel(user: bool = True, prefix: str | None = None) -> Path:
         for logo_file in logo_dir.glob("logo-*.png"):
             shutil.copy(logo_file, kernel_dir)
 
+    # Install IPython startup script for namespace injection
+    install_startup_script()
+
     print(f"Installed DataLab kernel to: {kernel_dir}")
+    print("Note: Uses xeus-python backend for improved performance and debugging")
     return kernel_dir
 
 
@@ -116,6 +237,8 @@ def uninstall_kernel(user: bool = True, prefix: str | None = None) -> bool:
     if kernel_dir.exists():
         shutil.rmtree(kernel_dir)
         print(f"Uninstalled DataLab kernel from: {kernel_dir}")
+        # Also remove startup script
+        remove_startup_script()
         return True
     print(f"DataLab kernel not found at: {kernel_dir}")
     return False