0.8 b1

.claude/settings.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(uv run:*)"
+    ]
+  }
+}

.env.template

@@ -197,11 +197,17 @@ OLLAMA_MODELS=qwen3:32b # models with good code generation capabilities recommen
 # Also configurable via CLI: --workspace-backend
 #
 # Choices:
-#   local      — (default) stores data under DATA_FORMULATOR_HOME
+#   local      — (default) durable storage under DATA_FORMULATOR_HOME; no TTL cleanup
 #   azure_blob — stores data in Azure Blob Storage (see below)
-#   ephemeral  — temp dirs; data does NOT survive restart (if you want to prevent user data retention on server disk for privacy reasons)
+#   ephemeral  — temporary server-local storage with TTL and LRU cleanup
 #
 # WORKSPACE_BACKEND=local
+#
+# Ephemeral-only retention settings (ignored by local and azure_blob):
+# EPHEMERAL_WORKSPACE_TTL_HOURS=24
+# EPHEMERAL_WORKSPACE_MAX_BYTES=10737418240
+# EPHEMERAL_WORKSPACE_CLEANUP_INTERVAL_SECONDS=1800
+# EPHEMERAL_WORKSPACE_ROOT=~/.data_formulator/ephemeral
 
 # -------------------------------------------------------------------
 # Azure Blob Storage settings (only when WORKSPACE_BACKEND=azure_blob)

.github/dependabot.yml

@@ -18,17 +18,40 @@ updates:
    target-branch: "dev"
    schedule:
      interval: weekly
+   groups:
+     frontend-minor-patch:
+       patterns:
+         - "*"
+       update-types:
+         - minor
+         - patch
 
  # Backend Python dependencies (pyproject.toml / uv.lock)
  - package-ecosystem: "uv"
    directory: "/"
    target-branch: "dev"
    schedule:
      interval: weekly
+   groups:
+     python-minor-patch:
+       patterns:
+         - "*"
+       update-types:
+         - minor
+         - patch
+   ignore:
+     # LiteLLM 1.92+ no longer provides portable Windows/macOS wheels.
+     - dependency-name: "litellm"
+       versions:
+         - ">=1.92"
 
  # GitHub Actions workflow dependencies
  - package-ecosystem: "github-actions"
    directory: "/"
    target-branch: "dev"
    schedule:
      interval: weekly
+   groups:
+     github-actions:
+       patterns:
+         - "*"

.github/workflows/desktop-build.yml

@@ -0,0 +1,124 @@
+name: desktop builds
+
+on:
+  push:
+    branches: [main]
+    tags: ['*']
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: desktop-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-desktop:
+    name: ${{ matrix.name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Windows x64
+            os: windows-latest
+            artifact: data-formulator-windows-x64
+          - name: macOS
+            os: macos-latest
+            artifact: data-formulator-macos
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Node.js 20
+        uses: actions/setup-node@v7
+        with:
+          node-version: 20
+          cache: yarn
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: '3.12'
+          enable-cache: true
+
+      - name: Install frontend dependencies
+        run: yarn install --frozen-lockfile
+
+      - name: Build frontend
+        run: yarn build
+
+      - name: Install desktop dependencies
+        run: uv sync --extra desktop --frozen
+
+      - name: Test desktop startup output
+        run: uv run pytest tests/backend/test_startup_spinner.py -q
+
+      - name: Build desktop application
+        run: uv run pyinstaller --noconfirm --clean packaging/data_formulator_desktop.spec
+
+      - name: Self-test packaged application (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        env:
+          DF_DESKTOP_SELF_TEST: '1'
+        run: |
+          & 'dist/Data Formulator/Data Formulator.exe' | Tee-Object -Variable output
+          if ($LASTEXITCODE -ne 0) { throw "self-test failed with exit code $LASTEXITCODE" }
+
+      - name: Self-test packaged application (macOS)
+        if: runner.os == 'macOS'
+        env:
+          DF_DESKTOP_SELF_TEST: '1'
+        run: |
+          './dist/Data Formulator.app/Contents/MacOS/Data Formulator'
+
+      - name: Archive Windows application
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          New-Item -ItemType Directory -Force release | Out-Null
+          Compress-Archive -Path 'dist/Data Formulator' -DestinationPath 'release/Data-Formulator-Windows-x64.zip'
+
+      - name: Archive macOS application
+        if: runner.os == 'macOS'
+        shell: bash
+        run: |
+          mkdir -p release
+          ditto -c -k --sequesterRsrc --keepParent \
+            'dist/Data Formulator.app' \
+            'release/Data-Formulator-macOS.zip'
+
+      - name: Upload desktop artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: ${{ matrix.artifact }}
+          path: release/*.zip
+          if-no-files-found: error
+          retention-days: 30
+
+  attach-to-release:
+    name: Attach desktop downloads to release
+    if: github.ref_type == 'tag'
+    needs: build-desktop
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Download desktop artifacts
+        uses: actions/download-artifact@v8
+        with:
+          pattern: data-formulator-*
+          path: release
+          merge-multiple: true
+
+      - name: Attach archives to GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: release/*.zip
+          fail_on_unmatched_files: true
+          generate_release_notes: true

.github/workflows/python-build.yml

@@ -20,24 +20,26 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set Node.js 20
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v7
       with:
         node-version: 20
         cache: 'yarn'
     - name: Set up uv
-      uses: astral-sh/setup-uv@v4
+      uses: astral-sh/setup-uv@v7
       with:
         python-version: 3.12
     - name: Install node dependencies
-      run: yarn install
+      run: yarn install --frozen-lockfile
     - name: Build frontend
       run: yarn build
     - name: Build python artifact
       run: uv build
+    - name: Validate python artifact
+      run: uv run --with twine twine check dist/*
     - name: Archive production artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: release-dist
         path: dist
@@ -54,7 +56,7 @@ jobs:
       id-token: write
     steps:
       - name: Retrieve release distributions
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: release-dist
           path: dist/

.gitignore

@@ -8,6 +8,10 @@
 build/
 dist/
 experiment_data/
+design-docs/
+deploy-scripts/
+test-data-loader/
+scripts/
 
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
@@ -413,8 +417,8 @@ venv
 # Temporary documentation directory
 tmp-docs/
 
-\.\NUL
+.NUL
 NUL
 
 # Package manager lock files (using yarn)
-package-lock.json
+package-lock.json

CHANGELOG.md

@@ -0,0 +1,63 @@
+# Changelog
+
+Notable changes to Data Formulator are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions follow
+[PEP 440](https://peps.python.org/pep-0440/).
+
+## [Unreleased]
+
+## [0.8.0b1] - 2026-08-15
+
+### Added
+
+- A unified Data Thread for questions, clarifications, explanations, tables,
+  charts, and branching exploration.
+- Conversational data loading with source discovery, plan review, and previews.
+- Persistent file attachments and local-folder access in the desktop app.
+- Databricks Unity Catalog support and expanded enterprise authentication.
+- Self-contained Windows and macOS preview builds.
+
+### Changed
+
+- Expanded the Flint-powered chart gallery, recommendations, themes, and style
+  refinement.
+- Improved catalog spacing on large displays and made code and identifier
+  typography consistent across operating systems.
+- Standardized button sizing, icon spacing, and action styles across display
+  densities, starting with model management.
+- Made sessions default to recent activity and simplified session sorting,
+  creation, and import controls.
+- Improved workspace persistence, credential handling, connector diagnostics,
+  and analyst context across an exploration.
+
+## [0.7] - 2026-05-28
+
+### Added
+
+- Governed data connectors, agent-assisted loading, a unified Data Thread,
+  persistent workspaces, chart refinement, reports, and multilingual UI.
+
+## [0.6] - 2026-01-26
+
+### Added
+
+- Live data from URLs and databases with automatic refresh.
+
+## [0.5.1] - 2025-12-08
+
+### Added
+
+- Community data loaders, US maps, pie charts, and editable reports.
+
+### Changed
+
+- Improved application responsiveness.
+
+For earlier releases, see the
+[GitHub release history](https://github.com/microsoft/data-formulator/releases).
+
+[Unreleased]: https://github.com/microsoft/data-formulator/compare/0.8.0b1...HEAD
+[0.8.0b1]: https://github.com/microsoft/data-formulator/releases/tag/0.8.0b1
+[0.7]: https://github.com/microsoft/data-formulator/releases/tag/0.7
+[0.6]: https://github.com/microsoft/data-formulator/releases/tag/0.6
+[0.5.1]: https://github.com/microsoft/data-formulator/releases/tag/0.5.1

DEVELOPMENT.md

@@ -372,7 +372,7 @@ data_formulator \
 
 ### Profile 2: Multi-User Anonymous (demo / public hosting)
 
-A shared server (e.g., for demos, workshops, public access). No login, no server-side state, no sensitive features.
+A shared server (e.g., for demos, workshops, public access). No login, short-lived server-side workspaces, and no sensitive features.
 
 ```bash
 data_formulator \
@@ -388,6 +388,10 @@ Or via environment variables:
 
 ```env
 WORKSPACE_BACKEND=ephemeral
+# Ephemeral retention only; local mode is durable and ignores these settings.
+EPHEMERAL_WORKSPACE_TTL_HOURS=24
+EPHEMERAL_WORKSPACE_MAX_BYTES=10737418240
+EPHEMERAL_WORKSPACE_CLEANUP_INTERVAL_SECONDS=1800
 DISABLE_DATA_CONNECTORS=true
 DISABLE_CUSTOM_MODELS=true
 DISABLE_DISPLAY_KEYS=true
@@ -400,14 +404,16 @@ OPENAI_MODELS=gpt-4.1
 | Setting | Value | Why |
 |---------|-------|-----|
 | `AUTH_PROVIDER` | *(unset)* | Anonymous access for demos |
-| `WORKSPACE_BACKEND` | `ephemeral` | No server-side persistence — data lives only in browser IndexedDB |
+| `WORKSPACE_BACKEND` | `ephemeral` | Temporary server-local workspaces with TTL/LRU cleanup |
 | `DISABLE_DATA_CONNECTORS` | `true` | **Critical** — prevents DB credential exposure via identity spoofing |
 | `DISABLE_CUSTOM_MODELS` | `true` | Prevents users from adding arbitrary LLM endpoints (SSRF risk) |
 | `DISABLE_DISPLAY_KEYS` | `true` | Hides server-configured API keys from UI |
 | Credential vault | N/A | No connectors → no credentials to store |
-| Identity | anonymous (`browser:<uuid>`) | Acceptable — no sensitive server-side state to protect |
+| Identity | anonymous (`browser:<uuid>`) | Isolates temporary workspaces by browser identity |
 
-**Security notes:** With data connectors disabled, the anonymous identity spoofing risk is eliminated — there are no DB credentials or persistent workspaces on the server to access. Each user's data lives entirely in their browser. The only server-side resource is the LLM proxy, which is locked down by `DF_ALLOWED_API_BASES`.
+**Retention notes:** Ephemeral workspaces may disappear after inactivity or when the configured byte cap is reached. The browser keeps only a row-free recovery snapshot for read-only viewing. Use `WORKSPACE_BACKEND=local` for durable workspaces; ephemeral TTL/LRU cleanup never scans or deletes local-mode workspaces.
+
+**Security notes:** Keep data connectors and custom models disabled for anonymous deployments. Browser identities are client-provided and are suitable for isolating disposable demo workspaces, not for protecting durable credentials or sensitive server-side state.
 
 ### Profile 3: Multi-User Authenticated (enterprise / team)
 

MANIFEST.in

@@ -1,4 +1,5 @@
 include py-src/data_formulator/dist/*
 include py-src/data_formulator/dist/assets/*
+recursive-include py-src/data_formulator/data_loader/guides *.md
 global-exclude .DS_Store
 exclude py-src/examples