feat: real-time MPK widget derivation — CREATE PAGE works without widget init

.claude/skills/mendix/custom-widgets.md

@@ -1,6 +1,6 @@
 ---
 name: mendix-custom-widgets
-description: Use when writing MDL for GALLERY, COMBOBOX, or third-party pluggable widgets in CREATE PAGE / ALTER PAGE statements. Covers built-in widget syntax, child slots (TEMPLATE/FILTER), adding new custom widgets via .def.json, and engine internals.
+description: Use when writing MDL for GALLERY, COMBOBOX, or third-party pluggable widgets in CREATE PAGE / ALTER PAGE statements. Covers built-in widget syntax, child slots (TEMPLATE/FILTER), real-time MPK discovery for project widgets, and adding custom widgets via .def.json.
 ---
 
 # Custom & Pluggable Widgets in MDL
@@ -59,7 +59,46 @@ combobox cmbCustomer (
 - `CaptionAttribute` is the display attribute on the **target** entity
 - In association mode, mapping order matters: DataSource must resolve before Association (sets entityContext)
 
-## Adding a Third-Party Widget
+## Project Widgets (Real-Time Discovery)
+
+**No extraction step required.** When `mxcli` runs a `CREATE PAGE` command against a project, it automatically scans `<project>/widgets/*.mpk` and makes every widget available by its derived MDL name — the last dot-segment of the widget ID, lowercased.
+
+```
+com.vendor.widget.web.MySlider.MySlider  →  MDL keyword: MYSLIDER
+com.example.QRScanner                   →  MDL keyword: QRSCANNER
+```
+
+### Using a project widget in MDL
+
+```sql
+-- No widget init needed. Just use the widget by its derived name.
+create page Module.MyPage (layout: Atlas_Default) {
+  dataview dv (entity: Module.Product) {
+    MYSLIDER slider1 (datasource: database Module.Product, attribute: Price)
+  }
+}
+```
+
+If the widget has a `datasource`, `attribute`, `association`, or `widgets` property in its XML, those are auto-mapped. For properties that need custom mapping (actions, expressions, textTemplates), see the extraction workflow below.
+
+### Checking what's available
+
+```bash
+# Lists all widgets: built-in + auto-discovered from project MPKs
+mxcli widget list -p App.mpr
+```
+
+### When auto-discovery isn't enough
+
+Extract a `.def.json` only if you need to:
+- Override the auto-inferred property mappings
+- Add support for `action`, `expression`, or `textTemplate` properties
+- Control the MDL keyword (the derived name doesn't match what you want)
+- Share a definition globally across projects (`~/.mxcli/widgets/`)
+
+---
+
+## Customizing a Widget (.def.json Workflow)
 
 ### Step 1 -- Extract .def.json from .mpk
 
@@ -291,15 +330,16 @@ project/widgets/*.mpk -> FindMPK(projectDir, widgetID) -> ParseMPK()
 
 This reduces CE0463 errors from widget version drift without requiring manual template re-extraction.
 
-### 3-Tier Registry
+### 4-Tier Registry
 
 | Priority | Location | Scope |
 |----------|----------|-------|
-| 1 (highest) | `<project>/.mxcli/widgets/*.def.json` | Project |
+| 1 (highest) | `<project>/.mxcli/widgets/*.def.json` | Project (hand-crafted) |
 | 2 | `~/.mxcli/widgets/*.def.json` | Global (user) |
-| 3 (lowest) | `sdk/widgets/definitions/*.def.json` (embedded) | Built-in |
+| 3 | `sdk/widgets/definitions/*.def.json` (embedded) | Built-in |
+| 4 (lowest) | `<project>/widgets/*.mpk` (real-time) | Project (auto-derived) |
 
-Higher priority definitions override lower ones with the same MDL name (case-insensitive).
+Higher priority definitions override lower ones. Real-time MPK derivation only activates when no definition exists at tiers 1–3. The MDL name derived from an MPK (lowercase last ID segment) is used as the key; if a built-in or hand-crafted definition uses the same name, the MPK entry is silently skipped.
 
 ## Verify & Debug
 
@@ -322,7 +362,7 @@ mxcli bson dump -p App.mpr --type page --object "Module.PageName" --format ndsl
 | Mistake | Fix |
 |---------|-----|
 | CE0463 after page creation | Template version mismatch -- extract fresh template from Studio Pro MPR, or ensure .mpk augmentation picks up new properties |
-| Widget not recognized | Check `mxcli widget list`; .def.json must be in `.mxcli/widgets/` with `.def.json` extension |
+| Widget not recognized | Run `mxcli widget list -p App.mpr` — project widgets are auto-discovered from `widgets/*.mpk`; if still missing, the .mpk may not exist or the widget ID differs from expected |
 | TEMPLATE content missing | Widget needs `childSlots` entry with `"mdlContainer": "template"` |
 | Association COMBOBOX shows enum behavior | Add `datasource` to trigger association mode (`hasDataSource` condition) |
 | Association mapping fails | Ensure DataSource mapping appears **before** Association mapping in the array |
@@ -334,10 +374,10 @@ mxcli bson dump -p App.mpr --type page --object "Module.PageName" --format ndsl
 | File | Purpose |
 |------|---------|
 | `mdl/executor/widget_engine.go` | PluggableWidgetEngine, 6 operations, Build() pipeline |
-| `mdl/executor/widget_registry.go` | 3-tier WidgetRegistry, definition validation |
+| `mdl/executor/widget_registry.go` | 4-tier WidgetRegistry: `SetProjectDir` triggers real-time MPK scan; `Get`/`GetByWidgetID` fall back to MPK derivation on miss |
 | `sdk/widgets/loader.go` | Template loading, ID remapping, MPK augmentation |
 | `sdk/widgets/mpk/mpk.go` | .mpk ZIP parsing, XML property extraction |
-| `cmd/mxcli/cmd_widget.go` | `mxcli widget extract/list` CLI commands |
+| `cmd/mxcli/cmd_widget.go` | `mxcli widget extract/list/init` CLI commands (`init` is now optional; use `--force` to overwrite) |
 | `sdk/widgets/definitions/*.def.json` | Built-in widget definitions (ComboBox, Gallery) |
 | `sdk/widgets/templates/mendix-11.6/*.json` | Embedded BSON templates |
 | `mdl/executor/cmd_pages_builder_input.go` | `updateWidgetPropertyValue()` -- TypePointer matching |

cmd/mxcli/cmd_extract_templates.go

@@ -21,9 +21,9 @@ var extractTemplatesCmd = &cobra.Command{
 	Long: `Extract pluggable widget type definitions from a Mendix project
 and save them as JSON templates for use in mxcli.
 
-This command searches for CustomWidgets in the project and extracts
-their type definitions, which can then be embedded in mxcli for
-consistent widget creation across projects.
+This command scans all pages and snippets in the project for pluggable
+widgets and extracts their type definitions. The resulting JSON files can
+be embedded in mxcli for consistent widget creation across projects.
 
 Example:
   mxcli extract-templates -p app.mpr -o templates/mendix-11.6/`,
@@ -38,8 +38,8 @@ func init() {
 	rootCmd.AddCommand(extractTemplatesCmd)
 }
 
-// WidgetTemplate is the JSON structure for a widget template file.
-type WidgetTemplate struct {
+// extractedWidgetTemplate is the JSON structure for a widget template file.
+type extractedWidgetTemplate struct {
 	WidgetID      string         `json:"widgetId"`
 	Name          string         `json:"name"`
 	Version       string         `json:"version"`
@@ -52,93 +52,72 @@ func runExtractTemplates(cmd *cobra.Command, args []string) error {
 	projectPath, _ := cmd.Flags().GetString("project")
 	outputDir, _ := cmd.Flags().GetString("output")
 
-	// Open the project
 	reader, err := mpr.Open(projectPath)
 	if err != nil {
 		return fmt.Errorf("failed to open project: %w", err)
 	}
 	defer reader.Close()
 
-	// Get Mendix version
 	version, _ := reader.GetMendixVersion()
 	fmt.Printf("Extracting templates from Mendix %s project\n", version)
 
-	// Create output directory
 	if err := os.MkdirAll(outputDir, 0755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
 	}
 
-	// Widget IDs to extract
-	widgetIDs := []struct {
-		id       string
-		filename string
-		name     string
-	}{
-		{"com.mendix.widget.web.combobox.Combobox", "combobox.json", "Combo box"},
-		{"com.mendix.widget.web.gallery.Gallery", "gallery.json", "Gallery"},
-		{"com.mendix.widget.web.datagrid.Datagrid", "datagrid.json", "Data grid 2"},
-		{"com.mendix.widget.web.datagridtextfilter.DatagridTextFilter", "datagrid-text-filter.json", "Text filter"},
-		{"com.mendix.widget.web.datagriddatefilter.DatagridDateFilter", "datagrid-date-filter.json", "Date filter"},
-		{"com.mendix.widget.web.datagriddropdownfilter.DatagridDropdownFilter", "datagrid-dropdown-filter.json", "Dropdown filter"},
-		{"com.mendix.widget.web.datagridnumberfilter.DatagridNumberFilter", "datagrid-number-filter.json", "Number filter"},
-		{"com.mendix.widget.web.image.Image", "image.json", "Image"},
+	widgets, err := reader.ListAllCustomWidgetTypes()
+	if err != nil {
+		return fmt.Errorf("failed to scan project for widget types: %w", err)
+	}
+	if len(widgets) == 0 {
+		fmt.Println("No pluggable widget types found in this project.")
+		return nil
 	}
+	fmt.Printf("Found %d widget type(s)\n", len(widgets))
 
 	extracted := 0
-	for _, w := range widgetIDs {
-		rawWidget, err := reader.FindCustomWidgetType(w.id)
-		if err != nil {
-			fmt.Printf("  [SKIP] %s: %v\n", w.name, err)
-			continue
-		}
-		if rawWidget == nil {
-			fmt.Printf("  [SKIP] %s: not found in project\n", w.name)
-			continue
-		}
-
-		// Convert BSON to JSON-compatible map
-		typeMap, err := bsonDToMap(rawWidget.RawType)
+	for _, w := range widgets {
+		typeMap, err := bsonDToMap(w.RawType)
 		if err != nil {
-			fmt.Printf("  [SKIP] %s: failed to convert type BSON: %v\n", w.name, err)
+			fmt.Printf("  [SKIP] %s: failed to convert type BSON: %v\n", w.WidgetID, err)
 			continue
 		}
 
 		var objectMap map[string]any
-		if rawWidget.RawObject != nil {
-			objectMap, err = bsonDToMap(rawWidget.RawObject)
+		if w.RawObject != nil {
+			objectMap, err = bsonDToMap(w.RawObject)
 			if err != nil {
-				fmt.Printf("  [SKIP] %s: failed to convert object BSON: %v\n", w.name, err)
+				fmt.Printf("  [SKIP] %s: failed to convert object BSON: %v\n", w.WidgetID, err)
 				continue
 			}
 		}
 
-		template := WidgetTemplate{
-			WidgetID:      w.id,
-			Name:          w.name,
+		template := extractedWidgetTemplate{
+			WidgetID:      w.WidgetID,
 			Version:       version,
-			ExtractedFrom: rawWidget.UnitID,
+			ExtractedFrom: w.UnitID,
 			Type:          typeMap,
 			Object:        objectMap,
 		}
 
-		// Write to file
-		outPath := filepath.Join(outputDir, w.filename)
+		filename := filenameFromWidgetID(w.WidgetID)
+		outPath := filepath.Join(outputDir, filename)
 		data, err := json.MarshalIndent(template, "", "  ")
 		if err != nil {
-			fmt.Printf("  [SKIP] %s: failed to marshal JSON: %v\n", w.name, err)
+			fmt.Printf("  [SKIP] %s: failed to marshal JSON: %v\n", w.WidgetID, err)
 			continue
 		}
 
 		if err := os.WriteFile(outPath, data, 0644); err != nil {
-			fmt.Printf("  [SKIP] %s: failed to write file: %v\n", w.name, err)
+			fmt.Printf("  [SKIP] %s: failed to write file: %v\n", w.WidgetID, err)
 			continue
 		}
 
-		fmt.Printf("  [OK] %s -> %s\n", w.name, w.filename)
+		fmt.Printf("  [OK] %s -> %s\n", w.WidgetID, filename)
 		extracted++
 	}
 
-	fmt.Printf("\nExtracted %d widget templates to %s\n", extracted, outputDir)
+	fmt.Printf("\nExtracted %d/%d widget templates to %s\n", extracted, len(widgets), outputDir)
 	return nil
 }
 
@@ -167,7 +146,6 @@ func convertBsonValue(v any) any {
 		}
 		return arr
 	case primitive.Binary:
-		// Convert binary IDs to hex strings
 		return fmt.Sprintf("%x", val.Data)
 	case []byte:
 		return fmt.Sprintf("%x", val)
@@ -176,12 +154,11 @@ func convertBsonValue(v any) any {
 	}
 }
 
-// filenameFromWidgetID generates a filename from a widget ID.
+// filenameFromWidgetID generates a kebab-case filename from a widget ID.
+// e.g. "com.mendix.widget.web.combobox.Combobox" -> "combobox.json"
 func filenameFromWidgetID(widgetID string) string {
-	// Extract the last part after the last dot
 	parts := strings.Split(widgetID, ".")
 	name := parts[len(parts)-1]
-	// Convert camelCase to kebab-case
 	var result strings.Builder
 	for i, r := range name {
 		if i > 0 && r >= 'A' && r <= 'Z' {