From eb6983b28db24d9d008c02d8bd4aa4e61ddb9fc2 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:38:52 -0400 Subject: [PATCH 1/3] Import portable browser profile data Import bookmarks, recent history, local storage, and selected Web Store extensions alongside cookies. Keep values local until final approval, enforce the API payload bounds in the CLI, and apply extension entitlements before upload. --- cmd/browser_import_managed_auth.go | 39 ++ cmd/browser_import_profile_data.go | 335 +++++++++++++++++ cmd/browser_import_profile_data_test.go | 55 +++ cmd/connector.go | 1 + cmd/profiles_import_local.go | 53 ++- cmd/profiles_import_local_test.go | 13 + go.mod | 2 + go.sum | 22 ++ internal/browserimport/bundle.go | 116 ++++-- internal/browserimport/bundle_test.go | 59 +-- internal/browserimport/chromium.go | 464 ++++++++++++++++++++++++ internal/browserimport/chromium_test.go | 126 +++++++ internal/browserimport/types.go | 55 +++ 13 files changed, 1288 insertions(+), 52 deletions(-) create mode 100644 cmd/browser_import_profile_data.go create mode 100644 cmd/browser_import_profile_data_test.go diff --git a/cmd/browser_import_managed_auth.go b/cmd/browser_import_managed_auth.go index 9a7081b..9329752 100644 --- a/cmd/browser_import_managed_auth.go +++ b/cmd/browser_import_managed_auth.go @@ -19,6 +19,11 @@ type managedAuthCapacity struct { unlimited bool } +type storedExtensionCapacity struct { + remaining int + unlimited bool +} + type orgLimitsGetter interface { Get(context.Context, ...option.RequestOption) (*kernel.OrgLimits, error) } @@ -57,6 +62,40 @@ func decodeManagedAuthCapacity(raw string) (managedAuthCapacity, error) { return managedAuthCapacity{remaining: max(0, maxConnections-usedConnections)}, nil } +func loadStoredExtensionCapacity(ctx context.Context, limits orgLimitsGetter) (storedExtensionCapacity, error) { + orgLimits, err := limits.Get(ctx) + if err != nil { + return storedExtensionCapacity{}, err + } + return decodeStoredExtensionCapacity(orgLimits.RawJSON()) +} + +func decodeStoredExtensionCapacity(raw string) (storedExtensionCapacity, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(raw), &fields); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode organization limits: %w", err) + } + maxRaw, hasMax := fields["max_stored_extensions"] + usedRaw, hasUsed := fields["stored_extensions_used"] + if !hasMax || !hasUsed { + return storedExtensionCapacity{}, fmt.Errorf("Kernel API does not expose stored extension capacity") + } + if string(maxRaw) == "null" { + return storedExtensionCapacity{unlimited: true}, nil + } + var maximum, used int + if err := json.Unmarshal(maxRaw, &maximum); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode max stored extensions: %w", err) + } + if err := json.Unmarshal(usedRaw, &used); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode used stored extensions: %w", err) + } + if maximum < 0 || used < 0 { + return storedExtensionCapacity{}, fmt.Errorf("Kernel API returned invalid stored extension capacity") + } + return storedExtensionCapacity{remaining: max(0, maximum-used)}, nil +} + type managedAuthProvisioner interface { Provision(context.Context, string, []passwordmanager.Record) ([]string, error) Existing(context.Context, string, []passwordmanager.Candidate) (map[string]bool, error) diff --git a/cmd/browser_import_profile_data.go b/cmd/browser_import_profile_data.go new file mode 100644 index 0000000..7311dfe --- /dev/null +++ b/cmd/browser_import_profile_data.go @@ -0,0 +1,335 @@ +package cmd + +import ( + "context" + "fmt" + "sort" + "time" + + localbrowser "github.com/kernel/cli/internal/browserimport" + "github.com/pterm/pterm" +) + +const ( + bookmarksChoice = "Bookmarks" + historyChoice = "History" + storageChoice = "Local storage" + extensionsChoice = "Browser extensions" +) + +type localProfileDataSelection struct { + data localbrowser.ProfileData + bookmarkCount int + historyCount int + history bool + storage bool + storageSites []string + storageBytes int64 +} + +func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, profile localbrowser.Profile, since time.Time, includeHistory, nonInteractive, humanOutput bool) (localProfileDataSelection, error) { + bookmarks, bookmarkCount, bookmarkErr := localbrowser.ExportBookmarks(profile) + historyCount, historyErr := localbrowser.HistoryCount(ctx, profile, since) + storageSites, storageErr := localbrowser.LocalStorageSites(ctx, profile) + extensions, extensionErr := localbrowser.DiscoverExtensions(profile) + extensionCapacity := storedExtensionCapacity{unlimited: true} + extensionCapacityKnown := c.extensionCapacity == nil + if extensionErr == nil && len(extensions) > 0 && c.extensionCapacity != nil { + var capacityErr error + extensionCapacity, capacityErr = c.extensionCapacity(ctx) + extensionCapacityKnown = capacityErr == nil + if capacityErr != nil && humanOutput { + pterm.Warning.Printf("extension capacity could not be checked; Kernel will enforce it during import: %v\n", capacityErr) + } + } + + if humanOutput { + warnUnavailableBrowserData("bookmarks", bookmarkErr) + warnUnavailableBrowserData("history", historyErr) + warnUnavailableBrowserData("local storage", storageErr) + warnUnavailableBrowserData("extensions", extensionErr) + } + + selection := localProfileDataSelection{history: includeHistory} + options := make([]string, 0, 4) + defaults := make([]string, 0, 4) + labels := make(map[string]string, 4) + if bookmarkErr == nil && bookmarkCount > 0 { + labels[bookmarksChoice] = fmt.Sprintf("%s — %d", bookmarksChoice, bookmarkCount) + options = append(options, labels[bookmarksChoice]) + defaults = append(defaults, labels[bookmarksChoice]) + } + if historyErr == nil && historyCount > 0 { + labels[historyChoice] = fmt.Sprintf("%s — %d visits", historyChoice, historyCount) + options = append(options, labels[historyChoice]) + if includeHistory { + defaults = append(defaults, labels[historyChoice]) + } + } + if storageErr == nil && len(storageSites) > 0 { + total := storageSiteBytes(storageSites) + labels[storageChoice] = fmt.Sprintf("%s — %s across %d origins", storageChoice, formatBinaryBytes(total), len(storageSites)) + options = append(options, labels[storageChoice]) + defaults = append(defaults, labels[storageChoice]) + } + if extensionErr == nil && len(extensions) > 0 { + labels[extensionsChoice] = fmt.Sprintf("%s — %d detected (plan limit applies)", extensionsChoice, len(extensions)) + options = append(options, labels[extensionsChoice]) + defaults = append(defaults, labels[extensionsChoice]) + } + + chosen := defaults + var err error + if !nonInteractive && len(options) > 0 { + chosen, err = c.prompter.MultiSelect("browser data", "use Space to exclude a category", "Choose browser data to import", options, defaults) + if err != nil { + return localProfileDataSelection{}, err + } + } + for _, choice := range chosen { + switch choice { + case labels[bookmarksChoice]: + selection.data.Bookmarks = &bookmarks + selection.bookmarkCount = bookmarkCount + case labels[historyChoice]: + selection.history = true + selection.historyCount = historyCount + case labels[storageChoice]: + selection.storage = true + selection.storageBytes = storageSiteBytes(storageSites) + case labels[extensionsChoice]: + selection.data.Extensions = append(selection.data.Extensions, extensions...) + } + } + + if selection.storage { + selection.storageSites, err = c.chooseLocalStorageSites(storageSites, nonInteractive) + if err != nil { + return localProfileDataSelection{}, err + } + selection.storageBytes = selectedStorageSiteBytes(storageSites, selection.storageSites) + } + if len(selection.data.Extensions) > 0 { + selection.data.Extensions, err = c.chooseProfileExtensions(selection.data.Extensions, extensionCapacity, extensionCapacityKnown, nonInteractive) + if err != nil { + return localProfileDataSelection{}, err + } + } + return selection, nil +} + +func warnUnavailableBrowserData(category string, err error) { + if err != nil { + pterm.Warning.Printf("%s could not be read and will be skipped: %v\n", category, err) + } +} + +func (c ProfilesImportLocalCmd) confirmBrowserImport(targetName string, cookies cookieImportSelection, cookieSites []localbrowser.Site, profileData localProfileDataSelection, logins pendingManagedAuth) (bool, error) { + pterm.Println() + pterm.Printf("Ready to import into profile %q\n\n", targetName) + if cookies.all { + pterm.Printf(" All cookies — %d across %d websites\n", selectedCookieCount(cookieSites, cookies.sites), len(cookies.sites)) + } else { + pterm.Printf(" Cookies from %d selected websites\n", len(cookies.sites)) + } + if profileData.bookmarkCount > 0 { + pterm.Printf(" Bookmarks — %d\n", profileData.bookmarkCount) + } + if profileData.history { + pterm.Printf(" History — %d visits\n", profileData.historyCount) + } + if profileData.storage { + pterm.Printf(" Local storage — %s across %d origins\n", formatBinaryBytes(profileData.storageBytes), len(profileData.storageSites)) + } + if count := len(profileData.data.Extensions); count > 0 { + pterm.Printf(" Browser extensions — %d\n", count) + } + loginCount := 0 + for _, provider := range logins.providers { + loginCount += len(provider.candidates) + } + if loginCount > 0 { + pterm.Printf(" Managed Auth connections — %d\n", loginCount) + } + pterm.Println() + return c.prompter.ConfirmDefault("import browser data", "Proceed?", true) +} + +func selectedCookieCount(sites []localbrowser.Site, selected []string) int { + set := make(map[string]struct{}, len(selected)) + for _, site := range selected { + set[site] = struct{}{} + } + total := 0 + for _, site := range sites { + if _, ok := set[site.Domain]; ok { + total += site.CookieCount + } + } + return total +} + +func (c ProfilesImportLocalCmd) chooseLocalStorageSites(sites []localbrowser.StorageSite, nonInteractive bool) ([]string, error) { + all := make([]string, 0, len(sites)) + for _, site := range sites { + all = append(all, site.Origin) + } + if storageSiteBytes(sites) <= localbrowser.MaxPortableStorageSize { + return all, nil + } + if nonInteractive { + return nil, fmt.Errorf("browser local storage exceeds Kernel's 64 MiB import limit; run interactively to choose websites or disable local storage") + } + + sorted := append([]localbrowser.StorageSite(nil), sites...) + sort.SliceStable(sorted, func(left, right int) bool { + if sorted[left].Bytes == sorted[right].Bytes { + return sorted[left].Origin < sorted[right].Origin + } + return sorted[left].Bytes > sorted[right].Bytes + }) + labels := make([]string, 0, len(sorted)) + byLabel := make(map[string]string, len(sorted)) + defaults := make([]string, 0, len(sorted)) + var selectedBytes int64 + for index, site := range sorted { + label := fmt.Sprintf("%d %s — %s", index+1, compactField(site.Origin, 46), formatBinaryBytes(site.Bytes)) + labels = append(labels, label) + byLabel[label] = site.Origin + if selectedBytes+site.Bytes <= localbrowser.MaxPortableStorageSize { + defaults = append(defaults, label) + selectedBytes += site.Bytes + } + } + chosen, err := c.prompter.MultiSelect("local storage", "deselect websites until the selection fits", "Choose local website data to import (64 MiB maximum)", labels, defaults) + if err != nil { + return nil, err + } + result := make([]string, 0, len(chosen)) + for _, label := range chosen { + result = append(result, byLabel[label]) + } + return result, nil +} + +func (c ProfilesImportLocalCmd) chooseProfileExtensions(extensions []localbrowser.Extension, capacity storedExtensionCapacity, capacityKnown, nonInteractive bool) ([]localbrowser.Extension, error) { + maximum := min(20, len(extensions)) + if capacityKnown && !capacity.unlimited { + maximum = min(maximum, capacity.remaining) + } + if nonInteractive { + if len(extensions) > maximum { + return nil, fmt.Errorf("%d extensions were selected, but only %d can be added under the profile and plan limits; run interactively to choose extensions", len(extensions), maximum) + } + return extensions, nil + } + labels := make([]string, 0, len(extensions)) + byLabel := make(map[string]localbrowser.Extension, len(extensions)) + for index, extension := range extensions { + name := extension.Name + if name == "" { + name = "Unnamed extension" + } + label := fmt.Sprintf("%d %s · %s", index+1, compactField(name, 42), extension.ID[len(extension.ID)-6:]) + labels = append(labels, label) + byLabel[label] = extension + } + defaults := labels[:maximum] + for { + prompt := fmt.Sprintf("Choose extensions to reinstall (select up to %d)", maximum) + chosen, err := c.prompter.MultiSelect("browser extensions", "your Kernel plan controls stored extension capacity", prompt, labels, defaults) + if err != nil { + return nil, err + } + if len(chosen) > maximum { + pterm.Warning.Printf("Select at most %d extension%s\n", maximum, pluralSuffix(maximum)) + defaults = chosen + continue + } + result := make([]localbrowser.Extension, 0, len(chosen)) + for _, label := range chosen { + result = append(result, byLabel[label]) + } + return result, nil + } +} + +func buildSelectedProfileData(ctx context.Context, profile localbrowser.Profile, selection localProfileDataSelection, cookies []localbrowser.Cookie, since time.Time) (localbrowser.ProfileData, map[string]int, error) { + data := selection.data + data.Cookies = cookies + counts := make(map[string]int, 5) + if len(cookies) > 0 { + counts["cookies"] = len(cookies) + } + if data.Bookmarks != nil { + counts["bookmarks"] = selection.bookmarkCount + } + if selection.history { + history, err := localbrowser.ExportHistory(ctx, profile, since) + if err != nil { + return localbrowser.ProfileData{}, nil, err + } + data.History = history + counts["history"] = len(history) + } + if selection.storage { + storage, err := localbrowser.ExportLocalStorage(ctx, profile, selection.storageSites) + if err != nil { + return localbrowser.ProfileData{}, nil, err + } + data.Storage = storage + counts["storage"] = len(storage) + } + if len(data.Extensions) > 0 { + counts["extensions"] = len(data.Extensions) + } + return data, counts, nil +} + +func selectedProfileCategories(counts map[string]int) []string { + order := []string{"cookies", "storage", "bookmarks", "history", "extensions"} + result := make([]string, 0, len(counts)) + for _, category := range order { + if count, selected := counts[category]; selected && count > 0 { + result = append(result, category) + } + } + return result +} + +func storageSiteBytes(sites []localbrowser.StorageSite) int64 { + var total int64 + for _, site := range sites { + total += site.Bytes + } + return total +} + +func selectedStorageSiteBytes(sites []localbrowser.StorageSite, selected []string) int64 { + set := make(map[string]struct{}, len(selected)) + for _, origin := range selected { + set[origin] = struct{}{} + } + var total int64 + for _, site := range sites { + if _, ok := set[site.Origin]; ok { + total += site.Bytes + } + } + return total +} + +func importedStorageOriginCount(records []localbrowser.StorageRecord) int { + origins := make(map[string]struct{}) + for _, record := range records { + origins[record.Origin] = struct{}{} + } + return len(origins) +} + +func formatBinaryBytes(bytes int64) string { + if bytes < 1<<20 { + return fmt.Sprintf("%.1f KiB", float64(bytes)/(1<<10)) + } + return fmt.Sprintf("%.1f MiB", float64(bytes)/(1<<20)) +} diff --git a/cmd/browser_import_profile_data_test.go b/cmd/browser_import_profile_data_test.go new file mode 100644 index 0000000..cf58bb6 --- /dev/null +++ b/cmd/browser_import_profile_data_test.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "strings" + "testing" + + localbrowser "github.com/kernel/cli/internal/browserimport" + "github.com/stretchr/testify/require" +) + +func TestLocalStorageSelectionUsesAllSitesWithinLimit(t *testing.T) { + sites := []localbrowser.StorageSite{ + {Origin: "https://example.com", Bytes: 1024}, + {Origin: "https://other.example", Bytes: 2048}, + } + + selected, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true) + require.NoError(t, err) + require.Equal(t, []string{"https://example.com", "https://other.example"}, selected) +} + +func TestLocalStorageSelectionRequiresReviewWhenOverLimit(t *testing.T) { + sites := []localbrowser.StorageSite{{Origin: "https://large.example", Bytes: localbrowser.MaxPortableStorageSize + 1}} + + _, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true) + require.ErrorContains(t, err, "run interactively to choose websites") +} + +func TestNonInteractiveExtensionSelectionHonorsCapacity(t *testing.T) { + extensions := []localbrowser.Extension{ + {ID: strings.Repeat("a", 32), Source: "chrome_web_store"}, + {ID: strings.Repeat("b", 32), Source: "chrome_web_store"}, + } + + _, err := (ProfilesImportLocalCmd{}).chooseProfileExtensions(extensions, storedExtensionCapacity{remaining: 1}, true, true) + require.ErrorContains(t, err, "only 1 can be added") +} + +func TestSelectedProfileCategoriesUsePortableApplyOrder(t *testing.T) { + categories := selectedProfileCategories(map[string]int{ + "extensions": 1, + "bookmarks": 2, + "cookies": 3, + "history": 4, + "storage": 5, + }) + + require.Equal(t, []string{"cookies", "storage", "bookmarks", "history", "extensions"}, categories) +} + +func TestProfilesImportLocalDefaultsHistoryOn(t *testing.T) { + flag := profilesImportLocalCmd.Flags().Lookup("history") + require.NotNil(t, flag) + require.Equal(t, "true", flag.DefValue) +} diff --git a/cmd/connector.go b/cmd/connector.go index 34d1621..cc66d60 100644 --- a/cmd/connector.go +++ b/cmd/connector.go @@ -59,6 +59,7 @@ func runConnectorOpen(cmd *cobra.Command, args []string) error { Version: metadata.Version, WaitTimeout: 30 * time.Minute, DashboardLaunch: true, + ImportHistory: true, } project, err := validateConnectorProject(cmd, input.ProjectID) if err != nil { diff --git a/cmd/profiles_import_local.go b/cmd/profiles_import_local.go index b06c013..e3493c4 100644 --- a/cmd/profiles_import_local.go +++ b/cmd/profiles_import_local.go @@ -41,6 +41,7 @@ type ProfilesImportLocalInput struct { PasswordManager string InstallAgentSkills bool DashboardLaunch bool + ImportHistory bool Project *kernel.Project } @@ -51,6 +52,7 @@ type ProfilesImportLocalCmd struct { providers func() []passwordmanager.Provider provisioner managedAuthProvisioner managedAuthCapacity func(context.Context) (managedAuthCapacity, error) + extensionCapacity func(context.Context) (storedExtensionCapacity, error) } type pendingManagedAuth struct { @@ -185,6 +187,11 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI if len(cookieSelection.sites) == 0 { return fmt.Errorf("select at least one website") } + since := c.now().AddDate(0, 0, -in.Days) + profileDataSelection, err := c.chooseLocalProfileData(ctx, profile, since, in.ImportHistory, nonInteractive, humanOutput) + if err != nil { + return err + } pendingLogins := pendingManagedAuth{} if managedAuthImportRequested(in.PasswordManager, nonInteractive) { phaseStarted = time.Now() @@ -196,6 +203,16 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI return err } } + if !nonInteractive { + proceed, err := c.confirmBrowserImport(targetName, cookieSelection, cookieSites, profileDataSelection, pendingLogins) + if err != nil { + return err + } + if !proceed { + pterm.Info.Println("Browser import canceled; no Kernel resources were changed") + return nil + } + } if humanOutput { pterm.Println() if cookieSelection.all { @@ -224,7 +241,12 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI version = "dev" } phaseStarted = time.Now() - bundle, err := localbrowser.BuildCookieBundle(ctx, profile, targetName, version, cookies) + profileData, itemCounts, err := buildSelectedProfileData(ctx, profile, profileDataSelection, cookies, since) + if err != nil { + return err + } + categories := selectedProfileCategories(itemCounts) + bundle, err := localbrowser.BuildProfileBundle(ctx, profile, targetName, version, profileData) timings["bundle"] = time.Since(phaseStarted) if err != nil { return err @@ -247,13 +269,13 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI } inventory := localbrowser.Inventory{Sources: []localbrowser.Source{{ ID: profile.ID, Kind: "browser", Name: profile.DisplayName(), Browser: profile.Browser.ID, - DataTypes: []string{"cookies"}, ItemCounts: map[string]int{"cookies": len(cookies)}, + DataTypes: categories, ItemCounts: itemCounts, }}} status, err := client.SubmitInventory(ctx, created.ID, created.HelperToken, inventory) if err != nil { return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err) } - selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: []string{"cookies"}}}, CredentialSources: make([]string, 0)} + selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: categories}}, CredentialSources: make([]string, 0)} status, err = client.SubmitSelection(ctx, created.ID, selection) if err != nil { return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err) @@ -275,6 +297,18 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI profileID := status.Applied.Profiles[0].ProfileID if humanOutput { pterm.Success.Printf("Imported %d cookies from %d websites\n", len(cookies), importedCookieSites) + if count := itemCounts["bookmarks"]; count > 0 { + pterm.Success.Printf("Imported %d bookmarks\n", count) + } + if count := itemCounts["history"]; count > 0 { + pterm.Success.Printf("Imported %d history entries\n", count) + } + if count := itemCounts["storage"]; count > 0 { + pterm.Success.Printf("Imported %d local storage keys from %d origins\n", count, importedStorageOriginCount(profileData.Storage)) + } + if count := itemCounts["extensions"]; count > 0 { + pterm.Success.Printf("Installed %d browser extensions\n", count) + } } connectionIDs := make([]string, 0) approvedLogins := make([]passwordmanager.Record, 0) @@ -325,7 +359,7 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI } } if in.Output == "json" { - data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": len(cookies), "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", " ") + data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": itemCounts["cookies"], "browser_data_imported": itemCounts, "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", " ") if err != nil { return err } @@ -1405,8 +1439,8 @@ var cuidLikeProfileName = regexp.MustCompile(`^[a-z0-9]{24}$`) var profilesImportLocalCmd = &cobra.Command{ Use: "import-local", - Short: "Import cookies from a local browser", - Long: "Import all cookies or selected websites from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.", + Short: "Import a local browser profile", + Long: "Import cookies and selected portable data from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.", Args: cobra.NoArgs, RunE: runProfilesImportLocal, } @@ -1427,6 +1461,7 @@ func init() { profilesImportLocalCmd.Flags().Int("days", 30, "Rank websites used during the last number of days (1-90)") profilesImportLocalCmd.Flags().Duration("wait-timeout", 30*time.Minute, "Maximum time to wait for the import to complete") profilesImportLocalCmd.Flags().BoolP("yes", "y", false, "Import all cookies and use unambiguous defaults without prompting") + profilesImportLocalCmd.Flags().Bool("history", true, "Include browsing history from the selected --days window") profilesImportLocalCmd.Flags().String("password-manager", "", "Password managers to search: bitwarden, 1password, both comma-separated, all, or none") profilesImportLocalCmd.Flags().Bool("install-agent-skills", false, "Install the Kernel Managed Auth skill into detected agent directories") addJSONOutputFlag(profilesImportLocalCmd) @@ -1442,12 +1477,13 @@ func runProfilesImportLocal(cmd *cobra.Command, _ []string) error { skipConfirm, _ := cmd.Flags().GetBool("yes") passwordManager, _ := cmd.Flags().GetString("password-manager") installAgentSkills, _ := cmd.Flags().GetBool("install-agent-skills") + importHistory, _ := cmd.Flags().GetBool("history") output, _ := cmd.Flags().GetString("output") project, _ := cmd.Flags().GetString("project") return runProfilesImportLocalWithInput(cmd, ProfilesImportLocalInput{ BrowserProfile: browserProfile, ProfileName: profileName, Sites: sites, Days: days, SkipConfirm: skipConfirm, Output: output, ProjectID: resolveProjectSelection(project), Version: metadata.Version, - WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, + WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, ImportHistory: importHistory, }) } @@ -1484,6 +1520,9 @@ func runProfilesImportLocalWithInput(cmd *cobra.Command, input ProfilesImportLoc providers: passwordmanager.Detect, provisioner: kernelManagedAuthProvisioner{credentials: &credentials, connections: &connections}, managedAuthCapacity: func(ctx context.Context) (managedAuthCapacity, error) { return loadManagedAuthCapacity(ctx, &limits) }, + extensionCapacity: func(ctx context.Context) (storedExtensionCapacity, error) { + return loadStoredExtensionCapacity(ctx, &limits) + }, } input.ProjectID = project.ID if input.Version == "" { diff --git a/cmd/profiles_import_local_test.go b/cmd/profiles_import_local_test.go index 9392f91..0a9bd3b 100644 --- a/cmd/profiles_import_local_test.go +++ b/cmd/profiles_import_local_test.go @@ -384,6 +384,19 @@ func TestDecodeManagedAuthCapacity(t *testing.T) { }) } +func TestDecodeStoredExtensionCapacity(t *testing.T) { + capacity, err := decodeStoredExtensionCapacity(`{"max_stored_extensions":5,"stored_extensions_used":3}`) + require.NoError(t, err) + assert.Equal(t, storedExtensionCapacity{remaining: 2}, capacity) + + capacity, err = decodeStoredExtensionCapacity(`{"max_stored_extensions":null,"stored_extensions_used":12}`) + require.NoError(t, err) + assert.Equal(t, storedExtensionCapacity{unlimited: true}, capacity) + + _, err = decodeStoredExtensionCapacity(`{"max_auth_connections":10}`) + require.ErrorContains(t, err, "does not expose stored extension capacity") +} + func TestChooseManagedAuthLoginsRejectsExplicitBatchAboveRemainingConnections(t *testing.T) { command := managedAuthTestCommand(func() []passwordmanager.Provider { return []passwordmanager.Provider{fakePasswordManager{candidates: []passwordmanager.Candidate{ diff --git a/go.mod b/go.mod index 3ac5d5c..88a084c 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.11.1 + github.com/syndtr/goleveldb v1.0.0 github.com/zalando/go-keyring v0.2.6 golang.org/x/crypto v0.52.0 golang.org/x/net v0.54.0 @@ -40,6 +41,7 @@ require ( github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/gookit/color v1.5.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect diff --git a/go.sum b/go.sum index 89114c2..f82040e 100644 --- a/go.sum +++ b/go.sum @@ -50,16 +50,22 @@ github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -95,6 +101,11 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -128,6 +139,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -153,6 +166,7 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -161,11 +175,13 @@ golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -201,7 +217,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/browserimport/bundle.go b/internal/browserimport/bundle.go index eeb24c9..11981ca 100644 --- a/internal/browserimport/bundle.go +++ b/internal/browserimport/bundle.go @@ -6,12 +6,17 @@ import ( "context" "encoding/json" "fmt" - "sort" "github.com/klauspost/compress/zstd" ) -const maxBundleBytes = 128 << 20 +const ( + maxBundleBytes = 128 << 20 + maxPortableFileBytes = 64 << 20 + maxPortableRecordBytes = 1 << 20 + maxPortableRecords = 100_000 + maxPortableDocumentBytes = 16 << 20 +) type Manifest struct { Version int `json:"version"` @@ -33,37 +38,109 @@ type BundleProfile struct { } type ProfileFiles struct { - Cookies string `json:"cookies,omitempty"` + Cookies string `json:"cookies,omitempty"` + Storage string `json:"storage,omitempty"` + Bookmarks string `json:"bookmarks,omitempty"` + History string `json:"history,omitempty"` + Extensions string `json:"extensions,omitempty"` +} + +type bundleFile struct { + path string + data []byte } -func BuildCookieBundle(ctx context.Context, profile Profile, targetName, version string, cookies []Cookie) ([]byte, error) { - if len(cookies) == 0 { - return nil, fmt.Errorf("no cookies were selected") +func BuildProfileBundle(ctx context.Context, profile Profile, targetName, version string, data ProfileData) ([]byte, error) { + if len(data.Cookies) == 0 && len(data.Storage) == 0 && data.Bookmarks == nil && len(data.History) == 0 && len(data.Extensions) == 0 { + return nil, fmt.Errorf("no browser data was selected") + } + files := ProfileFiles{} + payloads := make([]bundleFile, 0, 5) + addJSON := func(path, label string, value any) error { + encoded, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("encode browser %s: %w", label, err) + } + if len(encoded) > maxPortableDocumentBytes { + return fmt.Errorf("browser %s exceeds the 16 MiB import limit", label) + } + payloads = append(payloads, bundleFile{path: path, data: encoded}) + return nil + } + if len(data.Cookies) > 0 { + files.Cookies = "profiles/" + profile.ID + "/cookies.jsonl" + encoded, err := encodeJSONL("cookies", data.Cookies) + if err != nil { + return nil, fmt.Errorf("encode browser cookies: %w", err) + } + payloads = append(payloads, bundleFile{path: files.Cookies, data: encoded}) + } + if len(data.Storage) > 0 { + files.Storage = "profiles/" + profile.ID + "/storage.jsonl" + encoded, err := encodeJSONL("local storage", data.Storage) + if err != nil { + return nil, fmt.Errorf("encode browser local storage: %w", err) + } + payloads = append(payloads, bundleFile{path: files.Storage, data: encoded}) + } + if data.Bookmarks != nil { + files.Bookmarks = "profiles/" + profile.ID + "/bookmarks.json" + if err := addJSON(files.Bookmarks, "bookmarks", data.Bookmarks); err != nil { + return nil, err + } + } + if len(data.History) > 0 { + files.History = "profiles/" + profile.ID + "/history.jsonl" + encoded, err := encodeJSONL("history", data.History) + if err != nil { + return nil, fmt.Errorf("encode browser history: %w", err) + } + payloads = append(payloads, bundleFile{path: files.History, data: encoded}) + } + if len(data.Extensions) > 0 { + files.Extensions = "profiles/" + profile.ID + "/extensions.json" + if err := addJSON(files.Extensions, "extensions", data.Extensions); err != nil { + return nil, err + } } - cookiePath := "profiles/" + profile.ID + "/cookies.jsonl" manifest := Manifest{ Version: BundleVersion, Source: BundleSource{OS: "macos", HelperVersion: version}, Profiles: []BundleProfile{{ ID: profile.ID, Browser: profile.Browser.ID, SourceName: profile.DisplayName(), TargetName: targetName, - Files: ProfileFiles{Cookies: cookiePath}, + Files: files, }}, } manifestData, err := json.Marshal(manifest) if err != nil { return nil, fmt.Errorf("encode import manifest: %w", err) } - var cookieData bytes.Buffer - encoder := json.NewEncoder(&cookieData) - for _, cookie := range cookies { - if err := encoder.Encode(cookie); err != nil { - return nil, fmt.Errorf("encode browser cookie: %w", err) + return encodeBundle(ctx, manifestData, payloads) +} + +func encodeJSONL[T any](label string, records []T) ([]byte, error) { + if len(records) > maxPortableRecords { + return nil, fmt.Errorf("browser %s exceeds the %d record import limit", label, maxPortableRecords) + } + var output bytes.Buffer + for _, record := range records { + encoded, err := json.Marshal(record) + if err != nil { + return nil, err + } + if len(encoded)+1 > maxPortableRecordBytes { + return nil, fmt.Errorf("one browser %s record exceeds the 1 MiB import limit", label) } + if output.Len()+len(encoded)+1 > maxPortableFileBytes { + return nil, fmt.Errorf("browser %s exceeds the 64 MiB import limit", label) + } + output.Write(encoded) + output.WriteByte('\n') } - return encodeBundle(ctx, manifestData, map[string][]byte{cookiePath: cookieData.Bytes()}) + return output.Bytes(), nil } -func encodeBundle(ctx context.Context, manifest []byte, files map[string][]byte) ([]byte, error) { +func encodeBundle(ctx context.Context, manifest []byte, files []bundleFile) ([]byte, error) { var output bytes.Buffer zstdWriter, err := zstd.NewWriter(&output, zstd.WithEncoderConcurrency(1)) if err != nil { @@ -83,13 +160,8 @@ func encodeBundle(ctx context.Context, manifest []byte, files map[string][]byte) if err := write("manifest.json", manifest); err != nil { return nil, err } - paths := make([]string, 0, len(files)) - for path := range files { - paths = append(paths, path) - } - sort.Strings(paths) - for _, path := range paths { - if err := write(path, files[path]); err != nil { + for _, file := range files { + if err := write(file.path, file.data); err != nil { return nil, err } } diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index 72bdb6f..dc8b733 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -3,44 +3,57 @@ package browserimport import ( "archive/tar" "bytes" - "context" "encoding/json" "io" + "strings" "testing" "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestBuildCookieBundleMatchesServerContract(t *testing.T) { - profile := Profile{ID: "chrome-default-1234", Name: "Personal", Browser: Browser{ID: "chrome", Name: "Google Chrome"}} - bundle, err := BuildCookieBundle(context.Background(), profile, "my-browser", "test", []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret", Secure: true}}) +func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { + profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} + bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ + Cookies: []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret"}}, + Storage: []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, + Bookmarks: &BookmarkDocument{Roots: []BookmarkRoot{{Name: "bookmark_bar", Children: []BookmarkNode{{Title: "Kernel", URL: "https://onkernel.com"}}}}}, + Extensions: []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Source: "chrome_web_store"}}, + }) require.NoError(t, err) decoder, err := zstd.NewReader(bytes.NewReader(bundle)) require.NoError(t, err) defer decoder.Close() reader := tar.NewReader(decoder) - header, err := reader.Next() - require.NoError(t, err) - assert.Equal(t, "manifest.json", header.Name) - manifestData, err := io.ReadAll(reader) - require.NoError(t, err) + files := make(map[string][]byte) + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + files[header.Name], err = io.ReadAll(reader) + require.NoError(t, err) + } var manifest Manifest - require.NoError(t, json.Unmarshal(manifestData, &manifest)) - assert.Equal(t, BundleVersion, manifest.Version) - assert.Equal(t, "my-browser", manifest.Profiles[0].TargetName) - - header, err = reader.Next() - require.NoError(t, err) - assert.Equal(t, "profiles/chrome-default-1234/cookies.jsonl", header.Name) - var cookie Cookie - require.NoError(t, json.NewDecoder(reader).Decode(&cookie)) - assert.Equal(t, "secret", cookie.Value) + require.NoError(t, json.Unmarshal(files["manifest.json"], &manifest)) + require.NotEmpty(t, manifest.Profiles[0].Files.Cookies) + require.NotEmpty(t, manifest.Profiles[0].Files.Storage) + require.NotEmpty(t, manifest.Profiles[0].Files.Bookmarks) + require.Empty(t, manifest.Profiles[0].Files.History) + require.NotEmpty(t, manifest.Profiles[0].Files.Extensions) } -func TestBuildCookieBundleRequiresCookies(t *testing.T) { - _, err := BuildCookieBundle(context.Background(), Profile{}, "profile", "test", nil) - assert.EqualError(t, err, "no cookies were selected") +func TestEncodeJSONLEnforcesPortableRecordLimits(t *testing.T) { + _, err := encodeJSONL("history", make([]HistoryRecord, maxPortableRecords+1)) + require.ErrorContains(t, err, "100000 record import limit") + + _, err = encodeJSONL("local storage", []StorageRecord{{ + Origin: "https://example.com", + Kind: StorageKindLocal, + Key: "large", + Value: strings.Repeat("x", maxPortableRecordBytes), + }}) + require.ErrorContains(t, err, "1 MiB import limit") } diff --git a/internal/browserimport/chromium.go b/internal/browserimport/chromium.go index bffdaa6..cda6ecb 100644 --- a/internal/browserimport/chromium.go +++ b/internal/browserimport/chromium.go @@ -7,6 +7,7 @@ import ( "crypto/cipher" "crypto/sha1" "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -18,9 +19,13 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" "time" + "unicode/utf16" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" "golang.org/x/crypto/pbkdf2" "golang.org/x/net/publicsuffix" ) @@ -29,9 +34,14 @@ const ( maxDocumentBytes = 16 << 20 maxSQLiteOutput = 64 << 20 maxSQLiteBytes = 2 << 30 + maxStorageBytes = MaxPortableStorageSize + maxStorageSource = 512 << 20 + maxStorageRecord = 1 << 20 + maxStorageCount = 100_000 ) var profileIDCharacters = regexp.MustCompile(`[^a-zA-Z0-9._-]+`) +var chromeWebStoreExtensionID = regexp.MustCompile(`^[a-p]{32}$`) func DiscoverMacOSProfiles(home string) ([]Profile, error) { browsers := []Browser{ @@ -244,6 +254,373 @@ func ExportCookies(ctx context.Context, profile Profile, selectedSites []string) return cookies, nil } +func ExportBookmarks(profile Profile) (BookmarkDocument, int, error) { + payload, err := readFileBounded(filepath.Join(profile.Path, "Bookmarks"), maxDocumentBytes) + if errors.Is(err, os.ErrNotExist) { + return BookmarkDocument{}, 0, nil + } + if err != nil { + return BookmarkDocument{}, 0, fmt.Errorf("read browser bookmarks: %w", err) + } + var source struct { + Roots map[string]chromiumBookmarkNode `json:"roots"` + } + if err := json.Unmarshal(payload, &source); err != nil { + return BookmarkDocument{}, 0, fmt.Errorf("decode browser bookmarks: %w", err) + } + document := BookmarkDocument{Roots: make([]BookmarkRoot, 0, 3)} + count := 0 + for _, name := range []string{"bookmark_bar", "other", "synced"} { + root, ok := source.Roots[name] + if !ok { + continue + } + portableName := name + if name == "synced" { + portableName = "mobile" + } + document.Roots = append(document.Roots, BookmarkRoot{Name: portableName, Children: portableBookmarkNodes(root.Children, &count)}) + } + return document, count, nil +} + +type chromiumBookmarkNode struct { + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + DateAdded string `json:"date_added"` + DateLastUsed string `json:"date_last_used"` + Children []chromiumBookmarkNode `json:"children"` +} + +func portableBookmarkNodes(nodes []chromiumBookmarkNode, count *int) []BookmarkNode { + result := make([]BookmarkNode, 0, len(nodes)) + for _, node := range nodes { + if node.Type == "url" { + parsed, err := url.Parse(node.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + continue + } + *count++ + result = append(result, BookmarkNode{Title: node.Name, URL: node.URL, DateAdded: chromiumTimestamp(node.DateAdded), DateLastUsed: chromiumTimestamp(node.DateLastUsed)}) + continue + } + if node.Type != "folder" { + continue + } + children := portableBookmarkNodes(node.Children, count) + *count++ + result = append(result, BookmarkNode{Title: node.Name, Children: children, DateAdded: chromiumTimestamp(node.DateAdded)}) + } + return result +} + +func chromiumTimestamp(value string) *time.Time { + microseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || microseconds <= 0 { + return nil + } + parsed := chromiumTime(microseconds) + return &parsed +} + +func ExportHistory(ctx context.Context, profile Profile, since time.Time) ([]HistoryRecord, error) { + const chromeEpochMicros = int64(11_644_473_600_000_000) + cutoff := since.UnixMicro() + chromeEpochMicros + query := fmt.Sprintf(`SELECT urls.url, urls.title, MAX(visits.visit_time) AS last_visit_time, COUNT(*) AS visit_count FROM visits JOIN urls ON urls.id = visits.url WHERE visits.visit_time >= %d AND (urls.url LIKE 'http://%%' OR urls.url LIKE 'https://%%') GROUP BY urls.id ORDER BY last_visit_time DESC LIMIT 100001`, cutoff) + payload, err := sqliteSnapshotJSON(ctx, filepath.Join(profile.Path, "History"), query) + if err != nil { + return nil, fmt.Errorf("read browser history: %w", err) + } + var rows []struct { + URL string `json:"url"` + Title string `json:"title"` + VisitedAt int64 `json:"last_visit_time"` + Count int `json:"visit_count"` + } + if len(bytes.TrimSpace(payload)) != 0 { + if err := json.Unmarshal(payload, &rows); err != nil { + return nil, fmt.Errorf("decode browser history: %w", err) + } + } + if len(rows) > 100000 { + return nil, fmt.Errorf("browser history exceeds the 100000-record import limit; use fewer days") + } + records := make([]HistoryRecord, 0, len(rows)) + for _, row := range rows { + parsed, err := url.Parse(row.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + continue + } + records = append(records, HistoryRecord{URL: row.URL, Title: row.Title, VisitedAt: chromiumTime(row.VisitedAt), VisitCount: max(1, row.Count)}) + } + return records, nil +} + +func HistoryCount(ctx context.Context, profile Profile, since time.Time) (int, error) { + const chromeEpochMicros = int64(11_644_473_600_000_000) + cutoff := since.UnixMicro() + chromeEpochMicros + query := fmt.Sprintf(`SELECT COUNT(*) AS count FROM visits JOIN urls ON urls.id = visits.url WHERE visits.visit_time >= %d AND (urls.url LIKE 'http://%%' OR urls.url LIKE 'https://%%')`, cutoff) + payload, err := sqliteSnapshotJSON(ctx, filepath.Join(profile.Path, "History"), query) + if err != nil { + return 0, fmt.Errorf("count browser history: %w", err) + } + var rows []struct { + Count int `json:"count"` + } + if err := json.Unmarshal(payload, &rows); err != nil { + return 0, fmt.Errorf("decode browser history count: %w", err) + } + if len(rows) != 1 || rows[0].Count < 0 { + return 0, fmt.Errorf("browser history count is invalid") + } + return rows[0].Count, nil +} + +func LocalStorageSites(ctx context.Context, profile Profile) ([]StorageSite, error) { + database, cleanup, err := levelDBSnapshot(ctx, filepath.Join(profile.Path, "Local Storage", "leveldb")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("snapshot browser local storage: %w", err) + } + defer cleanup() + + db, err := leveldb.OpenFile(database, nil) + if err != nil { + return nil, fmt.Errorf("open browser local storage: %w", err) + } + defer db.Close() + + metadataBytes := make(map[string]int64) + recordBytes := make(map[string]int64) + iterator := db.NewIterator(nil, nil) + defer iterator.Release() + for iterator.Next() { + key := iterator.Key() + if bytes.HasPrefix(key, []byte("META:")) { + origin, ok := portableStorageOrigin(string(key[len("META:"):])) + if !ok { + continue + } + size, err := localStorageMetadataSize(iterator.Value()) + if err != nil { + return nil, fmt.Errorf("decode local storage metadata for %s: %w", origin, err) + } + metadataBytes[origin] = size + continue + } + if len(key) < 2 || key[0] != '_' { + continue + } + separator := bytes.IndexByte(key[1:], 0) + if separator < 1 { + continue + } + origin, ok := portableStorageOrigin(string(key[1 : 1+separator])) + if ok { + recordBytes[origin] += int64(len(key[2+separator:]) + len(iterator.Value())) + } + } + if err := iterator.Error(); err != nil { + return nil, fmt.Errorf("read browser local storage metadata: %w", err) + } + sites := make([]StorageSite, 0, len(metadataBytes)+len(recordBytes)) + for origin, bytes := range recordBytes { + if metadataBytes[origin] > bytes { + bytes = metadataBytes[origin] + } + sites = append(sites, StorageSite{Origin: origin, Bytes: bytes}) + delete(metadataBytes, origin) + } + for origin, bytes := range metadataBytes { + sites = append(sites, StorageSite{Origin: origin, Bytes: bytes}) + } + sort.Slice(sites, func(left, right int) bool { return sites[left].Origin < sites[right].Origin }) + return sites, nil +} + +func ExportLocalStorage(ctx context.Context, profile Profile, selectedOrigins []string) ([]StorageRecord, error) { + database, cleanup, err := levelDBSnapshot(ctx, filepath.Join(profile.Path, "Local Storage", "leveldb")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("snapshot browser local storage: %w", err) + } + defer cleanup() + + db, err := leveldb.OpenFile(database, nil) + if err != nil { + return nil, fmt.Errorf("open browser local storage: %w", err) + } + defer db.Close() + + selected := make(map[string]struct{}, len(selectedOrigins)) + for _, origin := range selectedOrigins { + selected[origin] = struct{}{} + } + records := make([]StorageRecord, 0) + encodedBytes := 0 + iterator := db.NewIterator(util.BytesPrefix([]byte("_")), nil) + defer iterator.Release() + for iterator.Next() { + key := iterator.Key()[1:] + separator := bytes.IndexByte(key, 0) + if separator < 1 || separator == len(key)-1 { + continue + } + origin, ok := portableStorageOrigin(string(key[:separator])) + if !ok { + continue + } + if len(selected) > 0 { + if _, ok := selected[origin]; !ok { + continue + } + } + scriptKey, err := decodeChromiumStorageString(key[separator+1:]) + if err != nil || scriptKey == "" { + continue + } + value, err := decodeChromiumStorageString(iterator.Value()) + if err != nil { + continue + } + record := StorageRecord{Origin: origin, Kind: StorageKindLocal, Key: scriptKey, Value: value} + encoded, err := json.Marshal(record) + if err != nil { + return nil, fmt.Errorf("encode browser local storage: %w", err) + } + if len(encoded)+1 > maxStorageRecord { + return nil, fmt.Errorf("local storage key %q for %s exceeds the 1 MiB record limit", scriptKey, origin) + } + encodedBytes += len(encoded) + 1 + if encodedBytes > maxStorageBytes { + return nil, fmt.Errorf("browser local storage exceeds the 64 MiB import limit; choose fewer websites") + } + records = append(records, record) + if len(records) > maxStorageCount { + return nil, fmt.Errorf("browser local storage exceeds the 100000-record import limit; choose fewer websites") + } + } + if err := iterator.Error(); err != nil { + return nil, fmt.Errorf("read browser local storage: %w", err) + } + sort.Slice(records, func(left, right int) bool { + if records[left].Origin == records[right].Origin { + return records[left].Key < records[right].Key + } + return records[left].Origin < records[right].Origin + }) + return records, nil +} + +func portableStorageOrigin(raw string) (string, bool) { + parsed, err := url.Parse(raw) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.User != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", false + } + origin := parsed.Scheme + "://" + parsed.Host + if raw != origin && raw != origin+"/" { + return "", false + } + return origin, true +} + +func decodeChromiumStorageString(raw []byte) (string, error) { + if len(raw) == 0 { + return "", fmt.Errorf("string is missing its encoding prefix") + } + switch raw[0] { + case 0: + if len(raw[1:])%2 != 0 { + return "", fmt.Errorf("UTF-16 string has an odd byte length") + } + units := make([]uint16, len(raw[1:])/2) + for index := range units { + units[index] = binary.LittleEndian.Uint16(raw[1+index*2:]) + } + return string(utf16.Decode(units)), nil + case 1: + runes := make([]rune, len(raw)-1) + for index, value := range raw[1:] { + runes[index] = rune(value) + } + return string(runes), nil + default: + return "", fmt.Errorf("unsupported string encoding prefix %d", raw[0]) + } +} + +func localStorageMetadataSize(raw []byte) (int64, error) { + for len(raw) > 0 { + tag, read := binary.Uvarint(raw) + if read <= 0 { + return 0, fmt.Errorf("invalid protobuf tag") + } + raw = raw[read:] + field, wire := tag>>3, tag&7 + if wire != 0 { + return 0, fmt.Errorf("unsupported protobuf wire type %d", wire) + } + value, read := binary.Uvarint(raw) + if read <= 0 { + return 0, fmt.Errorf("invalid protobuf value") + } + raw = raw[read:] + if field == 2 { + if value > uint64(^uint64(0)>>1) { + return 0, fmt.Errorf("size exceeds int64") + } + return int64(value), nil + } + } + return 0, fmt.Errorf("size field is missing") +} + +func DiscoverExtensions(profile Profile) ([]Extension, error) { + payload, err := readFileBounded(filepath.Join(profile.Path, "Secure Preferences"), maxDocumentBytes) + if errors.Is(err, os.ErrNotExist) { + payload, err = readFileBounded(filepath.Join(profile.Path, "Preferences"), maxDocumentBytes) + } + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read browser extensions: %w", err) + } + var source struct { + Extensions struct { + Settings map[string]struct { + State *int `json:"state"` + FromWebStore bool `json:"from_webstore"` + Manifest struct { + Name string `json:"name"` + } `json:"manifest"` + } `json:"settings"` + } `json:"extensions"` + } + if err := json.Unmarshal(payload, &source); err != nil { + return nil, fmt.Errorf("decode browser extensions: %w", err) + } + result := make([]Extension, 0) + for id, setting := range source.Extensions.Settings { + if (setting.State != nil && *setting.State != 1) || !setting.FromWebStore || !chromeWebStoreExtensionID.MatchString(id) { + continue + } + result = append(result, Extension{ID: id, Name: setting.Manifest.Name, Source: "chrome_web_store"}) + } + sort.Slice(result, func(i, j int) bool { + if result[i].Name == result[j].Name { + return result[i].ID < result[j].ID + } + return result[i].Name < result[j].Name + }) + return result, nil +} + // CookieSites returns every website with importable cookies, ranked by recent // browser use. It reads cookie metadata only; values are decrypted by // ExportCookies after the user chooses what to import. @@ -374,6 +751,93 @@ type fileFingerprint struct { modified time.Time } +type directoryFingerprint struct { + name string + size int64 + modified time.Time +} + +func levelDBSnapshot(ctx context.Context, sourceDirectory string) (string, func(), error) { + if _, err := os.Stat(sourceDirectory); err != nil { + return "", nil, err + } + for attempt := 0; attempt < 3; attempt++ { + before, err := fingerprintLevelDB(sourceDirectory) + if err != nil { + return "", nil, err + } + var total int64 + for _, file := range before { + total += file.size + } + if total > maxStorageSource { + return "", nil, fmt.Errorf("browser local storage source exceeds 512 MiB") + } + root, err := os.MkdirTemp("", "kernel-browser-import-leveldb-") + if err != nil { + return "", nil, err + } + destination := filepath.Join(root, "leveldb") + if err := os.Mkdir(destination, 0o700); err != nil { + _ = os.RemoveAll(root) + return "", nil, err + } + changed := false + for _, file := range before { + err := copyFileBounded(ctx, filepath.Join(sourceDirectory, file.name), filepath.Join(destination, file.name), file.size) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, errFileGrew) { + changed = true + break + } + if err != nil { + _ = os.RemoveAll(root) + return "", nil, fmt.Errorf("copy local storage %s: %w", file.name, err) + } + } + after, err := fingerprintLevelDB(sourceDirectory) + if !changed && err == nil && directoryFingerprintsEqual(before, after) { + return destination, func() { _ = os.RemoveAll(root) }, nil + } + _ = os.RemoveAll(root) + } + return "", nil, fmt.Errorf("browser local storage changed while it was being read; try again") +} + +func fingerprintLevelDB(directory string) ([]directoryFingerprint, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + result := make([]directoryFingerprint, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || entry.Name() == "LOCK" || entry.Name() == "LOG" || entry.Name() == "LOG.old" { + continue + } + info, err := entry.Info() + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + continue + } + result = append(result, directoryFingerprint{name: entry.Name(), size: info.Size(), modified: info.ModTime()}) + } + sort.Slice(result, func(left, right int) bool { return result[left].name < result[right].name }) + return result, nil +} + +func directoryFingerprintsEqual(left, right []directoryFingerprint) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + func sqliteSnapshot(ctx context.Context, databasePath string) (string, func(), error) { if !fileExists(databasePath) { return "", nil, fmt.Errorf("browser database %q was not found", filepath.Base(databasePath)) diff --git a/internal/browserimport/chromium_test.go b/internal/browserimport/chromium_test.go index 188046a..baa1ee8 100644 --- a/internal/browserimport/chromium_test.go +++ b/internal/browserimport/chromium_test.go @@ -6,6 +6,7 @@ import ( "crypto/aes" "crypto/cipher" "crypto/sha256" + "encoding/binary" "encoding/json" "fmt" "os" @@ -13,9 +14,11 @@ import ( "path/filepath" "testing" "time" + "unicode/utf16" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb" ) func TestDiscoverMacOSProfilesFindsChromeAndHelium(t *testing.T) { @@ -229,6 +232,92 @@ INSERT INTO cookies VALUES assert.Equal(t, []string{"github", "google"}, []string{cookies[0].Name, cookies[1].Name}) } +func TestExportBookmarksNormalizesPortableRoots(t *testing.T) { + profile := sqliteProfileFixture(t) + payload := `{"roots":{"bookmark_bar":{"children":[{"type":"url","name":"Kernel","url":"https://onkernel.com","date_added":"13400000000000000"},{"type":"url","name":"Local","url":"chrome://settings"}]},"other":{"children":[{"type":"folder","name":"Work","children":[{"type":"url","name":"GitHub","url":"https://github.com"}]}]}}}` + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Bookmarks"), []byte(payload), 0o600)) + + document, count, err := ExportBookmarks(profile) + require.NoError(t, err) + require.Equal(t, 3, count) + require.Len(t, document.Roots, 2) + require.Equal(t, "https://onkernel.com", document.Roots[0].Children[0].URL) + require.Len(t, document.Roots[0].Children, 1, "non-http bookmarks must not leave the laptop") +} + +func TestExportHistoryUsesRequestedWindow(t *testing.T) { + profile := sqliteProfileFixture(t) + database := filepath.Join(profile.Path, "History") + runSQLite(t, database, ` +CREATE TABLE urls (id INTEGER PRIMARY KEY, url TEXT, title TEXT, last_visit_time INTEGER, visit_count INTEGER); +CREATE TABLE visits (id INTEGER PRIMARY KEY, url INTEGER, visit_time INTEGER); +INSERT INTO urls VALUES + (1, 'https://recent.example', 'Recent', 13400000000000000, 4), + (2, 'chrome://settings', 'Private', 13400000001000000, 2), + (3, 'https://old.example', 'Old', 12000000000000000, 1); +INSERT INTO visits VALUES + (1, 1, 13400000000000000), + (2, 1, 13400000000000001), + (3, 2, 13400000001000000), + (4, 3, 12000000000000000); +`) + + records, err := ExportHistory(t.Context(), profile, time.UnixMicro(13400000000000000-11_644_473_600_000_000)) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, "https://recent.example", records[0].URL) + require.Equal(t, 2, records[0].VisitCount) + count, err := HistoryCount(t.Context(), profile, time.UnixMicro(13400000000000000-11_644_473_600_000_000)) + require.NoError(t, err) + require.Equal(t, 2, count) +} + +func TestLocalStorageSitesAndExportUseLivePortableRecords(t *testing.T) { + profile := sqliteProfileFixture(t) + databasePath := filepath.Join(profile.Path, "Local Storage", "leveldb") + require.NoError(t, os.MkdirAll(filepath.Dir(databasePath), 0o755)) + database, err := leveldb.OpenFile(databasePath, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + require.NoError(t, database.Put([]byte("META:https://example.com"), localStorageMetadataFixture(1234), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("https://example.com", "theme"), chromiumStorageStringFixture("dark"), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("https://other.example", "emoji"), chromiumStorageStringFixture("hello 世界"), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("chrome-extension://abcdefghijklmnopabcdefghijklmnop", "private"), chromiumStorageStringFixture("skip"), nil)) + sites, err := LocalStorageSites(t.Context(), profile) + require.NoError(t, err) + require.Len(t, sites, 2) + require.Equal(t, StorageSite{Origin: "https://example.com", Bytes: 1234}, sites[0]) + require.Equal(t, "https://other.example", sites[1].Origin) + require.Positive(t, sites[1].Bytes) + + records, err := ExportLocalStorage(t.Context(), profile, []string{"https://example.com"}) + require.NoError(t, err) + require.Equal(t, []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, records) + + records, err = ExportLocalStorage(t.Context(), profile, nil) + require.NoError(t, err) + require.Len(t, records, 2) + require.Equal(t, "hello 世界", records[1].Value) +} + +func TestDiscoverExtensionsUsesPortableAllowlist(t *testing.T) { + profile := sqliteProfileFixture(t) + payload := `{ + "browser":{"theme":{"color_scheme2":2},"show_home_button":true}, + "extensions":{"settings":{ + "abcdefghijklmnopabcdefghijklmnop":{"from_webstore":true,"manifest":{"name":"Eligible"}}, + "bcdefghijklmnopabcdefghijklmnopa":{"state":0,"from_webstore":true,"manifest":{"name":"Disabled"}}, + "cdefghijklmnopabcdefghijklmnopab":{"state":1,"from_webstore":false,"manifest":{"name":"Local"}} + }} +}` + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Preferences"), []byte(payload), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Secure Preferences"), []byte(payload), 0o600)) + + extensions, err := DiscoverExtensions(profile) + require.NoError(t, err) + require.Equal(t, []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Name: "Eligible", Source: "chrome_web_store"}}, extensions) +} + func TestSelectedCookieFilterEscapesInput(t *testing.T) { cte, clause := selectedCookieFilter([]string{"example.com' OR 1=1 --"}, false) assert.Contains(t, cte, "example.com'' or 1=1 --") @@ -314,3 +403,40 @@ func bytesRepeat(value byte, count int) []byte { } return result } + +func localStorageRecordKeyFixture(origin, key string) []byte { + result := append([]byte("_"+origin+"\x00"), chromiumStorageStringFixture(key)...) + return result +} + +func chromiumStorageStringFixture(value string) []byte { + runes := []rune(value) + latin1 := true + for _, value := range runes { + if value > 255 { + latin1 = false + break + } + } + if latin1 { + result := make([]byte, 1, len(runes)+1) + result[0] = 1 + for _, value := range runes { + result = append(result, byte(value)) + } + return result + } + units := utf16.Encode(runes) + result := make([]byte, 1+len(units)*2) + for index, value := range units { + binary.LittleEndian.PutUint16(result[1+index*2:], value) + } + return result +} + +func localStorageMetadataFixture(size uint64) []byte { + result := []byte{8, 1, 16} + buffer := make([]byte, binary.MaxVarintLen64) + written := binary.PutUvarint(buffer, size) + return append(result, buffer[:written]...) +} diff --git a/internal/browserimport/types.go b/internal/browserimport/types.go index 3d1f598..4f5db21 100644 --- a/internal/browserimport/types.go +++ b/internal/browserimport/types.go @@ -46,6 +46,61 @@ type Cookie struct { SameSite string `json:"same_site,omitempty"` } +type BookmarkDocument struct { + Roots []BookmarkRoot `json:"roots"` +} + +type BookmarkRoot struct { + Name string `json:"name"` + Children []BookmarkNode `json:"children"` +} + +type BookmarkNode struct { + Title string `json:"title"` + URL string `json:"url,omitempty"` + Children []BookmarkNode `json:"children"` + DateAdded *time.Time `json:"date_added,omitempty"` + DateLastUsed *time.Time `json:"date_last_used,omitempty"` +} + +type HistoryRecord struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` + VisitedAt time.Time `json:"visited_at"` + VisitCount int `json:"visit_count,omitempty"` +} + +const ( + StorageKindLocal = "local_storage" + MaxPortableStorageSize = 64 << 20 +) + +type StorageRecord struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Key string `json:"key"` + Value string `json:"value"` +} + +type StorageSite struct { + Origin string + Bytes int64 +} + +type Extension struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Source string `json:"source"` +} + +type ProfileData struct { + Cookies []Cookie + Storage []StorageRecord + Bookmarks *BookmarkDocument + History []HistoryRecord + Extensions []Extension +} + type Source struct { ID string `json:"id"` Kind string `json:"kind"` From a41e455056b41b5e8a443be284141f5af66a7195 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:38:22 -0400 Subject: [PATCH 2/3] Preserve local connector environment --- internal/connector/connector.go | 2 +- internal/connector/connector_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 38e7836..172d033 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -226,7 +226,7 @@ set kernelExecutable to "` + appleScriptString(executable) + `" set commandText to quoted form of kernelExecutable & " connector open " & quoted form of incomingURL set scriptPath to «event sysoexec» "/usr/bin/mktemp /tmp/kernel-connector.XXXXXX" set scriptFile to «event rdwropen» POSIX file scriptPath with «class perm» -«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile +«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile «event rdwrclos» scriptFile «event sysoexec» "/bin/chmod 700 " & quoted form of scriptPath «event sysoexec» "/usr/bin/open -a Terminal " & quoted form of scriptPath diff --git a/internal/connector/connector_test.go b/internal/connector/connector_test.go index f810cab..f626d3c 100644 --- a/internal/connector/connector_test.go +++ b/internal/connector/connector_test.go @@ -134,6 +134,8 @@ func TestMacOSAppleScriptShellQuotesURLAtRuntime(t *testing.T) { assert.Contains(t, script, `/usr/bin/open -a Terminal`) assert.Contains(t, script, `/bin/zsh -lic`) assert.Contains(t, script, `/usr/bin/mktemp /tmp/kernel-connector.XXXXXX`) + assert.Contains(t, script, `/bin/launchctl getenv`) + assert.Contains(t, script, `KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL`) assert.NotContains(t, script, `XXXXXX.command`) assert.Contains(t, script, `Kernel CLI was removed`) assert.Contains(t, script, `«event GURLGURL»`) From 60febc76fcd1bdbde990e44782555a21c854bbbf Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:41:29 -0400 Subject: [PATCH 3/3] Apply connector overrides after shell startup --- internal/connector/connector.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 172d033..c081050 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -223,10 +223,10 @@ func bundleIdentifier(contents []byte) (string, error) { func macOSAppleScript(executable string) string { return `on «event GURLGURL» incomingURL set kernelExecutable to "` + appleScriptString(executable) + `" -set commandText to quoted form of kernelExecutable & " connector open " & quoted form of incomingURL +set commandText to "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done; exec " & quoted form of kernelExecutable & " connector open " & quoted form of incomingURL set scriptPath to «event sysoexec» "/usr/bin/mktemp /tmp/kernel-connector.XXXXXX" set scriptFile to «event rdwropen» POSIX file scriptPath with «class perm» -«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile +«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile «event rdwrclos» scriptFile «event sysoexec» "/bin/chmod 700 " & quoted form of scriptPath «event sysoexec» "/usr/bin/open -a Terminal " & quoted form of scriptPath