diff --git a/cmd/browser_import_managed_auth.go b/cmd/browser_import_managed_auth.go index 9a7081b..9329752 100644 --- a/cmd/browser_import_managed_auth.go +++ b/cmd/browser_import_managed_auth.go @@ -19,6 +19,11 @@ type managedAuthCapacity struct { unlimited bool } +type storedExtensionCapacity struct { + remaining int + unlimited bool +} + type orgLimitsGetter interface { Get(context.Context, ...option.RequestOption) (*kernel.OrgLimits, error) } @@ -57,6 +62,40 @@ func decodeManagedAuthCapacity(raw string) (managedAuthCapacity, error) { return managedAuthCapacity{remaining: max(0, maxConnections-usedConnections)}, nil } +func loadStoredExtensionCapacity(ctx context.Context, limits orgLimitsGetter) (storedExtensionCapacity, error) { + orgLimits, err := limits.Get(ctx) + if err != nil { + return storedExtensionCapacity{}, err + } + return decodeStoredExtensionCapacity(orgLimits.RawJSON()) +} + +func decodeStoredExtensionCapacity(raw string) (storedExtensionCapacity, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(raw), &fields); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode organization limits: %w", err) + } + maxRaw, hasMax := fields["max_stored_extensions"] + usedRaw, hasUsed := fields["stored_extensions_used"] + if !hasMax || !hasUsed { + return storedExtensionCapacity{}, fmt.Errorf("Kernel API does not expose stored extension capacity") + } + if string(maxRaw) == "null" { + return storedExtensionCapacity{unlimited: true}, nil + } + var maximum, used int + if err := json.Unmarshal(maxRaw, &maximum); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode max stored extensions: %w", err) + } + if err := json.Unmarshal(usedRaw, &used); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode used stored extensions: %w", err) + } + if maximum < 0 || used < 0 { + return storedExtensionCapacity{}, fmt.Errorf("Kernel API returned invalid stored extension capacity") + } + return storedExtensionCapacity{remaining: max(0, maximum-used)}, nil +} + type managedAuthProvisioner interface { Provision(context.Context, string, []passwordmanager.Record) ([]string, error) Existing(context.Context, string, []passwordmanager.Candidate) (map[string]bool, error) diff --git a/cmd/browser_import_profile_data.go b/cmd/browser_import_profile_data.go new file mode 100644 index 0000000..7311dfe --- /dev/null +++ b/cmd/browser_import_profile_data.go @@ -0,0 +1,335 @@ +package cmd + +import ( + "context" + "fmt" + "sort" + "time" + + localbrowser "github.com/kernel/cli/internal/browserimport" + "github.com/pterm/pterm" +) + +const ( + bookmarksChoice = "Bookmarks" + historyChoice = "History" + storageChoice = "Local storage" + extensionsChoice = "Browser extensions" +) + +type localProfileDataSelection struct { + data localbrowser.ProfileData + bookmarkCount int + historyCount int + history bool + storage bool + storageSites []string + storageBytes int64 +} + +func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, profile localbrowser.Profile, since time.Time, includeHistory, nonInteractive, humanOutput bool) (localProfileDataSelection, error) { + bookmarks, bookmarkCount, bookmarkErr := localbrowser.ExportBookmarks(profile) + historyCount, historyErr := localbrowser.HistoryCount(ctx, profile, since) + storageSites, storageErr := localbrowser.LocalStorageSites(ctx, profile) + extensions, extensionErr := localbrowser.DiscoverExtensions(profile) + extensionCapacity := storedExtensionCapacity{unlimited: true} + extensionCapacityKnown := c.extensionCapacity == nil + if extensionErr == nil && len(extensions) > 0 && c.extensionCapacity != nil { + var capacityErr error + extensionCapacity, capacityErr = c.extensionCapacity(ctx) + extensionCapacityKnown = capacityErr == nil + if capacityErr != nil && humanOutput { + pterm.Warning.Printf("extension capacity could not be checked; Kernel will enforce it during import: %v\n", capacityErr) + } + } + + if humanOutput { + warnUnavailableBrowserData("bookmarks", bookmarkErr) + warnUnavailableBrowserData("history", historyErr) + warnUnavailableBrowserData("local storage", storageErr) + warnUnavailableBrowserData("extensions", extensionErr) + } + + selection := localProfileDataSelection{history: includeHistory} + options := make([]string, 0, 4) + defaults := make([]string, 0, 4) + labels := make(map[string]string, 4) + if bookmarkErr == nil && bookmarkCount > 0 { + labels[bookmarksChoice] = fmt.Sprintf("%s — %d", bookmarksChoice, bookmarkCount) + options = append(options, labels[bookmarksChoice]) + defaults = append(defaults, labels[bookmarksChoice]) + } + if historyErr == nil && historyCount > 0 { + labels[historyChoice] = fmt.Sprintf("%s — %d visits", historyChoice, historyCount) + options = append(options, labels[historyChoice]) + if includeHistory { + defaults = append(defaults, labels[historyChoice]) + } + } + if storageErr == nil && len(storageSites) > 0 { + total := storageSiteBytes(storageSites) + labels[storageChoice] = fmt.Sprintf("%s — %s across %d origins", storageChoice, formatBinaryBytes(total), len(storageSites)) + options = append(options, labels[storageChoice]) + defaults = append(defaults, labels[storageChoice]) + } + if extensionErr == nil && len(extensions) > 0 { + labels[extensionsChoice] = fmt.Sprintf("%s — %d detected (plan limit applies)", extensionsChoice, len(extensions)) + options = append(options, labels[extensionsChoice]) + defaults = append(defaults, labels[extensionsChoice]) + } + + chosen := defaults + var err error + if !nonInteractive && len(options) > 0 { + chosen, err = c.prompter.MultiSelect("browser data", "use Space to exclude a category", "Choose browser data to import", options, defaults) + if err != nil { + return localProfileDataSelection{}, err + } + } + for _, choice := range chosen { + switch choice { + case labels[bookmarksChoice]: + selection.data.Bookmarks = &bookmarks + selection.bookmarkCount = bookmarkCount + case labels[historyChoice]: + selection.history = true + selection.historyCount = historyCount + case labels[storageChoice]: + selection.storage = true + selection.storageBytes = storageSiteBytes(storageSites) + case labels[extensionsChoice]: + selection.data.Extensions = append(selection.data.Extensions, extensions...) + } + } + + if selection.storage { + selection.storageSites, err = c.chooseLocalStorageSites(storageSites, nonInteractive) + if err != nil { + return localProfileDataSelection{}, err + } + selection.storageBytes = selectedStorageSiteBytes(storageSites, selection.storageSites) + } + if len(selection.data.Extensions) > 0 { + selection.data.Extensions, err = c.chooseProfileExtensions(selection.data.Extensions, extensionCapacity, extensionCapacityKnown, nonInteractive) + if err != nil { + return localProfileDataSelection{}, err + } + } + return selection, nil +} + +func warnUnavailableBrowserData(category string, err error) { + if err != nil { + pterm.Warning.Printf("%s could not be read and will be skipped: %v\n", category, err) + } +} + +func (c ProfilesImportLocalCmd) confirmBrowserImport(targetName string, cookies cookieImportSelection, cookieSites []localbrowser.Site, profileData localProfileDataSelection, logins pendingManagedAuth) (bool, error) { + pterm.Println() + pterm.Printf("Ready to import into profile %q\n\n", targetName) + if cookies.all { + pterm.Printf(" All cookies — %d across %d websites\n", selectedCookieCount(cookieSites, cookies.sites), len(cookies.sites)) + } else { + pterm.Printf(" Cookies from %d selected websites\n", len(cookies.sites)) + } + if profileData.bookmarkCount > 0 { + pterm.Printf(" Bookmarks — %d\n", profileData.bookmarkCount) + } + if profileData.history { + pterm.Printf(" History — %d visits\n", profileData.historyCount) + } + if profileData.storage { + pterm.Printf(" Local storage — %s across %d origins\n", formatBinaryBytes(profileData.storageBytes), len(profileData.storageSites)) + } + if count := len(profileData.data.Extensions); count > 0 { + pterm.Printf(" Browser extensions — %d\n", count) + } + loginCount := 0 + for _, provider := range logins.providers { + loginCount += len(provider.candidates) + } + if loginCount > 0 { + pterm.Printf(" Managed Auth connections — %d\n", loginCount) + } + pterm.Println() + return c.prompter.ConfirmDefault("import browser data", "Proceed?", true) +} + +func selectedCookieCount(sites []localbrowser.Site, selected []string) int { + set := make(map[string]struct{}, len(selected)) + for _, site := range selected { + set[site] = struct{}{} + } + total := 0 + for _, site := range sites { + if _, ok := set[site.Domain]; ok { + total += site.CookieCount + } + } + return total +} + +func (c ProfilesImportLocalCmd) chooseLocalStorageSites(sites []localbrowser.StorageSite, nonInteractive bool) ([]string, error) { + all := make([]string, 0, len(sites)) + for _, site := range sites { + all = append(all, site.Origin) + } + if storageSiteBytes(sites) <= localbrowser.MaxPortableStorageSize { + return all, nil + } + if nonInteractive { + return nil, fmt.Errorf("browser local storage exceeds Kernel's 64 MiB import limit; run interactively to choose websites or disable local storage") + } + + sorted := append([]localbrowser.StorageSite(nil), sites...) + sort.SliceStable(sorted, func(left, right int) bool { + if sorted[left].Bytes == sorted[right].Bytes { + return sorted[left].Origin < sorted[right].Origin + } + return sorted[left].Bytes > sorted[right].Bytes + }) + labels := make([]string, 0, len(sorted)) + byLabel := make(map[string]string, len(sorted)) + defaults := make([]string, 0, len(sorted)) + var selectedBytes int64 + for index, site := range sorted { + label := fmt.Sprintf("%d %s — %s", index+1, compactField(site.Origin, 46), formatBinaryBytes(site.Bytes)) + labels = append(labels, label) + byLabel[label] = site.Origin + if selectedBytes+site.Bytes <= localbrowser.MaxPortableStorageSize { + defaults = append(defaults, label) + selectedBytes += site.Bytes + } + } + chosen, err := c.prompter.MultiSelect("local storage", "deselect websites until the selection fits", "Choose local website data to import (64 MiB maximum)", labels, defaults) + if err != nil { + return nil, err + } + result := make([]string, 0, len(chosen)) + for _, label := range chosen { + result = append(result, byLabel[label]) + } + return result, nil +} + +func (c ProfilesImportLocalCmd) chooseProfileExtensions(extensions []localbrowser.Extension, capacity storedExtensionCapacity, capacityKnown, nonInteractive bool) ([]localbrowser.Extension, error) { + maximum := min(20, len(extensions)) + if capacityKnown && !capacity.unlimited { + maximum = min(maximum, capacity.remaining) + } + if nonInteractive { + if len(extensions) > maximum { + return nil, fmt.Errorf("%d extensions were selected, but only %d can be added under the profile and plan limits; run interactively to choose extensions", len(extensions), maximum) + } + return extensions, nil + } + labels := make([]string, 0, len(extensions)) + byLabel := make(map[string]localbrowser.Extension, len(extensions)) + for index, extension := range extensions { + name := extension.Name + if name == "" { + name = "Unnamed extension" + } + label := fmt.Sprintf("%d %s · %s", index+1, compactField(name, 42), extension.ID[len(extension.ID)-6:]) + labels = append(labels, label) + byLabel[label] = extension + } + defaults := labels[:maximum] + for { + prompt := fmt.Sprintf("Choose extensions to reinstall (select up to %d)", maximum) + chosen, err := c.prompter.MultiSelect("browser extensions", "your Kernel plan controls stored extension capacity", prompt, labels, defaults) + if err != nil { + return nil, err + } + if len(chosen) > maximum { + pterm.Warning.Printf("Select at most %d extension%s\n", maximum, pluralSuffix(maximum)) + defaults = chosen + continue + } + result := make([]localbrowser.Extension, 0, len(chosen)) + for _, label := range chosen { + result = append(result, byLabel[label]) + } + return result, nil + } +} + +func buildSelectedProfileData(ctx context.Context, profile localbrowser.Profile, selection localProfileDataSelection, cookies []localbrowser.Cookie, since time.Time) (localbrowser.ProfileData, map[string]int, error) { + data := selection.data + data.Cookies = cookies + counts := make(map[string]int, 5) + if len(cookies) > 0 { + counts["cookies"] = len(cookies) + } + if data.Bookmarks != nil { + counts["bookmarks"] = selection.bookmarkCount + } + if selection.history { + history, err := localbrowser.ExportHistory(ctx, profile, since) + if err != nil { + return localbrowser.ProfileData{}, nil, err + } + data.History = history + counts["history"] = len(history) + } + if selection.storage { + storage, err := localbrowser.ExportLocalStorage(ctx, profile, selection.storageSites) + if err != nil { + return localbrowser.ProfileData{}, nil, err + } + data.Storage = storage + counts["storage"] = len(storage) + } + if len(data.Extensions) > 0 { + counts["extensions"] = len(data.Extensions) + } + return data, counts, nil +} + +func selectedProfileCategories(counts map[string]int) []string { + order := []string{"cookies", "storage", "bookmarks", "history", "extensions"} + result := make([]string, 0, len(counts)) + for _, category := range order { + if count, selected := counts[category]; selected && count > 0 { + result = append(result, category) + } + } + return result +} + +func storageSiteBytes(sites []localbrowser.StorageSite) int64 { + var total int64 + for _, site := range sites { + total += site.Bytes + } + return total +} + +func selectedStorageSiteBytes(sites []localbrowser.StorageSite, selected []string) int64 { + set := make(map[string]struct{}, len(selected)) + for _, origin := range selected { + set[origin] = struct{}{} + } + var total int64 + for _, site := range sites { + if _, ok := set[site.Origin]; ok { + total += site.Bytes + } + } + return total +} + +func importedStorageOriginCount(records []localbrowser.StorageRecord) int { + origins := make(map[string]struct{}) + for _, record := range records { + origins[record.Origin] = struct{}{} + } + return len(origins) +} + +func formatBinaryBytes(bytes int64) string { + if bytes < 1<<20 { + return fmt.Sprintf("%.1f KiB", float64(bytes)/(1<<10)) + } + return fmt.Sprintf("%.1f MiB", float64(bytes)/(1<<20)) +} diff --git a/cmd/browser_import_profile_data_test.go b/cmd/browser_import_profile_data_test.go new file mode 100644 index 0000000..cf58bb6 --- /dev/null +++ b/cmd/browser_import_profile_data_test.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "strings" + "testing" + + localbrowser "github.com/kernel/cli/internal/browserimport" + "github.com/stretchr/testify/require" +) + +func TestLocalStorageSelectionUsesAllSitesWithinLimit(t *testing.T) { + sites := []localbrowser.StorageSite{ + {Origin: "https://example.com", Bytes: 1024}, + {Origin: "https://other.example", Bytes: 2048}, + } + + selected, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true) + require.NoError(t, err) + require.Equal(t, []string{"https://example.com", "https://other.example"}, selected) +} + +func TestLocalStorageSelectionRequiresReviewWhenOverLimit(t *testing.T) { + sites := []localbrowser.StorageSite{{Origin: "https://large.example", Bytes: localbrowser.MaxPortableStorageSize + 1}} + + _, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true) + require.ErrorContains(t, err, "run interactively to choose websites") +} + +func TestNonInteractiveExtensionSelectionHonorsCapacity(t *testing.T) { + extensions := []localbrowser.Extension{ + {ID: strings.Repeat("a", 32), Source: "chrome_web_store"}, + {ID: strings.Repeat("b", 32), Source: "chrome_web_store"}, + } + + _, err := (ProfilesImportLocalCmd{}).chooseProfileExtensions(extensions, storedExtensionCapacity{remaining: 1}, true, true) + require.ErrorContains(t, err, "only 1 can be added") +} + +func TestSelectedProfileCategoriesUsePortableApplyOrder(t *testing.T) { + categories := selectedProfileCategories(map[string]int{ + "extensions": 1, + "bookmarks": 2, + "cookies": 3, + "history": 4, + "storage": 5, + }) + + require.Equal(t, []string{"cookies", "storage", "bookmarks", "history", "extensions"}, categories) +} + +func TestProfilesImportLocalDefaultsHistoryOn(t *testing.T) { + flag := profilesImportLocalCmd.Flags().Lookup("history") + require.NotNil(t, flag) + require.Equal(t, "true", flag.DefValue) +} diff --git a/cmd/connector.go b/cmd/connector.go index 34d1621..cc66d60 100644 --- a/cmd/connector.go +++ b/cmd/connector.go @@ -59,6 +59,7 @@ func runConnectorOpen(cmd *cobra.Command, args []string) error { Version: metadata.Version, WaitTimeout: 30 * time.Minute, DashboardLaunch: true, + ImportHistory: true, } project, err := validateConnectorProject(cmd, input.ProjectID) if err != nil { diff --git a/cmd/profiles_import_local.go b/cmd/profiles_import_local.go index b06c013..e3493c4 100644 --- a/cmd/profiles_import_local.go +++ b/cmd/profiles_import_local.go @@ -41,6 +41,7 @@ type ProfilesImportLocalInput struct { PasswordManager string InstallAgentSkills bool DashboardLaunch bool + ImportHistory bool Project *kernel.Project } @@ -51,6 +52,7 @@ type ProfilesImportLocalCmd struct { providers func() []passwordmanager.Provider provisioner managedAuthProvisioner managedAuthCapacity func(context.Context) (managedAuthCapacity, error) + extensionCapacity func(context.Context) (storedExtensionCapacity, error) } type pendingManagedAuth struct { @@ -185,6 +187,11 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI if len(cookieSelection.sites) == 0 { return fmt.Errorf("select at least one website") } + since := c.now().AddDate(0, 0, -in.Days) + profileDataSelection, err := c.chooseLocalProfileData(ctx, profile, since, in.ImportHistory, nonInteractive, humanOutput) + if err != nil { + return err + } pendingLogins := pendingManagedAuth{} if managedAuthImportRequested(in.PasswordManager, nonInteractive) { phaseStarted = time.Now() @@ -196,6 +203,16 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI return err } } + if !nonInteractive { + proceed, err := c.confirmBrowserImport(targetName, cookieSelection, cookieSites, profileDataSelection, pendingLogins) + if err != nil { + return err + } + if !proceed { + pterm.Info.Println("Browser import canceled; no Kernel resources were changed") + return nil + } + } if humanOutput { pterm.Println() if cookieSelection.all { @@ -224,7 +241,12 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI version = "dev" } phaseStarted = time.Now() - bundle, err := localbrowser.BuildCookieBundle(ctx, profile, targetName, version, cookies) + profileData, itemCounts, err := buildSelectedProfileData(ctx, profile, profileDataSelection, cookies, since) + if err != nil { + return err + } + categories := selectedProfileCategories(itemCounts) + bundle, err := localbrowser.BuildProfileBundle(ctx, profile, targetName, version, profileData) timings["bundle"] = time.Since(phaseStarted) if err != nil { return err @@ -247,13 +269,13 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI } inventory := localbrowser.Inventory{Sources: []localbrowser.Source{{ ID: profile.ID, Kind: "browser", Name: profile.DisplayName(), Browser: profile.Browser.ID, - DataTypes: []string{"cookies"}, ItemCounts: map[string]int{"cookies": len(cookies)}, + DataTypes: categories, ItemCounts: itemCounts, }}} status, err := client.SubmitInventory(ctx, created.ID, created.HelperToken, inventory) if err != nil { return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err) } - selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: []string{"cookies"}}}, CredentialSources: make([]string, 0)} + selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: categories}}, CredentialSources: make([]string, 0)} status, err = client.SubmitSelection(ctx, created.ID, selection) if err != nil { return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err) @@ -275,6 +297,18 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI profileID := status.Applied.Profiles[0].ProfileID if humanOutput { pterm.Success.Printf("Imported %d cookies from %d websites\n", len(cookies), importedCookieSites) + if count := itemCounts["bookmarks"]; count > 0 { + pterm.Success.Printf("Imported %d bookmarks\n", count) + } + if count := itemCounts["history"]; count > 0 { + pterm.Success.Printf("Imported %d history entries\n", count) + } + if count := itemCounts["storage"]; count > 0 { + pterm.Success.Printf("Imported %d local storage keys from %d origins\n", count, importedStorageOriginCount(profileData.Storage)) + } + if count := itemCounts["extensions"]; count > 0 { + pterm.Success.Printf("Installed %d browser extensions\n", count) + } } connectionIDs := make([]string, 0) approvedLogins := make([]passwordmanager.Record, 0) @@ -325,7 +359,7 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI } } if in.Output == "json" { - data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": len(cookies), "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", " ") + data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": itemCounts["cookies"], "browser_data_imported": itemCounts, "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", " ") if err != nil { return err } @@ -1405,8 +1439,8 @@ var cuidLikeProfileName = regexp.MustCompile(`^[a-z0-9]{24}$`) var profilesImportLocalCmd = &cobra.Command{ Use: "import-local", - Short: "Import cookies from a local browser", - Long: "Import all cookies or selected websites from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.", + Short: "Import a local browser profile", + Long: "Import cookies and selected portable data from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.", Args: cobra.NoArgs, RunE: runProfilesImportLocal, } @@ -1427,6 +1461,7 @@ func init() { profilesImportLocalCmd.Flags().Int("days", 30, "Rank websites used during the last number of days (1-90)") profilesImportLocalCmd.Flags().Duration("wait-timeout", 30*time.Minute, "Maximum time to wait for the import to complete") profilesImportLocalCmd.Flags().BoolP("yes", "y", false, "Import all cookies and use unambiguous defaults without prompting") + profilesImportLocalCmd.Flags().Bool("history", true, "Include browsing history from the selected --days window") profilesImportLocalCmd.Flags().String("password-manager", "", "Password managers to search: bitwarden, 1password, both comma-separated, all, or none") profilesImportLocalCmd.Flags().Bool("install-agent-skills", false, "Install the Kernel Managed Auth skill into detected agent directories") addJSONOutputFlag(profilesImportLocalCmd) @@ -1442,12 +1477,13 @@ func runProfilesImportLocal(cmd *cobra.Command, _ []string) error { skipConfirm, _ := cmd.Flags().GetBool("yes") passwordManager, _ := cmd.Flags().GetString("password-manager") installAgentSkills, _ := cmd.Flags().GetBool("install-agent-skills") + importHistory, _ := cmd.Flags().GetBool("history") output, _ := cmd.Flags().GetString("output") project, _ := cmd.Flags().GetString("project") return runProfilesImportLocalWithInput(cmd, ProfilesImportLocalInput{ BrowserProfile: browserProfile, ProfileName: profileName, Sites: sites, Days: days, SkipConfirm: skipConfirm, Output: output, ProjectID: resolveProjectSelection(project), Version: metadata.Version, - WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, + WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, ImportHistory: importHistory, }) } @@ -1484,6 +1520,9 @@ func runProfilesImportLocalWithInput(cmd *cobra.Command, input ProfilesImportLoc providers: passwordmanager.Detect, provisioner: kernelManagedAuthProvisioner{credentials: &credentials, connections: &connections}, managedAuthCapacity: func(ctx context.Context) (managedAuthCapacity, error) { return loadManagedAuthCapacity(ctx, &limits) }, + extensionCapacity: func(ctx context.Context) (storedExtensionCapacity, error) { + return loadStoredExtensionCapacity(ctx, &limits) + }, } input.ProjectID = project.ID if input.Version == "" { diff --git a/cmd/profiles_import_local_test.go b/cmd/profiles_import_local_test.go index 9392f91..0a9bd3b 100644 --- a/cmd/profiles_import_local_test.go +++ b/cmd/profiles_import_local_test.go @@ -384,6 +384,19 @@ func TestDecodeManagedAuthCapacity(t *testing.T) { }) } +func TestDecodeStoredExtensionCapacity(t *testing.T) { + capacity, err := decodeStoredExtensionCapacity(`{"max_stored_extensions":5,"stored_extensions_used":3}`) + require.NoError(t, err) + assert.Equal(t, storedExtensionCapacity{remaining: 2}, capacity) + + capacity, err = decodeStoredExtensionCapacity(`{"max_stored_extensions":null,"stored_extensions_used":12}`) + require.NoError(t, err) + assert.Equal(t, storedExtensionCapacity{unlimited: true}, capacity) + + _, err = decodeStoredExtensionCapacity(`{"max_auth_connections":10}`) + require.ErrorContains(t, err, "does not expose stored extension capacity") +} + func TestChooseManagedAuthLoginsRejectsExplicitBatchAboveRemainingConnections(t *testing.T) { command := managedAuthTestCommand(func() []passwordmanager.Provider { return []passwordmanager.Provider{fakePasswordManager{candidates: []passwordmanager.Candidate{ diff --git a/go.mod b/go.mod index 3ac5d5c..88a084c 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.11.1 + github.com/syndtr/goleveldb v1.0.0 github.com/zalando/go-keyring v0.2.6 golang.org/x/crypto v0.52.0 golang.org/x/net v0.54.0 @@ -40,6 +41,7 @@ require ( github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/gookit/color v1.5.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect diff --git a/go.sum b/go.sum index 89114c2..f82040e 100644 --- a/go.sum +++ b/go.sum @@ -50,16 +50,22 @@ github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -95,6 +101,11 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -128,6 +139,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -153,6 +166,7 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -161,11 +175,13 @@ golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -201,7 +217,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/browserimport/bundle.go b/internal/browserimport/bundle.go index eeb24c9..11981ca 100644 --- a/internal/browserimport/bundle.go +++ b/internal/browserimport/bundle.go @@ -6,12 +6,17 @@ import ( "context" "encoding/json" "fmt" - "sort" "github.com/klauspost/compress/zstd" ) -const maxBundleBytes = 128 << 20 +const ( + maxBundleBytes = 128 << 20 + maxPortableFileBytes = 64 << 20 + maxPortableRecordBytes = 1 << 20 + maxPortableRecords = 100_000 + maxPortableDocumentBytes = 16 << 20 +) type Manifest struct { Version int `json:"version"` @@ -33,37 +38,109 @@ type BundleProfile struct { } type ProfileFiles struct { - Cookies string `json:"cookies,omitempty"` + Cookies string `json:"cookies,omitempty"` + Storage string `json:"storage,omitempty"` + Bookmarks string `json:"bookmarks,omitempty"` + History string `json:"history,omitempty"` + Extensions string `json:"extensions,omitempty"` +} + +type bundleFile struct { + path string + data []byte } -func BuildCookieBundle(ctx context.Context, profile Profile, targetName, version string, cookies []Cookie) ([]byte, error) { - if len(cookies) == 0 { - return nil, fmt.Errorf("no cookies were selected") +func BuildProfileBundle(ctx context.Context, profile Profile, targetName, version string, data ProfileData) ([]byte, error) { + if len(data.Cookies) == 0 && len(data.Storage) == 0 && data.Bookmarks == nil && len(data.History) == 0 && len(data.Extensions) == 0 { + return nil, fmt.Errorf("no browser data was selected") + } + files := ProfileFiles{} + payloads := make([]bundleFile, 0, 5) + addJSON := func(path, label string, value any) error { + encoded, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("encode browser %s: %w", label, err) + } + if len(encoded) > maxPortableDocumentBytes { + return fmt.Errorf("browser %s exceeds the 16 MiB import limit", label) + } + payloads = append(payloads, bundleFile{path: path, data: encoded}) + return nil + } + if len(data.Cookies) > 0 { + files.Cookies = "profiles/" + profile.ID + "/cookies.jsonl" + encoded, err := encodeJSONL("cookies", data.Cookies) + if err != nil { + return nil, fmt.Errorf("encode browser cookies: %w", err) + } + payloads = append(payloads, bundleFile{path: files.Cookies, data: encoded}) + } + if len(data.Storage) > 0 { + files.Storage = "profiles/" + profile.ID + "/storage.jsonl" + encoded, err := encodeJSONL("local storage", data.Storage) + if err != nil { + return nil, fmt.Errorf("encode browser local storage: %w", err) + } + payloads = append(payloads, bundleFile{path: files.Storage, data: encoded}) + } + if data.Bookmarks != nil { + files.Bookmarks = "profiles/" + profile.ID + "/bookmarks.json" + if err := addJSON(files.Bookmarks, "bookmarks", data.Bookmarks); err != nil { + return nil, err + } + } + if len(data.History) > 0 { + files.History = "profiles/" + profile.ID + "/history.jsonl" + encoded, err := encodeJSONL("history", data.History) + if err != nil { + return nil, fmt.Errorf("encode browser history: %w", err) + } + payloads = append(payloads, bundleFile{path: files.History, data: encoded}) + } + if len(data.Extensions) > 0 { + files.Extensions = "profiles/" + profile.ID + "/extensions.json" + if err := addJSON(files.Extensions, "extensions", data.Extensions); err != nil { + return nil, err + } } - cookiePath := "profiles/" + profile.ID + "/cookies.jsonl" manifest := Manifest{ Version: BundleVersion, Source: BundleSource{OS: "macos", HelperVersion: version}, Profiles: []BundleProfile{{ ID: profile.ID, Browser: profile.Browser.ID, SourceName: profile.DisplayName(), TargetName: targetName, - Files: ProfileFiles{Cookies: cookiePath}, + Files: files, }}, } manifestData, err := json.Marshal(manifest) if err != nil { return nil, fmt.Errorf("encode import manifest: %w", err) } - var cookieData bytes.Buffer - encoder := json.NewEncoder(&cookieData) - for _, cookie := range cookies { - if err := encoder.Encode(cookie); err != nil { - return nil, fmt.Errorf("encode browser cookie: %w", err) + return encodeBundle(ctx, manifestData, payloads) +} + +func encodeJSONL[T any](label string, records []T) ([]byte, error) { + if len(records) > maxPortableRecords { + return nil, fmt.Errorf("browser %s exceeds the %d record import limit", label, maxPortableRecords) + } + var output bytes.Buffer + for _, record := range records { + encoded, err := json.Marshal(record) + if err != nil { + return nil, err + } + if len(encoded)+1 > maxPortableRecordBytes { + return nil, fmt.Errorf("one browser %s record exceeds the 1 MiB import limit", label) } + if output.Len()+len(encoded)+1 > maxPortableFileBytes { + return nil, fmt.Errorf("browser %s exceeds the 64 MiB import limit", label) + } + output.Write(encoded) + output.WriteByte('\n') } - return encodeBundle(ctx, manifestData, map[string][]byte{cookiePath: cookieData.Bytes()}) + return output.Bytes(), nil } -func encodeBundle(ctx context.Context, manifest []byte, files map[string][]byte) ([]byte, error) { +func encodeBundle(ctx context.Context, manifest []byte, files []bundleFile) ([]byte, error) { var output bytes.Buffer zstdWriter, err := zstd.NewWriter(&output, zstd.WithEncoderConcurrency(1)) if err != nil { @@ -83,13 +160,8 @@ func encodeBundle(ctx context.Context, manifest []byte, files map[string][]byte) if err := write("manifest.json", manifest); err != nil { return nil, err } - paths := make([]string, 0, len(files)) - for path := range files { - paths = append(paths, path) - } - sort.Strings(paths) - for _, path := range paths { - if err := write(path, files[path]); err != nil { + for _, file := range files { + if err := write(file.path, file.data); err != nil { return nil, err } } diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index 72bdb6f..dc8b733 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -3,44 +3,57 @@ package browserimport import ( "archive/tar" "bytes" - "context" "encoding/json" "io" + "strings" "testing" "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestBuildCookieBundleMatchesServerContract(t *testing.T) { - profile := Profile{ID: "chrome-default-1234", Name: "Personal", Browser: Browser{ID: "chrome", Name: "Google Chrome"}} - bundle, err := BuildCookieBundle(context.Background(), profile, "my-browser", "test", []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret", Secure: true}}) +func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { + profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} + bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ + Cookies: []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret"}}, + Storage: []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, + Bookmarks: &BookmarkDocument{Roots: []BookmarkRoot{{Name: "bookmark_bar", Children: []BookmarkNode{{Title: "Kernel", URL: "https://onkernel.com"}}}}}, + Extensions: []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Source: "chrome_web_store"}}, + }) require.NoError(t, err) decoder, err := zstd.NewReader(bytes.NewReader(bundle)) require.NoError(t, err) defer decoder.Close() reader := tar.NewReader(decoder) - header, err := reader.Next() - require.NoError(t, err) - assert.Equal(t, "manifest.json", header.Name) - manifestData, err := io.ReadAll(reader) - require.NoError(t, err) + files := make(map[string][]byte) + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + files[header.Name], err = io.ReadAll(reader) + require.NoError(t, err) + } var manifest Manifest - require.NoError(t, json.Unmarshal(manifestData, &manifest)) - assert.Equal(t, BundleVersion, manifest.Version) - assert.Equal(t, "my-browser", manifest.Profiles[0].TargetName) - - header, err = reader.Next() - require.NoError(t, err) - assert.Equal(t, "profiles/chrome-default-1234/cookies.jsonl", header.Name) - var cookie Cookie - require.NoError(t, json.NewDecoder(reader).Decode(&cookie)) - assert.Equal(t, "secret", cookie.Value) + require.NoError(t, json.Unmarshal(files["manifest.json"], &manifest)) + require.NotEmpty(t, manifest.Profiles[0].Files.Cookies) + require.NotEmpty(t, manifest.Profiles[0].Files.Storage) + require.NotEmpty(t, manifest.Profiles[0].Files.Bookmarks) + require.Empty(t, manifest.Profiles[0].Files.History) + require.NotEmpty(t, manifest.Profiles[0].Files.Extensions) } -func TestBuildCookieBundleRequiresCookies(t *testing.T) { - _, err := BuildCookieBundle(context.Background(), Profile{}, "profile", "test", nil) - assert.EqualError(t, err, "no cookies were selected") +func TestEncodeJSONLEnforcesPortableRecordLimits(t *testing.T) { + _, err := encodeJSONL("history", make([]HistoryRecord, maxPortableRecords+1)) + require.ErrorContains(t, err, "100000 record import limit") + + _, err = encodeJSONL("local storage", []StorageRecord{{ + Origin: "https://example.com", + Kind: StorageKindLocal, + Key: "large", + Value: strings.Repeat("x", maxPortableRecordBytes), + }}) + require.ErrorContains(t, err, "1 MiB import limit") } diff --git a/internal/browserimport/chromium.go b/internal/browserimport/chromium.go index bffdaa6..cda6ecb 100644 --- a/internal/browserimport/chromium.go +++ b/internal/browserimport/chromium.go @@ -7,6 +7,7 @@ import ( "crypto/cipher" "crypto/sha1" "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -18,9 +19,13 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" "time" + "unicode/utf16" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" "golang.org/x/crypto/pbkdf2" "golang.org/x/net/publicsuffix" ) @@ -29,9 +34,14 @@ const ( maxDocumentBytes = 16 << 20 maxSQLiteOutput = 64 << 20 maxSQLiteBytes = 2 << 30 + maxStorageBytes = MaxPortableStorageSize + maxStorageSource = 512 << 20 + maxStorageRecord = 1 << 20 + maxStorageCount = 100_000 ) var profileIDCharacters = regexp.MustCompile(`[^a-zA-Z0-9._-]+`) +var chromeWebStoreExtensionID = regexp.MustCompile(`^[a-p]{32}$`) func DiscoverMacOSProfiles(home string) ([]Profile, error) { browsers := []Browser{ @@ -244,6 +254,373 @@ func ExportCookies(ctx context.Context, profile Profile, selectedSites []string) return cookies, nil } +func ExportBookmarks(profile Profile) (BookmarkDocument, int, error) { + payload, err := readFileBounded(filepath.Join(profile.Path, "Bookmarks"), maxDocumentBytes) + if errors.Is(err, os.ErrNotExist) { + return BookmarkDocument{}, 0, nil + } + if err != nil { + return BookmarkDocument{}, 0, fmt.Errorf("read browser bookmarks: %w", err) + } + var source struct { + Roots map[string]chromiumBookmarkNode `json:"roots"` + } + if err := json.Unmarshal(payload, &source); err != nil { + return BookmarkDocument{}, 0, fmt.Errorf("decode browser bookmarks: %w", err) + } + document := BookmarkDocument{Roots: make([]BookmarkRoot, 0, 3)} + count := 0 + for _, name := range []string{"bookmark_bar", "other", "synced"} { + root, ok := source.Roots[name] + if !ok { + continue + } + portableName := name + if name == "synced" { + portableName = "mobile" + } + document.Roots = append(document.Roots, BookmarkRoot{Name: portableName, Children: portableBookmarkNodes(root.Children, &count)}) + } + return document, count, nil +} + +type chromiumBookmarkNode struct { + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + DateAdded string `json:"date_added"` + DateLastUsed string `json:"date_last_used"` + Children []chromiumBookmarkNode `json:"children"` +} + +func portableBookmarkNodes(nodes []chromiumBookmarkNode, count *int) []BookmarkNode { + result := make([]BookmarkNode, 0, len(nodes)) + for _, node := range nodes { + if node.Type == "url" { + parsed, err := url.Parse(node.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + continue + } + *count++ + result = append(result, BookmarkNode{Title: node.Name, URL: node.URL, DateAdded: chromiumTimestamp(node.DateAdded), DateLastUsed: chromiumTimestamp(node.DateLastUsed)}) + continue + } + if node.Type != "folder" { + continue + } + children := portableBookmarkNodes(node.Children, count) + *count++ + result = append(result, BookmarkNode{Title: node.Name, Children: children, DateAdded: chromiumTimestamp(node.DateAdded)}) + } + return result +} + +func chromiumTimestamp(value string) *time.Time { + microseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || microseconds <= 0 { + return nil + } + parsed := chromiumTime(microseconds) + return &parsed +} + +func ExportHistory(ctx context.Context, profile Profile, since time.Time) ([]HistoryRecord, error) { + const chromeEpochMicros = int64(11_644_473_600_000_000) + cutoff := since.UnixMicro() + chromeEpochMicros + query := fmt.Sprintf(`SELECT urls.url, urls.title, MAX(visits.visit_time) AS last_visit_time, COUNT(*) AS visit_count FROM visits JOIN urls ON urls.id = visits.url WHERE visits.visit_time >= %d AND (urls.url LIKE 'http://%%' OR urls.url LIKE 'https://%%') GROUP BY urls.id ORDER BY last_visit_time DESC LIMIT 100001`, cutoff) + payload, err := sqliteSnapshotJSON(ctx, filepath.Join(profile.Path, "History"), query) + if err != nil { + return nil, fmt.Errorf("read browser history: %w", err) + } + var rows []struct { + URL string `json:"url"` + Title string `json:"title"` + VisitedAt int64 `json:"last_visit_time"` + Count int `json:"visit_count"` + } + if len(bytes.TrimSpace(payload)) != 0 { + if err := json.Unmarshal(payload, &rows); err != nil { + return nil, fmt.Errorf("decode browser history: %w", err) + } + } + if len(rows) > 100000 { + return nil, fmt.Errorf("browser history exceeds the 100000-record import limit; use fewer days") + } + records := make([]HistoryRecord, 0, len(rows)) + for _, row := range rows { + parsed, err := url.Parse(row.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + continue + } + records = append(records, HistoryRecord{URL: row.URL, Title: row.Title, VisitedAt: chromiumTime(row.VisitedAt), VisitCount: max(1, row.Count)}) + } + return records, nil +} + +func HistoryCount(ctx context.Context, profile Profile, since time.Time) (int, error) { + const chromeEpochMicros = int64(11_644_473_600_000_000) + cutoff := since.UnixMicro() + chromeEpochMicros + query := fmt.Sprintf(`SELECT COUNT(*) AS count FROM visits JOIN urls ON urls.id = visits.url WHERE visits.visit_time >= %d AND (urls.url LIKE 'http://%%' OR urls.url LIKE 'https://%%')`, cutoff) + payload, err := sqliteSnapshotJSON(ctx, filepath.Join(profile.Path, "History"), query) + if err != nil { + return 0, fmt.Errorf("count browser history: %w", err) + } + var rows []struct { + Count int `json:"count"` + } + if err := json.Unmarshal(payload, &rows); err != nil { + return 0, fmt.Errorf("decode browser history count: %w", err) + } + if len(rows) != 1 || rows[0].Count < 0 { + return 0, fmt.Errorf("browser history count is invalid") + } + return rows[0].Count, nil +} + +func LocalStorageSites(ctx context.Context, profile Profile) ([]StorageSite, error) { + database, cleanup, err := levelDBSnapshot(ctx, filepath.Join(profile.Path, "Local Storage", "leveldb")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("snapshot browser local storage: %w", err) + } + defer cleanup() + + db, err := leveldb.OpenFile(database, nil) + if err != nil { + return nil, fmt.Errorf("open browser local storage: %w", err) + } + defer db.Close() + + metadataBytes := make(map[string]int64) + recordBytes := make(map[string]int64) + iterator := db.NewIterator(nil, nil) + defer iterator.Release() + for iterator.Next() { + key := iterator.Key() + if bytes.HasPrefix(key, []byte("META:")) { + origin, ok := portableStorageOrigin(string(key[len("META:"):])) + if !ok { + continue + } + size, err := localStorageMetadataSize(iterator.Value()) + if err != nil { + return nil, fmt.Errorf("decode local storage metadata for %s: %w", origin, err) + } + metadataBytes[origin] = size + continue + } + if len(key) < 2 || key[0] != '_' { + continue + } + separator := bytes.IndexByte(key[1:], 0) + if separator < 1 { + continue + } + origin, ok := portableStorageOrigin(string(key[1 : 1+separator])) + if ok { + recordBytes[origin] += int64(len(key[2+separator:]) + len(iterator.Value())) + } + } + if err := iterator.Error(); err != nil { + return nil, fmt.Errorf("read browser local storage metadata: %w", err) + } + sites := make([]StorageSite, 0, len(metadataBytes)+len(recordBytes)) + for origin, bytes := range recordBytes { + if metadataBytes[origin] > bytes { + bytes = metadataBytes[origin] + } + sites = append(sites, StorageSite{Origin: origin, Bytes: bytes}) + delete(metadataBytes, origin) + } + for origin, bytes := range metadataBytes { + sites = append(sites, StorageSite{Origin: origin, Bytes: bytes}) + } + sort.Slice(sites, func(left, right int) bool { return sites[left].Origin < sites[right].Origin }) + return sites, nil +} + +func ExportLocalStorage(ctx context.Context, profile Profile, selectedOrigins []string) ([]StorageRecord, error) { + database, cleanup, err := levelDBSnapshot(ctx, filepath.Join(profile.Path, "Local Storage", "leveldb")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("snapshot browser local storage: %w", err) + } + defer cleanup() + + db, err := leveldb.OpenFile(database, nil) + if err != nil { + return nil, fmt.Errorf("open browser local storage: %w", err) + } + defer db.Close() + + selected := make(map[string]struct{}, len(selectedOrigins)) + for _, origin := range selectedOrigins { + selected[origin] = struct{}{} + } + records := make([]StorageRecord, 0) + encodedBytes := 0 + iterator := db.NewIterator(util.BytesPrefix([]byte("_")), nil) + defer iterator.Release() + for iterator.Next() { + key := iterator.Key()[1:] + separator := bytes.IndexByte(key, 0) + if separator < 1 || separator == len(key)-1 { + continue + } + origin, ok := portableStorageOrigin(string(key[:separator])) + if !ok { + continue + } + if len(selected) > 0 { + if _, ok := selected[origin]; !ok { + continue + } + } + scriptKey, err := decodeChromiumStorageString(key[separator+1:]) + if err != nil || scriptKey == "" { + continue + } + value, err := decodeChromiumStorageString(iterator.Value()) + if err != nil { + continue + } + record := StorageRecord{Origin: origin, Kind: StorageKindLocal, Key: scriptKey, Value: value} + encoded, err := json.Marshal(record) + if err != nil { + return nil, fmt.Errorf("encode browser local storage: %w", err) + } + if len(encoded)+1 > maxStorageRecord { + return nil, fmt.Errorf("local storage key %q for %s exceeds the 1 MiB record limit", scriptKey, origin) + } + encodedBytes += len(encoded) + 1 + if encodedBytes > maxStorageBytes { + return nil, fmt.Errorf("browser local storage exceeds the 64 MiB import limit; choose fewer websites") + } + records = append(records, record) + if len(records) > maxStorageCount { + return nil, fmt.Errorf("browser local storage exceeds the 100000-record import limit; choose fewer websites") + } + } + if err := iterator.Error(); err != nil { + return nil, fmt.Errorf("read browser local storage: %w", err) + } + sort.Slice(records, func(left, right int) bool { + if records[left].Origin == records[right].Origin { + return records[left].Key < records[right].Key + } + return records[left].Origin < records[right].Origin + }) + return records, nil +} + +func portableStorageOrigin(raw string) (string, bool) { + parsed, err := url.Parse(raw) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.User != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", false + } + origin := parsed.Scheme + "://" + parsed.Host + if raw != origin && raw != origin+"/" { + return "", false + } + return origin, true +} + +func decodeChromiumStorageString(raw []byte) (string, error) { + if len(raw) == 0 { + return "", fmt.Errorf("string is missing its encoding prefix") + } + switch raw[0] { + case 0: + if len(raw[1:])%2 != 0 { + return "", fmt.Errorf("UTF-16 string has an odd byte length") + } + units := make([]uint16, len(raw[1:])/2) + for index := range units { + units[index] = binary.LittleEndian.Uint16(raw[1+index*2:]) + } + return string(utf16.Decode(units)), nil + case 1: + runes := make([]rune, len(raw)-1) + for index, value := range raw[1:] { + runes[index] = rune(value) + } + return string(runes), nil + default: + return "", fmt.Errorf("unsupported string encoding prefix %d", raw[0]) + } +} + +func localStorageMetadataSize(raw []byte) (int64, error) { + for len(raw) > 0 { + tag, read := binary.Uvarint(raw) + if read <= 0 { + return 0, fmt.Errorf("invalid protobuf tag") + } + raw = raw[read:] + field, wire := tag>>3, tag&7 + if wire != 0 { + return 0, fmt.Errorf("unsupported protobuf wire type %d", wire) + } + value, read := binary.Uvarint(raw) + if read <= 0 { + return 0, fmt.Errorf("invalid protobuf value") + } + raw = raw[read:] + if field == 2 { + if value > uint64(^uint64(0)>>1) { + return 0, fmt.Errorf("size exceeds int64") + } + return int64(value), nil + } + } + return 0, fmt.Errorf("size field is missing") +} + +func DiscoverExtensions(profile Profile) ([]Extension, error) { + payload, err := readFileBounded(filepath.Join(profile.Path, "Secure Preferences"), maxDocumentBytes) + if errors.Is(err, os.ErrNotExist) { + payload, err = readFileBounded(filepath.Join(profile.Path, "Preferences"), maxDocumentBytes) + } + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read browser extensions: %w", err) + } + var source struct { + Extensions struct { + Settings map[string]struct { + State *int `json:"state"` + FromWebStore bool `json:"from_webstore"` + Manifest struct { + Name string `json:"name"` + } `json:"manifest"` + } `json:"settings"` + } `json:"extensions"` + } + if err := json.Unmarshal(payload, &source); err != nil { + return nil, fmt.Errorf("decode browser extensions: %w", err) + } + result := make([]Extension, 0) + for id, setting := range source.Extensions.Settings { + if (setting.State != nil && *setting.State != 1) || !setting.FromWebStore || !chromeWebStoreExtensionID.MatchString(id) { + continue + } + result = append(result, Extension{ID: id, Name: setting.Manifest.Name, Source: "chrome_web_store"}) + } + sort.Slice(result, func(i, j int) bool { + if result[i].Name == result[j].Name { + return result[i].ID < result[j].ID + } + return result[i].Name < result[j].Name + }) + return result, nil +} + // CookieSites returns every website with importable cookies, ranked by recent // browser use. It reads cookie metadata only; values are decrypted by // ExportCookies after the user chooses what to import. @@ -374,6 +751,93 @@ type fileFingerprint struct { modified time.Time } +type directoryFingerprint struct { + name string + size int64 + modified time.Time +} + +func levelDBSnapshot(ctx context.Context, sourceDirectory string) (string, func(), error) { + if _, err := os.Stat(sourceDirectory); err != nil { + return "", nil, err + } + for attempt := 0; attempt < 3; attempt++ { + before, err := fingerprintLevelDB(sourceDirectory) + if err != nil { + return "", nil, err + } + var total int64 + for _, file := range before { + total += file.size + } + if total > maxStorageSource { + return "", nil, fmt.Errorf("browser local storage source exceeds 512 MiB") + } + root, err := os.MkdirTemp("", "kernel-browser-import-leveldb-") + if err != nil { + return "", nil, err + } + destination := filepath.Join(root, "leveldb") + if err := os.Mkdir(destination, 0o700); err != nil { + _ = os.RemoveAll(root) + return "", nil, err + } + changed := false + for _, file := range before { + err := copyFileBounded(ctx, filepath.Join(sourceDirectory, file.name), filepath.Join(destination, file.name), file.size) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, errFileGrew) { + changed = true + break + } + if err != nil { + _ = os.RemoveAll(root) + return "", nil, fmt.Errorf("copy local storage %s: %w", file.name, err) + } + } + after, err := fingerprintLevelDB(sourceDirectory) + if !changed && err == nil && directoryFingerprintsEqual(before, after) { + return destination, func() { _ = os.RemoveAll(root) }, nil + } + _ = os.RemoveAll(root) + } + return "", nil, fmt.Errorf("browser local storage changed while it was being read; try again") +} + +func fingerprintLevelDB(directory string) ([]directoryFingerprint, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + result := make([]directoryFingerprint, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || entry.Name() == "LOCK" || entry.Name() == "LOG" || entry.Name() == "LOG.old" { + continue + } + info, err := entry.Info() + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + continue + } + result = append(result, directoryFingerprint{name: entry.Name(), size: info.Size(), modified: info.ModTime()}) + } + sort.Slice(result, func(left, right int) bool { return result[left].name < result[right].name }) + return result, nil +} + +func directoryFingerprintsEqual(left, right []directoryFingerprint) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + func sqliteSnapshot(ctx context.Context, databasePath string) (string, func(), error) { if !fileExists(databasePath) { return "", nil, fmt.Errorf("browser database %q was not found", filepath.Base(databasePath)) diff --git a/internal/browserimport/chromium_test.go b/internal/browserimport/chromium_test.go index 188046a..baa1ee8 100644 --- a/internal/browserimport/chromium_test.go +++ b/internal/browserimport/chromium_test.go @@ -6,6 +6,7 @@ import ( "crypto/aes" "crypto/cipher" "crypto/sha256" + "encoding/binary" "encoding/json" "fmt" "os" @@ -13,9 +14,11 @@ import ( "path/filepath" "testing" "time" + "unicode/utf16" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb" ) func TestDiscoverMacOSProfilesFindsChromeAndHelium(t *testing.T) { @@ -229,6 +232,92 @@ INSERT INTO cookies VALUES assert.Equal(t, []string{"github", "google"}, []string{cookies[0].Name, cookies[1].Name}) } +func TestExportBookmarksNormalizesPortableRoots(t *testing.T) { + profile := sqliteProfileFixture(t) + payload := `{"roots":{"bookmark_bar":{"children":[{"type":"url","name":"Kernel","url":"https://onkernel.com","date_added":"13400000000000000"},{"type":"url","name":"Local","url":"chrome://settings"}]},"other":{"children":[{"type":"folder","name":"Work","children":[{"type":"url","name":"GitHub","url":"https://github.com"}]}]}}}` + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Bookmarks"), []byte(payload), 0o600)) + + document, count, err := ExportBookmarks(profile) + require.NoError(t, err) + require.Equal(t, 3, count) + require.Len(t, document.Roots, 2) + require.Equal(t, "https://onkernel.com", document.Roots[0].Children[0].URL) + require.Len(t, document.Roots[0].Children, 1, "non-http bookmarks must not leave the laptop") +} + +func TestExportHistoryUsesRequestedWindow(t *testing.T) { + profile := sqliteProfileFixture(t) + database := filepath.Join(profile.Path, "History") + runSQLite(t, database, ` +CREATE TABLE urls (id INTEGER PRIMARY KEY, url TEXT, title TEXT, last_visit_time INTEGER, visit_count INTEGER); +CREATE TABLE visits (id INTEGER PRIMARY KEY, url INTEGER, visit_time INTEGER); +INSERT INTO urls VALUES + (1, 'https://recent.example', 'Recent', 13400000000000000, 4), + (2, 'chrome://settings', 'Private', 13400000001000000, 2), + (3, 'https://old.example', 'Old', 12000000000000000, 1); +INSERT INTO visits VALUES + (1, 1, 13400000000000000), + (2, 1, 13400000000000001), + (3, 2, 13400000001000000), + (4, 3, 12000000000000000); +`) + + records, err := ExportHistory(t.Context(), profile, time.UnixMicro(13400000000000000-11_644_473_600_000_000)) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, "https://recent.example", records[0].URL) + require.Equal(t, 2, records[0].VisitCount) + count, err := HistoryCount(t.Context(), profile, time.UnixMicro(13400000000000000-11_644_473_600_000_000)) + require.NoError(t, err) + require.Equal(t, 2, count) +} + +func TestLocalStorageSitesAndExportUseLivePortableRecords(t *testing.T) { + profile := sqliteProfileFixture(t) + databasePath := filepath.Join(profile.Path, "Local Storage", "leveldb") + require.NoError(t, os.MkdirAll(filepath.Dir(databasePath), 0o755)) + database, err := leveldb.OpenFile(databasePath, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + require.NoError(t, database.Put([]byte("META:https://example.com"), localStorageMetadataFixture(1234), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("https://example.com", "theme"), chromiumStorageStringFixture("dark"), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("https://other.example", "emoji"), chromiumStorageStringFixture("hello 世界"), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("chrome-extension://abcdefghijklmnopabcdefghijklmnop", "private"), chromiumStorageStringFixture("skip"), nil)) + sites, err := LocalStorageSites(t.Context(), profile) + require.NoError(t, err) + require.Len(t, sites, 2) + require.Equal(t, StorageSite{Origin: "https://example.com", Bytes: 1234}, sites[0]) + require.Equal(t, "https://other.example", sites[1].Origin) + require.Positive(t, sites[1].Bytes) + + records, err := ExportLocalStorage(t.Context(), profile, []string{"https://example.com"}) + require.NoError(t, err) + require.Equal(t, []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, records) + + records, err = ExportLocalStorage(t.Context(), profile, nil) + require.NoError(t, err) + require.Len(t, records, 2) + require.Equal(t, "hello 世界", records[1].Value) +} + +func TestDiscoverExtensionsUsesPortableAllowlist(t *testing.T) { + profile := sqliteProfileFixture(t) + payload := `{ + "browser":{"theme":{"color_scheme2":2},"show_home_button":true}, + "extensions":{"settings":{ + "abcdefghijklmnopabcdefghijklmnop":{"from_webstore":true,"manifest":{"name":"Eligible"}}, + "bcdefghijklmnopabcdefghijklmnopa":{"state":0,"from_webstore":true,"manifest":{"name":"Disabled"}}, + "cdefghijklmnopabcdefghijklmnopab":{"state":1,"from_webstore":false,"manifest":{"name":"Local"}} + }} +}` + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Preferences"), []byte(payload), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Secure Preferences"), []byte(payload), 0o600)) + + extensions, err := DiscoverExtensions(profile) + require.NoError(t, err) + require.Equal(t, []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Name: "Eligible", Source: "chrome_web_store"}}, extensions) +} + func TestSelectedCookieFilterEscapesInput(t *testing.T) { cte, clause := selectedCookieFilter([]string{"example.com' OR 1=1 --"}, false) assert.Contains(t, cte, "example.com'' or 1=1 --") @@ -314,3 +403,40 @@ func bytesRepeat(value byte, count int) []byte { } return result } + +func localStorageRecordKeyFixture(origin, key string) []byte { + result := append([]byte("_"+origin+"\x00"), chromiumStorageStringFixture(key)...) + return result +} + +func chromiumStorageStringFixture(value string) []byte { + runes := []rune(value) + latin1 := true + for _, value := range runes { + if value > 255 { + latin1 = false + break + } + } + if latin1 { + result := make([]byte, 1, len(runes)+1) + result[0] = 1 + for _, value := range runes { + result = append(result, byte(value)) + } + return result + } + units := utf16.Encode(runes) + result := make([]byte, 1+len(units)*2) + for index, value := range units { + binary.LittleEndian.PutUint16(result[1+index*2:], value) + } + return result +} + +func localStorageMetadataFixture(size uint64) []byte { + result := []byte{8, 1, 16} + buffer := make([]byte, binary.MaxVarintLen64) + written := binary.PutUvarint(buffer, size) + return append(result, buffer[:written]...) +} diff --git a/internal/browserimport/types.go b/internal/browserimport/types.go index 3d1f598..4f5db21 100644 --- a/internal/browserimport/types.go +++ b/internal/browserimport/types.go @@ -46,6 +46,61 @@ type Cookie struct { SameSite string `json:"same_site,omitempty"` } +type BookmarkDocument struct { + Roots []BookmarkRoot `json:"roots"` +} + +type BookmarkRoot struct { + Name string `json:"name"` + Children []BookmarkNode `json:"children"` +} + +type BookmarkNode struct { + Title string `json:"title"` + URL string `json:"url,omitempty"` + Children []BookmarkNode `json:"children"` + DateAdded *time.Time `json:"date_added,omitempty"` + DateLastUsed *time.Time `json:"date_last_used,omitempty"` +} + +type HistoryRecord struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` + VisitedAt time.Time `json:"visited_at"` + VisitCount int `json:"visit_count,omitempty"` +} + +const ( + StorageKindLocal = "local_storage" + MaxPortableStorageSize = 64 << 20 +) + +type StorageRecord struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Key string `json:"key"` + Value string `json:"value"` +} + +type StorageSite struct { + Origin string + Bytes int64 +} + +type Extension struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Source string `json:"source"` +} + +type ProfileData struct { + Cookies []Cookie + Storage []StorageRecord + Bookmarks *BookmarkDocument + History []HistoryRecord + Extensions []Extension +} + type Source struct { ID string `json:"id"` Kind string `json:"kind"` diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 38e7836..c081050 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -223,7 +223,7 @@ func bundleIdentifier(contents []byte) (string, error) { func macOSAppleScript(executable string) string { return `on «event GURLGURL» incomingURL set kernelExecutable to "` + appleScriptString(executable) + `" -set commandText to quoted form of kernelExecutable & " connector open " & quoted form of incomingURL +set commandText to "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done; exec " & quoted form of kernelExecutable & " connector open " & quoted form of incomingURL set scriptPath to «event sysoexec» "/usr/bin/mktemp /tmp/kernel-connector.XXXXXX" set scriptFile to «event rdwropen» POSIX file scriptPath with «class perm» «event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile diff --git a/internal/connector/connector_test.go b/internal/connector/connector_test.go index f810cab..f626d3c 100644 --- a/internal/connector/connector_test.go +++ b/internal/connector/connector_test.go @@ -134,6 +134,8 @@ func TestMacOSAppleScriptShellQuotesURLAtRuntime(t *testing.T) { assert.Contains(t, script, `/usr/bin/open -a Terminal`) assert.Contains(t, script, `/bin/zsh -lic`) assert.Contains(t, script, `/usr/bin/mktemp /tmp/kernel-connector.XXXXXX`) + assert.Contains(t, script, `/bin/launchctl getenv`) + assert.Contains(t, script, `KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL`) assert.NotContains(t, script, `XXXXXX.command`) assert.Contains(t, script, `Kernel CLI was removed`) assert.Contains(t, script, `«event GURLGURL»`)