diff --git a/src/builder-utils.c b/src/builder-utils.c index 01d981b5..51bb7f1d 100644 --- a/src/builder-utils.c +++ b/src/builder-utils.c @@ -51,6 +51,11 @@ G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (yaml_parser_t, yaml_parser_delete) G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (yaml_document_t, yaml_document_delete) #endif +typedef struct { + int opath_fd; + char *name; +} LocaleFile; + char * builder_uri_to_filename (const char *uri) { @@ -283,78 +288,347 @@ directory_is_empty (const char *path) return empty; } +static void +locale_file_free (gpointer p) +{ + LocaleFile *lf = p; + glnx_close_fd (&lf->opath_fd); + g_free (lf->name); + g_free (lf); +} + static gboolean -migrate_locale_dir (GFile *source_dir, - GFile *separate_dir, - const char *subdir, - GError **error) +collect_dir_names (int dfd, + GPtrArray **out_names, + GError **error) { - g_autoptr(GFileEnumerator) dir_enum = NULL; - GFileInfo *next; - GError *temp_error = NULL; + g_auto(GLnxDirFdIterator) iter = { 0, }; + g_autoptr(GPtrArray) names = g_ptr_array_new_with_free_func (g_free); + struct stat dir_st; + struct dirent *dent; - dir_enum = g_file_enumerate_children (source_dir, "standard::name,standard::type", - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, NULL); - if (!dir_enum) - return TRUE; + if (!glnx_dirfd_iterator_init_at (dfd, ".", FALSE, &iter, error)) + return FALSE; + + if (fstat (iter.fd, &dir_st) < 0) + return glnx_throw_errno_prefix (error, "fstat dir"); - while ((next = g_file_enumerator_next_file (dir_enum, NULL, &temp_error))) + while (TRUE) { - g_autoptr(GFileInfo) child_info = next; - g_autoptr(GFile) locale_subdir = NULL; + glnx_autofd int opath_fd = -1; + struct stat st; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&iter, &dent, NULL, error)) + return FALSE; - if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY) + if (dent == NULL) + break; + + if (dent->d_type != DT_DIR) + continue; + + opath_fd = openat (iter.fd, dent->d_name, + O_PATH | O_NOFOLLOW | O_CLOEXEC); + if (opath_fd < 0) { - g_autoptr(GFile) child = NULL; - const char *name = g_file_info_get_name (child_info); - g_autofree char *language = g_strdup (name); - g_autofree char *relative = NULL; - g_autofree char *target = NULL; - char *c; - - c = strchr (language, '@'); - if (c != NULL) - *c = 0; - c = strchr (language, '_'); - if (c != NULL) - *c = 0; - c = strchr (language, '.'); - if (c != NULL) - *c = 0; - - /* We ship english and C locales always */ - if (strcmp (language, "C") == 0 || - strcmp (language, "en") == 0) + if (G_IN_SET (errno, ENOENT, EACCES)) continue; + return glnx_throw_errno_prefix (error, "openat %s", dent->d_name); + } - child = g_file_get_child (source_dir, g_file_info_get_name (child_info)); + if (fstat (opath_fd, &st) < 0) + return glnx_throw_errno_prefix (error, "fstat %s", dent->d_name); - relative = g_build_filename (language, subdir, name, NULL); - locale_subdir = g_file_resolve_relative_path (separate_dir, relative); - if (!flatpak_mkdir_p (locale_subdir, NULL, error)) - return FALSE; + if (!S_ISDIR (st.st_mode)) + continue; - if (!flatpak_cp_a (child, locale_subdir, NULL, - FLATPAK_CP_FLAGS_MERGE | FLATPAK_CP_FLAGS_MOVE, - NULL, NULL, error)) - return FALSE; + g_ptr_array_add (names, g_strdup (dent->d_name)); + } - target = g_build_filename ("../../share/runtime/locale", relative, NULL); + *out_names = g_steal_pointer (&names); + return TRUE; +} - if (!g_file_make_symbolic_link (child, target, - NULL, error)) - return FALSE; +static gboolean +migrate_lc_dir (int src_dfd, + const char *lc_name, + int dst_dfd, + GError **error) +{ + glnx_autofd int lc_src_dfd = -1; + glnx_autofd int lc_dst_dfd = -1; + g_auto(GLnxDirFdIterator) iter = { 0, }; + g_autoptr(GPtrArray) files = g_ptr_array_new_with_free_func (locale_file_free); + struct stat dir_st; + struct dirent *dent; + + if (!glnx_opendirat (src_dfd, lc_name, FALSE, &lc_src_dfd, error)) + return FALSE; + + if (!glnx_ensure_dir (dst_dfd, lc_name, 0755, error)) + return FALSE; + + lc_dst_dfd = openat (dst_dfd, lc_name, + O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (lc_dst_dfd < 0) + return glnx_throw_errno_prefix (error, "openat LC dir %s", lc_name); + + if (!glnx_dirfd_iterator_init_at (lc_src_dfd, ".", FALSE, &iter, error)) + return FALSE; + + if (fstat (iter.fd, &dir_st) < 0) + return glnx_throw_errno_prefix (error, "fstat LC dir %s", lc_name); + + while (TRUE) + { + glnx_autofd int opath_fd = -1; + struct stat st; + LocaleFile *lf; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&iter, &dent, NULL, error)) + return FALSE; + + if (dent == NULL) + break; + if (dent->d_type != DT_REG) + continue; + + opath_fd = openat (iter.fd, dent->d_name, + O_PATH | O_NOFOLLOW | O_CLOEXEC); + if (opath_fd < 0) + { + if (G_IN_SET (errno, ENOENT, EACCES)) + continue; + return glnx_throw_errno_prefix (error, "openat %s", dent->d_name); } + + if (fstat (opath_fd, &st) < 0) + return glnx_throw_errno_prefix (error, "fstat %s", dent->d_name); + + if (!S_ISREG (st.st_mode)) + continue; + + lf = g_new0 (LocaleFile, 1); + lf->opath_fd = g_steal_fd (&opath_fd); + lf->name = g_strdup (dent->d_name); + g_ptr_array_add (files, lf); + } + + for (size_t i = 0; i < files->len; i++) + { + LocaleFile *lf = g_ptr_array_index (files, i); + glnx_autofd int src_fd = -1; + glnx_autofd int dst_fd = -1; + + src_fd = glnx_fd_reopen (lf->opath_fd, O_RDONLY, error); + if (src_fd < 0) + return FALSE; + + dst_fd = openat (lc_dst_dfd, lf->name, + O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, 0644); + if (dst_fd < 0) + return glnx_throw_errno_prefix (error, "openat dest %s", lf->name); + + if (glnx_regfile_copy_bytes (src_fd, dst_fd, -1) < 0) + return glnx_throw_errno_prefix (error, "copying locale file %s", lf->name); + + if (unlinkat (lc_src_dfd, lf->name, 0) < 0) + return glnx_throw_errno_prefix (error, "unlinkat %s", lf->name); + } + + if (unlinkat (src_dfd, lc_name, AT_REMOVEDIR) < 0) + { + if (errno == ENOTEMPTY) + return TRUE; + return glnx_throw_errno_prefix (error, "unlinkat LC dir %s", lc_name); + } + + return TRUE; +} + +static gboolean +migrate_lang_dir (int source_dfd, + const char *lang_name, + int separate_dfd, + const char *subdir, + GError **error) +{ + glnx_autofd int child_dfd = -1; + glnx_autofd int locale_subdir_dfd = -1; + g_autofree char *language = g_strdup (lang_name); + g_autofree char *target = NULL; + g_autoptr(GPtrArray) lc_dirs = NULL; + char *c; + + c = strchr (language, '@'); + if (c != NULL) + *c = 0; + c = strchr (language, '_'); + if (c != NULL) + *c = 0; + c = strchr (language, '.'); + if (c != NULL) + *c = 0; + + const char *components[] = { language, subdir, lang_name, NULL }; + + if (!builder_ensure_dirs_at (separate_dfd, components, &locale_subdir_dfd, error)) + return FALSE; + + if (!glnx_opendirat (source_dfd, lang_name, FALSE, &child_dfd, error)) + return FALSE; + + if (!collect_dir_names (child_dfd, &lc_dirs, error)) + return FALSE; + + for (size_t i = 0; i < lc_dirs->len; i++) + { + if (!migrate_lc_dir (child_dfd, g_ptr_array_index (lc_dirs, i), + locale_subdir_dfd, error)) + return FALSE; + } + + { + g_auto(GLnxDirFdIterator) file_iter = { 0, }; + g_autoptr(GPtrArray) direct_files = g_ptr_array_new_with_free_func (locale_file_free); + struct dirent *fdent; + + if (!glnx_dirfd_iterator_init_at (child_dfd, ".", FALSE, &file_iter, error)) + return FALSE; + + while (TRUE) + { + glnx_autofd int opath_fd = -1; + struct stat st; + LocaleFile *lf; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&file_iter, &fdent, NULL, error)) + return FALSE; + + if (fdent == NULL) + break; + + if (fdent->d_type != DT_REG) + continue; + + opath_fd = openat (file_iter.fd, fdent->d_name, + O_PATH | O_NOFOLLOW | O_CLOEXEC); + if (opath_fd < 0) + { + if (G_IN_SET (errno, ENOENT, EACCES)) + continue; + return glnx_throw_errno_prefix (error, "openat %s", fdent->d_name); + } + + if (fstat (opath_fd, &st) < 0) + return glnx_throw_errno_prefix (error, "fstat %s", fdent->d_name); + + if (!S_ISREG (st.st_mode)) + continue; + + lf = g_new0 (LocaleFile, 1); + lf->opath_fd = g_steal_fd (&opath_fd); + lf->name = g_strdup (fdent->d_name); + g_ptr_array_add (direct_files, lf); + } + + for (size_t i = 0; i < direct_files->len; i++) + { + LocaleFile *lf = g_ptr_array_index (direct_files, i); + glnx_autofd int src_fd = -1; + glnx_autofd int dst_fd = -1; + + src_fd = glnx_fd_reopen (lf->opath_fd, O_RDONLY, error); + if (src_fd < 0) + return FALSE; + + dst_fd = openat (locale_subdir_dfd, lf->name, + O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, 0644); + if (dst_fd < 0) + return glnx_throw_errno_prefix (error, "openat dest %s", lf->name); + + if (glnx_regfile_copy_bytes (src_fd, dst_fd, -1) < 0) + return glnx_throw_errno_prefix (error, "copying locale file %s", lf->name); + + if (unlinkat (child_dfd, lf->name, 0) < 0) + return glnx_throw_errno_prefix (error, "unlinkat %s", lf->name); + } + } + + target = g_build_filename ("../../share/runtime/locale", + language, subdir, lang_name, NULL); + + if (unlinkat (source_dfd, lang_name, AT_REMOVEDIR) < 0) + { + if (errno == ENOTEMPTY) + return TRUE; + return glnx_throw_errno_prefix (error, "unlinkat LC dir %s", lang_name); } - if (temp_error != NULL) + if (symlinkat (target, source_dfd, lang_name) < 0) + return glnx_throw_errno_prefix (error, "symlinkat %s", lang_name); + + return TRUE; +} + +static gboolean +migrate_locale_dir (int root_dfd, + const char *source_rel, + int separate_dfd, + const char *subdir, + GError **error) +{ + glnx_autofd int chase_fd = -1; + glnx_autofd int source_dfd = -1; + g_autoptr(GPtrArray) lang_dirs = NULL; + + chase_fd = glnx_chaseat (root_dfd, source_rel, + GLNX_CHASE_RESOLVE_BENEATH | GLNX_CHASE_MUST_BE_DIRECTORY, + error); + if (chase_fd < 0) { - g_propagate_error (error, temp_error); + if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND) || + g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_DIRECTORY)) + { + g_clear_error (error); + return TRUE; + } return FALSE; } + source_dfd = openat (chase_fd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (source_dfd < 0) + return glnx_throw_errno_prefix (error, "failed to reopen locale dir %s", source_rel); + + if (!collect_dir_names (source_dfd, &lang_dirs, error)) + return FALSE; + + for (size_t i = 0; i < lang_dirs->len; i++) + { + const char *lang_name = g_ptr_array_index (lang_dirs, i); + g_autofree char *language = g_strdup (lang_name); + char *c; + + c = strchr (language, '@'); + if (c != NULL) + *c = 0; + c = strchr (language, '_'); + if (c != NULL) + *c = 0; + c = strchr (language, '.'); + if (c != NULL) + *c = 0; + + /* We ship english and C locales always */ + if (strcmp (language, "C") == 0 || + strcmp (language, "en") == 0) + continue; + + if (!migrate_lang_dir (source_dfd, lang_name, separate_dfd, subdir, error)) + return FALSE; + } + return TRUE; } @@ -395,18 +669,22 @@ gboolean builder_migrate_locale_dirs (GFile *root_dir, GError **error) { - g_autoptr(GFile) separate_dir = NULL; - g_autoptr(GFile) lib_locale_dir = NULL; - g_autoptr(GFile) share_locale_dir = NULL; + glnx_autofd int root_dfd = -1; + glnx_autofd int separate_dfd = -1; + const char *separate_components[] = { "share", "runtime", "locale", NULL }; - lib_locale_dir = g_file_resolve_relative_path (root_dir, "lib/locale"); - share_locale_dir = g_file_resolve_relative_path (root_dir, "share/locale"); - separate_dir = g_file_resolve_relative_path (root_dir, "share/runtime/locale"); + if (!glnx_opendirat (AT_FDCWD, + flatpak_file_get_path_cached (root_dir), + FALSE, &root_dfd, error)) + return FALSE; + + if (!builder_ensure_dirs_at (root_dfd, separate_components, &separate_dfd, error)) + return FALSE; - if (!migrate_locale_dir (lib_locale_dir, separate_dir, "lib", error)) + if (!migrate_locale_dir (root_dfd, "lib/locale", separate_dfd, "lib", error)) return FALSE; - if (!migrate_locale_dir (share_locale_dir, separate_dir, "share", error)) + if (!migrate_locale_dir (root_dfd, "share/locale", separate_dfd, "share", error)) return FALSE; return TRUE; diff --git a/tests/meson.build b/tests/meson.build index eda0e2d1..30c3a198 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -16,6 +16,7 @@ test_names = [ 'test-builder-cleanup', 'test-builder-licence-paths', 'test-builder-src-date-epoch', + 'test-builder-locale-migration', ] tap_test = find_program( diff --git a/tests/test-builder-locale-migration.sh b/tests/test-builder-locale-migration.sh new file mode 100755 index 00000000..d832df15 --- /dev/null +++ b/tests/test-builder-locale-migration.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# +# Copyright (C) 2026 Boudhayan Bhattacharya +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +skip_without_fuse + +echo "1..3" + +setup_repo +install_repo +setup_sdk_repo +install_sdk_repo + +cd "$TEST_DATA_DIR" + +run_build () { + local manifest=$1 + ${FLATPAK_BUILDER} --force-clean appdir "$manifest" >&2 +} + +cat > test-locale-migration.json <<'EOF' +{ + "app-id": "org.test.locale_migration", + "runtime": "org.test.Platform", + "sdk": "org.test.Sdk", + "modules": [{ + "name": "test", + "buildsystem": "simple", + "build-commands": [ + "mkdir -p /app/share/locale/de/LC_MESSAGES", + "echo 'de translation' > /app/share/locale/de/LC_MESSAGES/test.mo", + "mkdir -p /app/share/locale/C/LC_MESSAGES", + "echo 'C locale' > /app/share/locale/C/LC_MESSAGES/test.mo", + "mkdir -p /app/share/locale/en/LC_MESSAGES", + "echo 'en locale' > /app/share/locale/en/LC_MESSAGES/test.mo", + "mkdir -p /app/lib/locale/ru/LC_MESSAGES", + "echo 'ru translation' > /app/lib/locale/ru/LC_MESSAGES/test.mo", + "mkdir -p /app/share/locale/sr@latin/LC_MESSAGES", + "echo 'sr latin' > /app/share/locale/sr@latin/LC_MESSAGES/test.mo", + "mkdir -p /app/share/locale/pt_BR/LC_MESSAGES", + "echo 'pt_BR translation' > /app/share/locale/pt_BR/LC_MESSAGES/test.mo", + "mkdir -p /app/share/locale/de.utf8/LC_MESSAGES", + "echo 'de utf8' > /app/share/locale/de.utf8/LC_MESSAGES/test.mo", + "mkdir -p /app/share/locale/ko/LC_MESSAGES", + "echo 'app1' > /app/share/locale/ko/LC_MESSAGES/app1.mo", + "echo 'app2' > /app/share/locale/ko/LC_MESSAGES/app2.mo", + "mkdir -p /app/share/locale/es/LC_MESSAGES", + "echo 'es share' > /app/share/locale/es/LC_MESSAGES/test.mo", + "mkdir -p /app/lib/locale/es/LC_CTYPE", + "echo 'es lib' > /app/lib/locale/es/LC_CTYPE/test.mo", + "mkdir -p /app/share/locale/sv" + ] + }] +} +EOF + +run_build test-locale-migration.json + +# share/locale is migrated to share/runtime/locale/ +assert_has_symlink appdir/files/share/locale/de +assert_symlink_has_content appdir/files/share/locale/de 'share/runtime/locale/de' +assert_has_file appdir/files/share/runtime/locale/de/share/de/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/runtime/locale/de/share/de/LC_MESSAGES/test.mo 'de translation' + +# lib/locale is migrated to share/runtime/locale/ +assert_has_symlink appdir/files/lib/locale/ru +assert_symlink_has_content appdir/files/lib/locale/ru 'share/runtime/locale/ru' +assert_has_file appdir/files/share/runtime/locale/ru/lib/ru/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/runtime/locale/ru/lib/ru/LC_MESSAGES/test.mo 'ru translation' + +# C and en locales are not migrated +assert_has_dir appdir/files/share/locale/C +assert_has_file appdir/files/share/locale/C/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/locale/C/LC_MESSAGES/test.mo 'C locale' + +assert_has_dir appdir/files/share/locale/en +assert_has_file appdir/files/share/locale/en/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/locale/en/LC_MESSAGES/test.mo 'en locale' + +# '@' stripped: sr@latin -> sr +assert_has_symlink appdir/files/share/locale/sr@latin +assert_has_file appdir/files/share/runtime/locale/sr/share/sr@latin/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/runtime/locale/sr/share/sr@latin/LC_MESSAGES/test.mo 'sr latin' + +# '_' stripped: pt_BR -> pt +assert_has_symlink appdir/files/share/locale/pt_BR +assert_symlink_has_content appdir/files/share/locale/pt_BR 'share/runtime/locale/pt' +assert_has_file appdir/files/share/runtime/locale/pt/share/pt_BR/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/runtime/locale/pt/share/pt_BR/LC_MESSAGES/test.mo 'pt_BR translation' + +# '.' stripped: de.utf8 -> de +assert_has_symlink appdir/files/share/locale/de.utf8 +assert_has_file appdir/files/share/runtime/locale/de/share/de.utf8/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/runtime/locale/de/share/de.utf8/LC_MESSAGES/test.mo 'de utf8' + +# multiple files in a single locale dir are migrated +assert_has_symlink appdir/files/share/locale/ko +assert_has_file appdir/files/share/runtime/locale/ko/share/ko/LC_MESSAGES/app1.mo +assert_file_has_content appdir/files/share/runtime/locale/ko/share/ko/LC_MESSAGES/app1.mo 'app1' +assert_has_file appdir/files/share/runtime/locale/ko/share/ko/LC_MESSAGES/app2.mo +assert_file_has_content appdir/files/share/runtime/locale/ko/share/ko/LC_MESSAGES/app2.mo 'app2' + +# same language in share/locale and lib/locale is merged +assert_has_symlink appdir/files/share/locale/es +assert_has_file appdir/files/share/runtime/locale/es/share/es/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/runtime/locale/es/share/es/LC_MESSAGES/test.mo 'es share' +assert_has_symlink appdir/files/lib/locale/es +assert_has_file appdir/files/share/runtime/locale/es/lib/es/LC_CTYPE/test.mo +assert_file_has_content appdir/files/share/runtime/locale/es/lib/es/LC_CTYPE/test.mo 'es lib' + +# empty locale dir creates empty migration dir +assert_has_dir appdir/files/share/locale/sv +assert_has_dir appdir/files/share/runtime/locale/sv + +echo "ok locale dirs migrated" + + +cat > test-locale-disabled.json <<'EOF' +{ + "app-id": "org.test.locale_migration_disabled", + "runtime": "org.test.Platform", + "sdk": "org.test.Sdk", + "separate-locales": false, + "modules": [{ + "name": "test", + "buildsystem": "simple", + "build-commands": [ + "mkdir -p /app/share/locale/de/LC_MESSAGES", + "echo 'de translation' > /app/share/locale/de/LC_MESSAGES/test.mo" + ] + }] +} +EOF + +run_build test-locale-disabled.json + +assert_has_dir appdir/files/share/locale/de +assert_has_file appdir/files/share/locale/de/LC_MESSAGES/test.mo +assert_file_has_content appdir/files/share/locale/de/LC_MESSAGES/test.mo 'de translation' +assert_not_has_dir appdir/files/share/runtime/locale + +echo "ok locale dirs migration is disabled on separate-locales false" + +cat > test-locale-migration-runtime.json <<'EOF' +{ + "app-id": "org.test.locale_migration_runtime", + "runtime": "org.test.Platform", + "sdk": "org.test.Sdk", + "build-runtime": true, + "modules": [{ + "name": "test", + "buildsystem": "simple", + "build-commands": [ + "mkdir -p /usr/share/locale/pl/LC_MESSAGES", + "echo 'pl translation' > /usr/share/locale/pl/LC_MESSAGES/test.mo" + ] + }] +} +EOF + +run_build test-locale-migration-runtime.json + +assert_has_symlink appdir/usr/share/locale/pl +assert_has_file appdir/usr/share/runtime/locale/pl/share/pl/LC_MESSAGES/test.mo +assert_file_has_content appdir/usr/share/runtime/locale/pl/share/pl/LC_MESSAGES/test.mo 'pl translation' + +echo "ok locale dirs migration works with runtime"