branch-4.1: [feature](jsf) Treat JuiceFS (jfs://) as HDFS-compatible in FE/BE #61031

be/src/common/config.cpp

@@ -1496,6 +1496,41 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB
 /** Iceberg sink configurations **/
 DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB
 
+// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem.
+// Each entry uses the format "<scheme>=<file_type>", and file_type must be one of:
+// local, hdfs, s3, http, broker.
+DEFINE_Strings(paimon_file_system_scheme_mappings,
+               "file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs,"
+               "s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3,"
+               "abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http,"
+               "ofs=broker,gfs=broker");
+DEFINE_Validator(paimon_file_system_scheme_mappings,
+                 ([](const std::vector<std::string>& mappings) -> bool {
+                     doris::StringCaseUnorderedSet seen_schemes;
+                     static const doris::StringCaseUnorderedSet supported_types = {
+                             "local", "hdfs", "s3", "http", "broker"};
+                     for (const auto& raw_entry : mappings) {
+                         std::string_view entry = doris::trim(raw_entry);
+                         size_t separator = entry.find('=');
+                         if (separator == std::string_view::npos) {
+                             return false;
+                         }
+                         std::string scheme = std::string(doris::trim(entry.substr(0, separator)));
+                         std::string file_type =
+                                 std::string(doris::trim(entry.substr(separator + 1)));
+                         if (scheme.empty() || file_type.empty()) {
+                             return false;
+                         }
+                         if (supported_types.find(file_type) == supported_types.end()) {
+                             return false;
+                         }
+                         if (!seen_schemes.insert(scheme).second) {
+                             return false;
+                         }
+                     }
+                     return true;
+                 }));
+
 DEFINE_mInt32(thrift_client_open_num_tries, "1");
 
 DEFINE_Bool(enable_index_compaction, "false");

be/src/common/config.h

@@ -1592,6 +1592,9 @@ DECLARE_mInt64(hive_sink_max_file_size);
 /** Iceberg sink configurations **/
 DECLARE_mInt64(iceberg_sink_max_file_size);
 
+/** Paimon file system configurations **/
+DECLARE_Strings(paimon_file_system_scheme_mappings);
+
 // Number of open tries, default 1 means only try to open once.
 // Retry the Open num_retries time waiting 100 milliseconds between retries.
 DECLARE_mInt32(thrift_client_open_num_tries);

be/src/format/table/paimon_doris_file_system.cpp

@@ -19,8 +19,6 @@
 
 #include <gen_cpp/Types_types.h>
 
-#include <algorithm>
-#include <cctype>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -29,6 +27,7 @@
 #include <utility>
 #include <vector>
 
+#include "common/config.h"
 #include "common/status.h"
 #include "io/file_factory.h"
 #include "io/fs/file_reader.h"
@@ -40,6 +39,7 @@
 #include "paimon/fs/file_system_factory.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
+#include "util/string_util.h"
 
 namespace paimon {
 
@@ -48,12 +48,6 @@ struct ParsedUri {
     std::string authority;
 };
 
-std::string to_lower(std::string value) {
-    std::ranges::transform(value, value.begin(),
-                           [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
-    return value;
-}
-
 ParsedUri parse_uri(const std::string& path) {
     ParsedUri parsed;
     size_t scheme_pos = path.find("://");
@@ -65,7 +59,7 @@ ParsedUri parse_uri(const std::string& path) {
     if (scheme_pos == std::string::npos || scheme_pos == 0) {
         return parsed;
     }
-    parsed.scheme = to_lower(path.substr(0, scheme_pos));
+    parsed.scheme = doris::to_lower(path.substr(0, scheme_pos));
     size_t authority_start = scheme_pos + delim_len;
     if (authority_start >= path.size() || path[authority_start] == '/') {
         return parsed;
@@ -79,38 +73,58 @@ ParsedUri parse_uri(const std::string& path) {
     return parsed;
 }
 
-bool is_s3_scheme(const std::string& scheme) {
-    return scheme == "s3" || scheme == "s3a" || scheme == "s3n" || scheme == "oss" ||
-           scheme == "obs" || scheme == "cos" || scheme == "cosn" || scheme == "gs" ||
-           scheme == "abfs" || scheme == "abfss" || scheme == "wasb" || scheme == "wasbs";
-}
-
-bool is_hdfs_scheme(const std::string& scheme) {
-    return scheme == "hdfs" || scheme == "viewfs" || scheme == "local";
+bool parse_scheme_mapping_target(std::string_view raw_target, doris::TFileType::type* type) {
+    std::string target = doris::to_lower(std::string(doris::trim(raw_target)));
+    if (target == "local") {
+        *type = doris::TFileType::FILE_LOCAL;
+        return true;
+    }
+    if (target == "hdfs") {
+        *type = doris::TFileType::FILE_HDFS;
+        return true;
+    }
+    if (target == "s3") {
+        *type = doris::TFileType::FILE_S3;
+        return true;
+    }
+    if (target == "http") {
+        *type = doris::TFileType::FILE_HTTP;
+        return true;
+    }
+    if (target == "broker") {
+        *type = doris::TFileType::FILE_BROKER;
+        return true;
+    }
+    return false;
 }
 
-bool is_http_scheme(const std::string& scheme) {
-    return scheme == "http" || scheme == "https";
+bool parse_scheme_mapping_entry(std::string_view raw_entry, std::string* scheme,
+                                doris::TFileType::type* type) {
+    size_t separator = raw_entry.find('=');
+    if (separator == std::string_view::npos) {
+        return false;
+    }
+    *scheme = doris::to_lower(std::string(doris::trim(raw_entry.substr(0, separator))));
+    if (scheme->empty()) {
+        return false;
+    }
+    return parse_scheme_mapping_target(raw_entry.substr(separator + 1), type);
 }
 
 doris::TFileType::type map_scheme_to_file_type(const std::string& scheme) {
     if (scheme.empty()) {
         return doris::TFileType::FILE_HDFS;
     }
-    if (scheme == "file") {
-        return doris::TFileType::FILE_LOCAL;
-    }
-    if (is_hdfs_scheme(scheme)) {
-        return doris::TFileType::FILE_HDFS;
-    }
-    if (is_s3_scheme(scheme)) {
-        return doris::TFileType::FILE_S3;
-    }
-    if (is_http_scheme(scheme)) {
-        return doris::TFileType::FILE_HTTP;
-    }
-    if (scheme == "ofs" || scheme == "gfs" || scheme == "jfs") {
-        return doris::TFileType::FILE_BROKER;
+    std::string normalized_scheme = doris::to_lower(scheme);
+    for (const auto& mapping_entry : doris::config::paimon_file_system_scheme_mappings) {
+        std::string configured_scheme;
+        doris::TFileType::type configured_type;
+        if (!parse_scheme_mapping_entry(mapping_entry, &configured_scheme, &configured_type)) {
+            continue;
+        }
+        if (configured_scheme == normalized_scheme) {
+            return configured_type;
+        }
     }
     return doris::TFileType::FILE_HDFS;
 }
@@ -149,7 +163,7 @@ std::string normalize_path_for_type(const std::string& path, const std::string&
     if (type == doris::TFileType::FILE_LOCAL) {
         return normalize_local_path(path);
     }
-    if (type == doris::TFileType::FILE_S3 && scheme != "s3" && !is_http_scheme(scheme)) {
+    if (type == doris::TFileType::FILE_S3 && scheme != "s3") {
         return replace_scheme(path, "s3");
     }
     return path;

be/src/format/table/paimon_doris_file_system.h

@@ -17,9 +17,20 @@
 
 #pragma once
 
+#include <gen_cpp/Types_types.h>
+
+#include <string>
+
+namespace paimon {
+
+// Visible for tests: maps a URI scheme to the Doris file type used by paimon-cpp.
+doris::TFileType::type map_scheme_to_file_type(const std::string& scheme);
+
+} // namespace paimon
+
 namespace doris {
 
 // Force-link helper so the paimon-cpp file system factory registration is kept.
 void register_paimon_doris_file_system();
 
-} // namespace doris
+} // namespace doris

be/src/gen_cpp/CMakeLists.txt

@@ -23,6 +23,7 @@ file(GLOB SRC_FILES CONFIGURE_DEPENDS
     ${GEN_CPP_DIR}/*.cpp
     ${GEN_CPP_DIR}/*.cc
 )
+list(FILTER SRC_FILES EXCLUDE REGEX "_server\\.skeleton\\.cpp$")
 
 add_compile_options(-Wno-return-type)
 
@@ -43,4 +44,3 @@ endif()
 
 #add_dependencies(DorisGen thrift-cpp)
 #add_dependencies(Opcode function)
-

be/src/io/file_factory.h

@@ -119,6 +119,7 @@ class FileFactory {
         case TStorageBackendType::BROKER:
             return TFileType::FILE_BROKER;
         case TStorageBackendType::HDFS:
+        case TStorageBackendType::JFS:
             return TFileType::FILE_HDFS;
         default:
             return ResultError(Status::FatalError("not match type to convert, from type:{}", type));

be/test/format/table/paimon_doris_file_system_test.cpp

@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/table/paimon_doris_file_system.h"
+
+#include <gtest/gtest.h>
+
+#include <vector>
+
+#include "common/config.h"
+
+namespace doris {
+
+class PaimonDorisFileSystemTest : public testing::Test {
+protected:
+    void SetUp() override { saved_mappings_ = config::paimon_file_system_scheme_mappings; }
+
+    void TearDown() override { config::paimon_file_system_scheme_mappings = saved_mappings_; }
+
+    std::vector<std::string> saved_mappings_;
+};
+
+TEST_F(PaimonDorisFileSystemTest, UsesDefaultSchemeMappings) {
+    EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file"));
+    EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("jfs"));
+    EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("s3a"));
+    EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("gs"));
+    EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("https"));
+    EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("ofs"));
+    EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("unknown"));
+}
+
+TEST_F(PaimonDorisFileSystemTest, AllowsOverridingSchemeMappingsFromConfig) {
+    config::paimon_file_system_scheme_mappings = {"file=local", "jfs = s3", "gs = hdfs",
+                                                  "custom-http = http", "custom-broker = broker"};
+
+    EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file"));
+    EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("JFS"));
+    EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("gs"));
+    EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("custom-http"));
+    EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("custom-broker"));
+    EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("still-unknown"));
+}
+
+} // namespace doris

bin/start_be.sh

@@ -254,6 +254,14 @@ if [[ -d "${DORIS_HOME}/lib/java_extensions/jindofs" ]]; then
     done
 fi
 
+# add juicefs
+# should after jars in lib/hadoop_hdfs/, or it will override the hadoop jars in lib/hadoop_hdfs
+if [[ -d "${DORIS_HOME}/lib/java_extensions/juicefs" ]]; then
+    for f in "${DORIS_HOME}/lib/java_extensions/juicefs"/*.jar; do
+        DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
+    done
+fi
+
 # add custom_libs to CLASSPATH
 # ATTN, custom_libs is deprecated, use plugins/java_extensions
 if [[ -d "${DORIS_HOME}/custom_lib" ]]; then

bin/start_fe.sh

@@ -368,6 +368,14 @@ if [[ -d "${DORIS_HOME}/lib/jindofs" ]]; then
     done
 fi
 
+# add juicefs
+# should after jars in lib/, or it will override the hadoop jars in lib/
+if [[ -d "${DORIS_HOME}/lib/juicefs" ]]; then
+    for f in "${DORIS_HOME}/lib/juicefs"/*.jar; do
+        CLASSPATH="${CLASSPATH}:${f}"
+    done
+fi
+
 # add plugins/java_extensions to CLASSPATH
 if [[ -d "${DORIS_HOME}/plugins/java_extensions" ]]; then
     for f in "${DORIS_HOME}/plugins/java_extensions"/*.jar; do