Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions fe/fe-connector/fe-connector-adbc/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,7 @@ under the License.
<version>${project.version}</version>
</dependency>

<!-- fe-connector-cache: the shared external meta-cache framework (CacheSpec property validation +
MetaCacheEntry/CacheFactory), backing AdbcMetadataCache. fe-core does NOT depend on it, so under
the parent-first org.apache.doris.connector.* prefix these classes resolve parent→miss→CHILD,
i.e. they load from this plugin's own bundled jar and link against THIS plugin's Caffeine. -->
<!-- Shared declarative metadata-cache framework backing AdbcMetadataCache. -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>fe-connector-cache</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.connector.adbc;

import org.apache.doris.connector.cache.CatalogMetaCache;
import org.apache.doris.connector.spi.Connector;
import org.apache.doris.connector.spi.ConnectorContext;
import org.apache.doris.connector.spi.ConnectorMetadata;
Expand Down Expand Up @@ -46,6 +47,7 @@ public class AdbcConnector implements Connector {
private final AdbcSchemaStrategy schemaStrategy = new AdbcSchemaStrategy();
private final AdbcPartitionedReadSupport partitionedRead = new AdbcPartitionedReadSupport();
private final AdbcDialectSelector dialectSelector;
private final CatalogMetaCache metaCache = new CatalogMetaCache();
private final AdbcMetadataCache metadataCache;

private volatile AdbcClient client;
Expand All @@ -65,7 +67,7 @@ public AdbcConnector(Map<String, String> properties, ConnectorContext context) {
// The raw map, because the cache knobs are the shared framework's keys rather than this
// connector's: CacheSpec owns their names, and mirroring them as fields here would give the
// validator and the reader two things to drift apart.
this.metadataCache = new AdbcMetadataCache(props.getRaw());
this.metadataCache = new AdbcMetadataCache(metaCache, props.getRaw());
}

@Override
Expand Down Expand Up @@ -212,6 +214,7 @@ private AdbcClient getOrCreateClient() {
@Override
public synchronized void close() throws IOException {
closed = true;
metaCache.close();
if (client != null) {
client.close();
client = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,16 @@
package org.apache.doris.connector.adbc;

import org.apache.doris.connector.cache.CacheSpec;
import org.apache.doris.connector.cache.MetaCacheEntry;
import org.apache.doris.connector.cache.CatalogMetaCache;
import org.apache.doris.connector.cache.MetaCache;
import org.apache.doris.connector.cache.MetaCacheDefinition;
import org.apache.doris.connector.cache.ScopePath;

import org.apache.arrow.vector.types.pojo.Schema;

import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ForkJoinPool;
import java.util.function.Supplier;

/**
Expand Down Expand Up @@ -85,19 +87,33 @@ public final class AdbcMetadataCache {
/** The database listing is a single value, so it needs a single key. */
private static final String THE_ONLY_KEY = "";

private final MetaCacheEntry<String, List<AdbcNamespace>> namespaces;
private final MetaCacheEntry<AdbcNamespace, List<String>> tableNames;
private final MetaCacheEntry<TableKey, Schema> tableSchemas;
private final CatalogMetaCache owner;
private final MetaCache<String, List<AdbcNamespace>> namespaces;
private final MetaCache<TableNameKey, List<String>> tableNames;
private final MetaCache<TableKey, Schema> tableSchemas;

/**
* Built in {@link AdbcConnector}'s constructor, which also runs on an FE replaying the edit log, so this
* reads properties and nothing else -- no driver, no filesystem, no remote call.
*/
public AdbcMetadataCache(Map<String, String> properties) {
this(new CatalogMetaCache(), properties);
}

AdbcMetadataCache(CatalogMetaCache owner, Map<String, String> properties) {
this.owner = Objects.requireNonNull(owner, "owner can not be null");
CacheSpec spec = cacheSpec(properties);
this.namespaces = entry("adbc-namespaces", spec);
this.tableNames = entry("adbc-table-names", spec);
this.tableSchemas = entry("adbc-table-schema", spec);
this.namespaces = owner.create(MetaCacheDefinition
.<String, List<AdbcNamespace>>builder("adbc-namespaces", spec, ignored -> ScopePath.catalog())
.build());
this.tableNames = owner.create(MetaCacheDefinition
.<TableNameKey, List<String>>builder("adbc-table-names", spec,
key -> ScopePath.database(key.dorisDbName))
.build());
this.tableSchemas = owner.create(MetaCacheDefinition
.<TableKey, Schema>builder("adbc-table-schema", spec,
key -> ScopePath.table(key.dorisDbName, key.table))
.build());
}

static CacheSpec cacheSpec(Map<String, String> properties) {
Expand All @@ -114,15 +130,6 @@ static CacheSpec.PropertySpec propertySpec() {
CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY));
}

/**
* Contextual-only with manual miss load, as the iceberg caches are: the remote read runs OUTSIDE
* Caffeine's compute lock, so a slow source does not stall unrelated keys, the driver's own exception
* arrives unwrapped, and a load that failed is not remembered as an answer.
*/
private static <K, V> MetaCacheEntry<K, V> entry(String name, CacheSpec spec) {
return new MetaCacheEntry<>(name, null, spec, ForkJoinPool.commonPool(), false, true, 0L, true);
}

// ========= reads =========

List<AdbcNamespace> namespaces(Supplier<List<AdbcNamespace>> loader) {
Expand All @@ -136,12 +143,12 @@ List<AdbcNamespace> reloadNamespaces(Supplier<List<AdbcNamespace>> loader) {
}

List<String> tableNames(AdbcNamespace namespace, Supplier<List<String>> loader) {
return tableNames.get(namespace, ignored -> loader.get());
return tableNames.get(TableNameKey.forNamespace(namespace), ignored -> loader.get());
}

/** Re-reads one database's table listing, replacing whatever was remembered. See the class note. */
List<String> reloadTableNames(AdbcNamespace namespace, Supplier<List<String>> loader) {
tableNames.invalidateKey(namespace);
tableNames.invalidateKey(TableNameKey.forNamespace(namespace));
return tableNames(namespace, loader);
}

Expand All @@ -160,24 +167,55 @@ Schema tableSchema(AdbcTableHandle handle, Supplier<Schema> loader) {
* new name in, and the one the user tried would appear to do nothing.
*/
void invalidateTable(String dbName, String tableName) {
tableSchemas.invalidateIf(key -> key.is(dbName, tableName));
tableNames.invalidateIf(namespace -> namespace.dorisDatabaseName().equals(dbName));
owner.invalidateTable(dbName, tableName);
// A table refresh must also forget the parent name listing, so a remotely-created table can be found.
// This is an ancestor materialization, not a sibling-cache dependency, and therefore cannot be selected
// by the table's descendant scope.
tableNames.invalidateKey(TableNameKey.forDatabase(dbName));
}

/** {@code REFRESH DATABASE}: that database's table listing and every schema in it. */
void invalidateDb(String dbName) {
tableSchemas.invalidateIf(key -> key.isIn(dbName));
tableNames.invalidateIf(namespace -> namespace.dorisDatabaseName().equals(dbName));
owner.invalidateDatabase(dbName);
}

/**
* {@code REFRESH CATALOG}: everything, including which databases exist -- the only statement that both
* names no database to invalidate and is reached for when the catalog's own shape changed.
*/
void invalidateAll() {
namespaces.invalidateAll();
tableNames.invalidateAll();
tableSchemas.invalidateAll();
owner.invalidateCatalog();
}

/**
* A database's listing is addressed by the Doris database name. The remote namespace is carried only so
* the miss loader can query it; it is deliberately not part of identity because Doris cannot address two
* remote namespaces that project to the same database name separately.
*/
private static final class TableNameKey {
private final String dorisDbName;

private TableNameKey(String dorisDbName) {
this.dorisDbName = Objects.requireNonNull(dorisDbName, "dorisDbName can not be null");
}

private static TableNameKey forNamespace(AdbcNamespace namespace) {
return new TableNameKey(namespace.dorisDatabaseName());
}

private static TableNameKey forDatabase(String database) {
return new TableNameKey(database);
}

@Override
public boolean equals(Object o) {
return o instanceof TableNameKey && dorisDbName.equals(((TableNameKey) o).dorisDbName);
}

@Override
public int hashCode() {
return dorisDbName.hashCode();
}
}

/**
Expand All @@ -199,14 +237,6 @@ private TableKey(AdbcTableHandle handle) {
this.dorisDbName = handle.getDorisDbName();
}

private boolean is(String db, String tableName) {
return dorisDbName.equals(db) && table.equals(tableName);
}

private boolean isIn(String db) {
return dorisDbName.equals(db);
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand Down
19 changes: 8 additions & 11 deletions fe/fe-connector/fe-connector-cache/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,10 @@ under the License.
<packaging>jar</packaging>
<name>Doris FE Connector Cache Framework</name>
<description>
Connector-side meta-cache framework (CacheSpec + MetaCacheEntry + CacheFactory + MetaCacheEntryStats),
an INDEPENDENT copy of fe-core's `org.apache.doris.datasource.metacache` framework re-homed under the
`org.apache.doris.connector.*` prefix so the connector plugins can reuse it (they cannot import fe-core).
fe-core keeps its own copy untouched; the two live side-by-side until every connector has migrated, then
the fe-core copy is retired. fe-core does NOT depend on this module.

This module is bundled into each connector plugin zip (child-first), so it uses the plugin's own bundled
Caffeine at runtime; Caffeine is therefore `provided` here (compiled against, never packaged by this
module). The framework's public API (MetaCacheEntry) is Caffeine-free, and fe-core and the connectors
never share a cache object across the classloader boundary, so no Caffeine type crosses and there is no
split-brain. Two knobs fe-core reads from static Config are constructor-injected here.
Shared catalog-local metadata cache framework. Cache definitions declare their metadata scope, and every
physical cache in one catalog participates in hierarchical invalidation through a shared registry. The
public API is Caffeine-free so connector plugins can use their child-first Caffeine runtime without
exposing third-party types across the FE/connector classloader boundary.
</description>

<dependencies>
Expand All @@ -53,6 +46,10 @@ under the License.
<version>2.9.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,8 @@
/**
* Common cache specification for external metadata caches.
*
* <p>Connector-side copy of the meta-cache property model (independent-copy meta-cache migration). fe-core is
* NOT changed: it keeps its own {@code org.apache.doris.datasource.metacache.CacheSpec}; this is a separate
* class under {@code org.apache.doris.connector.*} used only by the connector plugins. Although that prefix is
* parent-first, fe-core does not depend on this module, so the class resolves parent → miss → CHILD and is
* child-loaded per plugin — fe-core and the plugins do NOT share one {@code Class} identity. It carries no
* third-party dependency (JDK only) and never crosses the fe-core↔connector boundary as an object (only its
* {@code IllegalArgumentException}, a JDK type, crosses), so it is safe on both classpaths.
* <p>This is the single property model shared by fe-core and connector plugins. It carries no fe-core dependency,
* so cache configuration and validation remain available to independently loaded connectors.
*
* <p>The {@code check*Property} validators throw {@link IllegalArgumentException} (fe-core's
* {@code PluginDrivenExternalCatalog.checkProperties} re-wraps it into a {@code DdlException} verbatim; the
Expand Down Expand Up @@ -204,7 +199,7 @@ public static boolean isMetaCacheKeyForEngine(String key, String engine) {
}

/**
* Convert ttlSecond to OptionalLong for CacheFactory.
* Convert ttlSecond to the optional expiry used by the cache runtime.
* ttlSecond=-1 means no expiration; ttlSecond=0 disables cache.
*/
public static OptionalLong toExpireAfterAccess(long ttlSecond) {
Expand Down
Loading
Loading