Skip to content

Commit 24f154e

Browse files
majin1102majin1102
committed
init
1 parent c69eff4 commit 24f154e

18 files changed

Lines changed: 956 additions & 74 deletions

File tree

‎amoro-ams/pom.xml‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
<properties>
3535
<swagger-ui.version>5.17.14</swagger-ui.version>
36+
<lance-arrow-version>18.3.0</lance-arrow-version>
3637
<git-commit-id-plugin.fail-on-no-git-dir>false</git-commit-id-plugin.fail-on-no-git-dir>
3738
</properties>
3839

@@ -231,7 +232,6 @@
231232
<dependency>
232233
<groupId>mysql</groupId>
233234
<artifactId>mysql-connector-java</artifactId>
234-
<scope>provided</scope>
235235
</dependency>
236236

237237
<dependency>
@@ -409,6 +409,13 @@
409409
<scope>test</scope>
410410
</dependency>
411411

412+
<dependency>
413+
<groupId>org.apache.amoro</groupId>
414+
<artifactId>amoro-format-lance</artifactId>
415+
<version>${project.version}</version>
416+
<scope>test</scope>
417+
</dependency>
418+
412419
<dependency>
413420
<groupId>org.apache.curator</groupId>
414421
<artifactId>curator-test</artifactId>
@@ -618,6 +625,66 @@
618625
</dependency>
619626
</dependencies>
620627
</profile>
628+
<profile>
629+
<id>support-lance-format</id>
630+
<dependencies>
631+
<dependency>
632+
<groupId>org.apache.amoro</groupId>
633+
<artifactId>amoro-format-lance</artifactId>
634+
</dependency>
635+
<dependency>
636+
<groupId>org.apache.arrow</groupId>
637+
<artifactId>arrow-vector</artifactId>
638+
</dependency>
639+
<dependency>
640+
<groupId>org.apache.arrow</groupId>
641+
<artifactId>arrow-memory-netty</artifactId>
642+
</dependency>
643+
<dependency>
644+
<groupId>org.apache.arrow</groupId>
645+
<artifactId>arrow-c-data</artifactId>
646+
</dependency>
647+
<dependency>
648+
<groupId>org.apache.arrow</groupId>
649+
<artifactId>arrow-dataset</artifactId>
650+
</dependency>
651+
</dependencies>
652+
<dependencyManagement>
653+
<dependencies>
654+
<dependency>
655+
<groupId>org.apache.arrow</groupId>
656+
<artifactId>arrow-bom</artifactId>
657+
<version>${lance-arrow-version}</version>
658+
<type>pom</type>
659+
<scope>import</scope>
660+
</dependency>
661+
</dependencies>
662+
</dependencyManagement>
663+
<build>
664+
<plugins>
665+
<plugin>
666+
<groupId>org.apache.maven.plugins</groupId>
667+
<artifactId>maven-enforcer-plugin</artifactId>
668+
<version>3.5.0</version>
669+
<executions>
670+
<execution>
671+
<id>enforce-arrow-dependency-convergence</id>
672+
<phase>validate</phase>
673+
<goals>
674+
<goal>enforce</goal>
675+
</goals>
676+
<configuration>
677+
<rules>
678+
<DependencyConvergence/>
679+
</rules>
680+
<fail>true</fail>
681+
</configuration>
682+
</execution>
683+
</executions>
684+
</plugin>
685+
</plugins>
686+
</build>
687+
</profile>
621688
<profile>
622689
<id>support-all-formats</id>
623690
<dependencies>
@@ -629,6 +696,10 @@
629696
<groupId>org.apache.amoro</groupId>
630697
<artifactId>amoro-format-hudi</artifactId>
631698
</dependency>
699+
<dependency>
700+
<groupId>org.apache.amoro</groupId>
701+
<artifactId>amoro-format-lance</artifactId>
702+
</dependency>
632703
</dependencies>
633704
</profile>
634705
<profile>
@@ -638,4 +709,4 @@
638709
</properties>
639710
</profile>
640711
</profiles>
641-
</project>
712+
</project>

‎amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,11 @@ public class CatalogBuilder {
4343
ImmutableMap.of(
4444
CATALOG_TYPE_HADOOP,
4545
Sets.newHashSet(
46-
TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON, TableFormat.HUDI),
46+
TableFormat.ICEBERG,
47+
TableFormat.MIXED_ICEBERG,
48+
TableFormat.PAIMON,
49+
TableFormat.HUDI,
50+
TableFormat.LANCE),
4751
CATALOG_TYPE_GLUE,
4852
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
4953
CATALOG_TYPE_REST,

‎amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import static org.apache.amoro.TableFormat.HUDI;
2222
import static org.apache.amoro.TableFormat.ICEBERG;
23+
import static org.apache.amoro.TableFormat.LANCE;
2324
import static org.apache.amoro.TableFormat.MIXED_HIVE;
2425
import static org.apache.amoro.TableFormat.MIXED_ICEBERG;
2526
import static org.apache.amoro.TableFormat.PAIMON;
@@ -48,6 +49,7 @@
4849
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_S3_ENDPOINT;
4950
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE;
5051
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP;
52+
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL;
5153
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_OSS;
5254
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3;
5355
import static org.apache.amoro.properties.CatalogMetaProperties.TABLE_FORMATS;
@@ -146,6 +148,12 @@ public class CatalogController {
146148
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON));
147149
VALIDATE_CATALOGS.add(
148150
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON));
151+
VALIDATE_CATALOGS.add(
152+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
153+
VALIDATE_CATALOGS.add(
154+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
155+
VALIDATE_CATALOGS.add(
156+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
149157
VALIDATE_CATALOGS.add(
150158
CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_S3, ICEBERG));
151159
VALIDATE_CATALOGS.add(
@@ -234,7 +242,7 @@ public void getCatalogTypeList(Context ctx) {
234242
String displayKey = "display";
235243
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_AMS, displayKey, "Amoro Metastore"));
236244
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HIVE, displayKey, "Hive Metastore"));
237-
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HADOOP, displayKey, "Filesystem"));
245+
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HADOOP, displayKey, "FileSystem"));
238246
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_GLUE, displayKey, "Glue"));
239247
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_REST, displayKey, "REST"));
240248
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_CUSTOM, displayKey, "Custom"));
@@ -519,6 +527,8 @@ private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info, CatalogMeta o
519527
STORAGE_CONFIGS_KEY_OSS_ENDPOINT,
520528
"fs.oss.endpoint");
521529
}
530+
} else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_LOCAL)) {
531+
// Local storage type does not require additional storage configs.
522532
} else {
523533
throw new RuntimeException("Invalid storage type " + storageType);
524534
}

‎amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ public AmoroTable<?> loadTable(String database, String table) {
120120
TableFormat.MIXED_ICEBERG,
121121
TableFormat.ICEBERG,
122122
TableFormat.PAIMON,
123-
TableFormat.HUDI)
123+
TableFormat.HUDI,
124+
TableFormat.LANCE)
124125
.map(
125126
formatCatalog -> {
126127
try {
@@ -147,7 +148,8 @@ public List<TableIDWithFormat> listTables(String database) {
147148
TableFormat.MIXED_ICEBERG,
148149
TableFormat.ICEBERG,
149150
TableFormat.PAIMON,
150-
TableFormat.HUDI
151+
TableFormat.HUDI,
152+
TableFormat.LANCE
151153
};
152154

153155
Map<String, TableFormat> tableNameToFormat = Maps.newHashMap();

‎amoro-common/src/main/java/org/apache/amoro/TableFormat.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public final class TableFormat implements Serializable {
4242
/** Open-source table formats */
4343
public static final TableFormat ICEBERG = register("ICEBERG");
4444

45+
public static final TableFormat LANCE = register("LANCE");
4546
public static final TableFormat MIXED_ICEBERG = register("MIXED_ICEBERG");
4647
public static final TableFormat MIXED_HIVE = register("MIXED_HIVE");
4748
public static final TableFormat PAIMON = register("PAIMON");

‎amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public class CatalogMetaProperties {
3333
public static final String STORAGE_CONFIGS_VALUE_TYPE_HADOOP = "Hadoop";
3434
public static final String STORAGE_CONFIGS_VALUE_TYPE_S3 = "S3";
3535
public static final String STORAGE_CONFIGS_VALUE_TYPE_OSS = "OSS";
36+
public static final String STORAGE_CONFIGS_VALUE_TYPE_LOCAL = "Local";
3637

3738
public static final String AUTH_CONFIGS_KEY_TYPE = "auth.type";
3839
public static final String AUTH_CONFIGS_KEY_PRINCIPAL = "auth.kerberos.principal";

‎amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java‎

Lines changed: 75 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.amoro.table.TableIdentifier;
2828
import org.apache.amoro.table.TableMetaStore;
2929
import org.apache.commons.lang3.StringUtils;
30+
import org.apache.hadoop.conf.Configuration;
3031
import org.slf4j.Logger;
3132
import org.slf4j.LoggerFactory;
3233

@@ -58,7 +59,8 @@ public static Set<TableFormat> tableFormats(
5859
.collect(Collectors.toSet());
5960
} else {
6061
// Generate table format from catalog type for compatibility with older versions
61-
switch (metastoreType) {
62+
String normalizedType = normalizeCatalogType(metastoreType);
63+
switch (normalizedType) {
6264
case CatalogMetaProperties.CATALOG_TYPE_AMS:
6365
return Sets.newHashSet(TableFormat.MIXED_ICEBERG);
6466
case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
@@ -111,83 +113,91 @@ public static Map<String, String> mergeCatalogPropertiesToTable(
111113
public static TableMetaStore buildMetaStore(CatalogMeta catalogMeta) {
112114
// load storage configs
113115
TableMetaStore.Builder builder = TableMetaStore.builder();
116+
boolean isLocalStorage = false;
114117
if (catalogMeta.getStorageConfigs() != null) {
115118
Map<String, String> storageConfigs = catalogMeta.getStorageConfigs();
116-
if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(
117-
CatalogUtil.getCompatibleStorageType(storageConfigs))) {
119+
String storageType = CatalogUtil.getCompatibleStorageType(storageConfigs);
120+
isLocalStorage =
121+
CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL.equalsIgnoreCase(storageType);
122+
if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(storageType)) {
118123
String coreSite = storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE);
119124
String hdfsSite = storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE);
120125
String hiveSite = storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HIVE_SITE);
121126
builder
122127
.withBase64CoreSite(coreSite)
123128
.withBase64MetaStoreSite(hiveSite)
124129
.withBase64HdfsSite(hdfsSite);
130+
} else if (isLocalStorage) {
131+
builder.withConfiguration(new Configuration());
125132
}
126133
}
127134

128-
boolean loadAuthFromAMS =
129-
propertyAsBoolean(
130-
catalogMeta.getCatalogProperties(),
131-
CatalogMetaProperties.LOAD_AUTH_FROM_AMS,
132-
CatalogMetaProperties.LOAD_AUTH_FROM_AMS_DEFAULT);
133-
// load auth configs from ams
134-
if (loadAuthFromAMS) {
135-
if (catalogMeta.getAuthConfigs() != null) {
136-
Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
137-
String authType = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
138-
LOG.info("TableMetaStore use auth config in catalog meta, authType is {}", authType);
135+
if (!isLocalStorage) {
136+
boolean loadAuthFromAMS =
137+
propertyAsBoolean(
138+
catalogMeta.getCatalogProperties(),
139+
CatalogMetaProperties.LOAD_AUTH_FROM_AMS,
140+
CatalogMetaProperties.LOAD_AUTH_FROM_AMS_DEFAULT);
141+
// load auth configs from ams
142+
if (loadAuthFromAMS) {
143+
if (catalogMeta.getAuthConfigs() != null) {
144+
Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
145+
String authType = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
146+
LOG.info("TableMetaStore use auth config in catalog meta, authType is {}", authType);
147+
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType)) {
148+
String hadoopUsername =
149+
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
150+
builder.withSimpleAuth(hadoopUsername);
151+
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
152+
authType)) {
153+
String krb5 = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
154+
String keytab = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
155+
String principal = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
156+
builder.withBase64KrbAuth(keytab, krb5, principal);
157+
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(
158+
authType)) {
159+
String accessKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
160+
String secretKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
161+
builder.withAkSkAuth(accessKey, secretKey);
162+
}
163+
}
164+
}
165+
166+
// cover auth configs from ams with auth configs in properties
167+
String authType =
168+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
169+
if (StringUtils.isNotEmpty(authType)) {
170+
LOG.info("TableMetaStore use auth config in properties, authType is {}", authType);
139171
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType)) {
140172
String hadoopUsername =
141-
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
173+
catalogMeta
174+
.getCatalogProperties()
175+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
142176
builder.withSimpleAuth(hadoopUsername);
143177
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
144178
authType)) {
145-
String krb5 = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
146-
String keytab = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
147-
String principal = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
179+
String krb5 =
180+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
181+
String keytab =
182+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
183+
String principal =
184+
catalogMeta
185+
.getCatalogProperties()
186+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
148187
builder.withBase64KrbAuth(keytab, krb5, principal);
149188
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(authType)) {
150-
String accessKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
151-
String secretKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
189+
String accessKey =
190+
catalogMeta
191+
.getCatalogProperties()
192+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
193+
String secretKey =
194+
catalogMeta
195+
.getCatalogProperties()
196+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
152197
builder.withAkSkAuth(accessKey, secretKey);
153198
}
154199
}
155200
}
156-
157-
// cover auth configs from ams with auth configs in properties
158-
String authType =
159-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
160-
if (StringUtils.isNotEmpty(authType)) {
161-
LOG.info("TableMetaStore use auth config in properties, authType is {}", authType);
162-
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType)) {
163-
String hadoopUsername =
164-
catalogMeta
165-
.getCatalogProperties()
166-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
167-
builder.withSimpleAuth(hadoopUsername);
168-
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
169-
authType)) {
170-
String krb5 =
171-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
172-
String keytab =
173-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
174-
String principal =
175-
catalogMeta
176-
.getCatalogProperties()
177-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
178-
builder.withBase64KrbAuth(keytab, krb5, principal);
179-
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(authType)) {
180-
String accessKey =
181-
catalogMeta
182-
.getCatalogProperties()
183-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
184-
String secretKey =
185-
catalogMeta
186-
.getCatalogProperties()
187-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
188-
builder.withAkSkAuth(accessKey, secretKey);
189-
}
190-
}
191201
return builder.build();
192202
}
193203

@@ -246,4 +256,14 @@ private static boolean propertyAsBoolean(
246256
}
247257
return defaultValue;
248258
}
259+
260+
private static String normalizeCatalogType(String metastoreType) {
261+
if (metastoreType == null) {
262+
return null;
263+
}
264+
if ("filesystem".equalsIgnoreCase(metastoreType)) {
265+
return CatalogMetaProperties.CATALOG_TYPE_HADOOP;
266+
}
267+
return metastoreType;
268+
}
249269
}

0 commit comments

Comments
 (0)