Skip to content

Commit 0de2543

Browse files
majin1102majin1102
committed
init
1 parent 29fed70 commit 0de2543

16 files changed

Lines changed: 866 additions & 73 deletions

File tree

‎amoro-ams/pom.xml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,6 @@
231231
<dependency>
232232
<groupId>mysql</groupId>
233233
<artifactId>mysql-connector-java</artifactId>
234-
<scope>provided</scope>
235234
</dependency>
236235

237236
<dependency>
@@ -409,6 +408,13 @@
409408
<scope>test</scope>
410409
</dependency>
411410

411+
<dependency>
412+
<groupId>org.apache.amoro</groupId>
413+
<artifactId>amoro-format-lance</artifactId>
414+
<version>${project.version}</version>
415+
<scope>test</scope>
416+
</dependency>
417+
412418
<dependency>
413419
<groupId>org.apache.curator</groupId>
414420
<artifactId>curator-test</artifactId>
@@ -618,6 +624,15 @@
618624
</dependency>
619625
</dependencies>
620626
</profile>
627+
<profile>
628+
<id>support-lance-format</id>
629+
<dependencies>
630+
<dependency>
631+
<groupId>org.apache.amoro</groupId>
632+
<artifactId>amoro-format-lance</artifactId>
633+
</dependency>
634+
</dependencies>
635+
</profile>
621636
<profile>
622637
<id>support-all-formats</id>
623638
<dependencies>
@@ -629,6 +644,10 @@
629644
<groupId>org.apache.amoro</groupId>
630645
<artifactId>amoro-format-hudi</artifactId>
631646
</dependency>
647+
<dependency>
648+
<groupId>org.apache.amoro</groupId>
649+
<artifactId>amoro-format-lance</artifactId>
650+
</dependency>
632651
</dependencies>
633652
</profile>
634653
<profile>

‎amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,11 @@ public class CatalogBuilder {
4343
ImmutableMap.of(
4444
CATALOG_TYPE_HADOOP,
4545
Sets.newHashSet(
46-
TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON, TableFormat.HUDI),
46+
TableFormat.ICEBERG,
47+
TableFormat.MIXED_ICEBERG,
48+
TableFormat.PAIMON,
49+
TableFormat.HUDI,
50+
TableFormat.LANCE),
4751
CATALOG_TYPE_GLUE,
4852
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
4953
CATALOG_TYPE_REST,

‎amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import static org.apache.amoro.TableFormat.HUDI;
2222
import static org.apache.amoro.TableFormat.ICEBERG;
23+
import static org.apache.amoro.TableFormat.LANCE;
2324
import static org.apache.amoro.TableFormat.MIXED_HIVE;
2425
import static org.apache.amoro.TableFormat.MIXED_ICEBERG;
2526
import static org.apache.amoro.TableFormat.PAIMON;
@@ -48,6 +49,7 @@
4849
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_S3_ENDPOINT;
4950
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE;
5051
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP;
52+
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL;
5153
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_OSS;
5254
import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3;
5355
import static org.apache.amoro.properties.CatalogMetaProperties.TABLE_FORMATS;
@@ -146,6 +148,12 @@ public class CatalogController {
146148
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON));
147149
VALIDATE_CATALOGS.add(
148150
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON));
151+
VALIDATE_CATALOGS.add(
152+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
153+
VALIDATE_CATALOGS.add(
154+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
155+
VALIDATE_CATALOGS.add(
156+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
149157
VALIDATE_CATALOGS.add(
150158
CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_S3, ICEBERG));
151159
VALIDATE_CATALOGS.add(
@@ -234,7 +242,7 @@ public void getCatalogTypeList(Context ctx) {
234242
String displayKey = "display";
235243
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_AMS, displayKey, "Amoro Metastore"));
236244
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HIVE, displayKey, "Hive Metastore"));
237-
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HADOOP, displayKey, "Filesystem"));
245+
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HADOOP, displayKey, "FileSystem"));
238246
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_GLUE, displayKey, "Glue"));
239247
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_REST, displayKey, "REST"));
240248
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_CUSTOM, displayKey, "Custom"));
@@ -519,6 +527,8 @@ private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info, CatalogMeta o
519527
STORAGE_CONFIGS_KEY_OSS_ENDPOINT,
520528
"fs.oss.endpoint");
521529
}
530+
} else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_LOCAL)) {
531+
// Local storage type does not require additional storage configs.
522532
} else {
523533
throw new RuntimeException("Invalid storage type " + storageType);
524534
}

‎amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ public AmoroTable<?> loadTable(String database, String table) {
120120
TableFormat.MIXED_ICEBERG,
121121
TableFormat.ICEBERG,
122122
TableFormat.PAIMON,
123-
TableFormat.HUDI)
123+
TableFormat.HUDI,
124+
TableFormat.LANCE)
124125
.map(
125126
formatCatalog -> {
126127
try {
@@ -147,7 +148,8 @@ public List<TableIDWithFormat> listTables(String database) {
147148
TableFormat.MIXED_ICEBERG,
148149
TableFormat.ICEBERG,
149150
TableFormat.PAIMON,
150-
TableFormat.HUDI
151+
TableFormat.HUDI,
152+
TableFormat.LANCE
151153
};
152154

153155
Map<String, TableFormat> tableNameToFormat = Maps.newHashMap();

‎amoro-common/src/main/java/org/apache/amoro/TableFormat.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public final class TableFormat implements Serializable {
4242
/** Open-source table formats */
4343
public static final TableFormat ICEBERG = register("ICEBERG");
4444

45+
public static final TableFormat LANCE = register("LANCE");
4546
public static final TableFormat MIXED_ICEBERG = register("MIXED_ICEBERG");
4647
public static final TableFormat MIXED_HIVE = register("MIXED_HIVE");
4748
public static final TableFormat PAIMON = register("PAIMON");

‎amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public class CatalogMetaProperties {
3333
public static final String STORAGE_CONFIGS_VALUE_TYPE_HADOOP = "Hadoop";
3434
public static final String STORAGE_CONFIGS_VALUE_TYPE_S3 = "S3";
3535
public static final String STORAGE_CONFIGS_VALUE_TYPE_OSS = "OSS";
36+
public static final String STORAGE_CONFIGS_VALUE_TYPE_LOCAL = "Local";
3637

3738
public static final String AUTH_CONFIGS_KEY_TYPE = "auth.type";
3839
public static final String AUTH_CONFIGS_KEY_PRINCIPAL = "auth.kerberos.principal";

‎amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java‎

Lines changed: 75 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.amoro.table.TableIdentifier;
2828
import org.apache.amoro.table.TableMetaStore;
2929
import org.apache.commons.lang3.StringUtils;
30+
import org.apache.hadoop.conf.Configuration;
3031
import org.slf4j.Logger;
3132
import org.slf4j.LoggerFactory;
3233

@@ -58,7 +59,8 @@ public static Set<TableFormat> tableFormats(
5859
.collect(Collectors.toSet());
5960
} else {
6061
// Generate table format from catalog type for compatibility with older versions
61-
switch (metastoreType) {
62+
String normalizedType = normalizeCatalogType(metastoreType);
63+
switch (normalizedType) {
6264
case CatalogMetaProperties.CATALOG_TYPE_AMS:
6365
return Sets.newHashSet(TableFormat.MIXED_ICEBERG);
6466
case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
@@ -111,83 +113,91 @@ public static Map<String, String> mergeCatalogPropertiesToTable(
111113
public static TableMetaStore buildMetaStore(CatalogMeta catalogMeta) {
112114
// load storage configs
113115
TableMetaStore.Builder builder = TableMetaStore.builder();
116+
boolean isLocalStorage = false;
114117
if (catalogMeta.getStorageConfigs() != null) {
115118
Map<String, String> storageConfigs = catalogMeta.getStorageConfigs();
116-
if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(
117-
CatalogUtil.getCompatibleStorageType(storageConfigs))) {
119+
String storageType = CatalogUtil.getCompatibleStorageType(storageConfigs);
120+
isLocalStorage =
121+
CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL.equalsIgnoreCase(storageType);
122+
if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(storageType)) {
118123
String coreSite = storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE);
119124
String hdfsSite = storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE);
120125
String hiveSite = storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HIVE_SITE);
121126
builder
122127
.withBase64CoreSite(coreSite)
123128
.withBase64MetaStoreSite(hiveSite)
124129
.withBase64HdfsSite(hdfsSite);
130+
} else if (isLocalStorage) {
131+
builder.withConfiguration(new Configuration());
125132
}
126133
}
127134

128-
boolean loadAuthFromAMS =
129-
propertyAsBoolean(
130-
catalogMeta.getCatalogProperties(),
131-
CatalogMetaProperties.LOAD_AUTH_FROM_AMS,
132-
CatalogMetaProperties.LOAD_AUTH_FROM_AMS_DEFAULT);
133-
// load auth configs from ams
134-
if (loadAuthFromAMS) {
135-
if (catalogMeta.getAuthConfigs() != null) {
136-
Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
137-
String authType = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
138-
LOG.info("TableMetaStore use auth config in catalog meta, authType is {}", authType);
135+
if (!isLocalStorage) {
136+
boolean loadAuthFromAMS =
137+
propertyAsBoolean(
138+
catalogMeta.getCatalogProperties(),
139+
CatalogMetaProperties.LOAD_AUTH_FROM_AMS,
140+
CatalogMetaProperties.LOAD_AUTH_FROM_AMS_DEFAULT);
141+
// load auth configs from ams
142+
if (loadAuthFromAMS) {
143+
if (catalogMeta.getAuthConfigs() != null) {
144+
Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
145+
String authType = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
146+
LOG.info("TableMetaStore use auth config in catalog meta, authType is {}", authType);
147+
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType)) {
148+
String hadoopUsername =
149+
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
150+
builder.withSimpleAuth(hadoopUsername);
151+
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
152+
authType)) {
153+
String krb5 = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
154+
String keytab = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
155+
String principal = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
156+
builder.withBase64KrbAuth(keytab, krb5, principal);
157+
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(
158+
authType)) {
159+
String accessKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
160+
String secretKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
161+
builder.withAkSkAuth(accessKey, secretKey);
162+
}
163+
}
164+
}
165+
166+
// cover auth configs from ams with auth configs in properties
167+
String authType =
168+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
169+
if (StringUtils.isNotEmpty(authType)) {
170+
LOG.info("TableMetaStore use auth config in properties, authType is {}", authType);
139171
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType)) {
140172
String hadoopUsername =
141-
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
173+
catalogMeta
174+
.getCatalogProperties()
175+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
142176
builder.withSimpleAuth(hadoopUsername);
143177
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
144178
authType)) {
145-
String krb5 = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
146-
String keytab = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
147-
String principal = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
179+
String krb5 =
180+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
181+
String keytab =
182+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
183+
String principal =
184+
catalogMeta
185+
.getCatalogProperties()
186+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
148187
builder.withBase64KrbAuth(keytab, krb5, principal);
149188
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(authType)) {
150-
String accessKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
151-
String secretKey = authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
189+
String accessKey =
190+
catalogMeta
191+
.getCatalogProperties()
192+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
193+
String secretKey =
194+
catalogMeta
195+
.getCatalogProperties()
196+
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
152197
builder.withAkSkAuth(accessKey, secretKey);
153198
}
154199
}
155200
}
156-
157-
// cover auth configs from ams with auth configs in properties
158-
String authType =
159-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
160-
if (StringUtils.isNotEmpty(authType)) {
161-
LOG.info("TableMetaStore use auth config in properties, authType is {}", authType);
162-
if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType)) {
163-
String hadoopUsername =
164-
catalogMeta
165-
.getCatalogProperties()
166-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
167-
builder.withSimpleAuth(hadoopUsername);
168-
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
169-
authType)) {
170-
String krb5 =
171-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
172-
String keytab =
173-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
174-
String principal =
175-
catalogMeta
176-
.getCatalogProperties()
177-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
178-
builder.withBase64KrbAuth(keytab, krb5, principal);
179-
} else if (CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(authType)) {
180-
String accessKey =
181-
catalogMeta
182-
.getCatalogProperties()
183-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
184-
String secretKey =
185-
catalogMeta
186-
.getCatalogProperties()
187-
.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
188-
builder.withAkSkAuth(accessKey, secretKey);
189-
}
190-
}
191201
return builder.build();
192202
}
193203

@@ -246,4 +256,14 @@ private static boolean propertyAsBoolean(
246256
}
247257
return defaultValue;
248258
}
259+
260+
private static String normalizeCatalogType(String metastoreType) {
261+
if (metastoreType == null) {
262+
return null;
263+
}
264+
if ("filesystem".equalsIgnoreCase(metastoreType)) {
265+
return CatalogMetaProperties.CATALOG_TYPE_HADOOP;
266+
}
267+
return metastoreType;
268+
}
249269
}

‎amoro-format-lance/pom.xml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>org.apache.amoro</groupId>
24+
<artifactId>amoro-parent</artifactId>
25+
<version>0.9-SNAPSHOT</version>
26+
<relativePath>../pom.xml</relativePath>
27+
</parent>
28+
29+
<artifactId>amoro-format-lance</artifactId>
30+
<name>Amoro Project Lance Format</name>
31+
32+
<dependencies>
33+
<dependency>
34+
<groupId>org.apache.amoro</groupId>
35+
<artifactId>amoro-common</artifactId>
36+
</dependency>
37+
38+
<!-- Lance format Java API -->
39+
<dependency>
40+
<groupId>org.lance</groupId>
41+
<artifactId>lance-core</artifactId>
42+
<version>2.0.0-beta.8</version>
43+
</dependency>
44+
45+
<!-- Lance namespace interfaces and models -->
46+
<dependency>
47+
<groupId>org.lance</groupId>
48+
<artifactId>lance-namespace-core</artifactId>
49+
<version>0.3.2</version>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.lance</groupId>
53+
<artifactId>lance-namespace-apache-client</artifactId>
54+
<version>0.3.2</version>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.apache.iceberg</groupId>
58+
<artifactId>iceberg-aws</artifactId>
59+
</dependency>
60+
<dependency>
61+
<groupId>org.apache.iceberg</groupId>
62+
<artifactId>iceberg-aliyun</artifactId>
63+
</dependency>
64+
</dependencies>
65+
66+
</project>

0 commit comments

Comments
 (0)