Skip to content

Commit 0d5f0b8

Browse files
UmeshPatil-1UmeshPatil-1
committed
ATLASS-4988 : Added force delete validation logic for BusinessMetadata.
1 parent 8478f2b commit 0d5f0b8

11 files changed

Lines changed: 188 additions & 83 deletions

File tree

‎client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,19 @@ public void deleteAtlasTypeDefs(AtlasTypesDef typesDef) throws AtlasServiceExcep
353353
}
354354

355355
public void deleteTypeByName(String typeName) throws AtlasServiceException {
356-
callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class) null, null, typeName);
356+
callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class<?>) null, null, typeName);
357+
}
358+
359+
public void deleteTypeByName(String typeName, boolean forceDelete) throws AtlasServiceException {
360+
if (forceDelete) {
361+
362+
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
363+
queryParams.add("force", "true");
364+
callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class<?>) null, queryParams, typeName);
365+
} else {
366+
367+
callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class<?>) null, null, typeName);
368+
}
357369
}
358370

359371
// Entity APIs

‎intg/src/main/java/org/apache/atlas/AtlasErrorCode.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,9 @@ public enum AtlasErrorCode {
257257
IMPORT_REGISTRATION_FAILED(500, "ATLAS-500-00-020", "Failed to register import request"),
258258
IMPORT_FAILED(500, "ATLAS-500-00-021", "Import with id {0} failed"),
259259
ABORT_IMPORT_FAILED(500, "ATLAS-500-00-022", "Failed to abort import with id {0}"),
260-
IMPORT_QUEUEING_FAILED(500, "ATLAS-500-00-023", "Failed to add import with id {0} to request queue, please try again later");
260+
IMPORT_QUEUEING_FAILED(500, "ATLAS-500-00-023", "Failed to add import with id {0} to request queue, please try again later"),
261+
262+
NON_INDEXABLE_BM_DELETE_NOT_ALLOWED(400, "ATLAS-400-00-106", "Deletion not allowed for non-indexable Business Metadata ''{0}'' without force-delete. Please use the force-delete parameter to remove.");
261263

262264
private static final Logger LOG = LoggerFactory.getLogger(AtlasErrorCode.class);
263265
private final String errorCode;

‎repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasDefStore.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,25 @@ public interface AtlasDefStore<T extends AtlasBaseTypeDef> {
4545

4646
AtlasVertex preDeleteByName(String name) throws AtlasBaseException;
4747

48-
void deleteByName(String name, AtlasVertex preDeleteResult) throws AtlasBaseException;
49-
5048
AtlasVertex preDeleteByGuid(String guid) throws AtlasBaseException;
5149

50+
void deleteByName(String name, AtlasVertex preDeleteResult) throws AtlasBaseException;
51+
5252
void deleteByGuid(String guid, AtlasVertex preDeleteResult) throws AtlasBaseException;
53+
54+
default AtlasVertex preDeleteByName(String name, boolean forceDelete) throws AtlasBaseException {
55+
return preDeleteByName(name);
56+
}
57+
58+
default void deleteByName(String name, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
59+
deleteByName(name, preDeleteResult);
60+
}
61+
62+
default AtlasVertex preDeleteByGuid(String guid, boolean forceDelete) throws AtlasBaseException {
63+
return preDeleteByGuid(guid);
64+
}
65+
66+
default void deleteByGuid(String guid, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
67+
deleteByGuid(guid, preDeleteResult);
68+
}
5369
}

‎repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -539,14 +539,20 @@ public AtlasTypesDef createUpdateTypesDef(AtlasTypesDef typesToCreate, AtlasType
539539
@Override
540540
@GraphTransaction
541541
public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
542+
deleteTypesDef(typesDef, false);
543+
}
544+
545+
@GraphTransaction
546+
public void deleteTypesDef(AtlasTypesDef typesDef, boolean forceDelete) throws AtlasBaseException {
542547
if (LOG.isDebugEnabled()) {
543-
LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, businessMetadataDefs={})",
548+
LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, businessMetadataDefs={}, forceDelete={})",
544549
CollectionUtils.size(typesDef.getEnumDefs()),
545550
CollectionUtils.size(typesDef.getStructDefs()),
546551
CollectionUtils.size(typesDef.getClassificationDefs()),
547552
CollectionUtils.size(typesDef.getEntityDefs()),
548553
CollectionUtils.size(typesDef.getRelationshipDefs()),
549-
CollectionUtils.size(typesDef.getBusinessMetadataDefs()));
554+
CollectionUtils.size(typesDef.getBusinessMetadataDefs()),
555+
forceDelete);
550556
}
551557

552558
AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit();
@@ -678,9 +684,9 @@ public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
678684
if (CollectionUtils.isNotEmpty(typesDef.getBusinessMetadataDefs())) {
679685
for (AtlasBusinessMetadataDef businessMetadataDef : typesDef.getBusinessMetadataDefs()) {
680686
if (StringUtils.isNotBlank(businessMetadataDef.getGuid())) {
681-
businessMetadataDefStore.deleteByGuid(businessMetadataDef.getGuid(), null);
687+
businessMetadataDefStore.deleteByGuid(businessMetadataDef.getGuid(), null, forceDelete);
682688
} else {
683-
businessMetadataDefStore.deleteByName(businessMetadataDef.getName(), null);
689+
businessMetadataDefStore.deleteByName(businessMetadataDef.getName(), null, forceDelete);
684690
}
685691
}
686692
}
@@ -777,7 +783,7 @@ public AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException {
777783

778784
@Override
779785
@GraphTransaction
780-
public void deleteTypeByName(String typeName) throws AtlasBaseException {
786+
public void deleteTypeByName(String typeName, boolean forceDelete) throws AtlasBaseException {
781787
AtlasType atlasType = typeRegistry.getType(typeName);
782788

783789
if (atlasType == null) {
@@ -801,7 +807,7 @@ public void deleteTypeByName(String typeName) throws AtlasBaseException {
801807
typesDef.setStructDefs(Collections.singletonList((AtlasStructDef) baseTypeDef));
802808
}
803809

804-
deleteTypesDef(typesDef);
810+
deleteTypesDef(typesDef, forceDelete);
805811
}
806812

807813
@Override

‎repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasAbstractDefStoreV2.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,24 +148,46 @@ public boolean isValidName(String typeName) {
148148

149149
@Override
150150
public void deleteByName(String name, AtlasVertex preDeleteResult) throws AtlasBaseException {
151-
LOG.debug("==> AtlasAbstractDefStoreV1.deleteByName({}, {})", name, preDeleteResult);
151+
LOG.debug("==> AtlasAbstractDefStoreV2.deleteByName({}, {})", name, preDeleteResult);
152152

153153
AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByName(name) : preDeleteResult;
154154

155155
typeDefStore.deleteTypeVertex(vertex);
156156

157-
LOG.debug("<== AtlasAbstractDefStoreV1.deleteByName({}, {})", name, preDeleteResult);
157+
LOG.debug("<== AtlasAbstractDefStoreV2.deleteByName({}, {})", name, preDeleteResult);
158158
}
159159

160160
@Override
161161
public void deleteByGuid(String guid, AtlasVertex preDeleteResult) throws AtlasBaseException {
162-
LOG.debug("==> AtlasAbstractDefStoreV1.deleteByGuid({}, {})", guid, preDeleteResult);
162+
LOG.debug("==> AtlasAbstractDefStoreV2.deleteByGuid({}, {})", guid, preDeleteResult);
163163

164164
AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByGuid(guid) : preDeleteResult;
165165

166166
typeDefStore.deleteTypeVertex(vertex);
167167

168-
LOG.debug("<== AtlasAbstractDefStoreV1.deleteByGuid({}, {})", guid, preDeleteResult);
168+
LOG.debug("<== AtlasAbstractDefStoreV2.deleteByGuid({}, {})", guid, preDeleteResult);
169+
}
170+
171+
@Override
172+
public void deleteByName(String name, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
173+
LOG.debug("==> AtlasAbstractDefStoreV2.deleteByName({}, {}, {})", name, preDeleteResult, forceDelete);
174+
175+
AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByName(name, forceDelete) : preDeleteResult;
176+
177+
typeDefStore.deleteTypeVertex(vertex);
178+
179+
LOG.debug("<== AtlasAbstractDefStoreV2.deleteByName({}, {}, {})", name, preDeleteResult, forceDelete);
180+
}
181+
182+
@Override
183+
public void deleteByGuid(String guid, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
184+
LOG.debug("==> AtlasAbstractDefStoreV2.deleteByGuid({}, {}, {})", guid, preDeleteResult, forceDelete);
185+
186+
AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByGuid(guid, forceDelete) : preDeleteResult;
187+
188+
typeDefStore.deleteTypeVertex(vertex);
189+
190+
LOG.debug("<== AtlasAbstractDefStoreV2.deleteByGuid({}, {}, {})", guid, preDeleteResult, forceDelete);
169191
}
170192

171193
public boolean isInvalidTypeDefName(String typeName) {

‎repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java‎

Lines changed: 65 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,14 @@ public AtlasBusinessMetadataDef updateByGuid(String guid, AtlasBusinessMetadataD
266266
return ret;
267267
}
268268

269+
@Override
269270
public AtlasVertex preDeleteByName(String name) throws AtlasBaseException {
270-
LOG.debug("==> AtlasBusinessMetadataDefStoreV2.preDeleteByName({})", name);
271+
return preDeleteByName(name, false);
272+
}
273+
274+
@Override
275+
public AtlasVertex preDeleteByName(String name, boolean forceDelete) throws AtlasBaseException {
276+
LOG.debug("==> AtlasBusinessMetadataDefStoreV2.preDeleteByName({}, {})", name, forceDelete);
271277

272278
AtlasBusinessMetadataDef existingDef = typeRegistry.getBusinessMetadataDefByName(name);
273279

@@ -279,15 +285,29 @@ public AtlasVertex preDeleteByName(String name) throws AtlasBaseException {
279285
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
280286
}
281287

282-
checkBusinessMetadataRef(existingDef.getName());
288+
if (!forceDelete) {
289+
boolean hasIndexableAttribute = hasIndexableAttribute(existingDef);
290+
291+
if (!hasIndexableAttribute) {
292+
LOG.warn("Deletion blocked for non-indexable Business Metadata '{}' without force-delete flag", name);
293+
throw new AtlasBaseException(AtlasErrorCode.NON_INDEXABLE_BM_DELETE_NOT_ALLOWED, name);
294+
}
295+
checkBusinessMetadataRef(existingDef.getName());
296+
}
283297

284-
LOG.debug("<== AtlasBusinessMetadataDefStoreV2.preDeleteByName({}): {}", name, ret);
298+
LOG.debug("<== AtlasBusinessMetadataDefStoreV2.preDeleteByName({}, {}): {}", name, forceDelete, ret);
285299

286300
return ret;
287301
}
288302

303+
@Override
289304
public AtlasVertex preDeleteByGuid(String guid) throws AtlasBaseException {
290-
LOG.debug("==> AtlasBusinessMetadataDefStoreV2.preDeleteByGuid({})", guid);
305+
return preDeleteByGuid(guid, false);
306+
}
307+
308+
@Override
309+
public AtlasVertex preDeleteByGuid(String guid, boolean forceDelete) throws AtlasBaseException {
310+
LOG.debug("==> AtlasBusinessMetadataDefStoreV2.preDeleteByGuid({}, {})", guid, forceDelete);
291311

292312
AtlasBusinessMetadataDef existingDef = typeRegistry.getBusinessMetadataDefByGuid(guid);
293313

@@ -299,15 +319,35 @@ public AtlasVertex preDeleteByGuid(String guid) throws AtlasBaseException {
299319
throw new AtlasBaseException(AtlasErrorCode.TYPE_GUID_NOT_FOUND, guid);
300320
}
301321

302-
if (existingDef != null) {
322+
if (existingDef != null && !forceDelete) {
323+
boolean hasIndexableAttribute = hasIndexableAttribute(existingDef);
324+
325+
if (!hasIndexableAttribute) {
326+
LOG.warn("Deletion blocked for non-indexable Business Metadata '{}' without force-delete flag", existingDef.getName());
327+
throw new AtlasBaseException(AtlasErrorCode.NON_INDEXABLE_BM_DELETE_NOT_ALLOWED, existingDef.getName());
328+
}
303329
checkBusinessMetadataRef(existingDef.getName());
304330
}
305331

306-
LOG.debug("<== AtlasBusinessMetadataDefStoreV2.preDeleteByGuid({}): ret={}", guid, ret);
332+
LOG.debug("<== AtlasBusinessMetadataDefStoreV2.preDeleteByGuid({}, {}): ret={}", guid, forceDelete, ret);
307333

308334
return ret;
309335
}
310336

337+
private boolean hasIndexableAttribute(AtlasBusinessMetadataDef bmDef) {
338+
if (bmDef == null || CollectionUtils.isEmpty(bmDef.getAttributeDefs())) {
339+
return false;
340+
}
341+
342+
for (AtlasStructDef.AtlasAttributeDef attributeDef : bmDef.getAttributeDefs()) {
343+
if (attributeDef.getIsIndexable()) {
344+
return true;
345+
}
346+
}
347+
348+
return false;
349+
}
350+
311351
@Override
312352
public void validateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
313353
super.validateType(typeDef);
@@ -392,14 +432,9 @@ private void validateAttributeReferences(AtlasBusinessMetadataDef bmDef, AtlasSt
392432
String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(bmDef, attributeDef.getName());
393433
String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(bmDef, attributeDef, qualifiedName);
394434

395-
boolean isPresent;
396435
long startTime = System.currentTimeMillis();
397436

398-
if (isArrayAttribute(attributeDef)) {
399-
isPresent = isBusinessAttributePresentInGraph(vertexPropertyName, allApplicableTypes);
400-
} else {
401-
isPresent = isBusinessAttributePresent(qualifiedName, allApplicableTypes);
402-
}
437+
boolean isPresent = isBusinessAttributePresentInGraph(vertexPropertyName, allApplicableTypes);
403438

404439
if (LOG.isDebugEnabled()) {
405440
LOG.info("Reference check for attribute {} took {} ms. Found: {}",
@@ -411,44 +446,34 @@ private void validateAttributeReferences(AtlasBusinessMetadataDef bmDef, AtlasSt
411446
}
412447
}
413448

414-
private boolean isBusinessAttributePresent(String attrName, Set<String> applicableTypes) throws AtlasBaseException {
415-
SearchParameters.FilterCriteria criteria = new SearchParameters.FilterCriteria();
416-
criteria.setAttributeName(attrName);
417-
criteria.setOperator(SearchParameters.Operator.NOT_EMPTY);
418-
419-
SearchParameters.FilterCriteria entityFilters = new SearchParameters.FilterCriteria();
420-
entityFilters.setCondition(SearchParameters.FilterCriteria.Condition.OR);
421-
entityFilters.setCriterion(Collections.singletonList(criteria));
422-
423-
SearchParameters searchParameters = new SearchParameters();
424-
searchParameters.setTypeName(String.join(SearchContext.TYPENAME_DELIMITER, applicableTypes));
425-
searchParameters.setExcludeDeletedEntities(false);
426-
searchParameters.setIncludeSubClassifications(false);
427-
searchParameters.setEntityFilters(entityFilters);
428-
searchParameters.setLimit(1);
429-
430-
AtlasSearchResult atlasSearchResult = entityDiscoveryService.searchWithParameters(searchParameters);
431-
return atlasSearchResult != null && CollectionUtils.isNotEmpty(atlasSearchResult.getEntities());
432-
}
433449

434450
private boolean isBusinessAttributePresentInGraph(String vertexPropertyName, Set<String> allApplicableTypes) {
435451
if (graph == null || CollectionUtils.isEmpty(allApplicableTypes)) {
436452
return false;
437453
}
438454

439455
try {
440-
Iterable<AtlasVertex> vertices = graph.query()
456+
List<String> typesList = new ArrayList<>(allApplicableTypes);
457+
458+
// 1. To Check if the BM property exists on a vertex where the direct type matches
459+
Iterable<AtlasVertex> verticesDirect = graph.query()
441460
.has(vertexPropertyName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, (Object) null)
461+
.in(Constants.ENTITY_TYPE_PROPERTY_KEY, typesList)
442462
.vertices();
443463

444-
if (vertices != null) {
445-
Iterator<AtlasVertex> iterator = vertices.iterator();
446-
if (iterator.hasNext()) {
447-
if (LOG.isDebugEnabled()) {
448-
LOG.info("Found ARRAY BM reference for property {}",vertexPropertyName);
449-
}
450-
return true;
451-
}
464+
if (verticesDirect != null && verticesDirect.iterator().hasNext()) {
465+
return true;
466+
}
467+
468+
// 2. To Check if the BM property exists on a vertex where it inherits from one of parent Types
469+
// This is crucial for Case 6 (Parent -> Child)
470+
Iterable<AtlasVertex> verticesInherited = graph.query()
471+
.has(vertexPropertyName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, (Object) null)
472+
.in(Constants.SUPER_TYPES_PROPERTY_KEY, typesList)
473+
.vertices();
474+
475+
if (verticesInherited != null && verticesInherited.iterator().hasNext()) {
476+
return true;
452477
}
453478
} catch (Exception e) {
454479
LOG.error("Error occurred while querying graph for references of property: {}", vertexPropertyName, e);
@@ -457,11 +482,6 @@ private boolean isBusinessAttributePresentInGraph(String vertexPropertyName, Set
457482
return false;
458483
}
459484

460-
private boolean isArrayAttribute(AtlasStructDef.AtlasAttributeDef attrDef) throws AtlasBaseException {
461-
AtlasType type = typeRegistry.getType(attrDef.getTypeName());
462-
return type instanceof AtlasArrayType;
463-
}
464-
465485
private Set<String> getApplicableTypesWithSubTypes(Set<String> applicableTypes) throws AtlasBaseException {
466486
Set<String> allTypes = new HashSet<>();
467487

‎repository/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ AtlasClassificationDef updateClassificationDefByGuid(String guid, AtlasClassific
109109

110110
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
111111

112-
void deleteTypeByName(String typeName) throws AtlasBaseException;
112+
void deleteTypeByName(String typeName, boolean forceDelete) throws AtlasBaseException;
113113

114114
void notifyLoadCompletion();
115115
}

‎repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -369,10 +369,10 @@ public void deleteTypeByName() throws IOException {
369369
AtlasTypesDef typesDef = TestResourceFileUtils.readObjectFromJson(".", hivedbV2Json, AtlasTypesDef.class);
370370

371371
typeDefStore.createTypesDef(typesDef);
372-
typeDefStore.deleteTypeByName(hiveDB2);
373-
typeDefStore.deleteTypeByName(relationshipDefName);
374-
typeDefStore.deleteTypeByName(hostEntityDef);
375-
typeDefStore.deleteTypeByName(clusterEntityDef);
372+
typeDefStore.deleteTypeByName(hiveDB2, false);
373+
typeDefStore.deleteTypeByName(relationshipDefName, false);
374+
typeDefStore.deleteTypeByName(hostEntityDef, false);
375+
typeDefStore.deleteTypeByName(clusterEntityDef, false);
376376
} catch (AtlasBaseException e) {
377377
fail("Deletion should've succeeded");
378378
}

0 commit comments

Comments
 (0)