diff --git a/src/datasource/catmaid/spatial_skeleton_commands.ts b/src/datasource/catmaid/spatial_skeleton_commands.ts
index de45323945..f9af10437e 100644
--- a/src/datasource/catmaid/spatial_skeleton_commands.ts
+++ b/src/datasource/catmaid/spatial_skeleton_commands.ts
@@ -846,6 +846,9 @@ function applyDeleteNodeToCache(
   if (remainingSegmentNodes.length === 0) {
     removeVisibleSegment(layer, node.segmentId, { deselect: true });
   }
+  layer
+    .getSpatiallyIndexedSkeletonLayer()
+    ?.removeRetainedOverlaySegmentIfEmpty(node.segmentId);
   layer.markSpatialSkeletonNodeDataChanged({
     invalidateFullSkeletonCache: false,
   });
diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts
index 28b7df3a29..e4b6fda2a7 100644
--- a/src/layer/segmentation/index.ts
+++ b/src/layer/segmentation/index.ts
@@ -1007,6 +1007,8 @@ export class SegmentationUserLayer extends Base {
   readonly spatialSkeletonState = this.registerDisposer(
     new SpatialSkeletonState(),
   );
+  private _currentSpatialSkeletonSource:
+    SpatiallyIndexedSkeletonSource | undefined = undefined;
   readonly selectedSpatialSkeletonNodeId = new WatchableValue<
     number | undefined
   >(undefined);
@@ -1563,12 +1565,13 @@ export class SegmentationUserLayer extends Base {
         break;
       }
     }
-    if (!hasSpatialSkeletonLayer) {
+    const newSource = this.getSpatialSkeletonCommandHistorySource();
+    const sourceChanged = newSource !== this._currentSpatialSkeletonSource;
+    this._currentSpatialSkeletonSource = newSource;
+    if (!hasSpatialSkeletonLayer || sourceChanged) {
       this.spatialSkeletonState.clearInspectedSkeletonCache();
     }
-    this.spatialSkeletonState.updateCommandHistorySource(
-      this.getSpatialSkeletonCommandHistorySource(),
-    );
+    this.spatialSkeletonState.updateCommandHistorySource(newSource);
   }
 
   private getSpatialSkeletonCommandHistorySource() {
diff --git a/src/skeleton/frontend.ts b/src/skeleton/frontend.ts
index 4867bf1a56..f549d3df24 100644
--- a/src/skeleton/frontend.ts
+++ b/src/skeleton/frontend.ts
@@ -2222,6 +2222,17 @@ export class SpatiallyIndexedSkeletonLayer
     return true;
   }
 
+  removeRetainedOverlaySegmentIfEmpty(segmentId: number) {
+    const nodes = this.inspectionState?.getCachedSegmentNodes(segmentId);
+    if (nodes === undefined || nodes.length > 0) return;
+    const next = this.retainedOverlaySegmentIds.filter(
+      (id) => id !== segmentId,
+    );
+    if (next.length === this.retainedOverlaySegmentIds.length) return;
+    this.retainedOverlaySegmentIds = next;
+    this.redrawNeeded.dispatch();
+  }
+
   suppressBrowseSegment(segmentId: number) {
     const normalizedSegmentId = Math.round(Number(segmentId));
     if (
diff --git a/src/skeleton/segment_overlay.ts b/src/skeleton/segment_overlay.ts
index 2d07645de3..91cb56c6c8 100644
--- a/src/skeleton/segment_overlay.ts
+++ b/src/skeleton/segment_overlay.ts
@@ -144,7 +144,7 @@ export function buildSpatiallyIndexedSkeletonOverlayGeometry(
   };
 }
 
-export const DEFAULT_MAX_RETAINED_OVERLAY_SEGMENTS = 4;
+export const DEFAULT_MAX_RETAINED_OVERLAY_SEGMENTS = 16;
 
 function normalizeSegmentId(segmentId: number) {
   const normalizedSegmentId = Math.round(Number(segmentId));