From 5d79b6d397f8a544b4bd750cce9adcaac34bb0a9 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 10 Aug 2026 21:39:30 +0530 Subject: [PATCH 1/3] Fix deletion of redirected URLs --- .../AbstractStatusUpdaterBolt.java | 6 +- .../AbstractStatusUpdaterBoltTest.java | 83 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java index 2fb09e36e..403772eaf 100644 --- a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java @@ -220,8 +220,10 @@ public void execute(Tuple tuple) { metadata.remove(Constants.STATUS_ERROR_CAUSE); metadata.remove(Constants.STATUS_ERROR_MESSAGE); metadata.remove(Constants.STATUS_ERROR_SOURCE); - } else if (status == Status.ERROR) { - // gone? notify any deleters. Doesn't need to be anchored + } + + if (status == Status.ERROR || status == Status.REDIRECTION) { + // gone or redirected? notify any deleters. Doesn't need to be anchored collector.emit(Constants.DELETION_STREAM_NAME, new Values(url, metadata)); } diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java new file mode 100644 index 000000000..6b040738f --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.persistence; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Tuple; +import org.apache.stormcrawler.Constants; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.TestOutputCollector; +import org.apache.stormcrawler.TestUtil; +import org.junit.jupiter.api.Test; + +class AbstractStatusUpdaterBoltTest { + + @Test + void testRedirectedUrlIsEmittedToDeletionStream() { + TestOutputCollector output = new TestOutputCollector(); + TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); + + Map config = new HashMap<>(); + config.put(AbstractStatusUpdaterBolt.useCacheParamName, false); + config.put( + "scheduler.class", + "org.apache.stormcrawler.persistence.DefaultScheduler"); + + bolt.prepare( + config, + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); + + String url = "http://example.com/old-page"; + Metadata metadata = new Metadata(); + + Map tupleValues = new HashMap<>(); + tupleValues.put("url", url); + tupleValues.put("status", Status.REDIRECTION); + tupleValues.put("metadata", metadata); + + Tuple tuple = TestUtil.getMockedTestTuple(tupleValues); + + bolt.execute(tuple); + + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); + + assertEquals(1, deletions.size()); + assertEquals(url, deletions.get(0).get(0)); + assertEquals(metadata, deletions.get(0).get(1)); + } + + private static class TestStatusUpdaterBolt extends AbstractStatusUpdaterBolt { + + @Override + protected void store( + String url, + Status status, + Metadata metadata, + java.util.Optional nextFetch, + Tuple tuple) { + collector.ack(tuple); + } + } +} \ No newline at end of file From 0d3857185c3e497fcb23f75050979d25f15bff37 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 11 Aug 2026 21:22:26 +0530 Subject: [PATCH 2/3] Format redirected URL fix --- .../persistence/AbstractStatusUpdaterBolt.java | 2 +- .../persistence/AbstractStatusUpdaterBoltTest.java | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java index 403772eaf..7231638c2 100644 --- a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java @@ -223,7 +223,7 @@ public void execute(Tuple tuple) { } if (status == Status.ERROR || status == Status.REDIRECTION) { - // gone or redirected? notify any deleters. Doesn't need to be anchored + // gone or redirected? notify any deleters. Doesn't need to be anchored collector.emit(Constants.DELETION_STREAM_NAME, new Values(url, metadata)); } diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java index 6b040738f..71acf8335 100644 --- a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java @@ -39,14 +39,9 @@ void testRedirectedUrlIsEmittedToDeletionStream() { Map config = new HashMap<>(); config.put(AbstractStatusUpdaterBolt.useCacheParamName, false); - config.put( - "scheduler.class", - "org.apache.stormcrawler.persistence.DefaultScheduler"); + config.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler"); - bolt.prepare( - config, - TestUtil.getMockedTopologyContext(), - new OutputCollector(output)); + bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); String url = "http://example.com/old-page"; Metadata metadata = new Metadata(); @@ -60,8 +55,7 @@ void testRedirectedUrlIsEmittedToDeletionStream() { bolt.execute(tuple); - List> deletions = - output.getEmitted(Constants.DELETION_STREAM_NAME); + List> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME); assertEquals(1, deletions.size()); assertEquals(url, deletions.get(0).get(0)); @@ -80,4 +74,4 @@ protected void store( collector.ack(tuple); } } -} \ No newline at end of file +} From 5b9de453d435f9f5673ad0debf3f594a4259c553 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2026 22:37:38 +0530 Subject: [PATCH 3/3] Handle permanent redirects in deletion stream --- .../AbstractStatusUpdaterBolt.java | 17 +- .../AbstractStatusUpdaterBoltTest.java | 173 ++++++++++++++++-- 2 files changed, 175 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java index 7231638c2..6579bcda9 100644 --- a/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java @@ -222,9 +222,22 @@ public void execute(Tuple tuple) { metadata.remove(Constants.STATUS_ERROR_SOURCE); } - if (status == Status.ERROR || status == Status.REDIRECTION) { - // gone or redirected? notify any deleters. Doesn't need to be anchored + if (status == Status.ERROR) { + // gone? notify any deleters. Doesn't need to be anchored collector.emit(Constants.DELETION_STREAM_NAME, new Values(url, metadata)); + } else if (status == Status.REDIRECTION) { + String statusCode = metadata.getFirstValue("fetch.statusCode"); + String redirection = metadata.getFirstValue("_redirTo"); + + // Delete permanently redirected URLs (301/308) and meta-refresh redirects. + boolean permanentRedirect = + "301".equals(statusCode) || "308".equals(statusCode); + boolean metaRefreshRedirect = + "200".equals(statusCode) && redirection != null; + + if (permanentRedirect || metaRefreshRedirect) { + collector.emit(Constants.DELETION_STREAM_NAME, new Values(url, metadata)); + } } // determine the value of the next fetch based on the status diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java index 71acf8335..7f00a4dac 100644 --- a/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java @@ -33,33 +33,180 @@ class AbstractStatusUpdaterBoltTest { @Test - void testRedirectedUrlIsEmittedToDeletionStream() { + void testPermanentRedirect301IsEmittedToDeletionStream() { TestOutputCollector output = new TestOutputCollector(); TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); - Map config = new HashMap<>(); - config.put(AbstractStatusUpdaterBolt.useCacheParamName, false); - config.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler"); + bolt.prepare( + createConfig(), + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); + + String url = "http://example.com/old-page"; + Metadata metadata = new Metadata(); + metadata.setValue("fetch.statusCode", "301"); + + Tuple tuple = createTuple(url, Status.REDIRECTION, metadata); - bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + bolt.execute(tuple); + + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); + + assertEquals(1, deletions.size()); + assertEquals(url, deletions.get(0).get(0)); + + Metadata emittedMetadata = (Metadata) deletions.get(0).get(1); + assertEquals("301", emittedMetadata.getFirstValue("fetch.statusCode")); + } + + @Test + void testPermanentRedirect308IsEmittedToDeletionStream() { + TestOutputCollector output = new TestOutputCollector(); + TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); + + bolt.prepare( + createConfig(), + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); String url = "http://example.com/old-page"; Metadata metadata = new Metadata(); + metadata.setValue("fetch.statusCode", "308"); - Map tupleValues = new HashMap<>(); - tupleValues.put("url", url); - tupleValues.put("status", Status.REDIRECTION); - tupleValues.put("metadata", metadata); + Tuple tuple = createTuple(url, Status.REDIRECTION, metadata); + + bolt.execute(tuple); + + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); + + assertEquals(1, deletions.size()); + assertEquals(url, deletions.get(0).get(0)); + + Metadata emittedMetadata = (Metadata) deletions.get(0).get(1); + assertEquals("308", emittedMetadata.getFirstValue("fetch.statusCode")); + } + + @Test + void testTemporaryRedirect302IsNotEmittedToDeletionStream() { + TestOutputCollector output = new TestOutputCollector(); + TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); + + bolt.prepare( + createConfig(), + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); + + String url = "http://example.com/old-page"; + Metadata metadata = new Metadata(); + metadata.setValue("fetch.statusCode", "302"); + + Tuple tuple = createTuple(url, Status.REDIRECTION, metadata); + + bolt.execute(tuple); + + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); + + assertEquals(0, deletions.size()); + } + + @Test + void testMetaRefreshRedirectIsEmittedToDeletionStream() { + TestOutputCollector output = new TestOutputCollector(); + TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); + + bolt.prepare( + createConfig(), + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); + + String url = "http://example.com/old-page"; + Metadata metadata = new Metadata(); + metadata.setValue("fetch.statusCode", "200"); + metadata.setValue("_redirTo", "http://example.com/new-page"); - Tuple tuple = TestUtil.getMockedTestTuple(tupleValues); + Tuple tuple = createTuple(url, Status.REDIRECTION, metadata); bolt.execute(tuple); - List> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME); + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); assertEquals(1, deletions.size()); assertEquals(url, deletions.get(0).get(0)); - assertEquals(metadata, deletions.get(0).get(1)); + + Metadata emittedMetadata = (Metadata) deletions.get(0).get(1); + assertEquals( + "http://example.com/new-page", + emittedMetadata.getFirstValue("_redirTo")); + } + + @Test + void testFetchedUrlIsNotEmittedToDeletionStream() { + TestOutputCollector output = new TestOutputCollector(); + TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); + + bolt.prepare( + createConfig(), + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); + + String url = "http://example.com/page"; + Metadata metadata = new Metadata(); + metadata.setValue("fetch.statusCode", "200"); + + Tuple tuple = createTuple(url, Status.FETCHED, metadata); + + bolt.execute(tuple); + + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); + + assertEquals(0, deletions.size()); + } + + @Test + void testErrorIsEmittedToDeletionStream() { + TestOutputCollector output = new TestOutputCollector(); + TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt(); + + bolt.prepare( + createConfig(), + TestUtil.getMockedTopologyContext(), + new OutputCollector(output)); + + String url = "http://example.com/error"; + Metadata metadata = new Metadata(); + + Tuple tuple = createTuple(url, Status.ERROR, metadata); + + bolt.execute(tuple); + + List> deletions = + output.getEmitted(Constants.DELETION_STREAM_NAME); + + assertEquals(1, deletions.size()); + assertEquals(url, deletions.get(0).get(0)); + } + + private static Map createConfig() { + Map config = new HashMap<>(); + config.put(AbstractStatusUpdaterBolt.useCacheParamName, false); + config.put( + "scheduler.class", + "org.apache.stormcrawler.persistence.DefaultScheduler"); + return config; + } + + private static Tuple createTuple(String url, Status status, Metadata metadata) { + Map tupleValues = new HashMap<>(); + tupleValues.put("url", url); + tupleValues.put("status", status); + tupleValues.put("metadata", metadata); + + return TestUtil.getMockedTestTuple(tupleValues); } private static class TestStatusUpdaterBolt extends AbstractStatusUpdaterBolt { @@ -74,4 +221,4 @@ protected void store( collector.ack(tuple); } } -} +} \ No newline at end of file