From 15d9652e85379f5857f63cab6b35960e5c504b83 Mon Sep 17 00:00:00 2001 From: John Niang Date: Thu, 13 Mar 2025 17:38:59 +0800 Subject: [PATCH] Fix the pending problem when requesting RSS endpoint --- app/build.gradle | 3 +- .../java/run/halo/feed/RssCacheManager.java | 22 +++--- .../java/run/halo/feed/RssXmlBuilder.java | 68 ++++++++++++++----- .../provider/AbstractPostRssProvider.java | 6 +- app/src/test/java/run/halo/feed/RSS2Test.java | 14 +++- .../java/run/halo/feed/RssXmlBuilderTest.java | 28 ++++++++ app/src/test/resources/logback.xml | 9 +++ 7 files changed, 113 insertions(+), 37 deletions(-) create mode 100644 app/src/test/java/run/halo/feed/RssXmlBuilderTest.java create mode 100644 app/src/test/resources/logback.xml diff --git a/app/build.gradle b/app/build.gradle index 3f9662c..aab275c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -15,7 +15,7 @@ jar { dependencies { implementation project(':api') compileOnly 'run.halo.app:api' - implementation 'org.dom4j:dom4j:2.1.3' + implementation 'org.dom4j:dom4j:2.1.4' implementation('org.apache.commons:commons-text:1.10.0') { // Because the transitive dependency already exists in halo. exclude group: 'org.apache.commons', module: 'commons-lang3' @@ -23,6 +23,7 @@ dependencies { testImplementation 'run.halo.app:api' testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'io.projectreactor:reactor-test' } tasks.register('copyUI', Copy) { diff --git a/app/src/main/java/run/halo/feed/RssCacheManager.java b/app/src/main/java/run/halo/feed/RssCacheManager.java index bd02623..f672e68 100644 --- a/app/src/main/java/run/halo/feed/RssCacheManager.java +++ b/app/src/main/java/run/halo/feed/RssCacheManager.java @@ -3,6 +3,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import java.time.Duration; +import java.util.concurrent.ExecutionException; import java.util.function.Predicate; import lombok.RequiredArgsConstructor; import org.springframework.context.event.EventListener; @@ -18,7 +19,7 @@ @RequiredArgsConstructor public class RssCacheManager { private final Duration expireMinutes = Duration.ofMinutes(60); - private final Cache cache = CacheBuilder.newBuilder() + private final Cache> cache = CacheBuilder.newBuilder() .expireAfterWrite(expireMinutes) .build(); @@ -26,13 +27,12 @@ public class RssCacheManager { private final ReactiveSettingFetcher settingFetcher; public Mono get(String requestPath, Mono loader) { - return Mono.fromCallable(() -> cache.get(requestPath, - () -> generateRssXml(requestPath, loader) - .doOnNext(xml -> cache.put(requestPath, xml)) - .block() - )) - .cache() - .subscribeOn(Schedulers.boundedElastic()); + try { + return cache.get(requestPath, () -> generateRssXml(requestPath, loader).cache()) + .subscribeOn(Schedulers.boundedElastic()); + } catch (ExecutionException e) { + return Mono.error(e); + } } private Mono generateRssXml(String requestPath, Mono loader) { @@ -51,10 +51,8 @@ private Mono generateRssXml(String requestPath, Mono loader) { .doOnNext(prop -> builder.withExtractRssTags(prop.getRssExtraTags())); return Mono.when(rssMono, generatorMono, extractTagsMono) - // toXmlString is a blocking operation - .then(Mono.fromCallable(builder::toXmlString) - .subscribeOn(Schedulers.boundedElastic()) - ); + .thenReturn(builder) + .flatMap(RssXmlBuilder::toXmlString); } @EventListener(PluginConfigUpdatedEvent.class) diff --git a/app/src/main/java/run/halo/feed/RssXmlBuilder.java b/app/src/main/java/run/halo/feed/RssXmlBuilder.java index 126d6e6..022fc8e 100644 --- a/app/src/main/java/run/halo/feed/RssXmlBuilder.java +++ b/app/src/main/java/run/halo/feed/RssXmlBuilder.java @@ -2,6 +2,7 @@ import com.google.common.base.Throwables; import java.io.StringReader; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; @@ -20,10 +21,14 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; import org.springframework.util.CollectionUtils; import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientException; import org.springframework.web.util.UriComponentsBuilder; import org.springframework.web.util.UriUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; import reactor.netty.http.client.HttpClient; import run.halo.feed.telemetry.TelemetryEndpoint; @@ -82,7 +87,7 @@ RssXmlBuilder withExternalUrl(String externalUrl) { return this; } - public String toXmlString() { + public Mono toXmlString() { Document document = DocumentHelper.createDocument(); Element root = DocumentHelper.createElement("rss"); @@ -136,9 +141,10 @@ public String toXmlString() { } var items = rss2.getItems(); - createItemElementsToChannel(channel, items); - return document.asXML(); + return createItemElementsToChannel(channel, items) + .thenReturn(document) + .map(Document::asXML); } private void copyAttributesAndChildren(Element target, Element source) { @@ -171,14 +177,16 @@ private Element parseXmlString(String xml) throws DocumentException { } } - private void createItemElementsToChannel(Element channel, List items) { + private Mono createItemElementsToChannel(Element channel, List items) { if (CollectionUtils.isEmpty(items)) { - return; + return Mono.empty(); } - items.forEach(item -> createItemElementToChannel(channel, item)); + return Flux.fromIterable(items) + .flatMap(item -> createItemElementToChannel(channel, item)) + .then(); } - private void createItemElementToChannel(Element channel, RSS2.Item item) { + private Mono createItemElementToChannel(Element channel, RSS2.Item item) { Element itemElement = channel.addElement("item"); itemElement.addElement("title") .addCDATA(XmlCharUtils.removeInvalidXmlChar(item.getTitle())); @@ -205,18 +213,19 @@ private void createItemElementToChannel(Element channel, RSS2.Item item) { .addText(item.getAuthor()); } + Mono handleEnclosure = Mono.empty(); if (StringUtils.isNotBlank(item.getEnclosureUrl())) { var enclosureElement = itemElement.addElement("enclosure") .addAttribute("url", item.getEnclosureUrl()) .addAttribute("type", item.getEnclosureType()); - var enclosureLength = item.getEnclosureLength(); + enclosureElement.addAttribute("length", enclosureLength); if (StringUtils.isBlank(enclosureLength)) { // https://www.rssboard.org/rss-validator/docs/error/MissingAttribute.html - var fileBytes = getFileSizeBytes(item.getEnclosureUrl()); - enclosureLength = String.valueOf(fileBytes); + handleEnclosure = getFileSizeBytes(item.getEnclosureUrl()) + .doOnNext(fileBytes -> enclosureElement.addAttribute("length", String.valueOf(fileBytes))) + .then(); } - enclosureElement.addAttribute("length", enclosureLength); } nullSafeList(item.getCategories()) @@ -264,6 +273,7 @@ private void createItemElementToChannel(Element channel, RSS2.Item item) { } } }); + return handleEnclosure; } private String getDescriptionWithTelemetry(RSS2.Item item) { @@ -298,23 +308,47 @@ static String instantToString(Instant instant) { .format(DateTimeFormatter.RFC_1123_DATE_TIME); } + @NonNull - private Long getFileSizeBytes(String url) { + Mono getFileSizeBytes(String url) { return webClient.get() - .uri(url) + .uri(URI.create(url)) .header(HttpHeaders.USER_AGENT, UA) // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range .header(HttpHeaders.RANGE, "bytes=0-0") + .headers(headers -> { + if (StringUtils.isNotBlank(externalUrl)) { + // For Referrer anti-hotlinking + headers.set(HttpHeaders.REFERER, externalUrl); + } + }) .retrieve() .toBodilessEntity() .map(HttpEntity::getHeaders) - .mapNotNull(headers -> headers.getFirst(HttpHeaders.CONTENT_LENGTH)) - .map(Long::parseLong) + .mapNotNull(headers -> headers.getFirst(HttpHeaders.CONTENT_RANGE)) + .mapNotNull(RssXmlBuilder::parseLengthOfContentRange) .doOnError(e -> log.debug("Failed to get file size from url: {}", url, Throwables.getRootCause(e)) ) .onErrorReturn(0L) - .blockOptional() - .orElse(0L); + .defaultIfEmpty(0L); } + + @Nullable + static Long parseLengthOfContentRange(@Nullable String contentRange) { + if (contentRange == null) { + return null; + } + // Refer to https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range#syntax + var range = contentRange.split("/"); + if (range.length != 2) { + return null; + } + var lengthStr = range[1]; + if ("*".equals(lengthStr)) { + return null; + } + return Long.parseLong(range[1]); + } + } diff --git a/app/src/main/java/run/halo/feed/provider/AbstractPostRssProvider.java b/app/src/main/java/run/halo/feed/provider/AbstractPostRssProvider.java index 79698f2..30c869b 100644 --- a/app/src/main/java/run/halo/feed/provider/AbstractPostRssProvider.java +++ b/app/src/main/java/run/halo/feed/provider/AbstractPostRssProvider.java @@ -126,8 +126,7 @@ protected Flux listPostsByFunc(Function> protected Mono fetchUser(String username) { return client.fetch(User.class, username) - .switchIfEmpty(Mono.error(new ServerWebInputException("User not found"))) - .subscribeOn(Schedulers.boundedElastic()); + .switchIfEmpty(Mono.error(new ServerWebInputException("User not found"))); } private Flux fetchCategoryDisplayName(List categoryNames) { @@ -137,8 +136,7 @@ private Flux fetchCategoryDisplayName(List categoryNames) { return client.listAll(Category.class, ListOptions.builder() .fieldQuery(QueryFactory.in("metadata.name", categoryNames)) .build(), ExtensionUtil.defaultSort()) - .map(category -> category.getSpec().getDisplayName()) - .subscribeOn(Schedulers.boundedElastic()); + .map(category -> category.getSpec().getDisplayName()); } @Data diff --git a/app/src/test/java/run/halo/feed/RSS2Test.java b/app/src/test/java/run/halo/feed/RSS2Test.java index da2330c..75c1570 100644 --- a/app/src/test/java/run/halo/feed/RSS2Test.java +++ b/app/src/test/java/run/halo/feed/RSS2Test.java @@ -6,6 +6,7 @@ import java.util.Arrays; import java.util.Collections; import org.junit.jupiter.api.Test; +import reactor.test.StepVerifier; class RSS2Test { @@ -80,7 +81,10 @@ void toXmlString() { """.formatted(lastBuildDate); - assertThat(rssXml).isEqualToIgnoringWhitespace(expected); + + StepVerifier.create(rssXml) + .assertNext(xml -> assertThat(xml).isEqualToIgnoringWhitespace(expected)) + .verifyComplete(); } @Test @@ -132,7 +136,9 @@ void extractRssTagsTest() { """.formatted(lastBuildDate); - assertThat(rssXml).isEqualToIgnoringWhitespace(expected); + StepVerifier.create(rssXml) + .assertNext(xml -> assertThat(xml).isEqualToIgnoringWhitespace(expected)) + .verifyComplete(); } @Test @@ -188,6 +194,8 @@ void invalidCharTest() { """.formatted(lastBuildDate); - assertThat(rssXml).isEqualToIgnoringWhitespace(expected); + StepVerifier.create(rssXml) + .assertNext(xml -> assertThat(xml).isEqualToIgnoringWhitespace(expected)) + .verifyComplete(); } } \ No newline at end of file diff --git a/app/src/test/java/run/halo/feed/RssXmlBuilderTest.java b/app/src/test/java/run/halo/feed/RssXmlBuilderTest.java new file mode 100644 index 0000000..c066feb --- /dev/null +++ b/app/src/test/java/run/halo/feed/RssXmlBuilderTest.java @@ -0,0 +1,28 @@ +package run.halo.feed; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static run.halo.feed.RssXmlBuilder.parseLengthOfContentRange; + +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class RssXmlBuilderTest { + + static Stream testParseLengthOfContentRange() { + return Stream.of( + arguments("bytes 0-0/123", 123L), + arguments("bytes */567", 567L), + arguments("bytes 0-0", null), + arguments("bytes 0-0/*", null) + ); + } + + @ParameterizedTest + @MethodSource + void testParseLengthOfContentRange(String contentRange, Long expected) { + assertEquals(expected, parseLengthOfContentRange(contentRange)); + } +} \ No newline at end of file diff --git a/app/src/test/resources/logback.xml b/app/src/test/resources/logback.xml new file mode 100644 index 0000000..1f21ed2 --- /dev/null +++ b/app/src/test/resources/logback.xml @@ -0,0 +1,9 @@ + + + + + + + + +