diff --git a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java
index 56aea37b74..a75f6d64ff 100644
--- a/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java
+++ b/application/src/main/java/org/togetherjava/tjbot/features/rss/RSSHandlerRoutine.java
@@ -2,6 +2,8 @@
import com.apptasticsoftware.rssreader.Item;
import com.apptasticsoftware.rssreader.RssReader;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import net.dv8tion.jda.api.EmbedBuilder;
import net.dv8tion.jda.api.JDA;
import net.dv8tion.jda.api.entities.channel.concrete.TextChannel;
@@ -48,7 +50,7 @@
*
* To include a new RSS feed, simply define an {@link RSSFeed} entry in the {@code "rssFeeds"} array
* within the configuration file, adhering to the format shown below:
- *
+ *
*
* {@code
* {
@@ -58,7 +60,7 @@
* }
* }
*
- *
+ *
* Where:
*
* - {@code url} represents the URL of the RSS feed.
@@ -70,6 +72,9 @@
*/
public final class RSSHandlerRoutine implements Routine {
+ private record FailureState(int count, ZonedDateTime lastFailure) {
+ }
+
private static final Logger logger = LoggerFactory.getLogger(RSSHandlerRoutine.class);
private static final int MAX_CONTENTS = 1000;
private static final ZonedDateTime ZONED_TIME_MIN =
@@ -84,6 +89,14 @@ public final class RSSHandlerRoutine implements Routine {
private final int interval;
private final Database database;
+ private final Cache circuitBreaker =
+ Caffeine.newBuilder().expireAfterWrite(7, TimeUnit.DAYS).maximumSize(500).build();
+
+ private static final int DEAD_RSS_FEED_FAILURE_THRESHOLD = 15;
+ private static final double BACKOFF_BASE = 2.0;
+ private static final double BACKOFF_EXPONENT_OFFSET = 1.0;
+ private static final double MAX_BACKOFF_HOURS = 24.0;
+
/**
* Constructs an RSSHandlerRoutine with the provided configuration and database.
*
@@ -117,7 +130,14 @@ public Schedule createSchedule() {
@Override
public void runRoutine(@Nonnull JDA jda) {
- this.config.feeds().forEach(feed -> sendRSS(jda, feed));
+ this.config.feeds().forEach(feed -> {
+ if (isBackingOff(feed.url())) {
+ logger.debug("Skipping RSS feed (Backing off): {}", feed.url());
+ return;
+ }
+
+ sendRSS(jda, feed);
+ });
}
/**
@@ -257,7 +277,6 @@ private void postItem(List textChannels, Item rssItem, RSSFeed feed
* @param rssFeedRecord the record representing the RSS feed, can be null if not found in the
* database
* @param lastPostedDate the last posted date to be updated
- *
* @throws DateTimeParseException if the date cannot be parsed
*/
private void updateLastDateToDatabase(RSSFeed feedConfig, @Nullable RssFeedRecord rssFeedRecord,
@@ -400,9 +419,26 @@ private MessageCreateData constructMessage(Item item, RSSFeed feedConfig) {
*/
private List- fetchRSSItemsFromURL(String rssUrl) {
try {
- return rssReader.read(rssUrl).toList();
+ List
- items = rssReader.read(rssUrl).toList();
+ circuitBreaker.invalidate(rssUrl);
+ return items;
} catch (IOException e) {
- logger.error("Could not fetch RSS from URL ({})", rssUrl, e);
+ FailureState oldState = circuitBreaker.getIfPresent(rssUrl);
+ int newCount = (oldState == null) ? 1 : oldState.count() + 1;
+
+ if (newCount >= DEAD_RSS_FEED_FAILURE_THRESHOLD) {
+ logger.error(
+ "Possibly dead RSS feed URL: {} - Failed {} times. Please remove it from config.",
+ rssUrl, newCount);
+ }
+ circuitBreaker.put(rssUrl, new FailureState(newCount, ZonedDateTime.now()));
+
+ long blacklistedHours = calculateWaitHours(newCount);
+
+ logger.warn(
+ "RSS fetch failed for {} (Attempt #{}). Backing off for {} hours. Reason: {}",
+ rssUrl, newCount, blacklistedHours, e.getMessage(), e);
+
return List.of();
}
}
@@ -424,4 +460,20 @@ private static ZonedDateTime getZonedDateTime(@Nullable String date, String form
return ZonedDateTime.parse(date, DateTimeFormatter.ofPattern(format));
}
+
+ private long calculateWaitHours(int failureCount) {
+ return (long) Math.min(Math.pow(BACKOFF_BASE, failureCount - BACKOFF_EXPONENT_OFFSET),
+ MAX_BACKOFF_HOURS);
+ }
+
+ private boolean isBackingOff(String url) {
+ FailureState state = circuitBreaker.getIfPresent(url);
+ if (state == null)
+ return false;
+
+ long waitHours = calculateWaitHours(state.count());
+ ZonedDateTime retryAt = state.lastFailure().plusHours(waitHours);
+
+ return ZonedDateTime.now().isBefore(retryAt);
+ }
}