Skip to content

NovelFire fix #1899

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nyagami merged 1 commit into from
Dec 4, 2025
Merged

NovelFire fix #1899

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 57 additions & 122 deletions plugins/english/novelfire.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ import { fetchApi } from '@libs/fetch';
import { Plugin } from '@/types/plugin';
import { NovelStatus } from '@libs/novelStatus';
import { Filters, FilterTypes } from '@libs/filterInputs';
import { defaultCover } from '@/types/constants';

class NovelFire implements Plugin.PluginBase {
class NovelFire implements Plugin.PagePlugin {
id = 'novelfire';
name = 'Novel Fire';
version = '1.0.4';
version = '1.1.0';
icon = 'src/en/novelfire/icon.png';
site = 'https://novelfire.net/';

Expand Down Expand Up @@ -57,7 +58,7 @@ class NovelFire implements Plugin.PluginBase {
return loadedCheerio('.novel-item')
.map((index, ele) => {
const novelName =
loadedCheerio(ele).find('.novel-title > a').attr('title') ||
loadedCheerio(ele).find('.novel-title > a').text() ||
'No Title Found';
const novelCover = loadedCheerio(ele)
.find('.novel-cover > img')
Expand All @@ -66,7 +67,7 @@ class NovelFire implements Plugin.PluginBase {
.find('.novel-title > a')
.attr('href');

if (!novelPath) return null;
if (!novelPath) return;

return {
name: novelName,
Expand All @@ -78,100 +79,13 @@ class NovelFire implements Plugin.PluginBase {
.filter(novel => novel !== null);
}

async parseChapters(
async parseNovel(
novelPath: string,
pages: number,
): Promise<Plugin.ChapterItem[]> {
const pagesArray = Array.from({ length: pages }, (_, i) => i + 1);
const allChapters: Plugin.ChapterItem[] = [];

// Function to parse a single page
const parsePage = async (page: number) => {
const url = `${this.site}${novelPath}/chapters?page=${page}`;
const result = await fetchApi(url);
const body = await result.text();

const loadedCheerio = load(body);

if (loadedCheerio.text().includes('You are being rate limited')) {
throw new NovelFireThrottlingError();
}

const chapters = loadedCheerio('.chapter-list li')
.map((index, ele) => {
const chapterName =
loadedCheerio(ele).find('a').attr('title') || 'No Title Found';
const chapterPath = loadedCheerio(ele).find('a').attr('href');

if (!chapterPath) return null;

return {
name: chapterName,
path: chapterPath.replace(this.site, ''),
};
})
.get()
.filter(chapter => chapter !== null) as Plugin.ChapterItem[];

return chapters;
};

// When pages > ~30, we get rate limited. To mitigate, split into chunks and retry chunk on rate limit with delay.
const chunkSize = 5; // 5 pages per chunk was tested to be a good balance between speed and rate limiting.
const retryCount = 10;
const sleepTime = 3.5; // Rate limit seems to be around ~10s, so usually 3 retries should be enough for another ~30 pages.

const chaptersArray: Plugin.ChapterItem[][] = [];

for (let i = 0; i < pagesArray.length; i += chunkSize) {
const pagesArrayChunk = pagesArray.slice(i, i + chunkSize);

const firstPage = pagesArrayChunk[0];
const lastPage = pagesArrayChunk[pagesArrayChunk.length - 1];

let attempt = 0;

while (attempt < retryCount) {
try {
// Parse all pages in chunk in parallel
const chaptersArrayChunk = await Promise.all(
pagesArrayChunk.map(parsePage),
);

chaptersArray.push(...chaptersArrayChunk);
break;
} catch (err) {
if (err instanceof NovelFireThrottlingError) {
attempt += 1;
console.warn(
`[pages=${firstPage}-${lastPage}] Novel Fire is rate limiting requests. Retry attempt ${attempt + 1} in ${sleepTime} seconds...`,
);
if (attempt === retryCount) {
throw err;
}

// Sleep for X second before retrying
await new Promise(resolve => setTimeout(resolve, sleepTime * 1000));
} else {
throw err;
}
}
}
}

// Merge all chapters into a single array
for (const chapters of chaptersArray) {
allChapters.push(...chapters);
}

return allChapters.length === 0 ? [] : allChapters;
}

async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
): Promise<Plugin.SourceNovel & { totalPages: number }> {
const $ = await this.getCheerio(this.site + novelPath, false);
const baseUrl = this.site;

const novel: Partial<Plugin.SourceNovel> = {
const novel: Partial<Plugin.SourceNovel & { totalPages: number }> = {
path: novelPath,
};

Expand All @@ -184,6 +98,8 @@ class NovelFire implements Plugin.PluginBase {

if (coverUrl) {
novel.cover = new URL(coverUrl, baseUrl).href;
} else {
novel.cover = defaultCover;
}

novel.genres = $('.categories .property-item')
Expand All @@ -200,8 +116,7 @@ class NovelFire implements Plugin.PluginBase {
novel.summary = 'No Summary Found';
}

novel.author =
$('.author .property-item > span').text() || 'No Author Found';
novel.author = $('.author .property-item > span').text();

const rawStatus =
$('.header-stats .ongoing').text() ||
Expand All @@ -217,41 +132,55 @@ class NovelFire implements Plugin.PluginBase {
};
novel.status = map[rawStatus.toLowerCase()] ?? NovelStatus.Unknown;

novel.rating = parseFloat($('.nub').text().trim());

const totalChapters = $('.header-stats .icon-book-open')
.parent()
.text()
.trim();
const pages = Math.ceil(parseInt(totalChapters) / 100);
novel.chapters = await this.parseChapters(novelPath, pages);

return novel as Plugin.SourceNovel;
novel.totalPages = Math.ceil(parseInt(totalChapters) / 100);

return novel as Plugin.SourceNovel & { totalPages: number };
}

async parseChapter(chapterPath: string): Promise<string> {
const url = this.site + chapterPath;
async parsePage(novelPath: string, page: string): Promise<Plugin.SourcePage> {
const url = `${this.site}${novelPath}/chapters?page=${page}`;
const result = await fetchApi(url);
const body = await result.text();

const loadedCheerio = load(body);

const bloatElements = [
'.box-ads',
'.box-notification',
/^nf/, // Regular expression to match tags starting with 'nf'
];
bloatElements.forEach(tag => {
if (tag instanceof RegExp) {
loadedCheerio('*')
.filter((_, el) =>
tag.test(loadedCheerio(el).prop('tagName')!.toLowerCase()),
)
.remove();
} else {
loadedCheerio(tag).remove();
}
});
const chapters = loadedCheerio('.chapter-list li')
.map((index, ele) => {
const chapterName =
loadedCheerio(ele).find('a').attr('title') || 'No Title Found';
const chapterPath = loadedCheerio(ele).find('a').attr('href');

if (!chapterPath) return null;

return loadedCheerio('#content').html()!;
return {
name: chapterName,
path: chapterPath.replace(this.site, ''),
};
})
.get()
.filter(chapter => chapter !== null) as Plugin.ChapterItem[];

return {
chapters,
};
}

async parseChapter(chapterPath: string): Promise<string> {
const url = this.site + chapterPath;
const loadedCheerio = await this.getCheerio(url, false);

const chapterText = loadedCheerio('#content');

loadedCheerio(chapterText).find('div').remove();

return chapterText.html() || '';
}

async searchNovels(
Expand Down Expand Up @@ -292,8 +221,12 @@ class NovelFire implements Plugin.PluginBase {
options: [
{ label: 'Rank (Top)', value: 'rank-top' },
{ label: 'Rating Score (Top)', value: 'rating-score-top' },
{ label: 'Bookmark Count (Most)', value: 'bookmark' },
{ label: 'Review Count (Most)', value: 'review' },
{ label: 'Comment Count (Most)', value: 'comment' },
{ label: 'Bookmark Count (Most)', value: 'bookmark' },
{ label: 'Today Views (Most)', value: 'today-view' },
{ label: 'Monthly Views (Most)', value: 'monthly-view' },
{ label: 'Total Views (Most)', value: 'total-view' },
{ label: 'Title (A>Z)', value: 'abc' },
{ label: 'Title (Z>A)', value: 'cba' },
{ label: 'Last Updated (Newest)', value: 'date' },
Expand All @@ -312,11 +245,12 @@ class NovelFire implements Plugin.PluginBase {
type: FilterTypes.Picker,
},
genre_operator: {
label: 'Genres (And/Or)',
label: 'Genres (And/Or/Exclude)',
value: 'and',
options: [
{ label: 'And', value: 'and' },
{ label: 'Or', value: 'or' },
{ label: 'AND', value: 'and' },
{ label: 'OR', value: 'or' },
{ label: 'EXCLUDE', value: 'exclude' },
],
type: FilterTypes.Picker,
},
Expand Down Expand Up @@ -353,6 +287,7 @@ class NovelFire implements Plugin.PluginBase {
{ label: 'Modern Life', value: '54' },
{ label: 'Movies', value: '55' },
{ label: 'Mystery', value: '16' },
{ label: 'Other', value: '64' },
{ label: 'Psychological', value: '9' },
{ label: 'Realistic Fiction', value: '56' },
{ label: 'Reincarnation', value: '43' },
Expand Down