feat: 新增角色下载脚本，取代此前浏览器手动下载逻辑
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const cheerio = require('cheerio');
|
||||
const {
|
||||
cleanImageUrl,
|
||||
generateImageName,
|
||||
toOutputBaseName,
|
||||
buildParseApiUrl,
|
||||
resolveImageUrl,
|
||||
getFileBaseNameFromUrl,
|
||||
listLocalCharacters,
|
||||
sleep,
|
||||
} = require('./utils');
|
||||
|
||||
/**
|
||||
* @typedef {Object} WikiConfig
|
||||
* @property {string} name 站点标识，用于日志和 CLI 筛选
|
||||
* @property {string} [baseUrl] Fandom 站点地
|
||||
* @property {string} [url] 完整 parse API 地址（与 baseUrl + wikiTitle 二选一）
|
||||
* @property {string} [wikiTitle] Wiki 页面标题
|
||||
* @property {string} savePath 图片保存目录
|
||||
* @property {string} imageSelector 图片 CSS 选择器
|
||||
* @property {string} [imageUrlAttr='src'] 图片 URL 所在属性，如 src、data-src
|
||||
* @property {(name: string) => string} [nameTransform] 文件名二次处理
|
||||
* @property {number} [downloadDelayMs] 下载间隔，默认 500ms
|
||||
* @property {string} [verifyApiUrl] 摄影集页面 API，配置后自动验证服务器已录入角色
|
||||
*/
|
||||
|
||||
/** @type {WikiConfig[]} */
|
||||
const wikiConfigs = [
|
||||
{
|
||||
name: 'nte',
|
||||
baseUrl: 'https://neverness-to-everness.fandom.com/wiki/Character/List',
|
||||
url: 'https://neverness-to-everness.fandom.com/api.php?action=parse&page=Character/List&prop=text&format=json',
|
||||
wikiTitle: 'Character/List',
|
||||
savePath: path.join(__dirname, 'output', 'nte'),
|
||||
imageSelector: 'table.fandom-table img[data-relevant="1"]',
|
||||
imageUrlAttr: 'data-src',
|
||||
nameTransform: (name) => name.replace(/-icon$/, ''),
|
||||
verifyApiUrl: 'https://paul.ren/api/page/get?slug=photo-collection-nte',
|
||||
},
|
||||
{
|
||||
name: 'ww',
|
||||
baseUrl: 'https://wutheringwaves.fandom.com',
|
||||
wikiTitle: 'Resonator/List',
|
||||
savePath: path.join(__dirname, 'output', 'wuthering-waves'),
|
||||
imageSelector: 'table.article-table img[data-relevant="1"]',
|
||||
nameTransform: (name) => name.replace(/^resonator-/, ''),
|
||||
verifyApiUrl: 'https://paul.ren/api/page/get?slug=photo-collection-ww',
|
||||
},
|
||||
// {
|
||||
// name: 'blue-archive',
|
||||
// baseUrl: 'https://bluearchive.fandom.com',
|
||||
// wikiTitle: 'Category:Students',
|
||||
// savePath: path.join(__dirname, 'output', 'blue-archive'),
|
||||
// imageSelector: 'table.article-table td:first-child img',
|
||||
// nameTransform: (name) => name.replace(/-icon$/, ''),
|
||||
// },
|
||||
];
|
||||
|
||||
function resolveApiUrl(config) {
|
||||
if (config.url) {
|
||||
return config.url;
|
||||
}
|
||||
if (config.baseUrl && config.wikiTitle) {
|
||||
return buildParseApiUrl(config.baseUrl, config.wikiTitle);
|
||||
}
|
||||
throw new Error(`[${config.name}] 需要提供 url，或同时提供 baseUrl 与 wikiTitle`);
|
||||
}
|
||||
|
||||
async function fetchWikiHtml(config) {
|
||||
const apiUrl = resolveApiUrl(config);
|
||||
console.log('apiUrl', apiUrl);
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
headers: {
|
||||
'user-agent': 'Mozilla/5.0 (compatible; download-character-img/1.0)',
|
||||
accept: 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`[${config.name}] 请求失败 ${response.status}: ${apiUrl}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const html = data?.parse?.text?.['*'];
|
||||
|
||||
if (!html) {
|
||||
throw new Error(`[${config.name}] API 响应中未找到页面 HTML`);
|
||||
}
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
function extractImages(html, imageSelector, imageUrlAttr = 'src') {
|
||||
const $ = cheerio.load(html);
|
||||
const results = [];
|
||||
const seenNames = new Set();
|
||||
|
||||
$(imageSelector).each((_, element) => {
|
||||
const $img = $(element);
|
||||
const rawUrl = resolveImageUrl($img, imageUrlAttr);
|
||||
|
||||
if (!rawUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cleanedUrl = cleanImageUrl(rawUrl);
|
||||
const rawName = generateImageName(cleanedUrl);
|
||||
|
||||
if (!rawName || seenNames.has(rawName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
seenNames.add(rawName);
|
||||
results.push({ url: cleanedUrl, rawName });
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function downloadImage(url, filePath) {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
|
||||
'user-agent': 'Mozilla/5.0 (compatible; download-character-img/1.0)',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
fs.writeFileSync(filePath, buffer);
|
||||
}
|
||||
|
||||
async function downloadWiki(config) {
|
||||
const delay = config.downloadDelayMs ?? 500;
|
||||
const nameTransform = config.nameTransform ?? ((name) => name);
|
||||
|
||||
console.log(`\n=== ${config.name} ===`);
|
||||
console.log(`保存目录: ${config.savePath}`);
|
||||
|
||||
fs.mkdirSync(config.savePath, { recursive: true });
|
||||
|
||||
const html = await fetchWikiHtml(config);
|
||||
const images = extractImages(html, config.imageSelector, config.imageUrlAttr ?? 'src');
|
||||
|
||||
console.log(`共找到 ${images.length} 张角色图`);
|
||||
|
||||
let downloaded = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const [index, image] of images.entries()) {
|
||||
const baseName = toOutputBaseName(image.rawName, nameTransform);
|
||||
const fileName = `${baseName}.webp`;
|
||||
const filePath = path.join(config.savePath, fileName);
|
||||
|
||||
if (fs.existsSync(filePath)) {
|
||||
console.log(
|
||||
`[${index + 1}/${images.length}] 跳过: ${fileName} 已存在（本地已有，原站可能有更新）`
|
||||
);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`[${index + 1}/${images.length}] 下载: ${fileName}`);
|
||||
console.log(` 来源: ${image.url}`);
|
||||
|
||||
try {
|
||||
await downloadImage(image.url, filePath);
|
||||
downloaded += 1;
|
||||
} catch (error) {
|
||||
console.error(` 失败: ${error.message}`);
|
||||
failed += 1;
|
||||
}
|
||||
|
||||
if (index < images.length - 1) {
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`完成: 下载 ${downloaded}，跳过 ${skipped}，失败 ${failed}`
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchServerCharacters(verifyApiUrl) {
|
||||
const response = await fetch(verifyApiUrl, {
|
||||
headers: {
|
||||
'user-agent': 'Mozilla/5.0 (compatible; download-character-img/1.0)',
|
||||
accept: 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`验证 API 请求失败 ${response.status}: ${verifyApiUrl}`);
|
||||
}
|
||||
|
||||
const res = await response.json();
|
||||
const content = JSON.parse(res.data.content);
|
||||
const components = content.filter((item) => item.type === 'component');
|
||||
const ids = new Set();
|
||||
|
||||
for (const component of components) {
|
||||
for (const item of component.props?.items ?? []) {
|
||||
if (item.url) {
|
||||
ids.add(getFileBaseNameFromUrl(item.url));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...ids].sort();
|
||||
}
|
||||
|
||||
async function verifyWiki(config) {
|
||||
console.log(`\n--- 验证 ${config.name} 服务器录入 ---`);
|
||||
|
||||
const localCharacters = listLocalCharacters(config.savePath);
|
||||
const serverCharacters = await fetchServerCharacters(config.verifyApiUrl);
|
||||
const serverSet = new Set(serverCharacters);
|
||||
const missingOnServer = localCharacters.filter((id) => !serverSet.has(id));
|
||||
const extraOnServer = serverCharacters.filter(
|
||||
(id) => !localCharacters.includes(id)
|
||||
);
|
||||
|
||||
console.log(`Wiki 角色数: ${localCharacters.length}`);
|
||||
console.log(`服务器已录入: ${serverCharacters.length}`);
|
||||
|
||||
if (missingOnServer.length === 0) {
|
||||
console.log('服务器已包含全部 Wiki 角色');
|
||||
} else {
|
||||
console.log(`\n服务器缺少 ${missingOnServer.length} 个角色，请手动补充：`);
|
||||
for (const id of missingOnServer) {
|
||||
console.log(` - ${id}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (extraOnServer.length > 0) {
|
||||
console.log(`\n服务器多余 ${extraOnServer.length} 个（Wiki 中未找到）：`);
|
||||
for (const id of extraOnServer) {
|
||||
console.log(` - ${id}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const verifyOnly = args.includes('--verify');
|
||||
const target = args.find((arg) => !arg.startsWith('--'));
|
||||
const configs = target
|
||||
? wikiConfigs.filter((config) => config.name === target)
|
||||
: wikiConfigs;
|
||||
|
||||
if (target && configs.length === 0) {
|
||||
const available = wikiConfigs.map((config) => config.name).join(', ');
|
||||
console.error(`未找到站点 "${target}"，可用: ${available}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const config of configs) {
|
||||
if (!verifyOnly) {
|
||||
await downloadWiki(config);
|
||||
}
|
||||
if (config.verifyApiUrl) {
|
||||
await verifyWiki(config);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n全部处理完成！');
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user