Node.js网站死链检测工具
爬取网站所有页面，检测失效链接(404/500)，生成死链报告，网站维护和SEO必备。
详细内容
// Node.js网站死链检测工具
// 依赖: npm install axios cheerio
// 使用方法: node link-checker.js https://example.com
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const BASE_URL = process.argv[2] || 'https://ozhan.com.cn';
const MAX_PAGES = 100; // 最大检测页面数
const TIMEOUT = 10000;
const visited = new Set();
const brokenLinks = [];
let pageCount = 0;
async function fetchPage(url) {
try {
const response = await axios.get(url, {
timeout: TIMEOUT,
headers: { 'User-Agent': 'LinkChecker/1.0' }
});
return response.data;
} catch (error) {
return null;
}
}
async function checkLink(url, referer) {
try {
const response = await axios.head(url, {
timeout: TIMEOUT,
validateStatus: () => true,
headers: { 'User-Agent': 'LinkChecker/1.0' }
});
return response.status;
} catch (error) {
return error.response?.status || 0;
}
}
function extractLinks(html, baseUrl) {
const $ = cheerio.load(html);
const links = [];
$('a[href]').each((i, el) => {
let href = $(el).attr('href');
if (href && !href.startsWith('#') && !href.startsWith('') && !href.startsWith('mailto:')) {
try {
const absoluteUrl = new URL(href, baseUrl).href;
links.push(absoluteUrl);
} catch (e) {}
}
});
return [...new Set(links)];
}
async function crawl(url) {
if (visited.has(url) || pageCount >= MAX_PAGES) return;
visited.add(url);
pageCount++;
console.log(`[${pageCount}/${MAX_PAGES}] 检测: ${url}`);
const html = await fetchPage(url);
if (!html) {
brokenLinks.push({ url, referer: '入口', status: '获取失败' });
return;
}
const links = extractLinks(html, url);
for (const link of links) {
// 只检查站内链接
if (link.startsWith(BASE_URL)) {
if (!visited.has(link)) {
await crawl(link);
}
} else {
// 检查外部链接状态
const status = await checkLink(link, url);
if (status >= 400 || status === 0) {
brokenLinks.push({ url: link, referer: url, status: status || '连接失败' });
console.log(` ❌ 死链: ${link} (状态: ${status})`);
}
}
}
}
async function main() {
console.log('========================================');
console.log(' 网站死链检测工具');
console.log('========================================');
console.log('目标网站:', BASE_URL);
console.log('开始时间:', new Date().toLocaleString());
console.log('----------------------------------------');
await crawl(BASE_URL);
// 生成报告
const report = {
site: BASE_URL,
checkedAt: new Date().toISOString(),
totalPages: pageCount,
brokenCount: brokenLinks.length,
brokenLinks: brokenLinks
};
fs.writeFileSync('broken-links-report.json', JSON.stringify(report, null, 2));
console.log('');
console.log('========================================');
console.log(' 检测完成！');
console.log('========================================');
console.log('检测页面数:', pageCount);
console.log('发现死链:', brokenLinks.length);
console.log('报告已保存: broken-links-report.json');
if (brokenLinks.length > 0) {
console.log('');
console.log('死链列表:');
brokenLinks.forEach((link, i) => {
console.log(`${i + 1}. [${link.status}] ${link.url}`);
console.log(` 来源: ${link.referer}`);
});
}
}
main().catch(console.error);