帖子导出工具

导出帖子内容到数据库
当前为 2022-07-18 提交的版本，查看最新版本。
换行
// ==UserScript==
// @name:zh-CN   帖子导出工具
// @name         Posts_Dumper
// @namespace    https://blog.chrxw.com
// @version      1.0
// @description:zh-CN  导出帖子内容到数据库
// @description  导出帖子内容到数据库
// @author       Chr_
// @match        https://keylol.com/*
// @match        https://dev.keylol.com/*
// @connect      127.0.0.1
// @connect      httpbin.org
// @license      AGPL-3.0
// @icon         https://blog.chrxw.com/favicon.ico
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_deleteValue
// @grant        GM_xmlhttpRequest
// @grant        GM_addStyle
// ==/UserScript==
 
setTimeout(async () => {
    'use strict';
 
    const port = 8000;
    const host = '127.0.0.1';
 
    const matchTid = new RegExp(/(?:t|tid=)(\d+)/);
 
    const treadList = document.querySelector("#threadlisttableid");
 
    if (treadList !== null) {//获取帖子列表
 
        function genBtn(name, foo) {
            const b = document.createElement('button');
            b.textContent = name;
            b.className = 'pd_btn';
            b.addEventListener('click', foo);
            return b;
        }
        function genDiv(cls) {
            const d = document.createElement('div');
            d.className = cls ?? 'pd_div';
            return d;
        }
        function genSpan(text) {
            const s = document.createElement('span');
            s.textContent = text;
            return s;
        }
        function genHr() {
            const b = document.createElement('hr');
            return b;
        }
        function genIframe() {
            const i = document.createElement('iframe');
            return i
        }
 
        const panel = genDiv('pd_panel');
 
        const tempIframe = genIframe();
 
        const status = await testBackend();
 
        const statusTips = genSpan(status ? '连接成功' : '连接失败');
 
        const btnGrubNew = genBtn('抓取尚未记录的', async () => {
            const postLists = treadList.querySelectorAll("tr>th:nth-child(2)>a.pd_not_added.xst");
            const total = postLists.length;
            if (total > 0) {
                statusTips.textContent = `开始抓取,共 ${total} 篇`;
 
                for (let i = 0; i < total; i++) {
                    const postTag = postLists[i];
                    const tid = grubTid(postTag.href);
                    const url = genUrl(tid) + '?utm=114514';
                    tempIframe.src = url;
                    const result = await waitUnitlDone(tid);
                    statusTips.textContent = `进度 ${i}/${total} TID ${tid} ${result}`;
                    GM_deleteValue(tid);
                    postTag.classList.remove('pd_not_added');
                    postTag.classList.add('pd_done');
                }
                statusTips.textContent = '抓取结束';
            } else {
                statusTips.textContent = '没有可以抓取的帖子';
            }
            await freshPostList();
        });
 
        const btnGrubAll = genBtn('抓取所有', async () => {
            const postLists = treadList.querySelectorAll("tr>th:nth-child(2)>a.xst");
            const total = postLists.length;
            if (total > 0) {
                statusTips.textContent = `开始抓取,共 ${total} 篇`;
 
                for (let i = 0; i < total; i++) {
                    const postTag = postLists[i];
                    const tid = grubTid(postTag.href);
                    const url = genUrl(tid) + '?utm=114514';
                    tempIframe.src = url;
                    const result = await waitUnitlDone(tid);
                    statusTips.textContent = `进度 ${i}/${total} TID ${tid} ${result}`;
                    GM_deleteValue(tid);
                    postTag.classList.remove('pd_not_added');
                    postTag.classList.remove('pd_added');
                    postTag.classList.add('pd_done');
                }
                statusTips.textContent = '抓取结束';
            } else {
                statusTips.textContent = '没有可以抓取的帖子';
            }
            await freshPostList();
        });
 
        const btnExportExcel = genBtn('导出Excel', () => {
            window.open(`http://${host}:${port}/api/excel`)
        });
 
        const btnExportBBCode = genBtn('导出BBCode', () => {
            window.open(`http://${host}:${port}/api/bbcode`)
        });
 
        const btnResetDB = genBtn('重置数据库(删除所有数据)', async () => {
            if (confirm('真的要删除所有数据吗?')) {
                await deleteAllData();
            }
        });
 
        const btnControl = genBtn('打开管理面板', () => {
            window.open(`http://${host}:${port}/index.html`);
        });
 
        panel.appendChild(statusTips);
        panel.appendChild(genHr());
        if (status) {
            panel.appendChild(btnGrubNew);
            panel.appendChild(btnGrubAll);
            panel.appendChild(genHr());
            panel.appendChild(btnExportExcel);
            panel.appendChild(btnExportBBCode);
            panel.appendChild(genHr());
            panel.appendChild(btnResetDB);
            panel.appendChild(genHr());
            panel.appendChild(btnControl);
            panel.appendChild(genHr());
            panel.appendChild(tempIframe);
        }
        else {
            panel.appendChild(genSpan('请检查软件是否运行以及端口是否被占用'));
        }
 
        document.body.appendChild(panel);
 
        if (status) {
            document.getElementById('autopbn').addEventListener('click', async () => {
                setTimeout(async () => {
                    await freshPostList();
                }, 500);
            });
 
            //判断是否已抓取
            await freshPostList();
        }
    } else if (ifNeedGrub()) {//抓取帖子内容
        const tid = grubTid(location.href);
        const post_url = genUrl(tid);
        const post_title = document.getElementById('thread_subject')?.textContent ?? '获取失败';
        const eleAuthor = document.querySelector('div.pi>div.authi>a.xw1');
        const author_nick = eleAuthor?.textContent ?? '获取失败';
        const author_uid = eleAuthor?.href.replace('https://keylol.com/suid-', '') ?? '获取失败';
        const post_date = document.querySelector('div.pti>div.authi>em[id]')?.textContent.substring(4) ?? '获取失败';
        const eleContent = document.querySelector('td[id^=postmessage');
        const nodes = eleContent?.childNodes ?? [];
        const contentLines = [];
 
        function node2text(node) {
            switch (node.nodeName) {
                case 'I':
                case 'A':
                case 'IFRAME':
                case 'STYLE':
                case 'IMG':
                    return;
            }
 
            if (node.nodeType === Node.TEXT_NODE) {
                const raw = node.textContent?.trim();
                if (raw && raw.length > 2 && raw.search('未经许可，严禁转载') === -1) {
                    contentLines.push(raw);
                }
            }
            else {
                if (node.childNodes?.length > 0) {
                    for (let child of node.childNodes) {
                        node2text(child);
                    }
                }
            }
        }
 
        for (let node of nodes) {
            node2text(node);
        }
        const content = contentLines.join('\n');
 
        const steamLinks = document.querySelectorAll("a[href^='https://store.steampowered.com/'],a[href^='https://steamdb.info/app/']");
        const grubAppid = new RegExp(/app\/(\d+)\/?/);
        const steamAppIDs = new Set();
        for (const ele of steamLinks) {
            const href = ele.href;
            if (href) {
                const appID = parseInt(grubAppid.exec(href)?.[1] ?? 0);
                if (appID > 0) {
                    steamAppIDs.add(appID);
                }
            }
        }
        const game_list = [...steamAppIDs].join('|');
        const data = { tid, post_url, post_title, author_nick, author_uid, post_date, content, game_list };
        console.log(data);
        try {
            GM_setValue(tid, '抓取完成');
            await savePostData(data);
        }
        catch (error) {
            GM_setValue(tid, error);
        }
    }
 
    //显示是否已经抓取
    async function freshPostList() {
        const tidSet = await getPostIds();
        const postLists = treadList.querySelectorAll("tr>th:nth-child(2)>a.xst");
        for (let postTag of postLists) {
            const tid = grubTid(postTag.href);
 
            postTag.classList.remove('pd_not_added');
            postTag.classList.remove('pd_added');
            postTag.classList.remove('pd_done');
 
            if (tidSet.has(tid)) {
                postTag.classList.add('pd_added');
                postTag.title = '【已抓取】';
            } else {
                postTag.classList.add('pd_not_added');
                postTag.title = '【未抓取】';
            }
        }
    }
 
    //判断是否需要抓取
    function ifNeedGrub() {
        if (location.search.endsWith('utm=114514')) {
            return matchTid.test(location.href) >= 0;
        } else {
            return false;
        }
    }
 
    //提取tid
    function grubTid(url) {
        return matchTid.exec(url)?.[1] ?? url.match(matchTid);
    }
 
    //生成链接
    function genUrl(tid) {
        return `https://keylol.com/t${tid}-1-1`;
    }
 
    //-----------------------------------
    //检测后台连通性
    function testBackend() {
        return new Promise((resolve, reject) => {
            $http.get(`http://${host}:${port}/api/test`)
                .then((response) => {
                    resolve(response?.code === 666)
                })
                .catch((reason) => {
                    resolve(false);
                });
        });
    }
    //检测是否抓取完成
    function waitUnitlDone(tid) {
        return new Promise((resolve, reject) => {
            let t1, t2;
 
            t1 = setInterval(() => {
                const fin = GM_getValue(tid);
                if (fin) {
                    clearInterval(t1);
                    clearInterval(t2);
                    resolve(fin);
                }
            }, 100);
 
            t2 = setTimeout(() => {
                clearInterval(t1);
                resolve('操作超时');
            }, 10000);
        });
    }
    //获取已抓取的帖子tid列表
    function getPostIds() {
        return new Promise((resolve, reject) => {
            $http.get(`http://${host}:${port}/api/posts/ids`)
                .then((response) => {
                    const tidSet = new Set();
                    if (response?.code !== 0) {
                        console.error(response?.msg ?? '消息为空');
                    } else {
                        const data = response?.data ?? [];
                        for (let o of data) {
                            tidSet.add(o);
                        }
                    }
                    resolve(tidSet);
                })
                .catch((reason) => {
                    reject(reason);
                });
        });
    }
    //上传抓取结果
    function savePostData(data) {
        return new Promise((resolve, reject) => {
            $http.post(`http://${host}:${port}/api/post`, JSON.stringify(data))
                .then((response) => {
                    console.log(response);
                    resolve(response?.code !== 0);
                })
                .catch((reason) => {
                    console.log(reason);
                    resolve(false);
                });
        });
    }
    //删除所有数据
    function deleteAllData() {
        return new Promise((resolve, reject) => {
            $http.delete(`http://${host}:${port}/api/posts`)
                .then((response) => {
                    console.log(response);
                    resolve(response?.code !== 0);
                })
                .catch((reason) => {
                    console.log(reason);
                    resolve(false);
                });
        });
    }
}, 500);
//-----------------------------------
class Request {
    'use strict';
    constructor(timeout = 3000) {
        this.timeout = timeout;
    }
    get(url, opt = {}) {
        return this.#baseRequest(url, 'GET', opt, 'json');
    }
    getHtml(url, opt = {}) {
        return this.#baseRequest(url, 'GET', opt, '');
    }
    getText(url, opt = {}) {
        return this.#baseRequest(url, 'GET', opt, 'text');
    }
    post(url, data, opt = {}) {
        opt.data = data;
        opt.headers = {
            "Content-Type": "application/json"
        }
        return this.#baseRequest(url, 'POST', opt, 'json');
    }
    delete(url, opt = {}) {
        return this.#baseRequest(url, 'DELETE', opt, 'json');
    }
    #baseRequest(url, method = 'GET', opt = {}, responseType = 'json') {
        Object.assign(opt, {
            url, method, responseType, timeout: this.timeout
        });
        return new Promise((resolve, reject) => {
            opt.ontimeout = opt.onerror = reject;
            opt.onload = ({ readyState, status, response, responseXML, responseText }) => {
                if (readyState === 4 && status === 200) {
                    if (responseType == 'json') {
                        resolve(response);
                    } else if (responseType == 'text') {
                        resolve(responseText);
                    } else {
                        resolve(responseXML);
                    }
                } else {
                    console.error('网络错误');
                    console.log(readyState);
                    console.log(status);
                    console.log(response);
                    reject('解析出错');
                }
            }
            GM_xmlhttpRequest(opt);
        });
    }
}
const $http = new Request();
 
//CSS表
GM_addStyle(`
.pd_div {
    vertical-align: middle;
  }
  
  .pd_panel {
    background: rgba(58, 58, 58, 0.5);
    position: fixed;
    top: 50%;
    right: 0px;
    text-align: center;
    transform: translate(0px, -50%);
    z-index: 100;
    padding: 5px;
    border-radius: 5px 0 0 5px;
  }
  
  .pd_panel > *:not(:last-child) {
    margin-right: 5px;
  }
  
  .pd_panel > hr {
    margin: 5px 0 5px;
  }
  
  .pd_panel > span {
    color: #fff;
  }
  
  .pd_panel > iframe {
    width: 200px;
    height: 200px;
  }
  
  .pd_added::before{
      content: "✅";
  }
  
  .pd_not_added::before{
    content: "❌";
  }
 
  .pd_done::before{
    content: "🤔";
  }
`);