☰

博客園新聞線上詞雲

博客園新聞抓取工具，詞雲生成器

// ==UserScript==
// @name         博客园新闻在线词云
// @name:en      Cnblogs News Scraper and WordCloud Generator
// @description:en Cnblogs News Scraper,WordCloud Generator
// @name:ar      مدونة الأخبار السحابية على الإنترنت
// @description:ar  مكشطة أخبار مدونة، مولد السحابة الكلمية
// @name:bg      Онлайн облак от думи за новини в Cnblogs
// @description:bg  Скрепер за новини от Cnblogs, генератор на облак от думи
// @name:cs      Online slovní oblak zpráv Cnblogs
// @description:cs  Skreper zpráv Cnblogs, generátor slovního oblaku
// @name:da      Cnblogs Nyheder Online Ordfold
// @description:da  Cnblogs Nyhedsskraper, Ordfoldsgenerator
// @name:de      Cnblogs Nachrichten Online-Wortwolke
// @description:de  Cnblogs Nachrichtenscraper, Wortwolkengenerator
// @name:el      Διαδικτυακό σύννεφο λέξεων ειδήσεων Cnblogs
// @description:el  Ξύστης ειδήσεων Cnblogs, Γεννήτρια σύννεφου λέξεων
// @name:eo      Interreta Vortonubo de Novaĵoj de Cnblogs
// @description:eo  Skrapilo de Novaĵoj de Cnblogs, Generilo de Vortonubo
// @name:es      Nube de palabras en línea de noticias de Cnblogs
// @description:es  Raspador de noticias de Cnblogs, Generador de nube de palabras
// @name:fi      Cnblogs-uutisten online-sanapilvi
// @description:fi  Cnblogs-uutiskaavin, sanapilvigeneraattori
// @name:fr      Nuage de mots en ligne des nouvelles de Cnblogs
// @description:fr  Scraper de nouvelles Cnblogs, Générateur de nuage de mots
// @name:fr-CA   Nuage de mots en ligne des nouvelles de Cnblogs
// @description:fr-CA  Grattoir de nouvelles Cnblogs, Générateur de nuage de mots
// @name:he      ענן מילים מקוון של חדשות Cnblogs
// @description:he  מגרד חדשות Cnblogs, מחולל ענן מילים
// @name:hr      Online oblak riječi vijesti Cnblogs
// @description:hr  Skreper vijesti Cnblogs, Generator oblaka riječi
// @name:hu      Cnblogs Hírek Online Szófelhő
// @description:hu  Cnblogs Hírek Kaparó, Szófelhő Generátor
// @name:id      Awan Kata Berita Cnblogs Online
// @description:id  Pengikis Berita Cnblogs, Pembuat Awan Kata
// @name:it      Nuvola di parole online delle notizie di Cnblogs
// @description:it  Scraper di notizie Cnblogs, Generatore di nuvole di parole
// @name:ja      Cnblogsニュースオンライン単語クラウド
// @description:ja  Cnblogsニューススクレーパー、単語クラウドジェネレーター
// @name:ka      Cnblogs სიახლეების ონლაინ სიტყვის ღრუბელი
// @description:ka  Cnblogs სიახლეების სკრაპერი, სიტყვის ღრუბლის გენერატორი
// @name:ko      Cnblogs 뉴스 온라인 워드클라우드
// @description:ko  Cnblogs 뉴스 스크레이퍼, 워드클라우드 생성기
// @name:nb      Cnblogs Nyheter Online Ordsky
// @description:nb  Cnblogs Nyhetsskraper, Ordskygenerator
// @name:nl      Cnblogs Nieuws Online Woordwolk
// @description:nl  Cnblogs Nieuwsscraper, Woordwolkgenerator
// @name:pl      Chmura słów online wiadomości Cnblogs
// @description:pl  Skraper wiadomości Cnblogs, Generator chmury słów
// @name:pt-BR   Nuvem de palavras online de notícias do Cnblogs
// @description:pt-BR  Raspador de notícias do Cnblogs, Gerador de nuvem de palavras
// @name:ro      Nor de cuvinte online al știrilor Cnblogs
// @description:ro  Scraper de știri Cnblogs, Generator de nori de cuvinte
// @name:ru      Онлайн-облако слов новостей Cnblogs
// @description:ru  Скрапер новостей Cnblogs, Генератор облака слов
// @name:sk      Online oblak slov správ Cnblogs
// @description:sk  Skreper správ Cnblogs, Generátor oblaku slov
// @name:sr      Онлајн облак речи вести Cnblogs
// @description:sr  Скрејпер вести Cnblogs, Генератор облака речи
// @name:sv      Cnblogs Nyheter Online Ordmoln
// @description:sv  Cnblogs Nyhetsskrapare, Ordmolnsgenerator
// @name:th      คลาวด์คำออนไลน์ของข่าว Cnblogs
// @description:th  ตัวขูดข่าว Cnblogs, ตัวสร้างคลาวด์คำ
// @name:tr      Cnblogs Haberler Çevrimiçi Kelime Bulutu
// @description:tr  Cnblogs Haber Kazıyıcı, Kelime Bulutu Oluşturucu
// @name:ug      Cnblogs خەۋەرلىرى تور سۆز بۇلۇتى
// @description:ug  Cnblogs خەۋەر قىرگۇچى، سۆز بۇلۇتى ياسىغۇچى
// @name:uk      Онлайн хмара слів новин Cnblogs
// @description:uk  Скрапер новин Cnblogs, Генератор хмари слів
// @name:vi      Đám mây từ trực tuyến của tin tức Cnblogs
// @description:vi  Công cụ cạo tin tức Cnblogs, Trình tạo đám mây từ
// @name:zh      博客园新闻在线词云
// @description:zh  博客园新闻抓取工具，词云生成器
// @name:zh-CN   博客园新闻在线词云
// @description:zh-CN  博客园新闻抓取工具，词云生成器
// @name:zh-HK   博客園新聞線上詞雲
// @description:zh-HK  博客園新聞抓取工具，詞雲生成器
// @name:zh-SG   博客园新闻在线词云
// @description:zh-SG  博客园新闻抓取工具，词云生成器
// @name:zh-TW   博客園新聞線上詞雲
// @description:zh-TW  博客園新聞抓取工具，詞雲生成器
// @namespace    http://tampermonkey.net/
// @version      1.2.2.1
// @description  Scrape news from cnblogs and generate word clouds
// @author       aspen138
// @icon         https://assets.cnblogs.com/favicon.ico
// @match        *://news.cnblogs.com/*
// @grant        GM_xmlhttpRequest
// @require      https://code.jquery.com/jquery-3.6.0.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/wordcloud2.js/1.1.2/wordcloud2.min.js
// @require      https://cdn.jsdelivr.net/npm/[email protected]/dist/umd/segmentit.min.js
// @connect      news.cnblogs.com
// @license      MIT
// ==/UserScript==


// Acknowledgement: o1-preview, Gemini 2.5 Flash Preview 04-17, Grok 3




var global_doesConsiderNewsBody = false; //default set "false"

/*-------------------------------预处理一下不纳入词云考虑的词表--------------------------------------------------------------*/
// Function to generate all Arabic numeral combinations of length 1 to 4
function generateArabicNumerals(maxLength) {
    const numerals = new Set();
    for (let len = 1; len <= maxLength; len++) {
        const maxNum = Math.pow(10, len);
        for (let i = 0; i < maxNum; i++) {
            const numStr = i.toString().padStart(len, '0'); // e.g., "0012" for 12
            numerals.add(numStr);
        }
    }
    return numerals;
}

// Function to convert a number to its Chinese reading
function numberToChinese(num) {
    const digits = ['零', '一', '二', '三', '四', '五', '六', '七', '八', '九'];
    const units = ['', '十', '百', '千', '万', '十', '百', '千', '亿'];

    if (num === 0) return '零';

    let str = '';
    let numStr = num.toString();
    let len = numStr.length;

    for (let i = 0; i < len; i++) {
        let digit = parseInt(numStr[i]);
        let unitIdx = len - 1 - i;

        // Add digit if not zero
        if (digit !== 0) {
            str += digits[digit] + units[unitIdx];
        } else if (i < len - 1 && parseInt(numStr[i + 1]) !== 0) {
            // Add zero if next digit is non-zero (e.g., 1001 -> 一千零一)
            str += digits[0];
        }
    }

    // Clean up: remove trailing units, handle special cases
    str = str.replace(/零+$/, ''); // Remove trailing zeros
    str = str.replace(/零{2,}/g, '零'); // Replace multiple zeros with single zero
    if (str.startsWith('一十')) {
        str = str.slice(1); // "一十" -> "十"
    }

    return str;
}

// Generate Chinese number readings for numbers 0 to 9999 with length <= 10
function generateChineseNumbers(maxNum, maxLength) {
    const chineseNums = new Set();
    for (let i = 0; i <= maxNum; i++) {
        const chinese = numberToChinese(i);
        if (chinese.length <= maxLength) {
            chineseNums.add(chinese);
        }
    }
    return chineseNums;
}
// Basic filter for common punctuation/symbols that might slip through or be single chars
const punctuationRegex = /^[.,!?;:"'，。！？；：‘’“”【】（）《》、]$/;
// Initialize custom stop words
const customStopWords = new Set(['的', '是', '和', '有', '在', '了', '中', '等', '与', '一']);

// Add Arabic numerals (length <= 4)
const arabicNumerals = generateArabicNumerals(4);
arabicNumerals.forEach(num => customStopWords.add(num));

// Add Chinese number readings (length <= 10)
const chineseNumbers = generateChineseNumbers(9999, 10);
chineseNumbers.forEach(num => customStopWords.add(num));
/*-------------------------------预处理一下不纳入词云考虑的词表--------------------------------------------------------------*/





function extract_newsIds() {
    // Select the `news_list` container
    const newsList = document.querySelector('#news_list');

    if (newsList) {
        // Find all <a> elements with href matching /n/{news_id}/
        const newsLinks = newsList.querySelectorAll('a[href^="/n/"][href$="/"]');

        // Extract the news_id from each matching link
        const newsIds = Array.from(newsLinks).map(link => {
            const match = link.getAttribute('href').match(/\/n\/(\d+)\//);
            return match ? match[1] : null;
        }).filter(Boolean); // Remove null values
        return newsIds;
    }

};


(function () {
    'use strict';

    // Function to load external scripts dynamically
    function loadScript(url, callback) {
        var script = document.createElement('script');
        script.src = url;
        script.type = 'text/javascript';
        script.onload = callback;
        document.head.appendChild(script);
    }

    // Load segmentit script
    loadScript('https://cdn.jsdelivr.net/npm/[email protected]/dist/umd/segmentit.min.js', function () {
        // Initialize segmentit after the script is loaded
        const segmentit = Segmentit.useDefault(new Segmentit.Segment());

        // Now start your main script
        main(segmentit);
    });

    function main(segmentit) {
        // Updated form HTML with horizontal layout and lower positioning

        const newsIds = extract_newsIds();

        // Assume `newsIds` might be undefined
        const defaultMin = 781100; // Default start news ID
        const defaultMax = 781159; // Default end news ID

        // Calculate min and max, falling back to defaults if newsIds is undefined or empty
        const minNewsId = Array.isArray(newsIds) && newsIds.length > 0 ? Math.min(...newsIds) : defaultMin;
        const maxNewsId = Array.isArray(newsIds) && newsIds.length > 0 ? Math.max(...newsIds) : defaultMax;
        const doesConsiderNewsBody = true; // 是否考虑新闻文本内容


        // var formHtml = `
        // <div id="news-scraper" style="position:fixed; top:50px; right:10px; background-color:#fff; padding:20px; border:1px solid #ccc; z-index:10000; display: flex; flex-direction: column; gap: 5px;">
        //     <h3 style="margin: 0; text-align: center;">News Scraper and WordCloud Generator</h3>
        //     <label style="display: flex; justify-content: space-between; align-items: center;">
        //         Start News ID:
        //         <input type="number" id="start-news-id" value="${minNewsId}" style="margin-left: 10px;" />
        //     </label>
        //     <label style="display: flex; justify-content: space-between; align-items: center;">
        //         End News ID:
        //         <input type="number" id="end-news-id" value="${maxNewsId}" style="margin-left: 10px;" />
        //     </label>
        //     <label style="display: flex; justify-content: space-between; align-items: center;">
        //         Consider News Body:
        //         <input type="checkbox" id="consider-news-body-checkbox" value="${doesConsiderNewsBody}" style="margin-left: 10px;" />
        //     </label>
        //     <button id="start-scraping" style="align-self: center; padding: 5px 10px;">Start Scraping</button>
        //     <div id="scraping-status" style="margin-top: 10px; text-align: center;"></div>
        // </div>
        // `;


        // $('body').append(formHtml);


        var formHtml = `
        <div id="news-scraper" style="position:fixed; top:50px; right:10px; background-color:#fff; padding:20px; border:1px solid #ccc; z-index:10000; display: flex; flex-direction: column;">
            <!-- Title Bar -->
            <div id="panel-title-bar" style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px;">
                <h3 style="margin: 0; text-align: left; flex-grow: 1; font-size: 1.2em;">News WordCloud</h3>
                <button id="minimize-button" style="background: none; border: none; font-size: 1.5em; cursor: pointer; padding: 0 5px; line-height: 1; color: #333;">-</button>
            </div>

            <!-- Panel Content (This part will be hidden when minimized) -->
            <div id="panel-content" style="display: flex; flex-direction: column; gap: 5px;">
                <label style="display: flex; justify-content: space-between; align-items: center;">
                    Start News ID:
                    <input type="number" id="start-news-id" value="${minNewsId}" style="margin-left: 10px; width: 80px;" />     <!-- Added width -->
                </label>
                <label style="display: flex; justify-content: space-between; align-items: center;">
                    End News ID:
                    <input type="number" id="end-news-id" value="${maxNewsId}" style="margin-left: 10px; width: 80px;" />      <!-- Added width -->
                </label>
                <label style="display: flex; justify-content: space-between; align-items: center;">
                    Consider News Body:
                    <input type="checkbox" id="consider-news-body-checkbox" ${doesConsiderNewsBody} style="margin-left: 10px;" /> <!-- Corrected checkbox value handling -->
                </label>
                <button id="start-scraping" style="align-self: center; padding: 5px 10px; margin-top: 10px;">Start Scraping</button>   <!-- Added margin-top -->
                <div id="scraping-status" style="margin-top: 10px; text-align: center;"></div>
            </div>
        </div>
        `;

        // Append the form HTML to the body
        $('body').append(formHtml);

        // --- Add the CSS for minimization ---
        // It's best to add a style block dynamically or ensure this CSS is in your page's head
        var style = document.createElement('style');
        style.type = 'text/css';
        style.innerHTML = `
        /* Hide the content section when the panel has the 'minimized' class */
        #news-scraper.minimized #panel-content {
            display: none;
        }
        /* Optional: Adjust padding or remove margin for a tighter minimized look */
        #news-scraper.minimized {
            /* padding-bottom: 10px; /* Example: reduce bottom padding */ */
        }
        #news-scraper.minimized #panel-title-bar {
            margin-bottom: 0; /* Remove margin below title bar when content is hidden */
        }
        `;
        document.head.appendChild(style);


        // --- Add the JavaScript logic for minimization ---
        $(document).ready(function () { // Ensure DOM is ready
            var $panel = $('#news-scraper');
            var $minimizeButton = $('#minimize-button');
            var $panelContent = $('#panel-content');
            var $titleBar = $('#panel-title-bar');

            // Set initial button text
            $minimizeButton.text('-'); // Panel starts expanded

            // Add click listener to the minimize button
            $minimizeButton.on('click', function () {
                // Toggle the 'minimized' class on the main panel
                $panel.toggleClass('minimized');

                // Check if the panel is now minimized
                if ($panel.hasClass('minimized')) {
                    // If minimized: hide content, change button text, adjust margin
                    $panelContent.hide(); // Use jQuery hide/show for potential animation later if desired
                    $minimizeButton.text('+');
                    $titleBar.css('margin-bottom', '0'); // Remove space below title
                } else {
                    // If expanded: show content, change button text, restore margin
                    $panelContent.show(); // Use jQuery hide/show
                    $minimizeButton.text('-');
                    $titleBar.css('margin-bottom', '10px'); // Restore space below title
                }
            });

            // Optional: Make the panel draggable (requires jQuery UI or similar library)
            // If you have jQuery UI loaded:
            /*
            $panel.draggable({
                handle: "#panel-title-bar", // Only drag by the title bar
                cancel: "#minimize-button" // Don't start dragging when clicking the button
            });
            */

            // Note: Existing scraping logic should be attached to #start-scraping button
            // outside of this minimization code block.
        });


        $('#start-scraping').click(function () {
            var startId = parseInt($('#start-news-id').val());
            var endId = parseInt($('#end-news-id').val());
            global_doesConsiderNewsBody = $('#consider-news-body-checkbox').is(':checked');

            // Validate input
            if (endId < startId) {
                alert('End News ID must be greater than or equal to Start News ID');
                return;
            }

            startScraping(startId, endId);
        });

        async function startScraping(startId, endId) {
            var newsIds = [];
            if (startId <= endId) {
                for (var i = startId; i <= endId; i++) {
                    newsIds.push(i);
                }
            } else {
                for (var ii = startId; ii >= endId; ii--) {
                    newsIds.push(ii);
                }
            }
            var totalNews = newsIds.length;
            var newsData = [];
            var completedRequests = 0;

            $('#scraping-status').text('Starting scraping...');

            var concurrencyLimit = 1024; // Adjust this number as needed
            var queue = newsIds.slice(); // Copy of newsIds

            async function worker() {
                while (queue.length > 0) {
                    var newsId = queue.shift();
                    await fetchNews(newsId).then(function (newsInfo) {
                        if (newsInfo) {
                            newsData.push(newsInfo);
                        }
                    });
                    completedRequests++;
                    $('#scraping-status').text('Scraped ' + completedRequests + ' of ' + totalNews);
                }
            }

            var workers = [];
            for (var j = 0; j < concurrencyLimit; j++) {
                workers.push(worker());
            }

            await Promise.all(workers);

            // All done
            processData(newsData);
        }

        // Function to fetch a single news page
        function fetchNews(newsId) {
            return new Promise(function (resolve) {
                var url = 'https://news.cnblogs.com/n/' + newsId + '/';

                GM_xmlhttpRequest({
                    method: 'GET',
                    url: url,
                    onload: function (response) {
                        if (response.status === 200) {
                            var parser = new DOMParser();
                            var doc = parser.parseFromString(response.responseText, 'text/html');

                            var newsInfo = getNewsInfo(doc, newsId, url);
                            resolve(newsInfo);
                        } else {
                            resolve(null);
                        }
                    },
                    onerror: function (error) {
                        resolve(null);
                    }
                });
            });
        }

        // Function to extract news information from the HTML document
        function getNewsInfo(doc, newsId, url) {
            var title = 'Not Found';
            var time_text = 'Not Found';
            var views = 'Not Found';
            var news_body = 'Not Found';

            var news_title_div = doc.querySelector('#news_title');
            if (news_title_div) {
                var a = news_title_div.querySelector('a');
                if (a) {
                    title = a.textContent.trim();
                }
            }

            var news_info_div = doc.querySelector('#news_info');
            if (news_info_div) {
                var time_span = news_info_div.querySelector('span.time');
                if (time_span) {
                    time_text = time_span.textContent.trim();
                }
                var view_span = news_info_div.querySelector('span.view#News_TotalView');
                if (view_span) {
                    views = view_span.textContent.trim();
                }
            }

            var news_body_div = doc.querySelector('#news_body');
            if (news_body_div) {
                news_body = news_body_div.innerText.trim();
            }

            return {
                news_id: newsId,
                title: title,
                time: time_text,
                views: views,
                news_body: news_body,
                url: url
            };
        }

        // Function to process the scraped data and generate word clouds
        function processData(newsData) {
            // Parse time and extract year_month
            for (var i = 0; i < newsData.length; i++) {
                var item = newsData[i];
                var timeStr = item.time; // e.g., "发布于 2023-09-30 12:34"
                var dateMatch = timeStr.match(/发布于\s+(\d{4}-\d{2}-\d{2})/);
                if (dateMatch) {
                    item.date = dateMatch[1];
                    var dateObj = new Date(item.date);
                    var year = dateObj.getFullYear();
                    var month = dateObj.getMonth() + 1; // Months are 0-based
                    item.year_month = year + '-' + (month < 10 ? '0' + month : month);
                } else {
                    item.date = null;
                    item.year_month = 'Unknown';
                }
            }

            // Group data by year_month
            var groupedData = {};
            for (var ii = 0; ii < newsData.length; ii++) {
                var my_item = newsData[ii];
                var key = my_item.year_month;
                if (!groupedData[key]) {
                    groupedData[key] = [];
                }
                groupedData[key].push(my_item);
            }

            // For each group, generate word cloud
            for (var my_key in groupedData) {
                var group = groupedData[key];
                var textArray = [];
                for (var j = 0; j < group.length; j++) {
                    var myitem = group[j];
                    // Combine title and news_body
                    var text
                    if (global_doesConsiderNewsBody) {
                        text = item.title + ' ' + item.news_body;
                    }
                    else {
                        text = myitem.title;
                    }
                    textArray.push(text);
                }
                var combinedText = textArray.join(' ');

                // Generate word cloud
                generateWordCloud(combinedText, key);
            }

            $('#scraping-status').text('All word clouds generated.');
        }

        // Function to generate word cloud using wordcloud2.js
        function generateWordCloud(text, title) {
            // Create a container div
            var container = $('<div></div>').css({
                'border': '1px solid #ccc',
                'margin': '10px',
                'padding': '10px'
            });
            // Add title
            var h3 = $('<h3></h3>').text(title);
            container.append(h3);
            // Create a canvas
            var canvas = $('<canvas></canvas>').attr('width', 500).attr('height', 500);
            container.append(canvas);

            $('#news-scraper').after(container);

            // Generate word cloud
            WordCloud(canvas[0], {
                list: getWordList(text),
                gridSize: 10,
                weightFactor: 5,
                fontFamily: 'Microsoft Yahei, SimHei, Arial, sans-serif',
                color: 'random-dark',
                backgroundColor: '#fff'
            });
        }

        // Function to segment text and generate word frequency list
        function getWordList(text) {
            // Use segmentit to segment Chinese text
            var segments = segmentit.doSegment(text);
            var words = segments.map(function (seg) {
                return seg.w;
            });

            // Count word frequencies
            var freqMap = {};
            words.forEach(function (word) {
                // Filter out empty strings, single characters, punctuation, and optionally custom stop words
                if (word.length > 1 && !punctuationRegex.test(word)
                    && !customStopWords.has(word)
                ) {
                    if (!freqMap[word]) {
                        freqMap[word] = 0;
                    }
                    freqMap[word]++;
                }
            });

            // Convert to list of [word, frequency] pairs
            var wordList = [];
            for (var word in freqMap) {
                wordList.push([word, freqMap[word]]);
            }

            // Sort by frequency
            wordList.sort(function (a, b) {
                return b[1] - a[1];
            });

            return wordList;
        }
    }
})();