☰

DSA Wiki Crawler

Crawling the Wiki and refactor it the good way

当前为 2017-02-20 提交的版本，查看 最新版本。

// ==UserScript==
// @name         DSA Wiki Crawler
// @namespace    http://tampermonkey.net/
// @version      0.1.1
// @description  Crawling the Wiki and refactor it the good way
// @author       David Mahl
// @match        http://www.ulisses-regelwiki.de/index.php/sonderfertigkeiten.html
// @grant        none
// @require      http://code.jquery.com/jquery-latest.js
// @require      https://cdn.jsdelivr.net/lodash/4.14.2/lodash.min.js
// ==/UserScript==

(function() {
    'use strict';
    var stop= false,
        storage = {},
        app = {
            getAllNavigationEntrys: function(result, entryPoint) {
                var $itemQuery = $('nav table a[role="menuitem"]'),
                    $article = $('.mod_article .ce_text'),
                    $content = "";

                if (result !== undefined) {
                    $itemQuery = $(result).find('nav table a[role="menuitem"]');
                    $article = $(result).find('.mod_article .ce_text');
                }

                if ($article.length === 1) {
                    $content = $article.find('p');
                    //_.update(storage, entryPoint+'.content', app.updateContent($content));
                    _.setWith(storage, entryPoint+'.content.text', $content);
                }

                $itemQuery.each(function(i, e) {
                    var title = $(this).attr('title'),
                        href= $(this).attr('href'),
                        objectPath= entryPoint === undefined?'root.'+title:entryPoint+'.children.'+title;

                    _.setWith(storage, objectPath, {'content': {'title':title, 'href':href, 'text': undefined}});
                    if (stop === false) {
                        app.crawlSubNavigation(_.get(storage, objectPath + '.content.href'), objectPath);
                    }
                });
                //stop = true;
            },
            crawlSubNavigation: function(url, parent) {
                app.loadPage(url)
                    .then(app.parsePage.bind(null, parent))
                    .fail(function (err) {
                    throw new Error('Error while Loading Time');
                });
            },
            updateContent: function($text){
                return {'text':$text};
            },
            parsePage: function(entryPoint, result) {
                app.getAllNavigationEntrys(result, entryPoint);
            },
            loadPage: function(url) {
                return $.ajax( {
                    url: url,
                    type: "POST",
                    dataType: "html"
                });
            },
            writeToLocalStorage() {
                // Put the object into storage
                var checksum = Object.keys(storage).length;
                // ToDo: add Checksum 
                console.log('checksum:',_.size(storage));
                localStorage.setItem('wikiData', JSON.stringify(storage));
                localStorage.setItem('wikiData.checksum','123');
            },
            init: function() {
                // Retrieve the object from storage
                var retrievedObject = localStorage.getItem('wikiData'),
                    checksum = localStorage.getItem('wikiData.checksum');

                if (checksum !== '1234') {
                    console.log('create new Local Storage');
                    app.parsePage();
                    app.writeToLocalStorage();
                } else {
                    console.log('create Storage from localStorage');
                    storage = JSON.parse(retrievedObject);
                }
                console.log('Storage:',storage);
            }
        };
    app.init();
})();