☰

DSA Wiki Crawler

Crawling the Wiki and refactor it the good way

// ==UserScript==
// @name         DSA Wiki Crawler
// @namespace    http://tampermonkey.net/
// @version      0.1.5
// @description  Crawling the Wiki and refactor it the good way
// @author       David Mahl
// @match        http://www.ulisses-regelwiki.de/index.php/za_zaubersprueche.html
// @grant        none
// @require      http://code.jquery.com/jquery-latest.js
// @require      https://cdn.jsdelivr.net/lodash/4.14.2/lodash.min.js
// ==/UserScript==

// Checksum should be "284666"
(function() {
    'use strict';
    var stop= false,
        storage = {},
        CONST = {
            'CONTENT' : {
                'PUBLIKATION':'Publikation',
                'ASP':'AsP-Kosten',
                'AP':'AP-Wert',
                'ERSCHWERNIS':'Erschwernis',
                'VORAUSSETZUNGEN':'Voraussetzungen',
                'KAMPFTECHNIKEN':'Kampftechniken',
                'REICHWEITE':'Reichweite'
            },
            'GLOBAL' : {
                'ADDTOSTARTINDEX': 2,
                'CUTPARAGRAPH': 4,
                'CUTSTRONG': 10,
                'ARTICLEQUERY':'.mod_article .ce_text',
                'ITEMQUERY':'nav table a[role="menuitem"]'
            }
        },
        app = {
            getAllNavigationEntrys: function(result, entryPoint) {
                var $itemQuery = $(CONST.GLOBAL.ITEMQUERY),
                    $article = $(CONST.GLOBAL.ARTICLEQUERY),
                    content = {
                        'Publikation': undefined,
                        'AsP-Kosten':undefined,
                        'AP-Wert':undefined,
                        'Erschwernis':undefined,
                        'Voraussetzungen':undefined,
                        'Kampftechniken':undefined,
                        'Reichweite':undefined,
                        'Info': ""
                    };

                if (result !== undefined) {
                    $itemQuery = $(result).find(CONST.GLOBAL.ITEMQUERY);
                    $article = $(result).find(CONST.GLOBAL.ARTICLEQUERY);
                }

                if ($article.length === 1) {
                    $article.find('p').each(function(i, e){
                        var element = $(e).prop('outerText'),
                            added = false;

                        $.each(CONST.CONTENT, function(key, value) {
                            var index = element.indexOf(value);

                            //console.log('found p | ', key,' | ', value);
                            if (index >= 0) {
                                content[value] = element.substring(index + value.length + CONST.GLOBAL.ADDTOSTARTINDEX, element.length);
                                added = true;
                                //console.log('Content:',content);
                                return false;
                            }
                        });
                        if (added !== true) {
                            content.Info = content.info + element.substring(0, element.length);
                        }
                    });
                    _.setWith(storage, entryPoint+'.content', content);
                    //console.log('storage', entryPoint, ' - ',storage);
                }

                $itemQuery.each(function(i, e) {
                    var title = $(this).attr('title'),
                        href= $(this).attr('href'),
                        objectPath= entryPoint === undefined?'root.'+title:entryPoint+'.children.'+title;

                    _.setWith(storage, objectPath, {'title':title, 'href':href});
                    app.crawlSubNavigation(_.get(storage, objectPath + '.href'), objectPath);
                });
                app.writeToLocalStorage();
            },
            crawlSubNavigation: function(url, entryPoint) {
                app.loadPage(url)
                    .then(app.parsePage.bind(null, entryPoint))
                    .fail(function (err) {
                    throw new Error('Error while Loading Time');
                });
            },
            parsePage: function(entryPoint, result) {
                app.getAllNavigationEntrys(result, entryPoint);
            },
            loadPage: function(url) {
                return $.ajax( {
                    url: url,
                    type: "POST",
                    dataType: "html"
                });
            },
            writeToLocalStorage() {
                var checksum = Object.keys(storage).length;
                localStorage.setItem('wikiData', JSON.stringify(storage));
                localStorage.setItem('wikiData.checksum',JSON.stringify(storage).length);
            },
            init: function() {
                // Retrieve the object from storage
                var retrievedObject = localStorage.getItem('wikiData'),
                    checksum = localStorage.getItem('wikiData.checksum');

                app.parsePage();
//                if (retrievedObject === null) {
//                    app.parsePage();
//                } else {
//                    storage = JSON.parse(retrievedObject);
//                }
                console.log('Storage:',storage);
            }
        };
    app.init();
})();