☰

DSA Wiki Crawler

Crawling the Wiki and refactor it the good way

当前为 2017-02-20 提交的版本，查看 最新版本。

  1. // ==UserScript==
  2. // @name         DSA Wiki Crawler
  3. // @namespace    http://tampermonkey.net/
  4. // @version      0.1.2
  5. // @description  Crawling the Wiki and refactor it the good way
  6. // @author       David Mahl
  7. // @match        http://www.ulisses-regelwiki.de/index.php/sonderfertigkeiten.html
  8. // @grant        none
  9. // @require      http://code.jquery.com/jquery-latest.js
  10. // @require      https://cdn.jsdelivr.net/lodash/4.14.2/lodash.min.js
  11. // ==/UserScript==
  12.  
  13. (function() {
  14.     'use strict';
  15.     var stop= false,
  16.         storage = {},
  17.         CONST = {
  18.             'CONTENT' : {
  19.                 'PUBLIKATION':'Publikation:',
  20.                 'ASP':'AsP-Kosten:',
  21.                 'AP':'AP-Wert:',
  22.                 'ERSCHWERNIS':'Erschwernis:',
  23.                 'VORAUSSETZUNGEN':'Voraussetzungen:',
  24.                 'KAMPFTECHNIKEN':'Kampftechniken:'
  25.             },
  26.             'GLOBAL' : {
  27.                 'CUTPARAGRAPH': 4,
  28.                 'CUTSTRONG': 10,
  29.                 'ARTICLEQUERY':'.mod_article .ce_text',
  30.                 'ITEMQUERY':'nav table a[role="menuitem"]'
  31.             }
  32.         },
  33.         app = {
  34.             getAllNavigationEntrys: function(result, entryPoint) {
  35.                 var $itemQuery = $(CONST.GLOBAL.ITEMQUERY),
  36.                     $article = $(CONST.GLOBAL.ARTICLEQUERY),
  37.                     content = {
  38.                         'publikation': undefined,
  39.                         'asp':undefined,
  40.                         'ap':undefined,
  41.                         'erschwernis':undefined,
  42.                         'vorraussetzungen':undefined,
  43.                         'kampftechniken':undefined,
  44.                         'info': ""
  45.                     };
  46.  
  47.                 if (result !== undefined) {
  48.                     $itemQuery = $(result).find(CONST.GLOBAL.ITEMQUERY);
  49.                     $article = $(result).find(CONST.GLOBAL.ARTICLEQUERY);
  50.                 }
  51.  
  52.                 //console.log('EntryPoint',entryPoint,' ',$article.length);
  53.                 if ($article.length === 1) {
  54.                     $article.find('p').each(function(i, e){
  55.                         var element = $(e).prop('outerText'),
  56.                             publikationIndex = element.indexOf(CONST.CONTENT.PUBLIKATION),
  57.                             aspIndex = element.indexOf(CONST.CONTENT.ASP),
  58.                             apIndex = element.indexOf(CONST.CONTENT.AP),
  59.                             erschwernisIndex = element.indexOf(CONST.CONTENT.ERSCHWERNIS),
  60.                             voraussetzungIndex = element.indexOf(CONST.CONTENT.VORAUSSETZUNGEN),
  61.                             kampftechIndex = element.indexOf(CONST.CONTENT.KAMPFTECHNIKEN),
  62.                             text = "";
  63.  
  64.                         if (publikationIndex >= 0) {
  65.                             text = element.substring(publikationIndex + CONST.CONTENT.PUBLIKATION.length +1, element.length);
  66.                             content.publikation = text;
  67.                         } else if (aspIndex >= 0) {
  68.                             content.asp = element.substring(aspIndex + CONST.CONTENT.ASP.length +1, element.length);
  69.                         } else if (apIndex >= 0) {
  70.                             content.ap = element.substring(apIndex + CONST.CONTENT.AP.length +1, element.length);
  71.                         } else if (erschwernisIndex >= 0) {
  72.                             content.ap = element.substring(erschwernisIndex + CONST.CONTENT.ERSCHWERNIS.length +1, element.length);
  73.                         } else if (voraussetzungIndex >= 0) {
  74.                             content.ap = element.substring(voraussetzungIndex + CONST.CONTENT.VORAUSSETZUNGEN.length +1, element.length);
  75.                         } else if (kampftechIndex >= 0) {
  76.                             content.ap = element.substring(kampftechIndex + CONST.CONTENT.KAMPFTECHNIKEN.length +1, element.length);
  77.                         } else {
  78.                             text = element.substring(0, element.length);
  79.                             content.info = content.info + text;
  80.                         }
  81.                     });
  82.                     _.setWith(storage, entryPoint+'.content', content);
  83.                    
  84.                 }
  85.  
  86.                 $itemQuery.each(function(i, e) {
  87.                     var title = $(this).attr('title'),
  88.                         href= $(this).attr('href'),
  89.                         objectPath= entryPoint === undefined?'root.'+title:entryPoint+'.children.'+title;
  90.  
  91.                     _.setWith(storage, objectPath, {'title':title, 'href':href});
  92.                     if (stop === false) {
  93.                          app.writeToLocalStorage();
  94.                         app.crawlSubNavigation(_.get(storage, objectPath + '.href'), objectPath);
  95.                     }
  96.                 });
  97.                 //stop = true;
  98.             },
  99.             crawlSubNavigation: function(url, parent) {
  100.                 app.loadPage(url)
  101.                     .then(app.parsePage.bind(null, parent))
  102.                     .fail(function (err) {
  103.                     throw new Error('Error while Loading Time');
  104.                 });
  105.             },
  106.             updateContent: function($text){
  107.                 return {'text':$text};
  108.             },
  109.             parsePage: function(entryPoint, result) {
  110.                 app.getAllNavigationEntrys(result, entryPoint);
  111.             },
  112.             loadPage: function(url) {
  113.                 return $.ajax( {
  114.                     url: url,
  115.                     type: "POST",
  116.                     dataType: "html"
  117.                 });
  118.             },
  119.             writeToLocalStorage() {
  120.                 // Put the object into storage
  121.                 var checksum = Object.keys(storage).length;
  122.                 // ToDo: add Checksum
  123.                 localStorage.setItem('wikiData', JSON.stringify(storage));
  124.                 localStorage.setItem('wikiData.checksum','123');
  125.             },
  126.             init: function() {
  127.                 // Retrieve the object from storage
  128.                 var retrievedObject = localStorage.getItem('wikiData'),
  129.                     checksum = localStorage.getItem('wikiData.checksum');
  130.  
  131.                 app.parsePage();
  132.                 if (retrievedObject === null) {
  133.                     //app.parsePage();
  134.                 } else {
  135.                     //storage = JSON.parse(retrievedObject);
  136.                 }
  137.                 console.log('Storage:',storage);
  138.             }
  139.         };
  140.     app.init();
  141. })();