☰

MTurk HIT Database Mk.II

Keep track of the HITs you've done (and more!). Cross browser compatible.

当前为 2016-05-01 提交的版本，查看 最新版本。

  1. // ==UserScript==
  2. // @name         MTurk HIT Database Mk.II
  3. // @author       feihtality
  4. // @namespace    https://greasyfork.org/en/users/12709
  5. // @version      1.1.012
  6. // @description  Keep track of the HITs you've done (and more!). Cross browser compatible.
  7. // @include      /^https://www\.mturk\.com/mturk/(dash|view|sort|find|prev|search|accept|cont).*/
  8. // @exclude      https://www.mturk.com/mturk/findhits?*hit_scraper
  9. // @grant        none
  10. // ==/UserScript==
  11.  
  12. /**\
  13.  ** 
  14.  ** This is a complete rewrite of the MTurk HIT Database script from the ground up, which
  15.  ** eliminates obsolete methods, fixes many bugs, and brings this script up-to-date 
  16.  ** with the modern browser environment.
  17.  **
  18. \**/ 
  19.  
  20.  
  21.  
  22. /*globals self*/
  23.  
  24. const DB_VERSION = 8;
  25. const DB_NAME = 'HITDB';
  26. const MTURK_BASE = 'https://www.mturk.com/mturk/';
  27.  
  28. /***************************      Native code modifications      *******************************/
  29. if (!NodeList.prototype[Symbol.iterator]) NodeList.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator];
  30. Number.prototype.toPadded = function(length) { // format leading zeros
  31.   'use strict';
  32.   length = length || 2;
  33.   return ("0000000"+this).substr(-length);
  34. };
  35. Math.decRound = function(v, shift) { // decimal rounding
  36.   'use strict';
  37.   v = Math.round(+(v+"e"+shift));
  38.   return +(v+"e"+-shift);
  39. };
  40. Date.prototype.toLocalISOString = function() { // ISOString by local timezone
  41.   'use strict';
  42.   var pad = function(num) { return Number(num).toPadded(); },
  43.       offset = pad(Math.floor(this.getTimezoneOffset()/60)) + pad(this.getTimezoneOffset()%60),
  44.       timezone = this.getTimezoneOffset() > 0 ? "-" + offset : "+" + offset;
  45.   return this.getFullYear() + "-" + pad(this.getMonth()+1) + "-" + pad(this.getDate()) +
  46.     "T" + pad(this.getHours()) + ":" + pad(this.getMinutes()) + ":" + pad(this.getSeconds()) + timezone;
  47. };
  48. if (!('includes' in Array.prototype)) Array.prototype.includes = function(arg) { 'use strict'; return Boolean(~this.indexOf(arg)); };
  49. /***********************************************************************************************/
  50.  
  51. (function() { // simplify strict scoping
  52.   'use strict';
  53.  
  54.   var qc = { 
  55.     //extraDays: !!localStorage.getItem("hitdb_extraDays") || false, 
  56.     save: function(key, name, isObj) {
  57.       if (isObj) 
  58.         localStorage.setItem(name, JSON.stringify(this[key]));
  59.       else 
  60.         localStorage.setItem(name, this[key]);
  61.     }
  62.   },
  63.       metrics = {};
  64.  
  65.   var 
  66.   HITStorage = { //{{{
  67.     data: {}, db: null,
  68.     versionChange: function hsversionChange() { //{{{
  69.       var db = this.result;
  70.       db.onversionchange = function(e) { console.log("detected version change??",console.dir(e)); db.close(); };
  71.       var dbo, idx;
  72.  
  73.       console.groupCollapsed("HITStorage.versionChange::onupgradeneeded");
  74.  
  75.       if (!db.objectStoreNames.contains("HIT")) { 
  76.         console.log("creating HIT OS");
  77.         dbo = db.createObjectStore("HIT", { keyPath: "hitId" });
  78.         for (idx of ['date', 'requesterName', 'title', 'reward', 'bonus', 'status', 'requesterId'])
  79.           dbo.createIndex(idx, idx, { unique: false });
  80.  
  81.         //localStorage.setItem("hitdb_extraDays", true);
  82.         //qc.extraDays = true;
  83.       } else if (!this.transaction.objectStore('HIT').indexNames.contains('bonus')) {
  84.         this.transaction.objectStore('HIT').createIndex('bonus','bonus',{ unique: false });
  85.       }
  86.       
  87.       if (!db.objectStoreNames.contains("STATS")) {
  88.         console.log("creating STATS OS");
  89.         dbo = db.createObjectStore("STATS", { keyPath: "date" });
  90.       }
  91.       if (this.transaction.objectStore("STATS").indexNames.length < 5) { // new in v5: schema additions
  92.         for (idx of ['approved', 'earnings', 'pending', 'rejected', 'submitted'])
  93.           this.transaction.objectStore("STATS").createIndex(idx, idx, { unique: false });
  94.       }
  95.  
  96.       if (db.objectStoreNames.contains("NOTES") && this.transaction.objectStore("NOTES").indexNames.length < 3)
  97.         db.deleteObjectStore("NOTES");
  98.  
  99.       if (!db.objectStoreNames.contains("NOTES")) { // new in v5; schema change
  100.         console.log("creating NOTES OS");
  101.         dbo = db.createObjectStore("NOTES", { keyPath: "id", autoIncrement: true });
  102.         dbo.createIndex("hitId", "hitId", { unique: false });
  103.         dbo.createIndex("requesterId", "requesterId", { unique: false });
  104.         dbo.createIndex("tags", "tags", { unique: false, multiEntry: true });
  105.         dbo.createIndex("date", "date", { unique: false });
  106.       }
  107.  
  108.       if (db.objectStoreNames.contains("BLOCKS"))
  109.         db.deleteObjectStore("BLOCKS");
  110.  
  111.       console.groupEnd();
  112.     }, // }}} versionChange
  113.     parseDOM: function(doc) {//{{{
  114.       Status.color = "black";
  115.  
  116.       var errorCheck = doc.querySelector('td[class="error_title"]'),
  117.           extraInfo = [];
  118.  
  119.       if (doc.title.search(/Status$/) > 0)            // status overview
  120.         parseStatus();
  121.       else if (doc.querySelector('td[colspan="4"]'))  // valid status detail, but no data
  122.         parseMisc("next");
  123.       else if (doc.title.search(/Status Detail/) > 0) // status detail with data
  124.         parseDetail();
  125.       else if (errorCheck) {                          // encountered an error page
  126.         // hit max request rate
  127.         if (~errorCheck.textContent.indexOf("page request rate")) {
  128.           var _d = doc.documentURI.match(/\d{8}/)[0],
  129.               _p = doc.documentURI.match(/ber=(\d+)/)[1];
  130.           metrics.dbupdate.mark("[PRE]"+_d+"p"+_p, "start");
  131.           console.log("exceeded max requests; refetching %sp%s", _d, _p);
  132.           Status.node.innerHTML = "Exceeded maximum server requests; retrying "+Utils.ISODate(_d)+" page "+_p+"."+
  133.             "<br>Please wait...";
  134.           setTimeout(HITStorage.fetch, 3050, doc.documentURI);
  135.           return;
  136.         }
  137.         // no more staus details left in range
  138.         else if (qc.extraDays)
  139.           parseMisc("end");
  140.         else
  141.           Utils.errorHandler(new Error("Failed to parse '" + doc.documentURI + "'"));
  142.       }
  143.       else 
  144.         Utils.errorHandler(new Error("Unhandled document '" + doc.docuemntURI + "'"));
  145.  
  146.       function parseStatus() {//{{{
  147.         HITStorage.data = { HIT: [], STATS: [] };
  148.         qc.seen = {};
  149.  
  150.         qc.aat = JSON.parse(localStorage.getItem("hitdb_autoAppTemp") || "{}");
  151.         qc.fetchData = JSON.parse(localStorage.getItem("hitdb_fetchData") || "{}");
  152.         ProjectedEarnings.clear();
  153.         var _pastDataExists = Boolean(Object.keys(qc.fetchData).length),
  154.             timeout = 0, scope = [],
  155.             range = _pastDataExists ? Object.keys(qc.fetchData).filter(v => !isNaN(v)) : [],
  156.             raw = { 
  157.           day: doc.querySelectorAll(".statusDateColumnValue"), 
  158.           sub: doc.querySelectorAll(".statusSubmittedColumnValue"),
  159.           app: doc.querySelectorAll(".statusApprovedColumnValue"),
  160.           rej: doc.querySelectorAll(".statusRejectedColumnValue"),
  161.           pen: doc.querySelectorAll(".statusPendingColumnValue"),
  162.           pay: doc.querySelectorAll(".statusEarningsColumnValue") 
  163.         };
  164.         for (var i=0;i<raw.day.length;i++) {
  165.           var d = {};
  166.           var _date = raw.day[i].childNodes[1].href.substr(53);
  167.           d.date      = Utils.ISODate(_date);
  168.           d.submitted = +raw.sub[i].textContent;
  169.           d.approved  = +raw.app[i].textContent;
  170.           d.rejected  = +raw.rej[i].textContent;
  171.           d.pending   = +raw.pen[i].textContent;
  172.           d.earnings  = +raw.pay[i].textContent.substr(1);
  173.           HITStorage.data.STATS.push(d);
  174.  
  175.           // bonus received on date with 0 HITs
  176.           if (!d.submitted && !d.pending) { if (_date in qc.fetchData) delete qc.fetchData[_date]; continue; }
  177.           // check whether or not we need to get status detail pages for date, then
  178.           // fetch status detail pages per date in range and slightly slow
  179.           // down GET requests to avoid making too many in too short an interval
  180.           var payload = { encodedDate: _date, pageNumber: 1, sortType: "All" };
  181.           if (_pastDataExists) {
  182.             // date not in range but is new date (or old date but we need updates)
  183.             // lastDate stored in ISO format, fetchData date keys stored in mturk's URI ecnodedDate format
  184.             if ( (d.date >= qc.fetchData.lastDate) || ~(Object.keys(qc.fetchData).indexOf(_date)) || 
  185.                 (d.pending && !~Object.keys(qc.fetchData).indexOf(_date))) {
  186.               setTimeout(HITStorage.fetch, timeout, MTURK_BASE+"statusdetail", payload);
  187.               timeout += 380;
  188.  
  189.               qc.fetchData[_date] = { submitted: d.submitted, pending: d.pending };
  190.               scope.push(_date);
  191.             } 
  192.           } else { // get everything
  193.             setTimeout(HITStorage.fetch, timeout, MTURK_BASE+"statusdetail", payload);
  194.             timeout += 380;
  195.  
  196.             qc.fetchData[_date] = { submitted: d.submitted, pending: d.pending };
  197.           }
  198.         } // for
  199.  
  200.         // remove out of range dates to prevent lockup when scanning after a long hiatus
  201.         range.filter(v => !scope.includes(v)).forEach(v => delete qc.fetchData[v]);
  202.  
  203.         // try for extra days
  204.         if (qc.extraDays === true) {
  205.           localStorage.removeItem("hitdb_extraDays");
  206.           d = _decDate(HITStorage.data.STATS[HITStorage.data.STATS.length-1].date);
  207.           qc.extraDays = d; // repurpose extraDays for QC
  208.           payload = { encodedDate: d, pageNumber: 1, sortType: "All" };
  209.           setTimeout(HITStorage.fetch, 1000, MTURK_BASE+"statusdetail", payload);
  210.         }
  211.         qc.fetchData.expectedTotal = _calcTotals(qc.fetchData);
  212.         qc.fetchData.lastDate = HITStorage.data.STATS[0].date; // most recent date seen
  213.         qc.save("fetchData", "hitdb_fetchData", true);
  214.  
  215.       }//}}} parseStatus
  216.       function parseDetail() {//{{{
  217.         var _date = doc.documentURI.replace(/.+(\d{8}).+/, "$1"),
  218.             _page = doc.documentURI.replace(/.+ber=(\d+).+/, "$1"),
  219.             getExtras = function(entry) {
  220.               return new Promise( function(y) {
  221.                 HITStorage.db.transaction('HIT', 'readonly').objectStore('HIT').get(entry.hitId).onsuccess = function() {
  222.                   if (this.result && +this.result.bonus)
  223.                     entry.bonus = +this.result.bonus;
  224.                   if (this.result && +this.result.autoAppTime)
  225.                     entry.autoAppTime = this.result.autoAppTime;
  226.                   HITStorage.data.HIT.push(entry); y(1);
  227.                 };
  228.               });
  229.             };
  230.  
  231.         metrics.dbupdate.mark("[PRE]"+_date+"p"+_page, "end");
  232.         Status.message = "Processing "+Utils.ISODate(_date)+" page "+_page;
  233.         var raw = {
  234.           req:      doc.querySelectorAll(".statusdetailRequesterColumnValue"),
  235.           title:    doc.querySelectorAll(".statusdetailTitleColumnValue"),
  236.           pay:      doc.querySelectorAll(".statusdetailAmountColumnValue"),
  237.           status:   doc.querySelectorAll(".statusdetailStatusColumnValue"),
  238.           feedback: doc.querySelectorAll(".statusdetailRequesterFeedbackColumnValue")
  239.         };
  240.  
  241.         for (var i=0;i<raw.req.length;i++) {
  242.           var d = {};
  243.           d.date          = Utils.ISODate(_date);
  244.           d.feedback      = raw.feedback[i].textContent.trim().replace(/[\n\t]/g, ' ');
  245.           d.hitId         = raw.req[i].childNodes[1].href.replace(/.+HIT\+(.+)/, "$1");
  246.           d.requesterId   = raw.req[i].childNodes[1].href.replace(/.+rId=(.+?)&.+/, "$1");
  247.           d.requesterName = raw.req[i].textContent.trim();
  248.           d.reward        = +raw.pay[i].textContent.substr(1);
  249.           d.status        = raw.status[i].textContent.replace(/\s/g, " "); // replace char160 spaces with char32 spaces
  250.           d.title         = raw.title[i].textContent.trim();
  251.  
  252.           // mturk apparently never marks $0.00 HITs as 'Paid' so we fix that
  253.           if (!d.reward && ~d.status.search(/approved/i)) d.status = "Paid";
  254.           // insert autoApproval times
  255.           d.autoAppTime = HITStorage.autoApprovals.getTime(_date,d.hitId);
  256.  
  257.           extraInfo.push(getExtras(d));
  258.  
  259.           if (!qc.seen[_date]) qc.seen[_date] = {};
  260.           qc.seen[_date] = { 
  261.             submitted:   qc.seen[_date].submitted + 1 || 1,
  262.             pending: ~d.status.search(/pending/i)  ? 
  263.               (qc.seen[_date].pending + 1 || 1) : (qc.seen[_date].pending || 0)
  264.           };
  265.  
  266.           ProjectedEarnings.updateValues(d);
  267.         } //for
  268.  
  269.         // additional pages remain; get them
  270.         var morePages = doc.querySelector('img[src="/media/right_dbl_arrow.gif"]')/* || doc.querySelector('a[href*="&pageNumber='+(_page+1)+'"]')*/;
  271.         if (morePages) {
  272.           var payload = { encodedDate: _date, pageNumber: +_page+1, sortType: "All" };
  273.           setTimeout(HITStorage.fetch, 380, MTURK_BASE+"statusdetail", payload);
  274.           return;
  275.         } else if (Utils.ISODate(_date) !== qc.fetchData.lastDate &&
  276.             qc.seen[_date].submitted === qc.fetchData[_date].submitted && qc.seen[_date].pending === 0) {
  277.           console.log("no more pending hits, removing",_date,"from fetchData");
  278.           delete qc.fetchData[_date];
  279.           qc.save("fetchData", "hitdb_fetchData", true);
  280.         }
  281.  
  282.         if (!qc.extraDays) { // not fetching extra days
  283.           console.log("date:", _date, "pages:", _page, "totals:", _calcTotals(qc.seen), "of", qc.fetchData.expectedTotal);
  284.           Status.message += " [ "+_calcTotals(qc.seen)+"/"+ qc.fetchData.expectedTotal+" ]";
  285.           if (_calcTotals(qc.seen) === qc.fetchData.expectedTotal) {
  286.             Status.message = "Writing to database...";
  287.             HITStorage.autoApprovals.purge();
  288.             Promise.all(extraInfo).then(function() { HITStorage.write(HITStorage.data, cbUpdate); });
  289.           } 
  290.         } else if (_date <= qc.extraDays) { // day is older than default range and still fetching extra days
  291.           parseMisc("next");
  292.           console.log("fetchrequest for", _decDate(Utils.ISODate(_date)));
  293.         }
  294.       }//}}} parseDetail
  295.       function parseMisc(type) {//{{{
  296.         var _d = doc.documentURI.match(/\d{8}/)[0],
  297.             _p = doc.documentURI.match(/ber=(\d+)/)[1];
  298.         metrics.dbupdate.mark("[PRE]"+_d+"p"+_p, "end");
  299.         var payload = { encodedDate: _decDate(Utils.ISODate(_d)), pageNumber: 1, sortType: "All" };
  300.  
  301.         if (type === "next" && +qc.extraDays > 1) {
  302.           setTimeout(HITStorage.fetch, 250, MTURK_BASE+"statusdetail", payload);
  303.           console.log("going to next page", payload.encodedDate);
  304.         } else if (type === "end" && +qc.extraDays > 1) {
  305.           Status.message = "Writing to database...";
  306.           Promise.all(extraInfo).then(function() { HITStorage.write(HITStorage.data, cbUpdate); });
  307.         } else 
  308.           Utils.errorHandler(new TypeError("Failed to execute '"+type+"' in '"+doc.documentURI+"'"));
  309.       }//}}}
  310.       function _decDate(date) {//{{{
  311.         var y = date.substr(0,4);
  312.         var m = date.substr(5,2);
  313.         var d = date.substr(8,2);
  314.         date = new Date(y,m-1,d-1);
  315.         return Number(date.getMonth()+1).toPadded() + Number(date.getDate()).toPadded() + date.getFullYear();
  316.       }//}}}
  317.       function _calcTotals(obj) {//{{{
  318.         var sum = 0;
  319.         for (var k in obj){
  320.           if (obj.hasOwnProperty(k) && !isNaN(+k)) 
  321.             sum += obj[k].submitted;
  322.         }
  323.         return sum;
  324.       }//}}}
  325.     },//}}} parseDOM
  326.     autoApprovals: {//{{{
  327.       getTime : function(date, hitId) {
  328.         if (qc.extraDays || !Object.keys(qc.aat).length) return "";
  329.         var found = false,
  330.             autoApp = "";
  331.  
  332.         if (!found && Object.keys(qc.aat).length) {
  333.           for (var key in qc.aat) { if (qc.aat.hasOwnProperty(key)) { // for all dates in aat
  334.             var id = Object.keys(qc.aat[key]).filter(id => id === hitId)[0];
  335.             autoApp = qc.aat[key][id] || "";
  336.             if (autoApp) {
  337.               found = true;
  338.               delete qc.aat[key][id];
  339.               qc.save("aat", "hitdb_autoAppTemp", true);
  340.               break;
  341.             }
  342.           }} // for key (dates)
  343.         } // if !found && aat not empty
  344.         return autoApp;
  345.       },// getTime
  346.       purge : function() {
  347.         if (!Object.keys(qc.aat).length) return; // nothing here
  348.         var pad = function(num) { return Number(num).toPadded(); },
  349.             _date = Date.parse(new Date().getFullYear() + "-" + pad(new Date().getMonth()+1) + "-" + pad(new Date().getDate()));
  350.  
  351.         for (var key of Object.keys(qc.aat)) {
  352.           if (_date - key > 169200000) delete qc.aat[key]; // at least 2 days old, no need to keep it around
  353.         }
  354.         qc.save("aat", "hitdb_autoAppTemp", true);
  355.       } // purge
  356.     },//}}} autoApprovals
  357.     fetch: function(url, payload) { //{{{
  358.       //format GET request with query payload
  359.       if (payload) {
  360.         var args = 0;
  361.         url += "?";
  362.         for (var k in payload) {
  363.           if (payload.hasOwnProperty(k)) {
  364.             if (args++) url += "&";
  365.             url += k + "=" + payload[k];
  366.           }
  367.         }
  368.       }
  369.       // defer XHR to a promise
  370.       var fetch = new Promise( function(fulfill, deny) {
  371.         var urlreq = new XMLHttpRequest();
  372.         urlreq.open("GET", url, true);
  373.         urlreq.responseType = "document";
  374.         urlreq.send();
  375.         urlreq.onload = function() { 
  376.           if (this.status === 200) {
  377.             fulfill(this.response);
  378.           } else {
  379.             deny(new Error(this.status + " - " + this.statusText));
  380.           }
  381.         };
  382.         urlreq.onerror   = function() { deny(new Error(this.status + " - " + this.statusText)); };
  383.         urlreq.ontimeout = function() { deny(new Error(this.status + " - " + this.statusText)); };
  384.       } );
  385.       fetch.then( HITStorage.parseDOM, Utils.errorHandler );
  386.  
  387.     }, //}}} fetch
  388.     write: function(input, callback) { //{{{
  389.       var counts = { requests: 0, total: 0 },
  390.           os = Object.keys(input),
  391.           dbo = [],
  392.           dbt = HITStorage.db.transaction(os, "readwrite");
  393.       for (var i=0;i<os.length;i++) { // cycle object stores
  394.         dbo[i] = dbt.objectStore(os[i]);
  395.         for (var k of input[os[i]]) { // cycle entries to put into object stores
  396.           if (typeof k.reward === 'object') { k.bonus = k.reward.bonus; k.reward = k.reward.pay; }
  397.           if (typeof callback === 'function' && ++counts.requests)
  398.             dbo[i].put(k).onsuccess = callback.bind(counts);
  399.           else
  400.             dbo[i].put(k);
  401.         }
  402.       }
  403.     }, //}}} write
  404.     recall: function(store, options) {//{{{
  405.       var _cb = function(cursor) {
  406.             try { Status.message = `Retrieving data... [ ${matches} / ${++total} ]`; } catch(e) {}
  407.             if (filter(cursor.value)) {
  408.               sr.include(cursor.value);
  409.               try { Status.message = `Retrieving data... [ ${++matches} / ${total} ]`; } catch(e) {}
  410.             }
  411.             cursor.continue();
  412.           },
  413.           o = Object.assign({
  414.         index: null, range: null, dir: 'next', limit: Infinity, progress: false, mode: 'readonly', callback: _cb,
  415.         status: null, query: null, date: null, reward: null, bonus: null, requesterId: null, requesterName: null
  416.       }, options || {});
  417.       if (o.status === '*') o.status = null; //if (o.query === '*') o.query = null;
  418.       if (o.progress) Progress.show();
  419.  
  420.       var sr = new DBResult(), matches = 0, total = 0,
  421.           filter = function(obj) {//{{{
  422.             var fields = ['status', 'query', 'reward', 'bonus', 'requesterId', 'requesterName'], matches = {};
  423.             // out of date range
  424.             if (o.date && (obj.date < (o.date[0]) || obj.date > (o.date[1]))) return false;
  425.  
  426.             for (var f of fields) {
  427.               matches[f] = false;
  428.               if (!o[f]) {
  429.                 matches[f] = true;
  430.               } else if (f === 'query') { // general search - title, rname, hitid
  431.                 if ((obj.title + obj.requesterName + obj.hitId).toLowerCase().includes(o[f].toLowerCase())) matches[f] = true;
  432.               } else if (!isNaN(obj[f])) {// number
  433.                 if (+obj[f] >= o[f][0] && +obj[f] <= o[f][1]) matches[f] = true;
  434.               } else { // text
  435.                 if (obj[f] && obj[f].toLowerCase().includes(o[f].toLowerCase())) matches[f] = true;
  436.               }
  437.             }
  438.             return fields.reduce((a,b) => a && matches[b], true);
  439.           };//}}}
  440.  
  441.       return new Promise( function(resolve) {
  442.         var dbo = HITStorage.db.transaction(store, o.mode).objectStore(store), dbq = null;
  443.         if (o.index) 
  444.           dbq = dbo.index(o.index).openCursor(o.range, o.dir);
  445.         else
  446.           dbq = dbo.openCursor(o.range, o.dir);
  447.         dbq.onsuccess = function() {
  448.           if (this. result && matches < o.limit)
  449.             o.callback(this.result);
  450.           else {
  451.             try { Status.message = "Done."; } catch(e) {}
  452.             resolve(sr);
  453.           }
  454.         }; // IDBCursor
  455.       }); // promise
  456.     },//}}} HITStorage::recall
  457.     backup: function(internal) {//{{{
  458.       var bData = {},
  459.           os    = ["STATS", "NOTES", "HIT"],
  460.           count = 0;
  461.  
  462.       Progress.show();
  463.       Status.push("Preparing backup...", "black");
  464.  
  465.       for (var store of os) 
  466.         HITStorage.db.transaction(os, "readonly").objectStore(store).openCursor().onsuccess = populateBackup;
  467.  
  468.       function populateBackup(e) {
  469.         var cursor = e.target.result;
  470.         if (cursor) {
  471.           if (!bData[cursor.source.name]) bData[cursor.source.name] = [];
  472.           bData[cursor.source.name].push(cursor.value);
  473.           cursor.continue();
  474.         } else 
  475.           if (++count === 3)
  476.             finalizeBackup();
  477.       }
  478.       function finalizeBackup() {
  479.         if (typeof internal === 'function') { qc.merge = bData; internal(true); return; }
  480.         var backupblob = new Blob([JSON.stringify(bData)], {type:"application/json"});
  481.         var date = new Date();
  482.         var dl = document.createElement("A");
  483.         date = date.getFullYear() + Number(date.getMonth()+1).toPadded() + Number(date.getDate()).toPadded();
  484.         dl.href = URL.createObjectURL(backupblob);
  485.         console.log(dl.href);
  486.         dl.download = "hitdb_"+date+".bak";
  487.         document.body.appendChild(dl); // FF doesn't support forced events unless element is part of the document
  488.         dl.click();                    // so we make it so and click,
  489.         dl.remove();                   // then immediately remove it
  490.         Progress.hide();
  491.         Status.push("Done!", "green");
  492.       }
  493.  
  494.     }//}}} backup
  495.   }, //}}} HITStorage
  496.  
  497.   Utils = { //{{{
  498.     disableButtons: function(arr, status) { //{{{
  499.       for (var b of arr) document.getElementById(b).disabled = status;
  500.     }, //}}}
  501.     ftime : function(t,options) {//{{{
  502.       if (t !== undefined && isNaN(t)) return;
  503.       options = Object.assign({verbose: false, partial: true}, options);
  504.       var units = ['day', 'hour', 'minute', 'second'];
  505.       units = units.reduce((a,b) => (a[b] = options.verbose ? ' '+b : b.charAt()) && a, {});
  506.       units.day = options.partial ? ' day' : units.day;
  507.       if (String(t).length && +t === 0) return [0 + units.second].map(v => options.verbose ? v+'s' : v);
  508.       if (!t) return ['n/a'];
  509.       var pluralize = (num, str) => num > 1 && str.length > 1 ? num + str + 's' : num + str,
  510.         time = [ pluralize(Math.floor(t/86400), units.day),
  511.           pluralize(Math.floor(t%86400/3600), units.hour),
  512.           pluralize(Math.floor(t%86400%3600/60), units.minute),
  513.           pluralize(t%86400%3600%60, units.second) ];
  514.  
  515.       return time.filter(v => +v.charAt() > 0);
  516.     },//}}}ftime
  517.     ISODate: function(date) { //{{{ MMDDYYYY <-> YYYY-MM-DD
  518.       if (date.length === 10)
  519.         return date.substr(5,2)+date.substr(-2)+date.substr(0,4);
  520.       else
  521.         return date.substr(4)+"-"+date.substr(0,2)+"-"+date.substr(2,2);
  522.     },//}}} ISODate
  523.     getPosition: function(element, includeHeight) {//{{{
  524.       var offsets = { x: 0, y: includeHeight ? element.offsetHeight : 0 };
  525.       do {
  526.         offsets.x += element.offsetLeft;
  527.         offsets.y += element.offsetTop;
  528.         element = element.offsetParent;
  529.       } while (element);
  530.       return offsets;
  531.     },//}}} getPosition
  532.     errorHandler: function(err) {//{{{
  533.       try { Status.push(err.name + ": " + err.message, "red"); }
  534.       catch(e) {}
  535.       finally { 
  536.         console.error(err);
  537.         if (err.message.includes('AccessViolation')) {
  538.           var _m = 'HITdb probably needs to run an internal update.\nPlease close all tabs running HITdb to complete the process. ' +
  539.               'This includes all other MTurk pages and all tabs running HIT Scraper.',
  540.               span = str => '<span style="color:#c60;margin-top:6px;width:400px;display:block;position:relative;left:50%;transform:translateX(-50%)">' + str + '</span>';
  541.           console.warn(_m);
  542.           try { Status.html = Status.html + span(_m); } catch(e) {}
  543.         }
  544.       }
  545.     },//}}}
  546.     updateTimestamp: function() {//{{{
  547.       var time = document.querySelector('time'),
  548.         then = Date.parse(time.getAttribute('datetime')),
  549.         now = Date.now(),
  550.         diff = Utils.ftime(Math.floor((now-then)/1000), {verbose:true});
  551.       if (!diff) return;
  552.       time.textContent = diff[0] + ' ago';
  553.       time.title = [new Date(then), '\n', 'Last updated:', diff.join(' '), 'ago'].join(' ');
  554.     }//}}}
  555.   }, //}}} Utils
  556.  
  557.   ProjectedEarnings = {//{{{
  558.     //
  559.     // TODO: definitely need to refactor... possibly into a factory
  560.     //
  561.     data: JSON.parse(localStorage.getItem("hitdb_projectedEarnings") || "{}"),
  562.     updateDate: function() {//{{{
  563.       var tableList = document.querySelectorAll(".metrics-table"), el, date;
  564.       try {
  565.         el   = tableList[5].rows[1].cells[0].children[0];
  566.         date = el.href.match(/\d{8}/)[0];
  567.       } catch(e1) {
  568.         try {
  569.           el   = tableList[3].rows[1].cells[0].children[0];
  570.           date = el.href.match(/\d{8}/)[0];
  571.         } catch(e2) {
  572.           for (var tbl of tableList) {
  573.             if (tbl.rows.length < 2 || tbl.rows[1].cells.length < 6) continue;
  574.             el   = tbl.rows[1].cells[0].children[0];
  575.             date = el.href.match(/\d{8}/)[0];
  576.           } //for
  577.         }//catch
  578.       }//catch
  579.       var day       = el ? el.textContent : null,
  580.           isToday   = day === "Today",
  581.           _date     = new Date(),
  582.           weekEnd   = null,
  583.           weekStart = null;
  584.  
  585.       _date.setDate(_date.getDate() - _date.getDay()); // sunday
  586.       weekStart = Date.parse(_date.toLocalISOString().slice(0,10));
  587.       _date.setDate(_date.getDate() + 7); // next sunday
  588.       weekEnd   = Date.parse(_date.toLocalISOString().slice(0,10));
  589.  
  590.       if (!Object.keys(this.data).length) {
  591.         this.data = {
  592.           today: date, weekStart: weekStart, weekEnd: weekEnd, day: new Date().getDay(), dbUpdated: "n/a",
  593.           pending: 0, approved: 0, earnings: {}, target: { day: 0, week: 0 }
  594.         };
  595.       }
  596.  
  597.       if ( (Date.parse(Utils.ISODate(date)) >= this.data.weekEnd) ||
  598.           (!isToday && new Date().getDay() < this.data.day) ) { // new week
  599.         this.data.earnings = {};
  600.         this.data.weekEnd = weekEnd;
  601.         this.data.weekStart = weekStart;
  602.       }
  603.       if ( (this.data.today === null && isToday) || (this.data.today !== null && (date !== this.data.today || !isToday)) ) { // new day
  604.         this.data.today = date === this.data.today ? null : date;
  605.         this.data.day = new Date().getDay();
  606.       }
  607.  
  608.       this.saveState();
  609.     },//}}} updateDate
  610.     draw: function(init) {//{{{
  611.       var parentTable = document.querySelector("#total_earnings_amount").offsetParent,
  612.           rowPending  = init ? parentTable.insertRow(-1) : parentTable.rows[4],
  613.           rowProjectedDay = init ? parentTable.insertRow(-1) : parentTable.rows[5],
  614.           rowProjectedWeek = init ? parentTable.insertRow(-1) : parentTable.rows[6],
  615.           title = "Click to set/change the target value",
  616.           weekTotal = this.getWeekTotal(),
  617.           dayTotal = this.data.earnings[this.data.today] || 0;
  618.  
  619.       if (init) {
  620.         rowPending.insertCell(-1);rowPending.insertCell(-1);rowPending.className = "even";
  621.         rowProjectedDay.insertCell(-1);rowProjectedDay.insertCell(-1);rowProjectedDay.className = "odd";
  622.         rowProjectedWeek.insertCell(-1);rowProjectedWeek.insertCell(-1);rowProjectedWeek.className = "even";
  623.         for (var i=0;i<rowPending.cells.length;i++) rowPending.cells[i].style.borderTop = "dotted 1px black";
  624.         rowPending.cells[0].className = "metrics-table-first-value";
  625.         rowProjectedDay.cells[0].className = "metrics-table-first-value";
  626.         rowProjectedWeek.cells[0].className = "metrics-table-first-value";
  627.       }
  628.  
  629.       rowPending.cells[1].title = "This value includes all earnings that are not yet fully cleared as 'Paid'\n" +
  630.         "\n   Pending Approval: $" + (this.data.pending - (this.data.approved || 0)).toFixed(2) +
  631.         "\n   Pending Payment: $" + (this.data.approved || 0).toFixed(2) +
  632.         "\n   Total Pending:        $" + this.data.pending.toFixed(2);
  633.       rowPending.cells[0].innerHTML = 'Pending earnings '+
  634.         `<span style="font-family:arial;font-size:10px;white-space:nowrap">[ Last updated: <time datetime="${this.data.dbUpdated}"></time> ]</span>`;
  635.       if (this.data.dbUpdated === 'n/a') rowPending.querySelector('time').textContent = 'Never';
  636.       else Utils.updateTimestamp();
  637.       rowPending.cells[1].textContent = "$" + this.data.pending.toFixed(2);
  638.       rowProjectedDay.cells[0].innerHTML = '<span>Projected earnings for the day</span>'+
  639.         '<div><meter id="projectedDayProgress" style="width:220px;" title="'+title+
  640.           '" value="'+dayTotal+'" max="'+this.data.target.day+'"></meter>'+
  641.         '<span style="color:blue;font-family:arial;font-size:10px;margin-left:3px">' +
  642.         (dayTotal-this.data.target.day).toFixed(2) + '</span></div>';
  643.       rowProjectedDay.cells[1].textContent = "$"+ dayTotal.toFixed(2);
  644.       rowProjectedWeek.cells[0].innerHTML = '<span>Projected earnings for the week</span>' +
  645.         '<div><meter id="projectedWeekProgress" style="width:220px;" title="'+title+
  646.           '" value="'+weekTotal+'" max="'+this.data.target.week+'"></meter>' +
  647.         '<span style="color:blue;font-family:arial;font-size:10px;margin-left:3px">' +
  648.         (weekTotal-this.data.target.week).toFixed(2) + '</span></div>';
  649.       rowProjectedWeek.cells[1].textContent = "$" + weekTotal.toFixed(2);
  650.       
  651.       document.querySelector("#projectedDayProgress").onclick = updateTargets.bind(this, "day");
  652.       document.querySelector("#projectedWeekProgress").onclick = updateTargets.bind(this, "week");
  653.  
  654.       function updateTargets(span, e) {
  655.         /*jshint validthis:true*/
  656.         var goal = prompt("Set your " + (span === "day" ? "daily" : "weekly") + " target:",
  657.             this.data.target[span === "day" ? "day" : "week"]);
  658.         if (goal && !isNaN(goal)) {
  659.           this.data.target[span === "day" ? "day" : "week"] = goal;
  660.           e.target.max = goal;
  661.           e.target.nextSibling.textContent = " " + ((span === "day" ? dayTotal : weekTotal) - goal).toFixed(2);
  662.           this.saveState();
  663.         }
  664.       }
  665.     },//}}} draw
  666.     updatePaint: function() {//{{{
  667.       var findRow = sel => { var el = document.querySelector(sel); while(el.tagName !== 'TR') el = el.parentElement; return el; },
  668.         rowPending = findRow('time'),
  669.         rowPD = findRow('#projectedDayProgress'),
  670.         rowPW = findRow('#projectedWeekProgress');
  671.  
  672.       rowPending.cells[1].title = "This value includes all earnings that are not yet fully cleared as 'Paid'\n" +
  673.         "\n   Pending Approval: $" + (this.data.pending - (this.data.approved || 0)).toFixed(2) +
  674.         "\n   Pending Payment: $" + (this.data.approved || 0).toFixed(2) +
  675.         "\n   Total Pending:        $" + this.data.pending.toFixed(2);
  676.       document.querySelector('time').setAttribute('datetime', this.data.dbUpdated);
  677.       Utils.updateTimestamp();
  678.       rowPending.cells[1].textContent = '$' + this.data.pending.toFixed(2);
  679.       rowPD.cells[1].textContent = '$' + (this.data.earnings[this.data.today] || 0).toFixed(2);
  680.       rowPW.cells[1].textContent = '$' + this.getWeekTotal().toFixed(2);
  681.       rowPD.querySelector('meter').value = this.data.earnings[this.data.today] || 0;
  682.       rowPW.querySelector('meter').value = this.getWeekTotal();
  683.     },//}}}
  684.     getWeekTotal: function() {//{{{
  685.       var totals = 0;
  686.       for (var k of Object.keys(this.data.earnings))
  687.         totals += this.data.earnings[k];
  688.  
  689.       return Math.decRound(totals, 2);
  690.     },//}}}
  691.     saveState: function() {//{{{
  692.       saveState("hitdb_projectedEarnings", JSON.stringify(this.data));
  693.     },//}}}
  694.     clear: function() {//{{{
  695.       this.data.pending = 0;
  696.       this.data.approved = 0;
  697.       for (var day of Object.keys(this.data.earnings))
  698.         if (day in qc.fetchData || day === this.data.today) this.data.earnings[day] = 0;
  699.     },//}}}
  700.     updateValues: function(obj) {//{{{
  701.       var vDate = Date.parse(obj.date), iDate = Utils.ISODate(obj.date);
  702.  
  703.       if (~obj.status.search(/pending/i)) // sum pending earnings (include approved until fully cleared as paid)
  704.         this.data.pending = Math.decRound(obj.reward+this.data.pending, 2);
  705.       if (~obj.status.search(/approved/i))
  706.         this.data.approved = Math.decRound(obj.reward+this.data.approved, 2);
  707.       if (vDate < this.data.weekEnd && vDate >= this.data.weekStart && !~obj.status.search(/rejected/i)){ // sum weekly earnings by day
  708.         this.data.earnings[iDate] = Math.decRound(obj.reward+(this.data.earnings[iDate] || 0), 2 );
  709.       }
  710.     }//}}}
  711.   },//}}} ProjectedEarnings
  712.  
  713.   DBResult = function(resArr, colObj) {//{{{
  714.     this.results = resArr || [];
  715.     this.collation = colObj || null;
  716.     this.formatHTML = function(type, simple) {//{{{
  717.       simple = simple || false;
  718.       var count = 0, htmlTxt = [], entry = null, _trClass = null;
  719.  
  720.       if (this.results.length < 1) return "<h2>No entries found matching your query.</h2>";
  721.  
  722.       if (type === "daily") {
  723.         htmlTxt.push('<thead><tr class="hdbHeaderRow"><th></th>'+
  724.             '<th>Date</th><th>Submitted</th><th>Approved</th><th>Rejected</th><th>Pending</th><th>Earnings</th></tr></thead><tbody>');
  725.         var r = this.collate(this.results,"stats");
  726.         for (entry of this.results) {
  727.           _trClass = (count++ % 2 === 0) ? 'class="even"' : 'class="odd"';
  728.           
  729.           htmlTxt.push('<tr '+_trClass+' style="text-align:right">' +
  730.               '<td><span class="hdbExpandRow">[+]</span></td>'+
  731.               '<td style="text-align:center;">' + entry.date + '</td><td>' + entry.submitted + '</td>' +
  732.               '<td>' + entry.approved + '</td><td>' + entry.rejected + '</td><td>' + entry.pending + '</td>' +
  733.               '<td>' + Number(entry.earnings).toFixed(2) + '</td></tr>');
  734.         }
  735.         htmlTxt.push('</tbody><tfoot><tr class="hdbTotalsRow" style="text-align:right;"><td>Totals:</td>' +
  736.             '<td>' + r.totalEntries + ' days</td><td>' + r.totalSub + '</td>' +
  737.             '<td>' + r.totalApp + '</td><td>' + r.totalRej + '</td>' +
  738.             '<td>' + r.totalPen + '</td><td>$' + 
  739.             Number(Math.decRound(r.totalPay,2)).toFixed(2) + '</td></tr></tfoot>');
  740.       } else if (type === "pending" || type === "requester") {
  741.         htmlTxt.push('<thead><tr data-sort="99999" class="hdbHeaderRow"><th width="160">Requester ID</th>' +
  742.             '<th>Requester</th><th>' + (type === "pending" ? 'Pending' : 'HITs') + '</th><th>Rewards</th></tr></thead><tbody>');
  743.         r = this.collate(this.results,"requesterId");
  744.         for (var k in r) {
  745.           if (!~k.search(/total/) && r.hasOwnProperty(k)) {
  746.             var tr = ['<tr data-sort="'+Math.decRound(r[k].pay,2)+'"><td>' +
  747.                 '<span class="hdbExpandRow" title="Display all pending HITs from this requester">' +
  748.                 '[+]</span> ' + r[k][0].requesterId + '</td><td>' + r[k][0].requesterName + '</td>' +
  749.                 '<td style="text-align:center;">' + r[k].length + '</td><td>' + Number(Math.decRound(r[k].pay,2)).toFixed(2) + '</td></tr>'];
  750.  
  751.             for (var hit of r[k]) {  // hits in range per requester id
  752.               tr.push('<tr data-rid="'+r[k][0].requesterId+'" style="color:#c60000;display:none;"><td style="text-align:right">' + 
  753.                   hit.date + '</td><td width="500" colspan="2" class="nowrap" style="max-width:520" title="'+hit.title+'">' +
  754.                   '[ <span class="helpSpan" title="Auto-approval time">AA: '+Utils.ftime(hit.autoAppTime).join(' ')+'</span> ] '+ 
  755.                   hit.title + '</td><td style="text-align:right">' + hit.reward.toFixed(2) + '</td></tr>');
  756.             }
  757.             htmlTxt.push(tr.join(''));
  758.           }
  759.         }
  760.         htmlTxt.sort(function(a,b) { return +b.substr(15,5).match(/\d+\.?\d*/) - +a.substr(15,5).match(/\d+\.?\d*/); });
  761.         htmlTxt.push('</tbody><tfoot><tr class="hdbTotalsRow"><td style="text-align:right;">Totals:</td>' +
  762.             '<td style="text-align:center;">' + (Object.keys(this.collation || r).length-7) + ' Requesters</td>' +
  763.             '<td style="text-align:right;">' + (this.collation || r).totalEntries + '</td>'+
  764.             '<td style="text-align:right;">$' + Number(Math.decRound((this.collation || r).totalPay,2)).toFixed(2) + '</td></tr></tfoot>');
  765.       } else { // default
  766.         if (!simple)
  767.           htmlTxt.push('<thead><tr class="hdbHeaderRow"><th colspan="3"></th>' +
  768.             '<th colspan="2" title="Bonuses must be added in manually.\n\nClick inside' +
  769.             'the cell to edit, click out of the cell to save">Reward</th><th colspan="3"></th></tr>'+
  770.             '<tr class="hdbHeaderRow">' +
  771.             '<th style="min-width:65">Date</th><th>Requester</th><th>HIT title</th><th style="font-size:10px;">Pay</th>'+
  772.             '<th style="font-size:10px;"><span class="helpSpan" title="Click the cell to edit.\nIts value is automatically saved">'+
  773.             'Bonus</span></th><th>Status</th><th>'+
  774.             '<span class="helpSpan" title="Auto-approval times">AA</span></th><th>Feedback</th></tr></thead><tbody>');
  775.  
  776.         this.results.sort(function(a,b) { return a.date === b.date ? 
  777.           (a.requesterName.toLowerCase() > b.requesterName.toLowerCase() ? 1 : -1) : a.date < b.date ? -1 : 1; });
  778.         for (entry of this.results) {
  779.           _trClass = (count++ % 2 === 0) ? 'class="even"' : 'class="odd"';
  780.           var _stColor = ~entry.status.search(/(paid|approved)/i) ? "green"  :
  781.                          entry.status === "Pending Approval"      ? "orange" : "red";
  782.           var href = MTURK_BASE+'contact?requesterId='+entry.requesterId+'&requesterName='+entry.requesterName+
  783.             '&subject=Regarding+Amazon+Mechanical+Turk+HIT+'+entry.hitId;
  784.  
  785.           if (!simple)
  786.             htmlTxt.push('<tr '+_trClass+' data-id="'+entry.hitId+'">'+
  787.               '<td width="74px">' + entry.date + '</td><td style="max-width:145px;">' +
  788.               '<a target="_blank" title="Contact this requester" href="'+href+'">' + entry.requesterName + '</a></td>' + 
  789.               '<td width="375px" title="HIT ID:   '+entry.hitId+'">' + 
  790.               '<span title="Add a note" id="note-'+entry.hitId+'" style="cursor:pointer;">&nbsp;&#128221;&nbsp;</span>' + 
  791.               entry.title + '</td><td style="text-align:right">' + entry.reward.toFixed(2) + '</td>' +
  792.               '<td style="text-align:right" class="bonusCell" title="Click to add/edit" contenteditable="true" data-hitid="'+entry.hitId+'">' + 
  793.               (entry.bonus ? entry.bonus.toFixed(2) : "") + 
  794.               '</td><td style="color:'+_stColor+';text-align:center">' + entry.status + '</td>' +
  795.               '<td>' + Utils.ftime(entry.autoAppTime).join(' ') + '</td><td>' + entry.feedback + '</td></tr>');
  796.           else
  797.             htmlTxt.push('<tr>' + '<td class="nowrap" title="'+entry.requesterName+'" style="max-width:130">'+entry.requesterName+'</td>' + 
  798.               '<td class="nowrap" title="'+entry.title+'" style="max-width:520">'+entry.title+'</td><td>'+ entry.reward.toFixed(2) + 
  799.               '</td><td class="nowrap" title="'+entry.status+'" style="max-width:60">'+ entry.status+'</td></tr>');
  800.         }
  801.  
  802.         if (!simple) {
  803.           r = this.collation || this.collate(this.results,"requesterId");
  804.           htmlTxt.push('</tbody><tfoot><tr class="hdbTotalsRow"><td></td>' +
  805.               '<td style="text-align:right">Totals:</td><td style="text-align:center;">' + r.totalEntries + ' HITs</td>' +
  806.               '<td style="text-align:right">$' + (+Math.decRound(r.totalPay,2)).toFixed(2) + '</td>' +
  807.               '<td style="text-align:right">$' + (+Math.decRound(r.totalBonus,2) || 0).toFixed(2) + '</td>' +
  808.               '<td colspan="3"></td></tr></tfoot>');
  809.         }
  810.       }
  811.       return htmlTxt.join('');
  812.     };//}}} formatHTML
  813.     this.formatCSV = function(type) {//{{{
  814.       var csvTxt = [], entry = null, delimiter="\t";
  815.       if (type === "daily") {
  816.         csvTxt.push( ["Date", "Submitted", "Approved", "Rejected", "Pending", "Earnings\n"].join(delimiter) );
  817.         for (entry of this.results) {
  818.           csvTxt.push( [entry.date, entry.submitted, entry.approved, entry.rejected, 
  819.               entry.pending, Number(entry.earnings).toFixed(2)+"\n"].join(delimiter) );
  820.         }
  821.         csvToFile(csvTxt, "hitdb_dailyOverview.csv");
  822.       } else if (type === "pending" || type === "requester") {
  823.         csvTxt.push( ["RequesterId","Requester", (type === "pending" ? "Pending" : "HITs"), "Rewards\n"].join(delimiter) );
  824.         var r = this.collation || this.collate(this.results,"requesterId");
  825.         for (var k in r) {
  826.           if (!~k.search(/total/) && r.hasOwnProperty(k))
  827.             csvTxt.push( [k, r[k][0].requesterName, r[k].length, Number(Math.decRound(r[k].pay,2)).toFixed(2)+"\n"].join(delimiter) );
  828.         }
  829.         csvToFile(csvTxt, "hitdb_"+type+"Overview.csv");
  830.       } else {
  831.         csvTxt.push(["hitId","date","requesterName","requesterId","title","pay","bonus","status","autoAppTime","feedback\n"].join(delimiter));
  832.         for (entry of this.results) {
  833.           csvTxt.push([entry.hitId, entry.date, entry.requesterName, entry.requesterId, entry.title, entry.reward.toFixed(2),
  834.               (entry.bonus ? entry.bonus.toFixed(2) : ''), entry.status, entry.autoAppTime, 
  835.               entry.feedback.replace(/[\t\n]/g,' ')+"\n"].join(delimiter));
  836.         }
  837.         csvToFile(csvTxt, "hitdb_queryResults.csv");
  838.       }
  839.  
  840.       return '';//"<pre>"+csvTxt.join('')+"</pre>";
  841.  
  842.       function csvToFile(csv, filename) {
  843.         var blob = new Blob(csv, {type: "text/csv", endings: "native"}),
  844.             dl   = document.createElement("A");
  845.         dl.href = URL.createObjectURL(blob);
  846.         dl.download = filename;
  847.         document.body.appendChild(dl); // FF doesn't support forced events unless element is part of the document
  848.         dl.click();                    // so we make it so and click,
  849.         dl.remove();                   // then immediately remove it
  850.         return dl;
  851.       }
  852.     };//}}} formatCSV
  853.     this.include = function(value) {
  854.       this.results.push(value);
  855.     };
  856.     this.collate = function(data, index) {//{{{
  857.       var r = { 
  858.         totalPay: 0, totalBonus: 0, totalEntries: data.length,
  859.         totalSub: 0, totalApp: 0, totalRej: 0, totalPen: 0
  860.       };
  861.       for (var e of data) {
  862.         if (!r[e[index]]) { 
  863.           r[e[index]] = [];
  864.           Object.defineProperty(r[e[index]], "pay", {value: 0, enumerable: false, configurable: true, writable: true});
  865.         }
  866.         r[e[index]].push(e);
  867.  
  868.         if (index === "stats") {
  869.           r.totalSub += e.submitted;
  870.           r.totalApp += e.approved;
  871.           r.totalRej += e.rejected;
  872.           r.totalPen += e.pending;
  873.           r.totalPay += e.earnings;
  874.         } else {
  875.           r[e[index]].pay += (+e.reward);
  876.           r.totalPay += (+e.reward);
  877.           r.totalBonus += (+e.bonus);
  878.         }
  879.       }
  880.       return r;
  881.     };//}}} _collate
  882.  
  883.   },//}}} databaseresult
  884.  
  885.   DashboardUI = {//{{{
  886.     draw: function() {//{{{
  887.       var controlPanel = document.createElement("TABLE"),
  888.           insertionNode = document.querySelector(".footer_separator").previousSibling;
  889.       document.body.insertBefore(controlPanel, insertionNode);
  890.       controlPanel.width = "760";
  891.       controlPanel.align = "center";
  892.       controlPanel.id = "hdbControlPanel";
  893.       controlPanel.cellSpacing = "0";
  894.       controlPanel.cellPadding = "0";
  895.       controlPanel.innerHTML = '<tr height="25px"><td width="10" bgcolor="#7FB448" style="padding-left: 10px;"></td>' +
  896.         '<td class="white_text_14_bold" style="padding-left:10px; background-color:#7FB448;">' +
  897.           'HIT Database Mk. II&nbsp;<a href="https://greasyfork.org/en/scripts/11733-mturk-hit-database-mk-ii#userGuide" '+
  898.             'class="whatis" target="turkPopUp" onclick="customPopup(this, 500, 400)">' +
  899.           '(What\'s this?)</a></td></tr>' +
  900.         '<tr><td class="container-content" colspan="2">' +
  901.         '<div style="text-align:center; position:relative" id="hdbDashboardInterface">' +
  902.          '<button id="hdbBackup" title="Export your entire database!\nPerfect for moving between computers or as a periodic backup">Create Backup</button>' +
  903.          '<button id="hdbRestore" title="Import data from an external file" style="margin:5px">Import</button>' +
  904.          '<button id="hdbUpdate" title="Update... the database" style="color:green;">Update Database</button>' +
  905.          '<input id="hdbFileInput" type="file" style="display:none"/>' +
  906.          '<br>'+
  907.          '<div style="position:absolute; top:0; right:0; text-align:initial">' +
  908.           //'<label title="Popout search results in a new window" style="vertical-align:middle;">popout' +
  909.           //'<input id="hdbPopout" type="checkbox" style="vertical-align:middle"></label>' +
  910.           '<label for="hdbCSVInput" title="Export results as CSV file" style="vertical-align:middle;">export CSV</label>' +
  911.           '<input id="hdbCSVInput" title="Export results as CSV file" type="checkbox" style="vertical-align:middle;">' +
  912.          '</div>' +
  913.          '<button id="hdbPending" title="Summary of all pending HITs\n Can be exported as CSV" style="margin: 0px 5px 5px;">Pending Overview</button>' +
  914.          '<button id="hdbRequester" title="Summary of all requesters\n Can be exported as CSV" style="margin: 0px 5px 5px;">Requester Overview</button>' +
  915.          '<button id="hdbDaily" title="Summary of each day you\'ve worked\nCan be exported as CSV" style="margin:0px 5px 5px;">Daily Overview</button>' +
  916.          '<br>' +
  917.          '<label>Find </label>' +
  918.          '<select id="hdbStatusSelect" style="width:100px"><option value="*">ALL</option>' +
  919.          '<option value="Pending Approval" style="color: orange;">Pending Approval</option>' +
  920.          '<option value="Rejected" style="color: red;">Rejected</option>' +
  921.          '<option value="Approved - Pending Payment" style="color:green;">Approved - Pending Payment</option>' +
  922.          '<option value="Paid" style="color:green;">Paid</option></select>' +
  923.          '<label> HITs from </label><input id="hdbMinDate" type="date" size="10" title="Specify a date, or leave blank">' +
  924.          '<label> to </label><input id="hdbMaxDate" type="date" size="10" title="Specify a date, or leave blank">' +
  925.          '<label> matching </label>'+
  926.          '<br>' +
  927.          '<input id="hdbSearchInput" style="width:400px" title="Query can be HIT title, HIT ID, or requester name" />' +
  928.          '<button id="hdbSearch" style="margin-left:5px">Search</button>' +
  929.          '<br>' +
  930.          '<label id="hdbStatusText"></label>' +
  931.          '<div id="hdbProgressBar">' +
  932.           '<div id="hdbB1" class="ball"></div><div id="hdbB2" class="ball"></div>' +
  933.           '<div id="hdbB3" class="ball"></div><div id="hdbB4" class="ball"></div>' +
  934.          '</div>' +
  935.         '</div></td></tr>';
  936.  
  937.       var searchResults = document.createElement("DIV");
  938.       searchResults.align = "center";
  939.       searchResults.id = "hdbSearchResults";
  940.       searchResults.style.display = "block";
  941.       searchResults.innerHTML = 
  942.         '<span class="hdbResControl" id="hdbResClear">[ clear results ]</span>' +
  943.         '<span class="hdbTablePagination" id="hdbPageTop"></span><br>' +
  944.         '<table cellSpacing="0" cellpadding="2" width="760" id="hdbResultsTable"></table>' +
  945.         '<span class="hdbResControl" id="hdbVpTop">Back to top</span>' +
  946.         '<span class="hdbTablePagination" id="hdbPageBot"></span><br>';
  947.       document.body.insertBefore(searchResults, insertionNode);
  948.     },//}}} dashboardUI::draw
  949.     initClickables: function() {//{{{
  950.       var updateBtn      = document.getElementById("hdbUpdate"),
  951.           backupBtn      = document.getElementById("hdbBackup"),
  952.           restoreBtn     = document.getElementById("hdbRestore"),
  953.           fileInput      = document.getElementById("hdbFileInput"),
  954.           exportCSVInput = document.getElementById("hdbCSVInput"),
  955.           searchBtn      = document.getElementById("hdbSearch"),
  956.           searchInput    = document.getElementById("hdbSearchInput"),
  957.           pendingBtn     = document.getElementById("hdbPending"),
  958.           reqBtn         = document.getElementById("hdbRequester"),
  959.           dailyBtn       = document.getElementById("hdbDaily"),
  960.           fromdate       = document.getElementById("hdbMinDate"),
  961.           todate         = document.getElementById("hdbMaxDate"),
  962.           statusSelect   = document.getElementById("hdbStatusSelect"),
  963.           searchResults  = document.getElementById("hdbSearchResults"),
  964.           resultsTable   = document.getElementById("hdbResultsTable"),
  965.           isGecko = /Gecko\/\d+/.test(navigator.userAgent);
  966.  
  967.       searchResults.firstChild.onclick = function() { //{{{ clear results
  968.         resultsTable.innerHTML = null; qc.sr = [];
  969.         for (var d of ["hdbResClear","hdbPageTop","hdbVpTop", "hdbPageBot"]) {
  970.           if (~d.search(/page/i)) searchResults.querySelector('#'+d).innerHTML = "";
  971.           document.getElementById(d).style.display = "none";
  972.         }
  973.       };//}}}
  974.       document.getElementById("hdbVpTop").onclick = function() { autoScroll("#hdbControlPanel"); };
  975.  
  976.       updateBtn.onclick = function() { //{{{
  977.         if (!HITStorage.db) { return Utils.errorHandler(new TypeError('(AccessViolation) Database is not defined')); }
  978.         Utils.disableButtons(['hdbUpdate'], true);
  979.         Progress.show();
  980.         metrics.dbupdate = new Metrics("database_update");
  981.         HITStorage.fetch(MTURK_BASE+"status");
  982.         Status.message = "fetching status page....";
  983.       };//}}}
  984.       exportCSVInput.addEventListener("click", function() {//{{{
  985.         var a = document.getElementById('hdbAnalytics');
  986.         if (a && a.checked) a.click();
  987.         if (exportCSVInput.checked) {
  988.           searchBtn.textContent = "Export CSV";
  989.           pendingBtn.textContent += " (csv)";
  990.           reqBtn.textContent += " (csv)";
  991.           dailyBtn.textContent += " (csv)";
  992.         }
  993.         else {
  994.           searchBtn.textContent = "Search";
  995.           pendingBtn.textContent = pendingBtn.textContent.replace(" (csv)","");
  996.           reqBtn.textContent = reqBtn.textContent.replace(" (csv)","");
  997.           dailyBtn.textContent = dailyBtn.textContent.replace(" (csv)", "");
  998.         }
  999.       });//}}}
  1000.       if (isGecko) {//{{{
  1001.         fromdate.addEventListener("focus", function() {
  1002.           var offsets = Utils.getPosition(this, true);
  1003.           new Calendar(offsets.x, offsets.y, this).drawCalendar();
  1004.         });
  1005.         todate.addEventListener("focus", function() {
  1006.           var offsets = Utils.getPosition(this, true);
  1007.           new Calendar(offsets.x, offsets.y, this).drawCalendar();
  1008.         });
  1009.       }//}}}
  1010.  
  1011.       backupBtn.onclick = HITStorage.backup;
  1012.       restoreBtn.onclick = function() { fileInput.value = ''; fileInput.click(); };
  1013.       fileInput.onchange = FileHandler.delegate;//processFile;
  1014.       searchInput.onkeydown = function(e) { if (e.keyCode === 13) searchBtn.click(); };
  1015.  
  1016.       searchBtn.addEventListener('click', function(e) {//{{{
  1017.         if (!/^[se]/i.test(e.target.textContent)) return;
  1018.         var opt = this.getRange(statusSelect.value, _getFilters(searchInput.value.trim()));
  1019.         opt.progress = true;
  1020.         if (opt.query && opt.query.length === 30 && !/\s/.test(opt.query)) {
  1021.           opt.range = window.IDBKeyRange.only(opt.query.toUpperCase());
  1022.           opt.index = null;
  1023.         }
  1024.         _dbaccess("search", ["HIT", opt], function(r) {
  1025.           var limiter = 500,
  1026.               _cb = function(slice) {
  1027.                 for (var _r of slice)
  1028.                   HITStorage.recall("NOTES", { index: "hitId", range: window.IDBKeyRange.only(_r.hitId) }).then(noteHandler.bind(null,"attach"));
  1029.                 
  1030.                 var _nodes   = [document.querySelectorAll(".bonusCell"), document.querySelectorAll('span[id^="note-"]')];
  1031.                 for (var i=0;i<_nodes[0].length;i++) {
  1032.                   var bonus = _nodes[0][i],
  1033.                       note  = _nodes[1][i];
  1034.                   bonus.dataset.initial = bonus.textContent;
  1035.                   bonus.onkeydown = updateBonus;
  1036.                   bonus.onblur    = updateBonus;
  1037.                   note.onclick    = noteHandler.bind(null,"new");
  1038.                 }
  1039.               };
  1040.           if (exportCSVInput.checked) 
  1041.             resultsTable.innerHTML = r.formatCSV();
  1042.           else if (r.results.length > limiter) {
  1043.             var collation = r.collate(r.results, "requesterId");
  1044.             do { qc.sr.push(new DBResult(r.results.splice(0,limiter), collation)) } while (r.results.length);
  1045.             resultConstrain(qc.sr, 0, "default", _cb);
  1046.           } else
  1047.             resultConstrain(r, 0, "default", _cb);
  1048.         });
  1049.       }.bind(this)); //}}} search button click event
  1050.       //{{{ overview buttons
  1051.       pendingBtn.onclick = function() {
  1052.         var opt = this.getRange('pending', _getFilters(searchInput.value.trim())),
  1053.             _opt = { index:'status', dir:'prev', range:window.IDBKeyRange.only('Pending Approval'), progress:true };
  1054.  
  1055.         opt = Object.assign(opt, _opt);
  1056.         _dbaccess("pending", ["HIT", opt], function(r) {
  1057.           resultsTable.innerHTML = exportCSVInput.checked ? r.formatCSV("pending") : r.formatHTML("pending");
  1058.           var expands = document.querySelectorAll(".hdbExpandRow");
  1059.           for (var el of expands)
  1060.             el.onclick = showHiddenRows;
  1061.         });
  1062.       }.bind(this); //pending overview click event
  1063.       reqBtn.onclick = function() {
  1064.         var opt = this.getRange(statusSelect.value, _getFilters(searchInput.value.trim()));
  1065.         opt.progress = true;
  1066.  
  1067.         _dbaccess("requester", ["HIT", opt], function(r) {
  1068.           var limiter = 100,
  1069.               _cb = function() {
  1070.                 var expands = document.querySelectorAll(".hdbExpandRow");
  1071.                 for (var el of expands) 
  1072.                   el.onclick = showHiddenRows;
  1073.               };
  1074.           if (exportCSVInput.checked)
  1075.             resultsTable.innerHTML = r.formatCSV("requester");
  1076.           else if (r.results.length > limiter) {
  1077.             var collation = r.collate(r.results, "requesterId"), _r = [], count = 0;
  1078.             var keys = Object.keys(collation)
  1079.               .filter(function(e) { return !/total/.test(e); })
  1080.               .sort(function(a,b) { return collation[b].pay - collation[a].pay; });
  1081.             keys.forEach(function(key){
  1082.               if (++count > limiter) {
  1083.                 qc.sr.push(new DBResult(_r, collation));
  1084.                 count = 0; _r = [];
  1085.               } else _r = _r.concat(collation[key]);
  1086.             });
  1087.             qc.sr.push(new DBResult(_r, collation));
  1088.             resultConstrain(qc.sr, 0, "requester", _cb);
  1089.           } else
  1090.             resultConstrain(r, 0, "requester", _cb);
  1091.         });
  1092.       }.bind(this); //requester overview click event
  1093.       dailyBtn.onclick = function() {
  1094.         var opt = Object.assign(this.getRange("*"), { index:null, dir:'prev', progress:true });
  1095.         _dbaccess("daily", ["STATS", opt], function(r) {
  1096.           resultsTable.innerHTML = exportCSVInput.checked ? r.formatCSV("daily") : r.formatHTML("daily");
  1097.           var expands = document.querySelectorAll(".hdbExpandRow");
  1098.           for (var el of expands)
  1099.             el.onclick = showHitsByDate;
  1100.         });
  1101.       }.bind(this); //daily overview click event
  1102.       //}}}
  1103.       function _getFilters(str) {//{{{
  1104.         var re = /(?:[rh][equstri]*(?:id|name)|bonus|reward|pay|req|id):[^;]+/ig,
  1105.             matches = str.match(re),
  1106.             filters = { query: str },
  1107.             _setRange = function(str) {
  1108.               var rng = str.split(/[><,]/).filter(v => v).sort(); rng.forEach((v,i,a) => a[i] = +v);
  1109.               if (rng.length === 1) {
  1110.                 if (str.startsWith('<')) {
  1111.                   rng[0] -= 0.01;
  1112.                   rng.unshift(0.01);
  1113.                 } else if (str.startsWith('>')) {
  1114.                   rng[0] += 0.01;
  1115.                   rng.push(Infinity);
  1116.                 } else rng.push(rng[0]);
  1117.               }
  1118.               return rng;
  1119.             };
  1120.  
  1121.         if (!matches) return filters;
  1122.         filters.query = str.slice(0,str.indexOf(matches[0])).trim();
  1123.         if (!filters.query.length) filters.query = null;
  1124.         for (var m of matches) {
  1125.           var _m = m.split(':');
  1126.           if (/(^req$|r[eqstr]*name)/i.test(_m[0])) filters.requesterName = _m[1].trimLeft();
  1127.           else if (/(^id$|hitid)/i.test(_m[0])) filters.hitId = _m[1].toUpperCase().trimLeft();
  1128.           else if (/r[eqstr]*id/i.test(_m[0])) filters.requesterId = _m[1].toUpperCase().trimLeft();
  1129.           else if (/(reward|pay)/i.test(_m[0])) filters.reward = _setRange(_m[1]);
  1130.           else if (_m[0].toLowerCase() === 'bonus') filters.bonus = _setRange(_m[1]);
  1131.         }
  1132.         return filters;
  1133.       }//}}}
  1134.       function _dbaccess(method, rargs, tfn) {//{{{
  1135.         if (!HITStorage.db) { Utils.errorHandler(new TypeError('(AccessViolation) Database is not defined')); return; }
  1136.         Utils.disableButtons(['hdbDaily','hdbRequester','hdbPending','hdbSearch'], true);
  1137.         searchResults.firstChild.click();
  1138.         Status.push("Preparing database...", "black");
  1139.         metrics.dbrecall = new Metrics("database_recall::"+method);
  1140.         metrics.dbrecall.mark("data retrieval", "start");
  1141.  
  1142.         HITStorage.recall(rargs[0],rargs[1]).then(function(r) {
  1143.           metrics.dbrecall.mark("data retrieval", "end");
  1144.           Status.message = "Building HTML...";
  1145.           try {
  1146.             for (var d of ["hdbResClear","hdbPageTop","hdbVpTop", "hdbPageBot"]) {
  1147.               if (exportCSVInput.checked || (~d.search(/page/i) && !/^[sr]/.test(method))) continue;
  1148.               document.getElementById(d).style.display = "initial";
  1149.             }
  1150.             metrics.dbrecall.mark("HTML construction", "start");
  1151.             tfn(r); 
  1152.             metrics.dbrecall.mark("HTML construction", "end");
  1153.           } catch(e) { 
  1154.             Utils.errorHandler(e);
  1155.           } finally {
  1156.             Utils.disableButtons(['hdbDaily','hdbRequester','hdbPending','hdbSearch'], false);
  1157.             autoScroll("#hdbSearchResults");
  1158.             Status.push("Done!", "green");
  1159.             Progress.hide();
  1160.             metrics.dbrecall.stop(); metrics.dbrecall.report();
  1161.           }
  1162.         });
  1163.       }//}}} _dbaccess
  1164.     },//}}} dashboardUI::initClickables
  1165.     getRange: function(status, filters) {//{{{
  1166.       var fromdate     = document.getElementById("hdbMinDate"),
  1167.           todate       = document.getElementById("hdbMaxDate"),
  1168.           statusSelect = document.getElementById("hdbStatusSelect"),
  1169.           obj = Object.assign({}, filters || {}), r = window.IDBKeyRange;
  1170.       obj.status = status || statusSelect.value;
  1171.       obj.date = [ (fromdate.value || '0000'), (todate.value || '9999') ];
  1172.       obj.index = obj.date[0] !== '0000' || obj.date[1] !== '9999' ? 'date' : 'status';
  1173.       if (filters) {
  1174.         var indexPriority = { hitId:100, bonus:80, date:70, status:60, requesterId:50, requesterName:40, reward:30,  },
  1175.             indices = Object.keys(filters);
  1176.         indices.push(obj.index);
  1177.         obj.index = indices.reduce((a,b) => indexPriority[a] || 0 > indexPriority[b] || 0 ? a : b);
  1178.       }
  1179.       obj.range = (function(i) {
  1180.         if (['date','reward','pay','bonus'].includes(i))
  1181.           return (obj[i] = obj[i].sort()) && r.bound(obj[i][0], obj[i][1]);
  1182.         else if (i === 'status' && status.length > 1)
  1183.           return r.only(status);
  1184.         else if (['hitId','requesterName','requesterId'].includes(i)) 
  1185.           return r.bound(obj[i], obj[i].slice(0,-1) + String.fromCharCode(obj[i].slice(-1).charCodeAt()+1));
  1186.       })(obj.index);
  1187.       if (obj.index === 'hitId' || (obj.index === 'status' && status.length === 1)) obj.index = null;
  1188.       return obj;
  1189.     }//}}} dashboardUI::getRange
  1190.   },//}}} dashboard
  1191.   
  1192.   FileHandler = { //{{{
  1193.     //
  1194.     // TODO: JSON integrity check
  1195.     //
  1196.     delegate: function(e) {//{{{
  1197.       var f = e.target.files;
  1198.       if (f.length && ~f[0].name.search(/\.(bak|csv|json)$/i)/* && ~f[0].type.search(/(text|json)/)*/) {
  1199.         var reader = new FileReader(), testing = true, isCsv = false;
  1200.         metrics.dbimport = new Metrics("file_import");
  1201.  
  1202.         reader.readAsText(f[0].slice(0,10));
  1203.         reader.onload = function(e) { 
  1204.           var r = e.target.result;
  1205.           if (testing && !~r.search(/(STATS|NOTES|HIT)/)) { // failed json check, test if csv
  1206.             console.log("failed json integrity:", r, "\nchecking csv schema...");
  1207.             if (!~r.search(/hitId/)) { // failed csv check, return error
  1208.               console.log("failed csv integrity:", r, "\naborting");
  1209.               return Utils.errorHandler(new TypeError("Invalid data structure"));
  1210.             } else { // passed initial csv check, parse full file
  1211.               console.log("deferring to csv parser");
  1212.               isCsv   = true;
  1213.               testing = false;
  1214.               Progress.show();
  1215.               reader.readAsText(f[0]);
  1216.             }
  1217.           } else if (testing) {
  1218.             testing = false;
  1219.             Progress.show();
  1220.             reader.readAsText(f[0]);
  1221.           } else {
  1222.             if (isCsv) this.csv.fromFile(r);
  1223.             else HITStorage.write(JSON.parse(r), cbImport);
  1224.           }
  1225.         }.bind(FileHandler); // reader.onload
  1226.       } else if (f.length)
  1227.         Utils.errorHandler(new TypeError("Unsupported file format"));
  1228.     },//}}}
  1229.     csv: {//{{{
  1230.       fromFile: function(r) {//{{{
  1231.         var validKeys  = ["autoAppTime","date","feedback","hitId","requesterId","requesterName","reward","pay","bonus","status","title"],
  1232.             //lines      = r.replace(/\r?\n^(?!"?[A-Z0-9]{30})/gm,' ').split(/\r?\n/);
  1233.             lines      = r.split(/\r?\n(?="?[A-Z0-9]{30})/);
  1234.         this.delimiter = /^"/.test(lines[0]) ? r.substr(7,1) : r.substr(5,1);
  1235.         this.header    = lines.splice(0,1)[0].replace(new RegExp(`([" ]|${this.delimiter}$)`,'g'),'').split(this.delimiter);
  1236.         this.data      = { HIT:[] };
  1237.  
  1238.         console.log('delimiter:',this.delimiter==='\t'?'tab':this.delimiter,'\nlines:',lines.length,'\nheader:',this.header);
  1239.         if (!lines.length) return Utils.errorHandler(new Error("CSV file must contain at least one record"));
  1240.         // make sure header keys are valid
  1241.         for (var key of this.header)
  1242.           if (!~validKeys.indexOf(key)) {
  1243.             Progress.hide();
  1244.             return Utils.errorHandler(new TypeError("Invalid key '"+key+"' found in column header"));
  1245.           }
  1246.         this.core(lines);
  1247.       },//}}}
  1248.       core: function(lr, syn) {//{{{
  1249.         syn = syn || false;
  1250.         var badLines = [],
  1251.             deq = function(str) { if (/^"/.test(str) && /"$/.test(str)) return str.replace(/(^"|"$)/g,''); else return str;},
  1252.             qfix = arr => arr.reduce((a,b) => {
  1253.               if (a.length && /^".+[^"]$/.test(a[a.length-1])) {
  1254.                 a[a.length-1] = a[a.length-1] + this.delimiter + b;
  1255.                 return a;
  1256.               } else return a.concat(b);
  1257.             }, []);
  1258.         for (var line of lr) {
  1259.           var record = {};
  1260.           line = line.split(this.delimiter);
  1261.           if (line.length <= 1) continue;
  1262.           if (line.length !== this.header.length) {
  1263.             // attempt to resolve delimiter conflicts within field values
  1264.             line = qfix(line);
  1265.             while (line.length > this.header.length) {
  1266.               var datum = line.pop();
  1267.               if (/\S/.test(deq(datum))) { line.push(datum); break; }
  1268.             }
  1269.             if (line.length !== this.header.length) { 
  1270.               badLines.push({record: line, reason: "SyntaxError: Number of field do not match number of columns"}); continue; }
  1271.           }
  1272.           // convert into usable JSON
  1273.           for (var i=0;i<line.length;i++) {
  1274.             if (/(pay|bonus|reward|autoAppTime)/.test(this.header[i]) && isNaN(+line[i])) { 
  1275.               badLines.push({record: line, reason: `TypeError: Value in '${this.header[i]}' is not a number.`}); break; }
  1276.             if (this.header[i] === 'hitId' && (/\W/.test(deq(line[i])) || deq(line[i]).length !== 30)) { 
  1277.               badLines.push({record: line, reason: "TypeError: Invalid hitId."}); break; }
  1278.             if (this.header[i] === 'date' && !/\d{4}-\d{2}-\d{2}/.test(line[i])) { 
  1279.               badLines.push({record: line, reason: "TypeError: Invalid date. Dates must be in ISO format (YYYY-MM-DD)."}); break; }
  1280.  
  1281.             if (this.header[i] === 'pay' || this.header[i] === 'reward')
  1282.               record.reward = +line[i];
  1283.             else if (this.header[i] === 'bonus')
  1284.               record.bonus = +line[i];
  1285.             else
  1286.               record[this.header[i]] = deq(line[i]);
  1287.           } // for each field
  1288.           if (!syn && !badLines.find(v => v.record === line)) this.data.HIT.push(record);
  1289.         } // for each record 
  1290.         if (syn) return !badLines.length;
  1291.         else if (badLines.length) { console.warn('SyntaxError'); console.dir(badLines); this.manualFix(badLines); }
  1292.         else HITStorage.write(this.data, cbImport);
  1293.       },//}}}
  1294.       manualFix: function(lr) {//{{{
  1295.         var div      = document.body.appendChild(document.createElement('DIV')),
  1296.             title    = div.appendChild(document.createElement('P')),
  1297.             divInner = div.appendChild(document.createElement('DIV')),
  1298.             buttons  = div.appendChild(document.createElement('P')),
  1299.             trimSansTab = function(str) {
  1300.               var c = "[ \f\n\r\v\u00a0\u1680\u180e\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+";
  1301.               return str.replace(new RegExp("^"+c),'').replace(new RegExp(c+"$"),'');
  1302.             },
  1303.             kdFn     = function(e) {
  1304.               if (e.keyCode === 9) {// tab
  1305.                 e.preventDefault();
  1306.                 var zs = e.target.selectionStart,
  1307.                     ze = e.target.selectionEnd;
  1308.                 e.target.value = e.target.value.substr(0,zs) + '\t' + e.target.value.substr(ze);
  1309.               }
  1310.               e.target.style.height = '1px';
  1311.               e.target.style.height = e.target.scrollHeight + 10 +'px';
  1312.             },
  1313.             blurFn   = function(e) {
  1314.               var check = this.core([trimSansTab(e.target.value)],true);
  1315.               if (check) e.target.style.background = '#9EFF9E'; else e.target.style.background = 'white';
  1316.             }.bind(this);
  1317.         title.outerHTML = '<p style="margin:0;text-align:center;font-weight:bold;font-size:1.2em">Failed lines</p>' +
  1318.           '<p style="text-align:center;font-weight:bold;font-size:0.9em;margin:1%">' + this.header.join(this.delimiter)+'</p>';
  1319.         div.style.cssText = "z-index:5; position:fixed; top:50%;left:50%; padding:0.7%; width:650px; resize:both; overflow:auto;" +
  1320.           "background:rgba(204,204,204,0.88); box-shadow: 0px 0px 15px 2px #000; margin-right:-50%; transform:translate(-50%, -50%);";
  1321.         divInner.style.cssText = "position:relative; max-height:350px; overflow:auto;";
  1322.         for (var v of lr) {
  1323.           divInner.appendChild(document.createTextNode(v.reason));
  1324.           var ta = divInner.appendChild(document.createElement('TEXTAREA'));
  1325.           ta.style.cssText = "resize:none; overflow:hidden; width:100%; display:block";
  1326.           ta.onkeydown = kdFn;
  1327.           ta.onblur = blurFn;
  1328.           ta.value = v.record.join(this.delimiter);
  1329.           ta.style.height = ta.scrollHeight + 10 + 'px';
  1330.         }
  1331.         buttons.style.cssText = "margin:1% auto; text-align:center;";
  1332.         buttons.innerHTML = '<button id="fretry" title="Retry failed lines.")">Retry</button> ' +
  1333.           '<button id="fskip" title="Skip these failed lines and import the rest to the database.">Skip</button> ' +
  1334.           '<button id="fcancel" title="Cancel the entire import process">Cancel</button><br>' +
  1335.           'Modify the above entries and retry, or skip them, or cancel the entire import.';
  1336.         buttons.querySelector('#fretry').onclick = function() {
  1337.           var l = [];
  1338.           for (var el of div.querySelectorAll('textarea')) l.push(trimSansTab(el.value));
  1339.           this.core(l);
  1340.           div.remove();
  1341.         }.bind(this);
  1342.         buttons.querySelector('#fskip').onclick = function() { div.remove(); HITStorage.write(this.data, cbImport); }.bind(this);
  1343.         buttons.querySelector('#fcancel').onclick = function() { div.remove(); this.data = null; Progress.hide();}.bind(this);
  1344.       }//}}}
  1345.     }//}}} FileHandler::csv
  1346.   };//}}}
  1347.  
  1348.   /* 
  1349.    *
  1350.    *
  1351.    *
  1352.    *
  1353.    *///{{{
  1354.   console.log('hdb hook');
  1355.   if (document.location.pathname === "/mturk/dashboard") {
  1356.     DashboardUI.draw();
  1357.     DashboardUI.initClickables();
  1358.     
  1359.     ProjectedEarnings.updateDate();
  1360.     ProjectedEarnings.draw(true);
  1361.     
  1362.     setInterval(Utils.updateTimestamp, 1000*60);
  1363.  
  1364.     var Status = {
  1365.       node: document.getElementById("hdbStatusText"),
  1366.       get message() { return this.node.textContent; },
  1367.       set message(str) { this.node.textContent = str; },
  1368.       get color() { return this.node.style.color; },
  1369.       set color(c) { this.node.style.color = c; },
  1370.       push: function(m,c) { c = c || "black"; this.message = m; this.color = c; },
  1371.       get html() { return this.node.innerHTML },
  1372.       set html(str) { this.node.innerHTML = str; }
  1373.     }, Progress = {
  1374.       node: document.getElementById("hdbProgressBar"),
  1375.       hide: function() { this.node.style.display = "none"; },
  1376.       show: function() { this.node.style.display = "block"; }
  1377.     };
  1378.  
  1379.     var dbh = window.indexedDB.open(DB_NAME, DB_VERSION);
  1380.     dbh.onerror = function(e) { Utils.errorHandler(e.target.error); };
  1381.     dbh.onupgradeneeded = HITStorage.versionChange;
  1382.     dbh.onsuccess = INITDB;
  1383.  
  1384.     // export some variables for external extensions
  1385.     self.Status = Status; self.Progress = Progress; self.Metrics = Metrics; self.Math.decRound = Math.decRound;
  1386.   } else { // page is not dashboard
  1387.     window.indexedDB.open(DB_NAME).onsuccess = function() { HITStorage.db = this.result; beenThereDoneThat(); };
  1388.   }
  1389.   /*}}}
  1390.    *
  1391.    *
  1392.    *
  1393.    *
  1394.    */
  1395.  
  1396.   function saveState(key, value) {//{{{
  1397.     try { 
  1398.       localStorage.setItem(key,value);
  1399.     } catch(err) {
  1400.       if (err.name !== 'QuotaExceededError') return Utils.errorHandler(err);
  1401.       try {
  1402.         localStorage.removeItem(key);
  1403.         localStorage.setItem(key, value);
  1404.       } catch(errr) {
  1405.         return Utils.errorHandler(errr);
  1406.       }
  1407.     }
  1408.   }//}}}
  1409.  
  1410.   // {{{ css injection
  1411.   var css = "<style type='text/css'>" +
  1412.   "#hdbProgressBar {margin:auto; width:250px; height:15px; position:relative; display:none;}" +
  1413.   ".ball {position:absolute; left:0; width:12px; height:12px; border-radius:5px;" +
  1414.     "animation:kfpballs 2s cubic-bezier(0.24,0.77,0.68,1) infinite;" +
  1415.     "background:linear-gradient(222deg, rgba(208,69,247,0), rgba(208,69,247,1), rgba(69,197,247,1), rgba(69,197,247,0))}" +
  1416.   "#hdbB2{animation-delay:.19s} #hdbB3{animation-delay:.38s} #hdbB4{animation-delay:.55s}" +
  1417.   "@keyframes kfpballs {0% {left:0%;opacity:1} 50% {left:98%;opacity:0.2} 100% {left:0%;opacity:1}}" +
  1418.   ".hitdbRTButtons {border:1px solid; font-size: 10px; height: 18px; padding-left: 5px; padding-right: 5px; background: pink;}" +
  1419.   ".hitdbRTButtons-green {background: lightgreen;}" +
  1420.   ".hitdbRTButtons-large {width:80px;}" +
  1421.   ".hdbCalControls {cursor:pointer;} .hdbCalControls:hover {color:#c27fcf;}" +
  1422.   ".hdbCalCells {background:#f0f6f9; height:19px}" +
  1423.   ".hdbCalDays {cursor:pointer; text-align:center;} .hdbCalDays:hover {background:#7fb4cf; color:white;}" +
  1424.   ".hdbDayHeader {width:26px; text-align:center; font-weight:bold; font-size:12px; background:#f0f6f9;}" +
  1425.   ".hdbCalHeader {background:#7fb4cf; color:white; font-weight:bold; text-align:center; font-size:11px; padding:3px 0px;}" +
  1426.   "#hdbCalendarPanel {position:absolute; z-index:10; box-shadow:-2px 3px 5px 0px rgba(0,0,0,0.68);}" +
  1427.   ".hdbExpandRow {cursor:pointer; color:blue;}" +
  1428.   ".hdbTotalsRow {background:#CCC; color:#369; font-weight:bold;}" +
  1429.   ".hdbHeaderRow {background:#7FB448; font-size:12px; color:white;}" +
  1430.   ".helpSpan {border-bottom:1px dotted; cursor:help;}" +
  1431.   ".hdbResControl {border-bottom:1px solid; color:#c60; cursor:pointer; display:none;}" +
  1432.   ".hdbTablePagination {margin-left:15em; color:#c60; display:none;}" +
  1433.   ".spin {animation: kfspin 0.7s infinite linear; font-weight:bold;}" +
  1434.   "@keyframes kfspin { 0% { transform: rotate(0deg) } 100% { transform: rotate(359deg) } }" +
  1435.   ".spin:before{content:'*'}" +
  1436.   ".nowrap {white-space:nowrap; overflow:hidden; text-overflow:ellipsis}" +
  1437.   "</style>";
  1438.   document.head.innerHTML += css;
  1439.   // }}}
  1440.  
  1441.   function resultConstrain(data, index, type, callback) {//{{{
  1442.     data = data || qc.sr;
  1443.  
  1444.     var table  = document.getElementById("hdbResultsTable"),
  1445.         rslice = data.length ? data[index].results : data.results,
  1446.         pager  = [document.getElementById("hdbPageTop"), document.getElementById("hdbPageBot")],
  1447.         sopt   = [],
  1448.         _f     = function(e) { resultConstrain(null,e.target.value,type,callback); };
  1449.     pager[0].innerHTML = ''; pager[1].innerHTML = '';
  1450.  
  1451.     if (data instanceof DBResult)
  1452.       table.innerHTML = data.formatHTML(type);
  1453.     else {
  1454.       table.innerHTML = data[index].formatHTML(type);
  1455.       pager[0].innerHTML = '<span style="cursor:pointer;">' + (index > 0 ? '&#9664; Prev' : '') + '</span> ' +
  1456.         '<span style="cursor:pointer;">' + (+index+1 === data.length ? '' : 'Next &#9654;') + '</span> &nbsp; || &nbsp; '+
  1457.         '<label>Select page: </label><select></select>';
  1458.       for (var i=0;i<data.length;i++) {
  1459.         if (i === +index)
  1460.           sopt.push('<option value="' + i + '" selected="selected">' + (i+1) + '</option>');
  1461.         else
  1462.           sopt.push('<option value="' + i + '">' + (i+1) + '</option>');
  1463.       }
  1464.       pager[0].lastChild.innerHTML = sopt.join('');
  1465.       pager[2] = pager[0].cloneNode(true);
  1466.       pager[2].id = "hdbPageBot";
  1467.       for (i of [0,2]) {
  1468.         pager[i].children[0].onclick = resultConstrain.bind(null,null,+index-1,type,callback);
  1469.         pager[i].children[1].onclick = resultConstrain.bind(null,null,+index+1,type,callback);
  1470.         pager[i].children[3].onchange = _f;
  1471.       }
  1472.       pager[0].parentNode.replaceChild(pager[2], pager[1]);
  1473.     }
  1474.  
  1475.     callback(rslice);
  1476.   }//}}} resultConstrain
  1477.  
  1478.   function beenThereDoneThat() {//{{{
  1479.     if (~document.location.pathname.search(/(accept|continue)/)) {
  1480.       if (!document.querySelector('input[name="hitAutoAppDelayInSeconds"]')) return;
  1481.  
  1482.       // capture autoapproval times
  1483.       var _aa = document.querySelector('input[name="hitAutoAppDelayInSeconds"]').value,
  1484.           _hid = document.querySelectorAll('input[name="hitId"]')[1].value,
  1485.           pad = function(num) { return Number(num).toPadded(); },
  1486.           _d  = Date.parse(new Date().getFullYear() + "-" + pad(new Date().getMonth()+1) + "-" + pad(new Date().getDate()));
  1487.       qc.aat = JSON.parse(localStorage.getItem("hitdb_autoAppTemp") || "{}");
  1488.  
  1489.       if (!qc.aat[_d]) qc.aat[_d] = {};
  1490.       qc.aat[_d][_hid] = _aa;
  1491.       qc.save("aat", "hitdb_autoAppTemp", true);
  1492.       return;
  1493.     }
  1494.     var qualNode = document.querySelector('td[colspan="11"]');
  1495.     if (qualNode) { // we're on the preview page!
  1496.       var requesterid   = document.querySelector('input[name="requesterId"]').value,
  1497.           requestername = document.querySelector('input[name="prevRequester"]').value,
  1498.           autoApproval  = document.querySelector('input[name="hitAutoAppDelayInSeconds"]').value,
  1499.           hitTitle      = document.querySelector('div[style*="ellipsis"]').textContent.trim(),
  1500.           insertionNode = qualNode.parentNode.parentNode;
  1501.       var row = document.createElement("TR"), cellL = document.createElement("TD"), cellR = document.createElement("TD");
  1502.       var resultsTableR = document.createElement("TABLE"),
  1503.           resultsTableT = document.createElement("TABLE");
  1504.       resultsTableR.dataset.rid = requesterid;
  1505.       resultsTableT.dataset.title = hitTitle;
  1506.       insertionNode.parentNode.parentNode.appendChild(resultsTableR);
  1507.       insertionNode.parentNode.parentNode.appendChild(resultsTableT);
  1508.  
  1509.       cellR.innerHTML = '<span class="capsule_field_title">Auto-Approval:</span>&nbsp;&nbsp;'+Utils.ftime(autoApproval).join(' ');
  1510.       var rbutton = document.createElement("BUTTON");
  1511.       rbutton.classList.add("hitdbRTButtons","hitdbRTButtons-large");
  1512.       rbutton.textContent = "Requester";
  1513.       rbutton.onclick = function(e) { e.preventDefault(); showResults.call(resultsTableR, "req", hitTitle); };
  1514.       var tbutton = rbutton.cloneNode(false);
  1515.       rbutton.title = "Show HITs completed from this requester";
  1516.       tbutton.textContent = "HIT Title";
  1517.       tbutton.onclick = function(e) { e.preventDefault(); showResults.call(resultsTableT, "title", requestername) };
  1518.       HITStorage.recall("HIT", {index: "requesterId", range: window.IDBKeyRange.only(requesterid), limit: 1})
  1519.         .then(processResults.bind(rbutton,resultsTableR));
  1520.       HITStorage.recall("HIT", {index: "title", range: window.IDBKeyRange.only(hitTitle), limit: 1})
  1521.         .then(processResults.bind(tbutton,resultsTableT));
  1522.       row.appendChild(cellL);
  1523.       row.appendChild(cellR);
  1524.       cellL.appendChild(rbutton);
  1525.       cellL.appendChild(tbutton);
  1526.       cellL.colSpan = "3";
  1527.       cellR.colSpan = "8";
  1528.       insertionNode.appendChild(row);
  1529.     } else { // browsing HITs n sutff 
  1530.       var titleNodes = document.querySelectorAll('a[class="capsulelink"]');
  1531.       if (titleNodes.length < 1) return; // nothing left to do here!
  1532.       var requesterNodes = document.querySelectorAll('a[href*="hitgroups&requester"]');
  1533.       var insertionNodes = [];
  1534.  
  1535.       for (var i=0;i<titleNodes.length;i++) {
  1536.         var _title = titleNodes[i].textContent.trim();
  1537.         var _tbutton = document.createElement("BUTTON");
  1538.         var _id = requesterNodes[i].href.replace(/.+Id=(.+)/, "$1");
  1539.         var _name = requesterNodes[i].textContent;
  1540.         var _rbutton = document.createElement("BUTTON");
  1541.         var _div = document.createElement("DIV"), _tr = document.createElement("TR");
  1542.         resultsTableR = document.createElement("TABLE");
  1543.         resultsTableR.dataset.rid = _id;
  1544.         resultsTableT = document.createElement("TABLE");
  1545.         resultsTableT.dataset.title = _title;
  1546.         insertionNodes.push(requesterNodes[i].parentNode.parentNode.parentNode);
  1547.         insertionNodes[i].offsetParent.offsetParent.offsetParent.offsetParent.appendChild(resultsTableR);
  1548.         insertionNodes[i].offsetParent.offsetParent.offsetParent.offsetParent.appendChild(resultsTableT);
  1549.  
  1550.         HITStorage.recall("HIT", {index: "title", range: window.IDBKeyRange.only(_title), limit: 1} )
  1551.           .then(processResults.bind(_tbutton,resultsTableT));
  1552.         HITStorage.recall("HIT", {index: "requesterId", range: window.IDBKeyRange.only(_id), limit: 1} )
  1553.           .then(processResults.bind(_rbutton,resultsTableR));
  1554.  
  1555.         _tr.appendChild(_div);
  1556.         _div.id = "hitdbRTInjection-"+i;
  1557.         _div.appendChild(_rbutton);
  1558.         _rbutton.textContent = 'R';
  1559.         _rbutton.classList.add("hitdbRTButtons");
  1560.         _rbutton.onclick = showResults.bind(resultsTableR, "req", _title);
  1561.         _rbutton.title = "Show HITs completed from this requester";
  1562.         _div.appendChild(_tbutton);
  1563.         _tbutton.textContent = 'T';
  1564.         _tbutton.classList.add("hitdbRTButtons");
  1565.         _tbutton.onclick = showResults.bind(resultsTableT, "title", _name);
  1566.         insertionNodes[i].appendChild(_tr);
  1567.       }
  1568.     } // else
  1569.  
  1570.     function showResults(type, match) {//{{{
  1571.       /*jshint validthis: true*/
  1572.       if (!this.dataset.hasResults) return;
  1573.       if (this.children.length) // table is populated
  1574.         this.innerHTML = '';
  1575.       else { // need to populate table
  1576.         var head = this.createTHead(),
  1577.             body = this.createTBody(),
  1578.             capt = this.createCaption(),
  1579.             style= "font-size:10px;font-weight:bold;text-align:center",
  1580.             validKeys = function(obj) { return Object.keys(obj).filter(function(v) { return !~v.search(/total[A-Z]/); }); };
  1581.  
  1582.         capt.innerHTML = '<span style="'+style+'">Loading...<label class="spin"></label></span>';
  1583.  
  1584.         if (type === "req") {
  1585.           HITStorage.recall("HIT", {index:"requesterId", range:window.IDBKeyRange.only(this.dataset.rid)})
  1586.             .then( function(r) {
  1587.               var cbydate = r.collate(r.results, "date"),
  1588.                   kbydate = validKeys(cbydate),
  1589.                   cbydatextitle, kbytitle, bodyHTML = [];
  1590.               kbydate.forEach(function(date) {
  1591.                 cbydatextitle = r.collate(cbydate[date], "title");
  1592.                 kbytitle = validKeys(cbydatextitle);
  1593.                 kbytitle.forEach(function(title) {
  1594.                   bodyHTML.push('<tr style="text-align:center;"><td>'+date+'</td>' +
  1595.                       '<td style="text-align:left">'+title.trim()+'</td><td>'+cbydatextitle[title].length+'</td>' +
  1596.                       '<td>'+Number(Math.decRound(cbydatextitle[title].pay,2)).toFixed(2)+'</td></tr>');
  1597.                 });
  1598.               });
  1599.               var help = "Total number of HITs submitted for a given date with the same title\n" +
  1600.                 "(aggregates results with the same title to simplify the table and reduce unnecessary spam for batch workers)"; 
  1601.               head.innerHTML = '<tr style="'+style+'"><th>Date</th><th>Title</th>' +
  1602.                 '<th><span class="helpSpan" title="'+help+'">#HITs</span></th><th>Total Rewards</th></tr>';
  1603.               body.innerHTML = bodyHTML.sort(function(a,b) {
  1604.                 return a.match(/\d{4}-\d{2}-\d{2}/)[0] < b.match(/\d{4}-\d{2}-\d{2}/)[0] ? 1 : -1;
  1605.               }).join('');
  1606.               capt.innerHTML = '<label style="'+style+'">HITs Matching This Requester</label>';
  1607.  
  1608.               var mrows = Array.prototype.filter.call(body.rows, function(v) {return v.cells[1].textContent === match});
  1609.               for (var row of mrows)
  1610.                 row.style.background = "lightgreen";
  1611.             });
  1612.         }
  1613.         else if (type === "title") {
  1614.           HITStorage.recall("HIT", {index:"title", range:window.IDBKeyRange.only(this.dataset.title)})
  1615.             .then( function(r) {
  1616.               var cbyreq = r.collate(r.results, "requesterName"),
  1617.                   kbyreq = validKeys(cbyreq),
  1618.                   bodyHTML = [];
  1619.               for (var key of kbyreq)
  1620.                 bodyHTML.push('<tr style="text-align:center;"><td>'+key+'</td><td>'+cbyreq[key].length+'</td>' +
  1621.                     '<td>'+Number(Math.decRound(cbyreq[key].pay,2)).toFixed(2)+'</td></tr>');
  1622.               var help = "Total number of HITs matching this title submitted for a given requester\n" +
  1623.                 "(aggregates results with the same requester name to simplify the table and reduce unnecessary spam for batch workers)";
  1624.               head.innerHTML = '<tr style="'+style+'"><th>Requester Name</th>' +
  1625.                 '<th><span class="helpSpan" title="'+help+'">#HITs</span></th><th>Total Rewards</th></tr>';
  1626.               body.innerHTML = bodyHTML.join('');
  1627.               capt.innerHTML = '<label style="'+style+'">Reqesters With HITs Matching This Title</label>';
  1628.  
  1629.               var mrows = Array.prototype.filter.call(body.rows, function(v) {return v.cells[0].textContent === match});
  1630.               for (var row of mrows)
  1631.                 row.style.background = "lightgreen";
  1632.             });
  1633.         } //if type === 'title'
  1634.       }//populate table
  1635.     }//}}} showResults
  1636.  
  1637.     function processResults(table, r) {
  1638.       /*jshint validthis: true*/
  1639.       if (r.results.length) {
  1640.         table.dataset.hasResults = "true";
  1641.         this.classList.add("hitdbRTButtons-green");
  1642.       }
  1643.     }
  1644.     
  1645.   }//}}} btdt
  1646.  
  1647.   function showHiddenRows(e) {//{{{
  1648.     var rid = e.target.parentNode.textContent.substr(4);
  1649.     var nodes = document.querySelectorAll('tr[data-rid="'+rid+'"]'), el = null;
  1650.     if (e.target.textContent === "[+]") {
  1651.       for (el of nodes)
  1652.         el.style.display="table-row";
  1653.       e.target.textContent = "[-]";
  1654.     } else {
  1655.       for (el of nodes)
  1656.         el.style.display="none";
  1657.       e.target.textContent = "[+]";
  1658.     }
  1659.   }//}}}
  1660.  
  1661.   function showHitsByDate(e) {//{{{
  1662.     var date = e.target.parentNode.nextSibling.textContent,
  1663.         row  = e.target.parentNode.parentNode,
  1664.         table= row.parentNode;
  1665.  
  1666.     if (e.target.textContent === "[+]") {
  1667.       e.target.textContent = "[-]";
  1668.       var nrow = table.insertBefore(document.createElement("TR"), row.nextSibling);
  1669.       nrow.className = row.className;
  1670.       nrow.innerHTML = '<td><b>Loading...<label class="spin"></label></b></td>';
  1671.       HITStorage.recall("HIT", {index: "date", range: window.IDBKeyRange.only(date)}).then( function(r) {
  1672.         nrow.innerHTML = '<td colspan="7"><table style="width:760;color:#c60;">' + r.formatHTML(null,true) + '</table></td>';
  1673.       });
  1674.     } else {
  1675.       e.target.textContent = "[+]";
  1676.       table.removeChild(row.nextSibling);
  1677.     }
  1678.   }//}}} showHitsByDate
  1679.  
  1680.   function updateBonus(e) {//{{{
  1681.     if (e instanceof window.KeyboardEvent && e.keyCode === 13) {
  1682.       e.target.blur();
  1683.       return false;
  1684.     } else if (e instanceof window.FocusEvent) {
  1685.       var _bonus = +e.target.textContent.replace(/[^\d.]/g,""),
  1686.           _tBonusCell = e.target.offsetParent.tFoot.rows[0].cells[4],
  1687.           _tBonus = +_tBonusCell.textContent.replace(/\$/,"");
  1688.       e.target.textContent = Number(_bonus).toFixed(2);
  1689.       _tBonusCell.textContent = '$'+Number(_tBonus-e.target.dataset.initial+_bonus).toFixed(2);
  1690.       if (_bonus !== +e.target.dataset.initial) {
  1691.         console.log("updating bonus to",_bonus,"from",e.target.dataset.initial,"("+e.target.dataset.hitid+")");
  1692.         e.target.dataset.initial = _bonus;
  1693.         var _range = window.IDBKeyRange.only(e.target.dataset.hitid);
  1694.  
  1695.         HITStorage.db.transaction("HIT", "readwrite").objectStore("HIT").openCursor(_range).onsuccess = function() {
  1696.           var c = this.result;
  1697.           if (c) {
  1698.             c.value.bonus = _bonus;
  1699.             c.update(c.value);
  1700.           } 
  1701.         }; // idbcursor
  1702.       } // bonus is new value
  1703.     } // keycode
  1704.   } //}}} updateBonus
  1705.  
  1706.   function noteHandler(type, e) {//{{{
  1707.     // 
  1708.     // TODO restructure event handling/logic tree
  1709.     //      combine save and delete; it's ugly :(
  1710.     //      actually this whole thing is messy and in need of refactoring
  1711.     //
  1712.     if (e instanceof window.KeyboardEvent) {
  1713.      if (e.keyCode === 13) {
  1714.         e.target.blur();
  1715.         return false;
  1716.       }
  1717.      return;
  1718.     }
  1719.  
  1720.     if (e instanceof window.FocusEvent) {
  1721.       if (e.target.textContent.trim() !== e.target.dataset.initial) {
  1722.         if (!e.target.textContent.trim()) { e.target.previousSibling.previousSibling.firstChild.click(); return; }
  1723.         var note   = e.target.textContent.trim(),
  1724.             _range = window.IDBKeyRange.only(e.target.dataset.id),
  1725.             inote  = e.target.dataset.initial,
  1726.             hitId  = e.target.dataset.id,
  1727.             date   = e.target.previousSibling.textContent;
  1728.  
  1729.         e.target.dataset.initial = note;
  1730.         HITStorage.db.transaction("NOTES", "readwrite").objectStore("NOTES").index("hitId").openCursor(_range).onsuccess = function() {
  1731.           if (this.result) {
  1732.             var r = this.result.value;
  1733.             if (r.note === inote) {  // note already exists in database, so we update its value
  1734.               r.note = note;
  1735.               this.result.update(r);
  1736.               return;
  1737.             }
  1738.             this.result.continue();
  1739.           } else {
  1740.             if (this.source instanceof window.IDBObjectStore)
  1741.               this.source.put({ note:note, date:date, hitId:hitId });
  1742.             else
  1743.               this.source.objectStore.put({ note:note, date:date, hitId:hitId });
  1744.           }
  1745.         };
  1746.       } 
  1747.       return; // end of save event; no need to proceed
  1748.     }
  1749.  
  1750.     if (type === "delete") {
  1751.       var tr       = e.target.parentNode.parentNode,
  1752.           noteCell = tr.lastChild;
  1753.           _range   = window.IDBKeyRange.only(noteCell.dataset.id);
  1754.       if (!noteCell.dataset.initial) tr.remove();
  1755.       else {
  1756.         HITStorage.db.transaction("NOTES", "readwrite").objectStore("NOTES").index("hitId").openCursor(_range).onsuccess = function() {
  1757.           if (this.result) {
  1758.             if (this.result.value.note === noteCell.dataset.initial) {
  1759.               this.result.delete();
  1760.               tr.remove();
  1761.               return;
  1762.             }
  1763.             this.result.continue();
  1764.           } 
  1765.         };
  1766.       }
  1767.       return; // end of deletion event; no need to proceed
  1768.     } else {
  1769.       if (type === "attach" && !e.results.length) return;
  1770.  
  1771.       var trow  = e instanceof window.MouseEvent ? e.target.parentNode.parentNode : null,
  1772.           tbody = trow ? trow.parentNode : null,
  1773.           row   = document.createElement("TR"),
  1774.           c1    = row.insertCell(0),
  1775.           c2    = row.insertCell(1),
  1776.           c3    = row.insertCell(2);
  1777.           date  = new Date();
  1778.           hitId = e instanceof window.MouseEvent ? e.target.id.substr(5) : null;
  1779.  
  1780.       c1.innerHTML = '<span class="removeNote" title="Delete this note" style="cursor:pointer;color:crimson;">[x]</span>';
  1781.       c1.firstChild.onclick = noteHandler.bind(null,"delete");
  1782.       c1.style.textAlign = "right";
  1783.       c2.title = "Date on which the note was added";
  1784.       c3.style.color = "crimson";
  1785.       c3.colSpan = "6";
  1786.       c3.contentEditable = "true";
  1787.       c3.onblur = noteHandler.bind(null,"blur");
  1788.       c3.onkeydown = noteHandler.bind(null, "kb");
  1789.       
  1790.       if (type === "new") {
  1791.         row.classList.add(trow.classList);
  1792.         tbody.insertBefore(row, trow.nextSibling);
  1793.         c2.textContent = date.getFullYear()+"-"+Number(date.getMonth()+1).toPadded()+"-"+Number(date.getDate()).toPadded();
  1794.         c3.dataset.initial = "";
  1795.         c3.dataset.id = hitId;
  1796.         c3.focus();
  1797.         return;
  1798.       }
  1799.  
  1800.       for (var entry of e.results) {
  1801.         trow  = document.querySelector('tr[data-id="'+entry.hitId+'"]');
  1802.         tbody = trow.parentNode;
  1803.         row   = row.cloneNode(true);
  1804.         c1    = row.firstChild;
  1805.         c2    = c1.nextSibling;
  1806.         c3    = row.lastChild;
  1807.         
  1808.         row.classList.add(trow.classList);
  1809.         tbody.insertBefore(row, trow.nextSibling);
  1810.  
  1811.         c1.firstChild.onclick = noteHandler.bind(null,"delete");
  1812.         c2.textContent = entry.date;
  1813.         c3.textContent = entry.note;
  1814.         c3.dataset.initial = entry.note;
  1815.         c3.dataset.id = entry.hitId;
  1816.         c3.onblur = noteHandler.bind(null,"blur");
  1817.         c3.onkeydown = noteHandler.bind(null, "kb");
  1818.       }
  1819.     } // new/attach
  1820.   }//}}} noteHandler
  1821.  
  1822.   // writing callback functions {{{
  1823.   function cbImport() {
  1824.     /*jshint validthis:true*/
  1825.     Status.push("Importing " + this.total + " entries");
  1826.     if (++this.total !== this.requests) return;
  1827.     Status.push("Importing " + this.total + " entries... Done!", "green");
  1828.     try { Progress.hide(); metrics.dbimport.stop(); metrics.dbimport.report(); } catch(err) {}
  1829.   }
  1830.   function cbUpdate() {
  1831.     /*jshint validthis:true*/
  1832.     if (++this.total !== this.requests) return;
  1833.     if (qc.extraDays) qc.extraDays = false;
  1834.     Status.push("Update Complete!", "green");
  1835.     ProjectedEarnings.data.dbUpdated = new Date().toLocalISOString();
  1836.     ProjectedEarnings.saveState();
  1837.     ProjectedEarnings.updatePaint();
  1838.     Utils.disableButtons(['hdbUpdate'], false);
  1839.     Progress.hide(); metrics.dbupdate.stop(); metrics.dbupdate.report();
  1840.   }
  1841.   //}}}
  1842.  
  1843.   function autoScroll(location, dt) {//{{{
  1844.     var target = document.querySelector(location).offsetTop,
  1845.         pos    = window.scrollY,
  1846.         dpos   = Math.ceil((target - pos)/3);
  1847.     dt = dt ? dt-1 : 25; // time step/max recursions
  1848.  
  1849.     if (target === pos || dpos === 0 || dt === 0) return;
  1850.  
  1851.     window.scrollBy(0, dpos);
  1852.     setTimeout(function() { autoScroll(location, dt); }, dt);
  1853.   }//}}}
  1854.  
  1855.   function Calendar(offsetX, offsetY, caller) {//{{{
  1856.     this.date = new Date();
  1857.     this.offsetX = offsetX;
  1858.     this.offsetY = offsetY;
  1859.     this.caller = caller;
  1860.     this.drawCalendar = function(year,month,day) {//{{{
  1861.       year = year || this.date.getFullYear();
  1862.       month = month || this.date.getMonth()+1;
  1863.       day = day || this.date.getDate();
  1864.       var longMonths = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
  1865.       var date = new Date(year,month-1,day);
  1866.       var anchors = _getAnchors(date);
  1867.  
  1868.       //make new container if one doesn't already exist
  1869.       var container = null;
  1870.       if (document.querySelector("#hdbCalendarPanel")) { 
  1871.         container = document.querySelector("#hdbCalendarPanel");
  1872.         container.removeChild( container.getElementsByTagName("TABLE")[0] );
  1873.       }
  1874.       else {
  1875.         container = document.createElement("DIV");
  1876.         container.id = "hdbCalendarPanel";
  1877.         document.body.appendChild(container);
  1878.       }
  1879.       container.style.left = this.offsetX;
  1880.       container.style.top = this.offsetY;
  1881.       var cal = document.createElement("TABLE");
  1882.       cal.cellSpacing = "0";
  1883.       cal.cellPadding = "0";
  1884.       cal.border = "0";
  1885.       container.appendChild(cal);
  1886.       cal.innerHTML = '<tr>' +
  1887.         '<th class="hdbCalHeader hdbCalControls" title="Previous month" style="text-align:right;"><span>&lt;</span></th>' +
  1888.         '<th class="hdbCalHeader hdbCalControls" title="Previous year" style="text-align:center;"><span>&#8810;</span></th>' +
  1889.         '<th colspan="3" id="hdbCalTableTitle" class="hdbCalHeader">'+date.getFullYear()+'<br>'+longMonths[date.getMonth()]+'</th>' +
  1890.         '<th class="hdbCalHeader hdbCalControls" title="Next year" style="text-align:center;"><span>&#8811;</span></th>' +
  1891.         '<th class="hdbCalHeader hdbCalControls" title="Next month" style="text-align:left;"><span>&gt;</span></th>' +
  1892.         '</tr><tr><th class="hdbDayHeader" style="color:red;">S</th><th class="hdbDayHeader">M</th>' +
  1893.         '<th class="hdbDayHeader">T</th><th class="hdbDayHeader">W</th><th class="hdbDayHeader">T</th>' +
  1894.         '<th class="hdbDayHeader">F</th><th class="hdbDayHeader">S</th></tr>';
  1895.       
  1896.       document.querySelector('th[title="Previous month"]').addEventListener( "click", function() { 
  1897.         this.drawCalendar(date.getFullYear(), date.getMonth(), 1);
  1898.       }.bind(this) );
  1899.       document.querySelector('th[title="Previous year"]').addEventListener( "click", function() {
  1900.         this.drawCalendar(date.getFullYear()-1, date.getMonth()+1, 1);
  1901.       }.bind(this) );
  1902.       document.querySelector('th[title="Next month"]').addEventListener( "click", function() {
  1903.         this.drawCalendar(date.getFullYear(), date.getMonth()+2, 1);
  1904.       }.bind(this) );
  1905.       document.querySelector('th[title="Next year"]').addEventListener( "click", function() {
  1906.         this.drawCalendar(date.getFullYear()+1, date.getMonth()+1, 1);
  1907.       }.bind(this) );
  1908.  
  1909.       var hasDay = false, thisDay = 1;
  1910.       for (var i=0;i<6;i++) { // cycle weeks
  1911.         var row = document.createElement("TR");
  1912.         for (var j=0;j<7;j++) { // cycle days
  1913.           if (!hasDay && j === anchors.first && thisDay < anchors.total)
  1914.             hasDay = true;
  1915.           else if (hasDay && thisDay > anchors.total)
  1916.             hasDay = false;
  1917.  
  1918.           var cell = document.createElement("TD");
  1919.           cell.classList.add("hdbCalCells");
  1920.           row.appendChild(cell);
  1921.           if (hasDay) {
  1922.             cell.classList.add("hdbCalDays");
  1923.             cell.textContent = thisDay;
  1924.             cell.addEventListener("click", _clickHandler.bind(this));
  1925.             cell.dataset.year = date.getFullYear();
  1926.             cell.dataset.month = date.getMonth()+1;
  1927.             cell.dataset.day = thisDay++;
  1928.           }
  1929.         } // for j
  1930.         cal.appendChild(row);
  1931.       } // for i
  1932.       var controls = cal.insertRow(-1);
  1933.       controls.insertCell(0);
  1934.       controls.cells[0].colSpan = "7";
  1935.       controls.cells[0].classList.add("hdbCalCells");
  1936.       controls.cells[0].innerHTML = ' &nbsp; &nbsp; <a href="javascript:void(0)" style="font-weight:bold;text-decoration:none;">Clear</a>' + 
  1937.         ' &nbsp; <a href="javascript:void(0)" style="font-weight:bold;text-decoration:none;">Close</a>';
  1938.       controls.cells[0].children[0].onclick = function() { this.caller.value = ""; }.bind(this);
  1939.       controls.cells[0].children[1].onclick = this.die;
  1940.  
  1941.       function _clickHandler(e) {
  1942.         /*jshint validthis:true*/
  1943.  
  1944.         var y = e.target.dataset.year;
  1945.         var m = Number(e.target.dataset.month).toPadded();
  1946.         var d = Number(e.target.dataset.day).toPadded();
  1947.         this.caller.value = y+"-"+m+"-"+d;
  1948.         this.die();
  1949.       }
  1950.  
  1951.       function _getAnchors(date) {
  1952.         var _anchors = {};
  1953.         date.setMonth(date.getMonth()+1);
  1954.         date.setDate(0);
  1955.         _anchors.total = date.getDate();
  1956.         date.setDate(1);
  1957.         _anchors.first = date.getDay();
  1958.         return _anchors;
  1959.       }
  1960.     };//}}} drawCalendar
  1961.  
  1962.     this.die = function() { document.getElementById('hdbCalendarPanel').remove(); };
  1963.  
  1964.   }//}}} Calendar
  1965.  
  1966.   // instance metrics apart from window scoped PerformanceTiming API
  1967.   function Metrics(name) {//{{{
  1968.     this.name = name || "undefined";
  1969.     this.marks = {};
  1970.     this.start = window.performance.now();
  1971.     this.end = null;
  1972.     this.stop = function(){
  1973.       if (!this.end) 
  1974.         this.end = window.performance.now();
  1975.       else
  1976.         Utils.errorHandler(new Error("Metrics::AccessViolation - end point cannot be overwritten"));
  1977.     };
  1978.     this.mark = function(name,position) {
  1979.       if (position === "end" && !this.marks[name]) return;
  1980.  
  1981.       if (!this.marks[name])
  1982.         this.marks[name] = {};
  1983.       if (!this.marks[name][position])
  1984.         this.marks[name][position] = window.performance.now();
  1985.     };
  1986.     this.report = function() {
  1987.       console.group("Metrics for",this.name.toUpperCase());
  1988.       console.log("Process completed in",+Number((this.end-this.start)/1000).toFixed(3),"seconds");
  1989.       for (var k in this.marks) {
  1990.         if (this.marks.hasOwnProperty(k)) {
  1991.           console.log(k,"occurred after",+Number((this.marks[k].start-this.start)/1000).toFixed(3),"seconds,",
  1992.               "resolving in", +Number((this.marks[k].end-this.marks[k].start)/1000).toFixed(3), "seconds");
  1993.         }
  1994.       }
  1995.       console.groupEnd();
  1996.     };
  1997.   }//}}}
  1998.  
  1999.   function INITDB() {//{{{
  2000.     HITStorage.db = this.result;
  2001.     self.HITStorage = {db: this.result};
  2002.     if (localStorage.getItem('hitdb_ridx') === 'true') return;
  2003.  
  2004.     Utils.disableButtons(['hdbDaily','hdbRequester','hdbPending','hdbSearch'], true);
  2005.     var count = 0;
  2006.     this.result.transaction('HIT', 'readwrite').objectStore('HIT').openCursor().onsuccess = function() {
  2007.       if (!this.result) {
  2008.         Status.push('Done.');
  2009.         Utils.disableButtons(['hdbDaily','hdbRequester','hdbPending','hdbSearch'], false);
  2010.         return localStorage.setItem('hitdb_ridx', 'true');
  2011.       }
  2012.       Status.push('Performing integrity check... ' + (++count));
  2013.       var r = this.result.value;
  2014.       if (typeof r.reward === 'object') {
  2015.         r.bonus = r.reward.bonus;
  2016.         r.reward = r.reward.pay;
  2017.         this.result.update(r);
  2018.       }
  2019.       this.result.continue();
  2020.     };
  2021.   }//}}}
  2022. })(); //scoping
  2023.  
  2024. // vim: ts=2:sw=2:et:fdm=marker:noai