☰

HIT Database Analytics

Analytics for HIT Database--makes pretty graphs

当前为 2015-09-23 提交的版本，查看 最新版本。

  1. // ==UserScript==
  2. // @name         HIT Database Analytics
  3. // @author       feihtality
  4. // @namespace    https://greasyfork.org/en/users/12709
  5. // @version      0.5.002
  6. // @description  Analytics for HIT Database--makes pretty graphs
  7. // @match        https://www.mturk.com/mturk/dashboard
  8. // @grant        none
  9. // ==/UserScript==
  10.  
  11.  
  12.  
  13. ((D) => {
  14.   'use strict';
  15.  
  16.   if (!('decRound' in Math) || !('Status' in window) || !('Progress' in window)) {
  17.       console.log('execution order is either too high or HITDB MKII is not detected');
  18.       return;
  19.   }
  20.  
  21.   const DB_NAME = "HITDB_TESTING";
  22.   Date.prototype.toISODateString = function() { return this.getUTCFullYear()+"-"+pad(this.getUTCMonth()+1)+"-"+pad(this.getUTCDate()); };
  23.   var pad = n => ("00"+n).substr(-2),
  24.       digitGroup = function(n) {
  25.         n = String(n).split('.');
  26.         if (n[0].length < 4) return n.join('.');
  27.         n[0] = n[0].replace(/(\d)(?=(\d{3})+$)/g, '$1,');
  28.         return n.join('.');
  29.       },
  30.       dec = (n,l) => Number(Math.decRound(n,l)).toFixed(l);
  31.  
  32.   // inject interface
  33.   var
  34.       insertion = D.getElementById('hdbCSVInput'),
  35.       searchbtn = D.getElementById('hdbSearch'),
  36.       acheckbox = insertion.parentNode.insertBefore(D.createElement('INPUT'), insertion.nextSibling),
  37.       alabel    = insertion.parentNode.insertBefore(D.createElement('LABEL'), insertion.nextSibling),
  38.       metrics   = null;
  39.  
  40.   acheckbox.type     = 'checkbox';
  41.   acheckbox.id       = 'hdbAnalytics';
  42.   acheckbox.style.verticalAlign = 'middle';
  43.   alabel.textContent = 'Analyze';
  44.   alabel.htmlFor     = 'hdbAnalytics';
  45.   alabel.style.verticalAlign = 'middle';
  46.  
  47.   acheckbox.onclick = function() {
  48.     if (searchbtn.textContent === "Export CSV") insertion.click();
  49.     if (searchbtn.textContent === "Analyze") searchbtn.textContent = 'Search';
  50.     else searchbtn.textContent = 'Analyze';
  51.   };
  52.   searchbtn.addEventListener('click', getData);
  53.  
  54.   function getData(e) {
  55.     if (e.target.textContent !== "Analyze") return;
  56.     
  57.     var range;
  58.     dbrange = [D.getElementById('hdbMinDate').value || undefined, D.getElementById('hdbMaxDate').value || undefined, ''];
  59.     if (!dbrange[0] && !dbrange[1]) {
  60.       dbrange[2] = 'ALL';
  61.       range = null;
  62.     } else if (dbrange[0] && !dbrange[1]) {
  63.       dbrange[2] = dbrange[0]+'>>';
  64.       range = window.IDBKeyRange.lowerBound(dbrange[0]);
  65.     } else if (!dbrange[0] && dbrange[1]) {
  66.       dbrange[2] = '<<'+dbrange[1];
  67.       range = window.IDBKeyRange.upperBound(dbrange[1]);
  68.     } else {
  69.       dbrange[2] = dbrange[0]+':'+dbrange[1];
  70.       range = window.IDBKeyRange.bound(dbrange[0],dbrange[1]);
  71.     }
  72.  
  73.     sets = new Sets();
  74.     window.Progress.show();
  75.     metrics = new window.Metrics('database_analytics');
  76.     metrics.mark('dbrecall', 'start');
  77.     dbrecall("HIT", {range: range}).then(analyzeData);//.then(drawCharts);
  78.   }
  79.  
  80. /* . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . */
  81.  
  82.   var
  83.     Sets = function() {//{{{ main data obj
  84.       this.year = {
  85.         aggregate: { def: {}, req: {} },
  86.         pay:   { labels: [], data: [] },
  87.         hits:  { labels: [], data: [] },
  88.         rpay:  {},
  89.         rhits: {}
  90.       };
  91.       this.month = {
  92.         aggregate: { def: {}, req: {} },
  93.         pay:   { labels: [], data: [] },
  94.         hits:  { labels: [], data: [] },
  95.         rpay:  {},
  96.         rhits: {}
  97.       };
  98.       this.week = {
  99.         aggregate: { def: {}, req: {} },
  100.         pay:   { labels: [], data: [] },
  101.         hits:  { labels: [], data: [] },
  102.         rpay:  {},
  103.         rhits: {}
  104.       };
  105.       this.day = {
  106.         aggregate: { def: {}, req: {} },
  107.         pay:   { labels: [], data: [] },
  108.         hits:  { labels: [], data: [] },
  109.         rpay:  {},
  110.         rhits: {}
  111.       };
  112.       this.all = {
  113.         aggregate: [],
  114.         distribution: {
  115.           hits: {'1':0,'2-5':0,'6-10':0,'11-15':0,'16-20':0,'21-25':0,'26-30':0,'31-35':0,
  116.             '36-40':0,'41-45':0,'46-50':0,'51-100':0,'101-150':0,'151-200':0,'201-250':0,'251-300':0,
  117.             '301-350':0,'351-400':0,'401-450':0,'451-500':0,'501-550':0,'551-600':0,'601-650':0,'651-700':0,
  118.             '701-750':0,'751-800':0,'801-850':0,'851-900':0,'901-950':0,'951-1000':0,'1000+':0},
  119.           pay: {'0':0, '0.01-0.05':0,'0.06-0.10':0,'0.11-0.15':0,'0.16-0.20':0,'0.21-0.25':0,
  120.             '0.26-0.30':0,'0.31-0.35':0,'0.36-0.40':0,'0.41-0.45':0,'0.46-0.50':0,'0.51-0.55':0,
  121.             '0.56-0.60':0,'0.61-0.65':0,'0.66-0.70':0,'0.71-0.75':0,'0.76-0.80':0,'0.81-0.85':0,
  122.             '0.86-0.90':0,'0.91-0.95':0,'0.96-1.00':0,'1.01-1.25':0,'1.26-1.50':0,'1.51-1.75':0,
  123.             '1.76-2.00':0,'2.01-2.25':0,'2.26-2.50':0,'2.51-2.75':0,'2.76-3.00':0,'3.01-3.25':0,
  124.             '3.26-3.50':0,'3.51-3.75':0,'3.76-4.00':0,'4.01-4.25':0,'4.26-4.50':0,'4.51-4.75':0,
  125.             '4.76-5.00':0,'5.01+':0},
  126.           data: { hits: [], labelsh: [], pay: [], labelsp: [] } },
  127.         hits: { total: 0, rejected: 0, pending: 0, titles: {}, batch: [] },
  128.         pay : { total: 0, rejected: 0, pending: 0 },
  129.         hitsPerRequester: { avg: 0, data: [], sd: 0, se: 0 },
  130.         payPerHit: { avg: 0, data: [], sd: 0, se: 0 }
  131.       };
  132.     },//}}}
  133.     sets   = new Sets(),
  134.     disth  = Object.keys(sets.all.distribution.hits),
  135.     distp  = Object.keys(sets.all.distribution.pay),
  136.     ymwdLabels = Object.keys(sets), dbrange;
  137.  
  138.   function dbrecall(os, options) {
  139.     options = options || {};
  140.     var 
  141.       index = options.index || "date",
  142.       range = options.range || null,
  143.       wRange = [null, null];
  144.  
  145.     var total = 0;
  146.     return new Promise( y => {
  147.       window.indexedDB.open(DB_NAME).onsuccess = function() {
  148.         this.result.transaction(os, "readonly").objectStore(os).index(index).openCursor(range).onsuccess = function() {
  149.           if (this.result) { 
  150.             window.Status.message = 'Aggregating data... [ '+(total++)+' ]';
  151.             wRange = aggregateCursor(this.result.value, wRange);
  152.             this.result.continue();
  153.           }
  154.           else y(1);
  155.         };
  156.         this.result.close();
  157.       };
  158.     });
  159.   }
  160.  
  161.   function analyzeData(r) {
  162.     void(r);
  163.     metrics.mark('dbrecall','end');
  164.     //var sets = data.getDatasets();
  165.     console.log(sets);
  166.     metrics.mark('dsWorker', 'start');
  167.     dsWorker.postMessage(sets);
  168.   }
  169.  
  170.   function drawCharts(sets) {//{{{
  171.     var 
  172.       hitlineopt = { 
  173.         label: 'HITs Submitted',
  174.         yAxixID: 'hits',
  175.         backgroundColor: 'rgba(149,89,240,0.1)'/*'rgba(42,161,152,0.1)'*/,
  176.         borderColor    : 'rgba(149,89,240,0.5)'/*'rgba(42,161,152,0.5)'*/,
  177.         pointBackgroundColor     : 'rgba(149,89,240,1)'/*'rgba(42,161,152,1)'*/,
  178.         pointHoverBackgroundColor: 'rgba(92,0,230,1)'/*'rgba(38,139,210,1)'*/
  179.       },
  180.       paylineopt = {
  181.         label: 'Total Pay',
  182.         yAxisID: 'pay',
  183.         backgroundColor: 'rgba(200,242,48,0.1)'/*'rgba(181,137,0,0.1)'*/,
  184.         borderColor    : 'rgba(200,242,48,0.5)'/*'rgba(181,137,0,0.5)'*/,
  185.         pointBackgroundColor     : 'rgba(200,242,48,1)'/*'rgba(181,137,0,1)'*/,
  186.         pointHoverBackgroundColor: 'rgba(129,163,5,1)'/*'rgba(133,153,0,1)'*/
  187.       },
  188.       chartopt = {};
  189.       
  190.     for (var k of Object.keys(sets)) {
  191.       if (k === 'all') {
  192.         var a = sets.all.hits;
  193.         chartopt.bsratio = {
  194.           type: 'pie', data: {
  195.             labels: ['batch %', 'survey %'],
  196.             datasets: [{ data: [Math.decRound(100*a.batch.total/a.total,2), Math.decRound(100*(a.total-a.batch.total)/a.total,2)],
  197.               backgroundColor: ['#4773ED', '#EDC147'] }]
  198.           }
  199.         };
  200.         chartopt.pdist = {
  201.           type: 'bar', data: {
  202.             labels: sets.all.distribution.data.labelsp,
  203.             datasets: [{ data: sets.all.distribution.data.pay,
  204.               backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }]
  205.           }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1 }] }}
  206.         };
  207.         chartopt.hdist = {
  208.           type: 'bar', data: {
  209.             labels: sets.all.distribution.data.labelsh,
  210.             datasets: [{ data: sets.all.distribution.data.hits,
  211.               backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }]
  212.           }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1}] }}
  213.         };
  214.         continue;
  215.       }
  216.       Object.assign(sets[k].hits, hitlineopt);
  217.       Object.assign(sets[k].pay, paylineopt);
  218.       chartopt[k] = {
  219.         type:'line', data: { labels: sets[k].hits.labels, datasets: [sets[k].hits, sets[k].pay] }, 
  220.         options: {
  221.           stacked: false, scales: {
  222.             xAxes: [{ gridLines: { offsetGridLines: false }, display: true, scaleSteps: 10,
  223.               labels: {fontSize: /*/^[ym]/.test(k) ? 5 :*/ 0, show: /^[ym]/.test(k) ? true : false} }],
  224.             yAxes: [{ type: 'linear', display: true, position: 'left', id: 'hits' },
  225.                     { type: 'linear', display: true, position: 'right', id: 'pay', gridLines: { drawOnChartArea: false } }]
  226.           }
  227.         }
  228.       };
  229.     }
  230.     var html = ['<head>',
  231.        '<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400" rel="stylesheet" type="text/css">',
  232.        '<script src="https://cdn.jsdelivr.net/chart.js/2.0.0-alpha3/Chart.min.js"></script>',
  233.        '<style>',
  234.         '.caption {width:60%;margin:auto;padding:5px;text-align:center;font-size:0.75em}',
  235.         '.hc {color:#9559F0} .pc {color:#C8F230}',
  236.         '.bc {color:#4773ED} .sc {color:#EDC147}',
  237.         '.container {display:flex;flex-wrap:wrap;align-content:center;justify-content:space-around}',
  238.         '.mi {font-size:0.6em}',
  239.         '.title {background-color:#EEE8D5;color:#073642;width:90%;margin:1% 6% 0% 5%;padding-left:2%;font-weight:400;}',
  240.        '</style>',
  241.        '<title>HIT Database Analytics | '+dbrange[2]+'</title>',
  242.       '</head>',
  243.       '<body style="color:#fff;background-color:#073642;font-size:100%;font-family:\'Open Sans\',Arial;font-weight:300">',
  244.       // '<i style="color:#fff;">Sigma = &#963;</i> <span style="color:#fff;font-family:"\'Times New Roman\', Arial;">&#963;</span>',
  245.        '<div class="title">GENERAL STATISTICS</div>',
  246.        '<div class="container">',
  247.         '<div style="margin-top:5%;margin-left:5%;flex:1">',
  248.          '<div class="container" style="margin-right:10%">',
  249.            '<span style="text-align:right;color:#BBD12A;flex:1">'+digitGroup(sets.all.hits.total)+'</span>',
  250.            '<span style="text-align:center;flex:1">SUBMITTED</span>',
  251.            '<span style="text-align:left;color:#BBD12A;flex:1">$'+digitGroup(dec(sets.all.pay.total,2))+'</span>',
  252.          '</div><div class="container" style="margin-right:10%">',
  253.            '<span style="text-align:right;color:#F23054;flex:1">'+digitGroup(sets.all.hits.rejected)+'</span>',
  254.            '<span style="text-align:center;flex:1">REJECTED</span>',
  255.            '<span style="text-align:left;color:#F23054;flex:1">$'+digitGroup(dec(sets.all.pay.rejected,2))+'</span>',
  256.          '</div><div class="container" style="margin-right:10%">',
  257.            '<span style="text-align:right;color:#E0B438;flex:1">'+digitGroup(sets.all.hits.pending)+'</span>',
  258.            '<span style="text-align:center;flex:1">PENDING</span>',
  259.            '<span style="text-align:left;color:#E0B438;flex:1">$'+digitGroup(dec(sets.all.pay.pending,2))+'</span>',
  260.          '</div>',
  261.          '<span class="mi" style="text-align:justify"><span style="color:#30CEF2">NOTE:</span> DOLLAR AMOUNT FOR \'PENDING\' MAY ',
  262.            'REFLECT VALUES FOR HITS WHICH HAVE ALREADY BEEN APPROVED. THESE FUNDS HAVE NOT BEEN FULLY CLEARED ',
  263.            'AND CREDITED TO YOUR ACCOUNT, AND THUS ARE STILL TECHNICALLY PENDING.</span>',
  264.          '<div style="margin:10% 10% 10% 0;flex:1;text-align:center;">',
  265.           'AVERAGE SUBMISSION OF <span style="color:#C7ED3E">'+dec(sets.all.hitsPerRequester.avg,3)+'</span> ',
  266.           '(<span style="color:#C7ED3E">&#177;'+dec(sets.all.hitsPerRequester.se,3)+'</span>) HITS PER REQUESTER',
  267.           '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.hitsPerRequester.sd,3)+'</span></span>',
  268.           '<hr style="width:45%;color:#fff;background-color:#fff">',
  269.           'AVERAGE PAY OF <span style="color:#C7ED3E">$'+dec(sets.all.payPerHit.avg,3)+'</span> ',
  270.           '(<span style="color:#C7ED3E">&#177;$'+dec(sets.all.payPerHit.se,3)+'</span>) PER HIT',
  271.           '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.payPerHit.sd,3)+'</span></span>',
  272.           '<hr style="width:45%;color:#fff;background-color:#fff">',
  273.           'AVERAGING <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg+sets.all.payPerHit.se))+'</span>',
  274.           '<span id="minbound"> TO <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg-sets.all.payPerHit.se))+
  275.             '</span></span> HITS PER $1.00',
  276.          '</div>',
  277.         '</div>',
  278.         '<div style="margin-top:1%;flex:1">',
  279.          '<div><canvas id="hitdist"></canvas><div class="caption">HIT DISTRIBUTION PER REQUESTER</div></div>',
  280.          '<div><canvas id="paydist"></canvas><div class="caption">PAY DISTRIBUTION PER HIT</div></div>',
  281.         '</div>',
  282.         '<div style="margin-top:10%;flex:1">',
  283.          '<canvas id="batchpie"></canvas>',
  284.          '<div class="caption"><span class="bc">BATCH</span> : <span class="sc">SURVEY</span><br>RATIO APPROXIMATION</div>',
  285.         '</div>',
  286.        '</div>',
  287.        '<div class="title">ACTIVITY</div>',
  288.        '<div class="container">',
  289.         '<div style="margin-top:1%;margin-left:5%;flex:1">',
  290.          '<canvas id="dailyline"></canvas>',
  291.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>DAILY</div>',
  292.         '</div>',
  293.         '<div style="margin-top:1%;margin-right:6%;flex:1">',
  294.          '<canvas id="weeklyline"></canvas>',
  295.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>WEEKLY</div>',
  296.         '</div>',
  297.        '</div>',
  298.        '<div class="container">',
  299.         '<div style="margin-top:1%;margin-left:5%;flex:1">',
  300.          '<canvas id="monthlyline"></canvas>',
  301.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>MONTHLY</div>',
  302.         '</div>',
  303.         '<div style="margin-top:1%;margin-right:6%;flex:1">',
  304.          '<canvas id="yearlyline"></canvas>',
  305.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>YEARLY</div>',
  306.         '</div>',
  307.        '</div>',
  308.        '<div class="title">TOP REQUESTERS</div>',
  309.        '<div class="container">',
  310.         '<table id="req10pay" style="font-weight:300;margin-top:1%;margin-left:5%;">',
  311.          '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">PAY</th></tr></thead><tbody></tbody></table>',
  312.         '<table id="req10hit" style="font-weight:300;margin-top:1%;margin-right:6%;">',
  313.          '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">HITS</th></tr></thead><tbody></tbody></table>',
  314.        '</div>',
  315.        '<script>',
  316.         'if (!NodeList.prototype[Symbol.iterator]) NodeList.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator];',
  317.         'var chartopt = '+JSON.stringify(chartopt)+',',
  318.         '    dLines = new Chart(document.getElementById("dailyline").getContext("2d"), chartopt.day),',
  319.         '    mLines = new Chart(document.getElementById("monthlyline").getContext("2d"), chartopt.month),',
  320.         '    wLines = new Chart(document.getElementById("weeklyline").getContext("2d"), chartopt.week),',
  321.         '    yLines = new Chart(document.getElementById("yearlyline").getContext("2d"), chartopt.year),',
  322.         '    bsPie  = new Chart(document.getElementById("batchpie").getContext("2d"), chartopt.bsratio),',
  323.         '    hdist  = new Chart(document.getElementById("hitdist").getContext("2d"), chartopt.hdist),',
  324.         '    pdist  = new Chart(document.getElementById("paydist").getContext("2d"), chartopt.pdist),',
  325.         '    r10hit = document.getElementById("req10hit").tBodies[0], r10hHtml = [],',
  326.         '    r10pay = document.getElementById("req10pay").tBodies[0], r10pHtml = [],',
  327.         '    payArr = '+JSON.stringify(sets.all.rpay)+',',
  328.         '    hitArr = '+JSON.stringify(sets.all.rhits)+',',
  329.         '    mb     = document.getElementById("minbound");',
  330.         'if (mb.children[0].textContent === mb.previousSibling.textContent) mb.style.display = "none";',
  331.         'payArr.forEach(v => r10pHtml',
  332.           '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">$"+v.pay+"</td><td>"+v.name+"</td></tr>"));',
  333.         'hitArr.forEach(v => r10hHtml',
  334.           '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">"+v.hits+"</td><td>"+v.name+"</td></tr>"));',
  335.         'r10hit.innerHTML = r10hHtml.join(""); r10pay.innerHTML = r10pHtml.join("");',
  336.        '</script>',
  337.       '</body>'];
  338.     var blob = new Blob(html, {type:'text/html'}), _a = D.body.appendChild(D.createElement('A'));
  339.     _a.href = URL.createObjectURL(blob);
  340.     _a.target = '_blank';
  341.     _a.click();
  342.     _a.remove();
  343.   }//}}}
  344.  
  345.   function aggregateCursor(dat, wRange) {//{{{
  346.     var ymwd = [dat.date.substr(0,4), dat.date.substr(0,7), null, dat.date],
  347.         pay = isNaN(dat.reward) ? +dat.reward.pay : +dat.reward;
  348.     // determine weekly scales
  349.     if (!wRange[0])
  350.       wRange = getWeekRange(dat.date);
  351.     if (dat.date >= wRange[1])
  352.       wRange = getWeekRange(dat.date);
  353.     ymwd[2] = wRange.join(':');
  354.  
  355.     // populate scaled aggregates
  356.     for (var i=0; i<ymwd.length; i++) {
  357.       // ... basic
  358.       if (!sets[ymwdLabels[i]].aggregate.def[ymwd[i]])
  359.         sets[ymwdLabels[i]].aggregate.def[ymwd[i]] = { hits:0, pay:0 };
  360.       sets[ymwdLabels[i]].aggregate.def[ymwd[i]].hits += 1;
  361.       sets[ymwdLabels[i]].aggregate.def[ymwd[i]].pay  += pay;
  362.       // ... by requester
  363.       if (i > 1) continue; // skip--don't need this for day or week
  364.       if (!sets[ymwdLabels[i]].aggregate.req[ymwd[i]]) {
  365.         sets[ymwdLabels[i]].aggregate.req[ymwd[i]] = [];
  366.         sets[ymwdLabels[i]].rpay[ymwd[i]] = [];
  367.         sets[ymwdLabels[i]].rhits[ymwd[i]] = [];
  368.       }
  369.       var reqArr = sets[ymwdLabels[i]].aggregate.req[ymwd[i]],
  370.           reqLoc = reqArr.findIndex(v => v.req === dat.requesterId);
  371.       if (~reqLoc) {
  372.         reqArr[reqLoc].hits += 1;
  373.         reqArr[reqLoc].pay += pay;
  374.       }
  375.       else
  376.         reqArr.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay });
  377.     }// end for ymwd loop
  378.     // repeat requester aggregation for 'all'
  379.     reqLoc = sets.all.aggregate.findIndex(v => v.req === dat.requesterId);
  380.     if (~reqLoc) {
  381.       sets.all.aggregate[reqLoc].hits += 1;
  382.       sets.all.aggregate[reqLoc].pay += pay;
  383.     }
  384.     else
  385.       sets.all.aggregate.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay });
  386.  
  387.     // populate pay distribution data
  388.     //   hit distribution needs to wait until later--after aggregation
  389.     sets.all.payPerHit.data.push(pay);
  390.     distp.forEach( v => {
  391.       var range = v.split('-');
  392.       if (pay === 0) { sets.all.distribution.pay['0'] += 1; return; }
  393.       if (pay > 5) { sets.all.distribution.pay['5.01+'] += 1; return; }
  394.       if (pay >= range[0] && pay <= range[1]) { sets.all.distribution.pay[v] += 1; return; }
  395.     });
  396.     // increment general stats
  397.     var _t = dat.title.trim();
  398.     sets.all.hits.total += 1;
  399.     sets.all.hits.rejected += dat.status === 'Rejected' ? 1 : 0;
  400.     sets.all.hits.pending += dat.status === 'Pending Approval' ? 1 : 0;
  401.     sets.all.hits.titles[_t] = sets.all.hits.titles[_t] + 1 || 1;
  402.     sets.all.pay.total += pay;
  403.     sets.all.pay.rejected += dat.status === 'Rejected' ? pay : 0;
  404.     sets.all.pay.pending += /pending/i.test(dat.status) ? pay : 0;
  405.     // populate data for batch approximation
  406.     var hitLoc = sets.all.hits.batch.findIndex(v => v.title === _t && v.req === dat.requesterId);
  407.     if (~hitLoc)
  408.       sets.all.hits.batch[hitLoc].count += 1;
  409.     else
  410.       sets.all.hits.batch.push({ title: _t, req: dat.requesterId, count: 1 });
  411.  
  412.     return wRange;
  413.   }//}}}
  414.  
  415.   function DBResult() {
  416.     this.results = [];
  417.     this.add = v => this.results.push(v);
  418.     this.getDatasets = () => {// {{{
  419.       
  420.       return sets;
  421.  
  422.     };//}}} getDatasets
  423.   }// DBResult
  424.  
  425.   function getWeekRange(date) {
  426.     var _d = new Date(date), _ws, _we;
  427.     _d.setUTCDate(_d.getUTCDate()-_d.getUTCDay());
  428.     _ws = _d.toISODateString();
  429.     _d.setUTCDate(_d.getUTCDate()+7);
  430.     _we = _d.toISODateString();
  431.     return [ _ws, _we ];
  432.   }
  433.  
  434.   //set up a worker thread to prevent interface locking on large datasets
  435.   var datasetsWorker = [//{{{
  436.     'var dec = '+dec+', digitGroup = '+digitGroup+';',
  437.     'Math.decRound = '+Math.decRound+';',
  438.     // calculate the standard deviation
  439.     'function stdev(avg, N, data) {',
  440.       'var sum = 0;',
  441.       'data.forEach(v => sum += Math.pow(v-avg, 2));',
  442.       'return Math.sqrt(sum/(N-1));',
  443.     '}',
  444.  
  445.     'onmessage = (e) => {',
  446.       'var sets = e.data, disth = '+JSON.stringify(disth)+', distp = '+JSON.stringify(distp)+';',
  447.       // sort aggregates
  448.       'for (var period of Object.keys(sets)) {',
  449.         'if (period === \'all\') { ',
  450.           // calculate averages, sd, se
  451.           'postMessage( {type: "status", data: "Calculating averages..." });',
  452.           'var _pph = sets.all.payPerHit, _hpr = sets.all.hitsPerRequester;',
  453.           '_pph.avg = sets.all.pay.total / sets.all.hits.total;',
  454.           '_pph.sd = stdev(_pph.avg, _pph.data.length, _pph.data);',
  455.           '_pph.se = _pph.sd/Math.sqrt(_pph.data.length);',
  456.           '_hpr.data = sets.all.aggregate.map( v => v.hits );',
  457.           '_hpr.avg = sets.all.hits.total / _hpr.data.length;',
  458.           '_hpr.sd = stdev(_hpr.avg, _hpr.data.length, _hpr.data);',
  459.           '_hpr.se = _hpr.sd/Math.sqrt(_hpr.data.length);',
  460.           // get hit distribution
  461.           'postMessage( {type: "status", data: "Populating distributions..." });',
  462.           'sets.all.aggregate.forEach(v => {',
  463.             'disth.forEach(w => {',
  464.               'var range = w.split(\'-\');',
  465.               'if (range.length === 1) range.push(\'1\');',
  466.               'if (v.hits > 1000) { sets.all.distribution.hits[\'1001+\'] += 1; return; }',
  467.               'if (v.hits >= range[0] && v.hits <= range[1]) { sets.all.distribution.hits[w] += 1; return; }',
  468.             '});',
  469.           '});',
  470.           // extract raw data for distributions
  471.           'disth.forEach(v => sets.all.distribution.data.hits.push(sets.all.distribution.hits[v]));',
  472.           'distp.forEach(v => sets.all.distribution.data.pay.push(sets.all.distribution.pay[v]));',
  473.           'sets.all.distribution.data.labelsh = Object.keys(sets.all.distribution.hits);',
  474.           'sets.all.distribution.data.labelsp = Object.keys(sets.all.distribution.pay);',
  475.           // get 'top' lists
  476.           'sets.all.rpay  = sets.all.aggregate.sort((a,b) => b.pay  - a.pay ).slice(0,10);',
  477.           'sets.all.rhits = sets.all.aggregate.sort((a,b) => b.hits - a.hits).slice(0,10);',
  478.           'sets.all.rpay.forEach(v => v.pay = dec(v.pay,2));',
  479.           'sets.all.rhits.forEach(v => v.hits = digitGroup(v.hits));',
  480.           'continue;',
  481.         '}',
  482.         // populate labels/data for graphing
  483.         'postMessage({ type: "status", data: "Sorting by "+period+"..." });',
  484.         'var labels = Object.keys(sets[period].aggregate.def);',
  485.         'for (k of labels) {',
  486.           'if (~[\'hits\',\'pay\'].indexOf(k)) continue;',
  487.           'sets[period].pay.labels.push(k);',
  488.           'sets[period].hits.labels.push(k);',
  489.           'sets[period].pay.data.push(Math.decRound(sets[period].aggregate.def[k].pay,2));',
  490.           'sets[period].hits.data.push(sets[period].aggregate.def[k].hits);',
  491.  
  492.           // sort req lists/period
  493.           'if (~[\'week\',\'day\'].indexOf(period)) continue;',
  494.           'var len = period === \'year\' ? 10 : 5;',
  495.           'reqArr = sets[period].aggregate.req[k];',
  496.           'sets[period].rpay[k] = reqArr.sort((a,b) => b.pay - a.pay).slice(0,len);',
  497.           'sets[period].rhits[k] = reqArr.map(v => v).sort((a,b) => b.hits - a.hits).slice(0,len);',
  498.         '}',
  499.       '}',
  500.       // if there are more than 5 hits with the same title under the same requester, assume it's a batch
  501.       'sets.all.hits.batch = sets.all.hits.batch.filter(v => v.count > 5);',
  502.       // workers fail to transfer nonenumerable properties :( 
  503.       //'Object.defineProperty(sets.all.hits.batch, \'total\', { configurable: true, writable: true, value: 0 });',
  504.       'sets.all.hits.batch.total = 0;',
  505.       'sets.all.hits.batch.forEach(v => {if (typeof v === "object") sets.all.hits.batch.total += v.count});',
  506.  
  507.       'postMessage({ type: "obj", data: sets });',
  508.  
  509.     '}'];//}}}
  510.   datasetsWorker = new Blob(datasetsWorker, {type:'text/javascript'});
  511.   var workerURL = URL.createObjectURL(datasetsWorker);
  512.   var dsWorker = new Worker(workerURL);
  513.   dsWorker.onmessage = (e) => {
  514.     if (e.data.type === 'status') {
  515.       window.Status.message = e.data.data;
  516.       console.log(e.data.data);
  517.     } else {
  518.       metrics.mark('dsWorker','end');
  519.       metrics.stop();metrics.report();
  520.       window.Status.message = 'Done!';
  521.       window.Progress.hide();
  522.       drawCharts(e.data.data);
  523.     }
  524.   };
  525.  
  526. })(document);
  527.  
  528. // vim: ts=2:sw=2:et:fdm=marker:noai
  529.  
  530. ((D) => {
  531.   'use strict';
  532.  
  533.   if (!('decRound' in Math) || !('Status' in window) || !('Progress' in window)) {
  534.       console.log('execution order is either too high or HITDB MKII is not detected');
  535.       return;
  536.   }
  537.  
  538.   const DB_NAME = "DATA_TESTING";
  539.   Date.prototype.toISODateString = function() { return this.getUTCFullYear()+"-"+pad(this.getUTCMonth()+1)+"-"+pad(this.getUTCDate()); };
  540.   var pad = n => ("00"+n).substr(-2),
  541.       digitGroup = function(n) {
  542.         n = String(n).split('.');
  543.         if (n[0].length < 4) return n.join('.');
  544.         n[0] = n[0].replace(/(\d)(?=(\d{3})+$)/g, '$1,');
  545.         return n.join('.');
  546.       },
  547.       dec = (n,l) => Number(Math.decRound(n,l)).toFixed(l);
  548.  
  549.   // inject interface
  550.   var
  551.       insertion = D.getElementById('hdbCSVInput'),
  552.       searchbtn = D.getElementById('hdbSearch'),
  553.       acheckbox = insertion.parentNode.insertBefore(D.createElement('INPUT'), insertion.nextSibling),
  554.       alabel    = insertion.parentNode.insertBefore(D.createElement('LABEL'), insertion.nextSibling),
  555.       metrics   = null;
  556.  
  557.   acheckbox.type     = 'checkbox';
  558.   acheckbox.id       = 'hdbAnalytics';
  559.   acheckbox.style.verticalAlign = 'middle';
  560.   alabel.textContent = 'Analyze';
  561.   alabel.htmlFor     = 'hdbAnalytics';
  562.   alabel.style.verticalAlign = 'middle';
  563.  
  564.   acheckbox.onclick = function() {
  565.     if (searchbtn.textContent === "Export CSV") insertion.click();
  566.     if (searchbtn.textContent === "Analyze") searchbtn.textContent = 'Search';
  567.     else searchbtn.textContent = 'Analyze';
  568.   };
  569.   searchbtn.addEventListener('click', getData);
  570.  
  571.   function getData(e) {
  572.     if (e.target.textContent !== "Analyze") return;
  573.     
  574.     var range;
  575.     dbrange = [D.getElementById('hdbMinDate').value || undefined, D.getElementById('hdbMaxDate').value || undefined, ''];
  576.     if (!dbrange[0] && !dbrange[1]) {
  577.       dbrange[2] = 'ALL';
  578.       range = null;
  579.     } else if (dbrange[0] && !dbrange[1]) {
  580.       dbrange[2] = dbrange[0]+'>>';
  581.       range = window.IDBKeyRange.lowerBound(dbrange[0]);
  582.     } else if (!dbrange[0] && dbrange[1]) {
  583.       dbrange[2] = '<<'+dbrange[1];
  584.       range = window.IDBKeyRange.upperBound(dbrange[1]);
  585.     } else {
  586.       dbrange[2] = dbrange[0]+':'+dbrange[1];
  587.       range = window.IDBKeyRange.bound(dbrange[0],dbrange[1]);
  588.     }
  589.  
  590.     sets = new Sets();
  591.     window.Progress.show();
  592.     metrics = new window.Metrics('database_analytics');
  593.     metrics.mark('dbrecall', 'start');
  594.     dbrecall("HIT", {range: range}).then(analyzeData);//.then(drawCharts);
  595.   }
  596.  
  597. /* . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . */
  598.  
  599.   var
  600.     Sets = function() {//{{{ main data obj
  601.       this.year = {
  602.         aggregate: { def: {}, req: {} },
  603.         pay:   { labels: [], data: [] },
  604.         hits:  { labels: [], data: [] },
  605.         rpay:  {},
  606.         rhits: {}
  607.       };
  608.       this.month = {
  609.         aggregate: { def: {}, req: {} },
  610.         pay:   { labels: [], data: [] },
  611.         hits:  { labels: [], data: [] },
  612.         rpay:  {},
  613.         rhits: {}
  614.       };
  615.       this.week = {
  616.         aggregate: { def: {}, req: {} },
  617.         pay:   { labels: [], data: [] },
  618.         hits:  { labels: [], data: [] },
  619.         rpay:  {},
  620.         rhits: {}
  621.       };
  622.       this.day = {
  623.         aggregate: { def: {}, req: {} },
  624.         pay:   { labels: [], data: [] },
  625.         hits:  { labels: [], data: [] },
  626.         rpay:  {},
  627.         rhits: {}
  628.       };
  629.       this.all = {
  630.         aggregate: [],
  631.         distribution: {
  632.           hits: {'1':0,'2-5':0,'6-10':0,'11-15':0,'16-20':0,'21-25':0,'26-30':0,'31-35':0,
  633.             '36-40':0,'41-45':0,'46-50':0,'51-100':0,'101-150':0,'151-200':0,'201-250':0,'251-300':0,
  634.             '301-350':0,'351-400':0,'401-450':0,'451-500':0,'501-550':0,'551-600':0,'601-650':0,'651-700':0,
  635.             '701-750':0,'751-800':0,'801-850':0,'851-900':0,'901-950':0,'951-1000':0,'1000+':0},
  636.           pay: {'0':0, '0.01-0.05':0,'0.06-0.10':0,'0.11-0.15':0,'0.16-0.20':0,'0.21-0.25':0,
  637.             '0.26-0.30':0,'0.31-0.35':0,'0.36-0.40':0,'0.41-0.45':0,'0.46-0.50':0,'0.51-0.55':0,
  638.             '0.56-0.60':0,'0.61-0.65':0,'0.66-0.70':0,'0.71-0.75':0,'0.76-0.80':0,'0.81-0.85':0,
  639.             '0.86-0.90':0,'0.91-0.95':0,'0.96-1.00':0,'1.01-1.25':0,'1.26-1.50':0,'1.51-1.75':0,
  640.             '1.76-2.00':0,'2.01-2.25':0,'2.26-2.50':0,'2.51-2.75':0,'2.76-3.00':0,'3.01-3.25':0,
  641.             '3.26-3.50':0,'3.51-3.75':0,'3.76-4.00':0,'4.01-4.25':0,'4.26-4.50':0,'4.51-4.75':0,
  642.             '4.76-5.00':0,'5.01+':0},
  643.           data: { hits: [], labelsh: [], pay: [], labelsp: [] } },
  644.         hits: { total: 0, rejected: 0, pending: 0, titles: {}, batch: [] },
  645.         pay : { total: 0, rejected: 0, pending: 0 },
  646.         hitsPerRequester: { avg: 0, data: [], sd: 0, se: 0 },
  647.         payPerHit: { avg: 0, data: [], sd: 0, se: 0 }
  648.       };
  649.     },//}}}
  650.     sets   = new Sets(),
  651.     disth  = Object.keys(sets.all.distribution.hits),
  652.     distp  = Object.keys(sets.all.distribution.pay),
  653.     ymwdLabels = Object.keys(sets), dbrange;
  654.  
  655.   function dbrecall(os, options) {
  656.     options = options || {};
  657.     var 
  658.       index = options.index || "date",
  659.       range = options.range || null,
  660.       wRange = [null, null];
  661.  
  662.     var total = 0;
  663.     return new Promise( y => {
  664.       window.indexedDB.open(DB_NAME).onsuccess = function() {
  665.         this.result.transaction(os, "readonly").objectStore(os).index(index).openCursor(range).onsuccess = function() {
  666.           if (this.result) { 
  667.             window.Status.message = 'Aggregating data... [ '+(total++)+' ]';
  668.             wRange = aggregateCursor(this.result.value, wRange);
  669.             this.result.continue();
  670.           }
  671.           else y(1);
  672.         };
  673.         this.result.close();
  674.       };
  675.     });
  676.   }
  677.  
  678.   function analyzeData(r) {
  679.     void(r);
  680.     metrics.mark('dbrecall','end');
  681.     //var sets = data.getDatasets();
  682.     console.log(sets);
  683.     metrics.mark('dsWorker', 'start');
  684.     dsWorker.postMessage(sets);
  685.   }
  686.  
  687.   function drawCharts(sets) {//{{{
  688.     var 
  689.       hitlineopt = { 
  690.         label: 'HITs Submitted',
  691.         yAxixID: 'hits',
  692.         backgroundColor: 'rgba(149,89,240,0.1)'/*'rgba(42,161,152,0.1)'*/,
  693.         borderColor    : 'rgba(149,89,240,0.5)'/*'rgba(42,161,152,0.5)'*/,
  694.         pointBackgroundColor     : 'rgba(149,89,240,1)'/*'rgba(42,161,152,1)'*/,
  695.         pointHoverBackgroundColor: 'rgba(92,0,230,1)'/*'rgba(38,139,210,1)'*/
  696.       },
  697.       paylineopt = {
  698.         label: 'Total Pay',
  699.         yAxisID: 'pay',
  700.         backgroundColor: 'rgba(200,242,48,0.1)'/*'rgba(181,137,0,0.1)'*/,
  701.         borderColor    : 'rgba(200,242,48,0.5)'/*'rgba(181,137,0,0.5)'*/,
  702.         pointBackgroundColor     : 'rgba(200,242,48,1)'/*'rgba(181,137,0,1)'*/,
  703.         pointHoverBackgroundColor: 'rgba(129,163,5,1)'/*'rgba(133,153,0,1)'*/
  704.       },
  705.       chartopt = {};
  706.       
  707.     for (var k of Object.keys(sets)) {
  708.       if (k === 'all') {
  709.         var a = sets.all.hits;
  710.         chartopt.bsratio = {
  711.           type: 'pie', data: {
  712.             labels: ['batch %', 'survey %'],
  713.             datasets: [{ data: [Math.decRound(100*a.batch.total/a.total,2), Math.decRound(100*(a.total-a.batch.total)/a.total,2)],
  714.               backgroundColor: ['#4773ED', '#EDC147'] }]
  715.           }
  716.         };
  717.         chartopt.pdist = {
  718.           type: 'bar', data: {
  719.             labels: sets.all.distribution.data.labelsp,
  720.             datasets: [{ data: sets.all.distribution.data.pay,
  721.               backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }]
  722.           }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1 }] }}
  723.         };
  724.         chartopt.hdist = {
  725.           type: 'bar', data: {
  726.             labels: sets.all.distribution.data.labelsh,
  727.             datasets: [{ data: sets.all.distribution.data.hits,
  728.               backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }]
  729.           }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1}] }}
  730.         };
  731.         continue;
  732.       }
  733.       Object.assign(sets[k].hits, hitlineopt);
  734.       Object.assign(sets[k].pay, paylineopt);
  735.       chartopt[k] = {
  736.         type:'line', data: { labels: sets[k].hits.labels, datasets: [sets[k].hits, sets[k].pay] }, 
  737.         options: {
  738.           stacked: false, scales: {
  739.             xAxes: [{ gridLines: { offsetGridLines: false }, display: true, scaleSteps: 10,
  740.               labels: {fontSize: /*/^[ym]/.test(k) ? 5 :*/ 0, show: /^[ym]/.test(k) ? true : false} }],
  741.             yAxes: [{ type: 'linear', display: true, position: 'left', id: 'hits' },
  742.                     { type: 'linear', display: true, position: 'right', id: 'pay', gridLines: { drawOnChartArea: false } }]
  743.           }
  744.         }
  745.       };
  746.     }
  747.     var html = ['<head>',
  748.        '<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400" rel="stylesheet" type="text/css">',
  749.        '<script src="https://cdn.jsdelivr.net/chart.js/2.0.0-alpha3/Chart.min.js"></script>',
  750.        '<style>',
  751.         '.caption {width:60%;margin:auto;padding:5px;text-align:center;font-size:0.75em}',
  752.         '.hc {color:#9559F0} .pc {color:#C8F230}',
  753.         '.bc {color:#4773ED} .sc {color:#EDC147}',
  754.         '.container {display:flex;flex-wrap:wrap;align-content:center;justify-content:space-around}',
  755.         '.mi {font-size:0.6em}',
  756.         '.title {background-color:#EEE8D5;color:#073642;width:90%;margin:1% 6% 0% 5%;padding-left:2%;font-weight:400;}',
  757.        '</style>',
  758.        '<title>HIT Database Analytics | '+dbrange[2]+'</title>',
  759.       '</head>',
  760.       '<body style="color:#fff;background-color:#073642;font-size:100%;font-family:\'Open Sans\',Arial;font-weight:300">',
  761.       // '<i style="color:#fff;">Sigma = &#963;</i> <span style="color:#fff;font-family:"\'Times New Roman\', Arial;">&#963;</span>',
  762.        '<div class="title">GENERAL STATISTICS</div>',
  763.        '<div class="container">',
  764.         '<div style="margin-top:5%;margin-left:5%;flex:1">',
  765.          '<div class="container" style="margin-right:10%">',
  766.            '<span style="text-align:right;color:#BBD12A;flex:1">'+digitGroup(sets.all.hits.total)+'</span>',
  767.            '<span style="text-align:center;flex:1">SUBMITTED</span>',
  768.            '<span style="text-align:left;color:#BBD12A;flex:1">$'+digitGroup(dec(sets.all.pay.total,2))+'</span>',
  769.          '</div><div class="container" style="margin-right:10%">',
  770.            '<span style="text-align:right;color:#F23054;flex:1">'+digitGroup(sets.all.hits.rejected)+'</span>',
  771.            '<span style="text-align:center;flex:1">REJECTED</span>',
  772.            '<span style="text-align:left;color:#F23054;flex:1">$'+digitGroup(dec(sets.all.pay.rejected,2))+'</span>',
  773.          '</div><div class="container" style="margin-right:10%">',
  774.            '<span style="text-align:right;color:#E0B438;flex:1">'+digitGroup(sets.all.hits.pending)+'</span>',
  775.            '<span style="text-align:center;flex:1">PENDING</span>',
  776.            '<span style="text-align:left;color:#E0B438;flex:1">$'+digitGroup(dec(sets.all.pay.pending,2))+'</span>',
  777.          '</div>',
  778.          '<span class="mi" style="text-align:justify"><span style="color:#30CEF2">NOTE:</span> DOLLAR AMOUNT FOR \'PENDING\' MAY ',
  779.            'REFLECT VALUES FOR HITS WHICH HAVE ALREADY BEEN APPROVED. THESE FUNDS HAVE NOT BEEN FULLY CLEARED ',
  780.            'AND CREDITED TO YOUR ACCOUNT, AND THUS ARE STILL TECHNICALLY PENDING.</span>',
  781.          '<div style="margin:10% 10% 10% 0;flex:1;text-align:center;">',
  782.           'AVERAGE SUBMISSION OF <span style="color:#C7ED3E">'+dec(sets.all.hitsPerRequester.avg,3)+'</span> ',
  783.           '(<span style="color:#C7ED3E">&#177;'+dec(sets.all.hitsPerRequester.se,3)+'</span>) HITS PER REQUESTER',
  784.           '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.hitsPerRequester.sd,3)+'</span></span>',
  785.           '<hr style="width:45%;color:#fff;background-color:#fff">',
  786.           'AVERAGE PAY OF <span style="color:#C7ED3E">$'+dec(sets.all.payPerHit.avg,3)+'</span> ',
  787.           '(<span style="color:#C7ED3E">&#177;$'+dec(sets.all.payPerHit.se,3)+'</span>) PER HIT',
  788.           '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.payPerHit.sd,3)+'</span></span>',
  789.           '<hr style="width:45%;color:#fff;background-color:#fff">',
  790.           'AVERAGING <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg+sets.all.payPerHit.se))+'</span>',
  791.           '<span id="minbound"> TO <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg-sets.all.payPerHit.se))+
  792.             '</span></span> HITS PER $1.00',
  793.          '</div>',
  794.         '</div>',
  795.         '<div style="margin-top:1%;flex:1">',
  796.          '<div><canvas id="hitdist"></canvas><div class="caption">HIT DISTRIBUTION PER REQUESTER</div></div>',
  797.          '<div><canvas id="paydist"></canvas><div class="caption">PAY DISTRIBUTION PER HIT</div></div>',
  798.         '</div>',
  799.         '<div style="margin-top:10%;flex:1">',
  800.          '<canvas id="batchpie"></canvas>',
  801.          '<div class="caption"><span class="bc">BATCH</span> : <span class="sc">SURVEY</span><br>RATIO APPROXIMATION</div>',
  802.         '</div>',
  803.        '</div>',
  804.        '<div class="title">ACTIVITY</div>',
  805.        '<div class="container">',
  806.         '<div style="margin-top:1%;margin-left:5%;flex:1">',
  807.          '<canvas id="dailyline"></canvas>',
  808.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>DAILY</div>',
  809.         '</div>',
  810.         '<div style="margin-top:1%;margin-right:6%;flex:1">',
  811.          '<canvas id="weeklyline"></canvas>',
  812.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>WEEKLY</div>',
  813.         '</div>',
  814.        '</div>',
  815.        '<div class="container">',
  816.         '<div style="margin-top:1%;margin-left:5%;flex:1">',
  817.          '<canvas id="monthlyline"></canvas>',
  818.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>MONTHLY</div>',
  819.         '</div>',
  820.         '<div style="margin-top:1%;margin-right:6%;flex:1">',
  821.          '<canvas id="yearlyline"></canvas>',
  822.          '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>YEARLY</div>',
  823.         '</div>',
  824.        '</div>',
  825.        '<div class="title">TOP REQUESTERS</div>',
  826.        '<div class="container">',
  827.         '<table id="req10pay" style="font-weight:300;margin-top:1%;margin-left:5%;">',
  828.          '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">PAY</th></tr></thead><tbody></tbody></table>',
  829.         '<table id="req10hit" style="font-weight:300;margin-top:1%;margin-right:6%;">',
  830.          '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">HITS</th></tr></thead><tbody></tbody></table>',
  831.        '</div>',
  832.        '<script>',
  833.         'if (!NodeList.prototype[Symbol.iterator]) NodeList.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator];',
  834.         'var chartopt = '+JSON.stringify(chartopt)+',',
  835.         '    dLines = new Chart(document.getElementById("dailyline").getContext("2d"), chartopt.day),',
  836.         '    mLines = new Chart(document.getElementById("monthlyline").getContext("2d"), chartopt.month),',
  837.         '    wLines = new Chart(document.getElementById("weeklyline").getContext("2d"), chartopt.week),',
  838.         '    yLines = new Chart(document.getElementById("yearlyline").getContext("2d"), chartopt.year),',
  839.         '    bsPie  = new Chart(document.getElementById("batchpie").getContext("2d"), chartopt.bsratio),',
  840.         '    hdist  = new Chart(document.getElementById("hitdist").getContext("2d"), chartopt.hdist),',
  841.         '    pdist  = new Chart(document.getElementById("paydist").getContext("2d"), chartopt.pdist),',
  842.         '    r10hit = document.getElementById("req10hit").tBodies[0], r10hHtml = [],',
  843.         '    r10pay = document.getElementById("req10pay").tBodies[0], r10pHtml = [],',
  844.         '    payArr = '+JSON.stringify(sets.all.rpay)+',',
  845.         '    hitArr = '+JSON.stringify(sets.all.rhits)+',',
  846.         '    mb     = document.getElementById("minbound");',
  847.         'if (mb.children[0].textContent === mb.previousSibling.textContent) mb.style.display = "none";',
  848.         'payArr.forEach(v => r10pHtml',
  849.           '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">$"+v.pay+"</td><td>"+v.name+"</td></tr>"));',
  850.         'hitArr.forEach(v => r10hHtml',
  851.           '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">"+v.hits+"</td><td>"+v.name+"</td></tr>"));',
  852.         'r10hit.innerHTML = r10hHtml.join(""); r10pay.innerHTML = r10pHtml.join("");',
  853.        '</script>',
  854.       '</body>'];
  855.     var blob = new Blob(html, {type:'text/html'}), _a = D.body.appendChild(D.createElement('A'));
  856.     _a.href = URL.createObjectURL(blob);
  857.     _a.target = '_blank';
  858.     _a.click();
  859.     _a.remove();
  860.   }//}}}
  861.  
  862.   function aggregateCursor(dat, wRange) {//{{{
  863.     var ymwd = [dat.date.substr(0,4), dat.date.substr(0,7), null, dat.date],
  864.         pay = isNaN(dat.reward) ? +dat.reward.pay : +dat.reward;
  865.     // determine weekly scales
  866.     if (!wRange[0])
  867.       wRange = getWeekRange(dat.date);
  868.     if (dat.date >= wRange[1])
  869.       wRange = getWeekRange(dat.date);
  870.     ymwd[2] = wRange.join(':');
  871.  
  872.     // populate scaled aggregates
  873.     for (var i=0; i<ymwd.length; i++) {
  874.       // ... basic
  875.       if (!sets[ymwdLabels[i]].aggregate.def[ymwd[i]])
  876.         sets[ymwdLabels[i]].aggregate.def[ymwd[i]] = { hits:0, pay:0 };
  877.       sets[ymwdLabels[i]].aggregate.def[ymwd[i]].hits += 1;
  878.       sets[ymwdLabels[i]].aggregate.def[ymwd[i]].pay  += pay;
  879.       // ... by requester
  880.       if (i > 1) continue; // skip--don't need this for day or week
  881.       if (!sets[ymwdLabels[i]].aggregate.req[ymwd[i]]) {
  882.         sets[ymwdLabels[i]].aggregate.req[ymwd[i]] = [];
  883.         sets[ymwdLabels[i]].rpay[ymwd[i]] = [];
  884.         sets[ymwdLabels[i]].rhits[ymwd[i]] = [];
  885.       }
  886.       var reqArr = sets[ymwdLabels[i]].aggregate.req[ymwd[i]],
  887.           reqLoc = reqArr.findIndex(v => v.req === dat.requesterId);
  888.       if (~reqLoc) {
  889.         reqArr[reqLoc].hits += 1;
  890.         reqArr[reqLoc].pay += pay;
  891.       }
  892.       else
  893.         reqArr.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay });
  894.     }// end for ymwd loop
  895.     // repeat requester aggregation for 'all'
  896.     reqLoc = sets.all.aggregate.findIndex(v => v.req === dat.requesterId);
  897.     if (~reqLoc) {
  898.       sets.all.aggregate[reqLoc].hits += 1;
  899.       sets.all.aggregate[reqLoc].pay += pay;
  900.     }
  901.     else
  902.       sets.all.aggregate.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay });
  903.  
  904.     // populate pay distribution data
  905.     //   hit distribution needs to wait until later--after aggregation
  906.     sets.all.payPerHit.data.push(pay);
  907.     distp.forEach( v => {
  908.       var range = v.split('-');
  909.       if (pay === 0) { sets.all.distribution.pay['0'] += 1; return; }
  910.       if (pay > 5) { sets.all.distribution.pay['5.01+'] += 1; return; }
  911.       if (pay >= range[0] && pay <= range[1]) { sets.all.distribution.pay[v] += 1; return; }
  912.     });
  913.     // increment general stats
  914.     var _t = dat.title.trim();
  915.     sets.all.hits.total += 1;
  916.     sets.all.hits.rejected += dat.status === 'Rejected' ? 1 : 0;
  917.     sets.all.hits.pending += dat.status === 'Pending Approval' ? 1 : 0;
  918.     sets.all.hits.titles[_t] = sets.all.hits.titles[_t] + 1 || 1;
  919.     sets.all.pay.total += pay;
  920.     sets.all.pay.rejected += dat.status === 'Rejected' ? pay : 0;
  921.     sets.all.pay.pending += /pending/i.test(dat.status) ? pay : 0;
  922.     // populate data for batch approximation
  923.     var hitLoc = sets.all.hits.batch.findIndex(v => v.title === _t && v.req === dat.requesterId);
  924.     if (~hitLoc)
  925.       sets.all.hits.batch[hitLoc].count += 1;
  926.     else
  927.       sets.all.hits.batch.push({ title: _t, req: dat.requesterId, count: 1 });
  928.  
  929.     return wRange;
  930.   }//}}}
  931.  
  932.   function DBResult() {
  933.     this.results = [];
  934.     this.add = v => this.results.push(v);
  935.     this.getDatasets = () => {// {{{
  936.       
  937.       return sets;
  938.  
  939.     };//}}} getDatasets
  940.   }// DBResult
  941.  
  942.   function getWeekRange(date) {
  943.     var _d = new Date(date), _ws, _we;
  944.     _d.setUTCDate(_d.getUTCDate()-_d.getUTCDay());
  945.     _ws = _d.toISODateString();
  946.     _d.setUTCDate(_d.getUTCDate()+7);
  947.     _we = _d.toISODateString();
  948.     return [ _ws, _we ];
  949.   }
  950.  
  951.   //set up a worker thread to prevent interface locking on large datasets
  952.   var datasetsWorker = [//{{{
  953.     'var dec = '+dec+', digitGroup = '+digitGroup+';',
  954.     'Math.decRound = '+Math.decRound+';',
  955.     // calculate the standard deviation
  956.     'function stdev(avg, N, data) {',
  957.       'var sum = 0;',
  958.       'data.forEach(v => sum += Math.pow(v-avg, 2));',
  959.       'return Math.sqrt(sum/(N-1));',
  960.     '}',
  961.  
  962.     'onmessage = (e) => {',
  963.       'var sets = e.data, disth = '+JSON.stringify(disth)+', distp = '+JSON.stringify(distp)+';',
  964.       // sort aggregates
  965.       'for (var period of Object.keys(sets)) {',
  966.         'if (period === \'all\') { ',
  967.           // calculate averages, sd, se
  968.           'postMessage( {type: "status", data: "Calculating averages..." });',
  969.           'var _pph = sets.all.payPerHit, _hpr = sets.all.hitsPerRequester;',
  970.           '_pph.avg = sets.all.pay.total / sets.all.hits.total;',
  971.           '_pph.sd = stdev(_pph.avg, _pph.data.length, _pph.data);',
  972.           '_pph.se = _pph.sd/Math.sqrt(_pph.data.length);',
  973.           '_hpr.data = sets.all.aggregate.map( v => v.hits );',
  974.           '_hpr.avg = sets.all.hits.total / _hpr.data.length;',
  975.           '_hpr.sd = stdev(_hpr.avg, _hpr.data.length, _hpr.data);',
  976.           '_hpr.se = _hpr.sd/Math.sqrt(_hpr.data.length);',
  977.           // get hit distribution
  978.           'postMessage( {type: "status", data: "Populating distributions..." });',
  979.           'sets.all.aggregate.forEach(v => {',
  980.             'disth.forEach(w => {',
  981.               'var range = w.split(\'-\');',
  982.               'if (range.length === 1) range.push(\'1\');',
  983.               'if (v.hits > 1000) { sets.all.distribution.hits[\'1001+\'] += 1; return; }',
  984.               'if (v.hits >= range[0] && v.hits <= range[1]) { sets.all.distribution.hits[w] += 1; return; }',
  985.             '});',
  986.           '});',
  987.           // extract raw data for distributions
  988.           'disth.forEach(v => sets.all.distribution.data.hits.push(sets.all.distribution.hits[v]));',
  989.           'distp.forEach(v => sets.all.distribution.data.pay.push(sets.all.distribution.pay[v]));',
  990.           'sets.all.distribution.data.labelsh = Object.keys(sets.all.distribution.hits);',
  991.           'sets.all.distribution.data.labelsp = Object.keys(sets.all.distribution.pay);',
  992.           // get 'top' lists
  993.           'sets.all.rpay  = sets.all.aggregate.sort((a,b) => b.pay  - a.pay ).slice(0,10);',
  994.           'sets.all.rhits = sets.all.aggregate.sort((a,b) => b.hits - a.hits).slice(0,10);',
  995.           'sets.all.rpay.forEach(v => v.pay = dec(v.pay,2));',
  996.           'sets.all.rhits.forEach(v => v.hits = digitGroup(v.hits));',
  997.           'continue;',
  998.         '}',
  999.         // populate labels/data for graphing
  1000.         'postMessage({ type: "status", data: "Sorting by "+period+"..." });',
  1001.         'var labels = Object.keys(sets[period].aggregate.def);',
  1002.         'for (k of labels) {',
  1003.           'if (~[\'hits\',\'pay\'].indexOf(k)) continue;',
  1004.           'sets[period].pay.labels.push(k);',
  1005.           'sets[period].hits.labels.push(k);',
  1006.           'sets[period].pay.data.push(Math.decRound(sets[period].aggregate.def[k].pay,2));',
  1007.           'sets[period].hits.data.push(sets[period].aggregate.def[k].hits);',
  1008.  
  1009.           // sort req lists/period
  1010.           'if (~[\'week\',\'day\'].indexOf(period)) continue;',
  1011.           'var len = period === \'year\' ? 10 : 5;',
  1012.           'reqArr = sets[period].aggregate.req[k];',
  1013.           'sets[period].rpay[k] = reqArr.sort((a,b) => b.pay - a.pay).slice(0,len);',
  1014.           'sets[period].rhits[k] = reqArr.map(v => v).sort((a,b) => b.hits - a.hits).slice(0,len);',
  1015.         '}',
  1016.       '}',
  1017.       // if there are more than 5 hits with the same title under the same requester, assume it's a batch
  1018.       'sets.all.hits.batch = sets.all.hits.batch.filter(v => v.count > 5);',
  1019.       // workers fail to transfer nonenumerable properties :( 
  1020.       //'Object.defineProperty(sets.all.hits.batch, \'total\', { configurable: true, writable: true, value: 0 });',
  1021.       'sets.all.hits.batch.total = 0;',
  1022.       'sets.all.hits.batch.forEach(v => {if (typeof v === "object") sets.all.hits.batch.total += v.count});',
  1023.  
  1024.       'postMessage({ type: "obj", data: sets });',
  1025.  
  1026.     '}'];//}}}
  1027.   datasetsWorker = new Blob(datasetsWorker, {type:'text/javascript'});
  1028.   var workerURL = URL.createObjectURL(datasetsWorker);
  1029.   var dsWorker = new Worker(workerURL);
  1030.   dsWorker.onmessage = (e) => {
  1031.     if (e.data.type === 'status') {
  1032.       window.Status.message = e.data.data;
  1033.       console.log(e.data.data);
  1034.     } else {
  1035.       metrics.mark('dsWorker','end');
  1036.       metrics.stop();metrics.report();
  1037.       window.Status.message = 'Done!';
  1038.       window.Progress.hide();
  1039.       drawCharts(e.data.data);
  1040.     }
  1041.   };
  1042.  
  1043. })(document);
  1044.  
  1045. // vim: ts=2:sw=2:et:fdm=marker:noai