您需要先安装一个扩展,例如 篡改猴、Greasemonkey 或 暴力猴,之后才能安装此脚本。
您需要先安装一个扩展,例如 篡改猴 或 暴力猴,之后才能安装此脚本。
您需要先安装一个扩展,例如 篡改猴 或 暴力猴,之后才能安装此脚本。
您需要先安装一个扩展,例如 篡改猴 或 Userscripts ,之后才能安装此脚本。
您需要先安装一款用户脚本管理器扩展,例如 Tampermonkey,才能安装此脚本。
您需要先安装用户脚本管理器扩展后才能安装此脚本。
Analytics for HIT Database--makes pretty graphs
当前为
// ==UserScript== // @name HIT Database Analytics // @author feihtality // @namespace https://greasyfork.org/en/users/12709 // @version 0.5.002 // @description Analytics for HIT Database--makes pretty graphs // @match https://www.mturk.com/mturk/dashboard // @grant none // ==/UserScript== ((D) => { 'use strict'; if (!('decRound' in Math) || !('Status' in window) || !('Progress' in window)) { console.log('execution order is either too high or HITDB MKII is not detected'); return; } const DB_NAME = "HITDB_TESTING"; Date.prototype.toISODateString = function() { return this.getUTCFullYear()+"-"+pad(this.getUTCMonth()+1)+"-"+pad(this.getUTCDate()); }; var pad = n => ("00"+n).substr(-2), digitGroup = function(n) { n = String(n).split('.'); if (n[0].length < 4) return n.join('.'); n[0] = n[0].replace(/(\d)(?=(\d{3})+$)/g, '$1,'); return n.join('.'); }, dec = (n,l) => Number(Math.decRound(n,l)).toFixed(l); // inject interface var insertion = D.getElementById('hdbCSVInput'), searchbtn = D.getElementById('hdbSearch'), acheckbox = insertion.parentNode.insertBefore(D.createElement('INPUT'), insertion.nextSibling), alabel = insertion.parentNode.insertBefore(D.createElement('LABEL'), insertion.nextSibling), metrics = null; acheckbox.type = 'checkbox'; acheckbox.id = 'hdbAnalytics'; acheckbox.style.verticalAlign = 'middle'; alabel.textContent = 'Analyze'; alabel.htmlFor = 'hdbAnalytics'; alabel.style.verticalAlign = 'middle'; acheckbox.onclick = function() { if (searchbtn.textContent === "Export CSV") insertion.click(); if (searchbtn.textContent === "Analyze") searchbtn.textContent = 'Search'; else searchbtn.textContent = 'Analyze'; }; searchbtn.addEventListener('click', getData); function getData(e) { if (e.target.textContent !== "Analyze") return; var range; dbrange = [D.getElementById('hdbMinDate').value || undefined, D.getElementById('hdbMaxDate').value || undefined, '']; if (!dbrange[0] && !dbrange[1]) { dbrange[2] = 'ALL'; range = null; } else if (dbrange[0] && !dbrange[1]) { dbrange[2] = dbrange[0]+'>>'; range = window.IDBKeyRange.lowerBound(dbrange[0]); } else if (!dbrange[0] && dbrange[1]) { dbrange[2] = '<<'+dbrange[1]; range = window.IDBKeyRange.upperBound(dbrange[1]); } else { dbrange[2] = dbrange[0]+':'+dbrange[1]; range = window.IDBKeyRange.bound(dbrange[0],dbrange[1]); } sets = new Sets(); window.Progress.show(); metrics = new window.Metrics('database_analytics'); metrics.mark('dbrecall', 'start'); dbrecall("HIT", {range: range}).then(analyzeData);//.then(drawCharts); } /* . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . */ var Sets = function() {//{{{ main data obj this.year = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.month = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.week = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.day = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.all = { aggregate: [], distribution: { hits: {'1':0,'2-5':0,'6-10':0,'11-15':0,'16-20':0,'21-25':0,'26-30':0,'31-35':0, '36-40':0,'41-45':0,'46-50':0,'51-100':0,'101-150':0,'151-200':0,'201-250':0,'251-300':0, '301-350':0,'351-400':0,'401-450':0,'451-500':0,'501-550':0,'551-600':0,'601-650':0,'651-700':0, '701-750':0,'751-800':0,'801-850':0,'851-900':0,'901-950':0,'951-1000':0,'1000+':0}, pay: {'0':0, '0.01-0.05':0,'0.06-0.10':0,'0.11-0.15':0,'0.16-0.20':0,'0.21-0.25':0, '0.26-0.30':0,'0.31-0.35':0,'0.36-0.40':0,'0.41-0.45':0,'0.46-0.50':0,'0.51-0.55':0, '0.56-0.60':0,'0.61-0.65':0,'0.66-0.70':0,'0.71-0.75':0,'0.76-0.80':0,'0.81-0.85':0, '0.86-0.90':0,'0.91-0.95':0,'0.96-1.00':0,'1.01-1.25':0,'1.26-1.50':0,'1.51-1.75':0, '1.76-2.00':0,'2.01-2.25':0,'2.26-2.50':0,'2.51-2.75':0,'2.76-3.00':0,'3.01-3.25':0, '3.26-3.50':0,'3.51-3.75':0,'3.76-4.00':0,'4.01-4.25':0,'4.26-4.50':0,'4.51-4.75':0, '4.76-5.00':0,'5.01+':0}, data: { hits: [], labelsh: [], pay: [], labelsp: [] } }, hits: { total: 0, rejected: 0, pending: 0, titles: {}, batch: [] }, pay : { total: 0, rejected: 0, pending: 0 }, hitsPerRequester: { avg: 0, data: [], sd: 0, se: 0 }, payPerHit: { avg: 0, data: [], sd: 0, se: 0 } }; },//}}} sets = new Sets(), disth = Object.keys(sets.all.distribution.hits), distp = Object.keys(sets.all.distribution.pay), ymwdLabels = Object.keys(sets), dbrange; function dbrecall(os, options) { options = options || {}; var index = options.index || "date", range = options.range || null, wRange = [null, null]; var total = 0; return new Promise( y => { window.indexedDB.open(DB_NAME).onsuccess = function() { this.result.transaction(os, "readonly").objectStore(os).index(index).openCursor(range).onsuccess = function() { if (this.result) { window.Status.message = 'Aggregating data... [ '+(total++)+' ]'; wRange = aggregateCursor(this.result.value, wRange); this.result.continue(); } else y(1); }; this.result.close(); }; }); } function analyzeData(r) { void(r); metrics.mark('dbrecall','end'); //var sets = data.getDatasets(); console.log(sets); metrics.mark('dsWorker', 'start'); dsWorker.postMessage(sets); } function drawCharts(sets) {//{{{ var hitlineopt = { label: 'HITs Submitted', yAxixID: 'hits', backgroundColor: 'rgba(149,89,240,0.1)'/*'rgba(42,161,152,0.1)'*/, borderColor : 'rgba(149,89,240,0.5)'/*'rgba(42,161,152,0.5)'*/, pointBackgroundColor : 'rgba(149,89,240,1)'/*'rgba(42,161,152,1)'*/, pointHoverBackgroundColor: 'rgba(92,0,230,1)'/*'rgba(38,139,210,1)'*/ }, paylineopt = { label: 'Total Pay', yAxisID: 'pay', backgroundColor: 'rgba(200,242,48,0.1)'/*'rgba(181,137,0,0.1)'*/, borderColor : 'rgba(200,242,48,0.5)'/*'rgba(181,137,0,0.5)'*/, pointBackgroundColor : 'rgba(200,242,48,1)'/*'rgba(181,137,0,1)'*/, pointHoverBackgroundColor: 'rgba(129,163,5,1)'/*'rgba(133,153,0,1)'*/ }, chartopt = {}; for (var k of Object.keys(sets)) { if (k === 'all') { var a = sets.all.hits; chartopt.bsratio = { type: 'pie', data: { labels: ['batch %', 'survey %'], datasets: [{ data: [Math.decRound(100*a.batch.total/a.total,2), Math.decRound(100*(a.total-a.batch.total)/a.total,2)], backgroundColor: ['#4773ED', '#EDC147'] }] } }; chartopt.pdist = { type: 'bar', data: { labels: sets.all.distribution.data.labelsp, datasets: [{ data: sets.all.distribution.data.pay, backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }] }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1 }] }} }; chartopt.hdist = { type: 'bar', data: { labels: sets.all.distribution.data.labelsh, datasets: [{ data: sets.all.distribution.data.hits, backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }] }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1}] }} }; continue; } Object.assign(sets[k].hits, hitlineopt); Object.assign(sets[k].pay, paylineopt); chartopt[k] = { type:'line', data: { labels: sets[k].hits.labels, datasets: [sets[k].hits, sets[k].pay] }, options: { stacked: false, scales: { xAxes: [{ gridLines: { offsetGridLines: false }, display: true, scaleSteps: 10, labels: {fontSize: /*/^[ym]/.test(k) ? 5 :*/ 0, show: /^[ym]/.test(k) ? true : false} }], yAxes: [{ type: 'linear', display: true, position: 'left', id: 'hits' }, { type: 'linear', display: true, position: 'right', id: 'pay', gridLines: { drawOnChartArea: false } }] } } }; } var html = ['<head>', '<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400" rel="stylesheet" type="text/css">', '<script src="https://cdn.jsdelivr.net/chart.js/2.0.0-alpha3/Chart.min.js"></script>', '<style>', '.caption {width:60%;margin:auto;padding:5px;text-align:center;font-size:0.75em}', '.hc {color:#9559F0} .pc {color:#C8F230}', '.bc {color:#4773ED} .sc {color:#EDC147}', '.container {display:flex;flex-wrap:wrap;align-content:center;justify-content:space-around}', '.mi {font-size:0.6em}', '.title {background-color:#EEE8D5;color:#073642;width:90%;margin:1% 6% 0% 5%;padding-left:2%;font-weight:400;}', '</style>', '<title>HIT Database Analytics | '+dbrange[2]+'</title>', '</head>', '<body style="color:#fff;background-color:#073642;font-size:100%;font-family:\'Open Sans\',Arial;font-weight:300">', // '<i style="color:#fff;">Sigma = σ</i> <span style="color:#fff;font-family:"\'Times New Roman\', Arial;">σ</span>', '<div class="title">GENERAL STATISTICS</div>', '<div class="container">', '<div style="margin-top:5%;margin-left:5%;flex:1">', '<div class="container" style="margin-right:10%">', '<span style="text-align:right;color:#BBD12A;flex:1">'+digitGroup(sets.all.hits.total)+'</span>', '<span style="text-align:center;flex:1">SUBMITTED</span>', '<span style="text-align:left;color:#BBD12A;flex:1">$'+digitGroup(dec(sets.all.pay.total,2))+'</span>', '</div><div class="container" style="margin-right:10%">', '<span style="text-align:right;color:#F23054;flex:1">'+digitGroup(sets.all.hits.rejected)+'</span>', '<span style="text-align:center;flex:1">REJECTED</span>', '<span style="text-align:left;color:#F23054;flex:1">$'+digitGroup(dec(sets.all.pay.rejected,2))+'</span>', '</div><div class="container" style="margin-right:10%">', '<span style="text-align:right;color:#E0B438;flex:1">'+digitGroup(sets.all.hits.pending)+'</span>', '<span style="text-align:center;flex:1">PENDING</span>', '<span style="text-align:left;color:#E0B438;flex:1">$'+digitGroup(dec(sets.all.pay.pending,2))+'</span>', '</div>', '<span class="mi" style="text-align:justify"><span style="color:#30CEF2">NOTE:</span> DOLLAR AMOUNT FOR \'PENDING\' MAY ', 'REFLECT VALUES FOR HITS WHICH HAVE ALREADY BEEN APPROVED. THESE FUNDS HAVE NOT BEEN FULLY CLEARED ', 'AND CREDITED TO YOUR ACCOUNT, AND THUS ARE STILL TECHNICALLY PENDING.</span>', '<div style="margin:10% 10% 10% 0;flex:1;text-align:center;">', 'AVERAGE SUBMISSION OF <span style="color:#C7ED3E">'+dec(sets.all.hitsPerRequester.avg,3)+'</span> ', '(<span style="color:#C7ED3E">±'+dec(sets.all.hitsPerRequester.se,3)+'</span>) HITS PER REQUESTER', '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.hitsPerRequester.sd,3)+'</span></span>', '<hr style="width:45%;color:#fff;background-color:#fff">', 'AVERAGE PAY OF <span style="color:#C7ED3E">$'+dec(sets.all.payPerHit.avg,3)+'</span> ', '(<span style="color:#C7ED3E">±$'+dec(sets.all.payPerHit.se,3)+'</span>) PER HIT', '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.payPerHit.sd,3)+'</span></span>', '<hr style="width:45%;color:#fff;background-color:#fff">', 'AVERAGING <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg+sets.all.payPerHit.se))+'</span>', '<span id="minbound"> TO <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg-sets.all.payPerHit.se))+ '</span></span> HITS PER $1.00', '</div>', '</div>', '<div style="margin-top:1%;flex:1">', '<div><canvas id="hitdist"></canvas><div class="caption">HIT DISTRIBUTION PER REQUESTER</div></div>', '<div><canvas id="paydist"></canvas><div class="caption">PAY DISTRIBUTION PER HIT</div></div>', '</div>', '<div style="margin-top:10%;flex:1">', '<canvas id="batchpie"></canvas>', '<div class="caption"><span class="bc">BATCH</span> : <span class="sc">SURVEY</span><br>RATIO APPROXIMATION</div>', '</div>', '</div>', '<div class="title">ACTIVITY</div>', '<div class="container">', '<div style="margin-top:1%;margin-left:5%;flex:1">', '<canvas id="dailyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>DAILY</div>', '</div>', '<div style="margin-top:1%;margin-right:6%;flex:1">', '<canvas id="weeklyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>WEEKLY</div>', '</div>', '</div>', '<div class="container">', '<div style="margin-top:1%;margin-left:5%;flex:1">', '<canvas id="monthlyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>MONTHLY</div>', '</div>', '<div style="margin-top:1%;margin-right:6%;flex:1">', '<canvas id="yearlyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>YEARLY</div>', '</div>', '</div>', '<div class="title">TOP REQUESTERS</div>', '<div class="container">', '<table id="req10pay" style="font-weight:300;margin-top:1%;margin-left:5%;">', '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">PAY</th></tr></thead><tbody></tbody></table>', '<table id="req10hit" style="font-weight:300;margin-top:1%;margin-right:6%;">', '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">HITS</th></tr></thead><tbody></tbody></table>', '</div>', '<script>', 'if (!NodeList.prototype[Symbol.iterator]) NodeList.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator];', 'var chartopt = '+JSON.stringify(chartopt)+',', ' dLines = new Chart(document.getElementById("dailyline").getContext("2d"), chartopt.day),', ' mLines = new Chart(document.getElementById("monthlyline").getContext("2d"), chartopt.month),', ' wLines = new Chart(document.getElementById("weeklyline").getContext("2d"), chartopt.week),', ' yLines = new Chart(document.getElementById("yearlyline").getContext("2d"), chartopt.year),', ' bsPie = new Chart(document.getElementById("batchpie").getContext("2d"), chartopt.bsratio),', ' hdist = new Chart(document.getElementById("hitdist").getContext("2d"), chartopt.hdist),', ' pdist = new Chart(document.getElementById("paydist").getContext("2d"), chartopt.pdist),', ' r10hit = document.getElementById("req10hit").tBodies[0], r10hHtml = [],', ' r10pay = document.getElementById("req10pay").tBodies[0], r10pHtml = [],', ' payArr = '+JSON.stringify(sets.all.rpay)+',', ' hitArr = '+JSON.stringify(sets.all.rhits)+',', ' mb = document.getElementById("minbound");', 'if (mb.children[0].textContent === mb.previousSibling.textContent) mb.style.display = "none";', 'payArr.forEach(v => r10pHtml', '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">$"+v.pay+"</td><td>"+v.name+"</td></tr>"));', 'hitArr.forEach(v => r10hHtml', '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">"+v.hits+"</td><td>"+v.name+"</td></tr>"));', 'r10hit.innerHTML = r10hHtml.join(""); r10pay.innerHTML = r10pHtml.join("");', '</script>', '</body>']; var blob = new Blob(html, {type:'text/html'}), _a = D.body.appendChild(D.createElement('A')); _a.href = URL.createObjectURL(blob); _a.target = '_blank'; _a.click(); _a.remove(); }//}}} function aggregateCursor(dat, wRange) {//{{{ var ymwd = [dat.date.substr(0,4), dat.date.substr(0,7), null, dat.date], pay = isNaN(dat.reward) ? +dat.reward.pay : +dat.reward; // determine weekly scales if (!wRange[0]) wRange = getWeekRange(dat.date); if (dat.date >= wRange[1]) wRange = getWeekRange(dat.date); ymwd[2] = wRange.join(':'); // populate scaled aggregates for (var i=0; i<ymwd.length; i++) { // ... basic if (!sets[ymwdLabels[i]].aggregate.def[ymwd[i]]) sets[ymwdLabels[i]].aggregate.def[ymwd[i]] = { hits:0, pay:0 }; sets[ymwdLabels[i]].aggregate.def[ymwd[i]].hits += 1; sets[ymwdLabels[i]].aggregate.def[ymwd[i]].pay += pay; // ... by requester if (i > 1) continue; // skip--don't need this for day or week if (!sets[ymwdLabels[i]].aggregate.req[ymwd[i]]) { sets[ymwdLabels[i]].aggregate.req[ymwd[i]] = []; sets[ymwdLabels[i]].rpay[ymwd[i]] = []; sets[ymwdLabels[i]].rhits[ymwd[i]] = []; } var reqArr = sets[ymwdLabels[i]].aggregate.req[ymwd[i]], reqLoc = reqArr.findIndex(v => v.req === dat.requesterId); if (~reqLoc) { reqArr[reqLoc].hits += 1; reqArr[reqLoc].pay += pay; } else reqArr.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay }); }// end for ymwd loop // repeat requester aggregation for 'all' reqLoc = sets.all.aggregate.findIndex(v => v.req === dat.requesterId); if (~reqLoc) { sets.all.aggregate[reqLoc].hits += 1; sets.all.aggregate[reqLoc].pay += pay; } else sets.all.aggregate.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay }); // populate pay distribution data // hit distribution needs to wait until later--after aggregation sets.all.payPerHit.data.push(pay); distp.forEach( v => { var range = v.split('-'); if (pay === 0) { sets.all.distribution.pay['0'] += 1; return; } if (pay > 5) { sets.all.distribution.pay['5.01+'] += 1; return; } if (pay >= range[0] && pay <= range[1]) { sets.all.distribution.pay[v] += 1; return; } }); // increment general stats var _t = dat.title.trim(); sets.all.hits.total += 1; sets.all.hits.rejected += dat.status === 'Rejected' ? 1 : 0; sets.all.hits.pending += dat.status === 'Pending Approval' ? 1 : 0; sets.all.hits.titles[_t] = sets.all.hits.titles[_t] + 1 || 1; sets.all.pay.total += pay; sets.all.pay.rejected += dat.status === 'Rejected' ? pay : 0; sets.all.pay.pending += /pending/i.test(dat.status) ? pay : 0; // populate data for batch approximation var hitLoc = sets.all.hits.batch.findIndex(v => v.title === _t && v.req === dat.requesterId); if (~hitLoc) sets.all.hits.batch[hitLoc].count += 1; else sets.all.hits.batch.push({ title: _t, req: dat.requesterId, count: 1 }); return wRange; }//}}} function DBResult() { this.results = []; this.add = v => this.results.push(v); this.getDatasets = () => {// {{{ return sets; };//}}} getDatasets }// DBResult function getWeekRange(date) { var _d = new Date(date), _ws, _we; _d.setUTCDate(_d.getUTCDate()-_d.getUTCDay()); _ws = _d.toISODateString(); _d.setUTCDate(_d.getUTCDate()+7); _we = _d.toISODateString(); return [ _ws, _we ]; } //set up a worker thread to prevent interface locking on large datasets var datasetsWorker = [//{{{ 'var dec = '+dec+', digitGroup = '+digitGroup+';', 'Math.decRound = '+Math.decRound+';', // calculate the standard deviation 'function stdev(avg, N, data) {', 'var sum = 0;', 'data.forEach(v => sum += Math.pow(v-avg, 2));', 'return Math.sqrt(sum/(N-1));', '}', 'onmessage = (e) => {', 'var sets = e.data, disth = '+JSON.stringify(disth)+', distp = '+JSON.stringify(distp)+';', // sort aggregates 'for (var period of Object.keys(sets)) {', 'if (period === \'all\') { ', // calculate averages, sd, se 'postMessage( {type: "status", data: "Calculating averages..." });', 'var _pph = sets.all.payPerHit, _hpr = sets.all.hitsPerRequester;', '_pph.avg = sets.all.pay.total / sets.all.hits.total;', '_pph.sd = stdev(_pph.avg, _pph.data.length, _pph.data);', '_pph.se = _pph.sd/Math.sqrt(_pph.data.length);', '_hpr.data = sets.all.aggregate.map( v => v.hits );', '_hpr.avg = sets.all.hits.total / _hpr.data.length;', '_hpr.sd = stdev(_hpr.avg, _hpr.data.length, _hpr.data);', '_hpr.se = _hpr.sd/Math.sqrt(_hpr.data.length);', // get hit distribution 'postMessage( {type: "status", data: "Populating distributions..." });', 'sets.all.aggregate.forEach(v => {', 'disth.forEach(w => {', 'var range = w.split(\'-\');', 'if (range.length === 1) range.push(\'1\');', 'if (v.hits > 1000) { sets.all.distribution.hits[\'1001+\'] += 1; return; }', 'if (v.hits >= range[0] && v.hits <= range[1]) { sets.all.distribution.hits[w] += 1; return; }', '});', '});', // extract raw data for distributions 'disth.forEach(v => sets.all.distribution.data.hits.push(sets.all.distribution.hits[v]));', 'distp.forEach(v => sets.all.distribution.data.pay.push(sets.all.distribution.pay[v]));', 'sets.all.distribution.data.labelsh = Object.keys(sets.all.distribution.hits);', 'sets.all.distribution.data.labelsp = Object.keys(sets.all.distribution.pay);', // get 'top' lists 'sets.all.rpay = sets.all.aggregate.sort((a,b) => b.pay - a.pay ).slice(0,10);', 'sets.all.rhits = sets.all.aggregate.sort((a,b) => b.hits - a.hits).slice(0,10);', 'sets.all.rpay.forEach(v => v.pay = dec(v.pay,2));', 'sets.all.rhits.forEach(v => v.hits = digitGroup(v.hits));', 'continue;', '}', // populate labels/data for graphing 'postMessage({ type: "status", data: "Sorting by "+period+"..." });', 'var labels = Object.keys(sets[period].aggregate.def);', 'for (k of labels) {', 'if (~[\'hits\',\'pay\'].indexOf(k)) continue;', 'sets[period].pay.labels.push(k);', 'sets[period].hits.labels.push(k);', 'sets[period].pay.data.push(Math.decRound(sets[period].aggregate.def[k].pay,2));', 'sets[period].hits.data.push(sets[period].aggregate.def[k].hits);', // sort req lists/period 'if (~[\'week\',\'day\'].indexOf(period)) continue;', 'var len = period === \'year\' ? 10 : 5;', 'reqArr = sets[period].aggregate.req[k];', 'sets[period].rpay[k] = reqArr.sort((a,b) => b.pay - a.pay).slice(0,len);', 'sets[period].rhits[k] = reqArr.map(v => v).sort((a,b) => b.hits - a.hits).slice(0,len);', '}', '}', // if there are more than 5 hits with the same title under the same requester, assume it's a batch 'sets.all.hits.batch = sets.all.hits.batch.filter(v => v.count > 5);', // workers fail to transfer nonenumerable properties :( //'Object.defineProperty(sets.all.hits.batch, \'total\', { configurable: true, writable: true, value: 0 });', 'sets.all.hits.batch.total = 0;', 'sets.all.hits.batch.forEach(v => {if (typeof v === "object") sets.all.hits.batch.total += v.count});', 'postMessage({ type: "obj", data: sets });', '}'];//}}} datasetsWorker = new Blob(datasetsWorker, {type:'text/javascript'}); var workerURL = URL.createObjectURL(datasetsWorker); var dsWorker = new Worker(workerURL); dsWorker.onmessage = (e) => { if (e.data.type === 'status') { window.Status.message = e.data.data; console.log(e.data.data); } else { metrics.mark('dsWorker','end'); metrics.stop();metrics.report(); window.Status.message = 'Done!'; window.Progress.hide(); drawCharts(e.data.data); } }; })(document); // vim: ts=2:sw=2:et:fdm=marker:noai ((D) => { 'use strict'; if (!('decRound' in Math) || !('Status' in window) || !('Progress' in window)) { console.log('execution order is either too high or HITDB MKII is not detected'); return; } const DB_NAME = "DATA_TESTING"; Date.prototype.toISODateString = function() { return this.getUTCFullYear()+"-"+pad(this.getUTCMonth()+1)+"-"+pad(this.getUTCDate()); }; var pad = n => ("00"+n).substr(-2), digitGroup = function(n) { n = String(n).split('.'); if (n[0].length < 4) return n.join('.'); n[0] = n[0].replace(/(\d)(?=(\d{3})+$)/g, '$1,'); return n.join('.'); }, dec = (n,l) => Number(Math.decRound(n,l)).toFixed(l); // inject interface var insertion = D.getElementById('hdbCSVInput'), searchbtn = D.getElementById('hdbSearch'), acheckbox = insertion.parentNode.insertBefore(D.createElement('INPUT'), insertion.nextSibling), alabel = insertion.parentNode.insertBefore(D.createElement('LABEL'), insertion.nextSibling), metrics = null; acheckbox.type = 'checkbox'; acheckbox.id = 'hdbAnalytics'; acheckbox.style.verticalAlign = 'middle'; alabel.textContent = 'Analyze'; alabel.htmlFor = 'hdbAnalytics'; alabel.style.verticalAlign = 'middle'; acheckbox.onclick = function() { if (searchbtn.textContent === "Export CSV") insertion.click(); if (searchbtn.textContent === "Analyze") searchbtn.textContent = 'Search'; else searchbtn.textContent = 'Analyze'; }; searchbtn.addEventListener('click', getData); function getData(e) { if (e.target.textContent !== "Analyze") return; var range; dbrange = [D.getElementById('hdbMinDate').value || undefined, D.getElementById('hdbMaxDate').value || undefined, '']; if (!dbrange[0] && !dbrange[1]) { dbrange[2] = 'ALL'; range = null; } else if (dbrange[0] && !dbrange[1]) { dbrange[2] = dbrange[0]+'>>'; range = window.IDBKeyRange.lowerBound(dbrange[0]); } else if (!dbrange[0] && dbrange[1]) { dbrange[2] = '<<'+dbrange[1]; range = window.IDBKeyRange.upperBound(dbrange[1]); } else { dbrange[2] = dbrange[0]+':'+dbrange[1]; range = window.IDBKeyRange.bound(dbrange[0],dbrange[1]); } sets = new Sets(); window.Progress.show(); metrics = new window.Metrics('database_analytics'); metrics.mark('dbrecall', 'start'); dbrecall("HIT", {range: range}).then(analyzeData);//.then(drawCharts); } /* . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . */ var Sets = function() {//{{{ main data obj this.year = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.month = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.week = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.day = { aggregate: { def: {}, req: {} }, pay: { labels: [], data: [] }, hits: { labels: [], data: [] }, rpay: {}, rhits: {} }; this.all = { aggregate: [], distribution: { hits: {'1':0,'2-5':0,'6-10':0,'11-15':0,'16-20':0,'21-25':0,'26-30':0,'31-35':0, '36-40':0,'41-45':0,'46-50':0,'51-100':0,'101-150':0,'151-200':0,'201-250':0,'251-300':0, '301-350':0,'351-400':0,'401-450':0,'451-500':0,'501-550':0,'551-600':0,'601-650':0,'651-700':0, '701-750':0,'751-800':0,'801-850':0,'851-900':0,'901-950':0,'951-1000':0,'1000+':0}, pay: {'0':0, '0.01-0.05':0,'0.06-0.10':0,'0.11-0.15':0,'0.16-0.20':0,'0.21-0.25':0, '0.26-0.30':0,'0.31-0.35':0,'0.36-0.40':0,'0.41-0.45':0,'0.46-0.50':0,'0.51-0.55':0, '0.56-0.60':0,'0.61-0.65':0,'0.66-0.70':0,'0.71-0.75':0,'0.76-0.80':0,'0.81-0.85':0, '0.86-0.90':0,'0.91-0.95':0,'0.96-1.00':0,'1.01-1.25':0,'1.26-1.50':0,'1.51-1.75':0, '1.76-2.00':0,'2.01-2.25':0,'2.26-2.50':0,'2.51-2.75':0,'2.76-3.00':0,'3.01-3.25':0, '3.26-3.50':0,'3.51-3.75':0,'3.76-4.00':0,'4.01-4.25':0,'4.26-4.50':0,'4.51-4.75':0, '4.76-5.00':0,'5.01+':0}, data: { hits: [], labelsh: [], pay: [], labelsp: [] } }, hits: { total: 0, rejected: 0, pending: 0, titles: {}, batch: [] }, pay : { total: 0, rejected: 0, pending: 0 }, hitsPerRequester: { avg: 0, data: [], sd: 0, se: 0 }, payPerHit: { avg: 0, data: [], sd: 0, se: 0 } }; },//}}} sets = new Sets(), disth = Object.keys(sets.all.distribution.hits), distp = Object.keys(sets.all.distribution.pay), ymwdLabels = Object.keys(sets), dbrange; function dbrecall(os, options) { options = options || {}; var index = options.index || "date", range = options.range || null, wRange = [null, null]; var total = 0; return new Promise( y => { window.indexedDB.open(DB_NAME).onsuccess = function() { this.result.transaction(os, "readonly").objectStore(os).index(index).openCursor(range).onsuccess = function() { if (this.result) { window.Status.message = 'Aggregating data... [ '+(total++)+' ]'; wRange = aggregateCursor(this.result.value, wRange); this.result.continue(); } else y(1); }; this.result.close(); }; }); } function analyzeData(r) { void(r); metrics.mark('dbrecall','end'); //var sets = data.getDatasets(); console.log(sets); metrics.mark('dsWorker', 'start'); dsWorker.postMessage(sets); } function drawCharts(sets) {//{{{ var hitlineopt = { label: 'HITs Submitted', yAxixID: 'hits', backgroundColor: 'rgba(149,89,240,0.1)'/*'rgba(42,161,152,0.1)'*/, borderColor : 'rgba(149,89,240,0.5)'/*'rgba(42,161,152,0.5)'*/, pointBackgroundColor : 'rgba(149,89,240,1)'/*'rgba(42,161,152,1)'*/, pointHoverBackgroundColor: 'rgba(92,0,230,1)'/*'rgba(38,139,210,1)'*/ }, paylineopt = { label: 'Total Pay', yAxisID: 'pay', backgroundColor: 'rgba(200,242,48,0.1)'/*'rgba(181,137,0,0.1)'*/, borderColor : 'rgba(200,242,48,0.5)'/*'rgba(181,137,0,0.5)'*/, pointBackgroundColor : 'rgba(200,242,48,1)'/*'rgba(181,137,0,1)'*/, pointHoverBackgroundColor: 'rgba(129,163,5,1)'/*'rgba(133,153,0,1)'*/ }, chartopt = {}; for (var k of Object.keys(sets)) { if (k === 'all') { var a = sets.all.hits; chartopt.bsratio = { type: 'pie', data: { labels: ['batch %', 'survey %'], datasets: [{ data: [Math.decRound(100*a.batch.total/a.total,2), Math.decRound(100*(a.total-a.batch.total)/a.total,2)], backgroundColor: ['#4773ED', '#EDC147'] }] } }; chartopt.pdist = { type: 'bar', data: { labels: sets.all.distribution.data.labelsp, datasets: [{ data: sets.all.distribution.data.pay, backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }] }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1 }] }} }; chartopt.hdist = { type: 'bar', data: { labels: sets.all.distribution.data.labelsh, datasets: [{ data: sets.all.distribution.data.hits, backgroundColor: '#FA5583', hoverBackgroundColor: '#C7ED3E' }] }, options: { scales: { xAxes: [{ labels: {fontSize: 0}, categorySpacing: 1}] }} }; continue; } Object.assign(sets[k].hits, hitlineopt); Object.assign(sets[k].pay, paylineopt); chartopt[k] = { type:'line', data: { labels: sets[k].hits.labels, datasets: [sets[k].hits, sets[k].pay] }, options: { stacked: false, scales: { xAxes: [{ gridLines: { offsetGridLines: false }, display: true, scaleSteps: 10, labels: {fontSize: /*/^[ym]/.test(k) ? 5 :*/ 0, show: /^[ym]/.test(k) ? true : false} }], yAxes: [{ type: 'linear', display: true, position: 'left', id: 'hits' }, { type: 'linear', display: true, position: 'right', id: 'pay', gridLines: { drawOnChartArea: false } }] } } }; } var html = ['<head>', '<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400" rel="stylesheet" type="text/css">', '<script src="https://cdn.jsdelivr.net/chart.js/2.0.0-alpha3/Chart.min.js"></script>', '<style>', '.caption {width:60%;margin:auto;padding:5px;text-align:center;font-size:0.75em}', '.hc {color:#9559F0} .pc {color:#C8F230}', '.bc {color:#4773ED} .sc {color:#EDC147}', '.container {display:flex;flex-wrap:wrap;align-content:center;justify-content:space-around}', '.mi {font-size:0.6em}', '.title {background-color:#EEE8D5;color:#073642;width:90%;margin:1% 6% 0% 5%;padding-left:2%;font-weight:400;}', '</style>', '<title>HIT Database Analytics | '+dbrange[2]+'</title>', '</head>', '<body style="color:#fff;background-color:#073642;font-size:100%;font-family:\'Open Sans\',Arial;font-weight:300">', // '<i style="color:#fff;">Sigma = σ</i> <span style="color:#fff;font-family:"\'Times New Roman\', Arial;">σ</span>', '<div class="title">GENERAL STATISTICS</div>', '<div class="container">', '<div style="margin-top:5%;margin-left:5%;flex:1">', '<div class="container" style="margin-right:10%">', '<span style="text-align:right;color:#BBD12A;flex:1">'+digitGroup(sets.all.hits.total)+'</span>', '<span style="text-align:center;flex:1">SUBMITTED</span>', '<span style="text-align:left;color:#BBD12A;flex:1">$'+digitGroup(dec(sets.all.pay.total,2))+'</span>', '</div><div class="container" style="margin-right:10%">', '<span style="text-align:right;color:#F23054;flex:1">'+digitGroup(sets.all.hits.rejected)+'</span>', '<span style="text-align:center;flex:1">REJECTED</span>', '<span style="text-align:left;color:#F23054;flex:1">$'+digitGroup(dec(sets.all.pay.rejected,2))+'</span>', '</div><div class="container" style="margin-right:10%">', '<span style="text-align:right;color:#E0B438;flex:1">'+digitGroup(sets.all.hits.pending)+'</span>', '<span style="text-align:center;flex:1">PENDING</span>', '<span style="text-align:left;color:#E0B438;flex:1">$'+digitGroup(dec(sets.all.pay.pending,2))+'</span>', '</div>', '<span class="mi" style="text-align:justify"><span style="color:#30CEF2">NOTE:</span> DOLLAR AMOUNT FOR \'PENDING\' MAY ', 'REFLECT VALUES FOR HITS WHICH HAVE ALREADY BEEN APPROVED. THESE FUNDS HAVE NOT BEEN FULLY CLEARED ', 'AND CREDITED TO YOUR ACCOUNT, AND THUS ARE STILL TECHNICALLY PENDING.</span>', '<div style="margin:10% 10% 10% 0;flex:1;text-align:center;">', 'AVERAGE SUBMISSION OF <span style="color:#C7ED3E">'+dec(sets.all.hitsPerRequester.avg,3)+'</span> ', '(<span style="color:#C7ED3E">±'+dec(sets.all.hitsPerRequester.se,3)+'</span>) HITS PER REQUESTER', '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.hitsPerRequester.sd,3)+'</span></span>', '<hr style="width:45%;color:#fff;background-color:#fff">', 'AVERAGE PAY OF <span style="color:#C7ED3E">$'+dec(sets.all.payPerHit.avg,3)+'</span> ', '(<span style="color:#C7ED3E">±$'+dec(sets.all.payPerHit.se,3)+'</span>) PER HIT', '<br><span class="mi">STANDARD DEVIATION: <span style="color:#FA5583">'+dec(sets.all.payPerHit.sd,3)+'</span></span>', '<hr style="width:45%;color:#fff;background-color:#fff">', 'AVERAGING <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg+sets.all.payPerHit.se))+'</span>', '<span id="minbound"> TO <span style="color:#C7ED3E">'+Math.ceil(1/(sets.all.payPerHit.avg-sets.all.payPerHit.se))+ '</span></span> HITS PER $1.00', '</div>', '</div>', '<div style="margin-top:1%;flex:1">', '<div><canvas id="hitdist"></canvas><div class="caption">HIT DISTRIBUTION PER REQUESTER</div></div>', '<div><canvas id="paydist"></canvas><div class="caption">PAY DISTRIBUTION PER HIT</div></div>', '</div>', '<div style="margin-top:10%;flex:1">', '<canvas id="batchpie"></canvas>', '<div class="caption"><span class="bc">BATCH</span> : <span class="sc">SURVEY</span><br>RATIO APPROXIMATION</div>', '</div>', '</div>', '<div class="title">ACTIVITY</div>', '<div class="container">', '<div style="margin-top:1%;margin-left:5%;flex:1">', '<canvas id="dailyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>DAILY</div>', '</div>', '<div style="margin-top:1%;margin-right:6%;flex:1">', '<canvas id="weeklyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>WEEKLY</div>', '</div>', '</div>', '<div class="container">', '<div style="margin-top:1%;margin-left:5%;flex:1">', '<canvas id="monthlyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>MONTHLY</div>', '</div>', '<div style="margin-top:1%;margin-right:6%;flex:1">', '<canvas id="yearlyline"></canvas>', '<div class="caption"><span class="hc">HITS SUBMITTED</span> | <span class="pc">TOTAL PAY</span><br>YEARLY</div>', '</div>', '</div>', '<div class="title">TOP REQUESTERS</div>', '<div class="container">', '<table id="req10pay" style="font-weight:300;margin-top:1%;margin-left:5%;">', '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">PAY</th></tr></thead><tbody></tbody></table>', '<table id="req10hit" style="font-weight:300;margin-top:1%;margin-right:6%;">', '<thead><tr><th colspan="2" style="text-align:left;padding-left:15">HITS</th></tr></thead><tbody></tbody></table>', '</div>', '<script>', 'if (!NodeList.prototype[Symbol.iterator]) NodeList.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator];', 'var chartopt = '+JSON.stringify(chartopt)+',', ' dLines = new Chart(document.getElementById("dailyline").getContext("2d"), chartopt.day),', ' mLines = new Chart(document.getElementById("monthlyline").getContext("2d"), chartopt.month),', ' wLines = new Chart(document.getElementById("weeklyline").getContext("2d"), chartopt.week),', ' yLines = new Chart(document.getElementById("yearlyline").getContext("2d"), chartopt.year),', ' bsPie = new Chart(document.getElementById("batchpie").getContext("2d"), chartopt.bsratio),', ' hdist = new Chart(document.getElementById("hitdist").getContext("2d"), chartopt.hdist),', ' pdist = new Chart(document.getElementById("paydist").getContext("2d"), chartopt.pdist),', ' r10hit = document.getElementById("req10hit").tBodies[0], r10hHtml = [],', ' r10pay = document.getElementById("req10pay").tBodies[0], r10pHtml = [],', ' payArr = '+JSON.stringify(sets.all.rpay)+',', ' hitArr = '+JSON.stringify(sets.all.rhits)+',', ' mb = document.getElementById("minbound");', 'if (mb.children[0].textContent === mb.previousSibling.textContent) mb.style.display = "none";', 'payArr.forEach(v => r10pHtml', '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">$"+v.pay+"</td><td>"+v.name+"</td></tr>"));', 'hitArr.forEach(v => r10hHtml', '.push("<tr><td style=\\"text-align:right;padding-right:10;color:#47EDE1\\">"+v.hits+"</td><td>"+v.name+"</td></tr>"));', 'r10hit.innerHTML = r10hHtml.join(""); r10pay.innerHTML = r10pHtml.join("");', '</script>', '</body>']; var blob = new Blob(html, {type:'text/html'}), _a = D.body.appendChild(D.createElement('A')); _a.href = URL.createObjectURL(blob); _a.target = '_blank'; _a.click(); _a.remove(); }//}}} function aggregateCursor(dat, wRange) {//{{{ var ymwd = [dat.date.substr(0,4), dat.date.substr(0,7), null, dat.date], pay = isNaN(dat.reward) ? +dat.reward.pay : +dat.reward; // determine weekly scales if (!wRange[0]) wRange = getWeekRange(dat.date); if (dat.date >= wRange[1]) wRange = getWeekRange(dat.date); ymwd[2] = wRange.join(':'); // populate scaled aggregates for (var i=0; i<ymwd.length; i++) { // ... basic if (!sets[ymwdLabels[i]].aggregate.def[ymwd[i]]) sets[ymwdLabels[i]].aggregate.def[ymwd[i]] = { hits:0, pay:0 }; sets[ymwdLabels[i]].aggregate.def[ymwd[i]].hits += 1; sets[ymwdLabels[i]].aggregate.def[ymwd[i]].pay += pay; // ... by requester if (i > 1) continue; // skip--don't need this for day or week if (!sets[ymwdLabels[i]].aggregate.req[ymwd[i]]) { sets[ymwdLabels[i]].aggregate.req[ymwd[i]] = []; sets[ymwdLabels[i]].rpay[ymwd[i]] = []; sets[ymwdLabels[i]].rhits[ymwd[i]] = []; } var reqArr = sets[ymwdLabels[i]].aggregate.req[ymwd[i]], reqLoc = reqArr.findIndex(v => v.req === dat.requesterId); if (~reqLoc) { reqArr[reqLoc].hits += 1; reqArr[reqLoc].pay += pay; } else reqArr.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay }); }// end for ymwd loop // repeat requester aggregation for 'all' reqLoc = sets.all.aggregate.findIndex(v => v.req === dat.requesterId); if (~reqLoc) { sets.all.aggregate[reqLoc].hits += 1; sets.all.aggregate[reqLoc].pay += pay; } else sets.all.aggregate.push({ req: dat.requesterId, name: dat.requesterName, hits: 1, pay: pay }); // populate pay distribution data // hit distribution needs to wait until later--after aggregation sets.all.payPerHit.data.push(pay); distp.forEach( v => { var range = v.split('-'); if (pay === 0) { sets.all.distribution.pay['0'] += 1; return; } if (pay > 5) { sets.all.distribution.pay['5.01+'] += 1; return; } if (pay >= range[0] && pay <= range[1]) { sets.all.distribution.pay[v] += 1; return; } }); // increment general stats var _t = dat.title.trim(); sets.all.hits.total += 1; sets.all.hits.rejected += dat.status === 'Rejected' ? 1 : 0; sets.all.hits.pending += dat.status === 'Pending Approval' ? 1 : 0; sets.all.hits.titles[_t] = sets.all.hits.titles[_t] + 1 || 1; sets.all.pay.total += pay; sets.all.pay.rejected += dat.status === 'Rejected' ? pay : 0; sets.all.pay.pending += /pending/i.test(dat.status) ? pay : 0; // populate data for batch approximation var hitLoc = sets.all.hits.batch.findIndex(v => v.title === _t && v.req === dat.requesterId); if (~hitLoc) sets.all.hits.batch[hitLoc].count += 1; else sets.all.hits.batch.push({ title: _t, req: dat.requesterId, count: 1 }); return wRange; }//}}} function DBResult() { this.results = []; this.add = v => this.results.push(v); this.getDatasets = () => {// {{{ return sets; };//}}} getDatasets }// DBResult function getWeekRange(date) { var _d = new Date(date), _ws, _we; _d.setUTCDate(_d.getUTCDate()-_d.getUTCDay()); _ws = _d.toISODateString(); _d.setUTCDate(_d.getUTCDate()+7); _we = _d.toISODateString(); return [ _ws, _we ]; } //set up a worker thread to prevent interface locking on large datasets var datasetsWorker = [//{{{ 'var dec = '+dec+', digitGroup = '+digitGroup+';', 'Math.decRound = '+Math.decRound+';', // calculate the standard deviation 'function stdev(avg, N, data) {', 'var sum = 0;', 'data.forEach(v => sum += Math.pow(v-avg, 2));', 'return Math.sqrt(sum/(N-1));', '}', 'onmessage = (e) => {', 'var sets = e.data, disth = '+JSON.stringify(disth)+', distp = '+JSON.stringify(distp)+';', // sort aggregates 'for (var period of Object.keys(sets)) {', 'if (period === \'all\') { ', // calculate averages, sd, se 'postMessage( {type: "status", data: "Calculating averages..." });', 'var _pph = sets.all.payPerHit, _hpr = sets.all.hitsPerRequester;', '_pph.avg = sets.all.pay.total / sets.all.hits.total;', '_pph.sd = stdev(_pph.avg, _pph.data.length, _pph.data);', '_pph.se = _pph.sd/Math.sqrt(_pph.data.length);', '_hpr.data = sets.all.aggregate.map( v => v.hits );', '_hpr.avg = sets.all.hits.total / _hpr.data.length;', '_hpr.sd = stdev(_hpr.avg, _hpr.data.length, _hpr.data);', '_hpr.se = _hpr.sd/Math.sqrt(_hpr.data.length);', // get hit distribution 'postMessage( {type: "status", data: "Populating distributions..." });', 'sets.all.aggregate.forEach(v => {', 'disth.forEach(w => {', 'var range = w.split(\'-\');', 'if (range.length === 1) range.push(\'1\');', 'if (v.hits > 1000) { sets.all.distribution.hits[\'1001+\'] += 1; return; }', 'if (v.hits >= range[0] && v.hits <= range[1]) { sets.all.distribution.hits[w] += 1; return; }', '});', '});', // extract raw data for distributions 'disth.forEach(v => sets.all.distribution.data.hits.push(sets.all.distribution.hits[v]));', 'distp.forEach(v => sets.all.distribution.data.pay.push(sets.all.distribution.pay[v]));', 'sets.all.distribution.data.labelsh = Object.keys(sets.all.distribution.hits);', 'sets.all.distribution.data.labelsp = Object.keys(sets.all.distribution.pay);', // get 'top' lists 'sets.all.rpay = sets.all.aggregate.sort((a,b) => b.pay - a.pay ).slice(0,10);', 'sets.all.rhits = sets.all.aggregate.sort((a,b) => b.hits - a.hits).slice(0,10);', 'sets.all.rpay.forEach(v => v.pay = dec(v.pay,2));', 'sets.all.rhits.forEach(v => v.hits = digitGroup(v.hits));', 'continue;', '}', // populate labels/data for graphing 'postMessage({ type: "status", data: "Sorting by "+period+"..." });', 'var labels = Object.keys(sets[period].aggregate.def);', 'for (k of labels) {', 'if (~[\'hits\',\'pay\'].indexOf(k)) continue;', 'sets[period].pay.labels.push(k);', 'sets[period].hits.labels.push(k);', 'sets[period].pay.data.push(Math.decRound(sets[period].aggregate.def[k].pay,2));', 'sets[period].hits.data.push(sets[period].aggregate.def[k].hits);', // sort req lists/period 'if (~[\'week\',\'day\'].indexOf(period)) continue;', 'var len = period === \'year\' ? 10 : 5;', 'reqArr = sets[period].aggregate.req[k];', 'sets[period].rpay[k] = reqArr.sort((a,b) => b.pay - a.pay).slice(0,len);', 'sets[period].rhits[k] = reqArr.map(v => v).sort((a,b) => b.hits - a.hits).slice(0,len);', '}', '}', // if there are more than 5 hits with the same title under the same requester, assume it's a batch 'sets.all.hits.batch = sets.all.hits.batch.filter(v => v.count > 5);', // workers fail to transfer nonenumerable properties :( //'Object.defineProperty(sets.all.hits.batch, \'total\', { configurable: true, writable: true, value: 0 });', 'sets.all.hits.batch.total = 0;', 'sets.all.hits.batch.forEach(v => {if (typeof v === "object") sets.all.hits.batch.total += v.count});', 'postMessage({ type: "obj", data: sets });', '}'];//}}} datasetsWorker = new Blob(datasetsWorker, {type:'text/javascript'}); var workerURL = URL.createObjectURL(datasetsWorker); var dsWorker = new Worker(workerURL); dsWorker.onmessage = (e) => { if (e.data.type === 'status') { window.Status.message = e.data.data; console.log(e.data.data); } else { metrics.mark('dsWorker','end'); metrics.stop();metrics.report(); window.Status.message = 'Done!'; window.Progress.hide(); drawCharts(e.data.data); } }; })(document); // vim: ts=2:sw=2:et:fdm=marker:noai