HIT Scraper WITH EXPORT

Snag HITs.

当前为 2014-06-02 提交的版本,查看 最新版本

  1. // ==UserScript==
  2. // @name HIT Scraper WITH EXPORT
  3. // @author Kerek and TJ
  4. // @description Snag HITs.
  5. // Based in part on code from mmmturkeybacon Export Mturk History and mmmturkeybacon Color Coded Search with Checkpoints
  6. // @namespace http://userscripts.org/users/536998
  7. // @match https://www.mturk.com/mturk/findhits?match=true#hit_scraper*
  8. // @match https://www.mturk.com/mturk/findhits?match=true?hit_scraper*
  9. // @version 1.3.0.2
  10. // @grant GM_xmlhttpRequest
  11. // @grant GM_getValue
  12. // @grant GM_setValue
  13. // @require http://code.jquery.com/jquery-latest.min.js
  14. // ==/UserScript==
  15.  
  16. //alter the requester ignore last as you desire, just follow the format below and use EXACT capitalization e.g., CrowdSource
  17. var ignore_list = ["oscar smith", "jonathon weber", "jerry torres", "crowdsource", "we-pay-you-fast", "turk experiment", "jon brelig"];
  18.  
  19. //this searches extra pages if you skip too much, helps fill out results if you hit a chunk of ignored HITs. Change to true for this behavior.
  20. var correct_for_skips = false;
  21.  
  22. //weight the four TO ratings for the coloring. Default has pay twice as important as fairness and nothing for communication and fast.
  23. var COMM_WEIGHT = 0;
  24. var PAY_WEIGHT = 10;
  25. var FAIR_WEIGHT = 5;
  26. var FAST_WEIGHT = 0;
  27.  
  28. //display your hitdb records if applicable
  29. var check_hitDB = true;
  30.  
  31. //default text size
  32. var default_text_size=11;
  33.  
  34.  
  35.  
  36. var HITStorage = {};
  37. var indexedDB = window.indexedDB || window.webkitIndexedDB ||
  38. window.mozIndexedDB;
  39. window.IDBTransaction = window.IDBTransaction || window.webkitIDBTransaction || window.mozIDBTransaction;
  40. window.IDBKeyRange = window.IDBKeyRange || window.webkitIDBKeyRange || window.mozIDBKeyRange;
  41. HITStorage.IDBTransactionModes = { "READ_ONLY": "readonly", "READ_WRITE": "readwrite", "VERSION_CHANGE": "versionchange" };
  42. var IDBKeyRange = window.IDBKeyRange;
  43.  
  44. HITStorage.indexedDB = {};
  45. HITStorage.indexedDB = {};
  46. HITStorage.indexedDB.db = null;
  47.  
  48. HITStorage.indexedDB.onerror = function(e) {
  49. console.log(e);
  50. };
  51.  
  52. var v=4;
  53.  
  54. HITStorage.indexedDB.checkTitle = function(title,button) {
  55. var request = indexedDB.open("HITDB", v);
  56. request.onsuccess = function(e) {
  57. HITStorage.indexedDB.db = e.target.result;
  58. var db = HITStorage.indexedDB.db;
  59. if (!db.objectStoreNames.contains("HIT"))
  60. {
  61. db.close();
  62. return;
  63. }
  64. var trans = db.transaction(["HIT"], HITStorage.IDBTransactionModes.READ_ONLY);
  65. var store = trans.objectStore("HIT");
  66.  
  67. var index = store.index("title");
  68. index.get(title).onsuccess = function(event)
  69. {
  70. if (event.target.result === undefined)
  71. {
  72. console.log(title + ' not found');
  73. history[button].titledb=false;
  74. }
  75. else
  76. {
  77. console.log(title + ' found');
  78. history[button].titledb=true;
  79. }
  80. db.close();
  81. };
  82. };
  83. request.onerror = HITStorage.indexedDB.onerror;
  84. };
  85.  
  86. HITStorage.indexedDB.checkRequester = function(id,button) {
  87. var request = indexedDB.open("HITDB", v);
  88. request.onsuccess = function(e) {
  89. HITStorage.indexedDB.db = e.target.result;
  90. var db = HITStorage.indexedDB.db;
  91. if (!db.objectStoreNames.contains("HIT"))
  92. {
  93. db.close();
  94. return;
  95. }
  96. var trans = db.transaction(["HIT"], HITStorage.IDBTransactionModes.READ_ONLY);
  97. var store = trans.objectStore("HIT");
  98.  
  99. var index = store.index("requesterId");
  100. index.get(id).onsuccess = function(event)
  101. {
  102. if (event.target.result === undefined)
  103. {history[button].reqdb=false;
  104. console.log(id + ' not found');
  105. }
  106. else
  107. {
  108. history[button].reqdb=true;
  109. console.log(id + ' found');
  110. }
  111. db.close();
  112. };
  113. };
  114. request.onerror = HITStorage.indexedDB.onerror;
  115. };
  116.  
  117. var PAGES_TO_SCRAPE = 3;
  118. var MINIMUM_HITS = 100;
  119. var SEARCH_REFRESH=0;
  120. var URL_BASE = "/mturk/searchbar?searchWords=&selectedSearchType=hitgroups";
  121. var initial_url = URL_BASE;
  122. var TO_REQ_URL = "http://turkopticon.ucsd.edu/reports?id=";
  123. var found_key_list=[];
  124. var last_clear_time = new Date().getTime();
  125. var searched_once = false;
  126. var save_new_results_time = 120;
  127. var save_results_time = 3600;
  128. var default_type = 0;
  129. var cur_loc = window.location.href;
  130. var time_input = document.createElement("INPUT");
  131. time_input.value = 0;
  132. var page_input = document.createElement("INPUT");
  133. page_input.value = 3;
  134. var min_input = document.createElement("INPUT");
  135. var new_time_display_input = document.createElement("INPUT");
  136. new_time_display_input.value = 300;
  137. var reward_input = document.createElement("INPUT");
  138. var qual_input = document.createElement("INPUT");
  139. qual_input.type = "checkbox";
  140. qual_input.checked = true;
  141. var masters_input = document.createElement("INPUT");
  142. masters_input.type = "checkbox";
  143. var sort_input1 = document.createElement("INPUT");
  144. sort_input1.type = "radio";
  145. sort_input1.name = "sort_type";
  146. sort_input1.value = "latest";
  147. sort_input1.checked = true;
  148. var sort_input2 = document.createElement("INPUT");
  149. sort_input2.type = "radio";
  150. sort_input2.name = "sort_type";
  151. sort_input2.value = "most";
  152. var sort_input3 = document.createElement("INPUT");
  153. sort_input3.type = "radio";
  154. sort_input3.name = "sort_type";
  155. sort_input3.value = "amount";
  156.  
  157. var search_input = document.createElement("INPUT");
  158.  
  159. var LINK_BASE = "https://www.mturk.com";
  160. var BACKGROUND_COLOR = "rgb(19, 19, 19)";
  161. var STATUSDETAIL_DELAY = 250;
  162. var MPRE_DELAY = 3000;
  163.  
  164. var next_page = 1;
  165.  
  166. var GREEN = '#66CC66'; // > 4
  167. var LIGHTGREEN = '#ADFF2F'; // > 3 GREEN YELLOW
  168. var YELLOW = '#FFD700';
  169. var ORANGE = '#FF9900'; // > 2
  170. var RED = '#FF3030'; // <= 2
  171. var BLUE = '#C0D9D9'; // no TO
  172. var GREY = 'lightGrey';
  173. var BROWN = '#94704D';
  174. var DARKGREY = '#9F9F9F';
  175. $('body').css('background', BACKGROUND_COLOR);
  176.  
  177. var API_PROXY_BASE = 'https://api.turkopticon.istrack.in/';
  178. var API_MULTI_ATTRS_URL = API_PROXY_BASE + 'multi-attrs.php?ids=';
  179. var REVIEWS_BASE = 'http://turkopticon.ucsd.edu/';
  180.  
  181. var control_panel_HTML = '<div id="control_panel" style="margin: 0 auto 0 auto;' +
  182. 'border-bottom: 1px solid #000000; margin-bottom: 5px; ' +
  183. 'background-color: ' + BACKGROUND_COLOR + ';"></div>';
  184. $('body > :not(#control_panel)').hide(); //hide all nodes directly under the body
  185. $('body').prepend(control_panel_HTML);
  186.  
  187. var control_panel = document.getElementById("control_panel");
  188. var big_red_button = document.createElement("BUTTON");
  189. var progress_report = document.createTextNode("Stopped");
  190. var text_area = document.createElement("TABLE");
  191. big_red_button.textContent = "Show Interface";
  192. big_red_button.onclick = function(){show_interface();};
  193. control_panel.appendChild(big_red_button);
  194.  
  195. show_interface();
  196.  
  197. var global_run = false;
  198. var statusdetail_loop_finished = false;
  199. var date_header = "";
  200. var history = {};
  201. var wait_loop;
  202.  
  203. function set_progress_report(text, force)
  204. {
  205. if (global_run == true || force == true)
  206. {
  207. progress_report.textContent = text;
  208. }
  209. }
  210.  
  211. function get_progress_report()
  212. {
  213. return progress_report.textContent;
  214. }
  215.  
  216. function wait_until_stopped()
  217. {
  218. if (global_run == true)
  219. {
  220. if (statusdetail_loop_finished == true)
  221. {
  222. big_red_button.textContent = "Start";
  223. set_progress_report("Finished", false);
  224. }
  225. else
  226. {
  227. setTimeout(function(){wait_until_stopped();}, 500);
  228. }
  229. }
  230. }
  231.  
  232. function display_wait_time(wait_time)
  233. {
  234. if (global_run == true)
  235. {
  236. var current_progress = get_progress_report();
  237. if (current_progress.indexOf("Searching again in")!==-1)
  238. {
  239. set_progress_report(current_progress.replace(/Searching again in \d+ seconds/ , "Searching again in " + wait_time + " seconds"),false);
  240. }
  241. else
  242. set_progress_report(current_progress + " Searching again in " + wait_time + " seconds.", false);
  243. if (wait_time>1)
  244. setTimeout(function(){display_wait_time(wait_time-1);}, 1000);
  245. }
  246. }
  247.  
  248. function dispArr(ar)
  249. {
  250. var disp = "";
  251. for (var z = 0; z < ar.length; z++)
  252. {
  253. disp += "id " + z + " is " + ar[z] + " ";
  254. }
  255. console.log(disp);
  256. }
  257.  
  258. function scrape($src)
  259. {
  260. var $requester = $src.find('a[href^="/mturk/searchbar?selectedSearchType=hitgroups&requester"]');
  261. var $title = $src.find('a[class="capsulelink"]');
  262. var $reward = $src.find('span[class="reward"]');
  263. var $preview = $src.find('a[href^="/mturk/preview?"]');
  264. var $qualified = $src.find('a[href^="/mturk/notqualified?"]');
  265. var $times = $src.find('a[id^="duration_to_complete"]');
  266. var $descriptions = $src.find('a[id^="description"]');
  267. var not_qualified_group_IDs=[];
  268. $qualified.each(function(){
  269. var groupy = $(this).attr('href');
  270. groupy = groupy.replace("/mturk/notqualified?hitId=","");
  271. not_qualified_group_IDs.push(groupy);
  272. });
  273. var $mixed = $src.find('a[href^="/mturk/preview?"],a[href^="/mturk/notqualified?"]');
  274. var listy =[];
  275. $mixed.each(function(){
  276. var groupy = $(this).attr('href');
  277. groupy = groupy.replace("/mturk/notqualified?hitId=","");
  278. groupy = groupy.replace("/mturk/preview?groupId=","");
  279. listy.push(groupy);
  280. });
  281. listy = listy.filter(function(elem, pos) {
  282. return listy.indexOf(elem) == pos;
  283. });
  284.  
  285. for (var j = 0; j < $requester.length; j++)
  286. {
  287. var $hits = $requester.eq(j).parent().parent().parent().parent().parent().parent().find('td[class="capsule_field_text"]');
  288. var requester_name = $requester.eq(j).text().trim();
  289. var requester_link = $requester.eq(j).attr('href');
  290. var group_ID=listy[j];
  291. var preview_link = "/mturk/preview?groupId=" + group_ID;
  292. var title = $title.eq(j).text().trim();
  293. var reward = $reward.eq(j).text().trim();
  294. var hits = $hits.eq(4).text().trim();
  295. var time = $times.eq(j).parent()[0].nextSibling.nextSibling.innerHTML;
  296. var description = $descriptions.eq(j).parent()[0].nextSibling.nextSibling.innerHTML;
  297. console.log(description);
  298. var requester_id = requester_link.replace('/mturk/searchbar?selectedSearchType=hitgroups&requesterId=','');
  299. var accept_link;
  300. accept_link = preview_link.replace('preview','previewandaccept');
  301. /*HIT SCRAPER ADDITION*/
  302. var qElements = document.getElementById('qualificationsRequired.tooltip--' + j).parentNode.parentNode.parentNode.getElementsByTagName('tr');
  303.  
  304. var qualifications = [];
  305. for (var i = 1; i < qElements.length; i++) {
  306. qualifications.push((qElements[i].childNodes[1].textContent.trim().replace(/\s+/g, ' ').indexOf("Masters") != -1 ? "[color=red][b]"+qElements[i].childNodes[1].textContent.trim().replace(/\s+/g, ' ')+"[/b][/color]" : qElements[i].childNodes[1].textContent.trim().replace(/\s+/g, ' ')));
  307. }
  308. var qualList = (qualifications.join(', ') ? qualifications.join(', ') : "None");
  309.  
  310. key = requester_name+title+reward+group_ID;
  311. found_key_list.push(key);
  312. if (history[key] == undefined)
  313. {
  314. history[key] = {requester:"", title:"", description:"", reward:"", hits:"", req_link:"", quals:"", prev_link:"", rid:"", acc_link:"", new_result:"", qualified:"", found_this_time:"", initial_time:"", reqdb:"",titledb:"",time:""};
  315. history[key].req_link = requester_link;
  316. history[key].prev_link = preview_link;
  317. history[key].requester = requester_name;
  318. history[key].title = title;
  319. history[key].reward = reward;
  320. history[key].hits = hits;
  321. history[key].rid = requester_id;
  322. history[key].acc_link = accept_link;
  323. history[key].time = time;
  324. history[key].quals = qualList;
  325. history[key].description = description;
  326. HITStorage.indexedDB.checkRequester(requester_id,key);
  327. HITStorage.indexedDB.checkTitle(title,key);
  328. if (searched_once)
  329. {
  330. history[key].initial_time = new Date().getTime();//-1000*(save_new_results_time - SEARCH_REFRESH);
  331. history[key].new_result = 0;
  332. }
  333. else
  334. {
  335. history[key].initial_time = new Date().getTime()-1000*save_new_results_time;
  336. history[key].new_result = 1000*save_new_results_time;
  337. }
  338. if (not_qualified_group_IDs.indexOf(group_ID)!==-1)
  339. history[key].qualified = false;
  340. else
  341. history[key].qualified = true;
  342.  
  343. history[key].found_this_time = true;
  344. }
  345. else
  346. {
  347. history[key].new_result = new Date().getTime() - history[key].initial_time;
  348. history[key].found_this_time = true;
  349. history[key].hits = hits;
  350. }
  351. }
  352. }
  353.  
  354. function statusdetail_loop(next_URL)
  355. {
  356. if (global_run == true)
  357. {
  358. if (next_URL.length != 0)
  359. {
  360. $.get(next_URL, function(data)
  361. {
  362. var $src = $(data);
  363. var maxpagerate = $src.find('td[class="error_title"]:contains("You have exceeded the maximum allowed page request rate for this website.")');
  364. if (maxpagerate.length == 0)
  365. {
  366. set_progress_report("Processing page " + next_page, false);
  367. scrape($src);
  368. $next_URL = $src.find('a[href^="/mturk/viewsearchbar"]:contains("Next")');
  369. next_URL = ($next_URL.length != 0) ? $next_URL.attr("href") : "";
  370. next_page++;
  371. if (default_type == 1)
  372. {
  373. var hmin = MINIMUM_HITS+1;
  374. for (j = 0; j < found_key_list.length; j++)
  375. {
  376. if (history[found_key_list[j]].hits < hmin)
  377. {
  378. next_URL = "";
  379. next_page = -1;
  380. break;
  381. }
  382. }
  383. }
  384. else if (next_page > PAGES_TO_SCRAPE && correct_for_skips)
  385. {
  386. var skipped_hits = 0;
  387. var added_pages = 0;
  388. for (j = 0; j < found_key_list.length; j++)
  389. {
  390. var obj = history[found_key_list[j]];
  391. if (! ignore_check(obj.requester,obj.title))
  392. skipped_hits++;
  393. }
  394. added_pages = Math.floor(skipped_hits/10);
  395. if (skipped_hits%10 >6)
  396. added_pages++;
  397. if (next_page > PAGES_TO_SCRAPE + added_pages)
  398. {
  399. next_URL = "";
  400. next_page = -1;
  401. }
  402. }
  403. else if (next_page > PAGES_TO_SCRAPE)
  404. {
  405. next_URL = "";
  406. next_page = -1;
  407. }
  408. setTimeout(function(){statusdetail_loop(next_URL);}, STATUSDETAIL_DELAY);
  409. }
  410. else
  411. {
  412. console.log("MPRE");
  413. setTimeout(function(){statusdetail_loop(next_URL);}, MPRE_DELAY);
  414. }
  415. });
  416. }
  417. else
  418. {
  419. searched_once = true;
  420. var found_hits = found_key_list.length;
  421. var shown_hits = 0;
  422. var new_hits = 0;
  423. var url = API_MULTI_ATTRS_URL;
  424. var rids = [];
  425. var lastRow = text_area.rows.length - 1;
  426. for (i = lastRow; i>0; i--)
  427. text_area.deleteRow(i);
  428. for (j = 0; j < found_key_list.length; j++)
  429. {
  430. //(function(url,rids,j) {
  431. var obj = history[found_key_list[j]];
  432. if (ignore_check(obj.requester,obj.title) && obj.found_this_time){
  433. ++shown_hits;
  434. //console.log(obj);
  435. //hit export will update col_heads[1]
  436. var col_heads = ["<a href='"+ LINK_BASE+obj.req_link +"' target='_blank'>" + obj.requester + "</a>","<a href='"+ LINK_BASE+obj.prev_link +"' target='_blank'>" + obj.title + "</a>",obj.reward,obj.hits,"TO down","<a href='"+ LINK_BASE+obj.acc_link +"' target='_blank'>Accept</a>"];
  437. var row = text_area.insertRow(text_area.rows.length);
  438. url += obj.rid + ',';
  439. rids.push(obj.rid);
  440. if (check_hitDB)
  441. {
  442. col_heads.push("R");
  443. col_heads.push("T");
  444. }
  445. if (!obj.qualified)
  446. {
  447. col_heads.push("Not Qualified");
  448. }
  449. for (i=0; i<col_heads.length; i++)
  450. {
  451. var this_cell = row.insertCell(i);
  452. row.cells[i].style.fontSize = default_text_size;
  453. this_cell.innerHTML = col_heads[i];
  454. if(i>1)
  455. this_cell.style.textAlign = 'center';
  456. if (check_hitDB)
  457. {
  458. if (i==6)
  459. {
  460. if (obj.reqdb)
  461. this_cell.style.backgroundColor = GREEN;
  462. else
  463. this_cell.style.backgroundColor = RED;
  464. }
  465. else if (i==7)
  466. {
  467. if (obj.titledb)
  468. this_cell.style.backgroundColor = GREEN;
  469. else
  470. this_cell.style.backgroundColor = RED;
  471. }
  472. else if (i==8)
  473. this_cell.style.backgroundColor = DARKGREY;
  474. }
  475. else if (i==6)
  476. this_cell.style.backgroundColor = DARKGREY;
  477. }
  478. if (Object.keys(history).length>0)
  479. {
  480. if (obj.new_result < 1000*save_new_results_time)
  481. {
  482. new_hits++;
  483. for (i in col_heads)
  484. {
  485. row.cells[i].style.fontSize = default_text_size + 1;
  486. row.cells[i].style.fontWeight = "bold";
  487. }
  488. }
  489. }
  490. button = document.createElement('button'); //HIT SCRAPER ADDITION
  491. button.textContent = 'vB';
  492. button.title = 'Export this HIT description as vBulletin formatted text';
  493. button.style.height = '14px';
  494. button.style.width = '30px';
  495. button.style.fontSize = '8px';
  496. button.style.border = '1px solid';
  497. button.style.padding = '0px';
  498. button.style.backgroundColor = 'transparent';
  499. //button.addEventListener("click", function() {export_func_deleg(j);}.bind(null,j), false);
  500. button.addEventListener("click", (function (obj,j) { return function() {export_func_deleg(obj,j);}})(obj,j));
  501. row.cells[1].appendChild(button);
  502. }
  503. //});
  504. }
  505. set_progress_report("Scrape complete. " + shown_hits + " HITs found (" + new_hits + " new results). " + (found_hits - shown_hits) + " HITs ignored.", false);
  506. url = url.substring(0,url.length - 1);
  507. //console.log(url);
  508. var success_flag = false;
  509. GM_xmlhttpRequest(
  510. {
  511. method: "GET",
  512. url: url,
  513. onload: function (results)
  514. {
  515. //console.log(results.responseText);
  516. rdata = $.parseJSON(results.responseText);
  517. for (i = 0; i < rids.length; i++)
  518. {
  519. text_area.rows[i+1].style.backgroundColor = GREY;
  520. if (rdata[rids[i]])
  521. {
  522. var pay = rdata[rids[i]].attrs.pay
  523. var reviews = rdata[rids[i]].reviews
  524. var average = 0;
  525. var sum = 0;
  526. var divisor = 0;
  527. var comm = rdata[rids[i]].attrs.comm;
  528. var fair = rdata[rids[i]].attrs.fair;
  529. var fast = rdata[rids[i]].attrs.fast;
  530. if (comm > 0)
  531. {
  532. sum += COMM_WEIGHT*comm;
  533. divisor += COMM_WEIGHT;
  534. }
  535. if (pay > 0)
  536. {
  537. sum += PAY_WEIGHT*pay;
  538. divisor += PAY_WEIGHT;
  539. }
  540. if (fair > 0)
  541. {
  542. sum += FAIR_WEIGHT*fair;
  543. divisor += FAIR_WEIGHT;
  544. }
  545. if (fast > 0)
  546. {
  547. sum += FAST_WEIGHT*fast;
  548. divisor += FAST_WEIGHT;
  549. }
  550. if (divisor > 0)
  551. {
  552. average = sum/divisor;
  553. }
  554. text_area.rows[i+1].cells[4].innerHTML = "<a href='"+ TO_REQ_URL+rids[i] +"' target='_blank'>" + pay + "</a>";
  555. if (reviews > 4)
  556. {
  557. if (average > 4.49)
  558. text_area.rows[i+1].style.backgroundColor = GREEN;
  559. else if (average > 3.49)
  560. text_area.rows[i+1].style.backgroundColor = LIGHTGREEN;
  561. //else if (average > 2.99)
  562. // text_area.rows[i+1].style.backgroundColor = YELLOW;
  563. else if (average > 1.99)
  564. text_area.rows[i+1].style.backgroundColor = ORANGE;
  565. else if (average > 0)
  566. text_area.rows[i+1].style.backgroundColor = RED;
  567. }
  568. }
  569. else
  570. {
  571. text_area.rows[i+1].cells[4].innerHTML = "No data";
  572. }
  573. }
  574. success_flag = true;
  575. }
  576. });
  577. if (!success_flag)
  578. for (i = 0; i < rids.length; i++) text_area.rows[i+1].style.backgroundColor = GREY;
  579. statusdetail_loop_finished = true;
  580. if (SEARCH_REFRESH>0)
  581. {
  582. wait_loop = setTimeout(function(){if (global_run) start_it();}, 1000*SEARCH_REFRESH);
  583. display_wait_time(SEARCH_REFRESH);
  584. }
  585. else
  586. {
  587. global_run = false;
  588. big_red_button.textContent = "Start";
  589. }
  590. }
  591. }
  592. }
  593.  
  594. function ignore_check(r,t){
  595. if (ignore_list.indexOf(r.toLowerCase())==-1)
  596. {
  597. return true;
  598. }
  599. return false;
  600. }
  601.  
  602. function start_running()
  603. {
  604. if (big_red_button.textContent == "Start")
  605. {
  606. global_run = true;
  607. initial_url = URL_BASE;
  608. if (search_input.value.length>0)
  609. {
  610. initial_url = initial_url.replace("searchWords=", "searchWords=" + search_input.value);
  611. }
  612. if (time_input.value.replace(/[^0-9]+/g,"") != "")
  613. {
  614. SEARCH_REFRESH = Number(time_input.value);
  615. }
  616. if (page_input.value.replace(/[^0-9]+/g,"") != "")
  617. {
  618. PAGES_TO_SCRAPE = Number(page_input.value);
  619. }
  620. if (min_input.value.replace(/[^0-9]+/g,"") != "")
  621. {
  622. MINIMUM_HITS = Number(min_input.value);
  623. }
  624. if (new_time_display_input.value.replace(/[^0-9]+/g,"") != "")
  625. {
  626. save_new_results_time = Number(new_time_display_input.value);
  627. }
  628. if (reward_input.value.replace(/[^0-9]+/g,"") != "")
  629. {
  630. initial_url += "&minReward=" + reward_input.value;
  631. }
  632. else
  633. {
  634. initial_url += "&minReward=0.00";
  635. }
  636. if (qual_input.checked)
  637. {
  638. initial_url += "&qualifiedFor=on"
  639. }
  640. else
  641. {
  642. initial_url += "&qualifiedFor=off"
  643. }
  644. if (masters_input.checked)
  645. {
  646. initial_url += "&requiresMasterQual=on"
  647. }
  648. if (sort_input1.checked)
  649. {
  650. initial_url+= "&sortType=LastUpdatedTime%3A1";
  651. default_type = 0;
  652. }
  653. else if (sort_input2.checked)
  654. {
  655. initial_url+= "&sortType=NumHITs%3A1";
  656. default_type = 1;
  657. }
  658. else if (sort_input3.checked)
  659. {
  660. initial_url+= "&sortType=Reward%3A1";
  661. default_type = 0;
  662. }
  663. initial_url+="&pageNumber=1&searchSpec=HITGroupSearch"
  664. start_it();
  665. }
  666. else
  667. {
  668. global_run = false;
  669. clearTimeout(wait_loop);
  670. big_red_button.textContent = "Start";
  671. set_progress_report("Stopped", true);
  672. }
  673. }
  674.  
  675. function start_it()
  676. {
  677. statusdetail_loop_finished = false;
  678. big_red_button.textContent = "Stop";
  679. found_key_list=[];
  680. var ctime = new Date().getTime()
  681. if (ctime - last_clear_time > save_results_time*666)
  682. {
  683. var last_history=history;
  684. history = {};
  685. for (var key in last_history)
  686. {
  687. if (last_history[key].new_result<save_results_time*1000)
  688. {
  689. history[key]=last_history[key];
  690. if (last_history[key].found_this_time)
  691. {
  692. last_history[key].found_this_time = false;
  693. if (last_history[key].new_result>save_new_results_time*1000)
  694. last_history[key].initial_time = ctime-1000*save_new_results_time;
  695. }
  696. }
  697.  
  698. }
  699. last_clear_time = ctime;
  700. }
  701. next_page = 1;
  702. statusdetail_loop(initial_url);
  703. }
  704.  
  705.  
  706. function show_interface()
  707. {
  708. control_panel.style.color = BROWN;
  709. control_panel.style.fontSize = 14;
  710. control_panel.removeChild(big_red_button);
  711. control_panel.appendChild(document.createTextNode("Auto-refresh delay: "));
  712. time_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  713. time_input.title = "Enter search refresh delay in seconds\n" + "Enter 0 for no auto-refresh\n" + "Default is 0 (no auto-refresh)";
  714. time_input.size = 3;
  715. control_panel.appendChild(time_input);
  716. control_panel.appendChild(document.createTextNode(" "));
  717. control_panel.appendChild(document.createTextNode("Pages to scrape: "));
  718. page_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  719. page_input.title = "Enter number of pages to scrape\n" + "Default is 4";
  720. page_input.size = 3;
  721. control_panel.appendChild(page_input);
  722. control_panel.appendChild(document.createTextNode(" "));
  723. control_panel.appendChild(document.createTextNode("Minimum batch size: "));
  724. min_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  725. min_input.title = "Enter minimum HITs for batch search\n" + "Default is 100";
  726. min_input.size = 3;
  727. control_panel.appendChild(min_input);
  728. control_panel.appendChild(document.createTextNode(" "));
  729. control_panel.appendChild(document.createTextNode("New HIT highlighting: "));
  730. new_time_display_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  731. new_time_display_input.title = "Enter time (in seconds) to keep new HITs highlighted\n" + "Default is 300 (5 minutes)";
  732. new_time_display_input.size = 6;
  733. control_panel.appendChild(new_time_display_input);
  734. control_panel.appendChild(document.createElement("P"));
  735. control_panel.appendChild(document.createTextNode("Minimum reward: "));
  736. reward_input.size = 6;
  737. control_panel.appendChild(reward_input);
  738. control_panel.appendChild(document.createTextNode(" "));
  739.  
  740. control_panel.appendChild(document.createTextNode("Qualified"));
  741. control_panel.appendChild(qual_input);
  742. control_panel.appendChild(document.createTextNode(" "));
  743. control_panel.appendChild(document.createTextNode("Masters"));
  744. control_panel.appendChild(masters_input);
  745. control_panel.appendChild(document.createTextNode(" "));
  746. control_panel.appendChild(document.createTextNode("Sort types: "));
  747. control_panel.appendChild(sort_input1);
  748. control_panel.appendChild(document.createTextNode("Latest"));
  749. control_panel.appendChild(sort_input2);
  750. control_panel.appendChild(document.createTextNode("Most Available"));
  751. control_panel.appendChild(sort_input3);
  752. control_panel.appendChild(document.createTextNode("Amount"));
  753. control_panel.appendChild(document.createElement("P"));
  754. control_panel.appendChild(search_input);
  755. search_input.size = 20;
  756. search_input.title = "Enter a search term to include\n" + "Default is blank (no included terms)";
  757. search_input.placeholder="Enter search terms here";
  758. control_panel.appendChild(document.createTextNode(" "));
  759. big_red_button.textContent = "Start";
  760. big_red_button.onclick = function(){start_running();};
  761. control_panel.appendChild(big_red_button);
  762. control_panel.appendChild(document.createTextNode(" "));
  763. control_panel.appendChild(progress_report);
  764. control_panel.appendChild(document.createElement("P"));
  765. text_area.style.fontWeight = 400;
  766. text_area.createCaption().innerHTML = "HITs";
  767. var col_heads = ['Requester','Title','Reward','HITs Available','TO pay',"Accept HIT"];
  768. var row = text_area.createTHead().insertRow(0);
  769. text_area.caption.style.fontWeight = 800;
  770. text_area.caption.style.color = BROWN;
  771. if (default_text_size > 10)
  772. text_area.cellPadding=Math.min(Math.max(1,Math.floor((default_text_size-10)/2)),5);
  773. console.log(text_area.cellPadding);
  774. //text_area.cellPadding=2;
  775. text_area.caption.style.fontSize = 28;
  776. text_area.rows[0].style.fontWeight = 800;
  777. text_area.rows[0].style.color = BROWN;
  778. for (i=0; i<col_heads.length; i++)
  779. {
  780. var this_cell = row.insertCell(i);
  781. this_cell.innerHTML = col_heads[i];
  782. this_cell.style.fontSize = 14;
  783. if (i > 1)
  784. this_cell.style.textAlign = 'center';
  785. }
  786. control_panel.appendChild(text_area);
  787. }
  788.  
  789.  
  790. /********HIT EXPORT ADDITIONS*****/
  791.  
  792. var EDIT = false;
  793. var HITS = [];
  794. var HIT;
  795.  
  796. var TO_BASE = "http://turkopticon.ucsd.edu/";
  797. var API_BASE = "https://api.turkopticon.istrack.in/";
  798. var API_URL = API_BASE + "multi-attrs.php?ids=";
  799. DEFAULT_TEMPLATE = '[table][tr][td][b]Title:[/b] [url={prev_link}][COLOR=blue]{title}[/COLOR][/url]\n';
  800. DEFAULT_TEMPLATE += '[b]Requester:[/b] [url=https://www.mturk.com/mturk/searchbar?selectedSearchType=hitgroups&requesterId={rid}][COLOR=blue]{requester}[/COLOR][/url]';
  801. DEFAULT_TEMPLATE += ' [{rid}] ([url='+TO_BASE+'{requesterId}][COLOR=blue]TO[/COLOR][/url])';
  802. DEFAULT_TEMPLATE += '\n[b]TO Ratings:[/b]{to_stuff}';
  803. DEFAULT_TEMPLATE += '\n[b]Description:[/b] {description}';
  804. DEFAULT_TEMPLATE += '\n[b]Time:[/b] {time}';
  805. DEFAULT_TEMPLATE += '\n[b]Reward:[/b] [COLOR=green][b]{reward}[/b][/COLOR]';
  806. DEFAULT_TEMPLATE += '\n[b]Qualifications:[/b] {quals}[/td][/tr][/table]';
  807.  
  808. var TEMPLATE;
  809. var EASYLINK;
  810.  
  811. if (typeof GM_getValue === 'undefined')
  812. TEMPLATE = null;
  813. else {
  814. TEMPLATE = GM_getValue('HITExport Template');
  815. EASYLINK = GM_getValue('HITExport Easylink');
  816. }
  817. if (TEMPLATE == null) {
  818. TEMPLATE = DEFAULT_TEMPLATE;
  819. }
  820.  
  821. function buildXhrUrl(rai) {
  822. var url = API_URL;
  823. var ri = rai;
  824. url += rai;
  825. return url;
  826. }
  827.  
  828. function makeXhrQuery(url) {
  829. var xhr = new XMLHttpRequest();
  830. try{
  831. xhr.open('GET', url, false);
  832. xhr.send(null);
  833. return $.parseJSON(xhr.response);
  834. }
  835. catch(err){
  836. return "TO DOWN";
  837. }
  838. }
  839.  
  840. function getNamesForEmptyResponses(rai, resp) {
  841. for (var rid in rai) {
  842. if (rai.hasOwnProperty(rid) && resp[rid] == "") {
  843. resp[rid] = $.parseJSON('{"name": "' + rai[rid][0].innerHTML + '"}');
  844. }
  845. }
  846. return resp;
  847. }
  848.  
  849. function getKeys(obj) {
  850. var keys = [];
  851. for (var key in obj) {
  852. keys.push(key);
  853. }
  854. return keys;
  855. }
  856.  
  857. function export_func_deleg(item,index) {
  858. console.log(item);
  859. export_func(item);
  860. }
  861.  
  862. function export_func(item) {
  863. HIT = item;
  864. EDIT=false;
  865. edit_button.textContent = 'Edit Template';
  866. apply_template(item);
  867. div.style.display = 'block';
  868. textarea.select();
  869. }
  870.  
  871. function apply_template(hit_data) {
  872. var txt = TEMPLATE;
  873.  
  874. var vars = ['title', 'requester', 'rid', 'description', 'reward', 'quals', 'prev_link', 'time', 'hits', 'to_stuff', 'to_text'];
  875.  
  876. var resp = null;
  877. if (txt.indexOf('{to_text}') >= 0 || txt.indexOf('{to_stuff}') >= 0){
  878. var url = buildXhrUrl(hit_data["rid"]);
  879. resp = makeXhrQuery(url);
  880. console.log(resp);
  881. }
  882. var toText = "";
  883. var toStuff = "";
  884. var toData = "";
  885. var numResp = (resp == null || resp == "TO DOWN" ? "n/a" : resp[hit_data["rid"]].reviews);
  886. if (resp == "TO DOWN"){
  887. toStuff = " [URL=\""+TO_BASE+hit_data['rid']+"\"]TO down.[/URL]";
  888. toText = toStuff;
  889. }
  890. else if (resp == null || resp[hit_data["rid"]].attrs == null && resp != "TO DOWN") {
  891. toStuff = " No TO ";
  892. toText = " No TO ";
  893. toStuff += "[URL=\""+TO_BASE+"report?requester[amzn_id]=" + hit_data['rid'] + "&requester[amzn_name]=" + hit_data['requester'] + "\"]";
  894. toStuff += "(Submit a new TO rating for this requester)[/URL]";
  895. }
  896. else {
  897. for (var key in resp[hit_data["rid"]].attrs) {
  898. //toText += "\n[*]"+key+": "+resp[hit_data["requesterId"]].attrs[key]+"\n";
  899. var i = 0;
  900. var color = "green";
  901. var name = key;
  902. var num = Math.floor(resp[hit_data["rid"]].attrs[key]);
  903. switch (key){
  904. case "comm":
  905. name = "Communicativity";
  906. break;
  907. case "pay":
  908. name = "Generosity";
  909. break;
  910. case "fast":
  911. name = "Promptness";
  912. break;
  913. case "fair":
  914. name = "Fairness";
  915. break;
  916. default:
  917. name = key;
  918. break;
  919. }
  920. switch (num){
  921. case 0:
  922. color = "red";
  923. break;
  924. case 1:
  925. color = "red";
  926. break;
  927. case 2:
  928. color = "orange";
  929. break;
  930. case 3:
  931. color = "yellow";
  932. break;
  933. default:
  934. break;
  935. }
  936. toText += (num > 0 ? "\n[color="+color+"]" : "\n");
  937. for (i; i < num; i++){
  938. toText += "[b]☢[/b]"
  939. }
  940. toText += (num > 0 ? "[/color]" : "")
  941. if (i < 5){
  942. toText += "[color=white]";
  943. for (i; i < 5; i++)
  944. toText += "[b]☢[/b]";
  945. toText += "[/color]";
  946. }
  947. toText += " "+Number(resp[hit_data["rid"]].attrs[key]).toFixed(2)+" "+name;
  948. toData += Number(resp[hit_data["rid"]].attrs[key]).toFixed(2) + ",";
  949. }
  950. //toText += "[/list]";
  951. toText += (txt.indexOf('{to_stuff}') >= 0 ? "" : "\nNumber of Reviews: "+numResp+"\n[URL=\""+TO_BASE+"report?requester[amzn_id]=" + hit_data['rid'] + "&requester[amzn_name]=" + hit_data['requester'] + "\"](Submit a new TO rating for this requester)[/URL]");
  952. toStuff = '\n[img]http://data.istrack.in/to/' + toData.slice(0,-1) + '.png[/img]';
  953. toStuff += (txt.indexOf('{to_stuff}') >= 0 ? (txt.indexOf('{to_text}') >= 0 ? "" : toText) : "");
  954. toStuff += "\nNumber of Reviews: "+numResp;
  955. toStuff += "[URL=\""+TO_BASE+"report?requester[amzn_id]=" + hit_data['rid'] + "&requester[amzn_name]=" + hit_data['requester'] + "\"]";
  956. toStuff += "\n(Submit a new TO rating for this requester)[/URL]";
  957. }
  958. for (var i = 0; i < vars.length; i++) {
  959. t = new RegExp('\{' + vars[i] + '\}', 'g');
  960. if (vars[i] == "to_stuff") {
  961. txt = txt.replace(t, toStuff);
  962. }
  963. else if (vars[i] == "to_text"){
  964. txt = txt.replace(t, toText);
  965. }
  966. else if (vars[i] == "prev_link"){
  967. txt = txt.replace(t,"https://www.mturk.com/"+hit_data[vars[i]]);
  968. }
  969. else
  970. txt = txt.replace(t, hit_data[vars[i]]);
  971. }
  972. textarea.value = txt;
  973. }
  974.  
  975. function hide_func(div) {
  976. return function () {
  977. if (EDIT == false)
  978. div.style.display = 'none';
  979. }
  980. }
  981.  
  982. function edit_func() {
  983. return function () {
  984. if (EDIT == true) {
  985. EDIT = false;
  986. TEMPLATE = textarea.value;
  987. edit_button.textContent = 'Edit Template';
  988. apply_template(HITS[HIT]);
  989. }
  990. else {
  991. EDIT = true;
  992. edit_button.textContent = 'Show Changes';
  993. save_button.disabled = false;
  994. textarea.value = TEMPLATE;
  995. }
  996. }
  997. }
  998.  
  999. function easy_func() {
  1000. return function () {
  1001. var input = prompt("Please enter your adfly easy link, or leave blank if you don't have one.");
  1002. if (input.substring(input.length - 1) != "/")
  1003. input += "/";
  1004. EASYLINK = input;
  1005. GM_setValue('HITExport Easylink', EASYLINK);
  1006. var url = EASYLINK + hit_data["link"];
  1007. hit_data["adfly"] = url;
  1008. apply_template(hit_data);
  1009. }
  1010. }
  1011.  
  1012. function default_func() {
  1013. return function () {
  1014. GM_deleteValue('HITExport Template');
  1015. TEMPLATE = DEFAULT_TEMPLATE;
  1016. EDIT = false;
  1017. edit_button.textContent = 'Edit Template';
  1018. apply_template(HITS[HIT]);
  1019. }
  1020. }
  1021.  
  1022. function save_func() {
  1023. return function () {
  1024. if (EDIT)
  1025. TEMPLATE = textarea.value;
  1026. GM_setValue('HITExport Template', TEMPLATE);
  1027. }
  1028. }
  1029.  
  1030. var div = document.createElement('div');
  1031. var textarea = document.createElement('textarea');
  1032. var div2 = document.createElement('label');
  1033.  
  1034. div.style.position = 'fixed';
  1035. div.style.width = '500px';
  1036. div.style.height = '235px';
  1037. div.style.left = '50%';
  1038. div.style.right = '50%';
  1039. div.style.margin = '-250px 0px 0px -250px';
  1040. div.style.top = '300px';
  1041. div.style.padding = '5px';
  1042. div.style.border = '2px';
  1043. div.style.backgroundColor = 'black';
  1044. div.style.color = 'white';
  1045. div.style.zIndex = '100';
  1046.  
  1047. textarea.style.padding = '2px';
  1048. textarea.style.width = '500px';
  1049. textarea.style.height = '200px';
  1050. textarea.title = '{title}\n{requester}\n{requesterId}\n{description}\n{reward}\n{qualifications}\n{link}\n{time}\n{hits_available}\n{to_stuff}\n{to_text}\n{adfly}';
  1051.  
  1052. div.textContent = 'Press Ctrl+C to copy to clipboard. Click textarea to close';
  1053. div.style.fontSize = '12px';
  1054. div.appendChild(textarea);
  1055.  
  1056. var edit_button = document.createElement('button');
  1057. var save_button = document.createElement('button');
  1058. var default_button = document.createElement('button');
  1059. var easy_button = document.createElement('button');
  1060.  
  1061. edit_button.textContent = 'Edit Template';
  1062. edit_button.setAttribute('id', 'edit_button');
  1063. edit_button.style.height = '18px';
  1064. edit_button.style.width = '100px';
  1065. edit_button.style.fontSize = '10px';
  1066. edit_button.style.paddingLeft = '3px';
  1067. edit_button.style.paddingRight = '3px';
  1068. edit_button.style.backgroundColor = 'white';
  1069.  
  1070. save_button.textContent = 'Save Template';
  1071. save_button.setAttribute('id', 'save_button');
  1072. save_button.style.height = '18px';
  1073. save_button.style.width = '100px';
  1074. save_button.style.fontSize = '10px';
  1075. save_button.style.paddingLeft = '3px';
  1076. save_button.style.paddingRight = '3px';
  1077. save_button.style.backgroundColor = 'white';
  1078. save_button.style.marginLeft = '5px';
  1079.  
  1080. easy_button.textContent = 'Change Adfly Url';
  1081. easy_button.setAttribute('id', 'easy_button');
  1082. easy_button.style.height = '18px';
  1083. easy_button.style.width = '100px';
  1084. easy_button.style.fontSize = '10px';
  1085. easy_button.style.paddingLeft = '3px';var div = document.createElement('div');
  1086. var textarea = document.createElement('textarea');
  1087. var div2 = document.createElement('label');
  1088.  
  1089. div.style.position = 'fixed';
  1090. div.style.width = '500px';
  1091. div.style.height = '235px';
  1092. div.style.left = '50%';
  1093. div.style.right = '50%';
  1094. div.style.margin = '-250px 0px 0px -250px';
  1095. div.style.top = '300px';
  1096. div.style.padding = '5px';
  1097. div.style.border = '2px';
  1098. div.style.backgroundColor = 'black';
  1099. div.style.color = 'white';
  1100. div.style.zIndex = '100';
  1101.  
  1102. textarea.style.padding = '2px';
  1103. textarea.style.width = '500px';
  1104. textarea.style.height = '200px';
  1105. textarea.title = '{title}\n{requester}\n{requesterId}\n{description}\n{reward}\n{qualifications}\n{link}\n{time}\n{hits_available}\n{to_stuff}\n{to_text}\n{adfly}';
  1106.  
  1107. div.textContent = 'Press Ctrl+C to copy to clipboard. Click textarea to close';
  1108. div.style.fontSize = '12px';
  1109. div.appendChild(textarea);
  1110.  
  1111. var edit_button = document.createElement('button');
  1112. var save_button = document.createElement('button');
  1113. var default_button = document.createElement('button');
  1114. var easy_button = document.createElement('button');
  1115.  
  1116. edit_button.textContent = 'Edit Template';
  1117. edit_button.setAttribute('id', 'edit_button');
  1118. edit_button.style.height = '18px';
  1119. edit_button.style.width = '100px';
  1120. edit_button.style.fontSize = '10px';
  1121. edit_button.style.paddingLeft = '3px';
  1122. edit_button.style.paddingRight = '3px';
  1123. edit_button.style.backgroundColor = 'white';
  1124.  
  1125. save_button.textContent = 'Save Template';
  1126. save_button.setAttribute('id', 'save_button');
  1127. save_button.style.height = '18px';
  1128. save_button.style.width = '100px';
  1129. save_button.style.fontSize = '10px';
  1130. save_button.style.paddingLeft = '3px';
  1131. save_button.style.paddingRight = '3px';
  1132. save_button.style.backgroundColor = 'white';
  1133. save_button.style.marginLeft = '5px';
  1134.  
  1135. easy_button.textContent = 'Change Adfly Url';
  1136. easy_button.setAttribute('id', 'easy_button');
  1137. easy_button.style.height = '18px';
  1138. easy_button.style.width = '100px';
  1139. easy_button.style.fontSize = '10px';
  1140. easy_button.style.paddingLeft = '3px';
  1141. easy_button.style.paddingRight = '3px';
  1142. easy_button.style.backgroundColor = 'white';
  1143. easy_button.style.marginLeft = '5px';
  1144.  
  1145. default_button.textContent = ' D ';
  1146. default_button.setAttribute('id', 'default_button');
  1147. default_button.style.height = '18px';
  1148. default_button.style.width = '20px';
  1149. default_button.style.fontSize = '10px';
  1150. default_button.style.paddingLeft = '3px';
  1151. default_button.style.paddingRight = '3px';
  1152. default_button.style.backgroundColor = 'white';
  1153. default_button.style.marginLeft = '5px';
  1154. default_button.title = 'Return default template';
  1155.  
  1156.  
  1157. div.appendChild(edit_button);
  1158. div.appendChild(save_button);
  1159. div.appendChild(default_button);
  1160. div.appendChild(easy_button);
  1161. save_button.disabled = true;
  1162.  
  1163. div.style.display = 'none';
  1164. textarea.addEventListener("click", hide_func(div), false);
  1165. edit_button.addEventListener("click", edit_func(), false);
  1166. save_button.addEventListener("click", save_func(), false);
  1167. default_button.addEventListener("click", default_func(), false);
  1168. easy_button.addEventListener("click", easy_func(), false);
  1169. document.body.insertBefore(div, document.body.firstChild);
  1170. easy_button.style.paddingRight = '3px';
  1171. easy_button.style.backgroundColor = 'white';
  1172. easy_button.style.marginLeft = '5px';
  1173.  
  1174. default_button.textContent = ' D ';
  1175. default_button.setAttribute('id', 'default_button');
  1176. default_button.style.height = '18px';
  1177. default_button.style.width = '20px';
  1178. default_button.style.fontSize = '10px';
  1179. default_button.style.paddingLeft = '3px';
  1180. default_button.style.paddingRight = '3px';
  1181. default_button.style.backgroundColor = 'white';
  1182. default_button.style.marginLeft = '5px';
  1183. default_button.title = 'Return default template';
  1184.  
  1185.  
  1186. div.appendChild(edit_button);
  1187. div.appendChild(save_button);
  1188. div.appendChild(default_button);
  1189. div.appendChild(easy_button);
  1190. save_button.disabled = true;
  1191.  
  1192. div.style.display = 'none';
  1193. textarea.addEventListener("click", hide_func(div), false);
  1194. edit_button.addEventListener("click", edit_func(), false);
  1195. save_button.addEventListener("click", save_func(), false);
  1196. default_button.addEventListener("click", default_func(), false);
  1197. easy_button.addEventListener("click", easy_func(), false);
  1198. document.body.insertBefore(div, document.body.firstChild);