HIT Scraper WITH EXPORT

Snag HITs.

当前为 2014-06-04 提交的版本,查看 最新版本

  1. // ==UserScript==
  2. // @name HIT Scraper WITH EXPORT
  3. // @author Kerek and TJ
  4. // @description Snag HITs.
  5. // Based in part on code from mmmturkeybacon Export Mturk History and mmmturkeybacon Color Coded Search with Checkpoints
  6. // @namespace http://userscripts.org/users/536998
  7. // @match https://www.mturk.com/mturk/findhits?match=true#hit_scraper*
  8. // @match https://www.mturk.com/mturk/findhits?match=true?hit_scraper*
  9. // @version 1.3.0.4
  10. // @grant GM_xmlhttpRequest
  11. // @grant GM_getValue
  12. // @grant GM_setValue
  13. // @grant GM_deleteValue
  14. // @require http://code.jquery.com/jquery-latest.min.js
  15. // ==/UserScript==
  16.  
  17. //alter the requester ignore last as you desire, just follow the format below and use EXACT capitalization e.g., CrowdSource
  18. var ignore_list = ["oscar smith", "jonathon weber", "jerry torres", "crowdsource", "we-pay-you-fast", "turk experiment", "jon brelig"];
  19.  
  20. //this searches extra pages if you skip too much, helps fill out results if you hit a chunk of ignored HITs. Change to true for this behavior.
  21. var correct_for_skips = false;
  22.  
  23. //weight the four TO ratings for the coloring. Default has pay twice as important as fairness and nothing for communication and fast.
  24. var COMM_WEIGHT = 0;
  25. var PAY_WEIGHT = 10;
  26. var FAIR_WEIGHT = 5;
  27. var FAST_WEIGHT = 0;
  28.  
  29. //display your hitdb records if applicable
  30. var check_hitDB = true;
  31.  
  32. //default text size
  33. var default_text_size=11;
  34.  
  35.  
  36.  
  37. var HITStorage = {};
  38. var indexedDB = window.indexedDB || window.webkitIndexedDB ||
  39. window.mozIndexedDB;
  40. window.IDBTransaction = window.IDBTransaction || window.webkitIDBTransaction || window.mozIDBTransaction;
  41. window.IDBKeyRange = window.IDBKeyRange || window.webkitIDBKeyRange || window.mozIDBKeyRange;
  42. HITStorage.IDBTransactionModes = { "READ_ONLY": "readonly", "READ_WRITE": "readwrite", "VERSION_CHANGE": "versionchange" };
  43. var IDBKeyRange = window.IDBKeyRange;
  44.  
  45. HITStorage.indexedDB = {};
  46. HITStorage.indexedDB = {};
  47. HITStorage.indexedDB.db = null;
  48.  
  49. HITStorage.indexedDB.onerror = function(e) {
  50. console.log(e);
  51. };
  52.  
  53. var v=4;
  54.  
  55. HITStorage.indexedDB.checkTitle = function(title,button) {
  56. var request = indexedDB.open("HITDB", v);
  57. request.onsuccess = function(e) {
  58. HITStorage.indexedDB.db = e.target.result;
  59. var db = HITStorage.indexedDB.db;
  60. if (!db.objectStoreNames.contains("HIT"))
  61. {
  62. db.close();
  63. return;
  64. }
  65. var trans = db.transaction(["HIT"], HITStorage.IDBTransactionModes.READ_ONLY);
  66. var store = trans.objectStore("HIT");
  67.  
  68. var index = store.index("title");
  69. index.get(title).onsuccess = function(event)
  70. {
  71. if (event.target.result === undefined)
  72. {
  73. console.log(title + ' not found');
  74. history[button].titledb=false;
  75. }
  76. else
  77. {
  78. console.log(title + ' found');
  79. history[button].titledb=true;
  80. }
  81. db.close();
  82. };
  83. };
  84. request.onerror = HITStorage.indexedDB.onerror;
  85. };
  86.  
  87. HITStorage.indexedDB.checkRequester = function(id,button) {
  88. var request = indexedDB.open("HITDB", v);
  89. request.onsuccess = function(e) {
  90. HITStorage.indexedDB.db = e.target.result;
  91. var db = HITStorage.indexedDB.db;
  92. if (!db.objectStoreNames.contains("HIT"))
  93. {
  94. db.close();
  95. return;
  96. }
  97. var trans = db.transaction(["HIT"], HITStorage.IDBTransactionModes.READ_ONLY);
  98. var store = trans.objectStore("HIT");
  99.  
  100. var index = store.index("requesterId");
  101. index.get(id).onsuccess = function(event)
  102. {
  103. if (event.target.result === undefined)
  104. {history[button].reqdb=false;
  105. console.log(id + ' not found');
  106. }
  107. else
  108. {
  109. history[button].reqdb=true;
  110. console.log(id + ' found');
  111. }
  112. db.close();
  113. };
  114. };
  115. request.onerror = HITStorage.indexedDB.onerror;
  116. };
  117.  
  118. var PAGES_TO_SCRAPE = 3;
  119. var MINIMUM_HITS = 100;
  120. var SEARCH_REFRESH=0;
  121. var URL_BASE = "/mturk/searchbar?searchWords=&selectedSearchType=hitgroups";
  122. var initial_url = URL_BASE;
  123. var TO_REQ_URL = "http://turkopticon.ucsd.edu/reports?id=";
  124. var found_key_list=[];
  125. var last_clear_time = new Date().getTime();
  126. var searched_once = false;
  127. var save_new_results_time = 120;
  128. var save_results_time = 3600;
  129. var default_type = 0;
  130. var cur_loc = window.location.href;
  131. var time_input = document.createElement("INPUT");
  132. time_input.value = 0;
  133. var page_input = document.createElement("INPUT");
  134. page_input.value = 3;
  135. var min_input = document.createElement("INPUT");
  136. var new_time_display_input = document.createElement("INPUT");
  137. new_time_display_input.value = 300;
  138. var reward_input = document.createElement("INPUT");
  139. var qual_input = document.createElement("INPUT");
  140. qual_input.type = "checkbox";
  141. qual_input.checked = true;
  142. var masters_input = document.createElement("INPUT");
  143. masters_input.type = "checkbox";
  144. var sort_input1 = document.createElement("INPUT");
  145. sort_input1.type = "radio";
  146. sort_input1.name = "sort_type";
  147. sort_input1.value = "latest";
  148. sort_input1.checked = true;
  149. var sort_input2 = document.createElement("INPUT");
  150. sort_input2.type = "radio";
  151. sort_input2.name = "sort_type";
  152. sort_input2.value = "most";
  153. var sort_input3 = document.createElement("INPUT");
  154. sort_input3.type = "radio";
  155. sort_input3.name = "sort_type";
  156. sort_input3.value = "amount";
  157.  
  158. var search_input = document.createElement("INPUT");
  159.  
  160. var LINK_BASE = "https://www.mturk.com";
  161. var BACKGROUND_COLOR = "rgb(19, 19, 19)";
  162. var STATUSDETAIL_DELAY = 250;
  163. var MPRE_DELAY = 3000;
  164.  
  165. var next_page = 1;
  166.  
  167. var GREEN = '#66CC66'; // > 4
  168. var LIGHTGREEN = '#ADFF2F'; // > 3 GREEN YELLOW
  169. var YELLOW = '#FFD700';
  170. var ORANGE = '#FF9900'; // > 2
  171. var RED = '#FF3030'; // <= 2
  172. var BLUE = '#C0D9D9'; // no TO
  173. var GREY = 'lightGrey';
  174. var BROWN = '#94704D';
  175. var DARKGREY = '#9F9F9F';
  176. $('body').css('background', BACKGROUND_COLOR);
  177.  
  178. var API_PROXY_BASE = 'https://api.turkopticon.istrack.in/';
  179. var API_MULTI_ATTRS_URL = API_PROXY_BASE + 'multi-attrs.php?ids=';
  180. var REVIEWS_BASE = 'http://turkopticon.ucsd.edu/';
  181.  
  182. var control_panel_HTML = '<div id="control_panel" style="margin: 0 auto 0 auto;' +
  183. 'border-bottom: 1px solid #000000; margin-bottom: 5px; ' +
  184. 'background-color: ' + BACKGROUND_COLOR + ';"></div>';
  185. $('body > :not(#control_panel)').hide(); //hide all nodes directly under the body
  186. $('body').prepend(control_panel_HTML);
  187.  
  188. var control_panel = document.getElementById("control_panel");
  189. var big_red_button = document.createElement("BUTTON");
  190. var progress_report = document.createTextNode("Stopped");
  191. var text_area = document.createElement("TABLE");
  192. big_red_button.textContent = "Show Interface";
  193. big_red_button.onclick = function(){show_interface();};
  194. control_panel.appendChild(big_red_button);
  195.  
  196. show_interface();
  197.  
  198. var global_run = false;
  199. var statusdetail_loop_finished = false;
  200. var date_header = "";
  201. var history = {};
  202. var wait_loop;
  203.  
  204. function set_progress_report(text, force)
  205. {
  206. if (global_run == true || force == true)
  207. {
  208. progress_report.textContent = text;
  209. }
  210. }
  211.  
  212. function get_progress_report()
  213. {
  214. return progress_report.textContent;
  215. }
  216.  
  217. function wait_until_stopped()
  218. {
  219. if (global_run == true)
  220. {
  221. if (statusdetail_loop_finished == true)
  222. {
  223. big_red_button.textContent = "Start";
  224. set_progress_report("Finished", false);
  225. }
  226. else
  227. {
  228. setTimeout(function(){wait_until_stopped();}, 500);
  229. }
  230. }
  231. }
  232.  
  233. function display_wait_time(wait_time)
  234. {
  235. if (global_run == true)
  236. {
  237. var current_progress = get_progress_report();
  238. if (current_progress.indexOf("Searching again in")!==-1)
  239. {
  240. set_progress_report(current_progress.replace(/Searching again in \d+ seconds/ , "Searching again in " + wait_time + " seconds"),false);
  241. }
  242. else
  243. set_progress_report(current_progress + " Searching again in " + wait_time + " seconds.", false);
  244. if (wait_time>1)
  245. setTimeout(function(){display_wait_time(wait_time-1);}, 1000);
  246. }
  247. }
  248.  
  249. function dispArr(ar)
  250. {
  251. var disp = "";
  252. for (var z = 0; z < ar.length; z++)
  253. {
  254. disp += "id " + z + " is " + ar[z] + " ";
  255. }
  256. console.log(disp);
  257. }
  258.  
  259. function scrape($src)
  260. {
  261. var $requester = $src.find('a[href^="/mturk/searchbar?selectedSearchType=hitgroups&requester"]');
  262. var $title = $src.find('a[class="capsulelink"]');
  263. var $reward = $src.find('span[class="reward"]');
  264. var $preview = $src.find('a[href^="/mturk/preview?"]');
  265. var $qualified = $src.find('a[href^="/mturk/notqualified?"]');
  266. var $times = $src.find('a[id^="duration_to_complete"]');
  267. var $descriptions = $src.find('a[id^="description"]');
  268. var not_qualified_group_IDs=[];
  269. $qualified.each(function(){
  270. var groupy = $(this).attr('href');
  271. groupy = groupy.replace("/mturk/notqualified?hitId=","");
  272. not_qualified_group_IDs.push(groupy);
  273. });
  274. var $mixed = $src.find('a[href^="/mturk/preview?"],a[href^="/mturk/notqualified?"]');
  275. var listy =[];
  276. $mixed.each(function(){
  277. var groupy = $(this).attr('href');
  278. groupy = groupy.replace("/mturk/notqualified?hitId=","");
  279. groupy = groupy.replace("/mturk/preview?groupId=","");
  280. listy.push(groupy);
  281. });
  282. listy = listy.filter(function(elem, pos) {
  283. return listy.indexOf(elem) == pos;
  284. });
  285.  
  286. for (var j = 0; j < $requester.length; j++)
  287. {
  288. var $hits = $requester.eq(j).parent().parent().parent().parent().parent().parent().find('td[class="capsule_field_text"]');
  289. var requester_name = $requester.eq(j).text().trim();
  290. var requester_link = $requester.eq(j).attr('href');
  291. var group_ID=listy[j];
  292. var preview_link = "/mturk/preview?groupId=" + group_ID;
  293. var title = $title.eq(j).text().trim();
  294. var reward = $reward.eq(j).text().trim();
  295. var hits = $hits.eq(4).text().trim();
  296. var time = $times.eq(j).parent()[0].nextSibling.nextSibling.innerHTML;
  297. var description = $descriptions.eq(j).parent()[0].nextSibling.nextSibling.innerHTML;
  298. console.log(description);
  299. var requester_id = requester_link.replace('/mturk/searchbar?selectedSearchType=hitgroups&requesterId=','');
  300. var accept_link;
  301. accept_link = preview_link.replace('preview','previewandaccept');
  302. /*HIT SCRAPER ADDITION*/
  303. var qElements = document.getElementById('qualificationsRequired.tooltip--' + j).parentNode.parentNode.parentNode.getElementsByTagName('tr');
  304.  
  305. var qualifications = [];
  306. for (var i = 1; i < qElements.length; i++) {
  307. qualifications.push((qElements[i].childNodes[1].textContent.trim().replace(/\s+/g, ' ').indexOf("Masters") != -1 ? "[color=red][b]"+qElements[i].childNodes[1].textContent.trim().replace(/\s+/g, ' ')+"[/b][/color]" : qElements[i].childNodes[1].textContent.trim().replace(/\s+/g, ' ')));
  308. }
  309. var qualList = (qualifications.join(', ') ? qualifications.join(', ') : "None");
  310.  
  311. key = requester_name+title+reward+group_ID;
  312. found_key_list.push(key);
  313. if (history[key] == undefined)
  314. {
  315. history[key] = {requester:"", title:"", description:"", reward:"", hits:"", req_link:"", quals:"", prev_link:"", rid:"", acc_link:"", new_result:"", qualified:"", found_this_time:"", initial_time:"", reqdb:"",titledb:"",time:""};
  316. history[key].req_link = requester_link;
  317. history[key].prev_link = preview_link;
  318. history[key].requester = requester_name;
  319. history[key].title = title;
  320. history[key].reward = reward;
  321. history[key].hits = hits;
  322. history[key].rid = requester_id;
  323. history[key].acc_link = accept_link;
  324. history[key].time = time;
  325. history[key].quals = qualList;
  326. history[key].description = description;
  327. HITStorage.indexedDB.checkRequester(requester_id,key);
  328. HITStorage.indexedDB.checkTitle(title,key);
  329. if (searched_once)
  330. {
  331. history[key].initial_time = new Date().getTime();//-1000*(save_new_results_time - SEARCH_REFRESH);
  332. history[key].new_result = 0;
  333. }
  334. else
  335. {
  336. history[key].initial_time = new Date().getTime()-1000*save_new_results_time;
  337. history[key].new_result = 1000*save_new_results_time;
  338. }
  339. if (not_qualified_group_IDs.indexOf(group_ID)!==-1)
  340. history[key].qualified = false;
  341. else
  342. history[key].qualified = true;
  343.  
  344. history[key].found_this_time = true;
  345. }
  346. else
  347. {
  348. history[key].new_result = new Date().getTime() - history[key].initial_time;
  349. history[key].found_this_time = true;
  350. history[key].hits = hits;
  351. }
  352. }
  353. }
  354.  
  355. function statusdetail_loop(next_URL)
  356. {
  357. if (global_run == true)
  358. {
  359. if (next_URL.length != 0)
  360. {
  361. $.get(next_URL, function(data)
  362. {
  363. var $src = $(data);
  364. var maxpagerate = $src.find('td[class="error_title"]:contains("You have exceeded the maximum allowed page request rate for this website.")');
  365. if (maxpagerate.length == 0)
  366. {
  367. set_progress_report("Processing page " + next_page, false);
  368. scrape($src);
  369. $next_URL = $src.find('a[href^="/mturk/viewsearchbar"]:contains("Next")');
  370. next_URL = ($next_URL.length != 0) ? $next_URL.attr("href") : "";
  371. next_page++;
  372. if (default_type == 1)
  373. {
  374. var hmin = MINIMUM_HITS+1;
  375. for (j = 0; j < found_key_list.length; j++)
  376. {
  377. if (history[found_key_list[j]].hits < hmin)
  378. {
  379. next_URL = "";
  380. next_page = -1;
  381. break;
  382. }
  383. }
  384. }
  385. else if (next_page > PAGES_TO_SCRAPE && correct_for_skips)
  386. {
  387. var skipped_hits = 0;
  388. var added_pages = 0;
  389. for (j = 0; j < found_key_list.length; j++)
  390. {
  391. var obj = history[found_key_list[j]];
  392. if (! ignore_check(obj.requester,obj.title))
  393. skipped_hits++;
  394. }
  395. added_pages = Math.floor(skipped_hits/10);
  396. if (skipped_hits%10 >6)
  397. added_pages++;
  398. if (next_page > PAGES_TO_SCRAPE + added_pages)
  399. {
  400. next_URL = "";
  401. next_page = -1;
  402. }
  403. }
  404. else if (next_page > PAGES_TO_SCRAPE)
  405. {
  406. next_URL = "";
  407. next_page = -1;
  408. }
  409. setTimeout(function(){statusdetail_loop(next_URL);}, STATUSDETAIL_DELAY);
  410. }
  411. else
  412. {
  413. console.log("MPRE");
  414. setTimeout(function(){statusdetail_loop(next_URL);}, MPRE_DELAY);
  415. }
  416. });
  417. }
  418. else
  419. {
  420. searched_once = true;
  421. var found_hits = found_key_list.length;
  422. var shown_hits = 0;
  423. var new_hits = 0;
  424. var url = API_MULTI_ATTRS_URL;
  425. var rids = [];
  426. var lastRow = text_area.rows.length - 1;
  427. for (i = lastRow; i>0; i--)
  428. text_area.deleteRow(i);
  429. for (j = 0; j < found_key_list.length; j++)
  430. {
  431. //(function(url,rids,j) {
  432. var obj = history[found_key_list[j]];
  433. if (ignore_check(obj.requester,obj.title) && obj.found_this_time){
  434. ++shown_hits;
  435. //console.log(obj);
  436. //hit export will update col_heads[1]
  437. var col_heads = ["<a href='"+ LINK_BASE+obj.req_link +"' target='_blank'>" + obj.requester + "</a>","<a href='"+ LINK_BASE+obj.prev_link +"' target='_blank'>" + obj.title + "</a>",obj.reward,obj.hits,"TO down","<a href='"+ LINK_BASE+obj.acc_link +"' target='_blank'>Accept</a>"];
  438. var row = text_area.insertRow(text_area.rows.length);
  439. url += obj.rid + ',';
  440. rids.push(obj.rid);
  441. if (check_hitDB)
  442. {
  443. col_heads.push("R");
  444. col_heads.push("T");
  445. }
  446. if (!obj.qualified)
  447. {
  448. col_heads.push("Not Qualified");
  449. }
  450. for (i=0; i<col_heads.length; i++)
  451. {
  452. var this_cell = row.insertCell(i);
  453. row.cells[i].style.fontSize = default_text_size;
  454. this_cell.innerHTML = col_heads[i];
  455. if(i>1)
  456. this_cell.style.textAlign = 'center';
  457. if (check_hitDB)
  458. {
  459. if (i==6)
  460. {
  461. if (obj.reqdb)
  462. this_cell.style.backgroundColor = GREEN;
  463. else
  464. this_cell.style.backgroundColor = RED;
  465. }
  466. else if (i==7)
  467. {
  468. if (obj.titledb)
  469. this_cell.style.backgroundColor = GREEN;
  470. else
  471. this_cell.style.backgroundColor = RED;
  472. }
  473. else if (i==8)
  474. this_cell.style.backgroundColor = DARKGREY;
  475. }
  476. else if (i==6)
  477. this_cell.style.backgroundColor = DARKGREY;
  478. }
  479. if (Object.keys(history).length>0)
  480. {
  481. if (obj.new_result < 1000*save_new_results_time)
  482. {
  483. new_hits++;
  484. for (i in col_heads)
  485. {
  486. row.cells[i].style.fontSize = default_text_size + 1;
  487. row.cells[i].style.fontWeight = "bold";
  488. }
  489. }
  490. }
  491. button = document.createElement('button'); //HIT SCRAPER ADDITION
  492. button.textContent = 'vB';
  493. button.title = 'Export this HIT description as vBulletin formatted text';
  494. button.style.height = '14px';
  495. button.style.width = '30px';
  496. button.style.fontSize = '8px';
  497. button.style.border = '1px solid';
  498. button.style.padding = '0px';
  499. button.style.backgroundColor = 'transparent';
  500. //button.addEventListener("click", function() {export_func_deleg(j);}.bind(null,j), false);
  501. button.addEventListener("click", (function (obj,j) { return function() {export_func_deleg(obj,j);}})(obj,j));
  502. row.cells[1].appendChild(button);
  503. }
  504. //});
  505. }
  506. set_progress_report("Scrape complete. " + shown_hits + " HITs found (" + new_hits + " new results). " + (found_hits - shown_hits) + " HITs ignored.", false);
  507. url = url.substring(0,url.length - 1);
  508. //console.log(url);
  509. var success_flag = false;
  510. GM_xmlhttpRequest(
  511. {
  512. method: "GET",
  513. url: url,
  514. onload: function (results)
  515. {
  516. //console.log(results.responseText);
  517. rdata = $.parseJSON(results.responseText);
  518. for (i = 0; i < rids.length; i++)
  519. {
  520. text_area.rows[i+1].style.backgroundColor = GREY;
  521. if (rdata[rids[i]])
  522. {
  523. var pay = rdata[rids[i]].attrs.pay
  524. var reviews = rdata[rids[i]].reviews
  525. var average = 0;
  526. var sum = 0;
  527. var divisor = 0;
  528. var comm = rdata[rids[i]].attrs.comm;
  529. var fair = rdata[rids[i]].attrs.fair;
  530. var fast = rdata[rids[i]].attrs.fast;
  531. if (comm > 0)
  532. {
  533. sum += COMM_WEIGHT*comm;
  534. divisor += COMM_WEIGHT;
  535. }
  536. if (pay > 0)
  537. {
  538. sum += PAY_WEIGHT*pay;
  539. divisor += PAY_WEIGHT;
  540. }
  541. if (fair > 0)
  542. {
  543. sum += FAIR_WEIGHT*fair;
  544. divisor += FAIR_WEIGHT;
  545. }
  546. if (fast > 0)
  547. {
  548. sum += FAST_WEIGHT*fast;
  549. divisor += FAST_WEIGHT;
  550. }
  551. if (divisor > 0)
  552. {
  553. average = sum/divisor;
  554. }
  555. text_area.rows[i+1].cells[4].innerHTML = "<a href='"+ TO_REQ_URL+rids[i] +"' target='_blank'>" + pay + "</a>";
  556. if (reviews > 4)
  557. {
  558. if (average > 4.49)
  559. text_area.rows[i+1].style.backgroundColor = GREEN;
  560. else if (average > 3.49)
  561. text_area.rows[i+1].style.backgroundColor = LIGHTGREEN;
  562. //else if (average > 2.99)
  563. // text_area.rows[i+1].style.backgroundColor = YELLOW;
  564. else if (average > 1.99)
  565. text_area.rows[i+1].style.backgroundColor = ORANGE;
  566. else if (average > 0)
  567. text_area.rows[i+1].style.backgroundColor = RED;
  568. }
  569. }
  570. else
  571. {
  572. text_area.rows[i+1].cells[4].innerHTML = "No data";
  573. }
  574. }
  575. success_flag = true;
  576. }
  577. });
  578. if (!success_flag)
  579. for (i = 0; i < rids.length; i++) text_area.rows[i+1].style.backgroundColor = GREY;
  580. statusdetail_loop_finished = true;
  581. if (SEARCH_REFRESH>0)
  582. {
  583. wait_loop = setTimeout(function(){if (global_run) start_it();}, 1000*SEARCH_REFRESH);
  584. display_wait_time(SEARCH_REFRESH);
  585. }
  586. else
  587. {
  588. global_run = false;
  589. big_red_button.textContent = "Start";
  590. }
  591. }
  592. }
  593. }
  594.  
  595. function ignore_check(r,t){
  596. if (ignore_list.indexOf(r.toLowerCase())==-1)
  597. {
  598. return true;
  599. }
  600. return false;
  601. }
  602.  
  603. function start_running()
  604. {
  605. if (big_red_button.textContent == "Start")
  606. {
  607. global_run = true;
  608. initial_url = URL_BASE;
  609. if (search_input.value.length>0)
  610. {
  611. initial_url = initial_url.replace("searchWords=", "searchWords=" + search_input.value);
  612. }
  613. if (time_input.value.replace(/[^0-9]+/g,"") != "")
  614. {
  615. SEARCH_REFRESH = Number(time_input.value);
  616. }
  617. if (page_input.value.replace(/[^0-9]+/g,"") != "")
  618. {
  619. PAGES_TO_SCRAPE = Number(page_input.value);
  620. }
  621. if (min_input.value.replace(/[^0-9]+/g,"") != "")
  622. {
  623. MINIMUM_HITS = Number(min_input.value);
  624. }
  625. if (new_time_display_input.value.replace(/[^0-9]+/g,"") != "")
  626. {
  627. save_new_results_time = Number(new_time_display_input.value);
  628. }
  629. if (reward_input.value.replace(/[^0-9]+/g,"") != "")
  630. {
  631. initial_url += "&minReward=" + reward_input.value;
  632. }
  633. else
  634. {
  635. initial_url += "&minReward=0.00";
  636. }
  637. if (qual_input.checked)
  638. {
  639. initial_url += "&qualifiedFor=on"
  640. }
  641. else
  642. {
  643. initial_url += "&qualifiedFor=off"
  644. }
  645. if (masters_input.checked)
  646. {
  647. initial_url += "&requiresMasterQual=on"
  648. }
  649. if (sort_input1.checked)
  650. {
  651. initial_url+= "&sortType=LastUpdatedTime%3A1";
  652. default_type = 0;
  653. }
  654. else if (sort_input2.checked)
  655. {
  656. initial_url+= "&sortType=NumHITs%3A1";
  657. default_type = 1;
  658. }
  659. else if (sort_input3.checked)
  660. {
  661. initial_url+= "&sortType=Reward%3A1";
  662. default_type = 0;
  663. }
  664. initial_url+="&pageNumber=1&searchSpec=HITGroupSearch"
  665. start_it();
  666. }
  667. else
  668. {
  669. global_run = false;
  670. clearTimeout(wait_loop);
  671. big_red_button.textContent = "Start";
  672. set_progress_report("Stopped", true);
  673. }
  674. }
  675.  
  676. function start_it()
  677. {
  678. statusdetail_loop_finished = false;
  679. big_red_button.textContent = "Stop";
  680. found_key_list=[];
  681. var ctime = new Date().getTime()
  682. if (ctime - last_clear_time > save_results_time*666)
  683. {
  684. var last_history=history;
  685. history = {};
  686. for (var key in last_history)
  687. {
  688. if (last_history[key].new_result<save_results_time*1000)
  689. {
  690. history[key]=last_history[key];
  691. if (last_history[key].found_this_time)
  692. {
  693. last_history[key].found_this_time = false;
  694. if (last_history[key].new_result>save_new_results_time*1000)
  695. last_history[key].initial_time = ctime-1000*save_new_results_time;
  696. }
  697. }
  698.  
  699. }
  700. last_clear_time = ctime;
  701. }
  702. next_page = 1;
  703. statusdetail_loop(initial_url);
  704. }
  705.  
  706.  
  707. function show_interface()
  708. {
  709. control_panel.style.color = BROWN;
  710. control_panel.style.fontSize = 14;
  711. control_panel.removeChild(big_red_button);
  712. control_panel.appendChild(document.createTextNode("Auto-refresh delay: "));
  713. time_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  714. time_input.title = "Enter search refresh delay in seconds\n" + "Enter 0 for no auto-refresh\n" + "Default is 0 (no auto-refresh)";
  715. time_input.size = 3;
  716. control_panel.appendChild(time_input);
  717. control_panel.appendChild(document.createTextNode(" "));
  718. control_panel.appendChild(document.createTextNode("Pages to scrape: "));
  719. page_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  720. page_input.title = "Enter number of pages to scrape\n" + "Default is 4";
  721. page_input.size = 3;
  722. control_panel.appendChild(page_input);
  723. control_panel.appendChild(document.createTextNode(" "));
  724. control_panel.appendChild(document.createTextNode("Minimum batch size: "));
  725. min_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  726. min_input.title = "Enter minimum HITs for batch search\n" + "Default is 100";
  727. min_input.size = 3;
  728. control_panel.appendChild(min_input);
  729. control_panel.appendChild(document.createTextNode(" "));
  730. control_panel.appendChild(document.createTextNode("New HIT highlighting: "));
  731. new_time_display_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  732. new_time_display_input.title = "Enter time (in seconds) to keep new HITs highlighted\n" + "Default is 300 (5 minutes)";
  733. new_time_display_input.size = 6;
  734. control_panel.appendChild(new_time_display_input);
  735. control_panel.appendChild(document.createElement("P"));
  736. control_panel.appendChild(document.createTextNode("Minimum reward: "));
  737. reward_input.size = 6;
  738. control_panel.appendChild(reward_input);
  739. control_panel.appendChild(document.createTextNode(" "));
  740.  
  741. control_panel.appendChild(document.createTextNode("Qualified"));
  742. control_panel.appendChild(qual_input);
  743. control_panel.appendChild(document.createTextNode(" "));
  744. control_panel.appendChild(document.createTextNode("Masters"));
  745. control_panel.appendChild(masters_input);
  746. control_panel.appendChild(document.createTextNode(" "));
  747. control_panel.appendChild(document.createTextNode("Sort types: "));
  748. control_panel.appendChild(sort_input1);
  749. control_panel.appendChild(document.createTextNode("Latest"));
  750. control_panel.appendChild(sort_input2);
  751. control_panel.appendChild(document.createTextNode("Most Available"));
  752. control_panel.appendChild(sort_input3);
  753. control_panel.appendChild(document.createTextNode("Amount"));
  754. control_panel.appendChild(document.createElement("P"));
  755. control_panel.appendChild(search_input);
  756. search_input.size = 20;
  757. search_input.title = "Enter a search term to include\n" + "Default is blank (no included terms)";
  758. search_input.placeholder="Enter search terms here";
  759. control_panel.appendChild(document.createTextNode(" "));
  760. big_red_button.textContent = "Start";
  761. big_red_button.onclick = function(){start_running();};
  762. control_panel.appendChild(big_red_button);
  763. control_panel.appendChild(document.createTextNode(" "));
  764. control_panel.appendChild(progress_report);
  765. control_panel.appendChild(document.createElement("P"));
  766. text_area.style.fontWeight = 400;
  767. text_area.createCaption().innerHTML = "HITs";
  768. var col_heads = ['Requester','Title','Reward','HITs Available','TO pay',"Accept HIT"];
  769. var row = text_area.createTHead().insertRow(0);
  770. text_area.caption.style.fontWeight = 800;
  771. text_area.caption.style.color = BROWN;
  772. if (default_text_size > 10)
  773. text_area.cellPadding=Math.min(Math.max(1,Math.floor((default_text_size-10)/2)),5);
  774. console.log(text_area.cellPadding);
  775. //text_area.cellPadding=2;
  776. text_area.caption.style.fontSize = 28;
  777. text_area.rows[0].style.fontWeight = 800;
  778. text_area.rows[0].style.color = BROWN;
  779. for (i=0; i<col_heads.length; i++)
  780. {
  781. var this_cell = row.insertCell(i);
  782. this_cell.innerHTML = col_heads[i];
  783. this_cell.style.fontSize = 14;
  784. if (i > 1)
  785. this_cell.style.textAlign = 'center';
  786. }
  787. control_panel.appendChild(text_area);
  788. }
  789.  
  790.  
  791. /********HIT EXPORT ADDITIONS*****/
  792.  
  793. var EDIT = false;
  794. var HIT;
  795.  
  796. var TO_BASE = "http://turkopticon.ucsd.edu/";
  797. var API_BASE = "https://api.turkopticon.istrack.in/";
  798. var API_URL = API_BASE + "multi-attrs.php?ids=";
  799. DEFAULT_TEMPLATE = '[table][tr][td][b]Title:[/b] [url={prev_link}][COLOR=blue]{title}[/COLOR][/url]\n';
  800. DEFAULT_TEMPLATE += '[b]Requester:[/b] [url=https://www.mturk.com/mturk/searchbar?selectedSearchType=hitgroups&requesterId={rid}][COLOR=blue]{requester}[/COLOR][/url]';
  801. DEFAULT_TEMPLATE += ' [{rid}] ([url='+TO_BASE+'{rId}][COLOR=blue]TO[/COLOR][/url])';
  802. DEFAULT_TEMPLATE += '\n[b]TO Ratings:[/b]{to_stuff}';
  803. DEFAULT_TEMPLATE += '\n[b]Description:[/b] {description}';
  804. DEFAULT_TEMPLATE += '\n[b]Time:[/b] {time}';
  805. DEFAULT_TEMPLATE += '\n[b]Reward:[/b] [COLOR=green][b]{reward}[/b][/COLOR]';
  806. DEFAULT_TEMPLATE += '\n[b]Qualifications:[/b] {quals}[/td][/tr][/table]';
  807.  
  808. var TEMPLATE;
  809. var EASYLINK;
  810.  
  811. if (typeof GM_getValue === 'undefined')
  812. TEMPLATE = null;
  813. else {
  814. TEMPLATE = GM_getValue('HITExport Template');
  815. EASYLINK = GM_getValue('HITExport Easylink');
  816. }
  817. if (TEMPLATE == null) {
  818. TEMPLATE = DEFAULT_TEMPLATE;
  819. }
  820.  
  821. function buildXhrUrl(rai) {
  822. var url = API_URL;
  823. var ri = rai;
  824. url += rai;
  825. return url;
  826. }
  827.  
  828. function makeXhrQuery(url) {
  829. var xhr = new XMLHttpRequest();
  830. try{
  831. xhr.open('GET', url, false);
  832. xhr.send(null);
  833. return $.parseJSON(xhr.response);
  834. }
  835. catch(err){
  836. return "TO DOWN";
  837. }
  838. }
  839.  
  840. function getNamesForEmptyResponses(rai, resp) {
  841. for (var rid in rai) {
  842. if (rai.hasOwnProperty(rid) && resp[rid] == "") {
  843. resp[rid] = $.parseJSON('{"name": "' + rai[rid][0].innerHTML + '"}');
  844. }
  845. }
  846. return resp;
  847. }
  848.  
  849. function getKeys(obj) {
  850. var keys = [];
  851. for (var key in obj) {
  852. keys.push(key);
  853. }
  854. return keys;
  855. }
  856.  
  857. function export_func_deleg(item,index) {
  858. console.log(item);
  859. export_func(item);
  860. }
  861.  
  862. function export_func(item) {
  863. HIT = item;
  864. edit_button.textContent = 'Edit Template';
  865. apply_template(item);
  866. div.style.display = 'block';
  867. textarea.select();
  868. }
  869.  
  870. function apply_template(hit_data) {
  871. var txt = TEMPLATE;
  872.  
  873. var vars = ['title', 'requester', 'rid', 'description', 'reward', 'quals', 'prev_link', 'time', 'hits', 'to_stuff', 'to_text'];
  874.  
  875. var resp = null;
  876. if (txt.indexOf('{to_text}') >= 0 || txt.indexOf('{to_stuff}') >= 0){
  877. var url = buildXhrUrl(hit_data["rid"]);
  878. resp = makeXhrQuery(url);
  879. console.log(resp);
  880. }
  881. var toText = "";
  882. var toStuff = "";
  883. var toData = "";
  884. var numResp = (resp == null || resp == "TO DOWN" ? "n/a" : resp[hit_data["rid"]].reviews);
  885. if (resp == "TO DOWN"){
  886. toStuff = " [URL=\""+TO_BASE+hit_data['rid']+"\"]TO down.[/URL]";
  887. toText = toStuff;
  888. }
  889. else if (resp == null || resp[hit_data["rid"]].attrs == null && resp != "TO DOWN") {
  890. toStuff = " No TO ";
  891. toText = " No TO ";
  892. toStuff += "[URL=\""+TO_BASE+"report?requester[amzn_id]=" + hit_data['rid'] + "&requester[amzn_name]=" + hit_data['requester'] + "\"]";
  893. toStuff += "(Submit a new TO rating for this requester)[/URL]";
  894. }
  895. else {
  896. for (var key in resp[hit_data["rid"]].attrs) {
  897. //toText += "\n[*]"+key+": "+resp[hit_data["requesterId"]].attrs[key]+"\n";
  898. var i = 0;
  899. var color = "green";
  900. var name = key;
  901. var num = Math.floor(resp[hit_data["rid"]].attrs[key]);
  902. switch (key){
  903. case "comm":
  904. name = "Communicativity";
  905. break;
  906. case "pay":
  907. name = "Generosity";
  908. break;
  909. case "fast":
  910. name = "Promptness";
  911. break;
  912. case "fair":
  913. name = "Fairness";
  914. break;
  915. default:
  916. name = key;
  917. break;
  918. }
  919. switch (num){
  920. case 0:
  921. color = "red";
  922. break;
  923. case 1:
  924. color = "red";
  925. break;
  926. case 2:
  927. color = "orange";
  928. break;
  929. case 3:
  930. color = "yellow";
  931. break;
  932. default:
  933. break;
  934. }
  935. toText += (num > 0 ? "\n[color="+color+"]" : "\n");
  936. for (i; i < num; i++){
  937. toText += "[b]☢[/b]"
  938. }
  939. toText += (num > 0 ? "[/color]" : "")
  940. if (i < 5){
  941. toText += "[color=white]";
  942. for (i; i < 5; i++)
  943. toText += "[b]☢[/b]";
  944. toText += "[/color]";
  945. }
  946. toText += " "+Number(resp[hit_data["rid"]].attrs[key]).toFixed(2)+" "+name;
  947. toData += Number(resp[hit_data["rid"]].attrs[key]).toFixed(2) + ",";
  948. }
  949. //toText += "[/list]";
  950. toText += (txt.indexOf('{to_stuff}') >= 0 ? "" : "\nNumber of Reviews: "+numResp+"\n[URL=\""+TO_BASE+"report?requester[amzn_id]=" + hit_data['rid'] + "&requester[amzn_name]=" + hit_data['requester'] + "\"](Submit a new TO rating for this requester)[/URL]");
  951. toStuff = '\n[img]http://data.istrack.in/to/' + toData.slice(0,-1) + '.png[/img]';
  952. toStuff += (txt.indexOf('{to_stuff}') >= 0 ? (txt.indexOf('{to_text}') >= 0 ? "" : toText) : "");
  953. toStuff += "\nNumber of Reviews: "+numResp;
  954. toStuff += "[URL=\""+TO_BASE+"report?requester[amzn_id]=" + hit_data['rid'] + "&requester[amzn_name]=" + hit_data['requester'] + "\"]";
  955. toStuff += "\n(Submit a new TO rating for this requester)[/URL]";
  956. }
  957. for (var i = 0; i < vars.length; i++) {
  958. t = new RegExp('\{' + vars[i] + '\}', 'g');
  959. if (vars[i] == "to_stuff") {
  960. txt = txt.replace(t, toStuff);
  961. }
  962. else if (vars[i] == "to_text"){
  963. txt = txt.replace(t, toText);
  964. }
  965. else if (vars[i] == "prev_link"){
  966. txt = txt.replace(t,"https://www.mturk.com/"+hit_data[vars[i]]);
  967. }
  968. else if (vars[i] == "acc_link"){
  969. txt = txt.replace(t,"https://www.mturk.com/"+hit_data[vars[i]]);
  970. }
  971. else
  972. txt = txt.replace(t, hit_data[vars[i]]);
  973. }
  974. textarea.value = txt;
  975. }
  976.  
  977. function hide_func(div) {
  978. if (EDIT == false)
  979. div.style.display = 'none';
  980. }
  981.  
  982. function edit_func() {
  983. if (EDIT == true) {
  984. EDIT = false;
  985. TEMPLATE = textarea.value;
  986. edit_button.textContent = 'Edit Template';
  987. apply_template(HIT);
  988. }
  989. else {
  990. console.log("Editing");
  991. EDIT = true;
  992. edit_button.textContent = 'Show Changes';
  993. save_button.disabled = false;
  994. textarea.value = TEMPLATE;
  995. }
  996. }
  997.  
  998. function default_func() {
  999. GM_deleteValue('HITExport Template');
  1000. TEMPLATE = DEFAULT_TEMPLATE;
  1001. EDIT = false;
  1002. edit_button.textContent = 'Edit Template';
  1003. apply_template(HIT);
  1004. }
  1005.  
  1006. function save_func() {
  1007. if (EDIT)
  1008. TEMPLATE = textarea.value;
  1009. GM_setValue('HITExport Template', TEMPLATE);
  1010. }
  1011.  
  1012. var div = document.createElement('div');
  1013. var textarea = document.createElement('textarea');
  1014. var div2 = document.createElement('label');
  1015.  
  1016. div.style.position = 'fixed';
  1017. div.style.width = '500px';
  1018. div.style.height = '235px';
  1019. div.style.left = '50%';
  1020. div.style.right = '50%';
  1021. div.style.margin = '-250px 0px 0px -250px';
  1022. div.style.top = '300px';
  1023. div.style.padding = '5px';
  1024. div.style.border = '2px';
  1025. div.style.backgroundColor = 'black';
  1026. div.style.color = 'white';
  1027. div.style.zIndex = '100';
  1028.  
  1029. textarea.style.padding = '2px';
  1030. textarea.style.width = '500px';
  1031. textarea.style.height = '200px';
  1032. textarea.title = '{title}\n{requester}\n{rId}\n{description}\n{reward}\n{quals}\n{prev_link}\n{time}\n{hit}\n{to_stuff}\n{to_text}';
  1033.  
  1034. div.textContent = 'Press Ctrl+C to copy to clipboard. Click textarea to close';
  1035. div.style.fontSize = '12px';
  1036. div.appendChild(textarea);
  1037.  
  1038. var edit_button = document.createElement('button');
  1039. var save_button = document.createElement('button');
  1040. var default_button = document.createElement('button');
  1041. var easy_button = document.createElement('button');
  1042.  
  1043. edit_button.textContent = 'Edit Template';
  1044. edit_button.setAttribute('id', 'edit_button');
  1045. edit_button.style.height = '18px';
  1046. edit_button.style.width = '100px';
  1047. edit_button.style.fontSize = '10px';
  1048. edit_button.style.paddingLeft = '3px';
  1049. edit_button.style.paddingRight = '3px';
  1050. edit_button.style.backgroundColor = 'white';
  1051.  
  1052. save_button.textContent = 'Save Template';
  1053. save_button.setAttribute('id', 'save_button');
  1054. save_button.style.height = '18px';
  1055. save_button.style.width = '100px';
  1056. save_button.style.fontSize = '10px';
  1057. save_button.style.paddingLeft = '3px';
  1058. save_button.style.paddingRight = '3px';
  1059. save_button.style.backgroundColor = 'white';
  1060. save_button.style.marginLeft = '5px';
  1061.  
  1062. easy_button.textContent = 'Change Adfly Url';
  1063. easy_button.setAttribute('id', 'easy_button');
  1064. easy_button.style.height = '18px';
  1065. easy_button.style.width = '100px';
  1066. easy_button.style.fontSize = '10px';
  1067. easy_button.style.paddingLeft = '3px';var div = document.createElement('div');
  1068. var textarea = document.createElement('textarea');
  1069. var div2 = document.createElement('label');
  1070.  
  1071. div.style.position = 'fixed';
  1072. div.style.width = '500px';
  1073. div.style.height = '235px';
  1074. div.style.left = '50%';
  1075. div.style.right = '50%';
  1076. div.style.margin = '-250px 0px 0px -250px';
  1077. div.style.top = '300px';
  1078. div.style.padding = '5px';
  1079. div.style.border = '2px';
  1080. div.style.backgroundColor = 'black';
  1081. div.style.color = 'white';
  1082. div.style.zIndex = '100';
  1083.  
  1084. textarea.style.padding = '2px';
  1085. textarea.style.width = '500px';
  1086. textarea.style.height = '200px';
  1087. textarea.title = '{title}\n{requester}\n{rid}\n{description}\n{reward}\n{quals}\n{prev_link}\n{time}\n{hits}\n{to_stuff}\n{to_text}\n{acc_link}';
  1088.  
  1089. div.textContent = 'Press Ctrl+C to copy to clipboard. Click textarea to close';
  1090. div.style.fontSize = '12px';
  1091. div.appendChild(textarea);
  1092.  
  1093. var edit_button = document.createElement('button');
  1094. var save_button = document.createElement('button');
  1095. var default_button = document.createElement('button');
  1096.  
  1097. edit_button.textContent = 'Edit Template';
  1098. edit_button.setAttribute('id', 'edit_button');
  1099. edit_button.style.height = '18px';
  1100. edit_button.style.width = '100px';
  1101. edit_button.style.fontSize = '10px';
  1102. edit_button.style.paddingLeft = '3px';
  1103. edit_button.style.paddingRight = '3px';
  1104. edit_button.style.backgroundColor = 'white';
  1105.  
  1106. save_button.textContent = 'Save Template';
  1107. save_button.setAttribute('id', 'save_button');
  1108. save_button.style.height = '18px';
  1109. save_button.style.width = '100px';
  1110. save_button.style.fontSize = '10px';
  1111. save_button.style.paddingLeft = '3px';
  1112. save_button.style.paddingRight = '3px';
  1113. save_button.style.backgroundColor = 'white';
  1114. save_button.style.marginLeft = '5px';
  1115.  
  1116. default_button.textContent = ' D ';
  1117. default_button.setAttribute('id', 'default_button');
  1118. default_button.style.height = '18px';
  1119. default_button.style.width = '20px';
  1120. default_button.style.fontSize = '10px';
  1121. default_button.style.paddingLeft = '3px';
  1122. default_button.style.paddingRight = '3px';
  1123. default_button.style.backgroundColor = 'white';
  1124. default_button.style.marginLeft = '5px';
  1125. default_button.title = 'Return default template';
  1126.  
  1127.  
  1128. div.appendChild(edit_button);
  1129. div.appendChild(save_button);
  1130. div.appendChild(default_button);
  1131. div.appendChild(easy_button);
  1132. save_button.disabled = true;
  1133.  
  1134. div.style.display = 'none';
  1135. textarea.addEventListener("click", function() {hide_func(div);}, false);
  1136. edit_button.addEventListener("click", function() {edit_func();}, false);
  1137. save_button.addEventListener("click", function() {save_func();}, false);
  1138. default_button.addEventListener("click", function() {default_func();}, false);
  1139. document.body.insertBefore(div, document.body.firstChild);