Mturk Qualification Database and Scraper

Scrape, display, sort and search your Mturk qualifications

当前为 2023-02-06 提交的版本,查看 最新版本

  1. // ==UserScript==
  2. // @name Mturk Qualification Database and Scraper
  3. // @namespace https://greasyfork.org/en/users/1004048-elias041
  4. // @version 0.7
  5. // @description Scrape, display, sort and search your Mturk qualifications
  6. // @author Elias041
  7. // @match https://worker.mturk.com/qualifications/assigned
  8. // @match https://worker.mturk.com/qt
  9. // @require https://code.jquery.com/jquery-3.6.3.js
  10. // @require https://code.jquery.com/ui/1.13.1/jquery-ui.min.js
  11. // @require https://unpkg.com/dexie/dist/dexie.js
  12. // @require https://unpkg.com/ag-grid-community@29.0.0/dist/ag-grid-community.min.js
  13. // @require https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.4/moment.min.js
  14. // @resource https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-grid.css
  15. // @resource https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-theme-apline.css
  16. // @icon https://www.google.com/s2/favicons?sz=64&domain=mturk.com
  17. // @license none
  18. // @grant none
  19. // ==/UserScript==
  20.  
  21.  
  22. let timeout = 1850;
  23. let counter = " ";
  24. let retry_count = 0;
  25. let page = "https://worker.mturk.com/qualifications/assigned.json?page_size=100";
  26. let timeoutId = undefined;
  27. let scraping = false
  28. window.onload = function ()
  29. {
  30.  
  31.  
  32.  
  33. let t = document.getElementsByClassName("col-xs-5 col-md-3 text-xs-right p-l-0")[0],
  34. e = t.parentNode,
  35. o = document.createElement("div");
  36. o.style.color = "#fff";
  37. o.style.padding = "10px";
  38. o.style.boxShadow = "2px 2px 4px #888888";
  39. o.style.background = "#33773A";
  40. o.style.opacity = "0.5";
  41. o.style.cursor = "pointer";
  42. o.id = "button";
  43. o.innerHTML = "Scrape&nbspQuals";
  44. e.insertBefore(o, t);
  45.  
  46. let c = document.createElement("div");
  47. c.style.color = "#fff";
  48. c.style.background = "#C78D99";
  49. c.style.padding = "10px";
  50. c.style.boxShadow = "2px 2px 4px #888888";
  51. c.style.background = "#383c44";
  52. c.style.opacity = "0.5";
  53. c.style.cursor = "pointer";
  54. c.innerHTML = "Cancel";
  55. c.id = "cancelButton";
  56. e.insertBefore(c, t);
  57.  
  58.  
  59. let d = document.createElement("div");
  60. d.style.color = "#fff";
  61. d.style.background = "#fc0f03";
  62. d.style.padding = "10px";
  63. d.style.boxShadow = "2px 2px 4px #888888";
  64. d.style.background = "#323552";
  65. d.style.opacity = "0.5";
  66. d.style.cursor = "pointer";
  67. d.innerHTML = "Database";
  68. d.id = "dbButton";
  69. e.insertBefore(d, t);
  70.  
  71. let f = document.createElement("div");
  72. f.style.color = "#fff";
  73. f.style.padding = "10px";
  74. f.style.boxShadow = "2px 2px 4px #888888";
  75. f.style.background = "#33773A";
  76. f.style.opacity = "0.5";
  77. f.id = "progress";
  78. f.innerHTML = "-";
  79. e.insertBefore(f, t);
  80.  
  81. document.getElementById("dbButton").addEventListener("click", function e()
  82. {
  83. window.open("https://worker.mturk.com/qt", "_blank")
  84. })
  85.  
  86.  
  87.  
  88. document.getElementById("cancelButton").addEventListener("click", function e()
  89. {
  90.  
  91. scraping = false
  92. $("#cancelButton").css('background', '#383c44')
  93. $("#button").css('background', '#33773A')
  94. $("#progress").html("-")
  95. })
  96. document.getElementById("button").addEventListener("click", function e()
  97. {
  98. scraping = true;
  99. $("#button").css('background', '#383c44')
  100. $("#cancelButton").css('background', '#CE3132')
  101.  
  102.  
  103.  
  104. /*init db*/
  105. var db = new Dexie("qualifications");
  106. db.version(1).stores(
  107. {
  108. quals: `
  109. id,
  110. requester,
  111. description,
  112. score,
  113. date,
  114. qualName,
  115. reqURL,
  116. reqQURL,
  117. retURL,
  118. canRetake,
  119. hasTest,
  120. canRequest,
  121. isSystem`
  122. });
  123.  
  124. /*main loop*/
  125. function getAssignedQualifications(nextPageToken = "")
  126. {
  127. if (!scraping)
  128. {
  129. return;
  130. } //cancel trap
  131. counter++
  132. $("#progress").html(counter);
  133. //$("#progressBar").html("&nbsp&nbsp&nbspProcessing&nbsppage&nbsp" + counter + "&nbsp&nbsp&nbsp");
  134. //console.log("scraping")
  135. $.getJSON(page)
  136.  
  137. .then(function (data)
  138. {
  139. data.assigned_qualifications.forEach(function (t)
  140. {
  141. db.quals.bulkAdd([
  142. {
  143. id: t.request_qualification_url,
  144. requester: t.creator_name,
  145. description: t.description,
  146. canRetake: t.can_retake_test_or_rerequest,
  147. retry: t.earliest_retriable_time,
  148. score: t.value,
  149. date: t.grant_time,
  150. qualName: t.name,
  151. reqURL: t.creator_url,
  152. retURL: t.retake_test_url,
  153. isSystem: t.is_system_qualification,
  154. canRequest: t.is_requestable,
  155. hasTest: t.has_test
  156. }])
  157. })
  158.  
  159. if (data.next_page_token !== null)
  160. {
  161. timeoutId = setTimeout(() =>
  162. {
  163. page = `https://worker.mturk.com/qualifications/assigned.json?page_size=100&next_token=${encodeURIComponent(data.next_page_token)}`
  164. getAssignedQualifications(data.next_page_token);
  165. }, timeout);
  166.  
  167.  
  168. }
  169. else
  170. {
  171. console.log("Scraping completed");
  172. console.log(counter + "pages");
  173. console.log("Timeout" + timeout);
  174. console.log(retry_count + "timeouts");
  175. $("#cancelButton").css('background', '#383c44');
  176. $("#progress").css('background', '#25dc12');
  177. $("#progress").html('✓');
  178. $("#dbButton$").css('background', '#25dc12');
  179. return;
  180.  
  181. }
  182. })
  183.  
  184. .catch(function (error)
  185. { //handle timeouts
  186. if (error.status === 429 && retry_count < 10)
  187. {
  188. retry_count++;
  189. timeout += 500;
  190. console.log("timed out, incrementing clock to " + timeout + " milliseconds")
  191. setTimeout(() =>
  192. {
  193. getAssignedQualifications(nextPageToken);
  194. }, 10000);
  195. }
  196. else
  197. {
  198. // $("#progressBar").html("Timed&nbspout&nbsp5&nbsptimes,&nbspaborting.&nbsp" +
  199. // timeout + "&nbspmilliseconds.");
  200. //alert("Timed out 10 times. Bailing out.");
  201. console.log("Timed out 10 times, aborting. " + timeout + " milliseconds.");
  202.  
  203. }
  204. /* $("#button").html("Retry?");
  205. $("#button").css("background-color", "#e80c0f");
  206. document.getElementById("button").addEventListener("click", function e() {
  207. location.reload()
  208.  
  209.  
  210.  
  211. }*/
  212.  
  213. }
  214.  
  215. )
  216.  
  217. }
  218.  
  219. getAssignedQualifications();
  220.  
  221. }
  222.  
  223.  
  224. )
  225.  
  226. };
  227.  
  228. /*ag-grid*/
  229. if (location.href === "https://worker.mturk.com/qt")
  230. {
  231. document.body.innerHTML = "";
  232. let gridDiv = document.createElement("div");
  233. gridDiv.setAttribute("id", "gridDiv");
  234. document.body.appendChild(gridDiv);
  235. document.title = "Qualifications";
  236.  
  237.  
  238. /*init db*/
  239. var db = new Dexie("qualifications");
  240. db.version(1).stores(
  241. {
  242. quals: `
  243. id,
  244. requester,
  245. description,
  246. score,
  247. date,
  248. qualName,
  249. reqURL,
  250. reqQURL,
  251. retURL,
  252. canRetake,
  253. hasTest,
  254. canRequest,
  255. isSystem`
  256. });
  257.  
  258. gridDiv.innerHTML = `
  259. <div id="myGrid" class="ag-theme-alpine">
  260. <style>
  261. .ag-theme-alpine {
  262. --ag-grid-size: 3px;
  263. width: 100%;
  264. height: 100%;
  265. position: absolute;
  266. top: 0;
  267. left: 0;
  268. right: 0;
  269. bottom: 0;
  270. }
  271. </style>
  272. </div>`
  273.  
  274.  
  275. const gridOptions = {
  276. columnDefs: [
  277. {
  278. headerName: 'Mturk Qualification Database and Scraper',
  279. children: [
  280. {
  281. field: "qualName"
  282. },
  283. {
  284. field: "requester"
  285. }]
  286. },
  287.  
  288.  
  289. {
  290. headerName: ' ',
  291. children: [
  292. {
  293.  
  294.  
  295. field: "description",
  296. width: 350
  297. },
  298. {
  299. headerName: "Value",
  300. field: "score",
  301. width: 100
  302. },
  303. {
  304. headerName: "Date",
  305. field: "date",
  306. width: 100,
  307. valueGetter: function (params)
  308. {
  309. var date = new Date(params.data.date);
  310. return (date.getMonth() + 1) + "/" + date.getDate() + "/" + date.getFullYear();
  311. },
  312. comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  313. {
  314. var dateA = new Date(valueA);
  315. var dateB = new Date(valueB);
  316. return dateA - dateB;
  317. },
  318. //valueFormatter: function(params) {
  319. // return new Date(params.value).toString().substring(4, 15);
  320. //}
  321. },
  322. {
  323.  
  324. headerName: "Requester ID",
  325. width: 150,
  326. field: "reqURL",
  327. valueFormatter: function (params)
  328. {
  329. var parts = params.value.split("/");
  330. return parts[2];
  331.  
  332. },
  333.  
  334. },
  335. {
  336. headerName: "Qual ID",
  337. field: "id",
  338.  
  339. valueFormatter: function (params)
  340. {
  341. if (!params.value || params.value === '') return '';
  342. var parts = params.value.split("/");
  343. return parts[2];
  344.  
  345. }
  346. }]
  347. },
  348. {
  349. headerName: 'More',
  350. children: [
  351. {
  352. headerName: " ",
  353. field: " ",
  354. width: 100,
  355. columnGroupShow: 'closed'
  356. },
  357. {
  358. headerName: "Retake",
  359. field: "canRetake",
  360. width: 100,
  361. columnGroupShow: 'open',
  362. suppressMenu: true
  363. },
  364. {
  365. headerName: "hasTest",
  366. field: "hasTest",
  367. width: 100,
  368. columnGroupShow: 'open',
  369. suppressMenu: true
  370. },
  371. {
  372. headerName: "canReq",
  373. field: "canRequest",
  374. width: 100,
  375. columnGroupShow: 'open',
  376. suppressMenu: true
  377. },
  378. {
  379. headerName: "System",
  380. field: "isSystem",
  381. width: 100,
  382. columnGroupShow: 'open',
  383. suppressMenu: true
  384. },
  385.  
  386. ]
  387. }
  388. ],
  389. defaultColDef:
  390. {
  391. sortable: true,
  392. filter: true,
  393. editable: true,
  394. resizable: true,
  395. },
  396. rowSelection: 'multiple',
  397. animateRows: true,
  398. rowData: []
  399. };
  400.  
  401. window.addEventListener('load', function ()
  402. {
  403. const gridDiv = document.querySelector('#myGrid');
  404. db.quals.toArray().then(data =>
  405. {
  406.  
  407. var filteredData = data.filter(function (row)
  408. {
  409. return !row.qualName.includes("Exc: [");
  410. });
  411. gridOptions.rowData = filteredData;
  412. new agGrid.Grid(gridDiv, gridOptions);
  413.  
  414. })
  415. })
  416. };
  417.  
  418.  
  419.