Greasy Fork 还支持 简体中文。

Mturk Qualification Database and Scraper

Scrape, display, sort and search your Mturk qualifications

目前為 2023-02-05 提交的版本,檢視 最新版本

  1. // ==UserScript==
  2. // @name Mturk Qualification Database and Scraper
  3. // @namespace https://greasyfork.org/en/users/1004048-elias041
  4. // @version 0.6
  5. // @description Scrape, display, sort and search your Mturk qualifications
  6. // @author Elias041
  7. // @match https://worker.mturk.com/qualifications/assigned
  8. // @match https://worker.mturk.com/qt
  9. // @require https://code.jquery.com/jquery-3.6.3.js
  10. // @require https://code.jquery.com/ui/1.13.1/jquery-ui.min.js
  11. // @require https://unpkg.com/dexie/dist/dexie.js
  12. // @require https://unpkg.com/ag-grid-community@29.0.0/dist/ag-grid-community.min.js
  13. // @require https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.4/moment.min.js
  14. // @resource https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-grid.css
  15. // @resource https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-theme-apline.css
  16. // @icon https://www.google.com/s2/favicons?sz=64&domain=mturk.com
  17. // @license none
  18. // @grant none
  19. // ==/UserScript==
  20.  
  21.  
  22. let timeout = 1850;
  23. let counter = " ";
  24. let retry_count = 0;
  25. let page = "https://worker.mturk.com/qualifications/assigned.json?page_size=100";
  26. let timeoutId = undefined;
  27. let scraping = false
  28. window.onload = function ()
  29. {
  30.  
  31.  
  32.  
  33. let t = document.getElementsByClassName("col-xs-5 col-md-3 text-xs-right p-l-0")[0],
  34. e = t.parentNode,
  35. o = document.createElement("div");
  36. o.style.color = "#fff";
  37. o.style.padding = "10px";
  38. o.style.boxShadow = "2px 2px 4px #888888";
  39. o.style.background = "#33773A";
  40. o.style.opacity = "0.5";
  41. o.style.cursor = "pointer";
  42. o.id = "button";
  43. o.innerHTML = "Scrape&nbspQuals";
  44. e.insertBefore(o, t);
  45.  
  46. let c = document.createElement("div");
  47. c.style.color = "#fff";
  48. c.style.background = "#C78D99";
  49. c.style.padding = "10px";
  50. c.style.boxShadow = "2px 2px 4px #888888";
  51. c.style.background = "#383c44";
  52. c.style.opacity = "0.5";
  53. c.style.cursor = "pointer";
  54. c.innerHTML = "Cancel";
  55. c.id = "cancelButton";
  56. e.insertBefore(c, t);
  57.  
  58.  
  59. let d = document.createElement("div");
  60. d.style.color = "#fff";
  61. d.style.background = "#fc0f03";
  62. d.style.padding = "10px";
  63. d.style.boxShadow = "2px 2px 4px #888888";
  64. d.style.background = "#323552";
  65. d.style.opacity = "0.5";
  66. d.style.cursor = "pointer";
  67. d.innerHTML = "Database";
  68. d.id = "dbButton";
  69. e.insertBefore(d, t);
  70.  
  71. let f = document.createElement("div");
  72. f.style.color = "#fff";
  73. f.style.padding = "10px";
  74. f.style.boxShadow = "2px 2px 4px #888888";
  75. f.style.background = "#33773A";
  76. f.style.opacity = "0.5";
  77. f.id = "progress";
  78. f.innerHTML = "-";
  79. e.insertBefore(f, t);
  80.  
  81. document.getElementById("dbButton").addEventListener("click", function e()
  82. {
  83. window.open("https://worker.mturk.com/qt", "_blank")
  84. })
  85.  
  86.  
  87.  
  88. document.getElementById("cancelButton").addEventListener("click", function e()
  89. {
  90.  
  91. scraping = false
  92. $("#cancelButton").css('background', '#383c44')
  93. $("#button").css('background', '#33773A')
  94. $("#progress").html("-")
  95. })
  96. document.getElementById("button").addEventListener("click", function e()
  97. {
  98. scraping = true;
  99. $("#button").css('background', '#383c44')
  100. $("#cancelButton").css('background', '#CE3132')
  101.  
  102.  
  103.  
  104. /*init db*/
  105. var db = new Dexie("qualifications");
  106. db.version(1).stores(
  107. {
  108. quals: `
  109. id,
  110. requester,
  111. description,
  112. score,
  113. date,
  114. qualName,
  115. reqURL,
  116. reqQURL,
  117. retURL,
  118. canRetake,
  119. hasTest,
  120. canRequest,
  121. isSystem`
  122. });
  123.  
  124. /*main loop*/
  125. function getAssignedQualifications(nextPageToken = "")
  126. {
  127. if (!scraping)
  128. {
  129. return;
  130. } //cancel trap
  131. counter++
  132. $("#progress").html(counter);
  133. //$("#progressBar").html("&nbsp&nbsp&nbspProcessing&nbsppage&nbsp" + counter + "&nbsp&nbsp&nbsp");
  134. //console.log("scraping")
  135. $.getJSON(page)
  136.  
  137. .then(function (data)
  138. {
  139. data.assigned_qualifications.forEach(function (t)
  140. {
  141. db.quals.bulkAdd([
  142. {
  143. id: t.request_qualification_url,
  144. requester: t.creator_name,
  145. description: t.description,
  146. canRetake: t.can_retake_test_or_rerequest,
  147. retry: t.earliest_retriable_time,
  148. score: t.value,
  149. date: t.grant_time,
  150. qualName: t.name,
  151. reqURL: t.creator_url,
  152. retURL: t.retake_test_url,
  153. isSystem: t.is_system_qualification,
  154. canRequest: t.is_requestable,
  155. hasTest: t.has_test
  156. }])
  157. })
  158.  
  159. if (data.next_page_token !== null)
  160. {
  161. timeoutId = setTimeout(() =>
  162. {
  163. page = `https://worker.mturk.com/qualifications/assigned.json?page_size=100&next_token=${encodeURIComponent(data.next_page_token)}`
  164. getAssignedQualifications(data.next_page_token);
  165. }, timeout);
  166.  
  167.  
  168. }
  169. else
  170. {
  171. console.log("Scraping completed");
  172. console.log(counter + "pages");
  173. console.log("Timeout" + timeout);
  174. console.log(retry_count + "timeouts");
  175. $("#cancelButton").css('background', '#383c44');
  176. $("#progress").css('background', '#25dc12');
  177. $("#progress").html('✓');
  178. $("#dbButton$").css('background', '#25dc12');
  179. }
  180. })
  181.  
  182. .catch(function (error)
  183. { //handle timeouts
  184. if (error.status === 429 && retry_count < 10)
  185. {
  186. retry_count++;
  187. timeout += 500;
  188. console.log("timed out, incrementing clock to " + timeout + " milliseconds")
  189. setTimeout(() =>
  190. {
  191. getAssignedQualifications(nextPageToken);
  192. }, 10000);
  193. }
  194. else
  195. {
  196. // $("#progressBar").html("Timed&nbspout&nbsp5&nbsptimes,&nbspaborting.&nbsp" +
  197. // timeout + "&nbspmilliseconds.");
  198. alert("Timed out 10 times. Bailing out.");
  199. console.log("Timed out 10 times, aborting. " + timeout + " milliseconds.");
  200.  
  201. }
  202. /* $("#button").html("Retry?");
  203. $("#button").css("background-color", "#e80c0f");
  204. document.getElementById("button").addEventListener("click", function e() {
  205. location.reload()
  206.  
  207.  
  208.  
  209. }*/
  210.  
  211. }
  212.  
  213. )
  214.  
  215. }
  216.  
  217. getAssignedQualifications();
  218.  
  219. }
  220.  
  221.  
  222. )
  223.  
  224. };
  225.  
  226. /*ag-grid*/
  227. if (location.href === "https://worker.mturk.com/qt")
  228. {
  229. document.body.innerHTML = "";
  230. let gridDiv = document.createElement("div");
  231. gridDiv.setAttribute("id", "gridDiv");
  232. document.body.appendChild(gridDiv);
  233. document.title = "Qualifications";
  234.  
  235.  
  236. /*init db*/
  237. var db = new Dexie("qualifications");
  238. db.version(1).stores(
  239. {
  240. quals: `
  241. id,
  242. requester,
  243. description,
  244. score,
  245. date,
  246. qualName,
  247. reqURL,
  248. reqQURL,
  249. retURL,
  250. canRetake,
  251. hasTest,
  252. canRequest,
  253. isSystem`
  254. });
  255.  
  256. gridDiv.innerHTML = `
  257. <div id="myGrid" class="ag-theme-alpine">
  258. <style>
  259. .ag-theme-alpine {
  260. --ag-grid-size: 3px;
  261. width: 100%;
  262. height: 100%;
  263. position: absolute;
  264. top: 0;
  265. left: 0;
  266. right: 0;
  267. bottom: 0;
  268. }
  269. </style>
  270. </div>`
  271.  
  272.  
  273. const gridOptions = {
  274. columnDefs: [
  275. {
  276. headerName: 'Mturk Qualification Database and Scraper',
  277. children: [
  278. {
  279. field: "qualName"
  280. },
  281. {
  282. field: "requester"
  283. }]
  284. },
  285.  
  286.  
  287. {
  288. headerName: ' ',
  289. children: [
  290. {
  291.  
  292.  
  293. field: "description",
  294. width: 350
  295. },
  296. {
  297. headerName: "Value",
  298. field: "score",
  299. width: 100
  300. },
  301. {
  302. headerName: "Date",
  303. field: "date",
  304. width: 100,
  305. valueGetter: function (params)
  306. {
  307. var date = new Date(params.data.date);
  308. return (date.getMonth() + 1) + "/" + date.getDate() + "/" + date.getFullYear();
  309. },
  310. comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  311. {
  312. var dateA = new Date(valueA);
  313. var dateB = new Date(valueB);
  314. return dateA - dateB;
  315. },
  316. //valueFormatter: function(params) {
  317. // return new Date(params.value).toString().substring(4, 15);
  318. //}
  319. },
  320. {
  321.  
  322. headerName: "Requester ID",
  323. width: 150,
  324. field: "reqURL",
  325. valueFormatter: function (params)
  326. {
  327. var parts = params.value.split("/");
  328. return parts[2];
  329.  
  330. },
  331.  
  332. },
  333. {
  334. headerName: "Qual ID",
  335. field: "id",
  336.  
  337. valueFormatter: function (params)
  338. {
  339. if (!params.value || params.value === '') return '';
  340. var parts = params.value.split("/");
  341. return parts[2];
  342.  
  343. }
  344. }]
  345. },
  346. {
  347. headerName: 'More',
  348. children: [
  349. {
  350. headerName: " ",
  351. field: " ",
  352. width: 100,
  353. columnGroupShow: 'closed'
  354. },
  355. {
  356. headerName: "Retake",
  357. field: "canRetake",
  358. width: 100,
  359. columnGroupShow: 'open',
  360. suppressMenu: true
  361. },
  362. {
  363. headerName: "hasTest",
  364. field: "hasTest",
  365. width: 100,
  366. columnGroupShow: 'open',
  367. suppressMenu: true
  368. },
  369. {
  370. headerName: "canReq",
  371. field: "canRequest",
  372. width: 100,
  373. columnGroupShow: 'open',
  374. suppressMenu: true
  375. },
  376. {
  377. headerName: "System",
  378. field: "isSystem",
  379. width: 100,
  380. columnGroupShow: 'open',
  381. suppressMenu: true
  382. },
  383.  
  384. ]
  385. }
  386. ],
  387. defaultColDef:
  388. {
  389. sortable: true,
  390. filter: true,
  391. editable: true,
  392. resizable: true,
  393. },
  394. rowSelection: 'multiple',
  395. animateRows: true,
  396. rowData: []
  397. };
  398.  
  399. window.addEventListener('load', function ()
  400. {
  401. const gridDiv = document.querySelector('#myGrid');
  402. db.quals.toArray().then(data =>
  403. {
  404.  
  405. var filteredData = data.filter(function (row)
  406. {
  407. return !row.qualName.includes("Exc: [");
  408. });
  409. gridOptions.rowData = filteredData;
  410. new agGrid.Grid(gridDiv, gridOptions);
  411.  
  412. })
  413. })
  414. };
  415.  
  416.  
  417.