☰

Mturk Qualification Database and Scraper

Scrape, display, sort and search your Mturk qualifications

  1. // ==UserScript==
  2. // @name         Mturk Qualification Database and Scraper
  3. // @namespace    https://greasyfork.org/en/users/1004048-elias041
  4. // @version      0.84
  5. // @description  Scrape, display, sort and search your Mturk qualifications
  6. // @author       Elias041
  7. // @match        https://worker.mturk.com/qualifications/assigned*
  8. // @match        https://worker.mturk.com/qt
  9. // @require      https://code.jquery.com/jquery-3.6.3.js
  10. // @require      https://code.jquery.com/ui/1.13.1/jquery-ui.min.js
  11. // @require      https://unpkg.com/ag-grid-community@29.0.0/dist/ag-grid-community.min.js
  12. // @icon         https://www.google.com/s2/favicons?sz=64&domain=mturk.com
  13. // @icon         data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==
  14. // @grant        none
  15. // ==/UserScript==
  16.  
  17. const buttonStyle = {
  18. 	color: "#fff",
  19. 	padding: "10px",
  20. 	boxShadow: "2px 2px 4px #888888",
  21. 	opacity: "0.5",
  22. 	cursor: "pointer"
  23. };
  24.  
  25. const BUTTON_IDS = {
  26. 	scrapeButton: "button",
  27. 	cancelButton: "cancelButton",
  28. 	progress: "progress",
  29. 	dbButton: "dbButton"
  30. };
  31.  
  32. let timeout = 1850;
  33. let counter = " ";
  34. let retry_count = 0;
  35. let error_count = 0;
  36. let scraping = false;
  37.  
  38.  
  39. function createButton(parent, id, text, color, clickHandler) {
  40. 	const btn = document.createElement("div");
  41. 	Object.assign(btn.style, buttonStyle);
  42. 	btn.style.background = color;
  43. 	btn.id = id;
  44. 	btn.innerHTML = text;
  45. 	parent.insertBefore(btn, parent.firstChild);
  46. 	btn.addEventListener("click", clickHandler);
  47. }
  48.  
  49. const parentDiv = document.getElementsByClassName("col-xs-5 col-md-3 text-xs-right p-l-0")[0]?.parentNode;
  50. if (parentDiv) {
  51. 	createButton(parentDiv, BUTTON_IDS.scrapeButton, "Scrape&nbspQuals", "#33773A", function() {
  52. 		localStorage.setItem('incompleteScrape', true);
  53. 		scraping = true;
  54. 		getAssignedQualifications();
  55. 		$("#button").css('background', '#383c44');
  56. 		$("#cancelButton").css('background', '#CE3132');
  57. 	});
  58.  
  59. 	createButton(parentDiv, BUTTON_IDS.cancelButton, "Cancel", "#383c44", function() {
  60. 		retry_count = 0;
  61. 		scraping = false;
  62. 		$("#cancelButton").css('background', '#383c44');
  63. 		$("#button").css('background', '#33773A');
  64. 		$("#progress").html("-");
  65. 	});
  66. 	createButton(parentDiv, BUTTON_IDS.dbButton, "Database", "#323552", function() {
  67. 		window.open("https://worker.mturk.com/qt", "_blank");
  68. 	});
  69.  
  70. 	createButton(parentDiv, BUTTON_IDS.progress, "-", "#323552", function() {
  71.  
  72. 	});
  73.  
  74. 	function initializeDatabase() {
  75. 		const dbName = "qualifications_v2";
  76. 		const storeName = "quals";
  77. 		const version = 2;
  78. 		const openRequest = indexedDB.open(dbName, version);
  79.  
  80. 		openRequest.onupgradeneeded = function(event) {
  81. 			const db = event.target.result;
  82.  
  83. 			if (!db.objectStoreNames.contains(storeName)) {
  84. 				const objectStore = db.createObjectStore(storeName, {
  85. 					keyPath: "id"
  86. 				});
  87.  
  88. 				objectStore.createIndex("id", "id", {
  89. 					unique: true
  90. 				});
  91. 				objectStore.createIndex("requester", "requester", {
  92. 					unique: false
  93. 				});
  94. 				objectStore.createIndex("description", "description", {
  95. 					unique: false
  96. 				});
  97. 				objectStore.createIndex("score", "score", {
  98. 					unique: false
  99. 				});
  100. 				objectStore.createIndex("date", "date", {
  101. 					unique: false
  102. 				});
  103. 				objectStore.createIndex("qualName", "qualName", {
  104. 					unique: false
  105. 				});
  106. 				objectStore.createIndex("reqURL", "reqURL", {
  107. 					unique: false
  108. 				});
  109. 				objectStore.createIndex("reqQURL", "reqQURL", {
  110. 					unique: false
  111. 				});
  112. 				objectStore.createIndex("retURL", "retURL", {
  113. 					unique: false
  114. 				});
  115. 				objectStore.createIndex("canRetake", "canRetake", {
  116. 					unique: false
  117. 				});
  118. 				objectStore.createIndex("hasTest", "hasTest", {
  119. 					unique: false
  120. 				});
  121. 				objectStore.createIndex("canRequest", "canRequest", {
  122. 					unique: false
  123. 				});
  124. 				objectStore.createIndex("isSystem", "isSystem", {
  125. 					unique: false
  126. 				});
  127. 			}
  128. 		};
  129. 	}
  130.  
  131.  
  132. 	initializeDatabase()
  133. }
  134.  
  135. function openDatabase() {
  136. 	return new Promise((resolve, reject) => {
  137. 		const dbName = "qualifications_v2";
  138. 		const openRequest = indexedDB.open(dbName);
  139.  
  140. 		openRequest.onsuccess = (event) => {
  141. 			resolve(event.target.result);
  142. 		};
  143.  
  144. 		openRequest.onerror = (event) => {
  145. 			reject(event.target.errorCode);
  146. 		};
  147. 	});
  148. }
  149.  
  150.  
  151.  
  152. function readDatabase() {
  153. 	return openDatabase().then((db) => {
  154. 		return new Promise((resolve, reject) => {
  155. 			const storeName = "quals";
  156. 			const transaction = db.transaction(storeName, "readonly");
  157. 			const objectStore = transaction.objectStore(storeName);
  158. 			const request = objectStore.getAll();
  159.  
  160. 			request.onsuccess = (event) => {
  161. 				resolve(event.target.result);
  162. 			};
  163.  
  164. 			request.onerror = (event) => {
  165. 				reject(event.target.errorCode);
  166. 			};
  167. 		});
  168. 	});
  169. }
  170.  
  171. async function compareDatabases(oldDBPromise) {
  172.  
  173. 	const newDB = await readDatabase()
  174. 	return oldDBPromise.then(oldDB => {
  175. 		let changes = [];
  176.  
  177. 		for (let i = 0; i < newDB.length; i++) {
  178. 			let newRecord = newDB[i];
  179. 			let oldRecord = oldDB.find(r => r.id === newRecord.id);
  180.  
  181.  
  182. 			if (oldRecord && oldRecord.score !== newRecord.score) {
  183. 				changes.push({
  184. 					id: newRecord.id,
  185. 					field: "score",
  186. 					requester: newRecord.requester,
  187. 					qualName: newRecord.qualName,
  188. 					oldValue: oldRecord.score,
  189. 					newValue: newRecord.score
  190. 				});
  191. 			}
  192. 		}
  193.  
  194. 		if (changes.length > 0) {
  195. 			localStorage.setItem("changes", JSON.stringify(changes));
  196. 			localStorage.setItem("hasChanges", true);
  197. 			return changes;
  198. 		}
  199. 	})
  200. }
  201.  
  202.  
  203. function checkFirstRun() {
  204. 	openDatabase()
  205. 		.then((db) => {
  206. 			const storeName = "quals";
  207. 			const transaction = db.transaction(storeName, "readonly");
  208. 			const objectStore = transaction.objectStore(storeName);
  209. 			const request = objectStore.count();
  210.  
  211. 			request.onsuccess = (event) => {
  212. 				const count = event.target.result;
  213.  
  214. 				if (count === 0) {
  215. 					localStorage.setItem("firstRun", true);
  216. 				} else {
  217. 					localStorage.setItem("firstRun", false);
  218. 				}
  219. 			};
  220.  
  221. 			request.onerror = (event) => {
  222. 				console.error("Error counting records:", event.target.errorCode);
  223. 			};
  224. 		})
  225. 		.catch((error) => {
  226. 			console.error("Error opening database:", error);
  227. 		});
  228. }
  229.  
  230. function addEntries(assigned_qualifications) {
  231. 	const dbName = "qualifications_v2";
  232. 	const storeName = "quals";
  233. 	const openRequest = indexedDB.open(dbName);
  234.  
  235. 	openRequest.onsuccess = function(event) {
  236. 		const db = event.target.result;
  237. 		const transaction = db.transaction(storeName, "readwrite");
  238. 		const objectStore = transaction.objectStore(storeName);
  239.  
  240. 		assigned_qualifications.forEach(function(t) {
  241. 			const entry = {
  242. 				id: t.request_qualification_url,
  243. 				requester: t.creator_name,
  244. 				description: t.description,
  245. 				canRetake: t.can_retake_test_or_rerequest,
  246. 				retry: t.earliest_retriable_time,
  247. 				score: t.value,
  248. 				date: t.grant_time,
  249. 				qualName: t.name,
  250. 				reqURL: t.creator_url,
  251. 				retURL: t.retake_test_url,
  252. 				isSystem: t.is_system_qualification,
  253. 				canRequest: t.is_requestable,
  254. 				hasTest: t.has_test,
  255. 			};
  256.  
  257. 			objectStore.put(entry);
  258. 		});
  259.  
  260. 		transaction.oncomplete = function() {
  261. 			console.log("All entries added successfully");
  262. 		};
  263.  
  264. 		transaction.onerror = function(event) {
  265. 			console.error("Error adding entries:", event.target.errorCode);
  266. 		};
  267. 	};
  268.  
  269. 	openRequest.onerror = function(event) {
  270. 		console.error("Error opening database:", event.target.errorCode);
  271. 	};
  272. }
  273. checkFirstRun();
  274.  
  275.  
  276.  
  277. let page = "https://worker.mturk.com/qualifications/assigned.json?page_size=100";
  278. let timeoutId;
  279. let oldDBPromise;
  280. let totalRetries = 0;
  281.  
  282. function getAssignedQualifications(nextPageToken = "") {
  283. 	if (oldDBPromise === undefined) {
  284. 		oldDBPromise = readDatabase();
  285.  
  286. 	}
  287. 	if (!scraping) {
  288. 		return;
  289. 	}
  290. 	$("#progress").html(counter);
  291. 	$.getJSON(page)
  292.  
  293. 		.then(function(data) {
  294. 			counter++
  295. 			retry_count = 0
  296.  
  297. 			addEntries(data.assigned_qualifications);
  298.  
  299.  
  300. 			if (data.next_page_token !== null) {
  301. 				timeoutId = setTimeout(() => {
  302. 					page = `https://worker.mturk.com/qualifications/assigned.json?page_size=100&next_token=${encodeURIComponent(data.next_page_token)}`
  303. 					getAssignedQualifications(data.next_page_token);
  304. 				}, timeout);
  305.  
  306. 			} else if (data.next_page_token === null) {
  307. 				console.log("Scraping completed");
  308. 				console.log(counter + " pages");
  309. 				console.log(totalRetries + " timeouts");
  310. 				console.log("Clock was " + timeout);
  311. 				if (localStorage.getItem("firstRun") === "false") {
  312.  
  313. 					compareDatabases(oldDBPromise)
  314. 				}
  315. 				localStorage.setItem('incompleteScrape', false);
  316. 				$("#cancelButton").css('background', '#383c44');
  317. 				$("#progress").css('background', '#25dc12');
  318. 				$("#progress").html('&#10003;');
  319. 				$("#dbButton").css('background', '#57ab4f');
  320.  
  321. 			} else {
  322. 				console.log("Timeout or abort. Clock was " + timeout);
  323. 				$("#progress").css('background', '#FF0000');
  324. 				$("#progress").html('&#88;');
  325. 				return;
  326. 			}
  327. 		})
  328.  
  329. 		.catch(function(error) {
  330. 			if (error.status === 429 && retry_count < 20) {
  331.  
  332. 				retry_count++
  333. 				totalRetries++
  334. 				setTimeout(() => {
  335. 					getAssignedQualifications(nextPageToken);
  336. 				}, 3000);
  337. 			} else if (error.status === 429 && retry_count > 20) {
  338. 				console.log("error " + error_count)
  339. 				error_count++;
  340. 				timeout += 1000
  341. 				setTimeout(() => {
  342. 					getAssignedQualifications(nextPageToken);
  343. 				}, 10000);
  344.  
  345. 			} else if (error.status === 429 && retry_count > 20 && error_count > 3) {
  346. 				alert("There was a problem accessing the Mturk website. Scraping halted.")
  347. 				scraping = false
  348. 				return;
  349.  
  350. 			} else if (error.status === 503) {
  351. 				$("#progress").css('background', '#FFFF00');
  352. 				$("#progress").html('&#33;');
  353. 				if (confirm("Mturk responded with 503: Service Unavailable. Retry?")) {
  354. 					$("#progress").css('background', '#33773A');
  355. 					setTimeout(() => {
  356. 						getAssignedQualifications(nextPageToken);
  357. 					}, 10000);
  358. 				} else {
  359. 					$("#progress").css('background', '#FF0000');
  360. 					$("#progress").html('&#88;');
  361. 					console.log("User declined retry.");
  362. 					return;
  363. 				}
  364. 			}
  365. 		})
  366. }
  367.  
  368.  
  369.  
  370. if (location.href === "https://worker.mturk.com/qt") {
  371. 	document.body.innerHTML = "";
  372. 	let gridDiv = document.createElement("div");
  373. 	gridDiv.setAttribute("id", "gridDiv");
  374. 	document.body.appendChild(gridDiv);
  375. 	document.title = "Qualifications";
  376. 	window.closeModal = function() {
  377. 		document.getElementById("changesModal").style.display = "none";
  378. 		localStorage.setItem("hasChanges", false);
  379.  
  380. 	}
  381. 	window.closeIModal = function() {
  382. 		document.getElementById("incompleteModal").style.display = "none";
  383. 	}
  384.  
  385.  
  386. 	function getDataFromDatabase() {
  387. 		return new Promise((resolve, reject) => {
  388. 			const dbName = "qualifications_v2";
  389. 			const storeName = "quals";
  390. 			const openRequest = indexedDB.open(dbName);
  391.  
  392. 			openRequest.onsuccess = function(event) {
  393. 				const db = event.target.result;
  394. 				const transaction = db.transaction(storeName, "readonly");
  395. 				const objectStore = transaction.objectStore(storeName);
  396. 				const request = objectStore.getAll();
  397.  
  398. 				request.onsuccess = function() {
  399. 					resolve(request.result);
  400. 				};
  401.  
  402. 				request.onerror = function() {
  403. 					reject(new Error("Error retrieving data from the database"));
  404. 				};
  405. 			};
  406.  
  407. 			openRequest.onerror = function(event) {
  408. 				reject(new Error("Error opening the database"));
  409. 			};
  410. 		});
  411. 	}
  412.  
  413. 	function displayChangeDetails() {
  414. 		if (localStorage.getItem("firstRun") === "true") {
  415. 			document.getElementById("changesModal").style.display = "none";
  416. 			localStorage.setItem("hasChanges", false);
  417. 			return;
  418. 		}
  419. 		if (localStorage.getItem("hasChanges") === "true") {
  420. 			let storedData = localStorage.getItem("changes");
  421. 			if (storedData) {
  422. 				let changeDetails = JSON.parse(storedData);
  423. 				let changesList = document.getElementById("changesList");
  424. 				changeDetails.forEach(function(detail) {
  425. 					let changeText = detail.requester + " - " + detail.qualName + " - " + detail.field + ": " + detail.oldValue + " -> " + detail.newValue;
  426. 					let changeItem = document.createElement("div");
  427. 					changeItem.textContent = changeText;
  428. 					changesList.appendChild(changeItem);
  429. 				});
  430. 				document.getElementById("changesModal").style.display = "block";
  431. 			}
  432. 		}
  433. 	}
  434.  
  435. 	function incompleteScrapeNotification() {
  436. 		if (localStorage.getItem("incompleteScrape") === "true") {
  437. 			document.getElementById("incompleteModal").style.display = "block";
  438. 		}
  439. 	}
  440.  
  441. 	gridDiv.innerHTML = `
  442. <div id="myGrid"  class="ag-theme-alpine">
  443. <style>
  444. .ag-theme-alpine {
  445.     --ag-grid-size: 3px;
  446.     width: 100%;
  447.     height: 100%;
  448.     position: absolute;
  449.     top: 0;
  450.     left: 0;
  451.     right: 0;
  452.     bottom: 0;
  453.  
  454.   .modal {
  455.     display: none;
  456.     position: fixed;
  457.     z-index: 1;
  458.     left: 0;
  459.     top: 0;
  460.     width: 100%;
  461.     height: 100%;
  462.     overflow: auto;
  463.     background-color: rgba(0, 0, 0, 0.4);
  464. }
  465.  
  466. .modal-content {
  467.     background-color: #fefefe;
  468.     margin: auto;
  469.     margin-top: 10%;
  470.     padding: 20px;
  471.     border: 1px solid #888;
  472.     width: 80%;
  473.     max-width: 600px;
  474. }
  475.  
  476. .modal-footer {
  477.     padding: 10px;
  478.     text-align: right;
  479. }
  480.  
  481. .modal-close {
  482.     background-color: #4CAF50;
  483.     border: none;
  484.     color: white;
  485.     padding: 8px 16px;
  486.     text-align: center;
  487.     text-decoration: none;
  488.     display: inline-block;
  489.     font-size: 16px;
  490.     margin: 4px 2px;
  491.     cursor: pointer;
  492. }
  493.  
  494.  
  495. @media screen and (min-height: 600px) {
  496.     .modal-content {
  497.         margin-top: 15%;
  498.     }}}
  499. </style>
  500. <div id="changesModal" class="modal">
  501.     <div class="modal-content">
  502.         <h4>Changes Detected</h4>
  503.         <p id="changesList"></p>
  504.     </div>
  505.     <div class="modal-footer">
  506.         <button class="modal-close" ">Close</button>
  507.     </div>
  508. </div>
  509.      </div>
  510.  
  511.  
  512. <div id="incompleteModal" class="modal">
  513.     <div class="modal-content">
  514.         <h4>Incomplete Scrape Detected</h4>
  515.         <p>A scrape is in progress or the last scrape was incomplete.</p>
  516.     </div>
  517.     <div class="modal-footer">
  518.         <button class="modal-close" ">Close</button>
  519.     </div>
  520. </div>
  521.      </div>
  522.      `
  523.  
  524.  
  525. 	const gridOptions = {
  526. 		columnDefs: [{
  527. 				headerName: 'Mturk Qualification Database and Scraper',
  528. 				children: [{
  529. 						field: "qualName",
  530. 						comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
  531. 							return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  532. 						}
  533. 					},
  534. 					{
  535. 						headerName: "Requester",
  536. 						field: "requester",
  537. 						comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
  538. 							return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  539. 						}
  540. 					}
  541. 				]
  542. 			},
  543.  
  544.  
  545. 			{
  546. 				headerName: ' ',
  547. 				children: [{
  548. 						field: "description",
  549. 						width: 350,
  550. 						cellRenderer: function(params) {
  551. 							return '<span title="' + params.value + '">' + params.value + '</span>';
  552. 						},
  553. 						comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
  554. 							return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  555. 						}
  556. 					},
  557. 					{
  558. 						headerName: "Value",
  559. 						field: "score",
  560. 						width: 100
  561. 					},
  562. 					{
  563. 						headerName: "Date",
  564. 						field: "date",
  565. 						width: 100,
  566. 						valueGetter: function(params) {
  567. 							var date = new Date(params.data.date);
  568. 							return (date.getMonth() + 1) + "/" + date.getDate() + "/" + date.getFullYear();
  569. 						},
  570. 						comparator: function(valueA, valueB, nodeA, nodeB, isInverted) {
  571. 							var dateA = new Date(valueA);
  572. 							var dateB = new Date(valueB);
  573. 							return dateA - dateB;
  574. 						},
  575. 					},
  576. 					{
  577.  
  578. 						headerName: "Requester ID",
  579. 						width: 150,
  580. 						field: "reqURL",
  581. 						valueFormatter: function(params) {
  582. 							var parts = params.value.split("/");
  583. 							return parts[2];
  584.  
  585. 						},
  586.  
  587. 					},
  588. 					{
  589. 						headerName: "Qual ID",
  590. 						field: "id",
  591.  
  592. 						valueFormatter: function(params) {
  593. 							if (!params.value || params.value === '') return '';
  594. 							var parts = params.value.split("/");
  595. 							return parts[2];
  596. 						}
  597. 					}
  598. 				]
  599. 			},
  600. 			{
  601. 				headerName: 'More',
  602. 				children: [{
  603. 						headerName: " ",
  604. 						field: " ",
  605. 						width: 100,
  606. 						columnGroupShow: 'closed'
  607. 					},
  608. 					{
  609. 						headerName: "Retake",
  610. 						field: "canRetake",
  611. 						width: 100,
  612. 						columnGroupShow: 'open',
  613. 						suppressMenu: true
  614. 					},
  615. 					{
  616. 						headerName: "hasTest",
  617. 						field: "hasTest",
  618. 						width: 100,
  619. 						columnGroupShow: 'open',
  620. 						suppressMenu: true
  621. 					},
  622. 					{
  623. 						headerName: "canReq",
  624. 						field: "canRequest",
  625. 						width: 100,
  626. 						columnGroupShow: 'open',
  627. 						suppressMenu: true
  628. 					},
  629. 					{
  630. 						headerName: "System",
  631. 						field: "isSystem",
  632. 						width: 100,
  633. 						columnGroupShow: 'open',
  634. 						suppressMenu: true
  635. 					},
  636. 				]
  637. 			}
  638. 		],
  639. 		defaultColDef: {
  640. 			sortable: true,
  641. 			filter: true,
  642. 			editable: true,
  643. 			resizable: true,
  644. 		},
  645. 		rowSelection: 'multiple',
  646. 		animateRows: true,
  647. 		rowData: []
  648. 	};
  649. 	const closeModalButtons = gridDiv.querySelectorAll(".modal-close");
  650. 	closeModalButtons.forEach((button) => {
  651. 		button.addEventListener("click", function() {
  652. 			const modal = button.closest(".modal");
  653. 			modal.style.display = "none";
  654. 			if (modal.id === "changesModal") {
  655. 				localStorage.setItem("hasChanges", false);
  656. 			}
  657. 		});
  658. 	});
  659.  
  660. 	function addCSS(url, callback) {
  661. 		const link = document.createElement('link');
  662. 		link.rel = 'stylesheet';
  663. 		link.href = url;
  664. 		link.onload = callback;
  665. 		document.head.appendChild(link);
  666. 	}
  667.  
  668. 	addCSS('https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-grid.css', function() {
  669. 		addCSS('https://cdn.jsdelivr.net/npm/ag-grid-community@29.2.0/styles/ag-theme-alpine.css', function() {
  670. 			initializeAgGrid();
  671. 		});
  672. 	});
  673. 	async function initializeAgGrid() {
  674. 		window.addEventListener("load", function() {
  675. 			displayChangeDetails();
  676. 			incompleteScrapeNotification();
  677. 			const gridDiv = document.querySelector("#myGrid");
  678. 			getDataFromDatabase()
  679. 				.then((data) => {
  680. 					var filteredData = data.filter(function(row) {
  681. 						return !row.qualName.includes("Exc: [");
  682. 					});
  683. 					gridOptions.rowData = filteredData;
  684. 					new agGrid.Grid(gridDiv, gridOptions);
  685. 				})
  686. 				.catch((error) => {
  687. 					console.error("Error loading data for ag-grid:", error);
  688. 				});
  689. 		});
  690. 	};
  691. }