☰

db testing

Scrape, display, sort and search your Mturk qualifications

当前为 2023-03-16 提交的版本，查看 最新版本。

  1. // ==UserScript==
  2. // @name         db testing
  3. // @namespace    https://greasyfork.org/en/users/1004048-elias041
  4. // @version      0.80
  5. // @description  Scrape, display, sort and search your Mturk qualifications
  6. // @author       Elias041
  7. // @match        https://worker.mturk.com/qualifications/assigned
  8. // @match        https://worker.mturk.com/qt
  9. // @require      https://code.jquery.com/jquery-3.6.3.js
  10. // @require      https://code.jquery.com/ui/1.13.1/jquery-ui.min.js
  11. // @require      https://unpkg.com/dexie/dist/dexie.js
  12. // @require      https://unpkg.com/ag-grid-community@29.0.0/dist/ag-grid-community.min.js
  13. // @resource     https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-grid.css
  14. // @resource     https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-theme-apline.css
  15. // @icon         https://www.google.com/s2/favicons?sz=64&domain=mturk.com
  16. // @license      none
  17. // @grant        none
  18. // ==/UserScript==
  19.  
  20. let scraping = false;
  21. window.onload = function ()
  22. {
  23. 	let t = document.getElementsByClassName("col-xs-5 col-md-3 text-xs-right p-l-0")[0],
  24. 	e = t.parentNode,
  25. 	o = document.createElement("div");
  26. 	o.style.color = "#fff";
  27. 	o.style.padding = "10px";
  28. 	o.style.boxShadow = "2px 2px 4px #888888";
  29. 	o.style.background = "#33773A";
  30. 	o.style.opacity = "0.5";
  31. 	o.style.cursor = "pointer";
  32. 	o.id = "button";
  33. 	o.innerHTML = "Scrape&nbspQuals";
  34. 	e.insertBefore(o, t);
  35.  
  36. 	let c = document.createElement("div");
  37. 	c.style.color = "#fff";
  38. 	c.style.background = "#C78D99";
  39. 	c.style.padding = "10px";
  40. 	c.style.boxShadow = "2px 2px 4px #888888";
  41. 	c.style.background = "#383c44";
  42. 	c.style.opacity = "0.5";
  43. 	c.style.cursor = "pointer";
  44. 	c.innerHTML = "Cancel";
  45. 	c.id = "cancelButton";
  46. 	e.insertBefore(c, t);
  47.  
  48.  
  49. 	let d = document.createElement("div");
  50. 	d.style.color = "#fff";
  51. 	d.style.background = "#fc0f03";
  52. 	d.style.padding = "10px";
  53. 	d.style.boxShadow = "2px 2px 4px #888888";
  54. 	d.style.background = "#323552";
  55. 	d.style.opacity = "0.5";
  56. 	d.style.cursor = "pointer";
  57. 	d.innerHTML = "Database";
  58. 	d.id = "dbButton";
  59. 	e.insertBefore(d, t);
  60.  
  61. 	let f = document.createElement("div");
  62. 	f.style.color = "#fff";
  63. 	f.style.padding = "10px";
  64. 	f.style.boxShadow = "2px 2px 4px #888888";
  65. 	f.style.background = "#33773A";
  66. 	f.style.opacity = "0.5";
  67. 	f.id = "progress";
  68. 	f.innerHTML = "-";
  69. 	e.insertBefore(f, t);
  70.  
  71. 	document.getElementById("dbButton").addEventListener("click", function e()
  72. 	{
  73. 		window.open("https://worker.mturk.com/qt", "_blank");
  74. 	});
  75.  
  76. 	let timeout = 1850;
  77. 	let counter = " ";
  78. 	let retry_count = 0;
  79. 	let error_count = 0;
  80. 	let page = "https://worker.mturk.com/qualifications/assigned.json?page_size=100";
  81.  
  82. 	document.getElementById("cancelButton").addEventListener("click", function e()
  83. 	{
  84. 		retry_count = 0;
  85. 		scraping = false;
  86. 		$("#cancelButton").css('background', '#383c44')
  87. 		$("#button").css('background', '#33773A')
  88. 		$("#progress").html("-")
  89. 	})
  90. 	document.getElementById("button").addEventListener("click", function e()
  91. 	{
  92. 		localStorage.setItem('incompleteScrape', true);
  93. 		scraping = true;
  94. 		$("#button").css('background', '#383c44')
  95. 		$("#cancelButton").css('background', '#CE3132')
  96.  
  97.  
  98.  
  99. 		/*init db*/
  100. 		var db = new Dexie("qualifications_v2");
  101. 		db.version(2).stores(
  102. 		{
  103. 			quals: `
  104.     id,
  105.     requester,
  106.     description,
  107.     score,
  108.     date,
  109.     qualName,
  110.     reqURL,
  111.     reqQURL,
  112.     retURL,
  113.     canRetake,
  114.     hasTest,
  115.     canRequest,
  116.     isSystem`
  117. 		});
  118.  
  119. 		function readDatabase()
  120. 		{
  121. 			return db.quals.toArray();
  122. 		}
  123.  
  124. 		async function compareDatabases(oldDBPromise)
  125. 		{
  126.  
  127. 			const newDB = await readDatabase()
  128. 			return oldDBPromise.then(oldDB =>
  129. 			{
  130. 				let changes = [];
  131.  
  132. 				for (let i = 0; i < newDB.length; i++)
  133. 				{
  134. 					let newRecord = newDB[i];
  135. 					let oldRecord = oldDB.find(r => r.id === newRecord.id);
  136.  
  137.  
  138. 					if (oldRecord && oldRecord.score !== newRecord.score)
  139. 					{
  140. 						changes.push(
  141. 						{
  142. 							id: newRecord.id,
  143. 							field: "score",
  144. 							requester: newRecord.requester,
  145. 							qualName: newRecord.qualName,
  146. 							oldValue: oldRecord.score,
  147. 							newValue: newRecord.score
  148. 						});
  149. 					}
  150. 				}
  151.  
  152. 				if (changes.length > 0)
  153. 				{
  154. 					localStorage.setItem("changes", JSON.stringify(changes));
  155. 					localStorage.setItem("hasChanges", true);
  156. 					return changes;
  157. 				}
  158. 			})
  159. 		}
  160.  
  161.  
  162. 		function checkFirstRun()
  163. 		{
  164. 			db.quals.count().then(count =>
  165. 			{
  166. 				if (count === 0)
  167. 				{
  168.  
  169. 					localStorage.setItem("firstRun", true);
  170. 				}
  171. 				else
  172. 				{
  173. 					localStorage.setItem("firstRun", false);
  174. 				}
  175. 			});
  176. 		}
  177.  
  178. 		checkFirstRun();
  179. 		let timeoutId;
  180. 		let oldDBPromise;
  181. 		let totalRetries = 0;
  182.  
  183. 		function getAssignedQualifications(nextPageToken = "")
  184. 		{
  185. 			if (oldDBPromise === undefined)
  186. 			{
  187. 				oldDBPromise = readDatabase();
  188.  
  189. 			}
  190. 			if (!scraping)
  191. 			{
  192. 				return;
  193. 			}
  194. 			$("#progress").html(counter);
  195. 			$.getJSON(page)
  196.  
  197. 				.then(function (data)
  198. 				{
  199. 					counter++
  200. 					retry_count = 0
  201. 					data.assigned_qualifications.forEach(function (t)
  202. 					{
  203.  
  204. 						db.quals.bulkPut([
  205. 						{
  206. 							id: t.request_qualification_url,
  207. 							requester: t.creator_name,
  208. 							description: t.description,
  209. 							canRetake: t.can_retake_test_or_rerequest,
  210. 							retry: t.earliest_retriable_time,
  211. 							score: t.value,
  212. 							date: t.grant_time,
  213. 							qualName: t.name,
  214. 							reqURL: t.creator_url,
  215. 							retURL: t.retake_test_url,
  216. 							isSystem: t.is_system_qualification,
  217. 							canRequest: t.is_requestable,
  218. 							hasTest: t.has_test
  219. 						}])
  220. 					})
  221.  
  222. 					if (data.next_page_token !== null)
  223. 					{
  224. 						timeoutId = setTimeout(() =>
  225. 						{
  226. 							page = `https://worker.mturk.com/qualifications/assigned.json?page_size=100&next_token=${encodeURIComponent(data.next_page_token)}`
  227. 							getAssignedQualifications(data.next_page_token);
  228. 						}, timeout);
  229.  
  230. 					}
  231. 					else if (data.next_page_token === null)
  232. 					{
  233. 						console.log("Scraping completed");
  234. 						console.log(counter + " pages");
  235. 						console.log(totalRetries + " timeouts");
  236. 						console.log("Clock was " + timeout);
  237. 						if (localStorage.getItem("firstRun") === "false")
  238. 						{
  239.  
  240. 							compareDatabases(oldDBPromise)
  241. 						}
  242. 						localStorage.setItem('incompleteScrape', false);
  243. 						$("#cancelButton").css('background', '#383c44');
  244. 						$("#progress").css('background', '#25dc12');
  245. 						$("#progress").html('&#10003;');
  246. 						$("#dbButton").css('background', '#57ab4f');
  247.  
  248. 					}
  249. 					else
  250. 					{
  251. 						console.log("Timeout or abort. Clock was " + timeout);
  252. 						$("#progress").css('background', '#FF0000');
  253. 						$("#progress").html('&#88;');
  254. 						return;
  255. 					}
  256. 				})
  257.  
  258. 				.catch(function (error)
  259. 				{
  260. 					if (error.status === 429 && retry_count < 20)
  261. 					{
  262.  
  263. 						retry_count++
  264. 						totalRetries++
  265. 						setTimeout(() =>
  266. 						{
  267. 							getAssignedQualifications(nextPageToken);
  268. 						}, 3000);
  269. 					}
  270. 					else if (error.status === 429 && retry_count > 20)
  271. 					{
  272. 						console.log("error " + error_count)
  273. 						error_count++;
  274. 						timeout += 1000
  275. 						setTimeout(() =>
  276. 						{
  277. 							getAssignedQualifications(nextPageToken);
  278. 						}, 10000);
  279.  
  280. 					}
  281. 					else if (error.status === 429 && retry_count > 20 && error_count > 3)
  282. 					{
  283. 						alert("There was a problem accessing the Mturk website. Scraping halted.")
  284. 						scraping = false
  285. 						return;
  286.  
  287. 					}
  288. 					else if (error.status === 503)
  289. 					{
  290. 						$("#progress").css('background', '#FFFF00');
  291. 						$("#progress").html('&#33;');
  292. 						if (confirm("Mturk responded with 503: Service Unavailable. Retry?"))
  293. 						{
  294. 							$("#progress").css('background', '#33773A');
  295. 							setTimeout(() =>
  296. 							{
  297. 								getAssignedQualifications(nextPageToken);
  298. 							}, 10000);
  299. 						}
  300. 						else
  301. 						{
  302. 							$("#progress").css('background', '#FF0000');
  303. 							$("#progress").html('&#88;');
  304. 							console.log("User declined retry.");
  305. 							return;
  306. 						}
  307. 					}
  308. 				})
  309. 		}
  310.  
  311. 		getAssignedQualifications();
  312.  
  313. 	})
  314. };
  315.  
  316.  
  317. if (location.href === "https://worker.mturk.com/qt")
  318. {
  319. 	document.body.innerHTML = "";
  320. 	let gridDiv = document.createElement("div");
  321. 	gridDiv.setAttribute("id", "gridDiv");
  322. 	document.body.appendChild(gridDiv);
  323. 	document.title = "Qualifications";
  324. 	window.closeModal = function ()
  325. 	{
  326. 		document.getElementById("changesModal").style.display = "none";
  327. 		localStorage.setItem("hasChanges", false);
  328.  
  329. 	}
  330. 	window.closeIModal = function ()
  331. 	{
  332. 		document.getElementById("incompleteModal").style.display = "none";
  333. 	}
  334. 	var db = new Dexie("qualifications_v2");
  335. 	db.version(2).stores(
  336. 	{
  337. 		quals: `
  338.         id,
  339.         requester,
  340.         description,
  341.         score,
  342.         date,
  343.         qualName,
  344.         reqURL,
  345.         reqQURL,
  346.         retURL,
  347.         canRetake,
  348.         hasTest,
  349.         canRequest,
  350.         isSystem`
  351. 	});
  352.  
  353. 	function displayChangeDetails()
  354. 	{
  355. 		if (localStorage.getItem("firstRun") === "true")
  356. 		{
  357. 			document.getElementById("changesModal").style.display = "none";
  358. 			localStorage.setItem("hasChanges", false);
  359. 			return;
  360. 		}
  361. 		if (localStorage.getItem("hasChanges") === "true")
  362. 		{
  363. 			let storedData = localStorage.getItem("changes");
  364. 			if (storedData)
  365. 			{
  366. 				let changeDetails = JSON.parse(storedData);
  367. 				let changesList = document.getElementById("changesList");
  368. 				changeDetails.forEach(function (detail)
  369. 				{
  370. 					let changeText = detail.requester + " - " + detail.qualName + " - " + detail.field + ": " + detail.oldValue + " -> " + detail.newValue;
  371. 					let changeItem = document.createElement("div");
  372. 					changeItem.textContent = changeText;
  373. 					changesList.appendChild(changeItem);
  374. 				});
  375. 				document.getElementById("changesModal").style.display = "block";
  376. 			}
  377. 		}
  378. 	}
  379.  
  380. 	function incompleteScrapeNotification()
  381. 	{
  382. 		if (localStorage.getItem("incompleteScrape") === "true")
  383. 		{
  384. 			document.getElementById("incompleteModal").style.display = "block";
  385. 		}
  386. 	}
  387.  
  388. 	gridDiv.innerHTML = `
  389. <div id="myGrid"  class="ag-theme-alpine">
  390. <style>
  391. .ag-theme-alpine {
  392.     --ag-grid-size: 3px;
  393.     width: 100%;
  394.     height: 100%;
  395.     position: absolute;
  396.     top: 0;
  397.     left: 0;
  398.     right: 0;
  399.     bottom: 0;
  400.  
  401.   .modal {
  402.     display: none;
  403.     position: fixed;
  404.     z-index: 1;
  405.     left: 0;
  406.     top: 0;
  407.     width: 100%;
  408.     height: 100%;
  409.     overflow: auto;
  410.     background-color: rgba(0, 0, 0, 0.4);
  411. }
  412.  
  413. .modal-content {
  414.     background-color: #fefefe;
  415.     margin: auto;
  416.     margin-top: 10%;
  417.     padding: 20px;
  418.     border: 1px solid #888;
  419.     width: 80%;
  420.     max-width: 600px;
  421. }
  422.  
  423. .modal-footer {
  424.     padding: 10px;
  425.     text-align: right;
  426. }
  427.  
  428. .modal-close {
  429.     background-color: #4CAF50;
  430.     border: none;
  431.     color: white;
  432.     padding: 8px 16px;
  433.     text-align: center;
  434.     text-decoration: none;
  435.     display: inline-block;
  436.     font-size: 16px;
  437.     margin: 4px 2px;
  438.     cursor: pointer;
  439. }
  440.  
  441. /* Center the modal content vertically */
  442. @media screen and (min-height: 600px) {
  443.     .modal-content {
  444.         margin-top: 15%;
  445.     }}}
  446. </style>
  447. <div id="changesModal" class="modal">
  448.     <div class="modal-content">
  449.         <h4>Changes Detected</h4>
  450.         <p id="changesList"></p>
  451.     </div>
  452.     <div class="modal-footer">
  453.         <button class="modal-close" onclick="closeModal()">Close</button>
  454.     </div>
  455. </div>
  456.      </div>
  457.  
  458.  
  459. <div id="incompleteModal" class="modal">
  460.     <div class="modal-content">
  461.         <h4>Incomplete Scrape Detected</h4>
  462.         <p>A scrape is in progress or the last scrape was incomplete.</p>
  463.     </div>
  464.     <div class="modal-footer">
  465.         <button class="modal-close" onclick="closeIModal()">Close</button>
  466.     </div>
  467. </div>
  468.      </div>
  469.      `
  470.  
  471.  
  472. 	const gridOptions = {
  473. 		columnDefs: [
  474. 			{
  475. 				headerName: 'Mturk Qualification Database and Scraper',
  476. 				children: [
  477. 				{
  478. 					field: "qualName",
  479. 					comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  480. 					{
  481. 						return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  482. 					}
  483. 				},
  484. 				{
  485. 					headerName: "Requester",
  486. 					field: "requester",
  487. 					comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  488. 					{
  489. 						return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  490. 					}
  491. 				}]
  492. 			},
  493.  
  494.  
  495. 			{
  496. 				headerName: ' ',
  497. 				children: [
  498. 				{
  499. 					field: "description",
  500. 					width: 350,
  501. 					cellRenderer: function (params)
  502. 					{
  503. 						return '<span title="' + params.value + '">' + params.value + '</span>';
  504. 					},
  505. 					comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  506. 					{
  507. 						return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  508. 					}
  509. 				},
  510. 				{
  511. 					headerName: "Value",
  512. 					field: "score",
  513. 					width: 100
  514. 				},
  515. 				{
  516. 					headerName: "Date",
  517. 					field: "date",
  518. 					width: 100,
  519. 					valueGetter: function (params)
  520. 					{
  521. 						var date = new Date(params.data.date);
  522. 						return (date.getMonth() + 1) + "/" + date.getDate() + "/" + date.getFullYear();
  523. 					},
  524. 					comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  525. 					{
  526. 						var dateA = new Date(valueA);
  527. 						var dateB = new Date(valueB);
  528. 						return dateA - dateB;
  529. 					},
  530. 				},
  531. 				{
  532.  
  533. 					headerName: "Requester ID",
  534. 					width: 150,
  535. 					field: "reqURL",
  536. 					valueFormatter: function (params)
  537. 					{
  538. 						var parts = params.value.split("/");
  539. 						return parts[2];
  540.  
  541. 					},
  542.  
  543. 				},
  544. 				{
  545. 					headerName: "Qual ID",
  546. 					field: "id",
  547.  
  548. 					valueFormatter: function (params)
  549. 					{
  550. 						if (!params.value || params.value === '') return '';
  551. 						var parts = params.value.split("/");
  552. 						return parts[2];
  553. 					}
  554. 				}]
  555. 			},
  556. 			{
  557. 				headerName: 'More',
  558. 				children: [
  559. 				{
  560. 					headerName: " ",
  561. 					field: " ",
  562. 					width: 100,
  563. 					columnGroupShow: 'closed'
  564. 				},
  565. 				{
  566. 					headerName: "Retake",
  567. 					field: "canRetake",
  568. 					width: 100,
  569. 					columnGroupShow: 'open',
  570. 					suppressMenu: true
  571. 				},
  572. 				{
  573. 					headerName: "hasTest",
  574. 					field: "hasTest",
  575. 					width: 100,
  576. 					columnGroupShow: 'open',
  577. 					suppressMenu: true
  578. 				},
  579. 				{
  580. 					headerName: "canReq",
  581. 					field: "canRequest",
  582. 					width: 100,
  583. 					columnGroupShow: 'open',
  584. 					suppressMenu: true
  585. 				},
  586. 				{
  587. 					headerName: "System",
  588. 					field: "isSystem",
  589. 					width: 100,
  590. 					columnGroupShow: 'open',
  591. 					suppressMenu: true
  592. 				}, ]
  593. 			}
  594. 		],
  595. 		defaultColDef:
  596. 		{
  597. 			sortable: true,
  598. 			filter: true,
  599. 			editable: true,
  600. 			resizable: true,
  601. 		},
  602. 		rowSelection: 'multiple',
  603. 		animateRows: true,
  604. 		rowData: []
  605. 	};
  606.  
  607.  
  608. 	window.addEventListener('load', function ()
  609. 	{
  610. 		displayChangeDetails();
  611. 		incompleteScrapeNotification()
  612. 		const gridDiv = document.querySelector('#myGrid');
  613. 		db.quals.toArray().then(data =>
  614. 		{
  615.  
  616. 			var filteredData = data.filter(function (row)
  617. 			{
  618. 				return !row.qualName.includes("Exc: [");
  619. 			});
  620. 			gridOptions.rowData = filteredData;
  621. 			new agGrid.Grid(gridDiv, gridOptions);
  622.  
  623.  
  624. 		})
  625. 	})
  626. };