Mturk Qualification Database and Scraper

Scrape, display, sort and search your Mturk qualifications

当前为 2023-04-11 提交的版本,查看 最新版本

  1. // ==UserScript==
  2. // @name Mturk Qualification Database and Scraper
  3. // @namespace https://greasyfork.org/en/users/1004048-elias041
  4. // @version 0.82
  5. // @description Scrape, display, sort and search your Mturk qualifications
  6. // @author Elias041
  7. // @match https://worker.mturk.com/qualifications/assigned
  8. // @match https://worker.mturk.com/qt
  9. // @require https://code.jquery.com/jquery-3.6.3.js
  10. // @require https://code.jquery.com/ui/1.13.1/jquery-ui.min.js
  11. // @require https://unpkg.com/ag-grid-community@29.0.0/dist/ag-grid-community.min.js
  12. // @icon https://www.google.com/s2/favicons?sz=64&domain=mturk.com
  13. // @license none
  14. // @grant none
  15. // ==/UserScript==
  16.  
  17. let scraping = false;
  18. window.onload = function ()
  19. {
  20. let t = document.getElementsByClassName("col-xs-5 col-md-3 text-xs-right p-l-0")[0],
  21. e = t.parentNode,
  22. o = document.createElement("div");
  23. o.style.color = "#fff";
  24. o.style.padding = "10px";
  25. o.style.boxShadow = "2px 2px 4px #888888";
  26. o.style.background = "#33773A";
  27. o.style.opacity = "0.5";
  28. o.style.cursor = "pointer";
  29. o.id = "button";
  30. o.innerHTML = "Scrape&nbspQuals";
  31. e.insertBefore(o, t);
  32.  
  33. let c = document.createElement("div");
  34. c.style.color = "#fff";
  35. c.style.background = "#C78D99";
  36. c.style.padding = "10px";
  37. c.style.boxShadow = "2px 2px 4px #888888";
  38. c.style.background = "#383c44";
  39. c.style.opacity = "0.5";
  40. c.style.cursor = "pointer";
  41. c.innerHTML = "Cancel";
  42. c.id = "cancelButton";
  43. e.insertBefore(c, t);
  44.  
  45.  
  46. let d = document.createElement("div");
  47. d.style.color = "#fff";
  48. d.style.background = "#fc0f03";
  49. d.style.padding = "10px";
  50. d.style.boxShadow = "2px 2px 4px #888888";
  51. d.style.background = "#323552";
  52. d.style.opacity = "0.5";
  53. d.style.cursor = "pointer";
  54. d.innerHTML = "Database";
  55. d.id = "dbButton";
  56. e.insertBefore(d, t);
  57.  
  58. let f = document.createElement("div");
  59. f.style.color = "#fff";
  60. f.style.padding = "10px";
  61. f.style.boxShadow = "2px 2px 4px #888888";
  62. f.style.background = "#33773A";
  63. f.style.opacity = "0.5";
  64. f.id = "progress";
  65. f.innerHTML = "-";
  66. e.insertBefore(f, t);
  67.  
  68. document.getElementById("dbButton").addEventListener("click", function e()
  69. {
  70. window.open("https://worker.mturk.com/qt", "_blank");
  71. });
  72.  
  73. let timeout = 1850;
  74. let counter = " ";
  75. let retry_count = 0;
  76. let error_count = 0;
  77. let page = "https://worker.mturk.com/qualifications/assigned.json?page_size=100";
  78.  
  79. document.getElementById("cancelButton").addEventListener("click", function e()
  80. {
  81. retry_count = 0;
  82. scraping = false;
  83. $("#cancelButton").css('background', '#383c44')
  84. $("#button").css('background', '#33773A')
  85. $("#progress").html("-")
  86. })
  87. document.getElementById("button").addEventListener("click", function e()
  88. {
  89. localStorage.setItem('incompleteScrape', true);
  90. scraping = true;
  91. $("#button").css('background', '#383c44')
  92. $("#cancelButton").css('background', '#CE3132')
  93.  
  94.  
  95.  
  96. /*init db*/
  97. const dbName = "qualifications_v2";
  98. const storeName = "quals";
  99. const version = 2;
  100.  
  101. const openRequest = indexedDB.open(dbName, version);
  102.  
  103. openRequest.onupgradeneeded = function (event) {
  104. const db = event.target.result;
  105.  
  106. if (!db.objectStoreNames.contains(storeName)) {
  107. const objectStore = db.createObjectStore(storeName, { keyPath: "id" });
  108.  
  109. objectStore.createIndex("id", "id", { unique: true });
  110. objectStore.createIndex("requester", "requester", { unique: false });
  111. objectStore.createIndex("description", "description", { unique: false });
  112. objectStore.createIndex("score", "score", { unique: false });
  113. objectStore.createIndex("date", "date", { unique: false });
  114. objectStore.createIndex("qualName", "qualName", { unique: false });
  115. objectStore.createIndex("reqURL", "reqURL", { unique: false });
  116. objectStore.createIndex("reqQURL", "reqQURL", { unique: false });
  117. objectStore.createIndex("retURL", "retURL", { unique: false });
  118. objectStore.createIndex("canRetake", "canRetake", { unique: false });
  119. objectStore.createIndex("hasTest", "hasTest", { unique: false });
  120. objectStore.createIndex("canRequest", "canRequest", { unique: false });
  121. objectStore.createIndex("isSystem", "isSystem", { unique: false });
  122. }
  123. };
  124.  
  125. openRequest.onsuccess = function (event) {
  126. const db = event.target.result;
  127. console.log("Database opened successfully");
  128. };
  129.  
  130. openRequest.onerror = function (event) {
  131. console.error("Error opening database:", event.target.errorCode);
  132. };
  133.  
  134. function openDatabase() {
  135. return new Promise((resolve, reject) => {
  136. const dbName = "qualifications_v2";
  137. const openRequest = indexedDB.open(dbName);
  138.  
  139. openRequest.onsuccess = (event) => {
  140. resolve(event.target.result);
  141. };
  142.  
  143. openRequest.onerror = (event) => {
  144. reject(event.target.errorCode);
  145. };
  146. });
  147. }
  148.  
  149.  
  150.  
  151. function readDatabase() {
  152. return openDatabase().then((db) => {
  153. return new Promise((resolve, reject) => {
  154. const storeName = "quals";
  155. const transaction = db.transaction(storeName, "readonly");
  156. const objectStore = transaction.objectStore(storeName);
  157. const request = objectStore.getAll();
  158.  
  159. request.onsuccess = (event) => {
  160. resolve(event.target.result);
  161. };
  162.  
  163. request.onerror = (event) => {
  164. reject(event.target.errorCode);
  165. };
  166. });
  167. });
  168. }
  169.  
  170.  
  171. async function compareDatabases(oldDBPromise)
  172. {
  173.  
  174. const newDB = await readDatabase()
  175. return oldDBPromise.then(oldDB =>
  176. {
  177. let changes = [];
  178.  
  179. for (let i = 0; i < newDB.length; i++)
  180. {
  181. let newRecord = newDB[i];
  182. let oldRecord = oldDB.find(r => r.id === newRecord.id);
  183.  
  184.  
  185. if (oldRecord && oldRecord.score !== newRecord.score)
  186. {
  187. changes.push(
  188. {
  189. id: newRecord.id,
  190. field: "score",
  191. requester: newRecord.requester,
  192. qualName: newRecord.qualName,
  193. oldValue: oldRecord.score,
  194. newValue: newRecord.score
  195. });
  196. }
  197. }
  198.  
  199. if (changes.length > 0)
  200. {
  201. localStorage.setItem("changes", JSON.stringify(changes));
  202. localStorage.setItem("hasChanges", true);
  203. return changes;
  204. }
  205. })
  206. }
  207.  
  208.  
  209. function checkFirstRun() {
  210. openDatabase()
  211. .then((db) => {
  212. const storeName = "quals";
  213. const transaction = db.transaction(storeName, "readonly");
  214. const objectStore = transaction.objectStore(storeName);
  215. const request = objectStore.count();
  216.  
  217. request.onsuccess = (event) => {
  218. const count = event.target.result;
  219.  
  220. if (count === 0) {
  221. localStorage.setItem("firstRun", true);
  222. } else {
  223. localStorage.setItem("firstRun", false);
  224. }
  225. };
  226.  
  227. request.onerror = (event) => {
  228. console.error("Error counting records:", event.target.errorCode);
  229. };
  230. })
  231. .catch((error) => {
  232. console.error("Error opening database:", error);
  233. });
  234. }
  235.  
  236. function addEntries(assigned_qualifications) {
  237. const dbName = "qualifications_v2";
  238. const storeName = "quals";
  239. const openRequest = indexedDB.open(dbName);
  240.  
  241. openRequest.onsuccess = function (event) {
  242. const db = event.target.result;
  243. const transaction = db.transaction(storeName, "readwrite");
  244. const objectStore = transaction.objectStore(storeName);
  245.  
  246. assigned_qualifications.forEach(function (t) {
  247. const entry = {
  248. id: t.request_qualification_url,
  249. requester: t.creator_name,
  250. description: t.description,
  251. canRetake: t.can_retake_test_or_rerequest,
  252. retry: t.earliest_retriable_time,
  253. score: t.value,
  254. date: t.grant_time,
  255. qualName: t.name,
  256. reqURL: t.creator_url,
  257. retURL: t.retake_test_url,
  258. isSystem: t.is_system_qualification,
  259. canRequest: t.is_requestable,
  260. hasTest: t.has_test,
  261. };
  262.  
  263. objectStore.put(entry);
  264. });
  265.  
  266. transaction.oncomplete = function () {
  267. console.log("All entries added successfully");
  268. };
  269.  
  270. transaction.onerror = function (event) {
  271. console.error("Error adding entries:", event.target.errorCode);
  272. };
  273. };
  274.  
  275. openRequest.onerror = function (event) {
  276. console.error("Error opening database:", event.target.errorCode);
  277. };
  278. }
  279. checkFirstRun();
  280. let timeoutId;
  281. let oldDBPromise;
  282. let totalRetries = 0;
  283.  
  284. function getAssignedQualifications(nextPageToken = "")
  285. {
  286. if (oldDBPromise === undefined)
  287. {
  288. oldDBPromise = readDatabase();
  289.  
  290. }
  291. if (!scraping)
  292. {
  293. return;
  294. }
  295. $("#progress").html(counter);
  296. $.getJSON(page)
  297.  
  298. .then(function (data)
  299. {
  300. counter++
  301. retry_count = 0
  302.  
  303. addEntries(data.assigned_qualifications);
  304.  
  305.  
  306. if (data.next_page_token !== null)
  307. {
  308. timeoutId = setTimeout(() =>
  309. {
  310. page = `https://worker.mturk.com/qualifications/assigned.json?page_size=100&next_token=${encodeURIComponent(data.next_page_token)}`
  311. getAssignedQualifications(data.next_page_token);
  312. }, timeout);
  313.  
  314. }
  315. else if (data.next_page_token === null)
  316. {
  317. console.log("Scraping completed");
  318. console.log(counter + " pages");
  319. console.log(totalRetries + " timeouts");
  320. console.log("Clock was " + timeout);
  321. if (localStorage.getItem("firstRun") === "false")
  322. {
  323.  
  324. compareDatabases(oldDBPromise)
  325. }
  326. localStorage.setItem('incompleteScrape', false);
  327. $("#cancelButton").css('background', '#383c44');
  328. $("#progress").css('background', '#25dc12');
  329. $("#progress").html('&#10003;');
  330. $("#dbButton").css('background', '#57ab4f');
  331.  
  332. }
  333. else
  334. {
  335. console.log("Timeout or abort. Clock was " + timeout);
  336. $("#progress").css('background', '#FF0000');
  337. $("#progress").html('&#88;');
  338. return;
  339. }
  340. })
  341.  
  342. .catch(function (error)
  343. {
  344. if (error.status === 429 && retry_count < 20)
  345. {
  346.  
  347. retry_count++
  348. totalRetries++
  349. setTimeout(() =>
  350. {
  351. getAssignedQualifications(nextPageToken);
  352. }, 3000);
  353. }
  354. else if (error.status === 429 && retry_count > 20)
  355. {
  356. console.log("error " + error_count)
  357. error_count++;
  358. timeout += 1000
  359. setTimeout(() =>
  360. {
  361. getAssignedQualifications(nextPageToken);
  362. }, 10000);
  363.  
  364. }
  365. else if (error.status === 429 && retry_count > 20 && error_count > 3)
  366. {
  367. alert("There was a problem accessing the Mturk website. Scraping halted.")
  368. scraping = false
  369. return;
  370.  
  371. }
  372. else if (error.status === 503)
  373. {
  374. $("#progress").css('background', '#FFFF00');
  375. $("#progress").html('&#33;');
  376. if (confirm("Mturk responded with 503: Service Unavailable. Retry?"))
  377. {
  378. $("#progress").css('background', '#33773A');
  379. setTimeout(() =>
  380. {
  381. getAssignedQualifications(nextPageToken);
  382. }, 10000);
  383. }
  384. else
  385. {
  386. $("#progress").css('background', '#FF0000');
  387. $("#progress").html('&#88;');
  388. console.log("User declined retry.");
  389. return;
  390. }
  391. }
  392. })
  393. }
  394.  
  395. getAssignedQualifications();
  396.  
  397. })
  398. };
  399.  
  400. if (location.href === "https://worker.mturk.com/qt")
  401. {
  402. document.body.innerHTML = "";
  403. let gridDiv = document.createElement("div");
  404. gridDiv.setAttribute("id", "gridDiv");
  405. document.body.appendChild(gridDiv);
  406. document.title = "Qualifications";
  407. window.closeModal = function ()
  408. {
  409. document.getElementById("changesModal").style.display = "none";
  410. localStorage.setItem("hasChanges", false);
  411.  
  412. }
  413. window.closeIModal = function ()
  414. {
  415. document.getElementById("incompleteModal").style.display = "none";
  416. }
  417.  
  418.  
  419. function getDataFromDatabase() {
  420. return new Promise((resolve, reject) => {
  421. const dbName = "qualifications_v2";
  422. const storeName = "quals";
  423. const openRequest = indexedDB.open(dbName);
  424.  
  425. openRequest.onsuccess = function (event) {
  426. const db = event.target.result;
  427. const transaction = db.transaction(storeName, "readonly");
  428. const objectStore = transaction.objectStore(storeName);
  429. const request = objectStore.getAll();
  430.  
  431. request.onsuccess = function () {
  432. resolve(request.result);
  433. };
  434.  
  435. request.onerror = function () {
  436. reject(new Error("Error retrieving data from the database"));
  437. };
  438. };
  439.  
  440. openRequest.onerror = function (event) {
  441. reject(new Error("Error opening the database"));
  442. };
  443. });
  444. }
  445.  
  446. function displayChangeDetails()
  447. {
  448. if (localStorage.getItem("firstRun") === "true")
  449. {
  450. document.getElementById("changesModal").style.display = "none";
  451. localStorage.setItem("hasChanges", false);
  452. return;
  453. }
  454. if (localStorage.getItem("hasChanges") === "true")
  455. {
  456. let storedData = localStorage.getItem("changes");
  457. if (storedData)
  458. {
  459. let changeDetails = JSON.parse(storedData);
  460. let changesList = document.getElementById("changesList");
  461. changeDetails.forEach(function (detail)
  462. {
  463. let changeText = detail.requester + " - " + detail.qualName + " - " + detail.field + ": " + detail.oldValue + " -> " + detail.newValue;
  464. let changeItem = document.createElement("div");
  465. changeItem.textContent = changeText;
  466. changesList.appendChild(changeItem);
  467. });
  468. document.getElementById("changesModal").style.display = "block";
  469. }
  470. }
  471. }
  472.  
  473. function incompleteScrapeNotification()
  474. {
  475. if (localStorage.getItem("incompleteScrape") === "true")
  476. {
  477. document.getElementById("incompleteModal").style.display = "block";
  478. }
  479. }
  480.  
  481. gridDiv.innerHTML = `
  482. <div id="myGrid" class="ag-theme-alpine">
  483. <style>
  484. .ag-theme-alpine {
  485. --ag-grid-size: 3px;
  486. width: 100%;
  487. height: 100%;
  488. position: absolute;
  489. top: 0;
  490. left: 0;
  491. right: 0;
  492. bottom: 0;
  493.  
  494. .modal {
  495. display: none;
  496. position: fixed;
  497. z-index: 1;
  498. left: 0;
  499. top: 0;
  500. width: 100%;
  501. height: 100%;
  502. overflow: auto;
  503. background-color: rgba(0, 0, 0, 0.4);
  504. }
  505.  
  506. .modal-content {
  507. background-color: #fefefe;
  508. margin: auto;
  509. margin-top: 10%;
  510. padding: 20px;
  511. border: 1px solid #888;
  512. width: 80%;
  513. max-width: 600px;
  514. }
  515.  
  516. .modal-footer {
  517. padding: 10px;
  518. text-align: right;
  519. }
  520.  
  521. .modal-close {
  522. background-color: #4CAF50;
  523. border: none;
  524. color: white;
  525. padding: 8px 16px;
  526. text-align: center;
  527. text-decoration: none;
  528. display: inline-block;
  529. font-size: 16px;
  530. margin: 4px 2px;
  531. cursor: pointer;
  532. }
  533.  
  534.  
  535. @media screen and (min-height: 600px) {
  536. .modal-content {
  537. margin-top: 15%;
  538. }}}
  539. </style>
  540. <div id="changesModal" class="modal">
  541. <div class="modal-content">
  542. <h4>Changes Detected</h4>
  543. <p id="changesList"></p>
  544. </div>
  545. <div class="modal-footer">
  546. <button class="modal-close" ">Close</button>
  547. </div>
  548. </div>
  549. </div>
  550.  
  551.  
  552. <div id="incompleteModal" class="modal">
  553. <div class="modal-content">
  554. <h4>Incomplete Scrape Detected</h4>
  555. <p>A scrape is in progress or the last scrape was incomplete.</p>
  556. </div>
  557. <div class="modal-footer">
  558. <button class="modal-close" ">Close</button>
  559. </div>
  560. </div>
  561. </div>
  562. `
  563.  
  564.  
  565. const gridOptions = {
  566. columnDefs: [
  567. {
  568. headerName: 'Mturk Qualification Database and Scraper',
  569. children: [
  570. {
  571. field: "qualName",
  572. comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  573. {
  574. return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  575. }
  576. },
  577. {
  578. headerName: "Requester",
  579. field: "requester",
  580. comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  581. {
  582. return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  583. }
  584. }]
  585. },
  586.  
  587.  
  588. {
  589. headerName: ' ',
  590. children: [
  591. {
  592. field: "description",
  593. width: 350,
  594. cellRenderer: function (params)
  595. {
  596. return '<span title="' + params.value + '">' + params.value + '</span>';
  597. },
  598. comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  599. {
  600. return valueA.toLowerCase().localeCompare(valueB.toLowerCase());
  601. }
  602. },
  603. {
  604. headerName: "Value",
  605. field: "score",
  606. width: 100
  607. },
  608. {
  609. headerName: "Date",
  610. field: "date",
  611. width: 100,
  612. valueGetter: function (params)
  613. {
  614. var date = new Date(params.data.date);
  615. return (date.getMonth() + 1) + "/" + date.getDate() + "/" + date.getFullYear();
  616. },
  617. comparator: function (valueA, valueB, nodeA, nodeB, isInverted)
  618. {
  619. var dateA = new Date(valueA);
  620. var dateB = new Date(valueB);
  621. return dateA - dateB;
  622. },
  623. },
  624. {
  625.  
  626. headerName: "Requester ID",
  627. width: 150,
  628. field: "reqURL",
  629. valueFormatter: function (params)
  630. {
  631. var parts = params.value.split("/");
  632. return parts[2];
  633.  
  634. },
  635.  
  636. },
  637. {
  638. headerName: "Qual ID",
  639. field: "id",
  640.  
  641. valueFormatter: function (params)
  642. {
  643. if (!params.value || params.value === '') return '';
  644. var parts = params.value.split("/");
  645. return parts[2];
  646. }
  647. }]
  648. },
  649. {
  650. headerName: 'More',
  651. children: [
  652. {
  653. headerName: " ",
  654. field: " ",
  655. width: 100,
  656. columnGroupShow: 'closed'
  657. },
  658. {
  659. headerName: "Retake",
  660. field: "canRetake",
  661. width: 100,
  662. columnGroupShow: 'open',
  663. suppressMenu: true
  664. },
  665. {
  666. headerName: "hasTest",
  667. field: "hasTest",
  668. width: 100,
  669. columnGroupShow: 'open',
  670. suppressMenu: true
  671. },
  672. {
  673. headerName: "canReq",
  674. field: "canRequest",
  675. width: 100,
  676. columnGroupShow: 'open',
  677. suppressMenu: true
  678. },
  679. {
  680. headerName: "System",
  681. field: "isSystem",
  682. width: 100,
  683. columnGroupShow: 'open',
  684. suppressMenu: true
  685. }, ]
  686. }
  687. ],
  688. defaultColDef:
  689. {
  690. sortable: true,
  691. filter: true,
  692. editable: true,
  693. resizable: true,
  694. },
  695. rowSelection: 'multiple',
  696. animateRows: true,
  697. rowData: []
  698. };
  699. const closeModalButtons = gridDiv.querySelectorAll(".modal-close");
  700. closeModalButtons.forEach((button) => {
  701. button.addEventListener("click", function () {
  702. const modal = button.closest(".modal");
  703. modal.style.display = "none";
  704. if (modal.id === "changesModal") {
  705. localStorage.setItem("hasChanges", false);
  706. }
  707. });
  708. });
  709. function addCSS(url, callback) {
  710. const link = document.createElement('link');
  711. link.rel = 'stylesheet';
  712. link.href = url;
  713. link.onload = callback;
  714. document.head.appendChild(link);
  715. }
  716.  
  717. addCSS('https://cdn.jsdelivr.net/npm/ag-grid-community/styles/ag-grid.css', function() {
  718. addCSS('https://cdn.jsdelivr.net/npm/ag-grid-community@29.2.0/styles/ag-theme-alpine.css', function() {
  719. initializeAgGrid();
  720. });
  721. });
  722. async function initializeAgGrid() {
  723. window.addEventListener("load", function () {
  724. displayChangeDetails();
  725. incompleteScrapeNotification();
  726. const gridDiv = document.querySelector("#myGrid");
  727. getDataFromDatabase()
  728. .then((data) => {
  729. var filteredData = data.filter(function (row) {
  730. return !row.qualName.includes("Exc: [");
  731. });
  732. gridOptions.rowData = filteredData;
  733. new agGrid.Grid(gridDiv, gridOptions);
  734. })
  735. .catch((error) => {
  736. console.error("Error loading data for ag-grid:", error);
  737. });
  738. });
  739. };
  740. }