Queue processor with gentle WIP backoff. CAPTCHA-safe behavior (pauses instead of reloading while CAPTCHA present).
当前为
// ==UserScript==
// @name Instagram to archive.today URL Queue Manager v1.1.7
// @namespace http://tampermonkey.net/
// @version 1.1.7
// @description Queue processor with gentle WIP backoff. CAPTCHA-safe behavior (pauses instead of reloading while CAPTCHA present).
// @author thomased (ChatGPT + Gemini + Grok)
// @match https://archive.ph/*
// @match https://archive.today/*
// @match https://archive.is/*
// @match https://archive.vn/*
// @grant none
// @license MIT
// ==/UserScript==
(function () {
'use strict';
/* ===== CONFIG: tweak these to make WIP even gentler ===== */
const HOMEPAGE = 'https://archive.ph/';
const BASE_WIP_POLL_MS = 60 * 1000; // base wait between WIP reloads (60s)
const MAX_BACKOFF_EXP = 6; // cap exponent (2^6 = 64x base -> ~64 min at base=60s)
const BACKOFF_JITTER_RATIO = 0.12; // +/- 12% random jitter
const PROCESS_DELAY = 900;
/* ===== Utils / storage ===== */
function log(...args) { console.log('[ArchiveQueue]', ...args); }
function dbg(...args) { console.debug('[ArchiveQueue]', ...args); }
function getQueue() { try { return JSON.parse(localStorage.getItem('archiveQueue') || '[]'); } catch (e) { return []; } }
function saveQueue(q) { localStorage.setItem('archiveQueue', JSON.stringify(q)); updateOverlay(); }
function getProcessed() { try { return JSON.parse(localStorage.getItem('processedUrls') || '[]'); } catch (e) { return []; } }
function saveProcessed(url) {
try {
const arr = getProcessed();
if (!arr.includes(url)) {
arr.push(url);
localStorage.setItem('processedUrls', JSON.stringify(arr.slice(-1500)));
}
updateOverlay();
} catch (e) { console.error(e); }
}
function getRestricted() { try { return JSON.parse(localStorage.getItem('restrictedUrls') || '[]'); } catch (e) { return []; } }
function saveRestricted(url, reason = 'unknown') {
try {
const arr = getRestricted();
if (!arr.includes(url)) arr.push(url);
localStorage.setItem('restrictedUrls', JSON.stringify(arr));
updateOverlay();
} catch (e) { console.error(e); }
}
function clearAll() {
if (!confirm('Clear queue, processed & restricted lists?')) return;
localStorage.removeItem('archiveQueue');
localStorage.removeItem('processedUrls');
localStorage.removeItem('restrictedUrls');
localStorage.removeItem('aq_last_wip_reload');
sessionStorage.removeItem('forceSaveUrl');
sessionStorage.removeItem('aq_processing');
sessionStorage.removeItem('processingPaused');
location.reload();
}
/* ===== UI ===== */
function createOverlay() {
if (document.getElementById('aq-overlay')) return;
const ov = document.createElement('div');
ov.id = 'aq-overlay';
Object.assign(ov.style, {
position: 'fixed', top: '18px', right: '18px', zIndex: 999999,
background: 'rgba(255,255,255,0.97)', border: '1px solid #888', padding: '10px',
fontFamily: 'sans-serif', fontSize: '13px', color: '#222', borderRadius: '8px',
boxShadow: '0 4px 18px rgba(0,0,0,0.2)', maxWidth: '380px', maxHeight: '80vh', overflowY: 'auto'
});
ov.innerHTML = `
<div style="display:flex;justify-content:space-between;align-items:center">
<strong>Archive Queue v1.1.7</strong>
<span id="aq-close" style="cursor:pointer;font-weight:bold">×</span>
</div>
<div style="display:grid;grid-template-columns:repeat(2,1fr);gap:6px;margin-top:8px">
<button id="aq-add">Add URLs</button>
<button id="aq-edit">Edit Queue</button>
<button id="aq-resume">Resume</button>
<button id="aq-pause">Pause</button>
<button id="aq-export">Export Restricted</button>
<button id="aq-clear">Clear All</button>
</div>
<div id="aq-input" style="display:none;margin-top:8px">
<textarea id="aq-text" style="width:100%;height:80px" placeholder="URLs, one per line"></textarea>
<div style="display:flex;gap:6px;margin-top:6px"><button id="aq-save">Save</button><button id="aq-cancel">Cancel</button></div>
</div>
<div id="aq-edit-area" style="display:none;margin-top:8px">
<textarea id="aq-edit-text" style="width:100%;height:120px"></textarea>
<div style="display:flex;gap:6px;margin-top:6px"><button id="aq-update">Update</button><button id="aq-edit-cancel">Cancel</button></div>
</div>
<pre id="aq-status" style="white-space:pre-wrap;margin-top:8px;padding:8px;background:#f6f6f6;border-radius:6px"></pre>
<div id="aq-message" style="font-size:12px;color:#b40010;margin-top:6px"></div>
`;
document.body.appendChild(ov);
ov.querySelector('#aq-close').onclick = () => ov.style.display = 'none';
ov.querySelector('#aq-add').onclick = () => { document.getElementById('aq-input').style.display = 'block'; document.getElementById('aq-edit-area').style.display = 'none'; };
ov.querySelector('#aq-edit').onclick = () => { document.getElementById('aq-edit-area').style.display = 'block'; document.getElementById('aq-input').style.display = 'none'; document.getElementById('aq-edit-text').value = getQueue().join('\n'); };
ov.querySelector('#aq-resume').onclick = () => { sessionStorage.removeItem('processingPaused'); updateOverlay(); processQueue(); };
ov.querySelector('#aq-pause').onclick = () => { sessionStorage.setItem('processingPaused', '1'); updateOverlay(); };
ov.querySelector('#aq-export').onclick = exportRestricted;
ov.querySelector('#aq-clear').onclick = clearAll;
ov.querySelector('#aq-save').onclick = saveInput;
ov.querySelector('#aq-cancel').onclick = () => document.getElementById('aq-input').style.display = 'none';
ov.querySelector('#aq-update').onclick = updateQueue;
ov.querySelector('#aq-edit-cancel').onclick = () => document.getElementById('aq-edit-area').style.display = 'none';
updateOverlay();
}
function updateOverlay() {
const q = getQueue().length;
const p = getProcessed().length;
const r = getRestricted().length;
const st = document.getElementById('aq-status');
if (st) st.textContent = `Queue: ${q}\nProcessed: ${p}\nRestricted: ${r}\nProcessing: ${sessionStorage.getItem('aq_processing') ? 'ACTIVE' : 'IDLE'}`;
const msg = document.getElementById('aq-message');
if (msg) {
const captcha = !!document.querySelector('iframe[src*="recaptcha"], .g-recaptcha, [data-sitekey]') ||
(document.body && (document.body.innerText || '').toLowerCase().includes("i'm not a robot"));
const paused = !!sessionStorage.getItem('processingPaused');
msg.textContent = paused && captcha ? 'PAUSED - CAPTCHA detected. Solve it and click Resume.' : (paused ? 'PAUSED by user' : '');
}
}
function saveInput() {
const lines = (document.getElementById('aq-text').value || '').split('\n').map(s => s.trim()).filter(Boolean);
if (lines.length) {
const q = getQueue();
saveQueue(q.concat(lines));
document.getElementById('aq-text').value = '';
document.getElementById('aq-input').style.display = 'none';
updateOverlay();
setTimeout(processQueue, 250);
}
}
function updateQueue() {
const lines = (document.getElementById('aq-edit-text').value || '').split('\n').map(s => s.trim()).filter(Boolean);
saveQueue(lines);
document.getElementById('aq-edit-area').style.display = 'none';
updateOverlay();
setTimeout(processQueue, 250);
}
// --- NEW HELPER FUNCTION TO FIND MOST COMMON USERNAME ---
function findMostCommonUsername(urls) {
const counts = {};
const regex = /instagram\.com\/([^/]+)\/p\//;
for (const url of urls) {
const match = url.match(regex);
if (match && match[1]) {
const username = match[1];
counts[username] = (counts[username] || 0) + 1;
}
}
const keys = Object.keys(counts);
if (!keys.length) return 'export';
return keys.reduce((a, b) => counts[a] > counts[b] ? a : b);
}
// --- MODIFIED EXPORT FUNCTION ---
function exportRestricted() {
const arr = getRestricted();
if (!arr.length) return alert('No restricted URLs.');
const username = findMostCommonUsername(arr);
const date = new Date().toISOString().slice(0, 10);
const filename = `${username}_restricted-or-unavailable-urls_${date}.txt`;
const content = arr.join('\n');
const blob = new Blob([content], { type: 'text/plain' });
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = filename;
a.click();
URL.revokeObjectURL(a.href);
}
/* ===== Core processing ===== */
function isPaused() { return !!sessionStorage.getItem('processingPaused'); }
function setProcessingFlag(val) { if (val) sessionStorage.setItem('aq_processing', '1'); else sessionStorage.removeItem('aq_processing'); }
function processQueue() {
if (isPaused()) { log('Processing is paused by user.'); setProcessingFlag(false); updateOverlay(); return; }
if (sessionStorage.getItem('aq_processing')) { dbg('Already processing on another page/load.'); return; }
const q = getQueue();
if (!q.length) { log('Queue empty'); setProcessingFlag(false); updateOverlay(); return; }
const next = q[0];
const processed = getProcessed();
const restricted = getRestricted();
if (processed.includes(next) || restricted.includes(next)) {
log('Next already handled, removing from queue:', next);
q.shift(); saveQueue(q); setTimeout(processQueue, PROCESS_DELAY); return;
}
sessionStorage.setItem('aq_processing', '1');
sessionStorage.setItem('forceSaveUrl', next);
setProcessingFlag(true);
updateOverlay();
const nav = HOMEPAGE + next;
log('Navigating to pre-check:', nav);
window.location.href = nav;
}
/* ===== Helpers for homepage submit reliability (changed to be CAPTCHA-safe) ===== */
function waitForInputAndSubmit(forcedUrl, onDone) {
const SAVE_CLICK_TIMEOUT_MS = 5000;
const SAVE_CLICK_POLL_MS = 250;
const start = Date.now();
let interval = setInterval(() => {
try {
// Detect CAPTCHA first — if present, PAUSE processing and don't navigate/reload
const pageBody = (document.body && (document.body.innerText || '').toLowerCase()) || '';
const captchaDetected = !!document.querySelector('iframe[src*="recaptcha"], .g-recaptcha, [data-sitekey]') ||
pageBody.includes("i'm not a robot") || pageBody.includes('captcha');
if (captchaDetected) {
clearInterval(interval);
log('waitForInputAndSubmit: CAPTCHA detected on homepage/submit page — pausing processing to allow manual solve.');
sessionStorage.setItem('processingPaused', '1');
updateOverlay();
alert('Archive Queue paused: CAPTCHA detected. Solve it in the page, then click Resume in the overlay.');
onDone && onDone(false);
return;
}
const input = document.querySelector('input#url, input[name="url"], input[type="text"][name="url"]');
const submit = Array.from(document.querySelectorAll('input[type="submit"], button[type="submit"]'))
.find(el => (el.value && el.value.toLowerCase().includes('save')) || (el.textContent && el.textContent.toLowerCase().includes('save')));
const value = input ? (input.value || '').trim() : '';
dbg('waitForInputAndSubmit: input-value=', value, 'forced=', forcedUrl, 'submit?', !!submit);
if (input && value && (!forcedUrl || value === forcedUrl || value.includes(forcedUrl))) {
if (submit) {
clearInterval(interval);
log('Found input with value and save button -> clicking submit.');
setTimeout(() => { try { submit.click(); } catch (e) { try { submit.dispatchEvent(new Event('click', { bubbles: true })); } catch(_){} } }, 80);
onDone && onDone(true);
return;
} else {
const form = input.closest('form') || document.querySelector('form[action*="/submit/"], form');
if (form) {
clearInterval(interval);
log('No explicit save button, submitting form as fallback.');
setTimeout(()=> { try { form.submit(); } catch(e){ try { const btn = form.querySelector('input[type="submit"], button[type="submit"]'); if(btn) btn.click(); } catch(_){} } }, 80);
onDone && onDone(true);
return;
}
}
}
if (Date.now() - start > SAVE_CLICK_TIMEOUT_MS) {
clearInterval(interval);
log('Timeout waiting for input/save on homepage.');
// Before navigating to /submit/, re-check CAPTCHA — if captcha appears (rate-limiting caused it) -> PAUSE instead of navigating
const pageBody2 = (document.body && (document.body.innerText || '').toLowerCase()) || '';
const captchaNow = !!document.querySelector('iframe[src*="recaptcha"], .g-recaptcha, [data-sitekey]') ||
pageBody2.includes("i'm not a robot") || pageBody2.includes('captcha');
if (captchaNow) {
log('Captcha detected at timeout — pausing instead of navigating.');
sessionStorage.setItem('processingPaused', '1');
updateOverlay();
alert('Archive Queue paused: CAPTCHA detected. Solve it in the page, then click Resume in the overlay.');
onDone && onDone(false);
return;
}
// No captcha — fallback navigate to submit path (same as before)
if (forcedUrl) {
window.location.href = `${HOMEPAGE}submit/?url=${encodeURIComponent(forcedUrl)}`;
onDone && onDone(false);
return;
}
onDone && onDone(false);
}
} catch (e) {
console.error(e);
clearInterval(interval);
onDone && onDone(false);
}
}, SAVE_CLICK_POLL_MS);
}
/* ===== MODIFIED: Gentle WIP handler with backoff + cross-tab scheduling (NO timeout marking) ===== */
function handleWipPage() {
log('handleWipPage (gentle backoff, no timeout-marking)');
const q = getQueue();
if (!q.length) {
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
return;
}
const forced = sessionStorage.getItem('forceSaveUrl') || null;
const retryKey = 'aq_wip_retries_' + location.pathname;
let retries = parseInt(sessionStorage.getItem(retryKey) || '0', 10) + 1;
sessionStorage.setItem(retryKey, String(retries));
log('WIP: attempt', retries, 'for', forced);
const share = document.getElementById('SHARE_LONGLINK');
if (share) {
const finalLink = share.querySelector('input')?.value || null;
if (forced) saveProcessed(forced);
sessionStorage.removeItem('forceSaveUrl');
sessionStorage.removeItem(retryKey);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
if (finalLink) {
log('WIP done -> navigate to final:', finalLink);
window.location.href = finalLink;
return;
} else {
setTimeout(processQueue, PROCESS_DELAY);
return;
}
}
const exp = Math.min(retries - 1, MAX_BACKOFF_EXP);
const base = BASE_WIP_POLL_MS;
let delay = base * Math.pow(2, exp);
const jitter = Math.floor((Math.random() * 2 - 1) * BACKOFF_JITTER_RATIO * delay);
delay = Math.max(1000, delay + jitter);
const lastScheduled = parseInt(localStorage.getItem('aq_last_wip_reload') || '0', 10);
const now = Date.now();
if (lastScheduled > now) {
const remaining = Math.max(1000, lastScheduled - now);
log('WIP: another tab already scheduled reload; waiting remaining ms =', remaining);
setTimeout(() => {
log('WIP: performing scheduled reload now (other-tab coordination).');
try { localStorage.removeItem('aq_last_wip_reload'); } catch(e){}
location.reload();
}, remaining + 200);
return;
} else {
const scheduledTime = now + delay;
try { localStorage.setItem('aq_last_wip_reload', String(scheduledTime)); } catch (e) { log('localStorage write failed', e); }
log('WIP: scheduling reload in ms =', delay, '(scheduled timestamp =', new Date(scheduledTime).toISOString(), ')');
setTimeout(() => {
try { localStorage.removeItem('aq_last_wip_reload'); } catch (e) {}
log('WIP: reloading page now');
location.reload();
}, delay + 50);
return;
}
}
/* ===== Other page handlers (unchanged) ===== */
function handlePreCheckPage() {
log('handlePreCheckPage');
const q = getQueue();
if (!q.length) { setProcessingFlag(false); return; }
const current = q[0];
if (!location.hostname.match(/archive\.(ph|today|is|vn)$/i)) {
log('On external domain — waiting.');
return;
}
const selectors = ['#CONTENT', '#content', 'div[role="main"]', 'main', 'body'];
let content = null;
for (const s of selectors) { content = document.querySelector(s); if (content) break; }
if (!content) {
log('No content container found — marking restricted as fallback.');
saveRestricted(current, 'no-content');
q.shift(); saveQueue(q);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
return;
}
const text = (content.innerText || '').toLowerCase();
if (content.querySelector('.THUMBS-BLOCK') || text.includes('thumbnail') || text.includes('thumb')) {
log('Archive list detected on pre-check.');
if (text.includes('redirected to') || text.includes('redirected')) {
saveRestricted(current, 'redirected');
} else if (text.includes('restricted photo') || text.includes('you must be 18') || text.includes('post isn\'t available')) {
saveRestricted(current, 'restricted-content');
} else {
saveProcessed(current);
}
q.shift(); saveQueue(q);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
return;
}
const archiveLink = Array.from(content.querySelectorAll('a'))
.find(a => {
const t = (a.textContent || '').trim().toLowerCase();
const h = (a.getAttribute && (a.getAttribute('href')||'')).toLowerCase();
return t === 'archive this url' || h.includes('?url=') || h.includes('/submit/');
});
if (archiveLink) {
log('Archive-this-url link found. Setting forced submit and clicking link.');
sessionStorage.setItem('forceSaveUrl', current);
try { archiveLink.click(); } catch (e) { window.location.href = HOMEPAGE; }
return;
}
log('Pre-check: No results -> mark restricted');
saveRestricted(current, 'no-results');
q.shift(); saveQueue(q);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
}
function handleHomepage() {
log('handleHomepage');
const forced = sessionStorage.getItem('forceSaveUrl') || null;
if (!forced) {
setProcessingFlag(false);
setTimeout(processQueue, 350);
return;
}
log('Homepage: forced submit for', forced);
waitForInputAndSubmit(forced, (clicked) => {
log('Homepage: waitForInputAndSubmit result clicked=', clicked);
});
}
function handleSubmitPage() {
log('handleSubmitPage');
const captcha = document.querySelector('iframe[src*="recaptcha"], .g-recaptcha, [data-sitekey]') ||
(document.body.innerText || '').toLowerCase().includes("i'm not a robot") ||
(document.body.innerText || '').toLowerCase().includes('captcha');
if (captcha) {
log('CAPTCHA detected -> pausing.');
sessionStorage.setItem('processingPaused', '1');
updateOverlay();
// Notify user; let them solve it without script interference.
alert('Archive Queue paused: CAPTCHA detected. Solve it manually then Resume in the overlay.');
return;
}
const already = document.querySelector('#DIVALREADY, #DIVALREADY2, div[role="dialog"]');
if (already && (already.innerText || '').toLowerCase().includes('this page was last archived')) {
log('Submit page shows already archived popup — mark processed.');
const forced = sessionStorage.getItem('forceSaveUrl') || null;
if (forced) {
saveProcessed(forced);
const q = getQueue(); if (q.length && q[0] === forced) { q.shift(); saveQueue(q); }
sessionStorage.removeItem('forceSaveUrl');
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
}
return;
}
const saveBtn = Array.from(document.querySelectorAll('input[type="submit"], button[type="submit"]'))
.find(el => ((el.value || '').toLowerCase().includes('save')) || ((el.textContent || '').toLowerCase().includes('save')));
if (saveBtn) {
log('Submit page: Save button found -> clicking.');
setTimeout(()=> { try { saveBtn.click(); } catch (e) { try { saveBtn.dispatchEvent(new Event('click', { bubbles: true })); } catch(_){} } }, 120);
} else {
log('Submit page: no save button found.');
}
}
function handleFinalPage() {
log('handleFinalPage');
const q = getQueue();
if (!q.length) { sessionStorage.removeItem('aq_processing'); setProcessingFlag(false); return; }
const current = q[0];
const body = (document.body.innerText || '').toLowerCase();
const already = document.querySelector('#DIVALREADY, #DIVALREADY2, div[role="dialog"]');
if (already && (already.innerText || '').toLowerCase().includes('this page was last archived')) {
log('Final: already archived popup -> processed');
saveProcessed(current);
q.shift(); saveQueue(q);
sessionStorage.removeItem('forceSaveUrl');
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
return;
}
if (body.includes('restricted photo') || body.includes('you must be 18') || body.includes('post isn\'t available') || body.includes('this link may be broken') || body.includes('profile may have been removed') || body.includes('not available')) {
log('Final: restricted/unavailable detected -> mark restricted');
saveRestricted(current, 'restricted-or-unavailable');
q.shift(); saveQueue(q);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
return;
}
if (document.getElementById('SHARE_LONGLINK') || document.getElementById('HEADER') || document.querySelector('.THUMBS-BLOCK')) {
log('Final: success detected -> processed');
saveProcessed(current);
q.shift(); saveQueue(q);
sessionStorage.removeItem('forceSaveUrl');
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
return;
}
if (body.includes('redirected to')) {
log('Final: redirected to -> restricted');
saveRestricted(current, 'redirected-final');
q.shift(); saveQueue(q);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
return;
}
const saveBtn = Array.from(document.querySelectorAll('input[type="submit"], button[type="submit"]'))
.find(el => ((el.value || '').toLowerCase().includes('save')) || ((el.textContent || '').toLowerCase().includes('save')));
if (saveBtn) {
log('Final: found save button -> clicking as fallback');
setTimeout(()=> { try { saveBtn.click(); } catch(e){ try { saveBtn.dispatchEvent(new Event('click', { bubbles: true })); } catch(_){} } }, 80);
return;
}
log('Final: unknown page -> mark restricted (fallback)');
saveRestricted(current, 'unknown-final');
q.shift(); saveQueue(q);
sessionStorage.removeItem('aq_processing');
setProcessingFlag(false);
setTimeout(processQueue, PROCESS_DELAY);
}
/* ===== Router ===== */
function mainRouter() {
createOverlay();
updateOverlay();
if (isPaused()) { log('Processing is paused (user).'); setProcessingFlag(false); return; }
const path = location.pathname || '/';
const search = location.search || '';
if (path.startsWith('/wip/')) { handleWipPage(); return; }
if (path.startsWith('/submit/')) { handleSubmitPage(); return; }
if (/^\/https?:\/\//i.test(path) || search.toLowerCase().includes('url=')) {
if (path === '/' && search.toLowerCase().includes('url=')) { handleHomepage(); return; }
if (path.startsWith('/https://') || path.startsWith('/http://')) { handlePreCheckPage(); return; }
}
if (path === '/' || path === '') { handleHomepage(); return; }
handleFinalPage();
}
if (document.readyState === 'loading') window.addEventListener('DOMContentLoaded', mainRouter);
else mainRouter();
setTimeout(() => { if (!isPaused()) processQueue(); }, 900);
})();