Fanbox Batch Downloader

Batch Download on creator, not post

当前为 2021-07-19 提交的版本,查看 最新版本

  1. // ==UserScript==
  2. // @name Fanbox Batch Downloader
  3. // @namespace http://tampermonkey.net/
  4. // @version 0.680
  5. // @description Batch Download on creator, not post
  6. // @author https://github.com/amarillys QQ 719862760
  7. // @require https://cdnjs.cloudflare.com/ajax/libs/jszip/3.2.2/jszip.min.js
  8. // @require https://cdnjs.cloudflare.com/ajax/libs/dat-gui/0.7.6/dat.gui.min.js
  9. // @match https://*.fanbox.cc/*
  10. // @match https://www.fanbox.cc/*
  11. // @grant GM_xmlhttpRequest
  12. // @grant GM_addStyle
  13. // @grant unsafeWindow
  14. // @run-at document-end
  15. // @license MIT
  16. // ==/UserScript==
  17.  
  18. /**
  19. * Update Log
  20. * > 210719
  21. * Fix some logic error.
  22. * > 210301
  23. * Move id behind title in folder name.
  24. * > 210216
  25. * Add semi-custom name function.
  26. * > 200514
  27. * Decrease the pack size to avoid the stack overflow.
  28. * > 200429
  29. * Bug Fixed!
  30. * > 200427
  31. * Adapt to new Fanbox Change!
  32. * Add post id to folder name!
  33. * > 200328
  34. * Improve file naming
  35. * Fix bugs that may cause files being skipped
  36. * Add text if exist in post
  37. * > 200226
  38. * Adapt to new Api! Add Error Tip!
  39. * More frequentyle progress bar!
  40. * More clearly status!
  41. * > 200224
  42. * More beautiful! UI Redesigned. --use dat.gui,
  43. * Performence Improved. -- multi-thread supported.
  44. * > 200222
  45. * Bug Fixed - Psd files download failure <Change download type from blob to arraybuffer, which cause low performence>
  46. * Bug Fixed - Display incorrect on partial download
  47. * > 200222
  48. * Bug Fixed - Post with '/' cause deep path in zip
  49. * > 200102
  50. * Bug Fixed - Caused by empty cover
  51. * > 191228
  52. * Bug Fixed
  53. * Correct filenames
  54. * > 191227
  55. * Code Reconstruct
  56. * Support downloading of artice
  57. * Correct filenames
  58. *
  59. * // 中文注释
  60. * 代码重构
  61. * 新增对文章的下载支持
  62. * > 200222
  63. * 偷懒,以后不加中文注释
  64. * > 191226
  65. * Support downloading by batch(default: 100 files per batch)
  66. * Support donwloading by specific index
  67. * // 中文注释
  68. * 新增支持分批下载的功能(默认100个文件一个批次)
  69. * 新增支持按索引下载的功能
  70. *
  71. * > 191223
  72. * Add support of files
  73. * Improve the detect of file extension
  74. * Change Download Request as await, for avoiding delaying.
  75. * Add manual package while click button use middle button of mouse
  76. * // 中文注释
  77. * 增加对附件下载的支持
  78. * 优化文件后缀名识别
  79. * 修改下载方式为按顺序下载,避免超时
  80. * 增加当鼠标中键点击时手动打包
  81. **/
  82.  
  83. /* global JSZip GM_xmlhttpRequest */
  84. ;(function() {
  85. 'use strict'
  86.  
  87. const apiUserUri = 'https://api.fanbox.cc/creator.get'
  88. const apiPostUri = 'https://api.fanbox.cc/post.listCreator'
  89. // set style
  90. GM_addStyle(`
  91. .dg.main{
  92. top: 16px;
  93. position: fixed;
  94. left: 20%;
  95. filter: drop-shadow(2px 4px 6px black);
  96. opacity: 0.8;
  97. z-index: 999;
  98. }
  99. li.cr.number.has-slider:nth-child(2) {
  100. pointer-events: none;
  101. }
  102. .slider-fg {
  103. transition: width 0.5s ease-out;
  104. }
  105. `)
  106.  
  107. window = unsafeWindow
  108. class ThreadPool {
  109. constructor(poolSize) {
  110. this.size = poolSize || 20
  111. this.running = 0
  112. this.waittingTasks = []
  113. this.callback = []
  114. this.tasks = []
  115. this.counter = 0
  116. this.sum = 0
  117. this.finished = false
  118. this.errorLog = ''
  119. this.step = () => {}
  120. this.timer = null
  121. this.callback.push(() =>
  122. console.log(this.errorLog)
  123. )
  124. }
  125.  
  126. status() {
  127. return ((this.counter / this.sum) * 100).toFixed(1) + '%'
  128. }
  129.  
  130. run() {
  131. if (this.finished) return
  132. if (this.waittingTasks.length === 0)
  133. if (this.running <= 0) {
  134. for (let m = 0; m < this.callback.length; ++m)
  135. this.callback[m] && this.callback[m]()
  136. this.finished = true
  137. } else return
  138.  
  139. while (this.running < this.size) {
  140. if (this.waittingTasks.length === 0) return
  141. let curTask = this.waittingTasks[0]
  142. curTask.do().then(
  143. onSucceed => {
  144. this.running--
  145. this.counter++
  146. this.step()
  147. this.run()
  148. typeof onSucceed === 'function' && onSucceed()
  149. },
  150. onFailed => {
  151. this.errorLog += onFailed + '\n'
  152. this.running--
  153. this.counter++
  154. this.step()
  155. this.run()
  156. curTask.err()
  157. }
  158. )
  159. this.waittingTasks.splice(0, 1)
  160. this.tasks.push(this.waittingTasks[0])
  161. this.running++
  162. }
  163. }
  164.  
  165. add(fn, errFn) {
  166. this.waittingTasks.push({ do: fn, err: errFn || (() => {}) })
  167. this.sum++
  168. clearTimeout(this.timer)
  169. this.timer = setTimeout(() => {
  170. this.run()
  171. clearTimeout(this.timer)
  172. }, this.autoStartTime)
  173. }
  174.  
  175. setAutoStart(time) {
  176. this.autoStartTime = time
  177. }
  178.  
  179. finish(callback) {
  180. this.callback.push(callback)
  181. }
  182.  
  183. isFinished() {
  184. return this.finished
  185. }
  186. }
  187.  
  188. class Zip {
  189. constructor(title) {
  190. this.title = title
  191. this.zip = new JSZip()
  192. this.size = 0
  193. this.partIndex = 0
  194. }
  195. file(filename, blob) {
  196. this.zip.file(filename, blob, {
  197. compression: 'STORE'
  198. })
  199. this.size += blob.size
  200. }
  201. add(folder, name, blob) {
  202. if (this.size + blob.size >= Zip.MAX_SIZE)
  203. this.pack()
  204. this.zip.folder(folder).file(name, blob, {
  205. compression: 'STORE'
  206. })
  207. this.size += blob.size
  208. }
  209. pack() {
  210. if (this.size === 0) return
  211. let index = this.partIndex
  212. this.zip
  213. .generateAsync({
  214. type: 'blob'
  215. })
  216. .then(zipBlob => saveBlob(zipBlob, `${this.title}-${index}.zip`))
  217. this.partIndex++
  218. this.zip = new JSZip()
  219. this.size = 0
  220. }
  221. }
  222. Zip.MAX_SIZE = 800000000/*1048576000*/
  223.  
  224. const creatorId = document.URL.startsWith('https://www') ?
  225. document.URL.match(/@([\w_-]+)\/?/)?.[1] : document.URL.match(/https:\/\/(.+).fanbox/)?.[1]
  226. if (!creatorId) return;
  227. let creatorInfo = null
  228. let options = {
  229. start: 1,
  230. end: 1,
  231. thread: 6,
  232. batch: 200,
  233. progress: 0,
  234. speed: 0,
  235. nameWithId: 0,
  236. nameWithDate: 1,
  237. nameWithTitle: 1
  238. }
  239.  
  240. const Text = {
  241. batch: '分批 / Batch',
  242. download: '点击这里下载',
  243. download_en: 'Click to Download',
  244. downloading: '下载中...',
  245. downloading_en: 'Downloading...',
  246. packing: '打包中...',
  247. packing_en: 'Packing...',
  248. packed: '打包完成',
  249. packed_en: 'Packed!',
  250. init: '初始化中...',
  251. init_en: 'Initilizing...',
  252. initFailed: '请求数据失败',
  253. initFailed_en: 'Failed to get Data',
  254. initFailed_0: '请检查网络',
  255. initFailed_0_en: 'check network',
  256. initFailed_1: '或Github联系作者',
  257. initFailed_1_en: 'or connect at Github',
  258. initFinished: '初始化完成',
  259. initFinished_en: 'Initilized',
  260. name_with_id: '文件名带ID',
  261. name_with_id_en: 'name with id',
  262. name_with_date: '文件名带日期',
  263. name_with_date_en: 'name with date',
  264. name_with_title: '文件名带名字',
  265. name_with_title_en: 'name with title',
  266. start: '起始 / start',
  267. end: '结束 / end',
  268. thread: '线程 / threads',
  269. pack: '手动打包(不推荐)',
  270. pack_en: 'manual pack(Not Rcm)',
  271. progress: '进度 / Progress',
  272. speed: '网速 / speed'
  273. }
  274. const EN_FIX = navigator.language.indexOf('zh') > -1 ? '' : '_en'
  275.  
  276. let label = null
  277. const gui = new dat.GUI({
  278. autoPlace: false,
  279. useLocalStorage: false
  280. })
  281.  
  282. const clickHandler = {
  283. text() {},
  284. download: () => {
  285. console.log('startDownloading')
  286. downloadByFanboxId(creatorInfo, creatorId)
  287. },
  288. pack() {
  289. label.name(Text['packing' + EN_FIX])
  290. zip.pack()
  291. label.name(Text['packed' + EN_FIX])
  292. }
  293. }
  294.  
  295. label = gui.add(clickHandler, 'text').name(Text['init' + EN_FIX])
  296. let progressCtl = null
  297.  
  298. let init = async () => {
  299. let base = unsafeWindow.document.querySelector('#root')
  300.  
  301. base.appendChild(gui.domElement)
  302. uiInited = true
  303.  
  304. try {
  305. creatorInfo = await getAllPostsByFanboxId(creatorId)
  306. label.name(Text['initFinished' + EN_FIX])
  307. } catch (e) {
  308. label.name(Text['initFailed' + EN_FIX])
  309. gui.add(clickHandler, 'text').name(Text['initFailed_0' + EN_FIX])
  310. gui.add(clickHandler, 'text').name(Text['initFailed_1' + EN_FIX])
  311. return
  312. }
  313.  
  314. // init dat gui
  315. const sum = creatorInfo.posts.length
  316. progressCtl = gui.add(options, 'progress', 0, 100, 0.01).name(Text.progress)
  317. const startCtl = gui.add(options, 'start', 1, sum, 1).name(Text.start)
  318. const endCtl = gui.add(options, 'end', 1, sum, 1).name(Text.end)
  319. gui.add(options, 'thread', 1, 20, 1).name(Text.thread)
  320. gui.add(options, 'batch', 10, 5000, 10).name(Text.batch)
  321. gui.add(options, 'nameWithId', 0, 1, 1).name(Text['name_with_id' + EN_FIX])
  322. gui.add(options, 'nameWithDate', 0, 1, 1).name(Text['name_with_date' + EN_FIX])
  323. // gui.add(options, 'nameWithTitle', 0, 1, 1).name(Text['name_with_title' + EN_FIX])
  324. gui.add(clickHandler, 'download').name(Text['download' + EN_FIX])
  325. gui.add(clickHandler, 'pack').name(Text['pack' + EN_FIX])
  326. endCtl.setValue(sum)
  327. startCtl.onChange(() => (options.start = options.start > options.end ? options.end : options.start))
  328. endCtl.onChange(() => (options.end = options.end < options.start ? options.start : options.end ))
  329. gui.open()
  330. }
  331.  
  332. // init global values
  333. let zip = null
  334. let amount = 1
  335. let pool = null
  336. let progressList = []
  337. let uiInited = false
  338.  
  339. const fetchOptions = {
  340. credentials: 'include',
  341. headers: {
  342. Accept: 'application/json, text/plain, */*'
  343. }
  344. }
  345.  
  346. const setProgress = amount => {
  347. let currentProgress = progressList.reduce((p, q) => p + q, 0) / amount * 100
  348. if (currentProgress > 0)
  349. progressCtl.setValue(currentProgress)
  350. }
  351.  
  352. window.onload = () => {
  353. init()
  354. let timer = setInterval(() => {
  355. (!uiInited && document.querySelector('.dg.main') === null) ? init() : clearInterval(timer)
  356. }, 3000)
  357. }
  358.  
  359. function gmRequireImage(url, index) {
  360. return new Promise((resolve, reject) =>
  361. GM_xmlhttpRequest({
  362. method: 'GET',
  363. url,
  364. overrideMimeType: 'application/octet-stream',
  365. responseType: 'blob',
  366. asynchrouns: true,
  367. onload: res => {
  368. progressList[index] = 1
  369. setProgress(amount)
  370. resolve(res.response)
  371. },
  372. onprogress: res => {
  373. progressList[index] = res.done / res.total
  374. setProgress(amount)
  375. },
  376. onerror: () =>
  377. GM_xmlhttpRequest({
  378. method: 'GET',
  379. url,
  380. overrideMimeType: 'application/octet-stream',
  381. responseType: 'arraybuffer',
  382. onload: res => {
  383. progressList[index] = 1
  384. setProgress(amount)
  385. resolve(new Blob([res.response]))
  386. },
  387. onprogress: res => {
  388. progressList[index] = res.done / res.total
  389. setProgress(amount)
  390. },
  391. onerror: res => reject(res)
  392. })
  393. })
  394. )
  395. }
  396.  
  397. async function downloadByFanboxId(creatorInfo, creatorId) {
  398. let processed = 0
  399. amount = 0
  400. label.name(Text['downloading' + EN_FIX])
  401. progressCtl.setValue(0)
  402. let { batch, end, start, thread } = options
  403. options.progress = 0
  404. zip = new Zip(`${creatorInfo.name}@${start}-${end}`)
  405. let stepped = 0
  406. // init pool
  407. pool = new ThreadPool(thread)
  408. pool.finish(() => {
  409. label.name(Text['packing' + EN_FIX])
  410. zip.pack()
  411. label.name(Text['packed' + EN_FIX])
  412. })
  413.  
  414. // for name exist detect
  415. let titles = []
  416. progressList = new Array(amount).fill(0)
  417. pool.step = () => {
  418. console.log(` Progress: ${processed} / ${amount}, Pool: ${pool.running} @ ${pool.sum}`)
  419. if (stepped >= batch) {
  420. zip.pack()
  421. stepped = 0
  422. }
  423. }
  424.  
  425. // start downloading
  426. for (let i = start - 1, p = creatorInfo.posts; i < end; ++i) {
  427. let folder = '';
  428. options.nameWithDate === 1 && (folder += `[${p[i].publishedDatetime.split('T')[0].replace(/-/g, '')}] - `);
  429. folder += p[i].title.replace(/\//g, '-');
  430. options.nameWithId === 1 && (folder += ` - ${p[i].id}`);
  431. let titleExistLength = titles.filter(title => title === folder).length
  432. if (titleExistLength > 0) folder += `-${titleExistLength}`
  433. titles.push(folder)
  434. if (!p[i].body) continue
  435. let { blocks, imageMap, fileMap, files, images, text } = p[i].body
  436. let picIndex = 0
  437. let imageList = []
  438. let fileList = []
  439.  
  440. if (p[i].type === 'article') {
  441. let article = `# ${p[i].title}\n`
  442. for (let j = 0; j < blocks.length; ++j) {
  443. switch (blocks[j].type) {
  444. case 'p': {
  445. article += `${blocks[j].text}\n\n`
  446. break
  447. }
  448. case 'image': {
  449. picIndex++
  450. let image = imageMap[blocks[j].imageId]
  451. imageList.push(image)
  452. article += `![${p[i].title} - P${picIndex}](${folder}_${j}.${image.extension})\n\n`
  453. break
  454. }
  455. case 'file': {
  456. let file = fileMap[blocks[j].fileId]
  457. fileList.push(file)
  458. article += `[${p[i].title} - ${file.name}](${creatorId}-${folder}-${file.name}.${file.extension})\n\n`
  459. break
  460. }
  461. }
  462. }
  463.  
  464. zip.add(folder, 'article.md', new Blob([article]))
  465. for (let j = 0; j < imageList.length; ++j) {
  466. let image = imageList[j]
  467. let index = amount
  468. amount++
  469. pool.add(() => new Promise((resolve, reject) => {
  470. gmRequireImage(image.originalUrl, index).then(blob => {
  471. processed++
  472. zip.add(folder, `${folder}_${j}.${image.extension}`, blob)
  473. stepped++
  474. resolve()
  475. }).catch(() => {
  476. console.log(`Failed to download: ${image.originalUrl}`)
  477. reject()
  478. })
  479. }))
  480. }
  481. for (let j = 0; j < fileList.length; ++j) {
  482. let file = fileList[j]
  483. let index = amount
  484. amount++
  485. pool.add(() => new Promise((resolve, reject) => {
  486. let fileIndexText = ''
  487. if (fileList.length > 1) fileIndexText = `-${j}`
  488. gmRequireImage(file.url, index).then(blob => {
  489. processed++
  490. saveBlob(blob, `${creatorInfo.name}-${folder}${fileIndexText}-${file.name}.${file.extension}`)
  491. stepped++
  492. resolve()
  493. }).catch(() => {
  494. console.log(`Failed to download: ${file.url}`)
  495. reject()
  496. })
  497. }))
  498. }
  499. }
  500.  
  501. if (files) {
  502. for (let j = 0; j < files.length; ++j) {
  503. let file = files[j]
  504. let index = amount
  505. amount++
  506. pool.add(() => new Promise((resolve, reject) => {
  507. gmRequireImage(file.url, index).then(blob => {
  508. processed++
  509. let fileIndexText = ''
  510. if (files.length > 1) fileIndexText = `-${j}`
  511. if (blob.size < 51200000 * 3)
  512. zip.add(folder, `${file.name}${fileIndexText}.${file.extension}`, blob)
  513. else
  514. saveBlob(blob, `${creatorInfo.name}@${folder}${fileIndexText}.${file.extension}`)
  515. stepped++
  516. resolve()
  517. }).catch(() => {
  518. console.log(`Failed to download: ${file.url}`)
  519. reject()
  520. })
  521. }))
  522. }
  523. }
  524. if (images) {
  525. for (let j = 0; j < images.length; ++j) {
  526. let image = images[j]
  527. let index = amount
  528. amount++
  529. pool.add(() => new Promise((resolve, reject) => {
  530. gmRequireImage(image.originalUrl, index).then(blob => {
  531. processed++
  532. zip.add(folder, `${folder}_${j}.${image.extension}`, blob)
  533. stepped++
  534. resolve()
  535. }).catch(() => {
  536. console.log(`Failed to download: ${image.url}`)
  537. reject()
  538. })
  539. }))
  540. }
  541. }
  542.  
  543. if (text) {
  544. let textBlob = new Blob([text], { type: 'text/plain' })
  545. zip.add(folder, `${creatorInfo.name}-${folder}.txt`, textBlob)
  546. }
  547. }
  548.  
  549. if (creatorInfo.cover)
  550. gmRequireImage(creatorInfo.cover, 0).then(blob => {
  551. zip.file('cover.jpg', blob)
  552. if (amount === 0) zip.pack()
  553. })
  554. }
  555.  
  556. async function getAllPostsByFanboxId(creatorId) {
  557. // request userinfo
  558. const userUri = `${apiUserUri}?creatorId=${creatorId}`
  559. const userData = await (await fetch(userUri, fetchOptions)).json()
  560. let creatorInfo = {
  561. cover: null,
  562. posts: []
  563. }
  564. const limit = 56
  565. creatorInfo.cover = userData.body.coverImageUrl
  566. creatorInfo.name = userData.body.user.name
  567.  
  568. // request post info
  569. let postData = await (await fetch(`${apiPostUri}?creatorId=${creatorId}&limit=${limit}`, fetchOptions)).json()
  570. creatorInfo.posts.push(...postData.body.items.filter(p => p.body))
  571. let nextPageUrl = postData.body.nextUrl
  572. while (nextPageUrl) {
  573. let nextData = await (await fetch(nextPageUrl, fetchOptions)).json()
  574. creatorInfo.posts.push(...nextData.body.items.filter(p => p.body))
  575. nextPageUrl = nextData.body.nextUrl
  576. }
  577. console.log(creatorInfo)
  578. return creatorInfo
  579. }
  580.  
  581. function saveBlob(blob, fileName) {
  582. let downloadDom = document.createElement('a')
  583. document.body.appendChild(downloadDom)
  584. downloadDom.style = `display: none`
  585. let url = window.URL.createObjectURL(blob)
  586. downloadDom.href = url
  587. downloadDom.download = fileName
  588. downloadDom.click()
  589. window.URL.revokeObjectURL(url)
  590. }
  591. })()