下载抖音用户主页数据!
当前为
// ==UserScript==
// @name douyin-user-data-download
// @namespace http://tampermonkey.net/
// @version 0.4.0
// @description 下载抖音用户主页数据!
// @author xxmdmst
// @match https://www.douyin.com/*
// @icon https://xxmdmst.oss-cn-beijing.aliyuncs.com/imgs/favicon.ico
// @grant none
// @require https://cdnjs.cloudflare.com/ajax/libs/jszip/3.6.0/jszip.min.js
// @license MIT
// ==/UserScript==
(function () {
let table;
function initGbkTable() {
// https://en.wikipedia.org/wiki/GBK_(character_encoding)#Encoding
const ranges = [
[0xA1, 0xA9, 0xA1, 0xFE],
[0xB0, 0xF7, 0xA1, 0xFE],
[0x81, 0xA0, 0x40, 0xFE],
[0xAA, 0xFE, 0x40, 0xA0],
[0xA8, 0xA9, 0x40, 0xA0],
[0xAA, 0xAF, 0xA1, 0xFE],
[0xF8, 0xFE, 0xA1, 0xFE],
[0xA1, 0xA7, 0x40, 0xA0],
];
const codes = new Uint16Array(23940);
let i = 0;
for (const [b1Begin, b1End, b2Begin, b2End] of ranges) {
for (let b2 = b2Begin; b2 <= b2End; b2++) {
if (b2 !== 0x7F) {
for (let b1 = b1Begin; b1 <= b1End; b1++) {
codes[i++] = b2 << 8 | b1
}
}
}
}
table = new Uint16Array(65536);
table.fill(0xFFFF);
const str = new TextDecoder('gbk').decode(codes);
for (let i = 0; i < str.length; i++) {
table[str.charCodeAt(i)] = codes[i]
}
}
function str2gbk(str, opt = {}) {
if (!table) {
initGbkTable()
}
const NodeJsBufAlloc = typeof Buffer === 'function' && Buffer.allocUnsafe;
const defaultOnAlloc = NodeJsBufAlloc
? (len) => NodeJsBufAlloc(len)
: (len) => new Uint8Array(len);
const defaultOnError = () => 63;
const onAlloc = opt.onAlloc || defaultOnAlloc;
const onError = opt.onError || defaultOnError;
const buf = onAlloc(str.length * 2);
let n = 0;
for (let i = 0; i < str.length; i++) {
const code = str.charCodeAt(i);
if (code < 0x80) {
buf[n++] = code;
continue
}
const gbk = table[code];
if (gbk !== 0xFFFF) {
buf[n++] = gbk;
buf[n++] = gbk >> 8
} else if (code === 8364) {
buf[n++] = 0x80
} else {
const ret = onError(i, str);
if (ret === -1) {
break
}
if (ret > 0xFF) {
buf[n++] = ret;
buf[n++] = ret >> 8
} else {
buf[n++] = ret
}
}
}
return buf.subarray(0, n)
}
function formatSeconds(seconds) {
const timeUnits = ['小时', '分', '秒'];
const timeValues = [
Math.floor(seconds / 3600),
Math.floor((seconds % 3600) / 60),
seconds % 60
];
return timeValues.map((value, index) => value > 0 ? value + timeUnits[index] : '').join('');
}
const timeFormat = (timestamp = null, fmt = 'yyyy-mm-dd') => {
// 其他更多是格式化有如下:
// yyyy:mm:dd|yyyy:mm|yyyy年mm月dd日|yyyy年mm月dd日 hh时MM分等,可自定义组合
timestamp = parseInt(timestamp);
// 如果为null,则格式化当前时间
if (!timestamp) timestamp = Number(new Date());
// 判断用户输入的时间戳是秒还是毫秒,一般前端js获取的时间戳是毫秒(13位),后端传过来的为秒(10位)
if (timestamp.toString().length === 10) timestamp *= 1000;
let date = new Date(timestamp);
let ret;
let opt = {
"y{4,}": date.getFullYear().toString(), // 年
"y+": date.getFullYear().toString().slice(2,), // 年
"m+": (date.getMonth() + 1).toString(), // 月
"d+": date.getDate().toString(), // 日
"h+": date.getHours().toString(), // 时
"M+": date.getMinutes().toString(), // 分
"s+": date.getSeconds().toString() // 秒
// 有其他格式化字符需求可以继续添加，必须转化成字符串
};
for (let k in opt) {
ret = new RegExp("(" + k + ")").exec(fmt);
if (ret) {
fmt = fmt.replace(ret[1], (ret[1].length === 1) ? (opt[k]) : (opt[k].padStart(ret[1].length, "0")))
}
}
return fmt
};
let aweme_list = [];
let userKey = [
"昵称", "关注", "粉丝", "获赞",
"抖音号", "IP属地", "性别",
"位置", "签名", "作品数", "主页"
];
let userData = [];
let createEachButtonTimer;
function copyText(text, node) {
let oldText = node.textContent;
navigator.clipboard.writeText(text).then(r => {
node.textContent = "复制成功";
}).catch((e) => {
node.textContent = "复制失败";
})
setTimeout(() => node.textContent = oldText, 2000);
}
function copyUserData(node) {
if (userData.length === 0) {
alert("没有捕获到用户数据！");
return;
}
let text = [];
for (let i = 0; i < userKey.length; i++) {
let key = userKey[i];
let value = userData[userData.length - 1][i];
if (value) text.push(key + ":" + value);
}
copyText(text.join("\n"), node);
}
function createVideoButton(text, top, func) {
const button = document.createElement("button");
button.textContent = text;
button.style.position = "absolute";
button.style.right = "0px";
button.style.top = top;
button.style.opacity = "0.5";
if (func) {
button.addEventListener("click", (event) => {
event.preventDefault();
event.stopPropagation();
func();
});
}
return button;
}
function createDownloadLink(blob, filename, ext, prefix = "") {
if (filename === null) {
filename = userData.length > 0 ? userData[userData.length - 1][0] : document.title;
}
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.download = prefix + filename.replace(/[\/:*?"<>|\s]/g, "").slice(0, 40) + "." + ext;
link.click();
URL.revokeObjectURL(url);
}
function txt2file(txt, filename, ext) {
createDownloadLink(new Blob([txt], {type: 'text/plain'}), filename, ext);
}
function getAwemeName(aweme) {
let name = aweme.item_title ? aweme.item_title : aweme.caption;
if (!name) name = aweme.desc ? aweme.desc : aweme.awemeId;
return `【${aweme.date.slice(0, 10)}】` + name.replace(/[\/:*?"<>|\s]+/g, "").slice(0, 27).replace(/\.\d+$/g, "");
}
function createEachButton() {
let targetNodes = document.querySelectorAll("div[data-e2e='user-post-list'] > ul[data-e2e='scroll-list'] > li a");
for (let i = 0; i < targetNodes.length; i++) {
let targetNode = targetNodes[i];
if (targetNode.dataset.added) {
continue;
}
let aweme = aweme_list[i];
let copyDescButton = createVideoButton("复制描述", "0px");
copyDescButton.addEventListener("click", (event) => {
event.preventDefault();
event.stopPropagation();
copyText(aweme.desc, copyDescButton);
})
targetNode.appendChild(copyDescButton);
targetNode.appendChild(createVideoButton("打开视频源", "21px", () => window.open(aweme.url)));
let downloadVideoButton = createVideoButton("下载视频", "42px", () => {
let xhr = new XMLHttpRequest();
xhr.open('GET', aweme.url.replace("http://", "https://"), true);
xhr.responseType = 'blob';
xhr.onload = (e) => {
createDownloadLink(xhr.response, getAwemeName(aweme), (aweme.images ? "mp3" : "mp4"));
};
xhr.onprogress = (event) => {
if (event.lengthComputable) {
downloadVideoButton.textContent = "下载" + (event.loaded * 100 / event.total).toFixed(1) + '%';
}
};
xhr.send();
});
targetNode.appendChild(downloadVideoButton);
if (aweme.images) {
targetNode.appendChild(createVideoButton("图片打包下载", "63px", () => {
const zip = new JSZip();
// console.log(aweme.images);
downloadVideoButton.textContent = "图片下载并打包中...";
const promises = aweme.images.map((link, index) => {
return fetch(link)
.then((response) => response.arrayBuffer())
.then((buffer) => {
downloadVideoButton.textContent = `图片已下载【${index + 1}/${aweme.images.length}】`;
zip.file(`image_${index + 1}.jpg`, buffer);
});
});
Promise.all(promises)
.then(() => {
return zip.generateAsync({type: "blob"});
})
.then((content) => {
createDownloadLink(content, getAwemeName(aweme), "zip", "【图文】");
downloadVideoButton.textContent = "图文打包完成";
});
}));
}
targetNode.dataset.added = "true";
}
}
function flush() {
if (createEachButtonTimer !== undefined) {
clearTimeout(createEachButtonTimer);
createEachButtonTimer = undefined;
}
createEachButtonTimer = setTimeout(createEachButton, 500);
data_button.p2.textContent = `${aweme_list.length}`;
let img_num = aweme_list.filter(a => a.images).length;
img_button.p2.textContent = `${img_num}`;
msg_pre.textContent = `已加载${aweme_list.length}个作品，${img_num}个图文\n激活上方头像可展开下载按钮`;
}
let flag = false;
function formatJsonData(json_data) {
return json_data.aweme_list.map(item => Object.assign(
{
"awemeId": item.aweme_id,
"item_title": item.item_title,
"caption": item.caption,
"desc": item.desc,
"tag": item.text_extra.map(tag => tag.hashtag_name).filter(tag => tag).join("#"),
"video_tag": item.video_tag.map(tag => tag.tag_name).filter(tag => tag).join("->")
},
{
"diggCount": item.statistics.digg_count,
"commentCount": item.statistics.comment_count,
"collectCount": item.statistics.collect_count,
"shareCount": item.statistics.share_count
},
{
"date": timeFormat(item.create_time, "yyyy-mm-dd hh:MM:ss"),
"duration": formatSeconds(Math.round(item.video.duration / 1000)),
"url": item.video.play_addr.url_list[0],
"cover": item.video.cover.url_list[0]
},
{
"images": item.images ? item.images.map(row => row.url_list.pop()) : null
}
));
}
function interceptResponse() {
const originalSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function () {
const self = this;
this.onreadystatechange = function () {
if (self.readyState === 4 && self._url) {
if (self._url.indexOf("/aweme/v1/web/aweme/post") > -1) {
aweme_list.push(...formatJsonData(JSON.parse(self.response)));
if (domLoadedTimer === null) {
flush();
} else {
flag = true;
}
} else if (self._url.indexOf("/aweme/v1/web/user/profile/other") > -1) {
let userInfo = JSON.parse(self.response).user;
for (let key in userInfo) {
if (!userInfo[key]) userInfo[key] = "";
}
if (userInfo.district) userInfo.city += "·" + userInfo.district;
userInfo.unique_id = '\t' + (userInfo.unique_id ? userInfo.unique_id : userInfo.short_id);
userData.push([
userInfo.nickname, userInfo.following_count, userInfo.mplatform_followers_count,
userInfo.total_favorited, userInfo.unique_id, userInfo.ip_location,
userInfo.gender === 2 ? "女" : "男",
userInfo.city, '"' + userInfo.signature + '"', userInfo.aweme_count, "https://www.douyin.com/user/" + userInfo.sec_uid
]);
}
}
};
originalSend.apply(this, arguments);
};
}
function downloadData(node, encoding) {
if (node.disabled) {
alert("下载正在处理中，请不要重复点击按钮！");
return;
}
node.disabled = true;
try {
// if (userData.length > 0) {
// text += userKey.join(",") + "\n";
// text += userData.map(row => row.join(",")).join("\n") + "\n\n";
// }
let text = "作品描述,作品链接,点赞数,评论数,收藏数,分享数,发布时间,时长,标签,分类,封面,下载链接\n";
aweme_list.forEach(aweme => {
text += ['"' + aweme.desc.replace(/,/g, '，').replace(/"/g, '""') + '"',
"https://www.douyin.com/video/" + aweme.awemeId,
aweme.diggCount, aweme.commentCount,
aweme.collectCount, aweme.shareCount, aweme.date,
aweme.duration, aweme.tag, aweme.video_tag,
aweme.cover, aweme.url].join(",") + "\n"
});
if (encoding === "gbk") {
text = str2gbk(text);
}
txt2file(text, null, "csv");
} finally {
node.disabled = false;
}
}
let img_button, data_button, msg_pre;
function createMsgBox() {
msg_pre = document.createElement('pre');
msg_pre.textContent = '等待上方头像加载完毕';
msg_pre.style.color = 'white';
msg_pre.style.position = 'fixed';
msg_pre.style.right = '5px';
msg_pre.style.top = '60px';
msg_pre.style.color = 'white';
msg_pre.style.zIndex = '90000';
msg_pre.style.opacity = "0.5";
document.body.appendChild(msg_pre);
}
function scrollPageToBottom(scroll_button) {
let scrollInterval;
function scrollLoop() {
let endText = document.querySelector("div[data-e2e='user-post-list'] > ul[data-e2e='scroll-list'] + div div").innerText;
if (endText || (userData.length > 0 && aweme_list.length > userData[userData.length - 1][9] - 5)) {
clearInterval(scrollInterval);
scrollInterval = null;
scroll_button.p1.textContent = "已加载全部！";
} else {
scrollTo(0, document.body.scrollHeight);
}
}
scroll_button.addEventListener('click', () => {
if (!scrollInterval) {
scrollInterval = setInterval(scrollLoop, 1200);
scroll_button.p1.textContent = "停止自动下拉";
} else {
clearInterval(scrollInterval);
scrollInterval = null;
scroll_button.p1.textContent = "开启自动下拉";
}
});
}
function createAllButton() {
let dom = document.querySelector("#douyin-header-menuCt pace-island > div > div:nth-last-child(1) ul a:nth-last-child(1)");
let baseNode = dom.cloneNode(true);
baseNode.removeAttribute("target");
baseNode.removeAttribute("rel");
baseNode.removeAttribute("href");
let svgChild = baseNode.querySelector("svg");
if (svgChild) baseNode.removeChild(svgChild);
function createNewButton(name, num = "0") {
let button = baseNode.cloneNode(true);
button.p1 = button.querySelector("p:nth-child(1)");
button.p2 = button.querySelector("p:nth-child(2)");
button.p1.textContent = name;
button.p2.textContent = num;
dom.after(button);
return button;
}
function createCommonElement(tagName, attrs = {}, text = "") {
const tag = document.createElement(tagName);
for (const [k, v] of Object.entries(attrs)) {
tag.setAttribute(k, v);
}
if (text) tag.textContent = text;
tag.addEventListener('click', (event) => event.stopPropagation());
return tag;
}
img_button = createNewButton("图文打包下载");
img_button.addEventListener('click', () => downloadImg(img_button));
let downloadCoverButton = createNewButton("封面打包下载", "");
downloadCoverButton.addEventListener('click', () => downloadCover(downloadCoverButton));
data_button = createNewButton("下载已加载的数据");
data_button.p1.after(createCommonElement("label", {'for': 'gbk'}, 'gbk'));
let checkbox = createCommonElement("input", {'type': 'checkbox', 'id': 'gbk'});
data_button.p1.after(checkbox);
data_button.addEventListener('click', () => downloadData(data_button, checkbox.checked ? "gbk" : "utf-8"));
scrollPageToBottom(createNewButton("开启自动下拉到底", ""));
let share_button = document.querySelector("#frame-user-info-share-button");
let node = share_button.cloneNode(true);
node.span = node.querySelector("span");
node.span.innerHTML = "复制作者信息";
node.addEventListener('click', () => copyUserData(node.span));
share_button.after(node);
}
async function downloadCover(node) {
if (node.disabled) {
alert("下载正在处理中，请不要重复点击按钮！");
return;
}
node.disabled = true;
try {
const zip = new JSZip();
msg_pre.textContent = `下载封面并打包中...`;
let promises = aweme_list.map((aweme, index) => {
let awemeName = getAwemeName(aweme) + ".jpg";
return fetch(aweme.cover)
.then(response => response.arrayBuffer())
.then(buffer => zip.file(awemeName, buffer))
.then(() => msg_pre.textContent = `${index + 1}/${aweme_list.length} ` + awemeName)
});
Promise.all(promises).then(() => {
return zip.generateAsync({type: "blob"})
}).then((content) => {
createDownloadLink(content, null, "zip", "【封面】");
msg_pre.textContent = "封面打包完成";
node.disabled = false;
})
} finally {
node.disabled = false;
}
}
async function downloadImg(node) {
if (node.disabled) {
alert("下载正在处理中，请不要重复点击按钮！");
return;
}
node.disabled = true;
try {
const zip = new JSZip();
let flag = true;
let aweme_img_list = aweme_list.filter(a => a.images);
for (let [i, aweme] of aweme_img_list.entries()) {
let awemeName = getAwemeName(aweme);
msg_pre.textContent = `${i + 1}/${aweme_img_list.length} ` + awemeName;
let folder = zip.folder(awemeName);
await Promise.all(aweme.images.map((link, index) => {
return fetch(link)
.then((res) => res.arrayBuffer())
.then((buffer) => {
folder.file(`image_${index + 1}.jpg`, buffer);
});
}));
flag = false;
}
if (flag) {
alert("当前页面未发现图文链接");
node.disabled = false;
return;
}
msg_pre.textContent = "图文打包中...";
zip.generateAsync({type: "blob"})
.then((content) => {
createDownloadLink(content, null, "zip", "【图文】");
msg_pre.textContent = "图文打包完成";
node.disabled = false;
});
} finally {
node.disabled = false;
}
}
function douyinVideoDownloader() {
function run() {
let downloadOption = [{name: '打开视频源', id: 'toLink'}];
let videoElements = document.querySelectorAll('video');
if (videoElements.length === 0) return;
//把自动播放的video标签选择出来
let playVideoElements = [];
videoElements.forEach(function (element) {
let autoplay = element.getAttribute('autoplay');
if (autoplay !== null) {
playVideoElements.push(element);
}
})
let videoContainer = location.href.indexOf('modal_id') !== -1
? playVideoElements[0]
: playVideoElements[playVideoElements.length - 1];
if (!videoContainer) return;
//获取视频播放地址
let url = videoContainer && videoContainer.children.length > 0 && videoContainer.children[0].src
? videoContainer.children[0].src
: videoContainer.src;
//获取视频ID，配合自定义id使用
let videoId;
let resp = url.match(/^(https:)?\/\/.+\.com\/([a-zA-Z0-9]+)\/[a-zA-Z0-9]+\/video/);
let res = url.match(/blob:https:\/\/www.douyin.com\/(.*)/);
if (resp && resp[2]) {
videoId = resp[2];
} else if (res && res[1]) {
videoId = res[1]
} else {
videoId = videoContainer.getAttribute('data-xgplayerid')
}
let playContainer = videoContainer.parentNode.parentNode.querySelector('.xg-right-grid');
if (!playContainer) return;
//在对主页就行视频浏览时会出现多个按钮，删除不需要的，只保留当前对应的
let videoDownloadDom = playContainer.querySelector('#scriptVideoDownload' + videoId);
if (videoDownloadDom) {
let dom = playContainer.querySelectorAll('.xgplayer-playclarity-setting');
dom.forEach(function (d) {
let btn = d.querySelector('.btn');
if (d.id !== 'scriptVideoDownload' + videoId && btn.innerText === '下载') {
d.parentNode.removeChild(d);
}
});
return;
}
if (videoContainer && playContainer) {
let playClarityDom = playContainer.querySelector('.xgplayer-playclarity-setting');
if (!playClarityDom) return;
let palyClarityBtn = playClarityDom.querySelector('.btn');
if (!palyClarityBtn) return;
let downloadDom = playClarityDom.cloneNode(true);
downloadDom.setAttribute('id', 'scriptVideoDownload' + videoId);
if (location.href.indexOf('search') === -1) {
downloadDom.style = 'margin-top:-68px;padding-top:100px;padding-left:20px;padding-right:20px;';
} else {
downloadDom.style = 'margin-top:0px;padding-top:100px;';
}
let downloadText = downloadDom.querySelector('.btn');
downloadText.innerText = '下载';
downloadText.style = 'font-size:14px;font-weight:600;';
downloadText.setAttribute('id', 'zhmDouyinDownload' + videoId);
let detail = playContainer.querySelector('xg-icon:nth-of-type(1)').children[0];
let linkUrl = detail.getAttribute('href') ? detail.getAttribute('href') : location.href;
if (linkUrl.indexOf('www.douyin.com') === -1) {
linkUrl = '//www.douyin.com' + linkUrl;
}
downloadText.setAttribute('data-url', linkUrl);
downloadText.removeAttribute('target');
downloadText.setAttribute('href', 'javascript:void(0);');
let virtualDom = downloadDom.querySelector('.virtual');
downloadDom.onmouseover = function () {
if (location.href.indexOf('search') === -1) {
virtualDom.style = 'display:block !important';
} else {
virtualDom.style = 'display:block !important;margin-bottom:37px;';
}
}
downloadDom.onmouseout = function () {
virtualDom.style = 'display:none !important';
}
let downloadHtml = '';
downloadOption.forEach(function (item) {
if (item.id === "toLink") {
downloadHtml += `<div style="text-align:center;" class="item ${item.id}" id="${item.id}${videoId}">${item.name}</div>`;
}
})
if (downloadDom.querySelector('.virtual')) {
downloadDom.querySelector('.virtual').innerHTML = downloadHtml;
}
playClarityDom.after(downloadDom);
//直接打开
let toLinkDom = playContainer.querySelector('#toLink' + videoId);
if (toLinkDom) {
toLinkDom.addEventListener('click', function () {
if (url.match(/^blob/)) {
alert("加密视频地址，无法直接打开");
} else {
window.open(url);
}
})
}
}
}
setInterval(run, 500);
}
if (document.title === "验证码中间页") {
return
}
createMsgBox();
interceptResponse();
douyinVideoDownloader();
let domLoadedTimer;
const checkElementLoaded = () => {
const element = document.querySelector('#douyin-header-menuCt pace-island > div > div:nth-last-child(1) ul a');
if (element) {
console.log('顶部栏加载完毕');
msg_pre.textContent = "头像加载完成\n若需要下载用户数据，需进入目标用户主页";
clearInterval(domLoadedTimer);
domLoadedTimer = null;
createAllButton();
// scrollPageToBottom();
if (flag) flush();
}
};
window.onload = () => {
domLoadedTimer = setInterval(checkElementLoaded, 700);
}
})();