旅游网站马蜂窝景点页面游客评论爬取下载
// ==UserScript==
// @name mafengwo
// @namespace https://www.mafengwo.cn/poi/*.html
// @version 0.1
// @description 旅游网站马蜂窝景点页面游客评论爬取下载
// @author gisgo njnu
// @match https://www.mafengwo.cn/poi/*.html
// @icon data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==
// @grant none
// ==/UserScript==
(function() {
function downloadFileByBlob (blobUrl, filename) {
if (window.navigator && window.navigator.msSaveOrOpenBlob) {
window.navigator.msSaveOrOpenBlob(blobUrl, filename)
} else {
let eleLink = document.createElement('a')
eleLink.download = filename
eleLink.style.display = 'none'
eleLink.href = blobUrl
document.body.appendChild(eleLink)
eleLink.click()
window.URL.revokeObjectURL(eleLink.href)
document.body.removeChild(eleLink)
}
}
class Comment {
constructor (name, content) {
this.游客姓名 = name || " "
this.游客评论 = content || " "
}
}
function getComment () {
let commentArray = []
let pushData = (name, revTxt) => {
for (let i = 0; i < name.length; i++) {
commentArray.push(new Comment(name[i].innerText, revTxt[i].innerText))
}
return commentArray
}
return new Promise((resolve, reject) =>{
let timeOutFunc = () => {
let revTxt = document.getElementsByClassName('rev-txt')
let name = document.getElementsByClassName('name')
let titleName = document.getElementById('_j_sharecnt').attributes[2].textContent
let pgNext = document.getElementsByClassName('pg-next')
if (name && revTxt && titleName && (pgNext.length != 0)) {
pushData(name, revTxt)
pgNext[0].click()
setTimeout(timeOutFunc, 5000 + Math.random() * 2000)
} else if (name && revTxt && titleName && (pgNext.length === 0)) {
let commentData = pushData(name, revTxt)
resolve({commentData, titleName})
} else {
reject('getDomObj error!')
}
}
setTimeout(timeOutFunc, 3000 + Math.random() * 2000)
})
}
function main () {
console.log('start main func \n')
let downloadFile = (data, fileName) => {
let r = confirm('数据爬取完成，是否下载数据？点击确定下载数据！')
if (r === true) {
let blobContent = new Blob(
[JSON.stringify(data, null, 2)],
{ type: 'text/plain' }
)
let blobUrl = window.URL.createObjectURL(blobContent)
downloadFileByBlob(blobUrl, `${fileName}_${new Date().valueOf()}.json`)
} else {
console.log(data)
}
}
getComment()
.then((commentData) => {
console.log(commentData)
downloadFile(commentData.commentData, commentData.titleName)
})
.catch((error) => {
console.log(error)
throw new Error(error)
})
console.log('\n end main func')
return -9999
}
let bt = document.createElement('button')
let fa = document.getElementsByClassName('head-logo')
let btc = document.createTextNode('爬取')
bt.appendChild(btc)
bt.style.cssText = "color: red; width: 50px; height: 50px; postion: position: absolute; top: 10px;"
bt.onclick = main
fa[0].appendChild(bt)
})();