cheerio爬取網頁資料,儲存到MySQL資料庫
阿新 • • 發佈:2018-12-17
最近在做物流專案成本分析,需要爬取柴油價格資料,使用到了cheerio,cheerio實現了jQuery核心的一個子集。以下為爬取程式碼。
//getHtml.js,獲取HTML頁面資料
var http = require("http");
function gethtml(url,callback){
http.get(url,function(res){
var data = "";
res.on('data',function(chunk){
data += chunk;
});
res.on('end' ,function(){
callback(data);
});
}).on("error",function(){
callback(null);
})
}
exports.gethtml = gethtml;
//getData.js,使用cheerio獲取資料並存儲到mysql資料庫
const cheerio = require('cheerio');
const server = require('./getHtml');
var url = 'http://ny.gold600.com/';
server.gethtml(url,function (data){
if(data){
var $ = cheerio.load(data);
var data = [];
var list = $('.oilTable tbody tr:nth-child(odd)');
list.each(function(index,element){
let city = $(element).find('a').text().replace(/\s/g,''); // 獲取標題
let price = $($(element).find('td' )[4]).text().replace(/\s/g,''); // 獲取價格
let time = $($(element).find('td')[5]).text().replace(/\s/g,''); // 獲取價格
data.push({city,price,time})
// console.log(price)
// console.log(element.children('td')[0].innerTEXT);
})
var str = '';
data.forEach(function(item,index){
if(index == 0){
str += 'VALUES("'+item.city+'",'+item.price+',"'+item.time+'")';
}else{
str += ',("'+item.city+'",'+item.price+',"'+item.time+'")';
}
})
console.log(str)
var connection = mysql.createConnection({
host:'127.0.0.1',
user:'root',
password:'******',
database:'price'
});
console.log(str)
connection.connect();
connection.query('DELETE FROM priceList');
connection.query('ALTER TABLE priceList AUTO_INCREMENT=1');
connection.query('INSERT priceList(city,price,time) '+str,function(error,results,fields){
if(error) throw error;
console.log('The solution is:',results);
let a = results[1];
console.log(a)
})
connection.end();
console.log(value); // Success!
}else{
console.log("error");
}
})