node.js 抓取代理ip实例代码

node.js实现抓取代理ip

主要文件：index.js

Promise.promisifyAll(mongoose);
let Schema=mongoose.Schema;
mongoose.connect('mongodb://localhost:27017/ipproxypool');
let IPpool=new Schema({
ip:{type:String,unique:true}
})
let Ipproxy=mongoose.model('IP',IPpool);

function fetchUrl(url){
fetch(url,{
method:'get',headers:{
}
})
.then(res=>res.text())
.then(body=>{
let $=cheerio.load(body);
let length=$('#list table tbody').find('tr').length;
for (let i=0;i<length;i++){
let ipaddress= $('#list table tbody').find('tr').eq(i).find('td').eq(0).text() ;
let port = $('#list table tbody').find('tr').eq(i).find('td').eq(1).text();
console.log(IP:${ipaddress}:${port});
let ip=${ipaddress}:${port}
let ippool=new Ipproxy({
ip:ip
})
ippool.save();
}
})
}

var sleep = function (time) {
return new Promise(function (resolve,reject) {
setTimeout(function () {
resolve('ok');
},time);
})
};
const pageNumber=10;
var start = async function(){
for(let j=1;j<pageNumber;j++){
console.log(当前是第${j}次等待..);
fetchUrl(http://www.kuaidaili.com/free/inha/${j}/);
await sleep(1500);
}
}
start();

包支持 ： package.json

rush:js;"> { "name": "demo-4-ipproxypool","version": "1.0.0","description": "","main": "index.js","scripts": { "test": "echo \"Error: no test specified\" && exit 1" },"author": "false-l","license": "","devDependencies": { "babel-preset-es2015": "^6.24.1","babel-preset-react": "^6.24.1","babel-preset-stage-3": "^6.24.1" },"dependencies": { "babel-core": "^6.24.1","bluebird": "^3.5.0","cheerio": "^0.22.0","koa": "^2.2.0","koa-router": "^7.1.1","mongoose": "^4.9.6","node-fetch": "^1.6.3" } }

本地需要安装mongodb数据库,用于存储抓取到的ip，目前还未实现ip验证。写这个主要是处于好奇。

上面的代码就可以实现抓取ip代理网站的ip并存到mongodb数据库中。

下面在放出一个基于koa2的api接口的简易服务器实现

server

<div class="jb51code">
<pre class="brush:js;">
const Promise=require('bluebird');
let mongoose=require('mongoose');
const koa=require('koa');
const app=new koa();
var router = require('koa-router')();
Promise.promisifyAll(mongoose);
let Schema=mongoose.Schema;
mongoose.connect('mongodb://localhost:27017/ipproxypool');
let IPpool=new Schema({
ip:{type:String,IPpool);

app.use(async (ctx,next) => {
await next();
var data=await Ipproxy.find({},function(err,ips){
var ipmap=[];
ips.forEach(function(ip){
ipmap[ip._id]=ip;
//console.log(ip)
});
})
var map=data.map(ip=>ip.ip);
ctx.response.type = 'text/json';
ctx.response.body = map;
});
app.listen(3000);
console.log('server listen:3000')