Nodejs reported an error when climbing the web page: (libuv) kqueue (): toomany openfiles in system

  node.js, question

Main code:

db_operation.db_getUrl('appsIndex_China', function(results){
    var arr = [],
        length = results.length;
    for(var i = 0; i < length; i加加)
    {
        var item = results[i];
        (function(i, item){
            request(item.url, function(error, response, body){
                if(!error && response.statusCode == 200)
                {
                    $ = cheerio.load(body);
                    getAppDetail('#content', function(data){
                        data.id = item.id;
                        arr.push(data);
                        
                    });
                    if(arr.length == 100)
                    {
                        db_operation.db_addBasciDetail('appBasicDetail_China', arr);
                    }
                }    
            })
        }(i, item));
    }
})

Length is about 1w6, there should be too many request addresses, how to solve it? Kneeling xie!

Baidu added google and said it would change the maximum number of files opened by the system, but the number of files I can open now is already 1,000,000:

clipboard.png

1) The number of files that the 1)OS can open at the same time is limited. Even if you increase the system value, there is a bottleneck.
2) If you want to control the quantity control of asynchronous requests sent within a certain range in the code, it is actually meaningless to send too many requests. If the server finds too many requests from the same IP, it will also reject your request.

You can try the following modifications:

//全部请求完成后的回调函数
function appsIndexChinaProcessDone(){
    
}

db_operation.db_getUrl('appsIndex_China', function(results){
    var arr = [],
        length = results.length;
    
    var correntRequest=1000;
    var finishedRequest=0;
    function processNext(){
        var item=results.shift();
        if(someData){
            request(item.url, function(error, response, body){
                if(!error && response.statusCode == 200)
                {
                    $ = cheerio.load(body);
                    getAppDetail('#content', function(data){
                        data.id = item.id;
                        arr.push(data);

                    });
                    if(arr.length == 100)
                    {
                        finishedRequest加加;
                        db_operation.db_addBasciDetail('appBasicDetail_China', arr);
                    }
                    //也可以加在此时,只要回来200,不管有没有取到需要的数据
                    //finishedRequest加加;
                }else{
                    //放回队列中等待下次再尝试执行
                    //也可以不处理,看你的业务需求
                    results.push(item);
                }
                //获取下一个数据,发送请求
                processNext();
            })
        }else{
            //因为异步请求,队列为空,不等于所有的请求处理完毕,需要判断时候完成的请求数量和实际要求的数量是否吻合
            if(finishedRequest===length){
                console.log('process done!');
                //全部执行完后执行回调,通知数据抓取完成
                appsIndexChinaProcessDone();
            }
        }
        
    }
    
    //设置并发的初始请求数量,后续的请求通过processNext依次执行,一直到队列为空
    for(var i=0;i<correntRequest;i加加){
        processNext();
    }
});