Http request error from inside a Mongoose forEach loop

Http request error from inside a Mongoose forEach loop
0

#1

Hey everyone, I am having some trouble updating documents in a mongo collection from data a receive from a geo coding API. I am looping through my collection with a forEach loop, then if the document returned doesn’t have geo coords, I call an API with the appropriate query string, get the geo coords, put them in to an array, and then update the appropriate mongo document. Here is the error:

Error: connect ECONNREFUSED 165.91.85.245:443
    at Object._errnoException (util.js:1024:11)
    at _exceptionWithHostPort (util.js:1046:20)
    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1182:14)

Here is my code:

    City.find().then((cities) => {
        cities.forEach((doc) => {
            let {city, state, _id, geo} = doc;

            if(geo === undefined || geo.length !== 2){
                let coordArr = [], count = 0;
                let queryString = url + 'city=' + city + '&state=' + state + '&apikey=' + process.env.TEXAS_AM_GEOCODING_API_KEY + '&format=json&version=4.01';
                request(queryString, (err, resp, body) => {
                    if(err) throw err;
                    body = JSON.parse(body);
                    count++;
                    let lat = body.OutputGeocodes[0].OutputGeocode.Latitude;
                    let long = body.OutputGeocodes[0].OutputGeocode.Longitude;
                    coordArr.push(long, lat);
                    console.log(coordArr);
                    City.update(
                        {_id: _id},
                        {$set: {geo: coordArr }},
                        (err, data) => {
                            err ? console.log(err) : console.log(data);
                        }
                    ); 
                })
            }
        });
    }).catch((err) => {
        throw err;
    });

I have tried refactoring several times, but I can’t seem to figure out what I am doing wrong. Any help would be appreciated. Thank you.


#2

The error message suggests you are having trouble connecting…can you post your whole github repo so we can see your set up and how you handle server/DB connections?


#3

This is actually just one file I am using to get my city data in to a workable state. My mongo connection is right above this and looks like this:

import request from 'request';
import City from './../../../models/city.model';
import mongoose from 'mongoose';

mongoose.Promise = global.Promise;

//db connection
mongoose.connect('mongodb://localhost:27017/step0',{
    useMongoClient: true
}, function(err){
    if(err) return err;
    console.log('Connected to mongodb');
});

function geoUpdate(){
City.find().then((cities) => {
        cities.forEach((doc) => {
            let {city, state, _id, geo} = doc;

            if(geo === undefined || geo.length !== 2){
                let coordArr = [], count = 0;
                let queryString = url + 'city=' + city + '&state=' + state + '&apikey=' + process.env.TEXAS_AM_GEOCODING_API_KEY + '&format=json&version=4.01';
                request(queryString, (err, resp, body) => {
                    if(err) throw err;
                    body = JSON.parse(body);
                    count++;
                    let lat = body.OutputGeocodes[0].OutputGeocode.Latitude;
                    let long = body.OutputGeocodes[0].OutputGeocode.Longitude;
                    coordArr.push(long, lat);
                    console.log(coordArr);
                    City.update(
                        {_id: _id},
                        {$set: {geo: coordArr }},
                        (err, data) => {
                            err ? console.log(err) : console.log(data);
                        }
                    ); 
                })
            }
        });
    }).catch((err) => {
        throw err;
    });
}

This code works when the item count is low, but when I feed it 200 items, I get the above error. Should I be manually closing connections to the DB? I feel like I am sending data to mongo too fast, and then it is refusing me. I am also getting a warning saying I am not handling the error from a promise. Where am I not handling that? Thanks in advance.


#4

I’ll take a proper look at this later :slight_smile:


#5

Ended up getting it to work. I changed the request library to the request-promise library and its working now. Here is the updated code:

//update the geoCoordinates of 
city.geocoding = () => {
    const url = 'https://geoservices.tamu.edu/Services/Geocode/WebService/GeocoderWebServiceHttpNonParsed_V04_01.aspx?';
    City.find()
        .then((cities) => {
            cities.forEach((doc) => {
                let {city, state, _id, geo} = doc;
                if(geo === undefined || geo.length !== 2){
                    let coordArr = [];
                    let queryString = url + 'city=' + city + '&state=' + state 
                    queryString = queryString + '&apikey=' + process.env.TEXAS_AM_GEOCODING_API_KEY + '&format=json&version=4.01';

                    rp(queryString)
                        .then((body) => {
                            body = JSON.parse(body);
                            let lat = body.OutputGeocodes[0].OutputGeocode.Latitude;
                            let long = body.OutputGeocodes[0].OutputGeocode.Longitude;
                            coordArr.push(long, lat);
                            console.log(coordArr);
                            
                            City
                                .update(
                                    {_id: _id},
                                    {$set: {geo: coordArr }}
                                )
                                .then((data) => {
                                    console.log(city + " successfully updated!");
                                })
                                .catch((err) =>{
                                    console.log(err);
                                })

                        })
                        .catch((err) => {
                            console.log(err);
                        }) 
                    
                } else {
                    console.log(city + ' already has geocoordinates');
                }
                
            });// end forEach

    })
    .catch((err) => {
        console.log(err);
    });
}

The only issue I saw was that I hammer the Texas A&M api too fast and sometime I get a connection refused error. Should I be putting the request piece of this on a short setTimeout?