HTTP request to an external API in Firebase Cloud Functions (Spark Tier) refused

Question

I am trying to call an external resource from the web and load the results into Dialogflow using the NodeJS Client V2 and Cloud Functions.

I have tried multiple combinations of this code, using promises, external functions, etc. No luck.

Test1

function myIntent(conv) {
            // console.log('conv ', conv)

            const config ={
                "headers": {'X-App-Token': dataSFAPITOken},
                "Content-Type": "application/json"
            }
            const url = "http://data.sfgov.org/resource/cuks-n6tp.json";
            return new Promise((resolve, reject) => {

                axios
                    .get(url)
                    .then(response => {
                        console.log(response);
                        conv.ask('I got your data back')
                        return resolve(response)
                    })
                    .catch(error => {
                        console.log(error);
                        conv.ask('Sorry! Something went wrong')
                        return reject(error)
                    });
            })

    }
 app.intent('My Intent', myIntent);  
 exports.myAgent = functions.https.onRequest(app);

Error

This is the error that I get on the Cloud Functions Dashboard, after I invoke myIntent .

{ Error: getaddrinfo ENOTFOUND data.sfgov.org data.sfgov.org:80
    at errnoException (dns.js:28:10)
    at GetAddrInfoReqWrap.onlookup [as oncomplete] (dns.js:76:26)
  code: 'ENOTFOUND',
  errno: 'ENOTFOUND',
  syscall: 'getaddrinfo',
  hostname: 'data.sfgov.org',
  host: 'data.sfgov.org',
  port: 80,
  config: 
   { adapter: [Function: httpAdapter],
     transformRequest: { '0': [Function: transformRequest] },
     transformResponse: { '0': [Function: transformResponse] },
     timeout: 0,
     xsrfCookieName: 'XSRF-TOKEN',
     xsrfHeaderName: 'X-XSRF-TOKEN',
     maxContentLength: -1,
     validateStatus: [Function: validateStatus],
     headers: 
      { Accept: 'application/json, text/plain, */*',
        'User-Agent': 'axios/0.18.0' },
     method: 'get',
     url: 'http://data.sfgov.org/resource/cuks-n6tp.json',
     data: undefined },
  request: 
   Writable {
     _writableState: 
      WritableState {
        objectMode: false,
        highWaterMark: 16384,
        needDrain: false,
        ending: false,
        ended: false,
        finished: false,
        decodeStrings: true,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        corked: 0,
        sync: true,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        bufferedRequest: null,
        lastBufferedRequest: null,
        pendingcb: 0,
        prefinished: false,
        errorEmitted: false,
        bufferedRequestCount: 0,
        corkedRequestsFree: [Object] },
     writable: true,
     domain: null,
     _events: 
      { response: [Function: handleResponse],
        error: [Function: handleRequestError] },
     _eventsCount: 2,
     _maxListeners: undefined,
     _options: 
      { protocol: 'http:',
        maxRedirects: 21,
        maxBodyLength: 10485760,
        path: '/resource/cuks-n6tp.json',
        method: 'get',
        headers: [Object],
        agent: undefined,
        auth: undefined,
        hostname: 'data.sfgov.org',
        port: null,
        nativeProtocols: [Object],
        pathname: '/resource/cuks-n6tp.json' },
     _redirectCount: 0,
     _requestBodyLength: 0,
     _requestBodyBuffers: [],
     _onNativeResponse: [Function],
     _currentRequest: 
      ClientRequest {
        domain: null,
        _events: [Object],
        _eventsCount: 6,
        _maxListeners: undefined,
        output: [],
        outputEncodings: [],
        outputCallbacks: [],
        outputSize: 0,
        writable: true,
        _last: true,
        upgrading: false,
        chunkedEncoding: false,
        shouldKeepAlive: false,
        useChunkedEncodingByDefault: false,
        sendDate: false,
        _removedHeader: {},
        _contentLength: 0,
        _hasBody: true,
        _trailer: '',
        finished: true,
        _headerSent: true,
        socket: [Object],
        connection: [Object],
        _header: 'GET /resource/cuks-n6tp.json HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nUser-Agent: axios/0.18.0\r\nHost: data.sfgov.org\r\nConnection: close\r\n\r\n',
        _headers: [Object],
        _headerNames: [Object],
        _onPendingData: null,
        agent: [Object],
        socketPath: undefined,
        timeout: undefined,
        method: 'GET',
        path: '/resource/cuks-n6tp.json',
        _ended: false,
        _redirectable: [Circular],
        parser: null },
     _currentUrl: 'http://data.sfgov.org/resource/cuks-n6tp.json' },
  response: undefined }  

Test2

const url = "data.sfgov.org";
var options = {
    protocol:'http:',
    host: url,
    port : 8080,
    path:'/resource/cuks-n6tp',
    headers: {
        'X-App-Token': dataSFAPITOken,
        "Content-Type": "application/json"
    }
};
http.get(options, (http_res) => {
    // initialize the container for our data
    var data = "";

    // this event fires many times, each time collecting another piece of the response
    http_res.on("data", (chunk) =>{
        // append this chunk to our growing `data` var
        data += chunk;
    });

    // this event fires *one* time, after all the `data` events/chunks have been gathered
    http_res.on("end", () => {
        // you can use res.send instead of console.log to output via express
        console.log(data);
    });
});

Here is the link to the official documentation HERE.

The crazy thing about this is that below is a similar implementation on the browser and it works. As a matter of fact, I don't need an API key to query the data. I can simply copy/paste the link.

$.ajax({
    url: "https://data.sfgov.org/resource/cuks-n6tp.json",
    type: "GET",
    data: {
      "$limit" : 50,
      "$$app_token" : "APPTOKEN"
    }
}).done(function(data) {
  // alert("Retrieved " + data.length + " records from the dataset!");
  console.log(data);
  document.getElementById('data').innerHTML = JSON.stringify(data, null,2)
});

But for some reasons, I can't make it work with Dialogflow NodeJS Client V2. I am pretty sure in V1, I can make it work.

My migration to V2 is a little painful. Please help.

Thanks.

Answer 1

The free Spark plan on Firebase Cloud Functions doesn't allow calls to domains outside Google.

You will need to upgrade to one of the paid plans such as the Flame or Blaze plan, which requires a credit card on file. For low levels of usage, however, you won't be charged on Blaze.

The image below shows Spark, Flame, and Blaze. Notice the Google services only for Spark. https://firebase.google.com/pricing/

Update, Nov 2020

Note that the Flame plan is no longer available. So if you want external access, you'll need to use the Blaze plan.