In C, how do you use libcurl to read a HTTP response into a string?

Question

I have homework where I need somehow to compare two HTTP responses. I am writing it on C and I use libcurl to make things easier. I am calling the function that uses libcurl to do a HTTP request and response from another function, and I want to return the HTTP response as a char *. Here is my code so far (it crashes):

#include <stdio.h>
#include <curl/curl.h>
#include <string.h>

size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
    size_t written;
    written = fwrite(ptr, size, nmemb, stream);
    return written;
}

char *handle_url(void) {
    CURL *curl;
    char *fp;
    CURLcode res;
    char *url = "http://www.yahoo.com";
    curl = curl_easy_init();
    if (curl) {
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
        res = curl_easy_perform(curl);
        if(res != CURLE_OK)
                fprintf(stderr, "curl_easy_perform() failed: %s\n",   curl_easy_strerror(res));

        curl_easy_cleanup(curl);

        //printf("\n%s", fp);
    }
    return fp;
}

This solution C libcurl get output into a string works, but not in my case because I just want to return the string to the calling function.

Any ideas?

Answer 1

Fixed it for you. You need to handle the case where the write_data() function is called multiple times, and pass it the right kind of parameter. You also need to keep track of how big a structure you've got, so you can allocate enough memory.

I left in a debug printf in the write_data function to help you understand how it works.

#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
#include <stdlib.h>

struct url_data {
    size_t size;
    char* data;
};

size_t write_data(void *ptr, size_t size, size_t nmemb, struct url_data *data) {
    size_t index = data->size;
    size_t n = (size * nmemb);
    char* tmp;

    data->size += (size * nmemb);

#ifdef DEBUG
    fprintf(stderr, "data at %p size=%ld nmemb=%ld\n", ptr, size, nmemb);
#endif
    tmp = realloc(data->data, data->size + 1); /* +1 for '\0' */

    if(tmp) {
        data->data = tmp;
    } else {
        if(data->data) {
            free(data->data);
        }
        fprintf(stderr, "Failed to allocate memory.\n");
        return 0;
    }

    memcpy((data->data + index), ptr, n);
    data->data[data->size] = '\0';

    return size * nmemb;
}

char *handle_url(char* url) {
    CURL *curl;

    struct url_data data;
    data.size = 0;
    data.data = malloc(4096); /* reasonable size initial buffer */
    if(NULL == data.data) {
        fprintf(stderr, "Failed to allocate memory.\n");
        return NULL;
    }

    data.data[0] = '\0';

    CURLcode res;

    curl = curl_easy_init();
    if (curl) {
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
        res = curl_easy_perform(curl);
        if(res != CURLE_OK) {
                fprintf(stderr, "curl_easy_perform() failed: %s\n",  
                        curl_easy_strerror(res));
        }

        curl_easy_cleanup(curl);

    }
    return data.data;
}

int main(int argc, char* argv[]) {
    char* data;

    if(argc < 2) {
        fprintf(stderr, "Must provide URL to fetch.\n");
        return 1;
    }
    data = handle_url(argv[1]);

    if(data) {
        printf("%s\n", data);
        free(data);
    }

    return 0;
}

Note: compile with gcc -o test test.c -lcurl (assuming you pasted into test.c). Use gcc -o test test.c -lcurl -DDEBUG to see the test printf() calls.

Disclaimer: this is ugly, quick-and-dirty code. There may be bugs. Please see the more robust, better commented example here.