2 * Copyright (C) 2011-2014 jeanfi@gmail.com
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License as
6 * published by the Free Software Foundation; either version 2 of the
7 * License, or (at your option) any later version.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 #define _(String) gettext(String)
27 #include <curl/curl.h>
33 static const int DEFAULT_FETCH_RETRIES = 10;
42 static size_t cbk_curl(void *buffer, size_t size, size_t nmemb, void *userp)
44 size_t realsize = size * nmemb;
45 struct ucontent *mem = (struct ucontent *)userp;
47 mem->data = realloc(mem->data, mem->len + realsize + 1);
49 memcpy(&(mem->data[mem->len]), buffer, realsize);
51 mem->data[mem->len] = 0;
59 log_debug(_("initializing CURL"));
60 curl_global_init(CURL_GLOBAL_ALL);
61 curl = curl_easy_init();
68 static char *fetch_url(const char *url)
70 struct ucontent *content = malloc(sizeof(struct ucontent));
76 log_debug(_("fetch_url(): %s"), url);
82 retries = DEFAULT_FETCH_RETRIES;
85 content->data = malloc(1);
86 content->data[0] = '\0';
89 curl_easy_setopt(curl, CURLOPT_URL, url);
90 curl_easy_setopt(curl, CURLOPT_VERBOSE, 0);
91 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cbk_curl);
92 curl_easy_setopt(curl, CURLOPT_WRITEDATA, content);
93 curl_easy_setopt(curl, CURLOPT_USERAGENT, "ppastats/0.0");
94 #ifdef CURLOPT_TRANSFER_ENCODING
95 /* added since Curl 7.21.7 */
96 curl_easy_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1);
99 if (curl_easy_perform(curl) == CURLE_OK) {
100 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
104 result = content->data;
110 log_err(_("Fetch failed with code %ld for URL %s"),
115 s = 2 * (DEFAULT_FETCH_RETRIES - retries) + 2;
116 log_debug(_("Wait %ds before retry"), s);
126 log_err(_("Fetch failed with code %ld for URL %s"),
142 log_debug(_("cleanup CURL"));
144 curl_easy_cleanup(curl);
145 curl_global_cleanup();
148 char *get_url_content(const char *url, unsigned int use_cache)
155 content = fcache_get(url + 7);
158 content = fetch_url(url);
160 if (use_cache && content)
161 fcache_put(url + 7, content);