30 {
"$Id: HTTPConnect.cc 21699 2009-11-05 00:06:01Z jimg $" 78 #define CLIENT_ERR_MIN 400 79 #define CLIENT_ERR_MAX 417 83 "Unauthorized: Contact the server administrator.",
85 "Forbidden: Contact the server administrator.",
86 "Not Found: The data source or server could not be found.\n\ 87 Often this means that the OPeNDAP server is missing or needs attention;\n\ 88 Please contact the server administrator.",
89 "Method Not Allowed.",
91 "Proxy Authentication Required.",
96 "Precondition Failed.",
97 "Request Entity Too Large.",
98 "Request URI Too Large.",
99 "Unsupported Media Type.",
100 "Requested Range Not Satisfiable.",
101 "Expectation Failed." 104 #define SERVER_ERR_MIN 500 105 #define SERVER_ERR_MAX 505 108 "Internal Server Error.",
111 "Service Unavailable.",
113 "HTTP Version Not Supported." 119 http_status_to_string(
int status)
126 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
133 class ParseHeader :
public unary_function<const string &, void>
141 ParseHeader() : type(
unknown_type), server(
"dods/0.0"), protocol(
"2.0")
144 void operator()(
const string &line)
148 if (name ==
"content-description") {
149 DBG2(cerr << name <<
": " << value << endl);
155 else if (name ==
"xdods-server" && server ==
"dods/0.0") {
156 DBG2(cerr << name <<
": " << value << endl);
159 else if (name ==
"xopendap-server") {
160 DBG2(cerr << name <<
": " << value << endl);
163 else if (name ==
"xdap") {
164 DBG2(cerr << name <<
": " << value << endl);
167 else if (server ==
"dods/0.0" && name ==
"server") {
168 DBG2(cerr << name <<
": " << value << endl);
171 else if (name ==
"location") {
172 DBG2(cerr << name <<
": " << value << endl);
176 && line.find(
"text/html") != string::npos) {
177 DBG2(cerr << name <<
": text/html..." << endl);
192 string get_protocol()
197 string get_location() {
219 save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
221 DBG2(cerr <<
"Inside the header parser." << endl);
222 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
225 string complete_line;
226 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) ==
'\r')
227 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
229 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
232 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
233 DBG(cerr <<
"Header line: " << complete_line << endl);
234 hdrs->push_back(complete_line);
242 curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
244 string message(msg, size);
248 cerr <<
"Text: " << message;
break;
249 case CURLINFO_HEADER_IN:
250 cerr <<
"Header in: " << message;
break;
251 case CURLINFO_HEADER_OUT:
252 cerr <<
"Header out: " << message;
break;
253 case CURLINFO_DATA_IN:
254 cerr <<
"Data in: " << message;
break;
255 case CURLINFO_DATA_OUT:
256 cerr <<
"Data out: " << message;
break;
258 cerr <<
"End: " << message;
break;
259 #ifdef CURLINFO_SSL_DATA_IN 260 case CURLINFO_SSL_DATA_IN:
261 cerr <<
"SSL Data in: " << message;
break;
263 #ifdef CURLINFO_SSL_DATA_OUT 264 case CURLINFO_SSL_DATA_OUT:
265 cerr <<
"SSL Data out: " << message;
break;
268 cerr <<
"Curl info: " << message;
break;
277 HTTPConnect::www_lib_init()
279 d_curl = curl_easy_init();
281 throw InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
287 if (!d_rcr->get_proxy_server_host().empty()) {
288 DBG(cerr <<
"Setting up a proxy server." << endl);
289 DBG(cerr <<
"Proxy host: " << d_rcr->get_proxy_server_host()
291 DBG(cerr <<
"Proxy port: " << d_rcr->get_proxy_server_port()
293 DBG(cerr <<
"Proxy pwd : " << d_rcr->get_proxy_server_userpw()
295 curl_easy_setopt(d_curl, CURLOPT_PROXY,
296 d_rcr->get_proxy_server_host().c_str());
297 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
298 d_rcr->get_proxy_server_port());
301 #ifdef CURLOPT_PROXYAUTH 302 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (
long)CURLAUTH_ANY);
306 if (!d_rcr->get_proxy_server_userpw().empty())
307 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
308 d_rcr->get_proxy_server_userpw().c_str());
311 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
314 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
319 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
321 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
322 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
323 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
328 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
329 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
332 if (!d_rcr->get_validate_ssl() == 0) {
333 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
334 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
341 if (!d_cookie_jar.empty()) {
342 DBG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
343 curl_easy_setopt(d_curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
344 curl_easy_setopt(d_curl, CURLOPT_COOKIESESSION, 1);
348 cerr <<
"Curl version: " << curl_version() << endl;
349 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
350 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
357 class BuildHeaders :
public unary_function<const string &, void>
359 struct curl_slist *d_cl;
362 BuildHeaders() : d_cl(0)
365 void operator()(
const string &header)
367 DBG(cerr <<
"Adding '" << header.c_str() <<
"' to the header list." 369 d_cl = curl_slist_append(d_cl, header.c_str());
372 struct curl_slist *get_headers()
393 HTTPConnect::read_url(
const string &url, FILE *stream,
394 vector<string> *resp_hdrs,
395 const vector<string> *headers)
397 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
407 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
408 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
410 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
413 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
414 ostream_iterator<string>(cerr,
"\n")));
416 BuildHeaders req_hdrs;
417 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
420 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
421 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
424 bool temporary_proxy =
false;
425 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
426 DBG(cerr <<
"Suppress proxy for url: " << url << endl);
427 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
430 string::size_type at_sign = url.find(
'@');
434 if (at_sign != url.npos)
435 d_upstring = url.substr(7, at_sign - 7);
437 if (!d_upstring.empty())
438 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
443 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
445 CURLcode res = curl_easy_perform(d_curl);
448 curl_slist_free_all(req_hdrs.get_headers());
449 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
452 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
453 curl_easy_setopt(d_curl, CURLOPT_PROXY,
454 d_rcr->get_proxy_server_host().c_str());
457 throw Error(d_error_buffer);
460 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
462 throw Error(d_error_buffer);
471 HTTPConnect::url_uses_proxy_for(
const string &url)
throw()
473 if (d_rcr->is_proxy_for_used()) {
474 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
475 int index = 0, matchlen;
476 return host_regex.search(url.c_str(), url.size(), matchlen, index)
487 HTTPConnect::url_uses_no_proxy_for(
const string &url)
throw()
489 return d_rcr->is_no_proxy_for_used()
490 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
501 HTTPConnect::HTTPConnect(
RCReader *rcr) : d_username(
""), d_password(
""),
503 d_dap_client_protocol_major(2),
504 d_dap_client_protocol_minor(0)
514 d_request_headers.push_back(
string(
"Pragma:"));
515 string user_agent = string(
"User-Agent: ") + string(
CNAME)
516 + string(
"/") + string(
CVER);
517 d_request_headers.push_back(user_agent);
518 if (d_accept_deflate)
519 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
528 DBG2(cerr <<
"Cache object created (" << hex << d_http_cache << dec
547 DBG2(cerr <<
"Entering the HTTPConnect dtor" << endl);
549 curl_easy_cleanup(d_curl);
551 DBG2(cerr <<
"Leaving the HTTPConnect dtor" << endl);
570 cout <<
"GET " << url <<
" HTTP/1.0" << endl;
576 stream = caching_fetch_url(url);
579 stream = plain_fetch_url(url);
584 ss <<
"HTTP/1.0 " << stream->
get_status() <<
" -" << endl;
585 for (
size_t i = 0; i < stream->
get_headers()->size(); i++) {
597 cout << endl << endl;
601 if (parser.get_location() !=
"" &&
602 url.substr(0,url.find(
"?",0)).compare(parser.get_location().substr(0,url.find(
"?",0))) != 0) {
606 stream->
set_type(parser.get_object_type());
627 get_tempfile_template(
const char *file_template)
633 Regex directory(
"[-a-zA-Z0-9_\\]*");
636 if (c && directory.match(c, strlen(c)) && (access(getenv(
"TEMP"), 6) == 0))
637 goto valid_temp_directory;
640 if (c && directory.match(c, strlen(c)) && (access(getenv(
"TEMP"), 6) == 0))
641 goto valid_temp_directory;
644 Regex directory(
"[-a-zA-Z0-9_/]*");
646 c = getenv(
"TMPDIR");
647 if (c && directory.match(c, strlen(c)) && (access(c, W_OK | R_OK) == 0))
648 goto valid_temp_directory;
651 if (access(P_tmpdir, W_OK | R_OK) == 0) {
653 goto valid_temp_directory;
661 valid_temp_directory:
663 int size = strlen(c) + strlen(file_template) + 2;
665 throw Error(
"Bad temporary file name.");
667 char *temp =
new char[size];
668 strncpy(temp, c, size-2);
671 strcat(temp, file_template);
698 char *dods_temp = get_tempfile_template(
"dodsXXXXXX");
701 #if defined(WIN32) || defined(TEST_WIN32_TEMPS) 702 stream = fopen(_mktemp(dods_temp),
"w+b");
704 stream = fdopen(mkstemp(dods_temp),
"w+");
708 throw InternalErr(
"I/O Error: Failed to open a temporary file for the data values.");
710 string dods_temp_s = dods_temp;
711 delete[] dods_temp; dods_temp = 0;
722 DBG(cerr <<
"Counld not close the temporary file: " << name << endl);
725 unlink(name.c_str());
750 HTTPConnect::caching_fetch_url(
const string &url)
752 DBG(cerr <<
"Is this URL (" << url <<
") in the cache?... ");
754 vector<string> *headers =
new vector<string> ;
758 DBGN(cerr <<
"no; getting response and caching." << endl);
759 time_t now = time(0);
767 DBGN(cerr <<
"yes... ");
770 DBGN(cerr <<
"and it's valid; using cached response." << endl);
776 DBGN(cerr <<
"but it's not valid; validating... ");
780 vector<string> *resp_hdrs =
new vector<string> ;
781 vector<string> cond_hdrs =
785 time_t now = time(0);
789 http_status = read_url(url, body, resp_hdrs, &cond_hdrs);
797 switch (http_status) {
799 DBGN(cerr <<
"read a new response; caching." << endl);
809 DBGN(cerr <<
"cached response valid; updating." << endl);
814 vector<string> *headers =
new vector<string>;
822 if (http_status >= 400) {
823 string msg =
"Error while reading the URL: ";
826 +=
".\nThe OPeNDAP server returned the following message:\n";
827 msg += http_status_to_string(http_status);
832 "Bad response from the HTTP server: " +
long_to_string(http_status));
839 throw InternalErr(__FILE__, __LINE__,
"Should never get here");
854 HTTPConnect::plain_fetch_url(
const string &url)
856 DBG(cerr <<
"Getting URL: " << url << endl);
859 vector<string> *resp_hdrs =
new vector<string>;
863 status = read_url(url, stream, resp_hdrs);
865 string msg =
"Error while reading the URL: ";
867 msg +=
".\nThe OPeNDAP server returned the following message:\n";
868 msg += http_status_to_string(status);
880 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
899 if (d_accept_deflate) {
900 if (find(d_request_headers.begin(), d_request_headers.end(),
901 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
902 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
903 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
904 ostream_iterator<string>(cerr,
"\n")));
907 vector<string>::iterator i;
908 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
909 bind2nd(equal_to<string>(),
910 string(
"Accept-Encoding: deflate, gzip, compress")));
911 d_request_headers.erase(i, d_request_headers.end());
916 class HeaderMatch :
public unary_function<const string &, bool> {
917 const string &d_header;
919 HeaderMatch(
const string &header) : d_header(header) {}
920 bool operator()(
const string &arg) {
return arg.find(d_header) == 0; }
935 vector<string>::iterator i;
936 i = find_if(d_request_headers.begin(), d_request_headers.end(),
937 HeaderMatch(
"XDAP-Accept:"));
938 if (i != d_request_headers.end())
939 d_request_headers.erase(i);
942 d_dap_client_protocol_major = major;
943 d_dap_client_protocol_minor = minor;
944 ostringstream xdap_accept;
945 xdap_accept <<
"XDAP-Accept: " << major <<
"." << minor;
947 d_request_headers.push_back(xdap_accept.str());
949 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
950 ostream_iterator<string>(cerr,
"\n")));
978 d_upstring = u +
":" + p;
vector< string > get_conditional_request_headers(const string &url)
string get_cookie_jar() const
void set_cache_enabled(bool mode)
bool is_url_valid(const string &url)
void set_credentials(const string &u, const string &p)
static HTTPCache * instance(const string &cache_root, bool force=false)
void set_max_size(unsigned long size)
int get_max_cache_size() const
virtual void set_type(ObjectType o)
bool size_ok(unsigned int sz, unsigned int nelem)
sanitize the size of an array. Test for integer overflow when dynamically allocating an array...
FILE * get_cached_response(const string &url, vector< string > &headers, string &cacheName)
ObjectType
The type of object in the stream coming from the data server.
HTTPResponse * fetch_url(const string &url)
virtual void set_version(const string &v)
A class for software fault reporting.
void parse_mime_header(const string &header, string &name, string &value)
virtual void set_protocol(const string &p)
int get_always_validate() const
ObjectType get_description_type(const string &value)
virtual FILE * get_stream() const
void update_response(const string &url, time_t request_time, const vector< string > &headers)
void close_temp(FILE *s, const string &name)
int get_ignore_expires() const
string get_temp_file(FILE *&stream)
bool cache_response(const string &url, time_t request_time, const vector< string > &headers, const FILE *body)
void set_accept_deflate(bool defalte)
bool get_use_cache() const
string long_to_string(long val, int base)
string get_dods_cache_root() const
bool is_cache_enabled() const
void set_always_validate(bool validate)
void set_xdap_protocol(int major, int minor)
void set_default_expiration(int exp_time)
void release_cached_response(FILE *response)
unsigned int get_max_cached_obj() const
A class for error processing.
void set_expire_ignored(bool mode)
virtual int get_status() const
void set_max_entry_size(unsigned long size)
virtual vector< string > * get_headers() const
int get_default_expires() const