libdap++  Updated for version 3.14.0
D4StreamMarshaller.cc
Go to the documentation of this file.
1 // D4StreamMarshaller.cc
2 
3 // -*- mode: c++; c-basic-offset:4 -*-
4 
5 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
6 // Access Protocol.
7 
8 // Copyright (c) 2012 OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 #include "config.h"
28 
29 #include <byteswap.h>
30 #include <cassert>
31 
32 #include <iostream>
33 #include <sstream>
34 #include <iomanip>
35 #include <limits>
36 
37 //#define DODS_DEBUG 1
38 
39 #include "D4StreamMarshaller.h"
40 
41 #if USE_XDR_FOR_IEEE754_ENCODING
42 #include "XDRUtils.h"
43 #include "util.h"
44 #endif
45 
46 #include "debug.h"
47 
48 using namespace std;
49 
50 namespace libdap {
51 
52 #if 0
53 // We decided to use int64_t to represent sizes of both arrays and strings,
54 // So this code is not used. jhrg 10/4/13
55 
56 // From the Google protobuf library
57 inline uint8_t* WriteVarint64ToArrayInline(uint64_t value, uint8_t* target) {
58  // Splitting into 32-bit pieces gives better performance on 32-bit
59  // processors.
60  uint32_t part0 = static_cast<uint32_t>(value );
61  uint32_t part1 = static_cast<uint32_t>(value >> 28);
62  uint32_t part2 = static_cast<uint32_t>(value >> 56);
63 
64  int size;
65 
66  // Here we can't really optimize for small numbers, since the value is
67  // split into three parts. Checking for numbers < 128, for instance,
68  // would require three comparisons, since you'd have to make sure part1
69  // and part2 are zero. However, if the caller is using 64-bit integers,
70  // it is likely that they expect the numbers to often be very large, so
71  // we probably don't want to optimize for small numbers anyway. Thus,
72  // we end up with a hard coded binary search tree...
73  if (part2 == 0) {
74  if (part1 == 0) {
75  if (part0 < (1 << 14)) {
76  if (part0 < (1 << 7)) {
77  size = 1; goto size1;
78  } else {
79  size = 2; goto size2;
80  }
81  } else {
82  if (part0 < (1 << 21)) {
83  size = 3; goto size3;
84  } else {
85  size = 4; goto size4;
86  }
87  }
88  } else {
89  if (part1 < (1 << 14)) {
90  if (part1 < (1 << 7)) {
91  size = 5; goto size5;
92  } else {
93  size = 6; goto size6;
94  }
95  } else {
96  if (part1 < (1 << 21)) {
97  size = 7; goto size7;
98  } else {
99  size = 8; goto size8;
100  }
101  }
102  }
103  } else {
104  if (part2 < (1 << 7)) {
105  size = 9; goto size9;
106  } else {
107  size = 10; goto size10;
108  }
109  }
110 
111  // GOOGLE_LOG(FATAL) << "Can't get here.";
112 
113  size10: target[9] = static_cast<uint8_t>((part2 >> 7) | 0x80);
114  size9 : target[8] = static_cast<uint8_t>((part2 ) | 0x80);
115  size8 : target[7] = static_cast<uint8_t>((part1 >> 21) | 0x80);
116  size7 : target[6] = static_cast<uint8_t>((part1 >> 14) | 0x80);
117  size6 : target[5] = static_cast<uint8_t>((part1 >> 7) | 0x80);
118  size5 : target[4] = static_cast<uint8_t>((part1 ) | 0x80);
119  size4 : target[3] = static_cast<uint8_t>((part0 >> 21) | 0x80);
120  size3 : target[2] = static_cast<uint8_t>((part0 >> 14) | 0x80);
121  size2 : target[1] = static_cast<uint8_t>((part0 >> 7) | 0x80);
122  size1 : target[0] = static_cast<uint8_t>((part0 ) | 0x80);
123 
124  target[size-1] &= 0x7F;
125  return target + size;
126 }
127 #endif
128 
129 #if USE_XDR_FOR_IEEE754_ENCODING
130 
135 void D4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int width, Type type)
136 {
137  dods_uint64 size = num * width;
138  // char *buf = (char*)malloc(size); jhrg 7/23/13
139  vector<char> buf(size);
140  XDR xdr;
141  xdrmem_create(&xdr, &buf[0], size, XDR_ENCODE);
142  try {
143  if(!xdr_array(&xdr, &val, (unsigned int *)&num, size, width, XDRUtils::xdr_coder(type)))
144  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
145 
146  if (xdr_getpos(&xdr) != size)
147  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
148 
149  // If this is a little-endian host, twiddle the bytes
150  static bool twiddle_bytes = !is_host_big_endian();
151  if (twiddle_bytes) {
152  if (width == 4) {
153  dods_float32 *lbuf = reinterpret_cast<dods_float32*>(&buf[0]);
154  while (num--) {
155  dods_int32 *i = reinterpret_cast<dods_int32*>(lbuf++);
156  *i = bswap_32(*i);
157  }
158  }
159  else { // width == 8
160  dods_float64 *lbuf = reinterpret_cast<dods_float64*>(&buf[0]);
161  while (num--) {
162  dods_int64 *i = reinterpret_cast<dods_int64*>(lbuf++);
163  *i = bswap_64(*i);
164  }
165  }
166  }
167 
168  d_out.write(&buf[0], size);
169  }
170  catch (...) {
171  xdr_destroy(&xdr);
172  throw;
173  }
174  xdr_destroy(&xdr);
175 }
176 #endif
177 
185 D4StreamMarshaller::D4StreamMarshaller(ostream &out, bool write_data) :
186  d_out(out), d_write_data(write_data)
187 {
188  assert(sizeof(std::streamsize) >= sizeof(int64_t));
189 
190 #if USE_XDR_FOR_IEEE754_ENCODING
191  // XDR is used if the call std::numeric_limits<double>::is_iec559()
192  // returns false indicating that the compiler is not using IEEE 754.
193  // If it is, we just write out the bytes.
194  xdrmem_create(&d_scalar_sink, d_ieee754_buf, sizeof(dods_float64), XDR_ENCODE);
195 #endif
196 
197  // This will cause exceptions to be thrown on i/o errors. The exception
198  // will be ostream::failure
199  out.exceptions(ostream::failbit | ostream::badbit);
200 }
201 
203 {
204 #if USE_XDR_FOR_IEEE754_ENCODING
205  xdr_destroy(&d_scalar_sink);
206 #endif
207 }
208 
212 {
213  d_checksum.Reset();
214 }
215 
227 {
228  ostringstream oss;
229  oss.setf(ios::hex, ios::basefield);
230  oss << setfill('0') << setw(8) << d_checksum.GetCrc32();
231 
232  return oss.str();
233 }
234 
242 {
243  Crc32::checksum chk = d_checksum.GetCrc32();
244  d_out.write(reinterpret_cast<char*>(&chk), sizeof(Crc32::checksum));
245 }
246 
247 void D4StreamMarshaller::checksum_update(const void *data, unsigned long len)
248 {
249  d_checksum.AddData(reinterpret_cast<const uint8_t*>(data), len);
250 }
251 
253 {
254  checksum_update(&val, sizeof(dods_byte));
255 
256  if (d_write_data) {
257  DBG( std::cerr << "put_byte: " << val << std::endl );
258 
259  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_byte));
260  }
261 }
262 
264 {
265  checksum_update(&val, sizeof(dods_int8));
266 
267  if (d_write_data) {
268  DBG( std::cerr << "put_int8: " << val << std::endl );
269 
270  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int8));
271  }
272 }
273 
275 {
276  checksum_update(&val, sizeof(dods_int16));
277 
278  if (d_write_data)
279  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int16));
280 }
281 
283 {
284  checksum_update(&val, sizeof(dods_int32));
285 
286  if (d_write_data)
287  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int32));
288 }
289 
291 {
292  checksum_update(&val, sizeof(dods_int64));
293 
294  if (d_write_data)
295  d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_int64));
296 }
297 
299 {
300 #if !USE_XDR_FOR_IEEE754_ENCODING
301  assert(std::numeric_limits<float>::is_iec559);
302 
303  checksum_update(&val, sizeof(dods_float32));
304 
305  if (d_write_data)
306  d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_float32));
307 
308 #else
309  // This code uses XDR to convert from a local representation to IEEE754;
310  // The extra 'twiddle' operation makes the byte-order correct for this
311  // host should it not be big-endian. Also note the assert() at the
312  // start of the method.
313 
314  if (d_write_data) {
315  if (std::numeric_limits<float>::is_iec559 ) {
316  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float32));
317  }
318  else {
319  if (!xdr_setpos(&d_scalar_sink, 0))
320  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
321 
322  if (!xdr_float(&d_scalar_sink, &val))
323  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
324 
325  if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float32))
326  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
327 
328  // If this is a little-endian host, twiddle the bytes
329  static bool twiddle_bytes = !is_host_big_endian();
330  if (twiddle_bytes) {
331  dods_int32 *i = reinterpret_cast<dods_int32*>(&d_ieee754_buf);
332  *i = bswap_32(*i);
333  }
334 
335  d_out.write(d_ieee754_buf, sizeof(dods_float32));
336  }
337  }
338 #endif
339 }
340 
342 {
343 #if !USE_XDR_FOR_IEEE754_ENCODING
344  assert(std::numeric_limits<double>::is_iec559);
345 
346  checksum_update(&val, sizeof(dods_float64));
347 
348  if (d_write_data)
349  d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_float64));
350 
351 #else
352  // See the comment above in put_float32()
353  if (d_write_data) {
354  if (std::numeric_limits<double>::is_iec559)
355  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float64));
356  else {
357  if (!xdr_setpos(&d_scalar_sink, 0))
358  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
359 
360  if (!xdr_double(&d_scalar_sink, &val))
361  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
362 
363  if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float64))
364  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
365 
366  // If this is a little-endian host, twiddle the bytes
367  static bool twiddle_bytes = !is_host_big_endian();
368  if (twiddle_bytes) {
369  dods_int64 *i = reinterpret_cast<dods_int64*>(&d_ieee754_buf);
370  *i = bswap_64(*i);
371  }
372 
373  d_out.write(d_ieee754_buf, sizeof(dods_float64));
374  }
375  }
376 #endif
377 }
378 
380 {
381  checksum_update(&val, sizeof(dods_uint16));
382 
383  if (d_write_data)
384  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint16));
385 }
386 
388 {
389  checksum_update(&val, sizeof(dods_uint32));
390 
391  if (d_write_data)
392  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint32));
393 }
394 
396 {
397  checksum_update(&val, sizeof(dods_uint64));
398 
399  if (d_write_data)
400  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint64));
401 }
402 
411 void D4StreamMarshaller::put_count(int64_t count)
412 {
413  d_out.write(reinterpret_cast<const char*>(&count), sizeof(int64_t));
414 }
415 
416 void D4StreamMarshaller::put_str(const string &val)
417 {
418  checksum_update(val.c_str(), val.length());
419 
420  if (d_write_data) {
421  int64_t len = val.length();
422 
423  d_out.write(reinterpret_cast<const char*>(&len), sizeof(int64_t));
424  d_out.write(val.data(), val.length());
425  }
426 }
427 
428 void D4StreamMarshaller::put_url(const string &val)
429 {
430  put_str(val);
431 }
432 
433 void D4StreamMarshaller::put_opaque_dap4(const char *val, int64_t len)
434 {
435  checksum_update(val, len);
436 
437  if (d_write_data) {
438  d_out.write(reinterpret_cast<const char*>(&len), sizeof(int64_t));
439  d_out.write(val, len);
440  }
441 }
442 
448 void D4StreamMarshaller::put_vector(char *val, int64_t num_bytes)
449 {
450  checksum_update(val, num_bytes);
451 
452  if (d_write_data)
453  d_out.write(val, num_bytes);
454 }
455 
456 void D4StreamMarshaller::put_vector(char *val, int64_t num_elem, int elem_size)
457 {
458  assert(val);
459  assert(num_elem >= 0);
460  assert(elem_size > 0);
461 
462  int64_t bytes;
463 
464  switch (elem_size) {
465  case 1:
466  assert(!"Don't call this method for bytes, use put_vector(val, bytes) instead");
467  bytes = num_elem;
468  break;
469  case 2:
470  // Don't bother testing the sign bit
471  assert(!(num_elem & 0x4000000000000000)); // 0x 40 00 --> 0100 0000
472  bytes = num_elem << 1;
473  break;
474  case 4:
475  assert(!(num_elem & 0x6000000000000000)); // 0x 60 00 --> 0110 0000
476  bytes = num_elem << 2;
477  break;
478  case 8:
479  assert(!(num_elem & 0x7000000000000000)); // 0111 0000
480  bytes = num_elem << 3;
481  break;
482  default:
483  bytes = num_elem * elem_size;
484  break;
485  }
486 
487  checksum_update(val, bytes);
488 
489  if (d_write_data)
490  d_out.write(val, bytes);
491 }
492 
502 void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem)
503 {
504 #if !USE_XDR_FOR_IEEE754_ENCODING
505 
506  assert(std::numeric_limits<float>::is_iec559);
507  assert(val);
508  assert(num_elem >= 0);
509  // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
510  // make sure that doesn't overflow a 63-bit integer (the max positive value in
511  // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
512  // to test that num can be multiplied by 4. A
513  assert(!(num_elem & 0xe000000000000000));
514 
515  num_elem = num_elem << 2; // num_elem is now the number of bytes
516 
517  checksum_update(val, num_elem);
518 
519  if (d_write_data)
520  d_out.write(val, num_elem);
521 
522 #else
523  assert(val);
524  assert(num_elem >= 0);
525  // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
526  // make sure that doesn't overflow a 63-bit integer (the max positive value in
527  // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
528  // to test that num can be multiplied by 4. A
529  assert(!(num_elem & 0xe000000000000000));
530 
531  int64_t bytes = num_elem << 2; // num_elem is now the number of bytes
532 
533  checksum_update(val, bytes);
534 
535  if (d_write_data) {
536  if (!std::numeric_limits<float>::is_iec559) {
537  // If not using IEEE 754, use XDR to get it that way.
538  m_serialize_reals(val, num_elem, 4, type);
539  }
540  else {
541  d_out.write(val, bytes);
542  }
543  }
544 #endif
545 }
546 
555 void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem)
556 {
557 #if !USE_XDR_FOR_IEEE754_ENCODING
558 
559  assert(std::numeric_limits<double>::is_iec559);
560  assert(val);
561  assert(num_elem >= 0);
562  // See comment above
563  assert(!(num_elem & 0xf000000000000000));
564 
565  num_elem = num_elem << 3; // num_elem is now the number of bytes
566 
567  checksum_update(val, num_elem);
568 
569  if (d_write_data)
570  d_out.write(val, num_elem);
571 #else
572  assert(val);
573  assert(num_elem >= 0);
574  // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
575  // make sure that doesn't overflow a 63-bit integer (the max positive value in
576  // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
577  // to test that num can be multiplied by 4. A
578  assert(!(num_elem & 0xe000000000000000));
579 
580  int64_t bytes = num_elem << 3; // num_elem is now the number of bytes
581 
582  checksum_update(val, bytes);
583 
584  if (d_write_data) {
585  if (!std::numeric_limits<double>::is_iec559) {
586  // If not using IEEE 754, use XDR to get it that way.
587  m_serialize_reals(val, num_elem, 8, type);
588  }
589  else {
590  d_out.write(val, bytes);
591  }
592  }
593 #endif
594 
595 }
596 
597 void D4StreamMarshaller::dump(ostream &strm) const
598 {
599  strm << DapIndent::LMarg << "D4StreamMarshaller::dump - (" << (void *) this << ")" << endl;
600 }
601 
602 } // namespace libdap
603 
virtual void put_opaque_dap4(const char *val, int64_t len)
virtual void put_float64(dods_float64 val)
checksum GetCrc32() const
Definition: crc.h:92
DINT32 dods_int32
virtual void put_int16(dods_int16 val)
uint8_t dods_byte
int64_t dods_int64
virtual void put_count(int64_t count)
DFLOAT32 dods_float32
DINT64 dods_int64
virtual void put_byte(dods_byte val)
virtual void dump(std::ostream &strm) const
dump the contents of this object to the specified ostream
virtual void put_vector_float32(char *val, int64_t num_elem)
Write a fixed size vector.
Type
Identifies the data type.
Definition: Type.h:94
uint16_t dods_uint16
virtual void put_uint16(dods_uint16 val)
uint32_t checksum
Definition: crc.h:79
uint64_t dods_uint64
A class for software fault reporting.
Definition: InternalErr.h:64
DUINT64 dods_uint64
#define DBG(x)
Definition: debug.h:58
double dods_float64
uint32_t dods_uint32
virtual void put_uint32(dods_uint32 val)
virtual void put_int8(dods_int8 val)
void AddData(const uint8_t *pData, const uint32_t length)
Definition: crc.h:84
static ostream & LMarg(ostream &strm)
Definition: DapIndent.cc:80
virtual void checksum_update(const void *data, unsigned long len)
virtual void put_uint64(dods_uint64 val)
int16_t dods_int16
virtual void put_vector(char *val, int64_t num_bytes)
Write a fixed size vector.
DFLOAT64 dods_float64
virtual void put_vector_float64(char *val, int64_t num_elem)
Write a fixed size vector of float64s.
virtual void put_checksum()
Write the checksum Write the checksum for the data sent since the last call to reset_checksum() to th...
bool is_host_big_endian()
Does this host use big-endian byte order?
Definition: util.cc:94
virtual void put_float32(dods_float32 val)
virtual void put_int32(dods_int32 val)
void Reset()
Definition: crc.h:83
virtual void put_str(const string &val)
virtual void put_int64(dods_int64 val)
virtual void put_url(const string &val)
int32_t dods_int32