18 #include <google/protobuf/message.h>
34 CFile(fname, rw, name)
44 void CProtobufFile::init()
47 message_size=1024*1024;
49 buffer=SG_MALLOC(uint8_t, message_size*
sizeof(uint32_t));
52 #define GET_VECTOR(sg_type) \
53 void CProtobufFile::get_vector(sg_type*& vector, int32_t& len) \
55 read_and_validate_global_header(ShogunVersion::VECTOR); \
56 VectorHeader data_header=read_vector_header(); \
57 len=data_header.len(); \
58 read_memory_block(vector, len, data_header.num_messages()); \
75 #define GET_MATRIX(read_func, sg_type) \
76 void CProtobufFile::get_matrix(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \
78 read_and_validate_global_header(ShogunVersion::MATRIX); \
79 MatrixHeader data_header=read_matrix_header(); \
80 num_feat=data_header.num_cols(); \
81 num_vec=data_header.num_rows(); \
82 read_memory_block(matrix, num_feat*num_vec, data_header.num_messages()); \
99 #define GET_NDARRAY(read_func, sg_type) \
100 void CProtobufFile::get_ndarray(sg_type*& array, int32_t*& dims, int32_t& num_dims) \
114 #define GET_SPARSE_MATRIX(sg_type) \
115 void CProtobufFile::get_sparse_matrix( \
116 SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \
118 read_and_validate_global_header(ShogunVersion::SPARSE_MATRIX); \
119 SparseMatrixHeader data_header=read_sparse_matrix_header(); \
120 num_feat=data_header.num_features(); \
121 num_vec=data_header.num_vectors(); \
122 read_sparse_matrix(matrix, data_header); \
138 #undef GET_SPARSE_MATRIX
140 #define SET_VECTOR(sg_type) \
141 void CProtobufFile::set_vector(const sg_type* vector, int32_t len) \
143 int32_t num_messages=compute_num_messages(len, sizeof(sg_type)); \
144 write_global_header(ShogunVersion::VECTOR); \
145 write_vector_header(len, num_messages); \
146 write_memory_block(vector, len, num_messages); \
163 #define SET_MATRIX(sg_type) \
164 void CProtobufFile::set_matrix(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \
166 int32_t num_messages=compute_num_messages(num_feat*num_vec, sizeof(sg_type)); \
167 write_global_header(ShogunVersion::MATRIX); \
168 write_matrix_header(num_feat, num_vec, num_messages); \
169 write_memory_block(matrix, num_feat*num_vec, num_messages); \
186 #define SET_SPARSE_MATRIX(sg_type) \
187 void CProtobufFile::set_sparse_matrix( \
188 const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
190 write_global_header(ShogunVersion::SPARSE_MATRIX); \
191 write_sparse_matrix_header(matrix, num_feat, num_vec); \
192 write_sparse_matrix(matrix, num_vec); \
208 #undef SET_SPARSE_MATRIX
210 #define GET_STRING_LIST(sg_type) \
211 void CProtobufFile::get_string_list( \
212 SGString<sg_type>*& strings, int32_t& num_str, \
213 int32_t& max_string_len) \
215 read_and_validate_global_header(ShogunVersion::STRING_LIST); \
216 StringListHeader data_header=read_string_list_header(); \
217 num_str=data_header.num_str(); \
218 max_string_len=data_header.max_string_len(); \
219 read_string_list(strings, data_header); \
234 #undef GET_STRING_LIST
236 #define SET_STRING_LIST(sg_type) \
237 void CProtobufFile::set_string_list( \
238 const SGString<sg_type>* strings, int32_t num_str) \
240 write_global_header(ShogunVersion::STRING_LIST); \
241 write_string_list_header(strings, num_str); \
242 write_string_list(strings, num_str); \
257 #undef SET_STRING_LIST
259 void CProtobufFile::write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size)
262 SG_ERROR(
"array is too small to write\n");
264 array[0]=(number>>24)&0xffu;
265 array[1]=(number>>16)&0xffu;
266 array[2]=(number>>8)&0xffu;
267 array[3]=number&0xffu;
270 uint32_t CProtobufFile::read_big_endian_uint(uint8_t* array, uint32_t size)
273 SG_ERROR(
"array is too small to read\n");
275 return (array[0]<<24) | (array[1]<<16) | (array[2]<<8) | array[3];
278 int32_t CProtobufFile::compute_num_messages(uint64_t len, int32_t sizeof_type)
const
280 uint32_t elements_in_message=message_size/sizeof_type;
281 uint32_t num_messages=len/elements_in_message;
282 if (len % elements_in_message > 0)
291 read_message(header);
301 write_message(header);
307 read_message(data_header);
315 read_message(data_header);
323 read_message(data_header);
331 read_message(data_header);
336 void CProtobufFile::write_vector_header(int32_t len, int32_t num_messages)
341 write_message(data_header);
344 void CProtobufFile::write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages)
350 write_message(data_header);
353 #define WRITE_SPARSE_MATRIX_HEADER(sg_type) \
354 void CProtobufFile::write_sparse_matrix_header( \
355 const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
357 SparseMatrixHeader data_header; \
358 data_header.set_num_features(num_feat); \
359 data_header.set_num_vectors(num_vec); \
360 for (int32_t i=0; i<num_vec; i++) \
362 data_header.add_num_feat_entries(matrix[i].num_feat_entries); \
365 write_message(data_header); \
381 #undef WRITE_SPARSE_MATRIX_HEADER
383 #define WRITE_STRING_LIST_HEADER(sg_type) \
384 void CProtobufFile::write_string_list_header(const SGString<sg_type>* strings, int32_t num_str) \
386 int32_t max_string_len=0; \
387 StringListHeader data_header; \
388 data_header.set_num_str(num_str); \
389 for (int32_t i=0; i<num_str; i++) \
391 data_header.add_str_len(strings[i].slen); \
392 if (strings[i].slen>max_string_len) \
393 max_string_len=strings[i].slen; \
395 data_header.set_max_string_len(max_string_len); \
396 write_message(data_header); \
411 #undef WRITE_STRING_LIST_HEADER
413 void CProtobufFile::read_message(google::protobuf::Message& message)
415 uint32_t bytes_read=0;
419 bytes_read=fread(uint_buffer,
sizeof(
char),
sizeof(uint32_t),
file);
420 REQUIRE(bytes_read==
sizeof(uint32_t),
"IO error\n");
421 msg_size=read_big_endian_uint(uint_buffer,
sizeof(uint32_t));
422 REQUIRE(msg_size>0,
"message size should be more than zero\n");
425 bytes_read=fread(buffer,
sizeof(
char), msg_size,
file);
426 REQUIRE(bytes_read==msg_size,
"IO error\n");
429 REQUIRE(message.ParseFromArray(buffer, msg_size),
"cannot parse header\n");
432 void CProtobufFile::write_message(
const google::protobuf::Message& message)
434 uint32_t bytes_write=0;
435 uint32_t msg_size=message.ByteSize();
438 write_big_endian_uint(msg_size, uint_buffer,
sizeof(uint32_t));
439 bytes_write=fwrite(uint_buffer,
sizeof(
char),
sizeof(uint32_t),
file);
440 REQUIRE(bytes_write==
sizeof(uint32_t),
"IO error\n");
443 message.SerializeToArray(buffer, msg_size);
444 bytes_write=fwrite(buffer,
sizeof(
char), msg_size,
file);
445 REQUIRE(bytes_write==msg_size,
"IO error\n");
448 #define READ_MEMORY_BLOCK(chunk_type, sg_type) \
449 void CProtobufFile::read_memory_block(sg_type*& vector, uint64_t len, int32_t num_messages) \
451 vector=SG_MALLOC(sg_type, len); \
454 int32_t elements_in_message=message_size/sizeof(sg_type); \
455 for (int32_t i=0; i<num_messages; i++) \
457 read_message(chunk); \
459 int32_t num_elements_to_read=0; \
460 if ((len-(i+1)*elements_in_message)<=0) \
461 num_elements_to_read=len-i*elements_in_message; \
463 num_elements_to_read=elements_in_message; \
465 for (int32_t j=0; j<num_elements_to_read; j++) \
466 vector[j+i*elements_in_message]=chunk.data(j); \
482 #undef READ_MEMORY_BLOCK
484 #define WRITE_MEMORY_BLOCK(chunk_type, sg_type) \
485 void CProtobufFile::write_memory_block(const sg_type* vector, uint64_t len, int32_t num_messages) \
488 int32_t elements_in_message=message_size/sizeof(sg_type); \
489 for (int32_t i=0; i<num_messages; i++) \
492 int32_t num_elements_to_write=0; \
493 if ((len-(i+1)*elements_in_message)<=0) \
494 num_elements_to_write=len-i*elements_in_message; \
496 num_elements_to_write=elements_in_message; \
498 for (int32_t j=0; j<num_elements_to_write; j++) \
499 chunk.add_data(vector[j+i*elements_in_message]); \
501 write_message(chunk); \
518 #undef WRITE_MEMORY_BLOCK
520 #define READ_SPARSE_MATRIX(chunk_type, sg_type) \
521 void CProtobufFile::read_sparse_matrix( \
522 SGSparseVector<sg_type>*& matrix, const SparseMatrixHeader& data_header) \
524 matrix=SG_MALLOC(SGSparseVector<sg_type>, data_header.num_vectors()); \
526 UInt64Chunk feat_index_chunk; \
527 chunk_type entry_chunk; \
528 read_message(feat_index_chunk); \
529 read_message(entry_chunk); \
531 int32_t elements_in_message=message_size/sizeof(sg_type); \
532 int32_t buffer_counter=0; \
533 for (uint32_t i=0; i<data_header.num_vectors(); i++) \
535 matrix[i]=SGSparseVector<sg_type>(data_header.num_feat_entries(i)); \
536 for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
538 matrix[i].features[j].feat_index=feat_index_chunk.data(buffer_counter); \
539 matrix[i].features[j].entry=entry_chunk.data(buffer_counter); \
542 if (buffer_counter==elements_in_message) \
544 read_message(feat_index_chunk); \
545 read_message(entry_chunk); \
565 #undef READ_SPARSE_MATRIX
567 #define WRITE_SPARSE_MATRIX(chunk_type, sg_type) \
568 void CProtobufFile::write_sparse_matrix( \
569 const SGSparseVector<sg_type>* matrix, int32_t num_vec) \
571 UInt64Chunk feat_index_chunk; \
572 chunk_type entry_chunk; \
573 int32_t elements_in_message=message_size/sizeof(sg_type); \
574 int32_t buffer_counter=0; \
575 for (int32_t i=0; i<num_vec; i++) \
577 for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
579 feat_index_chunk.add_data(matrix[i].features[j].feat_index); \
580 entry_chunk.add_data(matrix[i].features[j].entry); \
583 if (buffer_counter==elements_in_message) \
585 write_message(feat_index_chunk); \
586 write_message(entry_chunk); \
587 feat_index_chunk.Clear(); \
588 entry_chunk.Clear(); \
594 if (buffer_counter!=0) \
596 write_message(feat_index_chunk); \
597 write_message(entry_chunk); \
614 #undef WRITE_SPARSE_MATRIX
616 #define READ_STRING_LIST(chunk_type, sg_type) \
617 void CProtobufFile::read_string_list( \
618 SGString<sg_type>*& strings, const StringListHeader& data_header) \
620 strings=SG_MALLOC(SGString<sg_type>, data_header.num_str()); \
623 read_message(chunk); \
624 int32_t elements_in_message=message_size/sizeof(sg_type); \
625 int32_t buffer_counter=0; \
626 for (uint32_t i=0; i<data_header.num_str(); i++) \
628 strings[i]=SGString<sg_type>(data_header.str_len(i)); \
629 for (int32_t j=0; j<strings[i].slen; j++) \
631 strings[i].string[j]=chunk.data(buffer_counter); \
634 if (buffer_counter==elements_in_message) \
636 read_message(chunk); \
655 #undef READ_STRING_LIST
657 #define WRITE_STRING_LIST(chunk_type, sg_type) \
658 void CProtobufFile::write_string_list( \
659 const SGString<sg_type>* strings, int32_t num_str) \
662 int32_t elements_in_message=message_size/sizeof(sg_type); \
663 int32_t buffer_counter=0; \
664 for (int32_t i=0; i<num_str; i++) \
666 for (int32_t j=0; j<strings[i].slen; j++) \
668 chunk.add_data(strings[i].string[j]); \
671 if (buffer_counter==elements_in_message) \
673 write_message(chunk); \
680 if (buffer_counter!=0) \
681 write_message(chunk); \
696 #undef WRITE_STRING_LIST
inline::shogun::ShogunVersion_SGDataType data_type() const
void set_data_type(::shogun::ShogunVersion_SGDataType value)
#define SET_MATRIX(sg_type)
#define GET_MATRIX(read_func, sg_type)
#define GET_STRING_LIST(sg_type)
#define SET_STRING_LIST(sg_type)
#define GET_NDARRAY(read_func, sg_type)
#define GET_SPARSE_MATRIX(sg_type)
#define GET_VECTOR(sg_type)
#define READ_STRING_LIST(chunk_type, sg_type)
#define WRITE_STRING_LIST(chunk_type, sg_type)
#define SET_VECTOR(sg_type)
void set_version(::google::protobuf::int32 value)
A File access base class.
#define WRITE_SPARSE_MATRIX(chunk_type, sg_type)
all of classes and functions are contained in the shogun namespace
#define WRITE_SPARSE_MATRIX_HEADER(sg_type)
inline::google::protobuf::int32 version() const
#define WRITE_STRING_LIST_HEADER(sg_type)
#define WRITE_MEMORY_BLOCK(chunk_type, sg_type)
#define READ_SPARSE_MATRIX(chunk_type, sg_type)
#define READ_MEMORY_BLOCK(chunk_type, sg_type)
#define SET_SPARSE_MATRIX(sg_type)