#include <bmsparsevec_serial.h>

Inheritance diagram for bm::sparse_vector_serializer< SV >:

[legend]

Collaboration diagram for bm::sparse_vector_serializer< SV >:

[legend]

Public Types
typedef SV::bvector_type	bvector_type
typedef const bvector_type *	bvector_type_const_ptr
typedef bvector_type *	bvector_type_ptr
typedef SV::value_type	value_type
typedef SV::size_type	size_type
typedef bvector_type::allocator_type	alloc_type
typedef alloc_type::allocator_pool_type	allocator_pool_type
typedef bm::serializer< bvector_type >::bv_ref_vector_type	bv_ref_vector_type
typedef bm::serializer< bvector_type >::xor_sim_model_type	xor_sim_model_type

Public Member Functions
	sparse_vector_serializer ()
Compression settings
void	set_bookmarks (bool enable, unsigned bm_interval=256) BMNOEXCEPT
	Add skip-markers for faster range deserialization.
void	enable_xor_compression () BMNOEXCEPT
	Enable XOR compression on vector serialization.
void	disable_xor_compression () BMNOEXCEPT
	Disable XOR compression on serialization.
void	set_xor_ref (bool is_enabled) BMNOEXCEPT
	Turn ON and OFF XOR compression of sparse vectors Enables XOR reference compression for the sparse vector.
void	set_xor_ref (const bv_ref_vector_type *bv_ref_ptr) BMNOEXCEPT
	Enable external XOR serialization via external reference vectors (data frame ref.
void	compute_sim_model (xor_sim_model_type &sim_model, const bv_ref_vector_type &ref_vect, const bm::xor_sim_params &params)
	Calculate XOR similarity model for ref_vector refernece vector must be associated before.
void	set_sim_model (const xor_sim_model_type *sim_model) BMNOEXCEPT
	Attach serizalizer to a pre-computed similarity model.
bool	is_xor_ref () const BMNOEXCEPT
	Returns the XOR reference compression status (enabled/disabled).
Serialization
void	serialize (const SV &sv, sparse_vector_serial_layout< SV > &sv_layout)
	Serialize sparse vector into a memory buffer(s) structure.
bm::serializer< bvector_type > &	get_bv_serializer () BMNOEXCEPT
	Get access to the underlying bit-vector serializer This access can be used to fine tune compression settings.

Protected Types
typedef SV::remap_matrix_type	remap_matrix_type
typedef bm::heap_vector< unsigned, alloc_type, true >	u32_vector_type
typedef bm::serializer< bvector_type >	serializer_type
typedef serializer_type::buffer	buffer_type

Protected Member Functions
void	build_xor_ref_vector (const SV &sv)
void	encode_remap_matrix (bm::encoder &enc, const SV &sv)
	serialize the remap matrix used for SV encoding

Static Protected Member Functions
static void	build_plane_digest (bvector_type &digest_bv, const SV &sv)

Protected Attributes
bm::serializer< bvector_type >	bvs_
bvector_type	plane_digest_bv_
	bv.digest of bit-planes
buffer_type	plane_digest_buf_
	serialization buf
u32_vector_type	plane_off_vect_
u32_vector_type	remap_rlen_vect_
bool	is_xor_ref_
bv_ref_vector_type	bv_ref_
xor_sim_model_type	sim_model_
const bv_ref_vector_type *	bv_ref_ptr_
const xor_sim_model_type *	sim_model_ptr_

Detailed Description

template<typename SV>
class bm::sparse_vector_serializer< SV >

\brief Serialize sparse vector into a memory buffer(s) structure

Serialization format:

| HEADER | BIT-VECTORS ... | REMAP_MATRIX

Header structure:

BYTE+BYTE: Magic-signature 'BM' or 'BC' (c-compressed) BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian) { BYTE : Number of Bit-vector planes (total) (non-zero when < 255 planes) | BYTE: zero - flag of large plane matrix INT64: Nnmber of bit-vector planes } INT64: Vector size INT64: Offset of plane 0 from the header start (value 0 means plane is empty) INT64: Offset of plane 1 from ... INT32: reserved

Bit-vectors:

Based on current bit-vector serialization

Remap Matrix: SubHeader | Matrix BLOB

sub-header: BYTE: 'R' (remapping) or 'N' (no remapping) N - means no other info is saved on the stream INT64: remap matrix size

Examples: rscsample02.cpp, rscsample05.cpp, strsvsample05.cpp, svsample02.cpp, and svsample08.cpp.

Definition at line 175 of file bmsparsevec_serial.h.

Member Typedef Documentation

◆ alloc_type

template<typename SV>

typedef bvector_type::allocator_type bm::sparse_vector_serializer< SV >::alloc_type

Definition at line 183 of file bmsparsevec_serial.h.

◆ allocator_pool_type

template<typename SV>

typedef alloc_type::allocator_pool_type bm::sparse_vector_serializer< SV >::allocator_pool_type

Definition at line 184 of file bmsparsevec_serial.h.

◆ buffer_type

template<typename SV>

typedef serializer_type::buffer bm::sparse_vector_serializer< SV >::buffer_type

protected

Definition at line 300 of file bmsparsevec_serial.h.

◆ bv_ref_vector_type

template<typename SV>

typedef bm::serializer<bvector_type>::bv_ref_vector_type bm::sparse_vector_serializer< SV >::bv_ref_vector_type

Definition at line 186 of file bmsparsevec_serial.h.

◆ bvector_type

template<typename SV>

typedef SV::bvector_type bm::sparse_vector_serializer< SV >::bvector_type

Definition at line 178 of file bmsparsevec_serial.h.

◆ bvector_type_const_ptr

template<typename SV>

typedef const bvector_type* bm::sparse_vector_serializer< SV >::bvector_type_const_ptr

Definition at line 179 of file bmsparsevec_serial.h.

◆ bvector_type_ptr

template<typename SV>

typedef bvector_type* bm::sparse_vector_serializer< SV >::bvector_type_ptr

Definition at line 180 of file bmsparsevec_serial.h.

◆ remap_matrix_type

template<typename SV>

typedef SV::remap_matrix_type bm::sparse_vector_serializer< SV >::remap_matrix_type

protected

Definition at line 293 of file bmsparsevec_serial.h.

◆ serializer_type

template<typename SV>

typedef bm::serializer<bvector_type> bm::sparse_vector_serializer< SV >::serializer_type

protected

Definition at line 299 of file bmsparsevec_serial.h.

◆ size_type

template<typename SV>

typedef SV::size_type bm::sparse_vector_serializer< SV >::size_type

Definition at line 182 of file bmsparsevec_serial.h.

◆ u32_vector_type

template<typename SV>

typedef bm::heap_vector<unsigned, alloc_type, true> bm::sparse_vector_serializer< SV >::u32_vector_type

protected

Definition at line 298 of file bmsparsevec_serial.h.

◆ value_type

template<typename SV>

typedef SV::value_type bm::sparse_vector_serializer< SV >::value_type

Definition at line 181 of file bmsparsevec_serial.h.

◆ xor_sim_model_type

template<typename SV>

typedef bm::serializer<bvector_type>::xor_sim_model_type bm::sparse_vector_serializer< SV >::xor_sim_model_type

Definition at line 188 of file bmsparsevec_serial.h.

Constructor & Destructor Documentation

◆ sparse_vector_serializer()

template<typename SV>

bm::sparse_vector_serializer< SV >::sparse_vector_serializer ( )

Definition at line 779 of file bmsparsevec_serial.h.

References bv_ref_ptr_, bvs_, and is_xor_ref_.

Member Function Documentation

◆ build_plane_digest()

template<typename SV>

void bm::sparse_vector_serializer< SV >::build_plane_digest	(	bvector_type &	digest_bv,
		const SV &	sv )

staticprotected

Definition at line 924 of file bmsparsevec_serial.h.

Referenced by serialize().

◆ build_xor_ref_vector()

template<typename SV>

void bm::sparse_vector_serializer< SV >::build_xor_ref_vector ( const SV & sv )

protected

Definition at line 833 of file bmsparsevec_serial.h.

References bv_ref_.

Referenced by serialize().

◆ compute_sim_model()

template<typename SV>

void bm::sparse_vector_serializer< SV >::compute_sim_model	(	xor_sim_model_type &	sim_model,
		const bv_ref_vector_type &	ref_vect,
		const bm::xor_sim_params &	params )

Calculate XOR similarity model for ref_vector refernece vector must be associated before.

See also: set_ref_vectors, set_sim_model

Definition at line 813 of file bmsparsevec_serial.h.

References bvs_.

Referenced by serialize_df2().

◆ disable_xor_compression()

template<typename SV>

void bm::sparse_vector_serializer< SV >::disable_xor_compression ( )

inline

Disable XOR compression on serialization.

Definition at line 221 of file bmsparsevec_serial.h.

Referenced by SDemo2().

◆ enable_xor_compression()

template<typename SV>

void bm::sparse_vector_serializer< SV >::enable_xor_compression ( )

inline

Enable XOR compression on vector serialization.

See also: set_xor_ref; disable_xor_compression

Definition at line 215 of file bmsparsevec_serial.h.

Referenced by SDemo2().

◆ encode_remap_matrix()

template<typename SV>

void bm::sparse_vector_serializer< SV >::encode_remap_matrix	(	bm::encoder &	enc,
		const SV &	sv )

protected

serialize the remap matrix used for SV encoding

Definition at line 842 of file bmsparsevec_serial.h.

References BM_ASSERT, BMRESTRICT, bm::count_nz(), bm::bit_out< TEncoder >::gamma(), bm::encoder::memcpy(), bm::encoder::put_16(), bm::encoder::put_32(), bm::encoder::put_64(), bm::encoder::put_8(), and remap_rlen_vect_.

Referenced by serialize().

◆ get_bv_serializer()

template<typename SV>

bm::serializer< bvector_type > & bm::sparse_vector_serializer< SV >::get_bv_serializer ( )

inline

Get access to the underlying bit-vector serializer This access can be used to fine tune compression settings.

See also: bm::serializer::set_compression_level

Definition at line 281 of file bmsparsevec_serial.h.

◆ is_xor_ref()

template<typename SV>

bool bm::sparse_vector_serializer< SV >::is_xor_ref ( ) const

inline

Returns the XOR reference compression status (enabled/disabled).

Definition at line 260 of file bmsparsevec_serial.h.

Referenced by SDemo2(), and serialize().

◆ serialize()

template<typename SV>

void bm::sparse_vector_serializer< SV >::serialize	(	const SV &	sv,
		sparse_vector_serial_layout< SV > &	sv_layout )

Serialize sparse vector into a memory buffer(s) structure.

Parameters

sv	- sparse vector to serialize
sv_layout	- buffer structure to keep the result as defined in bm::serialization_flags

Examples: rscsample02.cpp, strsvsample05.cpp, and svsample08.cpp.

Definition at line 941 of file bmsparsevec_serial.h.

References bm::bit_out< TEncoder >::bic_encode_u32_cm(), BM_ASSERT, build_plane_digest(), build_xor_ref_vector(), bv_ref_, bv_ref_ptr_, bvs_, bm::globals< T >::byte_order(), bm::sparse_vector_serial_layout< SV >::capacity(), encode_remap_matrix(), bm::sparse_vector_serial_layout< SV >::get_plane(), bm::id_max32, is_xor_ref(), plane_digest_buf_, plane_digest_bv_, plane_off_vect_, bm::encoder::put_32(), bm::encoder::put_64(), bm::encoder::put_8(), bm::sparse_vector_serial_layout< SV >::reserve(), bm::sparse_vector_serial_layout< SV >::resize(), bm::sparse_vector_serial_layout< SV >::resize_slices(), bm::sparse_vector_serial_layout< SV >::set_plane(), sim_model_, sim_model_ptr_, bm::encoder::size(), and bm::conditional< b >::test().

Referenced by main(), SDemo2(), serialize_df0(), serialize_df2(), and bm::sparse_vector_serialize().

◆ set_bookmarks()

template<typename SV>

void bm::sparse_vector_serializer< SV >::set_bookmarks	(	bool	enable,
		unsigned	bm_interval = 256 )

inline

Add skip-markers for faster range deserialization.

Parameters

enable	- TRUE searilization will add bookmark codes
bm_interval	- bookmark interval in (number of blocks) (suggested between 4 and 512) smaller interval means more bookmarks added to the skip list thus more increasing the BLOB size