#include <emmintrin.h>
#include <immintrin.h>
#include "bmdef.h"
#include "bmbmi2.h"
#include "bmutil.h"

Include dependency graph for bmavx2.h:

This graph shows which files directly or indirectly include this file:

Namespaces
namespace	bm

Macros
#define	BM_CSA256(h, l, a, b, c)
#define	BM_AVX2_BIT_COUNT(ret, v)
#define	BM_AVX2_DECL_LOOKUP1
#define	BM_AVX2_DECL_LOOKUP2
#define	BM_AVX2_POPCNT_PROLOG
#define	VECT_XOR_ARR_2_MASK(dst, src, src_end, mask)
#define	VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, mask)
#define	VECT_BITCOUNT(first, last)
#define	VECT_BITCOUNT_AND(first, last, mask)
#define	VECT_BITCOUNT_OR(first, last, mask)
#define	VECT_BITCOUNT_XOR(first, last, mask)
#define	VECT_BITCOUNT_SUB(first, last, mask)
#define	VECT_INVERT_BLOCK(first)
#define	VECT_AND_BLOCK(dst, src)
#define	VECT_AND_DIGEST(dst, src)
#define	VECT_AND_DIGEST_2WAY(dst, src1, src2)
#define	VECT_AND_OR_DIGEST_2WAY(dst, src1, src2)
#define	VECT_AND_DIGEST_5WAY(dst, src1, src2, src3, src4)
#define	VECT_AND_DIGEST_3WAY(dst, src1, src2)
#define	VECT_OR_BLOCK(dst, src)
#define	VECT_OR_BLOCK_3WAY(dst, src1, src2)
#define	VECT_OR_BLOCK_2WAY(dst, src1, src2)
#define	VECT_OR_BLOCK_3WAY(dst, src1, src2)
#define	VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4)
#define	VECT_SUB_BLOCK(dst, src)
#define	VECT_SUB_DIGEST(dst, src)
#define	VECT_SUB_DIGEST_2WAY(dst, src1, src2)
#define	VECT_SUB_DIGEST_5WAY(dst, src1, src2, src3, src4)
#define	VECT_SUB_DIGEST_3WAY(dst, src1, src2)
#define	VECT_XOR_BLOCK(dst, src)
#define	VECT_XOR_BLOCK_2WAY(dst, src1, src2)
#define	VECT_COPY_BLOCK(dst, src)
#define	VECT_COPY_BLOCK_UNALIGN(dst, src)
#define	VECT_STREAM_BLOCK(dst, src)
#define	VECT_STREAM_BLOCK_UNALIGN(dst, src)
#define	VECT_SET_BLOCK(dst, value)
#define	VECT_IS_ZERO_BLOCK(dst)
#define	VECT_IS_ONE_BLOCK(dst)
#define	VECT_IS_DIGEST_ZERO(start)
#define	VECT_BLOCK_SET_DIGEST(dst, val)
#define	VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to)
#define	VECT_SHIFT_L1(b, acc, co)
#define	VECT_SHIFT_R1(b, acc, co)
#define	VECT_SHIFT_R1_AND(b, co, m, digest)
#define	VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start)
#define	VECT_SET_BLOCK_BITS(block, idx, start, stop)
#define	VECT_BLOCK_CHANGE(block, size)
#define	VECT_BLOCK_XOR_CHANGE(block, xor_block, size, gc, bc)
#define	VECT_BLOCK_CHANGE_BC(block, gc, bc)
#define	VECT_BIT_TO_GAP(dest, src, dest_len)
#define	VECT_BIT_FIND_FIRST(src1, off, pos)
#define	VECT_BIT_FIND_DIFF(src1, src2, pos)
#define	VECT_BIT_BLOCK_XOR(t, src, src_xor, d)
#define	VECT_BIT_BLOCK_XOR_2WAY(t, src_xor, d)
#define	VECT_GAP_BFIND(buf, pos, is_set)
#define	VECT_GAP_TEST(buf, pos)
#define	VECT_BIT_COUNT_DIGEST(blk, d)

Functions
bm::id_t	bm::avx2_bit_count (const __m256i BMRESTRICT block, const __m256i BMRESTRICT block_end)
	AVX2 Harley-Seal popcount The algorithm is based on the paper "Faster Population Counts using AVX2 Instructions" by Daniel Lemire, Nathan Kurz and Wojciech Mula (23 Nov 2016).
bm::id_t	bm::avx2_bit_block_count (const bm::word_t *const block, bm::id64_t digest)
	Calculate population count based on digest.
bm::id_t	bm::avx2_bit_count_and (const __m256i BMRESTRICT block, const __m256i BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
	AND bit count for two aligned bit-blocks.
bm::id_t	bm::avx2_bit_count_or (const __m256i BMRESTRICT block, const __m256i BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
bm::id_t	bm::avx2_bit_count_xor (const __m256i BMRESTRICT block, const __m256i BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
	XOR bit count for two aligned bit-blocks.
bm::id_t	bm::avx2_bit_count_sub (const __m256i BMRESTRICT block, const __m256i BMRESTRICT block_end, const __m256i *BMRESTRICT mask_block)
	AND NOT bit count for two aligned bit-blocks.
void	bm::avx2_xor_arr_2_mask (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src, const __m256i *BMRESTRICT src_end, bm::word_t mask)
	XOR array elements to specified mask dst = *src ^ mask.
void	bm::avx2_andnot_arr_2_mask (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src, const __m256i *BMRESTRICT src_end, bm::word_t mask)
	Inverts array elements and NOT them to specified mask dst = ~*src & mask.
unsigned	bm::avx2_and_block (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AND array elements against another array dst &= *src.
bool	bm::avx2_and_digest (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AND block digest stride dst &= *src.
bool	bm::avx2_and_digest_2way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	AND block digest stride 2 way dst = src1 & src2.
bool	bm::avx2_and_or_digest_2way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	AND-OR block digest stride 2 way dst \|= src1 & src2.
bool	bm::avx2_and_digest_5way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i BMRESTRICT src2, const __m256i BMRESTRICT src3, const __m256i *BMRESTRICT src4)
	AND block digest stride.
bool	bm::avx2_and_digest_3way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	AND block digest stride.
unsigned	bm::avx2_and_arr_unal (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src, const __m256i *BMRESTRICT src_end)
	AND array elements against another array (unaligned) dst &= *src.
bool	bm::avx2_or_block (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	OR array elements against another array dst \|= *src.
bool	bm::avx2_or_arr_unal (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src, const __m256i *BMRESTRICT src_end)
	OR array elements against another unaligned array dst \|= *src.
bool	bm::avx2_or_block_2way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	OR 2 arrays and copy to the destination dst = *src1 \| src2.
bool	bm::avx2_or_block_3way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	OR array elements against another 2 arrays dst \|= *src1 \| src2.
bool	bm::avx2_or_block_5way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i BMRESTRICT src2, const __m256i BMRESTRICT src3, const __m256i *BMRESTRICT src4)
	OR array elements against another 4 arrays dst \|= *src1 \| src2.
unsigned	bm::avx2_xor_block (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	XOR block against another dst ^= *src.
unsigned	bm::avx2_xor_block_2way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	3 operand XOR dst = *src1 ^ src2
unsigned	bm::avx2_sub_block (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AND-NOT (SUB) array elements against another array dst &= ~*src.
bool	bm::avx2_sub_digest (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	SUB (AND NOT) block digest stride dst &= ~*src.
bool	bm::avx2_sub_digest_2way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	2-operand SUB (AND NOT) block digest stride dst = src1 & ~src2
bool	bm::avx2_sub_digest_5way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i BMRESTRICT src2, const __m256i BMRESTRICT src3, const __m256i *BMRESTRICT src4)
	SUB block digest stride.
bool	bm::avx2_sub_digest_3way (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src1, const __m256i *BMRESTRICT src2)
	SUB block digest stride.
BMFORCEINLINE void	bm::avx2_set_block (__m256i *BMRESTRICT dst, bm::word_t value)
	AVX2 block memset dst = value.
void	bm::avx2_copy_block (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AVX2 block copy dst = *src.
void	bm::avx2_copy_block_unalign (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AVX2 block copy (unaligned SRC) dst = *src.
void	bm::avx2_stream_block (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AVX2 block copy dst = *src.
void	bm::avx2_stream_block_unalign (__m256i BMRESTRICT dst, const __m256i BMRESTRICT src)
	AVX2 block copy (unaligned SRC) dst = *src.
void	bm::avx2_invert_block (__m256i *BMRESTRICT dst)
	Invert bit-block dst = ~dst or dst ^= dst.
bool	bm::avx2_is_all_zero (const __m256i *BMRESTRICT block)
	check if block is all zero bits
bool	bm::avx2_is_digest_zero (const __m256i *BMRESTRICT block)
	check if digest stride is all zero bits
void	bm::avx2_block_set_digest (__m256i *dst, unsigned value)
	set digest stride to 0xFF.. or 0x0 value
bool	bm::avx2_is_all_one (const __m256i *BMRESTRICT block)
	check if block is all one bits
BMFORCEINLINE bool	bm::avx2_test_all_one_wave (const void *ptr)
	check if wave of pointers is all 0xFFF
BMFORCEINLINE bool	bm::avx2_test_all_zero_wave (const void *ptr)
	check if wave of pointers is all NULL
BMFORCEINLINE bool	bm::avx2_test_all_zero_wave2 (const void ptr0, const void ptr1)
	check if 2 wave of pointers are all NULL
BMFORCEINLINE bool	bm::avx2_test_all_eq_wave2 (const void ptr0, const void ptr1)
	check if 2 wave of pointers are all the same (NULL or FULL)
bool	bm::avx2_shift_l1 (__m256i block, bm::word_t empty_acc, unsigned co1)
	block shift left by 1
bool	bm::avx2_shift_r1 (__m256i block, bm::word_t empty_acc, unsigned co1)
	block shift right by 1
bool	bm::avx2_shift_r1_and (__m256i BMRESTRICT block, bm::word_t co1, const __m256i BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest)
	fused block shift right by 1 plus AND
unsigned	bm::avx2_bit_block_calc_change (const __m256i *BMRESTRICT block, unsigned size)
void	bm::avx2_bit_block_calc_xor_change (const __m256i BMRESTRICT block, const __m256i BMRESTRICT xor_block, unsigned size, unsigned BMRESTRICT gcount, unsigned BMRESTRICT bcount)
void	bm::avx2_bit_block_calc_change_bc (const __m256i BMRESTRICT block, unsigned gcount, unsigned *bcount)
bool	bm::avx2_bit_find_first_diff (const __m256i BMRESTRICT block1, const __m256i BMRESTRICT block2, unsigned *pos)
	Find first bit which is different between two bit-blocks.
bool	bm::avx2_bit_find_first (const __m256i BMRESTRICT block, unsigned off, unsigned pos)
	Find first bit set.
const bm::gap_word_t *	bm::avx2_gap_sum_arr (const bm::gap_word_t pbuf, unsigned avx_vect_waves, unsigned sum)
unsigned	bm::avx2_idx_arr_block_lookup (const unsigned *idx, unsigned size, unsigned nb, unsigned start)
void	bm::avx2_set_block_bits (bm::word_t BMRESTRICT block, const unsigned BMRESTRICT idx, unsigned start, unsigned stop)
BMFORCEINLINE __m256i	bm::avx2_setbit_256 (__m256i target, __m256i source)
	Set a bits in an AVX target, by indexes (int4) from the source.
void	bm::avx2_set_block_bits2 (bm::word_t BMRESTRICT block, const unsigned BMRESTRICT idx, unsigned start, unsigned stop)
	Experimental code to set bits via AVX strides.
void	bm::avx2_set_block_bits3 (bm::word_t BMRESTRICT block, const unsigned BMRESTRICT idx, unsigned start, unsigned stop)
	Experimental code to set bits via AVX strides.
__m256i	bm::avx2_setbit_to256 (unsigned i)
	Experiemntal.
int	bm::avx2_cmpge_u32 (__m256i vect8, unsigned value)
	Experimental (test) function to do SIMD vector search (lower bound) in sorted, growing array.
int	bm::avx2_cmpge_u16 (__m256i vect16, unsigned short value)
	Experimental (test) function to do SIMD vector search in sorted, growing array.
template<bool RET_TEST = false>
unsigned	bm::avx2_gap_bfind (const unsigned short BMRESTRICT buf, unsigned pos, unsigned BMRESTRICT is_set)
	Hybrid binary search, starts as binary, then switches to scan.
unsigned	bm::avx2_gap_test (const unsigned short *BMRESTRICT buf, unsigned pos)
	Hybrid binary search, starts as binary, then switches to scan.
unsigned	bm::avx2_lower_bound_scan_u32 (const unsigned *BMRESTRICT arr, unsigned target, unsigned from, unsigned to)
	lower bound (great or equal) linear scan in ascending order sorted array
void	bm::avx2_bit_block_gather_scatter (unsigned BMRESTRICT arr, const unsigned BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx)
unsigned	bm::avx2_bit_to_gap (gap_word_t BMRESTRICT dest, const unsigned BMRESTRICT block, unsigned dest_len)
	Convert bit block to GAP block.
void	bm::avx2_bit_block_xor (bm::word_t target_block, const bm::word_t block, const bm::word_t *xor_block, bm::id64_t digest)
	Build partial XOR product of 2 bit-blocks using digest mask.
void	bm::avx2_bit_block_xor_2way (bm::word_t target_block, const bm::word_t xor_block, bm::id64_t digest) BMNOEXCEPT
	Build partial XOR product of 2 bit-blocks using digest mask.

Macro Definition Documentation

◆ BM_AVX2_BIT_COUNT

#define BM_AVX2_BIT_COUNT	(		ret,
			v )

Value:

{ \
    __m256i lo = _mm256_and_si256(v, low_mask); \
    __m256i hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), low_mask); \
    __m256i cnt1 = _mm256_shuffle_epi8(lookup1, lo); \
    __m256i cnt2 = _mm256_shuffle_epi8(lookup2, hi); \
    ret = _mm256_sad_epu8(cnt1, cnt2); \
}

Definition at line 124 of file bmavx2.h.

Referenced by bm::avx2_bit_block_calc_change(), bm::avx2_bit_block_calc_change_bc(), bm::avx2_bit_block_calc_xor_change(), bm::avx2_bit_block_count(), bm::avx2_bit_count(), bm::avx2_bit_count_and(), bm::avx2_bit_count_or(), bm::avx2_bit_count_sub(), bm::avx2_bit_count_xor(), and bm::bit_block_calc_count_to().

◆ BM_AVX2_DECL_LOOKUP1

#define BM_AVX2_DECL_LOOKUP1

Value:

__m256i lookup1 = _mm256_setr_epi8(4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, \

4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8);

Definition at line 133 of file bmavx2.h.

◆ BM_AVX2_DECL_LOOKUP2

#define BM_AVX2_DECL_LOOKUP2

Value:

__m256i lookup2 = _mm256_setr_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, \

4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0);

Definition at line 136 of file bmavx2.h.

◆ BM_AVX2_POPCNT_PROLOG

#define BM_AVX2_POPCNT_PROLOG

Value:

  BM_AVX2_DECL_LOOKUP1 \
  BM_AVX2_DECL_LOOKUP2 \
  __m256i low_mask = _mm256_set1_epi8(0x0f); \
  __m256i bc;

Definition at line 140 of file bmavx2.h.

Referenced by bm::avx2_bit_block_calc_change(), bm::avx2_bit_block_calc_change_bc(), bm::avx2_bit_block_calc_xor_change(), bm::avx2_bit_block_count(), bm::avx2_bit_count(), bm::avx2_bit_count_and(), bm::avx2_bit_count_or(), bm::avx2_bit_count_sub(), bm::avx2_bit_count_xor(), and bm::bit_block_calc_count_to().

◆ BM_CSA256

#define BM_CSA256	(	h,
		l,
		a,
		b,
		c )

Value:

{ \
    __m256i u = _mm256_xor_si256(a, b); \
    h = _mm256_or_si256(_mm256_and_si256(a, b), _mm256_and_si256(u, c)); \
    l = _mm256_xor_si256(u, c); \
}

Definition at line 117 of file bmavx2.h.

Referenced by bm::avx2_bit_count().

◆ VECT_AND_BLOCK

#define VECT_AND_BLOCK	(		dst,
			src )

Value:

avx2_and_block((__m256i*) dst, (const __m256i*) (src))

Definition at line 3456 of file bmavx2.h.

Referenced by bm::bit_block_and().

◆ VECT_AND_DIGEST

#define VECT_AND_DIGEST	(		dst,
			src )

Value:

avx2_and_digest((__m256i*) dst, (const __m256i*) (src))

Definition at line 3459 of file bmavx2.h.

Referenced by bm::bit_block_and().

◆ VECT_AND_DIGEST_2WAY

#define VECT_AND_DIGEST_2WAY	(	dst,
		src1,
		src2 )

Value:

avx2_and_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3462 of file bmavx2.h.

Referenced by bm::bit_block_and_2way(), and bm::bit_block_init_and_2way().

◆ VECT_AND_DIGEST_3WAY

#define VECT_AND_DIGEST_3WAY	(	dst,
		src1,
		src2 )

Value:

avx2_and_digest_3way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3471 of file bmavx2.h.

Referenced by bm::bit_block_and_3way().

◆ VECT_AND_DIGEST_5WAY

#define VECT_AND_DIGEST_5WAY	(	dst,
		src1,
		src2,
		src3,
		src4 )

Value:

avx2_and_digest_5way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2), (const __m256i*) (src3), (const __m256i*) (src4))

Definition at line 3468 of file bmavx2.h.

Referenced by bm::bit_block_and_5way().

◆ VECT_AND_OR_DIGEST_2WAY

#define VECT_AND_OR_DIGEST_2WAY	(	dst,
		src1,
		src2 )

Value:

avx2_and_or_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3465 of file bmavx2.h.

Referenced by bm::bit_block_and_or_2way().

◆ VECT_ANDNOT_ARR_2_MASK

#define VECT_ANDNOT_ARR_2_MASK	(	dst,
		src,
		src_end,
		mask )

Value:

avx2_andnot_arr_2_mask((__m256i*)(dst), (__m256i*)(src), (__m256i*)(src_end), (bm::word_t)mask)

bm::word_t

unsigned int word_t

Definition bmconst.h:39

Definition at line 3435 of file bmavx2.h.

Referenced by bm::bit_andnot_arr_ffmask().

◆ VECT_ARR_BLOCK_LOOKUP

#define VECT_ARR_BLOCK_LOOKUP	(	idx,
		size,
		nb,
		start )

Value:

avx2_idx_arr_block_lookup(idx, size, nb, start)

Definition at line 3549 of file bmavx2.h.

Referenced by bm::idx_arr_block_lookup_u32().

◆ VECT_BIT_BLOCK_XOR

#define VECT_BIT_BLOCK_XOR	(	t,
		src,
		src_xor,
		d )

Value:

avx2_bit_block_xor(t, src, src_xor, d)

Definition at line 3573 of file bmavx2.h.

Referenced by bm::bit_block_xor().

◆ VECT_BIT_BLOCK_XOR_2WAY

#define VECT_BIT_BLOCK_XOR_2WAY	(	t,
		src_xor,
		d )

Value:

avx2_bit_block_xor_2way(t, src_xor, d)

Definition at line 3576 of file bmavx2.h.

Referenced by bm::bit_block_xor().

◆ VECT_BIT_COUNT_DIGEST

#define VECT_BIT_COUNT_DIGEST	(		blk,
			d )

Value:

avx2_bit_block_count(blk, d)

Definition at line 3586 of file bmavx2.h.

Referenced by bm::bit_block_count().

◆ VECT_BIT_FIND_DIFF

#define VECT_BIT_FIND_DIFF	(	src1,
		src2,
		pos )

Value:

avx2_bit_find_first_diff((__m256i*) src1, (__m256i*) (src2), pos)

Definition at line 3570 of file bmavx2.h.

Referenced by bm::bit_find_first_diff().

◆ VECT_BIT_FIND_FIRST

#define VECT_BIT_FIND_FIRST	(	src1,
		off,
		pos )

Value:

avx2_bit_find_first((__m256i*) src1, off, pos)

Definition at line 3567 of file bmavx2.h.

Referenced by bm::bit_find_first(), and bm::bit_find_first().

◆ VECT_BIT_TO_GAP

#define VECT_BIT_TO_GAP	(	dest,
		src,
		dest_len )

Value:

avx2_bit_to_gap(dest, src, dest_len)

Definition at line 3564 of file bmavx2.h.

Referenced by bm::bit_to_gap().

◆ VECT_BITCOUNT

#define VECT_BITCOUNT	(		first,
			last )

Value:

avx2_bit_count((__m256i*) (first), (__m256i*) (last))

Definition at line 3438 of file bmavx2.h.

Referenced by bm::bit_block_count().

◆ VECT_BITCOUNT_AND

#define VECT_BITCOUNT_AND	(	first,
		last,
		mask )

Value:

avx2_bit_count_and((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3441 of file bmavx2.h.

Referenced by bm::bit_block_and_count().

◆ VECT_BITCOUNT_OR

#define VECT_BITCOUNT_OR	(	first,
		last,
		mask )

Value:

avx2_bit_count_or((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3444 of file bmavx2.h.

Referenced by bm::bit_block_or_count().

◆ VECT_BITCOUNT_SUB

#define VECT_BITCOUNT_SUB	(	first,
		last,
		mask )

Value:

avx2_bit_count_sub((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3450 of file bmavx2.h.

Referenced by bm::bit_block_sub_count().

◆ VECT_BITCOUNT_XOR

#define VECT_BITCOUNT_XOR	(	first,
		last,
		mask )

Value:

avx2_bit_count_xor((__m256i*) (first), (__m256i*) (last), (__m256i*) (mask))

Definition at line 3447 of file bmavx2.h.

Referenced by bm::bit_block_xor_count().

◆ VECT_BLOCK_CHANGE

#define VECT_BLOCK_CHANGE	(		block,
			size )

Value:

avx2_bit_block_calc_change((__m256i*)block, size)

Definition at line 3555 of file bmavx2.h.

Referenced by bm::bit_block_calc_change(), and bm::compute_s_block_descr().

◆ VECT_BLOCK_CHANGE_BC

#define VECT_BLOCK_CHANGE_BC	(	block,
		gc,
		bc )

Value:

avx2_bit_block_calc_change_bc((__m256i*)block, gc, bc)

Definition at line 3561 of file bmavx2.h.

Referenced by bm::bit_block_change_bc().

◆ VECT_BLOCK_SET_DIGEST

#define VECT_BLOCK_SET_DIGEST	(		dst,
			val )

Value:

avx2_block_set_digest((__m256i*)dst, val)

Definition at line 3534 of file bmavx2.h.

Referenced by bm::bit_block_init_and_2way(), and bm::block_init_digest0().

◆ VECT_BLOCK_XOR_CHANGE

#define VECT_BLOCK_XOR_CHANGE	(	block,
		xor_block,
		size,
		gc,
		bc )

Value:

avx2_bit_block_calc_xor_change((__m256i*)block, (__m256i*)xor_block, size, gc, bc)

Definition at line 3558 of file bmavx2.h.

Referenced by bm::bit_block_xor_change().

◆ VECT_COPY_BLOCK

#define VECT_COPY_BLOCK	(		dst,
			src )

Value:

avx2_copy_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3510 of file bmavx2.h.

Referenced by bm::bit_block_copy().

◆ VECT_COPY_BLOCK_UNALIGN

#define VECT_COPY_BLOCK_UNALIGN	(		dst,
			src )

Value:

avx2_copy_block_unalign((__m256i*) dst, (__m256i*) (src))

Definition at line 3513 of file bmavx2.h.

Referenced by bm::bit_block_copy_unalign().

◆ VECT_GAP_BFIND

#define VECT_GAP_BFIND	(	buf,
		pos,
		is_set )

Value:

avx2_gap_bfind(buf, pos, is_set)

Definition at line 3579 of file bmavx2.h.

Referenced by bm::gap_bfind().

◆ VECT_GAP_TEST

#define VECT_GAP_TEST	(		buf,
			pos )

Value:

avx2_gap_test(buf, pos)

Definition at line 3582 of file bmavx2.h.

Referenced by bm::gap_test_unr().

◆ VECT_INVERT_BLOCK

#define VECT_INVERT_BLOCK ( first )

Value:

avx2_invert_block((__m256i*)first);

Definition at line 3453 of file bmavx2.h.

Referenced by bm::bit_invert().

◆ VECT_IS_DIGEST_ZERO

#define VECT_IS_DIGEST_ZERO ( start )

Value:

avx2_is_digest_zero((__m256i*)start)

Definition at line 3531 of file bmavx2.h.

Referenced by bm::calc_block_digest0(), and bm::update_block_digest0().

◆ VECT_IS_ONE_BLOCK

#define VECT_IS_ONE_BLOCK ( dst )

Value:

avx2_is_all_one((__m256i*) dst)

Definition at line 3528 of file bmavx2.h.

Referenced by bm::is_bits_one().

◆ VECT_IS_ZERO_BLOCK

#define VECT_IS_ZERO_BLOCK ( dst )

Value:

avx2_is_all_zero((__m256i*) dst)

Definition at line 3525 of file bmavx2.h.

Referenced by bm::bit_is_all_zero().

◆ VECT_LOWER_BOUND_SCAN_U32

#define VECT_LOWER_BOUND_SCAN_U32	(	arr,
		target,
		from,
		to )

Value:

avx2_lower_bound_scan_u32(arr, target, from, to)

Definition at line 3537 of file bmavx2.h.

Referenced by bm::lower_bound_linear_u32().

◆ VECT_OR_BLOCK

#define VECT_OR_BLOCK	(		dst,
			src )

Value:

avx2_or_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3474 of file bmavx2.h.

Referenced by bm::bit_block_or().

◆ VECT_OR_BLOCK_2WAY

#define VECT_OR_BLOCK_2WAY	(	dst,
		src1,
		src2 )

Value:

avx2_or_block_2way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3480 of file bmavx2.h.

Referenced by bm::bit_block_or_2way().

◆ VECT_OR_BLOCK_3WAY [1/2]

#define VECT_OR_BLOCK_3WAY	(	dst,
		src1,
		src2 )

Value:

avx2_or_block_3way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3477 of file bmavx2.h.

◆ VECT_OR_BLOCK_3WAY [2/2]

#define VECT_OR_BLOCK_3WAY	(	dst,
		src1,
		src2 )

Value:

avx2_or_block_3way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3477 of file bmavx2.h.

Referenced by bm::bit_block_or_3way().

◆ VECT_OR_BLOCK_5WAY

#define VECT_OR_BLOCK_5WAY	(	dst,
		src1,
		src2,
		src3,
		src4 )

Value:

avx2_or_block_5way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2), (__m256i*) (src3), (__m256i*) (src4))

Definition at line 3486 of file bmavx2.h.

Referenced by bm::bit_block_or_5way().

◆ VECT_SET_BLOCK

#define VECT_SET_BLOCK	(		dst,
			value )

Value:

avx2_set_block((__m256i*) dst, (value))

Definition at line 3522 of file bmavx2.h.

Referenced by bm::bit_block_set().

◆ VECT_SET_BLOCK_BITS

#define VECT_SET_BLOCK_BITS	(	block,
		idx,
		start,
		stop )

Value:

avx2_set_block_bits3(block, idx, start, stop)

Definition at line 3552 of file bmavx2.h.

Referenced by bm::set_block_bits_u32().

◆ VECT_SHIFT_L1

#define VECT_SHIFT_L1	(	b,
		acc,
		co )

Value:

avx2_shift_l1((__m256i*)b, acc, co)

Definition at line 3540 of file bmavx2.h.

Referenced by bm::bit_block_shift_l1_unr().

◆ VECT_SHIFT_R1

#define VECT_SHIFT_R1	(	b,
		acc,
		co )

Value:

avx2_shift_r1((__m256i*)b, acc, co)

Definition at line 3543 of file bmavx2.h.

Referenced by bm::bit_block_shift_r1_unr().

◆ VECT_SHIFT_R1_AND

#define VECT_SHIFT_R1_AND	(	b,
		co,
		m,
		digest )

Value:

avx2_shift_r1_and((__m256i*)b, co, (__m256i*)m, digest)

Definition at line 3546 of file bmavx2.h.

Referenced by bm::bit_block_shift_r1_and_unr().

◆ VECT_STREAM_BLOCK

#define VECT_STREAM_BLOCK	(		dst,
			src )

Value:

avx2_stream_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3516 of file bmavx2.h.

Referenced by bm::bit_block_stream().

◆ VECT_STREAM_BLOCK_UNALIGN

#define VECT_STREAM_BLOCK_UNALIGN	(		dst,
			src )

Value:

avx2_stream_block_unalign((__m256i*) dst, (__m256i*) (src))

Definition at line 3519 of file bmavx2.h.

Referenced by bm::bit_block_stream_unalign().

◆ VECT_SUB_BLOCK

#define VECT_SUB_BLOCK	(		dst,
			src )

Value:

avx2_sub_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3489 of file bmavx2.h.

Referenced by bm::bit_block_sub().

◆ VECT_SUB_DIGEST

#define VECT_SUB_DIGEST	(		dst,
			src )

Value:

avx2_sub_digest((__m256i*) dst, (const __m256i*) (src))

Definition at line 3492 of file bmavx2.h.

Referenced by bm::bit_block_sub().

◆ VECT_SUB_DIGEST_2WAY

#define VECT_SUB_DIGEST_2WAY	(	dst,
		src1,
		src2 )

Value:

avx2_sub_digest_2way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3495 of file bmavx2.h.

Referenced by bm::bit_block_sub_2way().

◆ VECT_SUB_DIGEST_3WAY

#define VECT_SUB_DIGEST_3WAY	(	dst,
		src1,
		src2 )

Value:

avx2_sub_digest_3way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2))

Definition at line 3501 of file bmavx2.h.

Referenced by bm::bit_block_sub_3way().

◆ VECT_SUB_DIGEST_5WAY

#define VECT_SUB_DIGEST_5WAY	(	dst,
		src1,
		src2,
		src3,
		src4 )

Value:

avx2_sub_digest_5way((__m256i*) dst, (const __m256i*) (src1), (const __m256i*) (src2), (const __m256i*) (src3), (const __m256i*) (src4))

Definition at line 3498 of file bmavx2.h.

Referenced by bm::bit_block_sub_5way().

◆ VECT_XOR_ARR_2_MASK

#define VECT_XOR_ARR_2_MASK	(	dst,
		src,
		src_end,
		mask )

Value:

avx2_xor_arr_2_mask((__m256i*)(dst), (__m256i*)(src), (__m256i*)(src_end), (bm::word_t)mask)

Definition at line 3432 of file bmavx2.h.

Referenced by bm::bvector< Alloc >::combine_operation_with_block().

◆ VECT_XOR_BLOCK

#define VECT_XOR_BLOCK	(		dst,
			src )

Value:

avx2_xor_block((__m256i*) dst, (__m256i*) (src))

Definition at line 3504 of file bmavx2.h.

Referenced by bm::bit_block_xor().

◆ VECT_XOR_BLOCK_2WAY

#define VECT_XOR_BLOCK_2WAY	(	dst,
		src1,
		src2 )

Value:

avx2_xor_block_2way((__m256i*) dst, (__m256i*) (src1), (__m256i*) (src2))

Definition at line 3507 of file bmavx2.h.

Referenced by bm::bit_block_xor_2way().

Namespaces

Macros

Functions

Macro Definition Documentation

◆ BM_AVX2_BIT_COUNT

◆ BM_AVX2_DECL_LOOKUP1

◆ BM_AVX2_DECL_LOOKUP2

◆ BM_AVX2_POPCNT_PROLOG

◆ BM_CSA256

◆ VECT_AND_BLOCK

◆ VECT_AND_DIGEST

◆ VECT_AND_DIGEST_2WAY

◆ VECT_AND_DIGEST_3WAY

◆ VECT_AND_DIGEST_5WAY

◆ VECT_AND_OR_DIGEST_2WAY

◆ VECT_ANDNOT_ARR_2_MASK

◆ VECT_ARR_BLOCK_LOOKUP

◆ VECT_BIT_BLOCK_XOR

◆ VECT_BIT_BLOCK_XOR_2WAY

◆ VECT_BIT_COUNT_DIGEST

◆ VECT_BIT_FIND_DIFF

◆ VECT_BIT_FIND_FIRST

◆ VECT_BIT_TO_GAP

◆ VECT_BITCOUNT

◆ VECT_BITCOUNT_AND

◆ VECT_BITCOUNT_OR

◆ VECT_BITCOUNT_SUB

◆ VECT_BITCOUNT_XOR

◆ VECT_BLOCK_CHANGE

◆ VECT_BLOCK_CHANGE_BC

◆ VECT_BLOCK_SET_DIGEST

◆ VECT_BLOCK_XOR_CHANGE

◆ VECT_COPY_BLOCK

◆ VECT_COPY_BLOCK_UNALIGN

◆ VECT_GAP_BFIND

◆ VECT_GAP_TEST

◆ VECT_INVERT_BLOCK

◆ VECT_IS_DIGEST_ZERO

◆ VECT_IS_ONE_BLOCK

◆ VECT_IS_ZERO_BLOCK

◆ VECT_LOWER_BOUND_SCAN_U32

◆ VECT_OR_BLOCK

◆ VECT_OR_BLOCK_2WAY

◆ VECT_OR_BLOCK_3WAY [1/2]

◆ VECT_OR_BLOCK_3WAY [2/2]

◆ VECT_OR_BLOCK_5WAY

◆ VECT_SET_BLOCK

◆ VECT_SET_BLOCK_BITS

◆ VECT_SHIFT_L1

◆ VECT_SHIFT_R1

◆ VECT_SHIFT_R1_AND

◆ VECT_STREAM_BLOCK

◆ VECT_STREAM_BLOCK_UNALIGN

◆ VECT_SUB_BLOCK

◆ VECT_SUB_DIGEST

◆ VECT_SUB_DIGEST_2WAY

◆ VECT_SUB_DIGEST_3WAY

◆ VECT_SUB_DIGEST_5WAY

◆ VECT_XOR_ARR_2_MASK

◆ VECT_XOR_BLOCK

◆ VECT_XOR_BLOCK_2WAY