asm_amd64.s

// Code generated by command: go run avo.go -out ../../asm_amd64.s -stubs ../../stubs_amd64.go -pkg vectorcmp --dispatcher-amd64 ../../dispatch_amd64.go --dispatcher-other ../../dispatch_other.go --generate-test ../../generated_test.go. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

DATA constants<>+0(SB)/4, $0x55555555
DATA constants<>+4(SB)/4, $0x11111111
DATA constants<>+8(SB)/4, $0x01010101
GLOBL constants<>(SB), RODATA|NOPTR, $12

DATA const_zeroes<>+0(SB)/4, $0x00000000
DATA const_zeroes<>+4(SB)/4, $0x00000000
DATA const_zeroes<>+8(SB)/4, $0x00000000
DATA const_zeroes<>+12(SB)/4, $0x00000000
GLOBL const_zeroes<>(SB), RODATA|NOPTR, $16

DATA const_onezero<>+0(SB)/4, $0x01000100
DATA const_onezero<>+4(SB)/4, $0x01000100
DATA const_onezero<>+8(SB)/4, $0x01000100
DATA const_onezero<>+12(SB)/4, $0x01000100
GLOBL const_onezero<>(SB), RODATA|NOPTR, $16

DATA const_three_through_zero<>+0(SB)/4, $0x03020100
DATA const_three_through_zero<>+4(SB)/4, $0x03020100
DATA const_three_through_zero<>+8(SB)/4, $0x03020100
DATA const_three_through_zero<>+12(SB)/4, $0x03020100
GLOBL const_three_through_zero<>(SB), RODATA|NOPTR, $16

DATA const_seven_through_zero<>+0(SB)/8, $0x0706050403020100
DATA const_seven_through_zero<>+8(SB)/8, $0x0706050403020100
GLOBL const_seven_through_zero<>(SB), RODATA|NOPTR, $16

DATA const_drop_half<>+0(SB)/8, $0x0e0c0a0806040200
DATA const_drop_half<>+8(SB)/8, $0xffffffffffffffff
GLOBL const_drop_half<>(SB), RODATA|NOPTR, $16

DATA const_drop_threequarters<>+0(SB)/8, $0xffffffff0c080400
DATA const_drop_threequarters<>+8(SB)/8, $0xffffffffffffffff
GLOBL const_drop_threequarters<>(SB), RODATA|NOPTR, $16

DATA const_drop_seveneight<>+0(SB)/8, $0xffffffffffff0800
DATA const_drop_seveneight<>+8(SB)/8, $0xffffffffffffffff
GLOBL const_drop_seveneight<>(SB), RODATA|NOPTR, $16

DATA const_high_bits_for_size_8<>+0(SB)/8, $0x8080808080808080
DATA const_high_bits_for_size_8<>+8(SB)/8, $0x8080808080808080
DATA const_high_bits_for_size_8<>+16(SB)/8, $0x8080808080808080
DATA const_high_bits_for_size_8<>+24(SB)/8, $0x8080808080808080
GLOBL const_high_bits_for_size_8<>(SB), RODATA|NOPTR, $32

DATA const_high_bits_for_size_16<>+0(SB)/8, $0x8000800080008000
DATA const_high_bits_for_size_16<>+8(SB)/8, $0x8000800080008000
DATA const_high_bits_for_size_16<>+16(SB)/8, $0x8000800080008000
DATA const_high_bits_for_size_16<>+24(SB)/8, $0x8000800080008000
GLOBL const_high_bits_for_size_16<>(SB), RODATA|NOPTR, $32

DATA const_high_bits_for_size_32<>+0(SB)/8, $0x8000000080000000
DATA const_high_bits_for_size_32<>+8(SB)/8, $0x8000000080000000
DATA const_high_bits_for_size_32<>+16(SB)/8, $0x8000000080000000
DATA const_high_bits_for_size_32<>+24(SB)/8, $0x8000000080000000
GLOBL const_high_bits_for_size_32<>(SB), RODATA|NOPTR, $32

DATA const_high_bits_for_size_64<>+0(SB)/8, $0x8000000000000000
DATA const_high_bits_for_size_64<>+8(SB)/8, $0x8000000000000000
DATA const_high_bits_for_size_64<>+16(SB)/8, $0x8000000000000000
DATA const_high_bits_for_size_64<>+24(SB)/8, $0x8000000000000000
GLOBL const_high_bits_for_size_64<>(SB), RODATA|NOPTR, $32

// func asmAVX2EqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2EqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB Y1, Y0, Y1
	VPCMPEQB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB X1, X0, X1
	VPCMPEQB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2NotEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2NotEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB Y1, Y0, Y1
	VPCMPEQB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get NotEquals semantics we need to invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXNotEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXNotEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB X1, X0, X1
	VPCMPEQB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get NotEquals semantics we need to invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2GreaterThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y1, Y0, Y1
	VPCMPGTB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), X3
	VPXOR   X3, X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X1, X0, X1
	VPCMPGTB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2LessThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y0, Y1, Y1
	VPCMPGTB Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), X3
	VPXOR   X3, X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X0, X1, X1
	VPCMPGTB X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2GreaterEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y0, Y1, Y1
	VPCMPGTB Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), X3
	VPXOR   X3, X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X0, X1, X1
	VPCMPGTB X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2LesserEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y1, Y0, Y1
	VPCMPGTB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_8<>+0(SB), X3
	VPXOR   X3, X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X1, X0, X1
	VPCMPGTB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW Y1, Y0, Y1
	VPCMPEQW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Drop every second byte from these registers
	VMOVDQU const_drop_half<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW X1, X0, X1
	VPCMPEQW X2, X0, X2

	// Drop every second byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2NotEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2NotEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW Y1, Y0, Y1
	VPCMPEQW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get NotEquals semantics we need to invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXNotEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, SSE2
TEXT ·asmAVXNotEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Drop every second byte from these registers
	VMOVDQU const_drop_half<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW X1, X0, X1
	VPCMPEQW X2, X0, X2

	// Drop every second byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get NotEquals semantics we need to invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y1, Y0, Y1
	VPCMPGTW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second byte from these registers
	VMOVDQU const_drop_half<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X1, X0, X1
	VPCMPGTW X2, X0, X2

	// Drop every second byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y0, Y1, Y1
	VPCMPGTW Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second byte from these registers
	VMOVDQU const_drop_half<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X0, X1, X1
	VPCMPGTW X0, X2, X2

	// Drop every second byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y0, Y1, Y1
	VPCMPGTW Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second byte from these registers
	VMOVDQU const_drop_half<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X0, X1, X1
	VPCMPGTW X0, X2, X2

	// Drop every second byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y1, Y0, Y1
	VPCMPGTW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_16<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second byte from these registers
	VMOVDQU const_drop_half<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X1, X0, X1
	VPCMPGTW X2, X0, X2

	// Drop every second byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD Y1, Y0, Y1
	VPCMPEQD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD X1, X0, X1
	VPCMPEQD X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x00, Y1, Y0, Y1
	VCMPPS $0x00, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x00, X1, X0, X1
	VCMPPS $0x00, X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2NotEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2NotEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD Y1, Y0, Y1
	VPCMPEQD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get NotEquals semantics we need to invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXNotEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, SSE2
TEXT ·asmAVXNotEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD X1, X0, X1
	VPCMPEQD X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get NotEquals semantics we need to invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2NotEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2NotEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x04, Y1, Y0, Y1
	VCMPPS $0x04, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXNotEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, SSE2
TEXT ·asmAVXNotEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x04, X1, X0, X1
	VCMPPS $0x04, X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y1, Y0, Y1
	VPCMPGTD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X1, X0, X1
	VPCMPGTD X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1e, Y1, Y0, Y1
	VCMPPS $0x1e, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1e, X1, X0, X1
	VCMPPS $0x1e, X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y0, Y1, Y1
	VPCMPGTD Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X0, X1, X1
	VPCMPGTD X0, X2, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x11, Y1, Y0, Y1
	VCMPPS $0x11, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x11, X1, X0, X1
	VCMPPS $0x11, X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y0, Y1, Y1
	VPCMPGTD Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X0, X1, X1
	VPCMPGTD X0, X2, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1d, Y1, Y0, Y1
	VCMPPS $0x1d, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1d, X1, X0, X1
	VCMPPS $0x1d, X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), Y3
	VPXOR   Y3, Y0, Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Also flip the high-bit of the data
	VPXOR Y3, Y1, Y1
	VPXOR Y3, Y2, Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y1, Y0, Y1
	VPCMPGTD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_32<>+0(SB), X3
	VPXOR   X3, X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X4

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Also flip the high-bit of the data
	VPXOR X3, X1, X1
	VPXOR X3, X2, X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X1, X0, X1
	VPCMPGTD X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x12, Y1, Y0, Y1
	VCMPPS $0x12, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X3

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x12, X1, X0, X1
	VCMPPS $0x12, X2, X0, X2

	// Drop every second-fourth byte from these registers
	VPSHUFB X3, X1, X1
	VPSHUFB X3, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ Y1, Y0, Y1
	VPCMPEQQ Y2, Y0, Y2
	VPCMPEQQ Y3, Y0, Y3
	VPCMPEQQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ X1, X0, X1
	VPCMPEQQ X2, X0, X2
	VPCMPEQQ X3, X0, X3
	VPCMPEQQ X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x00, Y1, Y0, Y1
	VCMPPD $0x00, Y2, Y0, Y2
	VCMPPD $0x00, Y3, Y0, Y3
	VCMPPD $0x00, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x00, X1, X0, X1
	VCMPPD $0x00, X2, X0, X2
	VCMPPD $0x00, X3, X0, X3
	VCMPPD $0x00, X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2NotEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2NotEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ Y1, Y0, Y1
	VPCMPEQQ Y2, Y0, Y2
	VPCMPEQQ Y3, Y0, Y3
	VPCMPEQQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get NotEquals semantics we need to invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	NOTB DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXNotEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, SSE2
TEXT ·asmAVXNotEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ X1, X0, X1
	VPCMPEQQ X2, X0, X2
	VPCMPEQQ X3, X0, X3
	VPCMPEQQ X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// To get NotEquals semantics we need to invert the result
	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2NotEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2NotEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x04, Y1, Y0, Y1
	VCMPPD $0x04, Y2, Y0, Y2
	VCMPPD $0x04, Y3, Y0, Y3
	VCMPPD $0x04, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXNotEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, SSE2
TEXT ·asmAVXNotEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x04, X1, X0, X1
	VCMPPD $0x04, X2, X0, X2
	VCMPPD $0x04, X3, X0, X3
	VCMPPD $0x04, X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), Y5
	VPXOR   Y5, Y0, Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Also flip the high-bit of the data
	VPXOR Y5, Y1, Y1
	VPXOR Y5, Y2, Y2
	VPXOR Y5, Y3, Y3
	VPXOR Y5, Y4, Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y1, Y0, Y1
	VPCMPGTQ Y2, Y0, Y2
	VPCMPGTQ Y3, Y0, Y3
	VPCMPGTQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), X5
	VPXOR   X5, X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X6

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Also flip the high-bit of the data
	VPXOR X5, X1, X1
	VPXOR X5, X2, X2
	VPXOR X5, X3, X3
	VPXOR X5, X4, X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X1, X0, X1
	VPCMPGTQ X2, X0, X2
	VPCMPGTQ X3, X0, X3
	VPCMPGTQ X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X6, X1, X1
	VPSHUFB X6, X2, X2
	VPSHUFB X6, X3, X3
	VPSHUFB X6, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1e, Y1, Y0, Y1
	VCMPPD $0x1e, Y2, Y0, Y2
	VCMPPD $0x1e, Y3, Y0, Y3
	VCMPPD $0x1e, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1e, X1, X0, X1
	VCMPPD $0x1e, X2, X0, X2
	VCMPPD $0x1e, X3, X0, X3
	VCMPPD $0x1e, X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), Y5
	VPXOR   Y5, Y0, Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Also flip the high-bit of the data
	VPXOR Y5, Y1, Y1
	VPXOR Y5, Y2, Y2
	VPXOR Y5, Y3, Y3
	VPXOR Y5, Y4, Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y0, Y1, Y1
	VPCMPGTQ Y0, Y2, Y2
	VPCMPGTQ Y0, Y3, Y3
	VPCMPGTQ Y0, Y4, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), X5
	VPXOR   X5, X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X6

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Also flip the high-bit of the data
	VPXOR X5, X1, X1
	VPXOR X5, X2, X2
	VPXOR X5, X3, X3
	VPXOR X5, X4, X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X0, X1, X1
	VPCMPGTQ X0, X2, X2
	VPCMPGTQ X0, X3, X3
	VPCMPGTQ X0, X4, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X6, X1, X1
	VPSHUFB X6, X2, X2
	VPSHUFB X6, X3, X3
	VPSHUFB X6, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x11, Y1, Y0, Y1
	VCMPPD $0x11, Y2, Y0, Y2
	VCMPPD $0x11, Y3, Y0, Y3
	VCMPPD $0x11, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x11, X1, X0, X1
	VCMPPD $0x11, X2, X0, X2
	VCMPPD $0x11, X3, X0, X3
	VCMPPD $0x11, X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), Y5
	VPXOR   Y5, Y0, Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Also flip the high-bit of the data
	VPXOR Y5, Y1, Y1
	VPXOR Y5, Y2, Y2
	VPXOR Y5, Y3, Y3
	VPXOR Y5, Y4, Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y0, Y1, Y1
	VPCMPGTQ Y0, Y2, Y2
	VPCMPGTQ Y0, Y3, Y3
	VPCMPGTQ Y0, Y4, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	NOTB DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), X5
	VPXOR   X5, X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X6

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Also flip the high-bit of the data
	VPXOR X5, X1, X1
	VPXOR X5, X2, X2
	VPXOR X5, X3, X3
	VPXOR X5, X4, X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X0, X1, X1
	VPCMPGTQ X0, X2, X2
	VPCMPGTQ X0, X3, X3
	VPCMPGTQ X0, X4, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X6, X1, X1
	VPSHUFB X6, X2, X2
	VPSHUFB X6, X3, X3
	VPSHUFB X6, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1d, Y1, Y0, Y1
	VCMPPD $0x1d, Y2, Y0, Y2
	VCMPPD $0x1d, Y3, Y0, Y3
	VCMPPD $0x1d, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1d, X1, X0, X1
	VCMPPD $0x1d, X2, X0, X2
	VCMPPD $0x1d, X3, X0, X3
	VCMPPD $0x1d, X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), Y5
	VPXOR   Y5, Y0, Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Also flip the high-bit of the data
	VPXOR Y5, Y1, Y1
	VPXOR Y5, Y2, Y2
	VPXOR Y5, Y3, Y3
	VPXOR Y5, Y4, Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y1, Y0, Y1
	VPCMPGTQ Y2, Y0, Y2
	VPCMPGTQ Y3, Y0, Y3
	VPCMPGTQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	NOTB DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Flip the high-bit so we can make unsigned comparisons with the signed VPCMP instructions
	VMOVDQU const_high_bits_for_size_64<>+0(SB), X5
	VPXOR   X5, X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X6

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Also flip the high-bit of the data
	VPXOR X5, X1, X1
	VPXOR X5, X2, X2
	VPXOR X5, X3, X3
	VPXOR X5, X4, X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X1, X0, X1
	VPCMPGTQ X2, X0, X2
	VPCMPGTQ X3, X0, X3
	VPCMPGTQ X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X6, X1, X1
	VPSHUFB X6, X2, X2
	VPSHUFB X6, X3, X3
	VPSHUFB X6, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x12, Y1, Y0, Y1
	VCMPPD $0x12, Y2, Y0, Y2
	VCMPPD $0x12, Y3, Y0, Y3
	VCMPPD $0x12, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X5

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x12, X1, X0, X1
	VCMPPD $0x12, X2, X0, X2
	VCMPPD $0x12, X3, X0, X3
	VCMPPD $0x12, X4, X0, X4

	// Drop every second-seventh byte from these registers
	VPSHUFB X5, X1, X1
	VPSHUFB X5, X2, X2
	VPSHUFB X5, X3, X3
	VPSHUFB X5, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2IsNaNFloat32(dstMask []byte, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2IsNaNFloat32(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x04, Y0, Y0, Y0
	VCMPPS $0x04, Y1, Y1, Y1

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y0, BX
	VPMOVMSKB Y1, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXIsNaNFloat32(dstMask []byte, rows []float32)
// Requires: AVX
TEXT ·asmAVXIsNaNFloat32(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Drop every second-fourth byte from these registers
	VMOVDQU const_drop_threequarters<>+0(SB), X2

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X0
	VMOVDQU 16(CX), X1

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x04, X0, X0, X0
	VCMPPS $0x04, X1, X1, X1

	// Drop every second-fourth byte from these registers
	VPSHUFB X2, X0, X0
	VPSHUFB X2, X1, X1

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X0, BX
	VPMOVMSKB X1, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2IsNaNFloat64(dstMask []byte, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2IsNaNFloat64(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1
	VMOVDQU 64(CX), Y2
	VMOVDQU 96(CX), Y3

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x04, Y0, Y0, Y0
	VCMPPD $0x04, Y1, Y1, Y1
	VCMPPD $0x04, Y2, Y2, Y2
	VCMPPD $0x04, Y3, Y3, Y3

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y0, BX
	VPMOVMSKB Y1, SI
	VPMOVMSKB Y2, DI
	VPMOVMSKB Y3, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXIsNaNFloat64(dstMask []byte, rows []float64)
// Requires: AVX
TEXT ·asmAVXIsNaNFloat64(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Drop every second-seventh byte from these registers
	VMOVDQU const_drop_seveneight<>+0(SB), X4

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X0
	VMOVDQU 16(CX), X1
	VMOVDQU 32(CX), X2
	VMOVDQU 48(CX), X3

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x04, X0, X0, X0
	VCMPPD $0x04, X1, X1, X1
	VCMPPD $0x04, X2, X2, X2
	VCMPPD $0x04, X3, X3, X3

	// Drop every second-seventh byte from these registers
	VPSHUFB X4, X0, X0
	VPSHUFB X4, X1, X1
	VPSHUFB X4, X2, X2
	VPSHUFB X4, X3, X3

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X0, BX
	VPMOVMSKB X1, SI
	VPMOVMSKB X2, DI
	VPMOVMSKB X3, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET