// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "textflag.h"

// castagnoliSSE42 updates the (non-inverted) crc with the given buffer.
//
// func castagnoliSSE42(crc uint32, p []byte) uint32
TEXT ·castagnoliSSE42(SB), NOSPLIT, $0
	MOVL crc+0(FP), AX    // CRC value
	MOVQ p+8(FP), SI      // data pointer
	MOVQ p_len+16(FP), CX // len(p)

	// If there are fewer than 8 bytes to process, skip alignment.
	CMPQ CX, $8
	JL   less_than_8

	MOVQ SI, BX
	ANDQ $7, BX
	JZ   aligned

	// Process the first few bytes to 8-byte align the input.

	// BX = 8 - BX. We need to process this many bytes to align.
	SUBQ $1, BX
	XORQ $7, BX

	BTQ $0, BX
	JNC align_2

	CRC32B (SI), AX
	DECQ   CX
	INCQ   SI

align_2:
	BTQ $1, BX
	JNC align_4

	CRC32W (SI), AX

	SUBQ $2, CX
	ADDQ $2, SI

align_4:
	BTQ $2, BX
	JNC aligned

	CRC32L (SI), AX

	SUBQ $4, CX
	ADDQ $4, SI

aligned:
	// The input is now 8-byte aligned and we can process 8-byte chunks.
	CMPQ CX, $8
	JL   less_than_8

	CRC32Q (SI), AX
	ADDQ   $8, SI
	SUBQ   $8, CX
	JMP    aligned

less_than_8:
	// We may have some bytes left over; process 4 bytes, then 2, then 1.
	BTQ $2, CX
	JNC less_than_4

	CRC32L (SI), AX
	ADDQ   $4, SI

less_than_4:
	BTQ $1, CX
	JNC less_than_2

	CRC32W (SI), AX
	ADDQ   $2, SI

less_than_2:
	BTQ $0, CX
	JNC done

	CRC32B (SI), AX

done:
	MOVL AX, ret+32(FP)
	RET

// castagnoliSSE42Triple updates three (non-inverted) crcs with (24*rounds)
// bytes from each buffer.
//
// func castagnoliSSE42Triple(
//     crc1, crc2, crc3 uint32,
//     a, b, c []byte,
//     rounds uint32,
// ) (retA uint32, retB uint32, retC uint32)
TEXT ·castagnoliSSE42Triple(SB), NOSPLIT, $0
	MOVL crcA+0(FP), AX
	MOVL crcB+4(FP), CX
	MOVL crcC+8(FP), DX

	MOVQ a+16(FP), R8  // data pointer
	MOVQ b+40(FP), R9  // data pointer
	MOVQ c+64(FP), R10 // data pointer

	MOVL rounds+88(FP), R11

loop:
	CRC32Q (R8), AX
	CRC32Q (R9), CX
	CRC32Q (R10), DX

	CRC32Q 8(R8), AX
	CRC32Q 8(R9), CX
	CRC32Q 8(R10), DX

	CRC32Q 16(R8), AX
	CRC32Q 16(R9), CX
	CRC32Q 16(R10), DX

	ADDQ $24, R8
	ADDQ $24, R9
	ADDQ $24, R10

	DECQ R11
	JNZ  loop

	MOVL AX, retA+96(FP)
	MOVL CX, retB+100(FP)
	MOVL DX, retC+104(FP)
	RET

// CRC32 polynomial data
//
// These constants are lifted from the
// Linux kernel, since they avoid the costly
// PSHUFB 16 byte reversal proposed in the
// original Intel paper.
DATA r2r1<>+0(SB)/8, $0x154442bd4
DATA r2r1<>+8(SB)/8, $0x1c6e41596
DATA r4r3<>+0(SB)/8, $0x1751997d0
DATA r4r3<>+8(SB)/8, $0x0ccaa009e
DATA rupoly<>+0(SB)/8, $0x1db710641
DATA rupoly<>+8(SB)/8, $0x1f7011641
DATA r5<>+0(SB)/8, $0x163cd6124

GLOBL r2r1<>(SB), RODATA, $16
GLOBL r4r3<>(SB), RODATA, $16
GLOBL rupoly<>(SB), RODATA, $16
GLOBL r5<>(SB), RODATA, $8

// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 64, and must be a multiple of 16.

// func ieeeCLMUL(crc uint32, p []byte) uint32
TEXT ·ieeeCLMUL(SB), NOSPLIT, $0
	MOVL crc+0(FP), X0    // Initial CRC value
	MOVQ p+8(FP), SI      // data pointer
	MOVQ p_len+16(FP), CX // len(p)

	MOVOU (SI), X1
	MOVOU 16(SI), X2
	MOVOU 32(SI), X3
	MOVOU 48(SI), X4
	PXOR  X0, X1
	ADDQ  $64, SI    // buf+=64
	SUBQ  $64, CX    // len-=64
	CMPQ  CX, $64    // Less than 64 bytes left
	JB    remain64

	MOVOA r2r1<>+0(SB), X0

loopback64:
	MOVOA X1, X5
	MOVOA X2, X6
	MOVOA X3, X7
	MOVOA X4, X8

	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0, X0, X2
	PCLMULQDQ $0, X0, X3
	PCLMULQDQ $0, X0, X4

	// Load next early
	MOVOU (SI), X11
	MOVOU 16(SI), X12
	MOVOU 32(SI), X13
	MOVOU 48(SI), X14

	PCLMULQDQ $0x11, X0, X5
	PCLMULQDQ $0x11, X0, X6
	PCLMULQDQ $0x11, X0, X7
	PCLMULQDQ $0x11, X0, X8

	PXOR X5, X1
	PXOR X6, X2
	PXOR X7, X3
	PXOR X8, X4

	PXOR X11, X1
	PXOR X12, X2
	PXOR X13, X3
	PXOR X14, X4

	ADDQ $0x40, DI
	ADDQ $64, SI    // buf+=64
	SUBQ $64, CX    // len-=64
	CMPQ CX, $64    // Less than 64 bytes left?
	JGE  loopback64

	// Fold result into a single register (X1)
remain64:
	MOVOA r4r3<>+0(SB), X0

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X2, X1

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X3, X1

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X4, X1

	// If there is less than 16 bytes left we are done
	CMPQ CX, $16
	JB   finish

	// Encode 16 bytes
remain16:
	MOVOU     (SI), X10
	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X10, X1
	SUBQ      $16, CX
	ADDQ      $16, SI
	CMPQ      CX, $16
	JGE       remain16

finish:
	// Fold final result into 32 bits and return it
	PCMPEQB   X3, X3
	PCLMULQDQ $1, X1, X0
	PSRLDQ    $8, X1
	PXOR      X0, X1

	MOVOA X1, X2
	MOVQ  r5<>+0(SB), X0

	// Creates 32 bit mask. Note that we don't care about upper half.
	PSRLQ $32, X3

	PSRLDQ    $4, X2
	PAND      X3, X1
	PCLMULQDQ $0, X0, X1
	PXOR      X2, X1

	MOVOA rupoly<>+0(SB), X0

	MOVOA     X1, X2
	PAND      X3, X1
	PCLMULQDQ $0x10, X0, X1
	PAND      X3, X1
	PCLMULQDQ $0, X0, X1
	PXOR      X2, X1

	PEXTRD $1, X1, AX
	MOVL   AX, ret+32(FP)

	RET

DATA r2r1X<>+0(SB)/8, $0x154442bd4
DATA r2r1X<>+8(SB)/8, $0x1c6e41596
DATA r2r1X<>+16(SB)/8, $0x154442bd4
DATA r2r1X<>+24(SB)/8, $0x1c6e41596
DATA r2r1X<>+32(SB)/8, $0x154442bd4
DATA r2r1X<>+40(SB)/8, $0x1c6e41596
DATA r2r1X<>+48(SB)/8, $0x154442bd4
DATA r2r1X<>+56(SB)/8, $0x1c6e41596
GLOBL r2r1X<>(SB), RODATA, $64

// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 128, and must be a multiple of 16.

// func ieeeCLMULAvx512(crc uint32, p []byte) uint32
TEXT ·ieeeCLMULAvx512(SB), NOSPLIT, $0
	MOVL crc+0(FP), AX    // Initial CRC value
	MOVQ p+8(FP), SI      // data pointer
	MOVQ p_len+16(FP), CX // len(p)

	VPXORQ    Z0, Z0, Z0
	VMOVDQU64 (SI), Z1
	VMOVQ     AX, X0
	VPXORQ    Z0, Z1, Z1 // Merge initial CRC value into Z1
	ADDQ      $64, SI    // buf+=64
	SUBQ      $64, CX    // len-=64

	VMOVDQU64 r2r1X<>+0(SB), Z0

loopback64:
	// Load next early
	VMOVDQU64 (SI), Z11

	VPCLMULQDQ $0x11, Z0, Z1, Z5
	VPCLMULQDQ $0, Z0, Z1, Z1

	VPTERNLOGD $0x96, Z11, Z5, Z1 // Combine results with xor into Z1

	ADDQ $0x40, DI
	ADDQ $64, SI    // buf+=64
	SUBQ $64, CX    // len-=64
	CMPQ CX, $64    // Less than 64 bytes left?
	JGE  loopback64

	// Fold result into a single register (X1)
remain64:
	VEXTRACTF32X4 $1, Z1, X2 // X2: Second 128-bit lane
	VEXTRACTF32X4 $2, Z1, X3 // X3: Third 128-bit lane
	VEXTRACTF32X4 $3, Z1, X4 // X4: Fourth 128-bit lane

	MOVOA r4r3<>+0(SB), X0

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X2, X1

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X3, X1

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X4, X1

	// If there is less than 16 bytes left we are done
	CMPQ CX, $16
	JB   finish

	// Encode 16 bytes
remain16:
	MOVOU     (SI), X10
	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X10, X1
	SUBQ      $16, CX
	ADDQ      $16, SI
	CMPQ      CX, $16
	JGE       remain16

finish:
	// Fold final result into 32 bits and return it
	PCMPEQB   X3, X3
	PCLMULQDQ $1, X1, X0
	PSRLDQ    $8, X1
	PXOR      X0, X1

	MOVOA X1, X2
	MOVQ  r5<>+0(SB), X0

	// Creates 32 bit mask. Note that we don't care about upper half.
	PSRLQ $32, X3

	PSRLDQ    $4, X2
	PAND      X3, X1
	PCLMULQDQ $0, X0, X1
	PXOR      X2, X1

	MOVOA rupoly<>+0(SB), X0

	MOVOA     X1, X2
	PAND      X3, X1
	PCLMULQDQ $0x10, X0, X1
	PAND      X3, X1
	PCLMULQDQ $0, X0, X1
	PXOR      X2, X1

	PEXTRD $1, X1, AX
	MOVL   AX, ret+32(FP)
	VZEROUPPER
	RET

// Castagonli Polynomial constants
DATA r2r1C<>+0(SB)/8, $0x0740eef02
DATA r2r1C<>+8(SB)/8, $0x09e4addf8
DATA r2r1C<>+16(SB)/8, $0x0740eef02
DATA r2r1C<>+24(SB)/8, $0x09e4addf8
DATA r2r1C<>+32(SB)/8, $0x0740eef02
DATA r2r1C<>+40(SB)/8, $0x09e4addf8
DATA r2r1C<>+48(SB)/8, $0x0740eef02
DATA r2r1C<>+56(SB)/8, $0x09e4addf8
GLOBL r2r1C<>(SB), RODATA, $64

DATA r4r3C<>+0(SB)/8, $0xf20c0dfe
DATA r4r3C<>+8(SB)/8, $0x14cd00bd6
DATA rupolyC<>+0(SB)/8, $0x105ec76f0
DATA rupolyC<>+8(SB)/8, $0xdea713f1
DATA r5C<>+0(SB)/8, $0xdd45aab8

GLOBL r4r3C<>(SB), RODATA, $16
GLOBL rupolyC<>(SB), RODATA, $16
GLOBL r5C<>(SB), RODATA, $8

// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 128, and must be a multiple of 16.

// func castagnoliCLMULAvx512(crc uint32, p []byte) uint32
TEXT ·castagnoliCLMULAvx512(SB), NOSPLIT, $0
	MOVL crc+0(FP), AX    // Initial CRC value
	MOVQ p+8(FP), SI      // data pointer
	MOVQ p_len+16(FP), CX // len(p)

	VPXORQ    Z0, Z0, Z0
	VMOVDQU64 (SI), Z1
	VMOVQ     AX, X0
	VPXORQ    Z0, Z1, Z1 // Merge initial CRC value into Z1
	ADDQ      $64, SI    // buf+=64
	SUBQ      $64, CX    // len-=64

	VMOVDQU64 r2r1C<>+0(SB), Z0

loopback64:
	// Load next early
	VMOVDQU64 (SI), Z11

	VPCLMULQDQ $0x11, Z0, Z1, Z5
	VPCLMULQDQ $0, Z0, Z1, Z1

	VPTERNLOGD $0x96, Z11, Z5, Z1 // Combine results with xor into Z1

	ADDQ $0x40, DI
	ADDQ $64, SI    // buf+=64
	SUBQ $64, CX    // len-=64
	CMPQ CX, $64    // Less than 64 bytes left?
	JGE  loopback64

	// Fold result into a single register (X1)
remain64:
	VEXTRACTF32X4 $1, Z1, X2 // X2: Second 128-bit lane
	VEXTRACTF32X4 $2, Z1, X3 // X3: Third 128-bit lane
	VEXTRACTF32X4 $3, Z1, X4 // X4: Fourth 128-bit lane

	MOVOA r4r3C<>+0(SB), X0

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X2, X1

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X3, X1

	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X4, X1

	// If there is less than 16 bytes left we are done
	CMPQ CX, $16
	JB   finish

	// Encode 16 bytes
remain16:
	MOVOU     (SI), X10
	MOVOA     X1, X5
	PCLMULQDQ $0, X0, X1
	PCLMULQDQ $0x11, X0, X5
	PXOR      X5, X1
	PXOR      X10, X1
	SUBQ      $16, CX
	ADDQ      $16, SI
	CMPQ      CX, $16
	JGE       remain16

finish:
	// Fold final result into 32 bits and return it
	PCMPEQB   X3, X3
	PCLMULQDQ $1, X1, X0
	PSRLDQ    $8, X1
	PXOR      X0, X1

	MOVOA X1, X2
	MOVQ  r5C<>+0(SB), X0

	// Creates 32 bit mask. Note that we don't care about upper half.
	PSRLQ $32, X3

	PSRLDQ    $4, X2
	PAND      X3, X1
	PCLMULQDQ $0, X0, X1
	PXOR      X2, X1

	MOVOA rupolyC<>+0(SB), X0

	MOVOA     X1, X2
	PAND      X3, X1
	PCLMULQDQ $0x10, X0, X1
	PAND      X3, X1
	PCLMULQDQ $0, X0, X1
	PXOR      X2, X1

	PEXTRD $1, X1, AX
	MOVL   AX, ret+32(FP)
	VZEROUPPER
	RET
