/* dct.S */

#define D(n)	INCFIRST*(n)(DREG)
#define D2(n)	INCFIRST/2*(n)(DREG)
#ifdef EIGHT_OUTPUT
#define OUT_OFFSET	96
#else
#define OUT_OFFSET	0
#endif

	movb	$8,count

	.align	2
Y(loop)

#ifdef ZEROCHECK1
	movw	D(1),%ax
	orw	D(2),%ax
	orw	D(3),%ax
	orw	D(4),%ax
	orw	D(5),%ax
	orw	D(6),%ax
	orw	D(7),%ax
	jz	Z(mostlyzero)
#endif

	xorl	%ecx,%ecx
	cmpw	D(7),%cx
	setne	%al
	cmpw	D(5),%cx
	setne	%bl
	cmpw	D(3),%cx
	leal	(%ebx,%eax,2),%eax
	setne	%bl
	cmpw	D(1),%cx
	leal	(%ebx,%eax,2),%eax
	setne	%bl
	cmpw	D(6),%cx
	leal	(%ebx,%eax,2),%eax
	setne	%bl
	cmpw	D(4),%cx
	leal	(%ebx,%eax,2),%eax
	setne	%bl
	cmpw	D(2),%cx
	leal	(%ebx,%eax,2),%eax
	setne	%bl
	cmpw	D(0),%cx
	leal	(%ebx,%eax,2),%eax
	setne	%bl
	leal	(%ebx,%eax,2),%eax

#ifndef ZEROCHECK1
	testb	$0xfe,%al
	jz	Z(mostlyzero)
#endif

	movb	%al,mask
	andl	$0x0f,%eax
	movl	Z(jumptable1)(,%eax,4),%eax
	jmp	*%eax

	.align	2

Y(mostlyzero)
#ifdef ZEROCHECK1
	orw	D(0),%ax
#else
	movw	D(0),%ax
	orw	%ax,%ax
#endif
#ifndef EIGHT_OUTPUT
	jz	Z(continue)
#else
	jnz	Z(not_zero)
	movl	$ OUT_OFFSET<<24|OUT_OFFSET<<16|OUT_OFFSET<<8|OUT_OFFSET,(DREG)
	movl	$ OUT_OFFSET<<24|OUT_OFFSET<<16|OUT_OFFSET<<8|OUT_OFFSET,4(DREG)
	jmp	Z(continue)
	.align	2
Y(not_zero)
#endif
#if SHIFT < 0
	salw	$-SHIFT,%ax
#else
#if SHIFT > 0
#if OUT_OFFSET != 0
	addw	$(1+OUT_OFFSET*2)<<(SHIFT-1),%ax
#endif
	sarw	$ SHIFT,%ax
#endif
#endif

#ifdef EIGHT_OUTPUT
	movb	%al,%ah
	movw	%ax,(DREG)
	movw	%ax,2(DREG)
	movw	%ax,4(DREG)
	movw	%ax,6(DREG)
#else
	movw	%ax,D(0)
	movw	%ax,D(1)
	movw	%ax,D(2)
	movw	%ax,D(3)
	movw	%ax,D(4)
	movw	%ax,D(5)
	movw	%ax,D(6)
	movw	%ax,D(7)
#endif
	jmp	Z(continue)

	.align	2

Y(jumptable1)
	.long	Z(l_0),Z(l_1),Z(l_2),Z(l_3)
	.long	Z(l_4),Z(l_5),Z(l_6),Z(l_7)
	.long	Z(l_8),Z(l_9),Z(l_a),Z(l_b)
	.long	Z(l_c),Z(l_d),Z(l_e),Z(l_f)

Y(jumptable2)
	.long	Z(m_0),Z(m_1),Z(m_2),Z(m_3)
	.long	Z(m_4),Z(m_5),Z(m_6),Z(m_7)
	.long	Z(m_8),Z(m_9),Z(m_a),Z(m_b)
	.long	Z(m_c),Z(m_d),Z(m_e),Z(m_f)

	.align	2

Y(l_f)
	movswl	D(2),J
	movswl	D(6),L
	movswl	D(4),K
	movswl	D(0),I
	leal	(J,L),M
	imull	$ K0_7654,J
	imull	$ K0_5412,M
	imull	$-K1_8478,L
	leal	(I,K),N
	addl	M,J
	sall	$ CONST_BITS,N
	addl	L,M
	leal	(J,N),L
	subl	K,I
	movl	L,tmp10
	subl	J,N
	sall	$ CONST_BITS,I
	movl	N,tmp13
	leal	(I,M),J
	subl	M,I
	movl	J,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_e)
	movswl	D(4),J
	movswl	D(2),I
	movswl	D(6),K
	sall	$ CONST_BITS,J
	leal	(I,K),L
	imul	$ K0_7654,I
	imul	$ K0_5412,L
	movl	J,N
	addl	L,I
	imul	$-K1_8478,K
	subl	I,N
	leal	(J,I),M
	movl	N,tmp13
	negl	J
	addl	K,L
	movl	M,tmp10
	leal	(L,J),I
	subl	L,J
	movl	I,tmp11
	movl	J,tmp12
	jmp	Z(odd)

	.align	2

Y(l_d)
	movswl	D(0),I
	movswl	D(4),J
	movswl	D(6),K
	leal	(I,J),M
	imul	$ K0_5412,K,L
	sall	$ CONST_BITS,M
	imul	$-K1_3066,K
	leal	(M,L),N
	subl	J,I
	subl	L,M
	movl	N,tmp10
	sall	$ CONST_BITS,I
	movl	M,tmp13
	leal	(I,K),J
	subl	K,I
	movl	J,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_b)
	movswl	D(0),I
	movswl	D(2),J
	movswl	D(6),K
	sall	$ CONST_BITS,I
	leal	(J,K),L
	movl	I,N
	imul	$ K0_5412,L
	imul	$-K1_8478,K
	imul	$ K0_7654,J
	addl	L,K
	addl	J,L
	subl	K,N
	leal	(I,K),M
	movl	N,tmp12
	movl	M,tmp11
	leal	(I,L),J
	subl	L,I
	movl	J,tmp10
	movl	I,tmp13
	jmp	Z(odd)

	.align	2

Y(l_7)
	movswl	D(0),I
	movswl	D(4),K
	movswl	D(2),J
	leal	(I,K),M
	imul	$ K1_3066,J,L
	sall	$ CONST_BITS,M
	subl	K,I
	leal	(M,L),N
	subl	L,M
	imul	$ K0_5412,J
	sall	$ CONST_BITS,I
	movl	N,tmp10
	movl	M,tmp13
	leal	(I,J),K
	subl	J,I
	movl	K,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_c)
	movswl	D(4),I
	movswl	D(6),J
	sall	$ CONST_BITS,I
	imul	$ K0_5412,J,K
	imul	$-K1_3066,J
	leal	(I,K),L
	leal	(I,J),M
	movl	L,tmp10
	subl	I,J
	subl	K,I
	movl	J,tmp11
	negl	M
	movl	I,tmp13
	movl	M,tmp12
	jmp	Z(odd)

	.align	2

Y(l_a)
	movswl	D(2),I
	movswl	D(6),J
/**/
	leal	(I,J),K
	imul	$ K0_7654,I
	imul	$ K0_5412,K
	imul	$-K1_8478,J
	addl	K,I
	addl	K,J
	movl	I,tmp10
	movl	J,tmp11
	negl	I
	negl	J
	movl	I,tmp13
	movl	J,tmp12
	jmp	Z(odd)

	.align	2

Y(l_9)
	movswl	D(0),I
	movswl	D(6),J
	sall	$ CONST_BITS,I
	imul	$ K0_5412,J,K
	movl	I,N
	imul	$-K1_3066,J
	leal	(I,K),L
	subl	K,N
	movl	L,tmp10
	leal	(I,J),M
	movl	N,tmp13
	subl	J,I
	movl	M,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_6)
	movswl	D(4),I
	movswl	D(2),J
	sall	$ CONST_BITS,I
	imul	$ K1_3066,J,K
	imul	$ K0_5412,J
	leal	(I,K),M
	leal	(I,J),L
	subl	I,J
	movl	M,tmp10
	movl	J,tmp11
	negl	L
	subl	K,I
	movl	L,tmp12
	movl	I,tmp13
	jmp	Z(odd)

	.align	2

Y(l_5)
	movswl	D(0),I
	movswl	D(4),J
/**/
	leal	(I,J),K
	subl	J,I
	sall	$ CONST_BITS,K
	sall	$ CONST_BITS,I
	movl	K,tmp10
	movl	K,tmp13
	movl	I,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_3)
	movswl	D(0),I
	movswl	D(2),K
	sall	$ CONST_BITS,I
	imul	$ K1_3066,K,J
	movl	I,N
	imul	$ K0_5412,K
	subl	J,N
	leal	(I,J),M
	movl	N,tmp13
	leal	(I,K),L
	movl	M,tmp10
	subl	K,I
	movl	L,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_8)
	movswl	D(6),I
/**/
	imul	$ K0_5412,I,J
	imul	$ K1_3066,I
	movl	J,tmp10
	movl	I,tmp12
	negl	J
	negl	I
	movl	J,tmp13
	movl	I,tmp11
	jmp	Z(odd)

	.align	2

Y(l_4)
	movswl	D(4),I
/**/
	sall	$ CONST_BITS,I
/**/
	movl	I,tmp10
	movl	I,tmp13
	negl	I
/**/
	movl	I,tmp11
	movl	I,tmp12
	jmp	Z(odd)

	.align	2

Y(l_2)
	movswl	D(2),J
/**/
	imul	$ K1_3066,J,I
	imul	$ K0_5412,J
	movl	I,tmp10
	movl	J,tmp11
	negl	I
	negl	J
	movl	I,tmp13
	movl	J,tmp12
	jmp	Z(odd)

	.align	2

Y(l_0)
	xorl	%eax,%eax
	jmp	Z(l_00)

	.align	2

Y(l_1)
	movswl	D(0),%eax
/**/
	sall	$ CONST_BITS,%eax
/**/
Y(l_00)
	movl	%eax,tmp10
	movl	%eax,tmp11
	movl	%eax,tmp12
	movl	%eax,tmp13

Y(odd)
	movb	mask,%al
	andl	$0xf0,%eax
	sarl	$2,%eax
	movl	Z(jumptable2)(%eax),%eax
	jmp	*%eax

	.align	2

Y(m_0)
	xorl	%eax,%eax
/**/
	movl	%eax,tmp0
	movl	%eax,tmp1
	movl	%eax,tmp2
	movl	%eax,tmp3
	jmp	Z(done)

	.align	2

Y(m_1)
	movswl	D(1),I
/**/
	imul	$ K0_2759,I,J
	imul	$ K0_7857,I,K
	movl	J,tmp0
	movl	K,tmp1
	imul	$ K1_1759,I,J
	imul	$ K1_3870,I
	movl	J,tmp2
	movl	I,tmp3
	jmp	Z(done)

	.align	2

Y(m_2)
	movswl	D(3),I
/**/
	imul	$-K0_7857,I,J
	imul	$-K1_3870,I,K
	movl	J,tmp0
	movl	K,tmp1
	imul	$-K0_2759,I,J
	imul	$ K1_1759,I
	movl	J,tmp2
	movl	I,tmp3
	jmp	Z(done)

	.align	2

Y(m_3)
	movswl	D(1),I
	movswl	D(3),J
/**/
	leal	(I,J),K
	imul	$ K1_0616,I,M
	imul	$ K0_7857,K,L
	imul	$ K0_2112,I
	imul	$-K2_1727,J,N
	subl	L,M
	addl	L,N
	movl	M,tmp0
	movl	N,tmp1
	imul	$ K1_1759,K
	imul	$-K1_4518,J
	addl	K,I
	addl	K,J
	movl	I,tmp3
	movl	J,tmp2
	jmp	Z(done)

	.align	2

Y(m_4)
	movswl	D(5),I
/**/
	imul	$ K1_1759,I,J
	imul	$ K0_2759,I,K
	movl	J,tmp0
	movl	K,tmp1
	imul	$-K1_3870,I,J
	imul	$ K0_7857,I
	movl	J,tmp2
	movl	I,tmp3
	jmp	Z(done)

	.align	2

Y(m_5)
	movswl	D(1),I
	movswl	D(5),J
/**/
	leal	(I,J),K
	imul	$-K0_9000,I,M
	imul	$ K1_1759,K,L
	imul	$-K2_5629,J,N
	imul	$ K0_6013,I
	addl	L,M
	addl	L,N
	movl	M,tmp0
	movl	N,tmp2
	imul	$ K0_7857,K
	imul	$-K0_5098,J
	addl	K,I
	addl	K,J
	movl	I,tmp3
	movl	J,tmp1
	jmp	Z(done)

	.align	2

Y(m_6)
	movswl	D(3),I
	movswl	D(5),J
/**/
	leal	(I,J),K
	imul	$-K1_9616,I,M
	imul	$-K0_3902,J,N
	imul	$ K1_1759,K,L
	imul	$ K1_1111,I
	addl	L,M
	addl	L,N
	movl	M,tmp0
	movl	N,tmp3
	imul	$-K1_3870,K
	imul	$ K1_6629,J
	addl	K,I
	addl	K,J
	movl	I,tmp2
	movl	J,tmp1
	jmp	Z(done)

	.align	2

Y(m_7)
	movswl	D(1),I
	movswl	D(5),K
	movswl	D(3),J
	leal	(I,K),L
	imul	$-K1_9616,J,N
	leal	(L,J),M
	imul	$-K0_3902,L
	imul	$ K1_1759,M
/**//* oh well... */
	addl	M,L
	addl	N,M
	imul	$-K0_9000,I,N
	imul	$ K0_6013,I
	addl	M,N
	addl	L,I
	movl	N,tmp0
	movl	I,tmp3
	leal	(J,K),I
	imul	$ K2_0531,K
	imul	$ K3_0727,J
	imul	$-K2_5629,I
	addl	L,K
	addl	I,J
	addl	I,K
	addl	M,J
	movl	K,tmp1
	movl	J,tmp2
	jmp	Z(done)

	.align	2

Y(m_8)
	movswl	D(7),I
/**/
	imul	$-K1_3870,I,J
	imul	$ K1_1759,I,K
	movl	J,tmp0
	movl	K,tmp1
	imul	$-K0_7857,I,J
	imul	$ K0_2759,I
	movl	J,tmp2
	movl	I,tmp3
	jmp	Z(done)

	.align	2

Y(m_9)
	movswl	D(1),I
	movswl	D(7),J
/**/
	leal	(I,J),K
	imul	$-K0_3902,I,M
	imul	$ K1_1759,K,L
	imul	$ K0_2759,K
	addl	L,M
	imul	$-K1_9616,J,N
	imul	$ K1_1111,I
	imul	$-K1_6629,J
	addl	L,N
	movl	M,tmp1
	movl	N,tmp2
	addl	K,I
	addl	K,J
	movl	I,tmp3
	movl	J,tmp0
	jmp	Z(done)

	.align	2

Y(m_a)
	movswl	D(3),I
	movswl	D(7),J
/**/
	leal	(I,J),K
	imul	$-K2_5629,I,M
	imul	$ K1_1759,K,L
	imul	$-K0_7857,K
	addl	L,M
	imul	$ K0_5098,I
	movl	M,tmp1
	imul	$-K0_9000,J,M
	addl	L,M
	imul	$-K0_6013,J
	movl	M,tmp3
	addl	K,I
	addl	K,J
	movl	I,tmp2
	movl	J,tmp0
	jmp	Z(done)

	.align	2

Y(m_b)
	movswl	D(3),J
	movswl	D(7),K
	movswl	D(1),I
	leal	(J,K),L
	imul	$-K0_3902,I,N
	leal	(L,I),M
	imul	$-K1_9616,L
	imul	$ K1_1759,M
/**//* oh well... */
	addl	M,L
	addl	N,M
	leal	(I,K),N
	imul	$ K0_2986,K
	imul	$-K0_9000,N
	addl	L,K
	imul	$ K1_5013,I
	addl	N,K
	addl	N,I
	movl	K,tmp0
	addl	M,I
	imul	$-K2_5629,J,N
	imul	$ K0_5098,J
	movl	I,tmp3
	leal	(N,M),K
	addl	L,J
	movl	K,tmp1
	movl	J,tmp2
	jmp	Z(done)

	.align	2

Y(m_c)
	movswl	D(5),I
	movswl	D(7),J
/**/
	leal	(I,J),K
	imul	$-K1_9616,J,L
	imul	$ K1_1759,K
	imul	$-K0_6013,J,N
	addl	K,L
	imul	$-K2_5629,I,M
	addl	L,N
	addl	L,M
	movl	N,tmp0
	imul	$-K0_3902,I,L
	movl	M,tmp2
	imul	$-K0_5098,I
	addl	L,K
	imul	$-K0_9000,J
	addl	K,I
	addl	K,J
	movl	I,tmp1
	movl	J,tmp3
	jmp	Z(done)

	.align	2

Y(m_d)
	movswl	D(1),I
	movswl	D(5),J
	movswl	D(7),K
	leal	(I,J),L
/**/
	leal	(K,L),M
	imul	$-K0_3902,L
	imul	$ K1_1759,M
	imul	$-K1_9616,K,N
	addl	M,L
	addl	N,M
	leal	(I,K),N
	imul	$ K0_2986,K
	imul	$-K0_9000,N
	imul	$ K1_5013,I
	addl	N,K
	addl	N,I
	addl	M,K
	addl	L,I
	movl	K,tmp0
	imul	$-K2_5629,J,N
	imul	$-K0_5098,J
	addl	N,M
	movl	I,tmp3
	addl	L,J
	movl	M,tmp2
	movl	J,tmp1
	jmp	Z(done)

	.align	2

Y(m_e)
	movswl	D(3),I
	movswl	D(7),K
	movswl	D(5),J
	leal	(I,K),L
	leal	(L,J),M
	imul	$-K1_9616,L
	imul	$ K1_1759,M
	imul	$-K0_3902,J,N
	addl	M,L
	addl	N,M
	imul	$-K0_9000,K,N
	imul	$-K0_6013,K
	addl	M,N
	addl	L,K
	movl	N,tmp3
	movl	K,tmp0
	leal	(I,J),K
	imul	$ K2_0531,J
	imul	$-K2_5629,K
	imul	$ K3_0727,I
	addl	K,J
	addl	K,I
	addl	M,J
	addl	L,I
	movl	J,tmp1
	movl	I,tmp2
	jmp	Z(done)

	.align	2

Y(m_f)
	movswl	D(3),J
	movswl	D(7),L
	movswl	D(5),K
	leal	(J,L),M
	movswl	D(1),I
	leal	(K,M),N
	imull	$-K1_9616,M
	addl	I,N
	addl	I,K
	imull	$ K1_1759,N
	imull	$-K0_3902,K
	addl	N,M
	addl	K,N
	leal	(L,I),K
	imull	$ K0_2986,L
	imull	$-K0_9000,K
	imull	$ K1_5013,I
	addl	K,L
	addl	K,I
	addl	M,L
	addl	N,I
	movl	L,tmp0
	movswl	D(5),K
	movl	I,tmp3
	leal	(J,K),I
	imull	$ K2_0531,K
	imull	$-K2_5629,I
	imull	$ K3_0727,J
	addl	I,K
	addl	I,J
	addl	N,K
	addl	M,J
	movl	K,tmp1
	movl	J,tmp2

Y(done)

#ifdef EIGHT_OUTPUT
	movl	tmp10,%ebx
	movl	tmp3,%esi
	movl	tmp11,%edx
	movl	tmp2,%ebp
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%ebx,%esi),%eax
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%edx,%ebp),%ecx
	subl	%esi,%ebx
	subl	%ebp,%edx
	sarl	$ CONST_BITS+SHIFT,%eax
	sarl	$ CONST_BITS+SHIFT,%ecx
#if OUT_OFFSET != 0
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%ebx
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%edx
#endif
	movb	%ax,D2(0)
	movb	%cx,D2(1)
	sarl	$ CONST_BITS+SHIFT,%ebx
	sarl	$ CONST_BITS+SHIFT,%edx
	movb	%bx,D2(7)
	movb	%dx,D2(6)

	movl	tmp12,%ebx
	movl	tmp1,%esi
	movl	tmp13,%edx
	movl	tmp0,%ebp
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%ebx,%esi),%eax
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%edx,%ebp),%ecx
	subl	%esi,%ebx
	subl	%ebp,%edx
	sarl	$ CONST_BITS+SHIFT,%eax
	sarl	$ CONST_BITS+SHIFT,%ecx
#if OUT_OFFSET != 0
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%ebx
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%edx
#endif
	movb	%ax,D2(2)
	movb	%cx,D2(3)
	sarl	$ CONST_BITS+SHIFT,%ebx
	sarl	$ CONST_BITS+SHIFT,%edx
	movb	%bx,D2(5)
	movb	%dx,D2(4)
#else
	movl	tmp10,%ebx
	movl	tmp3,%esi
	movl	tmp11,%edx
	movl	tmp2,%ebp
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%ebx,%esi),%eax
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%edx,%ebp),%ecx
	subl	%esi,%ebx
	subl	%ebp,%edx
	sarl	$ CONST_BITS+SHIFT,%eax
	sarl	$ CONST_BITS+SHIFT,%ecx
#if OUT_OFFSET != 0
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%ebx
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%edx
#endif
	movw	%ax,D(0)
	movw	%cx,D(1)
	sarl	$ CONST_BITS+SHIFT,%ebx
	sarl	$ CONST_BITS+SHIFT,%edx
	movw	%bx,D(7)
	movw	%dx,D(6)

	movl	tmp12,%ebx
	movl	tmp1,%esi
	movl	tmp13,%edx
	movl	tmp0,%ebp
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%ebx,%esi),%eax
	leal	(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1)(%edx,%ebp),%ecx
	subl	%esi,%ebx
	subl	%ebp,%edx
	sarl	$ CONST_BITS+SHIFT,%eax
	sarl	$ CONST_BITS+SHIFT,%ecx
#if OUT_OFFSET != 0
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%ebx
	addl	$(1+OUT_OFFSET*2)<<(CONST_BITS+SHIFT-1),%edx
#endif
	movw	%ax,D(2)
	movw	%cx,D(3)
	sarl	$ CONST_BITS+SHIFT,%ebx
	sarl	$ CONST_BITS+SHIFT,%edx
	movw	%bx,D(5)
	movw	%dx,D(4)
#endif

Y(continue)

	addl	$ INCSECOND,DREG
	decb	count
	jnz	Z(loop)

#undef D
#undef OUT_OFFSET
