/*
 * Based on John the Ripper and modified to integrate with aircrack
 *
 * 	John the Ripper copyright and license.
 *
 * John the Ripper password cracker,
 * Copyright (c) 1996-2013 by Solar Designer.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * As a special exception to the GNU General Public License terms,
 * permission is hereby granted to link the code of this program, with or
 * without modification, with any version of the OpenSSL library and/or any
 * version of unRAR, and to distribute such linked combinations.  You must
 * obey the GNU GPL in all respects for all of the code used other than
 * OpenSSL and unRAR.  If you modify this program, you may extend this
 * exception to your version of the program, but you are not obligated to
 * do so.  (In other words, you may release your derived work under pure
 * GNU GPL version 2 or later as published by the FSF.)
 *
 * (This exception from the GNU GPL is not required for the core tree of
 * John the Ripper, but arguably it is required for -jumbo.)
 *
 * 	Relaxed terms for certain components.
 *
 * In addition or alternatively to the license above, many components are
 * available to you under more relaxed terms (most commonly under cut-down
 * BSD license) as specified in the corresponding source files.
 *
 * For more information on John the Ripper licensing please visit:
 *
 * http://www.openwall.com/john/doc/LICENSE.shtml
 *
 * This software is
 * Copyright (c) 2010 bartavelle, <bartavelle at bandecon.com>,
 * Copyright (c) 2012 Solar Designer,
 * Copyright (c) 2011-2015 JimF,
 * Copyright (c) 2011-2015 magnum,
 * and it is hereby released to the general public under the following terms:
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted.
 *
 * SHA-2 Copyright 2013, epixoip. Redistribution and use in source and binary
 * forms, with or without modification, are permitted provided that
 * redistribution of source retains the above copyright.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <assert.h>
#include <string.h>
#include <stdint.h>

#include "aircrack-ng/ce-wpa/arch.h"
#include "aircrack-ng/ce-wpa/pseudo_intrinsics.h"
#include "aircrack-ng/ce-wpa/memory.h"
#include "aircrack-ng/ce-wpa/johnswap.h"
#include "aircrack-ng/ce-wpa/simd-intrinsics-load-flags.h"
#include "aircrack-ng/ce-wpa/aligned.h"
#include "aircrack-ng/ce-wpa/misc.h"

/* Shorter names for use in index calculations */
#define VS32 SIMD_COEF_32
#define VS64 SIMD_COEF_64

#if SIMD_PARA_MD5
#define MD5_SSE_NUM_KEYS (SIMD_COEF_32 * SIMD_PARA_MD5)
#define MD5_PARA_DO(x) for ((x) = 0; (x) < SIMD_PARA_MD5; (x)++)

#define MD5_F(x, y, z) tmp[i] = vcmov((y[i]), (z[i]), (x[i]));

#define MD5_G(x, y, z) tmp[i] = vcmov((x[i]), (y[i]), (z[i]));

#if __AVX512F__
#define MD5_H(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0x96);

#define MD5_H2(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0x96);
#elif 1
#define MD5_H(x, y, z)                                                         \
	tmp2[i] = vxor((x[i]), (y[i]));                                            \
	tmp[i] = vxor(tmp2[i], (z[i]));

#define MD5_H2(x, y, z) tmp[i] = vxor((x[i]), tmp2[i]);
#else
#define MD5_H(x, y, z)                                                         \
	tmp[i] = vxor((x[i]), (y[i]));                                             \
	tmp[i] = vxor((tmp[i]), (z[i]));

#define MD5_H2(x, y, z)                                                        \
	tmp[i] = vxor((y[i]), (z[i]));                                             \
	tmp[i] = vxor((tmp[i]), (x[i]));
#endif

#if __AVX512F__
#define MD5_I(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0x39);
#elif __ARM_NEON__
#define MD5_I(x, y, z)                                                         \
	tmp[i] = vorn((x[i]), (z[i]));                                             \
	tmp[i] = vxor((tmp[i]), (y[i]));
#elif !VCMOV_EMULATED
#define MD5_I(x, y, z)                                                         \
	tmp[i] = vcmov((x[i]), mask, (z[i]));                                      \
	tmp[i] = vxor((tmp[i]), (y[i]));
#else
#define MD5_I(x, y, z)                                                         \
	tmp[i] = vandnot((z[i]), mask);                                            \
	tmp[i] = vor((tmp[i]), (x[i]));                                            \
	tmp[i] = vxor((tmp[i]), (y[i]));
#endif

#define MD5_STEP(f, a, b, c, d, x, t, s)                                       \
	MD5_PARA_DO(i)                                                             \
	{                                                                          \
		a[i] = vadd_epi32(a[i], vset1_epi32(t));                               \
		a[i] = vadd_epi32(a[i], data[i * 16 + x]);                             \
		f((b), (c), (d)) a[i] = vadd_epi32(a[i], tmp[i]);                      \
		a[i] = vroti_epi32(a[i], (s));                                         \
		a[i] = vadd_epi32(a[i], b[i]);                                         \
	}

#define MD5_STEP_r16(f, a, b, c, d, x, t, s)                                   \
	MD5_PARA_DO(i)                                                             \
	{                                                                          \
		a[i] = vadd_epi32(a[i], vset1_epi32(t));                               \
		a[i] = vadd_epi32(a[i], data[i * 16 + x]);                             \
		f((b), (c), (d)) a[i] = vadd_epi32(a[i], tmp[i]);                      \
		a[i] = vroti16_epi32(a[i], (s));                                       \
		a[i] = vadd_epi32(a[i], b[i]);                                         \
	}

#define INIT_A 0x67452301

void md5_reverse(uint32_t * hash) { hash[0] -= INIT_A; }

void md5_unreverse(uint32_t * hash) { hash[0] += INIT_A; }

#undef INIT_A

void SIMDmd5body(vtype * _data,
				 unsigned int * out,
				 ARCH_WORD_32 * reload_state,
				 unsigned SSEi_flags)
{
	vtype w[16 * SIMD_PARA_MD5];
	vtype a[SIMD_PARA_MD5];
	vtype b[SIMD_PARA_MD5];
	vtype c[SIMD_PARA_MD5];
	vtype d[SIMD_PARA_MD5];
	vtype tmp[SIMD_PARA_MD5];
#if !__AVX512F__
	vtype tmp2[SIMD_PARA_MD5];
#endif
	unsigned int i;
	vtype * data;

#if !__AVX512F__ && !__ARM_NEON__
	vtype mask;
	mask = vset1_epi32(0xffffffff);
#endif

	if (SSEi_flags & SSEi_FLAT_IN)
	{
// Move _data to __data, mixing it SIMD_COEF_32 wise.
#if __SSE4_1__ || __MIC__
		unsigned k;
		vtype * W = w;
		ARCH_WORD_32 * saved_key = (ARCH_WORD_32 *) _data;
		MD5_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (i = 0; i < 16; ++i)
				{
					GATHER_4x(W[i], saved_key, i);
				}
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (i = 0; i < 16; ++i)
				{
					GATHER_2x(W[i], saved_key, i);
				}
				saved_key += (VS32 << 5);
			}
			else
			{
				for (i = 0; i < 16; ++i)
				{
					GATHER(W[i], saved_key, i);
				}
				saved_key += (VS32 << 4);
			}
			W += 16;
		}
#else
		unsigned j, k;
		ARCH_WORD_32 * p = (ARCH_WORD_32 *) w;
		vtype * W = w;
		ARCH_WORD_32 * saved_key = (ARCH_WORD_32 *) _data;
		MD5_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 6) + j];
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 5) + j];
				saved_key += (VS32 << 5);
			}
			else
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 4) + j];
				saved_key += (VS32 << 4);
			}
			W += 16;
		}
#endif
		// now set our data pointer to point to this 'mixed' data.
		data = w;
	}
	else
		data = _data;

	if ((SSEi_flags & SSEi_RELOAD) == 0)
	{
		MD5_PARA_DO(i)
		{
			a[i] = vset1_epi32(0x67452301);
			b[i] = vset1_epi32(0xefcdab89);
			c[i] = vset1_epi32(0x98badcfe);
			d[i] = vset1_epi32(0x10325476);
		}
	}
	else
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			MD5_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]);
			}
		}
		else
		{
			MD5_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 3 * VS32]);
			}
		}
	}

	/* Round 1 */
	MD5_STEP(MD5_F, a, b, c, d, 0, 0xd76aa478, 7)
	MD5_STEP(MD5_F, d, a, b, c, 1, 0xe8c7b756, 12)
	MD5_STEP(MD5_F, c, d, a, b, 2, 0x242070db, 17)
	MD5_STEP(MD5_F, b, c, d, a, 3, 0xc1bdceee, 22)
	MD5_STEP(MD5_F, a, b, c, d, 4, 0xf57c0faf, 7)
	MD5_STEP(MD5_F, d, a, b, c, 5, 0x4787c62a, 12)
	MD5_STEP(MD5_F, c, d, a, b, 6, 0xa8304613, 17)
	MD5_STEP(MD5_F, b, c, d, a, 7, 0xfd469501, 22)
	MD5_STEP(MD5_F, a, b, c, d, 8, 0x698098d8, 7)
	MD5_STEP(MD5_F, d, a, b, c, 9, 0x8b44f7af, 12)
	MD5_STEP(MD5_F, c, d, a, b, 10, 0xffff5bb1, 17)
	MD5_STEP(MD5_F, b, c, d, a, 11, 0x895cd7be, 22)
	MD5_STEP(MD5_F, a, b, c, d, 12, 0x6b901122, 7)
	MD5_STEP(MD5_F, d, a, b, c, 13, 0xfd987193, 12)
	MD5_STEP(MD5_F, c, d, a, b, 14, 0xa679438e, 17)
	MD5_STEP(MD5_F, b, c, d, a, 15, 0x49b40821, 22)

	/* Round 2 */
	MD5_STEP(MD5_G, a, b, c, d, 1, 0xf61e2562, 5)
	MD5_STEP(MD5_G, d, a, b, c, 6, 0xc040b340, 9)
	MD5_STEP(MD5_G, c, d, a, b, 11, 0x265e5a51, 14)
	MD5_STEP(MD5_G, b, c, d, a, 0, 0xe9b6c7aa, 20)
	MD5_STEP(MD5_G, a, b, c, d, 5, 0xd62f105d, 5)
	MD5_STEP(MD5_G, d, a, b, c, 10, 0x02441453, 9)
	MD5_STEP(MD5_G, c, d, a, b, 15, 0xd8a1e681, 14)
	MD5_STEP(MD5_G, b, c, d, a, 4, 0xe7d3fbc8, 20)
	MD5_STEP(MD5_G, a, b, c, d, 9, 0x21e1cde6, 5)
	MD5_STEP(MD5_G, d, a, b, c, 14, 0xc33707d6, 9)
	MD5_STEP(MD5_G, c, d, a, b, 3, 0xf4d50d87, 14)
	MD5_STEP(MD5_G, b, c, d, a, 8, 0x455a14ed, 20)
	MD5_STEP(MD5_G, a, b, c, d, 13, 0xa9e3e905, 5)
	MD5_STEP(MD5_G, d, a, b, c, 2, 0xfcefa3f8, 9)
	MD5_STEP(MD5_G, c, d, a, b, 7, 0x676f02d9, 14)
	MD5_STEP(MD5_G, b, c, d, a, 12, 0x8d2a4c8a, 20)

	/* Round 3 */
	MD5_STEP(MD5_H, a, b, c, d, 5, 0xfffa3942, 4)
	MD5_STEP(MD5_H2, d, a, b, c, 8, 0x8771f681, 11)
	MD5_STEP_r16(MD5_H, c, d, a, b, 11, 0x6d9d6122, 16) MD5_STEP(
		MD5_H2, b, c, d, a, 14, 0xfde5380c, 23)
		MD5_STEP(MD5_H, a, b, c, d, 1, 0xa4beea44, 4) MD5_STEP(
			MD5_H2, d, a, b, c, 4, 0x4bdecfa9, 11)
			MD5_STEP_r16(MD5_H, c, d, a, b, 7, 0xf6bb4b60, 16) MD5_STEP(
				MD5_H2, b, c, d, a, 10, 0xbebfbc70, 23)
				MD5_STEP(MD5_H, a, b, c, d, 13, 0x289b7ec6, 4) MD5_STEP(
					MD5_H2, d, a, b, c, 0, 0xeaa127fa, 11)
					MD5_STEP_r16(MD5_H, c, d, a, b, 3, 0xd4ef3085, 16) MD5_STEP(
						MD5_H2, b, c, d, a, 6, 0x04881d05, 23)
						MD5_STEP(MD5_H, a, b, c, d, 9, 0xd9d4d039, 4) MD5_STEP(
							MD5_H2, d, a, b, c, 12, 0xe6db99e5, 11)
							MD5_STEP_r16(MD5_H, c, d, a, b, 15, 0x1fa27cf8, 16)
								MD5_STEP(MD5_H2, b, c, d, a, 2, 0xc4ac5665, 23)

		/* Round 4 */
		MD5_STEP(MD5_I, a, b, c, d, 0, 0xf4292244, 6) MD5_STEP(
			MD5_I, d, a, b, c, 7, 0x432aff97, 10)
			MD5_STEP(MD5_I, c, d, a, b, 14, 0xab9423a7, 15) MD5_STEP(
				MD5_I, b, c, d, a, 5, 0xfc93a039, 21)
				MD5_STEP(MD5_I, a, b, c, d, 12, 0x655b59c3, 6) MD5_STEP(
					MD5_I, d, a, b, c, 3, 0x8f0ccc92, 10)
					MD5_STEP(MD5_I, c, d, a, b, 10, 0xffeff47d, 15) MD5_STEP(
						MD5_I, b, c, d, a, 1, 0x85845dd1, 21)
						MD5_STEP(MD5_I, a, b, c, d, 8, 0x6fa87e4f, 6) MD5_STEP(
							MD5_I, d, a, b, c, 15, 0xfe2ce6e0, 10)
							MD5_STEP(MD5_I, c, d, a, b, 6, 0xa3014314, 15)
								MD5_STEP(MD5_I, b, c, d, a, 13, 0x4e0811a1, 21)
									MD5_STEP(
										MD5_I, a, b, c, d, 4, 0xf7537e82, 6)

										if (SSEi_flags & SSEi_REVERSE_STEPS)
	{
		MD5_PARA_DO(i)
		{
			vstore((vtype *) &out[i * 4 * VS32 + 0 * VS32], a[i]);
		}
		return;
	}

	MD5_STEP(MD5_I, d, a, b, c, 11, 0xbd3af235, 10)
	MD5_STEP(MD5_I, c, d, a, b, 2, 0x2ad7d2bb, 15)
	MD5_STEP(MD5_I, b, c, d, a, 9, 0xeb86d391, 21)

	if ((SSEi_flags & SSEi_RELOAD) == 0)
	{
		MD5_PARA_DO(i)
		{
			a[i] = vadd_epi32(a[i], vset1_epi32(0x67452301));
			b[i] = vadd_epi32(b[i], vset1_epi32(0xefcdab89));
			c[i] = vadd_epi32(c[i], vset1_epi32(0x98badcfe));
			d[i] = vadd_epi32(d[i], vset1_epi32(0x10325476));
		}
	}
	else
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			MD5_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]));
			}
		}
		else
		{
			MD5_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 3 * VS32]));
			}
		}
	}

#if USE_EXPERIMENTAL
	/*
 * This is currently not used for MD5, and was observed to result
 * in a significant performance regression (at least on XOP) just by sitting
 * here. http://www.openwall.com/lists/john-dev/2015/09/05/5
 * NOTE the regression might be gone now anyway since we went from -O3 to -O2.
 */
	if (SSEi_flags & SSEi_FLAT_OUT)
	{
		MD5_PARA_DO(i)
		{
			uint32_t * o = (uint32_t *) &out[i * 4 * VS32];
#if __AVX512F__ || __MIC__
			vtype idxs = vset_epi32(15 * 4,
									14 * 4,
									13 * 4,
									12 * 4,
									11 * 4,
									10 * 4,
									9 * 4,
									8 * 4,
									7 * 4,
									6 * 4,
									5 * 4,
									4 * 4,
									3 * 4,
									2 * 4,
									1 * 4,
									0 * 4);

			vscatter_epi32(o + 0, idxs, a[i], 4);
			vscatter_epi32(o + 1, idxs, b[i], 4);
			vscatter_epi32(o + 2, idxs, c[i], 4);
			vscatter_epi32(o + 3, idxs, d[i], 4);
#else
			uint32_t j, k;
			union {
				vtype v[4];
				uint32_t s[4 * VS32];
			} tmp;

			tmp.v[0] = a[i];
			tmp.v[1] = b[i];
			tmp.v[2] = c[i];
			tmp.v[3] = d[i];

			for (j = 0; j < VS32; j++)
				for (k = 0; k < 4; k++) o[j * 4 + k] = tmp.s[k * VS32 + j];
#endif
		}
	}
	else
#endif
		if (SSEi_flags & SSEi_OUTPUT_AS_INP_FMT)
	{
		if ((SSEi_flags & SSEi_OUTPUT_AS_2BUF_INP_FMT)
			== SSEi_OUTPUT_AS_2BUF_INP_FMT)
		{
			MD5_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 32 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 3 * VS32], d[i]);
			}
		}
		else
		{
			MD5_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 16 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 3 * VS32], d[i]);
			}
		}
	}
	else
	{
		MD5_PARA_DO(i)
		{
			vstore((vtype *) &out[i * 4 * VS32 + 0 * VS32], a[i]);
			vstore((vtype *) &out[i * 4 * VS32 + 1 * VS32], b[i]);
			vstore((vtype *) &out[i * 4 * VS32 + 2 * VS32], c[i]);
			vstore((vtype *) &out[i * 4 * VS32 + 3 * VS32], d[i]);
		}
	}
}

#define GETPOS(i, index)                                                       \
	((index & (VS32 - 1)) * 4 + (i & (0xffffffff - 3)) * VS32 + ((i) &3))

static MAYBE_INLINE void mmxput(void * buf,
								unsigned int index,
								unsigned int bid,
								unsigned int offset,
								unsigned char * src,
								unsigned int len)
{
	unsigned char * nbuf;
	unsigned int i;

	nbuf = ((unsigned char *) buf) + index / VS32 * 64 * VS32
		   + bid * 64 * MD5_SSE_NUM_KEYS;
	for (i = 0; i < len; i++) nbuf[GETPOS((offset + i), index)] = src[i];
}

static MAYBE_INLINE void mmxput2(void * buf, unsigned int bid, void * src)
{
	unsigned char * nbuf;
	unsigned int i;

	nbuf = ((unsigned char *) buf) + bid * 64 * MD5_SSE_NUM_KEYS;
	MD5_PARA_DO(i)
	memcpy(nbuf + i * 64 * VS32,
		   ((unsigned char *) src) + i * 16 * VS32,
		   16 * VS32);
}

#if (ARCH_SIZE >= 8) || defined(__i386__) || defined(__ARM_NEON__)
#define BITALIGN(hi, lo, s) ((((uint64_t)(hi) << 32) | (lo)) >> (s))
#else
#define BITALIGN(hi, lo, s) (((hi) << (32 - (s))) | ((lo) >> (s)))
#endif

static MAYBE_INLINE void mmxput3(void * buf,
								 unsigned int bid,
								 unsigned int * offset,
								 unsigned int mult,
								 unsigned int saltlen,
								 void * src)
{
	unsigned int j;

	MD5_PARA_DO(j)
	{
		unsigned int i;
		unsigned int jm = j * VS32 * 4;
		unsigned char * nbuf
			= ((unsigned char *) buf) + bid * (64 * MD5_SSE_NUM_KEYS) + jm * 16;
		unsigned int * s = (unsigned int *) src + jm;
		for (i = 0; i < VS32; i++, s++)
		{
			unsigned int n = offset[i + jm / 4] * mult + saltlen;
			unsigned int * d = (unsigned int *) (nbuf + (n & ~3U) * VS32) + i;

			switch (n &= 3)
			{
				case 0:
					d[0] = s[0];
					d[1 * VS32] = s[1 * VS32];
					d[2 * VS32] = s[2 * VS32];
					d[3 * VS32] = s[3 * VS32];
					break;
#if 0
			default:
				n <<= 3;
				{
					unsigned int m = 32 - n;
					d[0] = (d[0] & (0xffffffffU >> m)) | (s[0] << n);
					d[1 * VS32] = BITALIGN(s[1 * VS32], s[0], m);
					d[2 * VS32] = BITALIGN(s[2 * VS32], s[1 * VS32], m);
					d[3 * VS32] = BITALIGN(s[3 * VS32], s[2 * VS32], m);
					d[4 * VS32] = (d[4 * VS32] & (0xffffffffU << n)) | (s[3 * VS32] >> m);
				}
#else
				case 1:
					d[0] = (d[0] & 0xffU) | (s[0] << 8);
					d[1 * VS32] = BITALIGN(s[1 * VS32], s[0], 24);
					d[2 * VS32] = BITALIGN(s[2 * VS32], s[1 * VS32], 24);
					d[3 * VS32] = BITALIGN(s[3 * VS32], s[2 * VS32], 24);
					d[4 * VS32]
						= (d[4 * VS32] & 0xffffff00U) | (s[3 * VS32] >> 24);
					break;
				case 2:
					d[0] = (d[0] & 0xffffU) | (s[0] << 16);
					d[1 * VS32] = BITALIGN(s[1 * VS32], s[0], 16);
					d[2 * VS32] = BITALIGN(s[2 * VS32], s[1 * VS32], 16);
					d[3 * VS32] = BITALIGN(s[3 * VS32], s[2 * VS32], 16);
					d[4 * VS32]
						= (d[4 * VS32] & 0xffff0000U) | (s[3 * VS32] >> 16);
					break;
				case 3:
					d[0] = (d[0] & 0xffffffU) | (s[0] << 24);
					d[1 * VS32] = BITALIGN(s[1 * VS32], s[0], 8);
					d[2 * VS32] = BITALIGN(s[2 * VS32], s[1 * VS32], 8);
					d[3 * VS32] = BITALIGN(s[3 * VS32], s[2 * VS32], 8);
					d[4 * VS32]
						= (d[4 * VS32] & 0xff000000U) | (s[3 * VS32] >> 8);
#endif
			}
		}
	}
}

static MAYBE_INLINE void dispatch(unsigned char buffers[8]
													   [64 * MD5_SSE_NUM_KEYS],
								  unsigned int f[4 * MD5_SSE_NUM_KEYS],
								  unsigned int length[MD5_SSE_NUM_KEYS],
								  unsigned int saltlen)
{
	unsigned int i, j;
	unsigned int bufferid;

	i = 1000 / 42;
	j = 0;
	do
	{
		switch (j)
		{
			case 0:
				bufferid = 0;
				mmxput2(buffers, bufferid, f);
				break;
			case 21:
				bufferid = 1;
				mmxput3(buffers, bufferid, length, 1, 0, f);
				break;
			case 3:
			case 9:
			case 15:
			case 27:
			case 33:
			case 39:
				bufferid = 2;
				mmxput3(buffers, bufferid, length, 2, 0, f);
				break;
			case 6:
			case 12:
			case 18:
			case 24:
			case 30:
			case 36:
				bufferid = 3;
				mmxput2(buffers, bufferid, f);
				break;
			case 7:
			case 35:
				bufferid = 4;
				mmxput3(buffers, bufferid, length, 1, saltlen, f);
				break;
			case 14:
			case 28:
				bufferid = 5;
				mmxput2(buffers, bufferid, f);
				break;
			case 2:
			case 4:
			case 8:
			case 10:
			case 16:
			case 20:
			case 22:
			case 26:
			case 32:
			case 34:
			case 38:
			case 40:
				bufferid = 6;
				mmxput2(buffers, bufferid, f);
				break;
			default:
				bufferid = 7;
				mmxput3(buffers, bufferid, length, 2, saltlen, f);
				break;
		}
		SIMDmd5body((vtype *) &buffers[bufferid], f, NULL, SSEi_MIXED_IN);
		if (j++ < 1000 % 42 - 1) continue;
		if (j == 1000 % 42)
		{
			if (!i) break;
			i--;
			continue;
		}
		if (j >= 42) j = 0;
	} while (1);
}

void md5cryptsse(unsigned char pwd[MD5_SSE_NUM_KEYS][16],
				 unsigned char * salt,
				 char * out,
				 unsigned int md5_type)
{
	unsigned int length[MD5_SSE_NUM_KEYS];
	unsigned int saltlen;
	unsigned int i, j;
	MD5_CTX ctx;
	MD5_CTX tctx;
	JTR_ALIGN(MEM_ALIGN_SIMD)
	unsigned char buffers[8][64 * MD5_SSE_NUM_KEYS] = {{0}};
	JTR_ALIGN(MEM_ALIGN_SIMD) unsigned int F[4 * MD5_SSE_NUM_KEYS];

	saltlen = strlen((char *) salt);
	for (i = 0; i < MD5_SSE_NUM_KEYS; i++)
	{
		unsigned int length_i = strlen((char *) pwd[i]);
		unsigned int * bt;
		unsigned int tf[4];

		/* cas 0 fs */
		mmxput(buffers, i, 0, 16, pwd[i], length_i);
		mmxput(buffers, i, 0, length_i + 16, (unsigned char *) "\x80", 1);
		/* cas 1 sf */
		mmxput(buffers, i, 1, 0, pwd[i], length_i);
		mmxput(buffers, i, 1, length_i + 16, (unsigned char *) "\x80", 1);
		/* cas 2 ssf */
		mmxput(buffers, i, 2, 0, pwd[i], length_i);
		mmxput(buffers, i, 2, length_i, pwd[i], length_i);
		mmxput(buffers, i, 2, length_i * 2 + 16, (unsigned char *) "\x80", 1);
		/* cas 3 fss */
		mmxput(buffers, i, 3, 16, pwd[i], length_i);
		mmxput(buffers, i, 3, 16 + length_i, pwd[i], length_i);
		mmxput(buffers, i, 3, length_i * 2 + 16, (unsigned char *) "\x80", 1);
		/* cas 4 scf */
		mmxput(buffers, i, 4, 0, pwd[i], length_i);
		mmxput(buffers, i, 4, length_i, salt, saltlen);
		mmxput(buffers,
			   i,
			   4,
			   saltlen + length_i + 16,
			   (unsigned char *) "\x80",
			   1);
		/* cas 5 fcs */
		mmxput(buffers, i, 5, 16, salt, saltlen);
		mmxput(buffers, i, 5, 16 + saltlen, pwd[i], length_i);
		mmxput(buffers,
			   i,
			   5,
			   saltlen + length_i + 16,
			   (unsigned char *) "\x80",
			   1);
		/* cas 6 fcss */
		mmxput(buffers, i, 6, 16, salt, saltlen);
		mmxput(buffers, i, 6, 16 + saltlen, pwd[i], length_i);
		mmxput(buffers, i, 6, 16 + saltlen + length_i, pwd[i], length_i);
		mmxput(buffers,
			   i,
			   6,
			   saltlen + 2 * length_i + 16,
			   (unsigned char *) "\x80",
			   1);
		/* cas 7 scsf */
		mmxput(buffers, i, 7, 0, pwd[i], length_i);
		mmxput(buffers, i, 7, length_i, salt, saltlen);
		mmxput(buffers, i, 7, length_i + saltlen, pwd[i], length_i);
		mmxput(buffers,
			   i,
			   7,
			   saltlen + 2 * length_i + 16,
			   (unsigned char *) "\x80",
			   1);

		bt = (unsigned int *) &buffers[0];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i + 16) << 3;
		bt = (unsigned int *) &buffers[1];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i + 16) << 3;
		bt = (unsigned int *) &buffers[2];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i * 2 + 16) << 3;
		bt = (unsigned int *) &buffers[3];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i * 2 + 16) << 3;
		bt = (unsigned int *) &buffers[4];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i + saltlen + 16) << 3;
		bt = (unsigned int *) &buffers[5];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i + saltlen + 16) << 3;
		bt = (unsigned int *) &buffers[6];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i * 2 + saltlen + 16) << 3;
		bt = (unsigned int *) &buffers[7];
		bt[14 * VS32 + (i & (VS32 - 1)) + i / VS32 * 16 * VS32]
			= (length_i * 2 + saltlen + 16) << 3;

		MD5_Init(&ctx);
		MD5_Update(&ctx, pwd[i], length_i);
		if (md5_type == MD5_TYPE_STD)
			MD5_Update(&ctx, "$1$", 3);
		else if (md5_type == MD5_TYPE_APACHE)
			MD5_Update(&ctx, "$apr1$", 6);
		// else it's AIX and no prefix included
		MD5_Update(&ctx, salt, saltlen);
		MD5_Init(&tctx);
		MD5_Update(&tctx, pwd[i], length_i);
		MD5_Update(&tctx, salt, saltlen);
		MD5_Update(&tctx, pwd[i], length_i);
		MD5_Final((unsigned char *) tf, &tctx);
		MD5_Update(&ctx, tf, length_i);
		length[i] = length_i;
		for (j = length_i; j; j >>= 1)
			if (j & 1)
				MD5_Update(&ctx, "\0", 1);
			else
				MD5_Update(&ctx, pwd[i], 1);
		MD5_Final((unsigned char *) tf, &ctx);
		F[i / VS32 * 4 * VS32 + (i & (VS32 - 1)) + 0 * VS32] = tf[0];
		F[i / VS32 * 4 * VS32 + (i & (VS32 - 1)) + 1 * VS32] = tf[1];
		F[i / VS32 * 4 * VS32 + (i & (VS32 - 1)) + 2 * VS32] = tf[2];
		F[i / VS32 * 4 * VS32 + (i & (VS32 - 1)) + 3 * VS32] = tf[3];
	}
	dispatch(buffers, F, length, saltlen);
	memcpy(out, F, MD5_SSE_NUM_KEYS * 16);
}
#endif /* SIMD_PARA_MD5 */

#if SIMD_PARA_MD4
#define MD4_SSE_NUM_KEYS (SIMD_COEF_32 * SIMD_PARA_MD4)
#define MD4_PARA_DO(x) for ((x) = 0; (x) < SIMD_PARA_MD4; (x)++)

#define MD4_F(x, y, z) tmp[i] = vcmov((y[i]), (z[i]), (x[i]));

#if __AVX512F__
#define MD4_G(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0xE8);
#elif !VCMOV_EMULATED
#define MD4_G(x, y, z)                                                         \
	tmp[i] = vxor((y[i]), (z[i]));                                             \
	tmp[i] = vcmov((x[i]), (z[i]), (tmp[i]));
#else
#define MD4_G(x, y, z)                                                         \
	tmp[i] = vor((y[i]), (z[i]));                                              \
	tmp2[i] = vand((y[i]), (z[i]));                                            \
	tmp[i] = vand((tmp[i]), (x[i]));                                           \
	tmp[i] = vor((tmp[i]), (tmp2[i]));
#endif

#if __AVX512F__
#define MD4_H(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0x96);

#define MD4_H2(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0x96);
#elif SIMD_PARA_MD4 < 3
#define MD4_H(x, y, z)                                                         \
	tmp2[i] = vxor((x[i]), (y[i]));                                            \
	tmp[i] = vxor(tmp2[i], (z[i]));

#define MD4_H2(x, y, z) tmp[i] = vxor((x[i]), tmp2[i]);
#else
#define MD4_H(x, y, z)                                                         \
	tmp[i] = vxor((x[i]), (y[i]));                                             \
	tmp[i] = vxor((tmp[i]), (z[i]));

#define MD4_H2(x, y, z)                                                        \
	tmp[i] = vxor((y[i]), (z[i]));                                             \
	tmp[i] = vxor((tmp[i]), (x[i]));
#endif

#define MD4_STEP(f, a, b, c, d, x, t, s)                                       \
	MD4_PARA_DO(i)                                                             \
	{                                                                          \
		a[i] = vadd_epi32(a[i], t);                                            \
		f((b), (c), (d)) a[i] = vadd_epi32(a[i], tmp[i]);                      \
		a[i] = vadd_epi32(a[i], data[i * 16 + x]);                             \
		a[i] = vroti_epi32(a[i], (s));                                         \
	}

#define MD4_REV_STEP(f, a, b, c, d, x, t, s)                                   \
	MD4_PARA_DO(i)                                                             \
	{                                                                          \
		f((b), (c), (d)) a[i] = vadd_epi32(a[i], tmp[i]);                      \
		a[i] = vadd_epi32(a[i], data[i * 16 + x]);                             \
	}

#define INIT_A 0x67452301
#define INIT_B 0xefcdab89
#define INIT_C 0x98badcfe
#define INIT_D 0x10325476
#define SQRT_3 0x6ed9eba1

void md4_reverse(uint32_t * hash)
{
	hash[0] -= INIT_A;
	hash[1] -= INIT_B;
	hash[2] -= INIT_C;
	hash[3] -= INIT_D;
	hash[1] = (hash[1] >> 15) | (hash[1] << 17);
	hash[1] -= SQRT_3 + (hash[2] ^ hash[3] ^ hash[0]);
	hash[1] = (hash[1] >> 15) | (hash[1] << 17);
	hash[1] -= SQRT_3;
}

void md4_unreverse(uint32_t * hash)
{
	hash[1] += SQRT_3;
	hash[1] = (hash[1] >> 17) | (hash[1] << 15);
	hash[1] += SQRT_3 + (hash[2] ^ hash[3] ^ hash[0]);
	hash[1] = (hash[1] >> 17) | (hash[1] << 15);
	hash[3] += INIT_D;
	hash[2] += INIT_C;
	hash[1] += INIT_B;
	hash[0] += INIT_A;
}

#undef SQRT_3
#undef INIT_D
#undef INIT_C
#undef INIT_B
#undef INIT_A

void SIMDmd4body(vtype * _data,
				 unsigned int * out,
				 ARCH_WORD_32 * reload_state,
				 unsigned SSEi_flags)
{
	vtype w[16 * SIMD_PARA_MD4];
	vtype a[SIMD_PARA_MD4];
	vtype b[SIMD_PARA_MD4];
	vtype c[SIMD_PARA_MD4];
	vtype d[SIMD_PARA_MD4];
	vtype tmp[SIMD_PARA_MD4];
#if (SIMD_PARA_MD4 < 3 || VCMOV_EMULATED) && !__AVX512F__
	vtype tmp2[SIMD_PARA_MD4];
#endif
	vtype cst;
	unsigned int i;
	vtype * data;

	if (SSEi_flags & SSEi_FLAT_IN)
	{
// Move _data to __data, mixing it SIMD_COEF_32 wise.
#if __SSE4_1__ || __MIC__
		unsigned k;
		vtype * W = w;
		ARCH_WORD_32 * saved_key = (ARCH_WORD_32 *) _data;
		MD4_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (i = 0; i < 16; ++i)
				{
					GATHER_4x(W[i], saved_key, i);
				}
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (i = 0; i < 16; ++i)
				{
					GATHER_2x(W[i], saved_key, i);
				}
				saved_key += (VS32 << 5);
			}
			else
			{
				for (i = 0; i < 16; ++i)
				{
					GATHER(W[i], saved_key, i);
				}
				saved_key += (VS32 << 4);
			}
			W += 16;
		}
#else
		unsigned j, k;
		ARCH_WORD_32 * p = (ARCH_WORD_32 *) w;
		vtype * W = w;
		ARCH_WORD_32 * saved_key = (ARCH_WORD_32 *) _data;
		MD4_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 6) + j];
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 5) + j];
				saved_key += (VS32 << 5);
			}
			else
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 4) + j];
				saved_key += (VS32 << 4);
			}
			W += 16;
		}
#endif
		// now set our data pointer to point to this 'mixed' data.
		data = w;
	}
	else
		data = _data;

	if ((SSEi_flags & SSEi_RELOAD) == 0)
	{
		MD4_PARA_DO(i)
		{
			a[i] = vset1_epi32(0x67452301);
			b[i] = vset1_epi32(0xefcdab89);
			c[i] = vset1_epi32(0x98badcfe);
			d[i] = vset1_epi32(0x10325476);
		}
	}
	else
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			MD4_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]);
			}
		}
		else
		{
			MD4_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 4 * VS32 + 3 * VS32]);
			}
		}
	}

	/* Round 1 */
	cst = vsetzero();
	MD4_STEP(MD4_F, a, b, c, d, 0, cst, 3)
	MD4_STEP(MD4_F, d, a, b, c, 1, cst, 7)
	MD4_STEP(MD4_F, c, d, a, b, 2, cst, 11)
	MD4_STEP(MD4_F, b, c, d, a, 3, cst, 19)
	MD4_STEP(MD4_F, a, b, c, d, 4, cst, 3)
	MD4_STEP(MD4_F, d, a, b, c, 5, cst, 7)
	MD4_STEP(MD4_F, c, d, a, b, 6, cst, 11)
	MD4_STEP(MD4_F, b, c, d, a, 7, cst, 19)
	MD4_STEP(MD4_F, a, b, c, d, 8, cst, 3)
	MD4_STEP(MD4_F, d, a, b, c, 9, cst, 7)
	MD4_STEP(MD4_F, c, d, a, b, 10, cst, 11)
	MD4_STEP(MD4_F, b, c, d, a, 11, cst, 19)
	MD4_STEP(MD4_F, a, b, c, d, 12, cst, 3)
	MD4_STEP(MD4_F, d, a, b, c, 13, cst, 7)
	MD4_STEP(MD4_F, c, d, a, b, 14, cst, 11)
	MD4_STEP(MD4_F, b, c, d, a, 15, cst, 19)

	/* Round 2 */
	cst = vset1_epi32(0x5A827999L);
	MD4_STEP(MD4_G, a, b, c, d, 0, cst, 3)
	MD4_STEP(MD4_G, d, a, b, c, 4, cst, 5)
	MD4_STEP(MD4_G, c, d, a, b, 8, cst, 9)
	MD4_STEP(MD4_G, b, c, d, a, 12, cst, 13)
	MD4_STEP(MD4_G, a, b, c, d, 1, cst, 3)
	MD4_STEP(MD4_G, d, a, b, c, 5, cst, 5)
	MD4_STEP(MD4_G, c, d, a, b, 9, cst, 9)
	MD4_STEP(MD4_G, b, c, d, a, 13, cst, 13)
	MD4_STEP(MD4_G, a, b, c, d, 2, cst, 3)
	MD4_STEP(MD4_G, d, a, b, c, 6, cst, 5)
	MD4_STEP(MD4_G, c, d, a, b, 10, cst, 9)
	MD4_STEP(MD4_G, b, c, d, a, 14, cst, 13)
	MD4_STEP(MD4_G, a, b, c, d, 3, cst, 3)
	MD4_STEP(MD4_G, d, a, b, c, 7, cst, 5)
	MD4_STEP(MD4_G, c, d, a, b, 11, cst, 9)
	MD4_STEP(MD4_G, b, c, d, a, 15, cst, 13)

	/* Round 3 */
	cst = vset1_epi32(0x6ED9EBA1L);
	MD4_STEP(MD4_H, a, b, c, d, 0, cst, 3)
	MD4_STEP(MD4_H2, d, a, b, c, 8, cst, 9)
	MD4_STEP(MD4_H, c, d, a, b, 4, cst, 11)
	MD4_STEP(MD4_H2, b, c, d, a, 12, cst, 15)
	MD4_STEP(MD4_H, a, b, c, d, 2, cst, 3)
	MD4_STEP(MD4_H2, d, a, b, c, 10, cst, 9)
	MD4_STEP(MD4_H, c, d, a, b, 6, cst, 11)
	MD4_STEP(MD4_H2, b, c, d, a, 14, cst, 15)
	MD4_STEP(MD4_H, a, b, c, d, 1, cst, 3)
	MD4_STEP(MD4_H2, d, a, b, c, 9, cst, 9)
	MD4_STEP(MD4_H, c, d, a, b, 5, cst, 11)

	if (SSEi_flags & SSEi_REVERSE_STEPS)
	{
		MD4_REV_STEP(MD4_H2, b, c, d, a, 13, cst, 15)
		MD4_PARA_DO(i)
		{
			vstore((vtype *) &out[i * 4 * VS32 + 1 * VS32], b[i]);
		}
		return;
	}

	MD4_STEP(MD4_H2, b, c, d, a, 13, cst, 15)
	MD4_STEP(MD4_H, a, b, c, d, 3, cst, 3)
	MD4_STEP(MD4_H2, d, a, b, c, 11, cst, 9)
	MD4_STEP(MD4_H, c, d, a, b, 7, cst, 11)
	MD4_STEP(MD4_H2, b, c, d, a, 15, cst, 15)

	if ((SSEi_flags & SSEi_RELOAD) == 0)
	{
		MD4_PARA_DO(i)
		{
			a[i] = vadd_epi32(a[i], vset1_epi32(0x67452301));
			b[i] = vadd_epi32(b[i], vset1_epi32(0xefcdab89));
			c[i] = vadd_epi32(c[i], vset1_epi32(0x98badcfe));
			d[i] = vadd_epi32(d[i], vset1_epi32(0x10325476));
		}
	}
	else
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			MD4_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]));
			}
		}
		else
		{
			MD4_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 4 * VS32 + 3 * VS32]));
			}
		}
	}

#if USE_EXPERIMENTAL
	/*
 * This is currently not used for MD4, and was observed to result
 * in a significant performance regression (at least on XOP) just by sitting
 * here. http://www.openwall.com/lists/john-dev/2015/09/05/5
 * NOTE the regression might be gone now anyway since we went from -O3 to -O2.
 */
	if (SSEi_flags & SSEi_FLAT_OUT)
	{
		MD4_PARA_DO(i)
		{
			uint32_t * o = (uint32_t *) &out[i * 4 * VS32];
#if __AVX512F__ || __MIC__
			vtype idxs = vset_epi32(15 * 4,
									14 * 4,
									13 * 4,
									12 * 4,
									11 * 4,
									10 * 4,
									9 * 4,
									8 * 4,
									7 * 4,
									6 * 4,
									5 * 4,
									4 * 4,
									3 * 4,
									2 * 4,
									1 * 4,
									0 * 4);

			vscatter_epi32(o + 0, idxs, a[i], 4);
			vscatter_epi32(o + 1, idxs, b[i], 4);
			vscatter_epi32(o + 2, idxs, c[i], 4);
			vscatter_epi32(o + 3, idxs, d[i], 4);
#else
			uint32_t j, k;
			union {
				vtype v[4];
				uint32_t s[4 * VS32];
			} tmp;

			tmp.v[0] = a[i];
			tmp.v[1] = b[i];
			tmp.v[2] = c[i];
			tmp.v[3] = d[i];

			for (j = 0; j < VS32; j++)
				for (k = 0; k < 4; k++) o[j * 4 + k] = tmp.s[k * VS32 + j];
#endif
		}
	}
	else
#endif
		if (SSEi_flags & SSEi_OUTPUT_AS_INP_FMT)
	{
		if ((SSEi_flags & SSEi_OUTPUT_AS_2BUF_INP_FMT)
			== SSEi_OUTPUT_AS_2BUF_INP_FMT)
		{
			MD4_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 32 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 3 * VS32], d[i]);
			}
		}
		else
		{
			MD4_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 16 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 3 * VS32], d[i]);
			}
		}
	}
	else
	{
		MD4_PARA_DO(i)
		{
			vstore((vtype *) &out[i * 4 * VS32 + 0 * VS32], a[i]);
			vstore((vtype *) &out[i * 4 * VS32 + 1 * VS32], b[i]);
			vstore((vtype *) &out[i * 4 * VS32 + 2 * VS32], c[i]);
			vstore((vtype *) &out[i * 4 * VS32 + 3 * VS32], d[i]);
		}
	}
}

#endif /* SIMD_PARA_MD4 */

#if SIMD_PARA_SHA1
#define SHA1_SSE_NUM_KEYS (SIMD_COEF_32 * SIMD_PARA_SHA1)
#define SHA1_PARA_DO(x) for ((x) = 0; (x) < SIMD_PARA_SHA1; (x)++)

#define SHA1_F(x, y, z) tmp[i] = vcmov((y[i]), (z[i]), (x[i]));

#if __AVX512F__
#define SHA1_G(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0x96);
#else
#define SHA1_G(x, y, z)                                                        \
	tmp[i] = vxor((y[i]), (z[i]));                                             \
	tmp[i] = vxor((tmp[i]), (x[i]));
#endif

#if __AVX512F__
#define SHA1_H(x, y, z) tmp[i] = vternarylogic(x[i], y[i], z[i], 0xE8);
#elif !VCMOV_EMULATED
#define SHA1_H(x, y, z)                                                        \
	tmp[i] = vxor((z[i]), (y[i]));                                             \
	tmp[i] = vcmov((x[i]), (y[i]), tmp[i]);
#else
#define SHA1_H(x, y, z)                                                        \
	tmp[i] = vand((x[i]), (y[i]));                                             \
	tmp[i] = vor((tmp[i]), vand(vor((x[i]), (y[i])), (z[i])));
#endif

#define SHA1_I(x, y, z) SHA1_G(x, y, z)

#define SHA1_EXPAND2a(t)                                                       \
	tmp[i] = vxor(data[i * 16 + t - 3], data[i * 16 + t - 8]);                 \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 14]);                              \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 16]);                              \
	w[i * 16 + ((t) &0xF)] = vroti_epi32(tmp[i], 1);

#define SHA1_EXPAND2b(t)                                                       \
	tmp[i] = vxor(w[i * 16 + ((t - 3) & 0xF)], data[i * 16 + t - 8]);          \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 14]);                              \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 16]);                              \
	w[i * 16 + ((t) &0xF)] = vroti_epi32(tmp[i], 1);

#define SHA1_EXPAND2c(t)                                                       \
	tmp[i] = vxor(w[i * 16 + ((t - 3) & 0xF)], w[i * 16 + ((t - 8) & 0xF)]);   \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 14]);                              \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 16]);                              \
	w[i * 16 + ((t) &0xF)] = vroti_epi32(tmp[i], 1);

#define SHA1_EXPAND2d(t)                                                       \
	tmp[i] = vxor(w[i * 16 + ((t - 3) & 0xF)], w[i * 16 + ((t - 8) & 0xF)]);   \
	tmp[i] = vxor(tmp[i], w[i * 16 + ((t - 14) & 0xF)]);                       \
	tmp[i] = vxor(tmp[i], data[i * 16 + t - 16]);                              \
	w[i * 16 + ((t) &0xF)] = vroti_epi32(tmp[i], 1);

#define SHA1_EXPAND2(t)                                                        \
	tmp[i] = vxor(w[i * 16 + ((t - 3) & 0xF)], w[i * 16 + ((t - 8) & 0xF)]);   \
	tmp[i] = vxor(tmp[i], w[i * 16 + ((t - 14) & 0xF)]);                       \
	tmp[i] = vxor(tmp[i], w[i * 16 + ((t - 16) & 0xF)]);                       \
	w[i * 16 + ((t) &0xF)] = vroti_epi32(tmp[i], 1);

#define SHA1_ROUND2a(a, b, c, d, e, F, t)                                      \
	SHA1_PARA_DO(i)                                                            \
	{                                                                          \
		F(b, c, d)                                                             \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		tmp[i] = vroti_epi32(a[i], 5);                                         \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		e[i] = vadd_epi32(e[i], cst);                                          \
		e[i] = vadd_epi32(e[i], data[i * 16 + t]);                             \
		b[i] = vroti_epi32(b[i], 30);                                          \
		SHA1_EXPAND2a(t + 16)                                                  \
	}

#define SHA1_ROUND2b(a, b, c, d, e, F, t)                                      \
	SHA1_PARA_DO(i)                                                            \
	{                                                                          \
		F(b, c, d)                                                             \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		tmp[i] = vroti_epi32(a[i], 5);                                         \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		e[i] = vadd_epi32(e[i], cst);                                          \
		e[i] = vadd_epi32(e[i], data[i * 16 + t]);                             \
		b[i] = vroti_epi32(b[i], 30);                                          \
		SHA1_EXPAND2b(t + 16)                                                  \
	}

#define SHA1_ROUND2c(a, b, c, d, e, F, t)                                      \
	SHA1_PARA_DO(i)                                                            \
	{                                                                          \
		F(b, c, d)                                                             \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		tmp[i] = vroti_epi32(a[i], 5);                                         \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		e[i] = vadd_epi32(e[i], cst);                                          \
		e[i] = vadd_epi32(e[i], data[i * 16 + t]);                             \
		b[i] = vroti_epi32(b[i], 30);                                          \
		SHA1_EXPAND2c(t + 16)                                                  \
	}

#define SHA1_ROUND2d(a, b, c, d, e, F, t)                                      \
	SHA1_PARA_DO(i)                                                            \
	{                                                                          \
		F(b, c, d)                                                             \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		tmp[i] = vroti_epi32(a[i], 5);                                         \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		e[i] = vadd_epi32(e[i], cst);                                          \
		e[i] = vadd_epi32(e[i], data[i * 16 + t]);                             \
		b[i] = vroti_epi32(b[i], 30);                                          \
		SHA1_EXPAND2d(t + 16)                                                  \
	}

#define SHA1_ROUND2(a, b, c, d, e, F, t)                                       \
	SHA1_PARA_DO(i)                                                            \
	{                                                                          \
		F(b, c, d)                                                             \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		tmp[i] = vroti_epi32(a[i], 5);                                         \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		e[i] = vadd_epi32(e[i], cst);                                          \
		e[i] = vadd_epi32(e[i], w[i * 16 + (t & 0xF)]);                        \
		b[i] = vroti_epi32(b[i], 30);                                          \
		SHA1_EXPAND2(t + 16)                                                   \
	}

#define SHA1_ROUND2x(a, b, c, d, e, F, t)                                      \
	SHA1_PARA_DO(i)                                                            \
	{                                                                          \
		F(b, c, d)                                                             \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		tmp[i] = vroti_epi32(a[i], 5);                                         \
		e[i] = vadd_epi32(e[i], tmp[i]);                                       \
		e[i] = vadd_epi32(e[i], cst);                                          \
		e[i] = vadd_epi32(e[i], w[i * 16 + (t & 0xF)]);                        \
		b[i] = vroti_epi32(b[i], 30);                                          \
	}

#define INIT_E 0xC3D2E1F0

void sha1_reverse(uint32_t * hash)
{
	hash[4] -= INIT_E;
	hash[4] = (hash[4] << 2) | (hash[4] >> 30);
}

void sha1_unreverse(uint32_t * hash)
{
	hash[4] = (hash[4] << 30) | (hash[4] >> 2);
	hash[4] += INIT_E;
}

#undef INIT_E

// SSEi_MIXED_IN | SSEi_RELOAD | SSEi_OUTPUT_AS_INP_FMT

void SIMDSHA1body(vtype * _data,
				  ARCH_WORD_32 * out,
				  ARCH_WORD_32 * reload_state,
				  unsigned SSEi_flags)
{
#if __ALTIVEC__
	vtype w[16 * SIMD_PARA_SHA1] = {{{0}}};
	vtype a[SIMD_PARA_SHA1] = {{{0}}};
	vtype b[SIMD_PARA_SHA1] = {{{0}}};
	vtype c[SIMD_PARA_SHA1] = {{{0}}};
	vtype d[SIMD_PARA_SHA1] = {{{0}}};
	vtype e[SIMD_PARA_SHA1] = {{{0}}};
#else
	vtype w[16 * SIMD_PARA_SHA1] = {(vtype){0}};
	vtype a[SIMD_PARA_SHA1] = {(vtype){0}};
	vtype b[SIMD_PARA_SHA1] = {(vtype){0}};
	vtype c[SIMD_PARA_SHA1] = {(vtype){0}};
	vtype d[SIMD_PARA_SHA1] = {(vtype){0}};
	vtype e[SIMD_PARA_SHA1] = {(vtype){0}};
#endif
	vtype tmp[SIMD_PARA_SHA1];
	vtype cst;
	unsigned int i;
	vtype * data;

#ifdef NDEBUG
	(void) SSEi_flags;
#endif

#if 0
	if (SSEi_flags & SSEi_FLAT_IN)
	{
// Move _data to __data, mixing it SIMD_COEF_32 wise.
#if __SSE4_1__ || __MIC__
		unsigned k;
		vtype * W = w;
		ARCH_WORD_32 * saved_key = (ARCH_WORD_32 *) _data;
		SHA1_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (i = 0; i < 14; ++i)
				{
					GATHER_4x(W[i], saved_key, i);
					vswap32(W[i]);
				}
				GATHER_4x(W[14], saved_key, 14);
				GATHER_4x(W[15], saved_key, 15);
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (i = 0; i < 14; ++i)
				{
					GATHER_2x(W[i], saved_key, i);
					vswap32(W[i]);
				}
				GATHER_2x(W[14], saved_key, 14);
				GATHER_2x(W[15], saved_key, 15);
				saved_key += (VS32 << 5);
			}
			else
			{
				for (i = 0; i < 14; ++i)
				{
					GATHER(W[i], saved_key, i);
					vswap32(W[i]);
				}
				GATHER(W[14], saved_key, 14);
				GATHER(W[15], saved_key, 15);
				saved_key += (VS32 << 4);
			}
			if (((SSEi_flags & SSEi_2BUF_INPUT_FIRST_BLK)
				 == SSEi_2BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_4BUF_INPUT_FIRST_BLK)
					== SSEi_4BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_FLAT_RELOAD_SWAPLAST)
					== SSEi_FLAT_RELOAD_SWAPLAST))
			{
				vswap32(W[14]);
				vswap32(W[15]);
			}
			W += 16;
		}
#else
		unsigned j, k;
		ARCH_WORD_32 * p = (ARCH_WORD_32 *) w;
		vtype * W = w;
		ARCH_WORD_32 * saved_key = (ARCH_WORD_32 *) _data;
		SHA1_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 6) + j];
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 5) + j];
				saved_key += (VS32 << 5);
			}
			else
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 4) + j];
				saved_key += (VS32 << 4);
			}
			for (i = 0; i < 14; i++) vswap32(W[i]);
			if (((SSEi_flags & SSEi_2BUF_INPUT_FIRST_BLK)
				 == SSEi_2BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_4BUF_INPUT_FIRST_BLK)
					== SSEi_4BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_FLAT_RELOAD_SWAPLAST)
					== SSEi_FLAT_RELOAD_SWAPLAST))
			{
				vswap32(W[14]);
				vswap32(W[15]);
			}
			W += 16;
		}
#endif

		// now set our data pointer to point to this 'mixed' data.
		data = w;
	}
	else
#else
	assert((SSEi_flags & SSEi_FLAT_IN) == 0);
#endif
	data = _data;

#if 0
	if ((SSEi_flags & SSEi_RELOAD) == 0)
	{
		SHA1_PARA_DO(i)
		{
			a[i] = vset1_epi32(0x67452301);
			b[i] = vset1_epi32(0xefcdab89);
			c[i] = vset1_epi32(0x98badcfe);
			d[i] = vset1_epi32(0x10325476);
			e[i] = vset1_epi32(0xC3D2E1F0);
		}
	}
	else
	{
#endif
	assert((SSEi_flags & SSEi_RELOAD) != 0);

#if 0
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			SHA1_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]);
				e[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 4 * VS32]);
			}
		}
		else
		{
#endif
	SHA1_PARA_DO(i)
	{
		a[i] = vload((vtype *) &reload_state[i * 5 * VS32 + 0 * VS32]);
		b[i] = vload((vtype *) &reload_state[i * 5 * VS32 + 1 * VS32]);
		c[i] = vload((vtype *) &reload_state[i * 5 * VS32 + 2 * VS32]);
		d[i] = vload((vtype *) &reload_state[i * 5 * VS32 + 3 * VS32]);
		e[i] = vload((vtype *) &reload_state[i * 5 * VS32 + 4 * VS32]);
	}
#if 0
		}
#endif
#if 0
	}
#endif

	cst = vset1_epi32(0x5A827999);
	SHA1_ROUND2a(a, b, c, d, e, SHA1_F, 0);
	SHA1_ROUND2a(e, a, b, c, d, SHA1_F, 1);
	SHA1_ROUND2a(d, e, a, b, c, SHA1_F, 2);
	SHA1_ROUND2b(c, d, e, a, b, SHA1_F, 3);
	SHA1_ROUND2b(b, c, d, e, a, SHA1_F, 4);
	SHA1_ROUND2b(a, b, c, d, e, SHA1_F, 5);
	SHA1_ROUND2b(e, a, b, c, d, SHA1_F, 6);
	SHA1_ROUND2b(d, e, a, b, c, SHA1_F, 7);
	SHA1_ROUND2c(c, d, e, a, b, SHA1_F, 8);
	SHA1_ROUND2c(b, c, d, e, a, SHA1_F, 9);
	SHA1_ROUND2c(a, b, c, d, e, SHA1_F, 10);
	SHA1_ROUND2c(e, a, b, c, d, SHA1_F, 11);
	SHA1_ROUND2c(d, e, a, b, c, SHA1_F, 12);
	SHA1_ROUND2c(c, d, e, a, b, SHA1_F, 13);
	SHA1_ROUND2d(b, c, d, e, a, SHA1_F, 14);
	SHA1_ROUND2d(a, b, c, d, e, SHA1_F, 15);
	SHA1_ROUND2(e, a, b, c, d, SHA1_F, 16);
	SHA1_ROUND2(d, e, a, b, c, SHA1_F, 17);
	SHA1_ROUND2(c, d, e, a, b, SHA1_F, 18);
	SHA1_ROUND2(b, c, d, e, a, SHA1_F, 19);

	cst = vset1_epi32(0x6ED9EBA1);
	SHA1_ROUND2(a, b, c, d, e, SHA1_G, 20);
	SHA1_ROUND2(e, a, b, c, d, SHA1_G, 21);
	SHA1_ROUND2(d, e, a, b, c, SHA1_G, 22);
	SHA1_ROUND2(c, d, e, a, b, SHA1_G, 23);
	SHA1_ROUND2(b, c, d, e, a, SHA1_G, 24);
	SHA1_ROUND2(a, b, c, d, e, SHA1_G, 25);
	SHA1_ROUND2(e, a, b, c, d, SHA1_G, 26);
	SHA1_ROUND2(d, e, a, b, c, SHA1_G, 27);
	SHA1_ROUND2(c, d, e, a, b, SHA1_G, 28);
	SHA1_ROUND2(b, c, d, e, a, SHA1_G, 29);
	SHA1_ROUND2(a, b, c, d, e, SHA1_G, 30);
	SHA1_ROUND2(e, a, b, c, d, SHA1_G, 31);
	SHA1_ROUND2(d, e, a, b, c, SHA1_G, 32);
	SHA1_ROUND2(c, d, e, a, b, SHA1_G, 33);
	SHA1_ROUND2(b, c, d, e, a, SHA1_G, 34);
	SHA1_ROUND2(a, b, c, d, e, SHA1_G, 35);
	SHA1_ROUND2(e, a, b, c, d, SHA1_G, 36);
	SHA1_ROUND2(d, e, a, b, c, SHA1_G, 37);
	SHA1_ROUND2(c, d, e, a, b, SHA1_G, 38);
	SHA1_ROUND2(b, c, d, e, a, SHA1_G, 39);

	cst = vset1_epi32(0x8F1BBCDC);
	SHA1_ROUND2(a, b, c, d, e, SHA1_H, 40);
	SHA1_ROUND2(e, a, b, c, d, SHA1_H, 41);
	SHA1_ROUND2(d, e, a, b, c, SHA1_H, 42);
	SHA1_ROUND2(c, d, e, a, b, SHA1_H, 43);
	SHA1_ROUND2(b, c, d, e, a, SHA1_H, 44);
	SHA1_ROUND2(a, b, c, d, e, SHA1_H, 45);
	SHA1_ROUND2(e, a, b, c, d, SHA1_H, 46);
	SHA1_ROUND2(d, e, a, b, c, SHA1_H, 47);
	SHA1_ROUND2(c, d, e, a, b, SHA1_H, 48);
	SHA1_ROUND2(b, c, d, e, a, SHA1_H, 49);
	SHA1_ROUND2(a, b, c, d, e, SHA1_H, 50);
	SHA1_ROUND2(e, a, b, c, d, SHA1_H, 51);
	SHA1_ROUND2(d, e, a, b, c, SHA1_H, 52);
	SHA1_ROUND2(c, d, e, a, b, SHA1_H, 53);
	SHA1_ROUND2(b, c, d, e, a, SHA1_H, 54);
	SHA1_ROUND2(a, b, c, d, e, SHA1_H, 55);
	SHA1_ROUND2(e, a, b, c, d, SHA1_H, 56);
	SHA1_ROUND2(d, e, a, b, c, SHA1_H, 57);
	SHA1_ROUND2(c, d, e, a, b, SHA1_H, 58);
	SHA1_ROUND2(b, c, d, e, a, SHA1_H, 59);

	cst = vset1_epi32(0xCA62C1D6);
	SHA1_ROUND2(a, b, c, d, e, SHA1_I, 60);
	SHA1_ROUND2(e, a, b, c, d, SHA1_I, 61);
	SHA1_ROUND2(d, e, a, b, c, SHA1_I, 62);
	SHA1_ROUND2(c, d, e, a, b, SHA1_I, 63);
	SHA1_ROUND2x(b, c, d, e, a, SHA1_I, 64);
	SHA1_ROUND2x(a, b, c, d, e, SHA1_I, 65);
	SHA1_ROUND2x(e, a, b, c, d, SHA1_I, 66);
	SHA1_ROUND2x(d, e, a, b, c, SHA1_I, 67);
	SHA1_ROUND2x(c, d, e, a, b, SHA1_I, 68);
	SHA1_ROUND2x(b, c, d, e, a, SHA1_I, 69);
	SHA1_ROUND2x(a, b, c, d, e, SHA1_I, 70);
	SHA1_ROUND2x(e, a, b, c, d, SHA1_I, 71);
	SHA1_ROUND2x(d, e, a, b, c, SHA1_I, 72);
	SHA1_ROUND2x(c, d, e, a, b, SHA1_I, 73);
	SHA1_ROUND2x(b, c, d, e, a, SHA1_I, 74);
	SHA1_ROUND2x(a, b, c, d, e, SHA1_I, 75);

#if 0
	if (SSEi_flags & SSEi_REVERSE_STEPS)
	{
		SHA1_PARA_DO(i)
		{
			vstore((vtype *) &out[i * 5 * VS32 + 4 * VS32], e[i]);
		}
		return;
	}
#endif

	SHA1_ROUND2x(e, a, b, c, d, SHA1_I, 76);
	SHA1_ROUND2x(d, e, a, b, c, SHA1_I, 77);
	SHA1_ROUND2x(c, d, e, a, b, SHA1_I, 78);
	SHA1_ROUND2x(b, c, d, e, a, SHA1_I, 79);

#if 0
	if ((SSEi_flags & SSEi_RELOAD) == 0)
	{
		SHA1_PARA_DO(i)
		{
			a[i] = vadd_epi32(a[i], vset1_epi32(0x67452301));
			b[i] = vadd_epi32(b[i], vset1_epi32(0xefcdab89));
			c[i] = vadd_epi32(c[i], vset1_epi32(0x98badcfe));
			d[i] = vadd_epi32(d[i], vset1_epi32(0x10325476));
			e[i] = vadd_epi32(e[i], vset1_epi32(0xC3D2E1F0));
		}
	}
	else
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			SHA1_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]));
				e[i] = vadd_epi32(
					e[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 4 * VS32]));
			}
		}
		else
		{
#endif
	SHA1_PARA_DO(i)
	{
		a[i] = vadd_epi32(
			a[i], vload((vtype *) &reload_state[i * 5 * VS32 + 0 * VS32]));
		b[i] = vadd_epi32(
			b[i], vload((vtype *) &reload_state[i * 5 * VS32 + 1 * VS32]));
		c[i] = vadd_epi32(
			c[i], vload((vtype *) &reload_state[i * 5 * VS32 + 2 * VS32]));
		d[i] = vadd_epi32(
			d[i], vload((vtype *) &reload_state[i * 5 * VS32 + 3 * VS32]));
		e[i] = vadd_epi32(
			e[i], vload((vtype *) &reload_state[i * 5 * VS32 + 4 * VS32]));
	}
#if 0
		}
	}
#endif

#if 0
	if (SSEi_flags & SSEi_FLAT_OUT)
	{
		SHA1_PARA_DO(i)
		{
			uint32_t * o = (uint32_t *) &out[i * 5 * VS32];
#if __AVX512F__ || __MIC__
			vtype idxs = vset_epi32(15 * 5,
									14 * 5,
									13 * 5,
									12 * 5,
									11 * 5,
									10 * 5,
									9 * 5,
									8 * 5,
									7 * 5,
									6 * 5,
									5 * 5,
									4 * 5,
									3 * 5,
									2 * 5,
									1 * 5,
									0 * 5);

			vscatter_epi32((((uint32_t *) o) + 0), idxs, vswap32(a[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 1), idxs, vswap32(b[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 2), idxs, vswap32(c[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 3), idxs, vswap32(d[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 4), idxs, vswap32(e[i]), 4);
#else
			uint32_t j, k;
			union {
				vtype v[5];
				uint32_t s[5 * VS32];
			} tmp;

			tmp.v[0] = vswap32(a[i]);
			tmp.v[1] = vswap32(b[i]);
			tmp.v[2] = vswap32(c[i]);
			tmp.v[3] = vswap32(d[i]);
			tmp.v[4] = vswap32(e[i]);

			for (j = 0; j < VS32; j++)
				for (k = 0; k < 5; k++) o[j * 5 + k] = tmp.s[k * VS32 + j];
#endif
		}
	}
	else if (SSEi_flags & SSEi_OUTPUT_AS_INP_FMT)
	{
		if ((SSEi_flags & SSEi_OUTPUT_AS_2BUF_INP_FMT)
			== SSEi_OUTPUT_AS_2BUF_INP_FMT)
		{
			SHA1_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 32 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 3 * VS32], d[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 4 * VS32], e[i]);
			}
		}
		else
		{
#endif
	assert((SSEi_flags & SSEi_OUTPUT_AS_INP_FMT) != 0
		   && (SSEi_flags & SSEi_OUTPUT_AS_2BUF_INP_FMT)
				  != SSEi_OUTPUT_AS_2BUF_INP_FMT);

	SHA1_PARA_DO(i)
	{
		vstore((vtype *) &out[i * 16 * VS32 + 0 * VS32], a[i]);
		vstore((vtype *) &out[i * 16 * VS32 + 1 * VS32], b[i]);
		vstore((vtype *) &out[i * 16 * VS32 + 2 * VS32], c[i]);
		vstore((vtype *) &out[i * 16 * VS32 + 3 * VS32], d[i]);
		vstore((vtype *) &out[i * 16 * VS32 + 4 * VS32], e[i]);
	}
#if 0
		}
	}
	else
	{
		SHA1_PARA_DO(i)
		{
			vstore((vtype *) &out[i * 5 * VS32 + 0 * VS32], a[i]);
			vstore((vtype *) &out[i * 5 * VS32 + 1 * VS32], b[i]);
			vstore((vtype *) &out[i * 5 * VS32 + 2 * VS32], c[i]);
			vstore((vtype *) &out[i * 5 * VS32 + 3 * VS32], d[i]);
			vstore((vtype *) &out[i * 5 * VS32 + 4 * VS32], e[i]);
		}
	}
#endif

#if defined(__AVX2__) || defined(__AVX512F__)
	/* The fix for known issues with mixing AVX2+ and SSE. */
	_mm256_zeroupper();
#endif
}
#endif /* SIMD_PARA_SHA1 */

#if SIMD_PARA_SHA256

/*
 * These optimized Sigma alternatives are from "Fast SHA-256 Implementations
 * on Intel Architecture Processors" whitepaper by Intel. They should result
 * in less register copy operations but in our case they definitely cause a
 * regression. Not sure why.
 */
#if 0
#define S0(x)                                                                  \
	vroti_epi32(vxor(vroti_epi32(vxor(vroti_epi32(x, -9), x), -11), x), -2)
#define S1(x)                                                                  \
	vroti_epi32(vxor(vroti_epi32(vxor(vroti_epi32(x, -14), x), -5), x), -6)
#else
#define S0(x)                                                                  \
	(vxor(vroti_epi32(x, -22), vxor(vroti_epi32(x, -2), vroti_epi32(x, -13))))

#define S1(x)                                                                  \
	(vxor(vroti_epi32(x, -25), vxor(vroti_epi32(x, -6), vroti_epi32(x, -11))))
#endif

#define s0(x)                                                                  \
	(vxor(vsrli_epi32(x, 3), vxor(vroti_epi32(x, -7), vroti_epi32(x, -18))))

#define s1(x)                                                                  \
	(vxor(vsrli_epi32(x, 10), vxor(vroti_epi32(x, -17), vroti_epi32(x, -19))))

#if __AVX512F__
#define Maj(x, y, z) vternarylogic(x, y, z, 0xE8)
#elif !VCMOV_EMULATED
#define Maj(x, y, z) vcmov(x, y, vxor(z, y))
#else
#define Maj(x, y, z) vor(vand(x, y), vand(vor(x, y), z))
#endif

#define Ch(x, y, z) vcmov(y, z, x)

#undef R
#define R(t)                                                                   \
	{                                                                          \
		tmp1[i] = vadd_epi32(s1(w[(t - 2) & 0xf]), w[(t - 7) & 0xf]);          \
		tmp2[i] = vadd_epi32(s0(w[(t - 15) & 0xf]), w[(t - 16) & 0xf]);        \
		w[(t) &0xf] = vadd_epi32(tmp1[i], tmp2[i]);                            \
	}

#define SHA256_PARA_DO(x) for (x = 0; x < SIMD_PARA_SHA256; ++x)

#define SHA256_STEP(a, b, c, d, e, f, g, h, x, K)                              \
	{                                                                          \
		SHA256_PARA_DO(i)                                                      \
		{                                                                      \
			w = _w[i].w;                                                       \
			tmp1[i] = vadd_epi32(h[i], S1(e[i]));                              \
			tmp1[i] = vadd_epi32(tmp1[i], Ch(e[i], f[i], g[i]));               \
			tmp1[i] = vadd_epi32(tmp1[i], vset1_epi32(K));                     \
			tmp1[i] = vadd_epi32(tmp1[i], w[(x) &0xf]);                        \
			tmp2[i] = vadd_epi32(S0(a[i]), Maj(a[i], b[i], c[i]));             \
			d[i] = vadd_epi32(tmp1[i], d[i]);                                  \
			h[i] = vadd_epi32(tmp1[i], tmp2[i]);                               \
			if (x < 48) R(x);                                                  \
		}                                                                      \
	}

#define INIT_A 0x6a09e667
#define INIT_B 0xbb67ae85
#define INIT_C 0x3c6ef372
#define INIT_D 0xa54ff53a
#define INIT_E 0x510e527f
#define INIT_F 0x9b05688c
#define INIT_G 0x1f83d9ab
#define INIT_H 0x5be0cd19

#define ror(x, n) (((x) >> (n)) | ((x) << (32 - (n))))

void sha256_reverse(uint32_t * hash)
{
	uint32_t a, b, c, d, e, f, g, h, s0, maj, tmp;

	a = hash[0] - INIT_A;
	b = hash[1] - INIT_B;
	c = hash[2] - INIT_C;
	d = hash[3] - INIT_D;
	e = hash[4] - INIT_E;
	f = hash[5] - INIT_F;
	g = hash[6] - INIT_G;
	h = hash[7] - INIT_H;

	s0 = ror(b, 2) ^ ror(b, 13) ^ ror(b, 22);
	maj = (b & c) ^ (b & d) ^ (c & d);
	tmp = d;
	d = e - (a - (s0 + maj));
	e = f;
	f = g;
	g = h;
	a = b;
	b = c;
	c = tmp;

	s0 = ror(b, 2) ^ ror(b, 13) ^ ror(b, 22);
	maj = (b & c) ^ (b & d) ^ (c & d);
	tmp = d;
	d = e - (a - (s0 + maj));
	e = f;
	f = g;
	a = b;
	b = c;
	c = tmp;

	s0 = ror(b, 2) ^ ror(b, 13) ^ ror(b, 22);
	maj = (b & c) ^ (b & d) ^ (c & d);
	tmp = d;
	d = e - (a - (s0 + maj));
	e = f;
	a = b;
	b = c;
	c = tmp;

	s0 = ror(b, 2) ^ ror(b, 13) ^ ror(b, 22);
	maj = (b & c) ^ (b & d) ^ (c & d);

	hash[0] = e - (a - (s0 + maj));
}

void sha256_unreverse(void)
{
	fprintf(stderr, "sha256_unreverse() not implemented\n");
	perror("sha256_unreverse");
}

#undef ror

#undef INIT_H
#undef INIT_G
#undef INIT_F
#undef INIT_E
#undef INIT_D
#undef INIT_C
#undef INIT_B
#undef INIT_A

#define INIT_D 0xf70e5939

void sha224_reverse(uint32_t * hash) { hash[3] -= INIT_D; }

void sha224_unreverse(uint32_t * hash) { hash[3] += INIT_D; }

#undef INIT_D

void SIMDSHA256body(vtype * data,
					ARCH_WORD_32 * out,
					ARCH_WORD_32 * reload_state,
					unsigned SSEi_flags)
{
	vtype a[SIMD_PARA_SHA256], b[SIMD_PARA_SHA256], c[SIMD_PARA_SHA256],
		d[SIMD_PARA_SHA256], e[SIMD_PARA_SHA256], f[SIMD_PARA_SHA256],
		g[SIMD_PARA_SHA256], h[SIMD_PARA_SHA256];
	union {
		vtype w[16];
		ARCH_WORD_32 p[16 * sizeof(vtype) / sizeof(ARCH_WORD_32)];
	} _w[SIMD_PARA_SHA256];
	vtype tmp1[SIMD_PARA_SHA256], tmp2[SIMD_PARA_SHA256], *w = NULL;
	ARCH_WORD_32 * saved_key = 0;

	unsigned int i, k;
	if (SSEi_flags & SSEi_FLAT_IN)
	{

#if __SSE4_1__ || __MIC__
		saved_key = (ARCH_WORD_32 *) data;
		SHA256_PARA_DO(k)
		{
			w = _w[k].w;
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (i = 0; i < 14; ++i)
				{
					GATHER_4x(w[i], saved_key, i);
					vswap32(w[i]);
				}
				GATHER_4x(w[14], saved_key, 14);
				GATHER_4x(w[15], saved_key, 15);
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (i = 0; i < 14; ++i)
				{
					GATHER_2x(w[i], saved_key, i);
					vswap32(w[i]);
				}
				GATHER_2x(w[14], saved_key, 14);
				GATHER_2x(w[15], saved_key, 15);
				saved_key += (VS32 << 5);
			}
			else
			{
				for (i = 0; i < 14; ++i)
				{
					GATHER(w[i], saved_key, i);
					vswap32(w[i]);
				}
				GATHER(w[14], saved_key, 14);
				GATHER(w[15], saved_key, 15);
				saved_key += (VS32 << 4);
			}
			if (((SSEi_flags & SSEi_2BUF_INPUT_FIRST_BLK)
				 == SSEi_2BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_4BUF_INPUT_FIRST_BLK)
					== SSEi_4BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_FLAT_RELOAD_SWAPLAST)
					== SSEi_FLAT_RELOAD_SWAPLAST))
			{
				vswap32(w[14]);
				vswap32(w[15]);
			}
		}
#else
		unsigned int j;
		saved_key = (ARCH_WORD_32 *) data;
		SHA256_PARA_DO(k)
		{
			ARCH_WORD_32 * p = _w[k].p;
			w = _w[k].w;
			if (SSEi_flags & SSEi_4BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 6) + j];
				saved_key += (VS32 << 6);
			}
			else if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 5) + j];
				saved_key += (VS32 << 5);
			}
			else
			{
				for (j = 0; j < 16; j++)
					for (i = 0; i < VS32; i++) *p++ = saved_key[(i << 4) + j];
				saved_key += (VS32 << 4);
			}
			for (i = 0; i < 14; i++) vswap32(w[i]);
			if (((SSEi_flags & SSEi_2BUF_INPUT_FIRST_BLK)
				 == SSEi_2BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_4BUF_INPUT_FIRST_BLK)
					== SSEi_4BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_FLAT_RELOAD_SWAPLAST)
					== SSEi_FLAT_RELOAD_SWAPLAST))
			{
				vswap32(w[14]);
				vswap32(w[15]);
			}
		}
#endif
	}
	else
		memcpy(_w, data, 16 * sizeof(vtype) * SIMD_PARA_SHA256);

	//	dump_stuff_shammx(w, 64, 0);

	if (SSEi_flags & SSEi_RELOAD)
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			SHA256_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]);
				e[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 4 * VS32]);
				f[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 5 * VS32]);
				g[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 6 * VS32]);
				h[i] = vload((vtype *) &reload_state[i * 16 * VS32 + 7 * VS32]);
			}
		}
		else
		{
			SHA256_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 0 * VS32]);
				b[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 1 * VS32]);
				c[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 2 * VS32]);
				d[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 3 * VS32]);
				e[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 4 * VS32]);
				f[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 5 * VS32]);
				g[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 6 * VS32]);
				h[i] = vload((vtype *) &reload_state[i * 8 * VS32 + 7 * VS32]);
			}
		}
	}
	else
	{
		if (SSEi_flags & SSEi_CRYPT_SHA224)
		{
			SHA256_PARA_DO(i)
			{
				/* SHA-224 IV */
				a[i] = vset1_epi32(0xc1059ed8);
				b[i] = vset1_epi32(0x367cd507);
				c[i] = vset1_epi32(0x3070dd17);
				d[i] = vset1_epi32(0xf70e5939);
				e[i] = vset1_epi32(0xffc00b31);
				f[i] = vset1_epi32(0x68581511);
				g[i] = vset1_epi32(0x64f98fa7);
				h[i] = vset1_epi32(0xbefa4fa4);
			}
		}
		else
		{
			SHA256_PARA_DO(i)
			{
				// SHA-256 IV */
				a[i] = vset1_epi32(0x6a09e667);
				b[i] = vset1_epi32(0xbb67ae85);
				c[i] = vset1_epi32(0x3c6ef372);
				d[i] = vset1_epi32(0xa54ff53a);
				e[i] = vset1_epi32(0x510e527f);
				f[i] = vset1_epi32(0x9b05688c);
				g[i] = vset1_epi32(0x1f83d9ab);
				h[i] = vset1_epi32(0x5be0cd19);
			}
		}
	}

	SHA256_STEP(a, b, c, d, e, f, g, h, 0, 0x428a2f98);
	SHA256_STEP(h, a, b, c, d, e, f, g, 1, 0x71374491);
	SHA256_STEP(g, h, a, b, c, d, e, f, 2, 0xb5c0fbcf);
	SHA256_STEP(f, g, h, a, b, c, d, e, 3, 0xe9b5dba5);
	SHA256_STEP(e, f, g, h, a, b, c, d, 4, 0x3956c25b);
	SHA256_STEP(d, e, f, g, h, a, b, c, 5, 0x59f111f1);
	SHA256_STEP(c, d, e, f, g, h, a, b, 6, 0x923f82a4);
	SHA256_STEP(b, c, d, e, f, g, h, a, 7, 0xab1c5ed5);
	SHA256_STEP(a, b, c, d, e, f, g, h, 8, 0xd807aa98);
	SHA256_STEP(h, a, b, c, d, e, f, g, 9, 0x12835b01);
	SHA256_STEP(g, h, a, b, c, d, e, f, 10, 0x243185be);
	SHA256_STEP(f, g, h, a, b, c, d, e, 11, 0x550c7dc3);
	SHA256_STEP(e, f, g, h, a, b, c, d, 12, 0x72be5d74);
	SHA256_STEP(d, e, f, g, h, a, b, c, 13, 0x80deb1fe);
	SHA256_STEP(c, d, e, f, g, h, a, b, 14, 0x9bdc06a7);
	SHA256_STEP(b, c, d, e, f, g, h, a, 15, 0xc19bf174);

	SHA256_STEP(a, b, c, d, e, f, g, h, 16, 0xe49b69c1);
	SHA256_STEP(h, a, b, c, d, e, f, g, 17, 0xefbe4786);
	SHA256_STEP(g, h, a, b, c, d, e, f, 18, 0x0fc19dc6);
	SHA256_STEP(f, g, h, a, b, c, d, e, 19, 0x240ca1cc);
	SHA256_STEP(e, f, g, h, a, b, c, d, 20, 0x2de92c6f);
	SHA256_STEP(d, e, f, g, h, a, b, c, 21, 0x4a7484aa);
	SHA256_STEP(c, d, e, f, g, h, a, b, 22, 0x5cb0a9dc);
	SHA256_STEP(b, c, d, e, f, g, h, a, 23, 0x76f988da);
	SHA256_STEP(a, b, c, d, e, f, g, h, 24, 0x983e5152);
	SHA256_STEP(h, a, b, c, d, e, f, g, 25, 0xa831c66d);
	SHA256_STEP(g, h, a, b, c, d, e, f, 26, 0xb00327c8);
	SHA256_STEP(f, g, h, a, b, c, d, e, 27, 0xbf597fc7);
	SHA256_STEP(e, f, g, h, a, b, c, d, 28, 0xc6e00bf3);
	SHA256_STEP(d, e, f, g, h, a, b, c, 29, 0xd5a79147);
	SHA256_STEP(c, d, e, f, g, h, a, b, 30, 0x06ca6351);
	SHA256_STEP(b, c, d, e, f, g, h, a, 31, 0x14292967);

	SHA256_STEP(a, b, c, d, e, f, g, h, 32, 0x27b70a85);
	SHA256_STEP(h, a, b, c, d, e, f, g, 33, 0x2e1b2138);
	SHA256_STEP(g, h, a, b, c, d, e, f, 34, 0x4d2c6dfc);
	SHA256_STEP(f, g, h, a, b, c, d, e, 35, 0x53380d13);
	SHA256_STEP(e, f, g, h, a, b, c, d, 36, 0x650a7354);
	SHA256_STEP(d, e, f, g, h, a, b, c, 37, 0x766a0abb);
	SHA256_STEP(c, d, e, f, g, h, a, b, 38, 0x81c2c92e);
	SHA256_STEP(b, c, d, e, f, g, h, a, 39, 0x92722c85);
	SHA256_STEP(a, b, c, d, e, f, g, h, 40, 0xa2bfe8a1);
	SHA256_STEP(h, a, b, c, d, e, f, g, 41, 0xa81a664b);
	SHA256_STEP(g, h, a, b, c, d, e, f, 42, 0xc24b8b70);
	SHA256_STEP(f, g, h, a, b, c, d, e, 43, 0xc76c51a3);
	SHA256_STEP(e, f, g, h, a, b, c, d, 44, 0xd192e819);
	SHA256_STEP(d, e, f, g, h, a, b, c, 45, 0xd6990624);
	SHA256_STEP(c, d, e, f, g, h, a, b, 46, 0xf40e3585);
	SHA256_STEP(b, c, d, e, f, g, h, a, 47, 0x106aa070);

	SHA256_STEP(a, b, c, d, e, f, g, h, 48, 0x19a4c116);
	SHA256_STEP(h, a, b, c, d, e, f, g, 49, 0x1e376c08);
	SHA256_STEP(g, h, a, b, c, d, e, f, 50, 0x2748774c);
	SHA256_STEP(f, g, h, a, b, c, d, e, 51, 0x34b0bcb5);
	SHA256_STEP(e, f, g, h, a, b, c, d, 52, 0x391c0cb3);
	SHA256_STEP(d, e, f, g, h, a, b, c, 53, 0x4ed8aa4a);
	SHA256_STEP(c, d, e, f, g, h, a, b, 54, 0x5b9cca4f);
	SHA256_STEP(b, c, d, e, f, g, h, a, 55, 0x682e6ff3);
	SHA256_STEP(a, b, c, d, e, f, g, h, 56, 0x748f82ee);

	if (SSEi_flags & SSEi_REVERSE_STEPS && !(SSEi_flags & SSEi_CRYPT_SHA224))
	{
		SHA256_PARA_DO(i)
		{
			vstore((vtype *) &(out[i * 8 * VS32 + 0 * VS32]), h[i]);
		}
		return;
	}

	SHA256_STEP(h, a, b, c, d, e, f, g, 57, 0x78a5636f);
	SHA256_STEP(g, h, a, b, c, d, e, f, 58, 0x84c87814);
	SHA256_STEP(f, g, h, a, b, c, d, e, 59, 0x8cc70208);
	SHA256_STEP(e, f, g, h, a, b, c, d, 60, 0x90befffa);

	if (SSEi_flags & SSEi_REVERSE_STEPS)
	{
		SHA256_PARA_DO(i)
		{
			vstore((vtype *) &(out[i * 8 * VS32 + 3 * VS32]), d[i]);
		}
		return;
	}

	SHA256_STEP(d, e, f, g, h, a, b, c, 61, 0xa4506ceb);
	SHA256_STEP(c, d, e, f, g, h, a, b, 62, 0xbef9a3f7);
	SHA256_STEP(b, c, d, e, f, g, h, a, 63, 0xc67178f2);

	if (SSEi_flags & SSEi_RELOAD)
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			SHA256_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 3 * VS32]));
				e[i] = vadd_epi32(
					e[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 4 * VS32]));
				f[i] = vadd_epi32(
					f[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 5 * VS32]));
				g[i] = vadd_epi32(
					g[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 6 * VS32]));
				h[i] = vadd_epi32(
					h[i],
					vload((vtype *) &reload_state[i * 16 * VS32 + 7 * VS32]));
			}
		}
		else
		{
			SHA256_PARA_DO(i)
			{
				a[i] = vadd_epi32(
					a[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 0 * VS32]));
				b[i] = vadd_epi32(
					b[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 1 * VS32]));
				c[i] = vadd_epi32(
					c[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 2 * VS32]));
				d[i] = vadd_epi32(
					d[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 3 * VS32]));
				e[i] = vadd_epi32(
					e[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 4 * VS32]));
				f[i] = vadd_epi32(
					f[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 5 * VS32]));
				g[i] = vadd_epi32(
					g[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 6 * VS32]));
				h[i] = vadd_epi32(
					h[i],
					vload((vtype *) &reload_state[i * 8 * VS32 + 7 * VS32]));
			}
		}
	}
	else
	{
		if (SSEi_flags & SSEi_CRYPT_SHA224)
		{
			SHA256_PARA_DO(i)
			{
				/* SHA-224 IV */
				a[i] = vadd_epi32(a[i], vset1_epi32(0xc1059ed8));
				b[i] = vadd_epi32(b[i], vset1_epi32(0x367cd507));
				c[i] = vadd_epi32(c[i], vset1_epi32(0x3070dd17));
				d[i] = vadd_epi32(d[i], vset1_epi32(0xf70e5939));
				e[i] = vadd_epi32(e[i], vset1_epi32(0xffc00b31));
				f[i] = vadd_epi32(f[i], vset1_epi32(0x68581511));
				g[i] = vadd_epi32(g[i], vset1_epi32(0x64f98fa7));
				h[i] = vadd_epi32(h[i], vset1_epi32(0xbefa4fa4));
			}
		}
		else
		{
			SHA256_PARA_DO(i)
			{
				/* SHA-256 IV */
				a[i] = vadd_epi32(a[i], vset1_epi32(0x6a09e667));
				b[i] = vadd_epi32(b[i], vset1_epi32(0xbb67ae85));
				c[i] = vadd_epi32(c[i], vset1_epi32(0x3c6ef372));
				d[i] = vadd_epi32(d[i], vset1_epi32(0xa54ff53a));
				e[i] = vadd_epi32(e[i], vset1_epi32(0x510e527f));
				f[i] = vadd_epi32(f[i], vset1_epi32(0x9b05688c));
				g[i] = vadd_epi32(g[i], vset1_epi32(0x1f83d9ab));
				h[i] = vadd_epi32(h[i], vset1_epi32(0x5be0cd19));
			}
		}
	}

	if (SSEi_flags & SSEi_FLAT_OUT)
	{
		SHA256_PARA_DO(i)
		{
			uint32_t * o = (uint32_t *) &out[i * 8 * VS32];
#if __AVX512F__ || __MIC__
			vtype idxs = vset_epi32(15 << 3,
									14 << 3,
									13 << 3,
									12 << 3,
									11 << 3,
									10 << 3,
									9 << 3,
									8 << 3,
									7 << 3,
									6 << 3,
									5 << 3,
									4 << 3,
									3 << 3,
									2 << 3,
									1 << 3,
									0 << 3);

			vscatter_epi32((((uint32_t *) o) + 0), idxs, vswap32(a[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 1), idxs, vswap32(b[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 2), idxs, vswap32(c[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 3), idxs, vswap32(d[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 4), idxs, vswap32(e[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 5), idxs, vswap32(f[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 6), idxs, vswap32(g[i]), 4);
			vscatter_epi32((((uint32_t *) o) + 7), idxs, vswap32(h[i]), 4);
#else
			uint32_t j, k;
			union {
				vtype v[8];
				uint32_t s[8 * VS32];
			} tmp;

			tmp.v[0] = vswap32(a[i]);
			tmp.v[1] = vswap32(b[i]);
			tmp.v[2] = vswap32(c[i]);
			tmp.v[3] = vswap32(d[i]);
			tmp.v[4] = vswap32(e[i]);
			tmp.v[5] = vswap32(f[i]);
			tmp.v[6] = vswap32(g[i]);
			tmp.v[7] = vswap32(h[i]);

			for (j = 0; j < VS32; j++)
				for (k = 0; k < 8; k++) o[j * 8 + k] = tmp.s[k * VS32 + j];
#endif
		}
	}
	else if (SSEi_flags & SSEi_OUTPUT_AS_INP_FMT)
	{
		if ((SSEi_flags & SSEi_OUTPUT_AS_2BUF_INP_FMT)
			== SSEi_OUTPUT_AS_2BUF_INP_FMT)
		{
			SHA256_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 32 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 3 * VS32], d[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 4 * VS32], e[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 5 * VS32], f[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 6 * VS32], g[i]);
				vstore((vtype *) &out[i * 32 * VS32 + 7 * VS32], h[i]);
			}
		}
		else
		{
			SHA256_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 16 * VS32 + 0 * VS32], a[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 1 * VS32], b[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 2 * VS32], c[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 3 * VS32], d[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 4 * VS32], e[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 5 * VS32], f[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 6 * VS32], g[i]);
				vstore((vtype *) &out[i * 16 * VS32 + 7 * VS32], h[i]);
			}
		}
	}
	else
	{
		SHA256_PARA_DO(i)
		{
			vstore((vtype *) &(out[i * 8 * VS32 + 0 * VS32]), a[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 1 * VS32]), b[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 2 * VS32]), c[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 3 * VS32]), d[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 4 * VS32]), e[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 5 * VS32]), f[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 6 * VS32]), g[i]);
			vstore((vtype *) &(out[i * 8 * VS32 + 7 * VS32]), h[i]);
		}
	}
}
#endif /* SIMD_PARA_SHA256 */

#if SIMD_PARA_SHA512

#undef S0
#define S0(x)                                                                  \
	(vxor(vroti_epi64(x, -39), vxor(vroti_epi64(x, -28), vroti_epi64(x, -34))))

#undef S1
#define S1(x)                                                                  \
	(vxor(vroti_epi64(x, -41), vxor(vroti_epi64(x, -14), vroti_epi64(x, -18))))

/*
 * These optimized sigma alternatives are from "Fast SHA-512 Implementations
 * on Intel Architecture Processors" whitepaper by Intel. They result in less
 * register copy operations so is faster despite using more ops. Slight boost
 * indeed seen on intel core i7.
 */
#if 1
#undef s0
#define s0(x)                                                                  \
	(vxor(vsrli_epi64(vxor(vsrli_epi64(vxor(vsrli_epi64(x, 1), x), 6), x), 1), \
		  vslli_epi64(vxor(vslli_epi64(x, 7), x), 56)))
#undef s1
#define s1(x)                                                                  \
	(vxor(                                                                     \
		vsrli_epi64(vxor(vsrli_epi64(vxor(vsrli_epi64(x, 42), x), 13), x), 6), \
		vslli_epi64(vxor(vslli_epi64(x, 42), x), 3)))
#else
#undef s0
#define s0(x)                                                                  \
	(vxor(vsrli_epi64(x, 7), vxor(vroti_epi64(x, -1), vroti_epi64(x, -8))))

#undef s1
#define s1(x)                                                                  \
	(vxor(vsrli_epi64(x, 6), vxor(vroti_epi64(x, -19), vroti_epi64(x, -61))))
#endif

#if __AVX512F__
#define Maj(x, y, z) vternarylogic(x, y, z, 0xE8)
#elif !VCMOV_EMULATED
#define Maj(x, y, z) vcmov(x, y, vxor(z, y))
#else
#define Maj(x, y, z) vor(vand(x, y), vand(vor(x, y), z))
#endif

#define Ch(x, y, z) vcmov(y, z, x)

#define SHA512_PARA_DO(x) for (x = 0; x < SIMD_PARA_SHA512; ++x)

#undef R
#define R(t)                                                                   \
	{                                                                          \
		tmp1[i] = vadd_epi64(s1(w[i][(t - 2) & 0xf]), w[i][(t - 7) & 0xf]);    \
		tmp2[i] = vadd_epi64(s0(w[i][(t - 15) & 0xf]), w[i][(t - 16) & 0xf]);  \
		w[i][(t) &0xf] = vadd_epi64(tmp1[i], tmp2[i]);                         \
	}

#define SHA512_STEP(a, b, c, d, e, f, g, h, x, K)                              \
	{                                                                          \
		SHA512_PARA_DO(i)                                                      \
		{                                                                      \
			tmp1[i] = vadd_epi64(h[i], w[i][(x) &0xf]);                        \
			tmp2[i] = vadd_epi64(S1(e[i]), vset1_epi64(K));                    \
			tmp1[i] = vadd_epi64(tmp1[i], Ch(e[i], f[i], g[i]));               \
			tmp1[i] = vadd_epi64(tmp1[i], tmp2[i]);                            \
			tmp2[i] = vadd_epi64(S0(a[i]), Maj(a[i], b[i], c[i]));             \
			d[i] = vadd_epi64(tmp1[i], d[i]);                                  \
			h[i] = vadd_epi64(tmp1[i], tmp2[i]);                               \
			if (x < 64) R(x);                                                  \
		}                                                                      \
	}

#define INIT_A 0x6a09e667f3bcc908ULL
#define INIT_B 0xbb67ae8584caa73bULL
#define INIT_C 0x3c6ef372fe94f82bULL
#define INIT_D 0xa54ff53a5f1d36f1ULL
#define INIT_E 0x510e527fade682d1ULL
#define INIT_F 0x9b05688c2b3e6c1fULL
#define INIT_G 0x1f83d9abfb41bd6bULL
#define INIT_H 0x5be0cd19137e2179ULL

#define ror(x, n) (((x) >> (n)) | ((x) << (64 - (n))))

void sha512_reverse(uint64_t * hash)
{
	uint64_t a, b, c, d, e, f, g, h, s0, maj, tmp;

	a = hash[0] - INIT_A;
	b = hash[1] - INIT_B;
	c = hash[2] - INIT_C;
	d = hash[3] - INIT_D;
	e = hash[4] - INIT_E;
	f = hash[5] - INIT_F;
	g = hash[6] - INIT_G;
	h = hash[7] - INIT_H;

	s0 = ror(b, 28) ^ ror(b, 34) ^ ror(b, 39);
	maj = (b & c) ^ (b & d) ^ (c & d);
	tmp = d;
	d = e - (a - (s0 + maj));
	e = f;
	f = g;
	g = h;
	a = b;
	b = c;
	c = tmp;

	s0 = ror(b, 28) ^ ror(b, 34) ^ ror(b, 39);
	maj = (b & c) ^ (b & d) ^ (c & d);
	tmp = d;
	d = e - (a - (s0 + maj));
	e = f;
	f = g;
	a = b;
	b = c;
	c = tmp;

	s0 = ror(b, 28) ^ ror(b, 34) ^ ror(b, 39);
	maj = (b & c) ^ (b & d) ^ (c & d);
	tmp = d;
	d = e - (a - (s0 + maj));
	e = f;
	a = b;
	b = c;
	c = tmp;

	s0 = ror(b, 28) ^ ror(b, 34) ^ ror(b, 39);
	maj = (b & c) ^ (b & d) ^ (c & d);

	hash[0] = e - (a - (s0 + maj));
}

void sha512_unreverse(void)
{
	fprintf(stderr, "sha512_unreverse() not implemented\n");
	perror("sha512");
}

#undef ror

#undef INIT_H
#undef INIT_G
#undef INIT_F
#undef INIT_E
#undef INIT_D
#undef INIT_C
#undef INIT_B
#undef INIT_A

#define INIT_D 0x152fecd8f70e5939ULL

void sha384_reverse(ARCH_WORD_64 * hash) { hash[3] -= INIT_D; }

void sha384_unreverse(ARCH_WORD_64 * hash) { hash[3] += INIT_D; }

#undef INIT_D

void SIMDSHA512body(vtype * data,
					ARCH_WORD_64 * out,
					ARCH_WORD_64 * reload_state,
					unsigned SSEi_flags)
{
	unsigned int i, k;

	vtype a[SIMD_PARA_SHA512], b[SIMD_PARA_SHA512], c[SIMD_PARA_SHA512],
		d[SIMD_PARA_SHA512], e[SIMD_PARA_SHA512], f[SIMD_PARA_SHA512],
		g[SIMD_PARA_SHA512], h[SIMD_PARA_SHA512];
	vtype w[SIMD_PARA_SHA512][16];
	vtype tmp1[SIMD_PARA_SHA512], tmp2[SIMD_PARA_SHA512];

	if (SSEi_flags & SSEi_FLAT_IN)
	{
		ARCH_WORD_64 * _data = (ARCH_WORD_64 *) data;
		SHA512_PARA_DO(k)
		{
			if (SSEi_flags & SSEi_2BUF_INPUT)
			{
				ARCH_WORD_64(*saved_key)[32] = (ARCH_WORD_64(*)[32]) _data;
				for (i = 0; i < 14; i += 2)
				{
					GATHER64(tmp1[k], saved_key, i);
					GATHER64(tmp2[k], saved_key, i + 1);
					vswap64(tmp1[k]);
					vswap64(tmp2[k]);
					w[k][i] = tmp1[k];
					w[k][i + 1] = tmp2[k];
				}
				GATHER64(tmp1[k], saved_key, 14);
				GATHER64(tmp2[k], saved_key, 15);
				_data += (VS64 << 5);
			}
			else
			{
				ARCH_WORD_64(*saved_key)[16] = (ARCH_WORD_64(*)[16]) _data;
				for (i = 0; i < 14; i += 2)
				{
					GATHER64(tmp1[k], saved_key, i);
					GATHER64(tmp2[k], saved_key, i + 1);
					vswap64(tmp1[k]);
					vswap64(tmp2[k]);
					w[k][i] = tmp1[k];
					w[k][i + 1] = tmp2[k];
				}
				GATHER64(tmp1[k], saved_key, 14);
				GATHER64(tmp2[k], saved_key, 15);
				_data += (VS64 << 4);
			}
			if (((SSEi_flags & SSEi_2BUF_INPUT_FIRST_BLK)
				 == SSEi_2BUF_INPUT_FIRST_BLK)
				|| ((SSEi_flags & SSEi_FLAT_RELOAD_SWAPLAST)
					== SSEi_FLAT_RELOAD_SWAPLAST))
			{
				vswap64(tmp1[k]);
				vswap64(tmp2[k]);
			}
			w[k][14] = tmp1[k];
			w[k][15] = tmp2[k];
		}
	}
	else
		memcpy(w, data, 16 * sizeof(vtype) * SIMD_PARA_SHA512);

	// dump_stuff_shammx64_msg("\nindex 2", w, 128, 2);

	if (SSEi_flags & SSEi_RELOAD)
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			SHA512_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 0 * VS64]);
				b[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 1 * VS64]);
				c[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 2 * VS64]);
				d[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 3 * VS64]);
				e[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 4 * VS64]);
				f[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 5 * VS64]);
				g[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 6 * VS64]);
				h[i] = vload((vtype *) &reload_state[i * 16 * VS64 + 7 * VS64]);
			}
		}
		else
		{
			SHA512_PARA_DO(i)
			{
				a[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 0 * VS64]);
				b[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 1 * VS64]);
				c[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 2 * VS64]);
				d[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 3 * VS64]);
				e[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 4 * VS64]);
				f[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 5 * VS64]);
				g[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 6 * VS64]);
				h[i] = vload((vtype *) &reload_state[i * 8 * VS64 + 7 * VS64]);
			}
		}
	}
	else
	{
		if (SSEi_flags & SSEi_CRYPT_SHA384)
		{
			SHA512_PARA_DO(i)
			{
				/* SHA-384 IV */
				a[i] = vset1_epi64(0xcbbb9d5dc1059ed8ULL);
				b[i] = vset1_epi64(0x629a292a367cd507ULL);
				c[i] = vset1_epi64(0x9159015a3070dd17ULL);
				d[i] = vset1_epi64(0x152fecd8f70e5939ULL);
				e[i] = vset1_epi64(0x67332667ffc00b31ULL);
				f[i] = vset1_epi64(0x8eb44a8768581511ULL);
				g[i] = vset1_epi64(0xdb0c2e0d64f98fa7ULL);
				h[i] = vset1_epi64(0x47b5481dbefa4fa4ULL);
			}
		}
		else
		{
			SHA512_PARA_DO(i)
			{
				/* SHA-512 IV */
				a[i] = vset1_epi64(0x6a09e667f3bcc908ULL);
				b[i] = vset1_epi64(0xbb67ae8584caa73bULL);
				c[i] = vset1_epi64(0x3c6ef372fe94f82bULL);
				d[i] = vset1_epi64(0xa54ff53a5f1d36f1ULL);
				e[i] = vset1_epi64(0x510e527fade682d1ULL);
				f[i] = vset1_epi64(0x9b05688c2b3e6c1fULL);
				g[i] = vset1_epi64(0x1f83d9abfb41bd6bULL);
				h[i] = vset1_epi64(0x5be0cd19137e2179ULL);
			}
		}
	}

	SHA512_STEP(a, b, c, d, e, f, g, h, 0, 0x428a2f98d728ae22ULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 1, 0x7137449123ef65cdULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 2, 0xb5c0fbcfec4d3b2fULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 3, 0xe9b5dba58189dbbcULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 4, 0x3956c25bf348b538ULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 5, 0x59f111f1b605d019ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 6, 0x923f82a4af194f9bULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 7, 0xab1c5ed5da6d8118ULL);
	SHA512_STEP(a, b, c, d, e, f, g, h, 8, 0xd807aa98a3030242ULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 9, 0x12835b0145706fbeULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 10, 0x243185be4ee4b28cULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 11, 0x550c7dc3d5ffb4e2ULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 12, 0x72be5d74f27b896fULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 13, 0x80deb1fe3b1696b1ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 14, 0x9bdc06a725c71235ULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 15, 0xc19bf174cf692694ULL);

	SHA512_STEP(a, b, c, d, e, f, g, h, 16, 0xe49b69c19ef14ad2ULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 17, 0xefbe4786384f25e3ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 18, 0x0fc19dc68b8cd5b5ULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 19, 0x240ca1cc77ac9c65ULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 20, 0x2de92c6f592b0275ULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 21, 0x4a7484aa6ea6e483ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 22, 0x5cb0a9dcbd41fbd4ULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 23, 0x76f988da831153b5ULL);
	SHA512_STEP(a, b, c, d, e, f, g, h, 24, 0x983e5152ee66dfabULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 25, 0xa831c66d2db43210ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 26, 0xb00327c898fb213fULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 27, 0xbf597fc7beef0ee4ULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 28, 0xc6e00bf33da88fc2ULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 29, 0xd5a79147930aa725ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 30, 0x06ca6351e003826fULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 31, 0x142929670a0e6e70ULL);

	SHA512_STEP(a, b, c, d, e, f, g, h, 32, 0x27b70a8546d22ffcULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 33, 0x2e1b21385c26c926ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 34, 0x4d2c6dfc5ac42aedULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 35, 0x53380d139d95b3dfULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 36, 0x650a73548baf63deULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 37, 0x766a0abb3c77b2a8ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 38, 0x81c2c92e47edaee6ULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 39, 0x92722c851482353bULL);
	SHA512_STEP(a, b, c, d, e, f, g, h, 40, 0xa2bfe8a14cf10364ULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 41, 0xa81a664bbc423001ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 42, 0xc24b8b70d0f89791ULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 43, 0xc76c51a30654be30ULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 44, 0xd192e819d6ef5218ULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 45, 0xd69906245565a910ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 46, 0xf40e35855771202aULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 47, 0x106aa07032bbd1b8ULL);

	SHA512_STEP(a, b, c, d, e, f, g, h, 48, 0x19a4c116b8d2d0c8ULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 49, 0x1e376c085141ab53ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 50, 0x2748774cdf8eeb99ULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 51, 0x34b0bcb5e19b48a8ULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 52, 0x391c0cb3c5c95a63ULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 53, 0x4ed8aa4ae3418acbULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 54, 0x5b9cca4f7763e373ULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 55, 0x682e6ff3d6b2b8a3ULL);
	SHA512_STEP(a, b, c, d, e, f, g, h, 56, 0x748f82ee5defb2fcULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 57, 0x78a5636f43172f60ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 58, 0x84c87814a1f0ab72ULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 59, 0x8cc702081a6439ecULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 60, 0x90befffa23631e28ULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 61, 0xa4506cebde82bde9ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 62, 0xbef9a3f7b2c67915ULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 63, 0xc67178f2e372532bULL);

	SHA512_STEP(a, b, c, d, e, f, g, h, 64, 0xca273eceea26619cULL);
	SHA512_STEP(h, a, b, c, d, e, f, g, 65, 0xd186b8c721c0c207ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 66, 0xeada7dd6cde0eb1eULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 67, 0xf57d4f7fee6ed178ULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 68, 0x06f067aa72176fbaULL);
	SHA512_STEP(d, e, f, g, h, a, b, c, 69, 0x0a637dc5a2c898a6ULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 70, 0x113f9804bef90daeULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 71, 0x1b710b35131c471bULL);
	SHA512_STEP(a, b, c, d, e, f, g, h, 72, 0x28db77f523047d84ULL);

	if (SSEi_flags & SSEi_REVERSE_STEPS && !(SSEi_flags & SSEi_CRYPT_SHA384))
	{
		SHA512_PARA_DO(i)
		{
			vstore((vtype *) &(out[i * 8 * VS64 + 0 * VS64]), h[i]);
		}
		return;
	}

	SHA512_STEP(h, a, b, c, d, e, f, g, 73, 0x32caab7b40c72493ULL);
	SHA512_STEP(g, h, a, b, c, d, e, f, 74, 0x3c9ebe0a15c9bebcULL);
	SHA512_STEP(f, g, h, a, b, c, d, e, 75, 0x431d67c49c100d4cULL);
	SHA512_STEP(e, f, g, h, a, b, c, d, 76, 0x4cc5d4becb3e42b6ULL);

	if (SSEi_flags & SSEi_REVERSE_STEPS)
	{
		SHA512_PARA_DO(i)
		{
			vstore((vtype *) &(out[i * 8 * VS64 + 3 * VS64]), d[i]);
		}
		return;
	}

	SHA512_STEP(d, e, f, g, h, a, b, c, 77, 0x597f299cfc657e2aULL);
	SHA512_STEP(c, d, e, f, g, h, a, b, 78, 0x5fcb6fab3ad6faecULL);
	SHA512_STEP(b, c, d, e, f, g, h, a, 79, 0x6c44198c4a475817ULL);

	if (SSEi_flags & SSEi_RELOAD)
	{
		if ((SSEi_flags & SSEi_RELOAD_INP_FMT) == SSEi_RELOAD_INP_FMT)
		{
			SHA512_PARA_DO(i)
			{
				a[i] = vadd_epi64(
					a[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 0 * VS64]));
				b[i] = vadd_epi64(
					b[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 1 * VS64]));
				c[i] = vadd_epi64(
					c[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 2 * VS64]));
				d[i] = vadd_epi64(
					d[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 3 * VS64]));
				e[i] = vadd_epi64(
					e[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 4 * VS64]));
				f[i] = vadd_epi64(
					f[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 5 * VS64]));
				g[i] = vadd_epi64(
					g[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 6 * VS64]));
				h[i] = vadd_epi64(
					h[i],
					vload((vtype *) &reload_state[i * 16 * VS64 + 7 * VS64]));
			}
		}
		else
		{
			SHA512_PARA_DO(i)
			{
				a[i] = vadd_epi64(
					a[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 0 * VS64]));
				b[i] = vadd_epi64(
					b[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 1 * VS64]));
				c[i] = vadd_epi64(
					c[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 2 * VS64]));
				d[i] = vadd_epi64(
					d[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 3 * VS64]));
				e[i] = vadd_epi64(
					e[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 4 * VS64]));
				f[i] = vadd_epi64(
					f[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 5 * VS64]));
				g[i] = vadd_epi64(
					g[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 6 * VS64]));
				h[i] = vadd_epi64(
					h[i],
					vload((vtype *) &reload_state[i * 8 * VS64 + 7 * VS64]));
			}
		}
	}
	else
	{
		if (SSEi_flags & SSEi_CRYPT_SHA384)
		{
			SHA512_PARA_DO(i)
			{
				/* SHA-384 IV */
				a[i] = vadd_epi64(a[i], vset1_epi64(0xcbbb9d5dc1059ed8ULL));
				b[i] = vadd_epi64(b[i], vset1_epi64(0x629a292a367cd507ULL));
				c[i] = vadd_epi64(c[i], vset1_epi64(0x9159015a3070dd17ULL));
				d[i] = vadd_epi64(d[i], vset1_epi64(0x152fecd8f70e5939ULL));
				e[i] = vadd_epi64(e[i], vset1_epi64(0x67332667ffc00b31ULL));
				f[i] = vadd_epi64(f[i], vset1_epi64(0x8eb44a8768581511ULL));
				g[i] = vadd_epi64(g[i], vset1_epi64(0xdb0c2e0d64f98fa7ULL));
				h[i] = vadd_epi64(h[i], vset1_epi64(0x47b5481dbefa4fa4ULL));
			}
		}
		else
		{
			SHA512_PARA_DO(i)
			{
				/* SHA-512 IV */
				a[i] = vadd_epi64(a[i], vset1_epi64(0x6a09e667f3bcc908ULL));
				b[i] = vadd_epi64(b[i], vset1_epi64(0xbb67ae8584caa73bULL));
				c[i] = vadd_epi64(c[i], vset1_epi64(0x3c6ef372fe94f82bULL));
				d[i] = vadd_epi64(d[i], vset1_epi64(0xa54ff53a5f1d36f1ULL));
				e[i] = vadd_epi64(e[i], vset1_epi64(0x510e527fade682d1ULL));
				f[i] = vadd_epi64(f[i], vset1_epi64(0x9b05688c2b3e6c1fULL));
				g[i] = vadd_epi64(g[i], vset1_epi64(0x1f83d9abfb41bd6bULL));
				h[i] = vadd_epi64(h[i], vset1_epi64(0x5be0cd19137e2179ULL));
			}
		}
	}

	if (SSEi_flags & SSEi_FLAT_OUT)
	{
		SHA512_PARA_DO(i)
		{
			uint64_t * o = (uint64_t *) &out[i * 8 * VS64];
#if __AVX512F__ || __MIC__
			vtype idxs = vset_epi64(
				7 << 3, 6 << 3, 5 << 3, 4 << 3, 3 << 3, 2 << 3, 1 << 3, 0 << 3);

			vscatter_epi64((((uint64_t *) o) + 0), idxs, vswap64(a[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 1), idxs, vswap64(b[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 2), idxs, vswap64(c[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 3), idxs, vswap64(d[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 4), idxs, vswap64(e[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 5), idxs, vswap64(f[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 6), idxs, vswap64(g[i]), 8);
			vscatter_epi64((((uint64_t *) o) + 7), idxs, vswap64(h[i]), 8);
#else
			uint64_t j, k;
			union {
				vtype v[8];
				uint64_t s[8 * VS64];
			} tmp;

			tmp.v[0] = vswap64(a[i]);
			tmp.v[1] = vswap64(b[i]);
			tmp.v[2] = vswap64(c[i]);
			tmp.v[3] = vswap64(d[i]);
			tmp.v[4] = vswap64(e[i]);
			tmp.v[5] = vswap64(f[i]);
			tmp.v[6] = vswap64(g[i]);
			tmp.v[7] = vswap64(h[i]);

			for (j = 0; j < VS64; j++)
				for (k = 0; k < 8; k++) o[j * 8 + k] = tmp.s[k * VS64 + j];
#endif
		}
	}
	else if (SSEi_flags & SSEi_OUTPUT_AS_INP_FMT)
	{
		if ((SSEi_flags & SSEi_OUTPUT_AS_2BUF_INP_FMT)
			== SSEi_OUTPUT_AS_2BUF_INP_FMT)
		{
			SHA512_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 32 * VS64 + 0 * VS64], a[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 1 * VS64], b[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 2 * VS64], c[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 3 * VS64], d[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 4 * VS64], e[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 5 * VS64], f[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 6 * VS64], g[i]);
				vstore((vtype *) &out[i * 32 * VS64 + 7 * VS64], h[i]);
			}
		}
		else
		{
			SHA512_PARA_DO(i)
			{
				vstore((vtype *) &out[i * 16 * VS64 + 0 * VS64], a[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 1 * VS64], b[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 2 * VS64], c[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 3 * VS64], d[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 4 * VS64], e[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 5 * VS64], f[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 6 * VS64], g[i]);
				vstore((vtype *) &out[i * 16 * VS64 + 7 * VS64], h[i]);
			}
		}
	}
	else
	{
		SHA512_PARA_DO(i)
		{
			vstore((vtype *) &(out[i * 8 * VS64 + 0 * VS64]), a[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 1 * VS64]), b[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 2 * VS64]), c[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 3 * VS64]), d[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 4 * VS64]), e[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 5 * VS64]), f[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 6 * VS64]), g[i]);
			vstore((vtype *) &(out[i * 8 * VS64 + 7 * VS64]), h[i]);
		}
	}
}

#endif /* SIMD_PARA_SHA512 */
