[catacomb] / base / dispatch.c

/* -*-c-*-
 *
 * CPU-specific dispatch
 *
 * (c) 2015 Straylight/Edgeware
 */

/*----- Licensing notice --------------------------------------------------*
 *
 * This file is part of Catacomb.
 *
 * Catacomb is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * Catacomb is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with Catacomb; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

/*----- Header files ------------------------------------------------------*/

#include "config.h"

#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <mLib/macros.h>

#include "dispatch.h"

/*----- Intel x86/AMD64 feature probing -----------------------------------*/

#if CPUFAM_X86 || CPUFAM_AMD64

#  define EFLAGS_ID (1u << 21)
#  define CPUID1D_SSE2 (1u << 26)
#  define CPUID1D_FXSR (1u << 24)
#  define CPUID1C_PCLMUL (1u << 1)
#  define CPUID1C_SSSE3 (1u << 9)
#  define CPUID1C_AESNI (1u << 25)
#  define CPUID1C_AVX (1u << 28)
#  define CPUID1C_RDRAND (1u << 30)

struct cpuid { unsigned a, b, c, d; };

/* --- @cpuid@ --- *
 *
 * Arguments:	@struct cpuid *cc@ = where to write the result
 *		@unsigned a, c@ = EAX and ECX registers to set
 *
 * Returns:	---
 *
 * Use:		Minimal C wrapper around the x86 `CPUID' instruction.  Checks
 *		that the instruction is actually available before invoking
 *		it; fills the output structure with zero if it's not going to
 *		work.
 */

#ifdef __GNUC__
#  if CPUFAM_X86
static __inline__ unsigned getflags(void)
  { unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); }
static __inline__ unsigned setflags(unsigned f)
{
  unsigned ff;
  __asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf"
	   : "=r" (ff)
	   : "r" (f));
  return (ff);
}
#  else
static __inline__ unsigned long getflags(void)
  { unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); }
static __inline__ unsigned long long setflags(unsigned long f)
{
  unsigned long ff;
  __asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf"
	   : "=r" (ff)
	   : "r" (f));
  return (ff);
}
#  endif
#endif

static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
{
#ifdef __GNUC__
  unsigned f;
#endif

  cc->a = cc->b = cc->c = cc->d = 0;

#ifdef __GNUC__
  /* Stupid dance to detect whether the CPUID instruction is available. */
  f = getflags();
  if (!(setflags(f |  EFLAGS_ID) & EFLAGS_ID) ||
	setflags(f & ~EFLAGS_ID) & EFLAGS_ID) {
    dispatch_debug("CPUID instruction not available");
    return;
  }
  setflags(f);

  /* Alas, EBX is magical in PIC code, so abuse ESI instead.  This isn't
   * pretty, but it works.
   */
#  if CPUFAM_X86
  __asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx"
	   : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	   : "a" (a) , "c" (c));
#  elif CPUFAM_AMD64
  __asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx"
	   : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	   : "a" (a) , "c" (c));
#  else
#    error "I'm confused."
#  endif
  dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
		 a, c, cc->a, cc->b, cc->c, cc->d);
#else
  dispatch_debug("GNU inline assembler not available; can't CPUID");
#endif
}

static unsigned cpuid_maxleaf(void)
  { struct cpuid c; cpuid(&c, 0, 0); return (c.a); }

/* --- @cpuid_features_p@ --- *
 *
 * Arguments:	@unsigned dbits@ = bits to check in EDX
 *		@unsigned cbits@ = bits to check in ECX
 *
 * Returns:	Nonzero if all the requested bits are set in the CPUID result
 *		on leaf 1.
 */

static int cpuid_features_p(unsigned dbits, unsigned cbits)
{
  struct cpuid c;
  if (cpuid_maxleaf() < 1) return (0);
  cpuid(&c, 1, 0);
  return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
}

/* --- @xmm_registers_available_p@ --- *
 *
 * Arguments:	---
 *
 * Returns:	Nonzero if the operating system has made the XMM registers
 *		available for use.
 */

static int xmm_registers_available_p(void)
{
#ifdef __GNUC__
  unsigned f;
  /* This hack is by Agner Fog.  Use FXSAVE/FXRSTOR to figure out whether the
   * XMM registers are actually alive.
   */
  if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0);
#  if CPUFAM_X86
  __asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n"
	   "fxsave (%%esp)\n"
	   "movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n"
	   "fxrstor (%%esp); fxsave (%%esp)\n"
	   "movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n"
	   "fxrstor (%%esp); movl %%edx, %%esp\n"
	   "xorl %%ecx, %%eax"
	   : "=a" (f)
	   : /* no inputs */
	   : "%ecx", "%edx");
#  elif CPUFAM_AMD64
  __asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n"
	   "fxsave (%%rsp)\n"
	   "movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n"
	   "fxrstor (%%rsp); fxsave (%%rsp)\n"
	   "movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n"
	   "fxrstor (%%rsp); movq %%rdx, %%rsp\n"
	   "xorl %%ecx, %%eax"
	   : "=a" (f)
	   : /* no inputs */
	   : "%ecx", "%rdx");
#  else
#    error "I'm confused."
#  endif
  dispatch_debug("XMM registers %savailable", f ? "" : "not ");
  return (f);
#else
  dispatch_debug("GNU inline assembler not available; can't check for XMM");
  return (0);
#endif
}

/* --- @rdrand_works_p@ --- *
 *
 *
 * Arguments:	---
 *
 * Returns:	Nonzero if the `rdrand' instruction actually works.  Assumes
 *		that it's already been verified to be safe to issue.
 */

#ifdef __GNUC__
static int rdrand(unsigned *x)
{
  int i, rc;
  unsigned _t;

  i = 16;
  __asm__ ("" : "=g" (_t));
  __asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n"
	   "mov $-1, %0; jmp 9f\n"
	   "1: movl %2, (%3); xorl %0, %0\n"
	   "9:"
	   : "=r" (rc), "+r" (i), "+r" (_t)
	   : "r" (x)
	   : "cc");
  return (rc);
}
#endif

static int rdrand_works_p(void)
{
  unsigned ref, x, i;

  /* Check that it doesn't always give the same answer.  Try four times: this
   * will fail with probability %$2^{-128}$% with a truly random generator,
   * which seems fair enough.
   */
  if (rdrand(&ref)) goto fail;
  for (i = 0; i < 4; i++) {
    if (rdrand(&x)) goto fail;
    if (x != ref) goto not_stuck;
  }
  dispatch_debug("RDRAND always returns 0x%08x!", ref);
  return (0);

not_stuck:
  dispatch_debug("RDRAND instruction looks plausible");
  return (1);

fail:
  dispatch_debug("RDRAND instruction fails too often");
  return (0);
}

#endif

/*----- General feature probing using auxiliary vectors -------------------*/

/* Try to find the system's definitions for auxiliary vector entries. */
#ifdef HAVE_SYS_AUXV_H
#  include <sys/auxv.h>
#endif
#ifdef HAVE_LINUX_AUXVEC_H
#  include <linux/auxvec.h>
#endif
#ifdef HAVE_ASM_HWCAP_H
#  include <asm/hwcap.h>
#endif

/* The type of entries in the auxiliary vector.  I'm assuming that `unsigned
 * long' matches each platform's word length; if this is false then we'll
 * need some host-specific tweaking here.
 */
union auxval { long i; unsigned long u; const void *p; };
struct auxentry { unsigned long type; union auxval value; };

/* Register each CPU family's interest in the auxiliary vector.  Make sure
 * that the necessary entry types are defined.  This is primarily ordered by
 * entry type to minimize duplication.
 */
#if defined(AT_HWCAP) && CPUFAM_ARMEL
#  define WANT_ANY 1
#  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
#endif

#if defined(AT_HWCAP) && CPUFAM_ARM64
#  define WANT_ANY 1
#  define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
#endif

#if defined(AT_HWCAP2) && CPUFAM_ARMEL
#  define WANT_ANY 1
#  define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
#endif

/* If we couldn't find any interesting entries then we can switch all of this
 * machinery off.  Also do that if we have no means for atomic updates.
 */
#if WANT_ANY && CPU_DISPATCH_P

/* The main output of this section is a bitmask of detected features.  The
 * least significant bit will be set if we've tried to probe.  Always access
 * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
 */
static unsigned hwcaps = 0;

/* For each potentially interesting type which turned out not to exist or be
 * wanted, define a dummy macro for the sake of the next step.
 */
#ifndef WANT_AT_HWCAP
#  define WANT_AT_HWCAP(_)
#endif
#ifndef WANT_AT_HWCAP2
#  define WANT_AT_HWCAP2(_)
#endif

/* For each CPU family, define two lists.
 *
 *   * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
 *     family tried to register interest in above.  Each entry contains the
 *     interesting auxiliary vector entry type, the name of the union branch
 *     for its value, and the name of the slot in `struct auxprobe' in which
 *     to store the value.
 *
 *   * `CAPMAP' is a list describing the output features which the CPU family
 *     intends to satisfy from the auxiliary vector.  Each entry contains a
 *     feature name suffix, and the token name (for `check_env').
 */
#if CPUFAM_ARMEL
#  define WANTAUX(_)							\
	WANT_AT_HWCAP(_)						\
	WANT_AT_HWCAP2(_)
#  define CAPMAP(_)							\
	_(ARM_VFP, "arm:vfp")						\
	_(ARM_NEON, "arm:neon")						\
	_(ARM_V4, "arm:v4")						\
	_(ARM_D32, "arm:d32")						\
	_(ARM_AES, "arm:aes")						\
	_(ARM_PMULL, "arm:pmull")
#endif
#if CPUFAM_ARM64
#  define WANTAUX(_)							\
	WANT_AT_HWCAP(_)
#  define CAPMAP(_)							\
	_(ARM_NEON, "arm:neon")						\
	_(ARM_AES, "arm:aes")						\
	_(ARM_PMULL, "arm:pmull")
#endif

/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
enum {
  HFI_PROBED = 0,
#define HFI__ENUM(feat, tok) HFI_##feat,
  CAPMAP(HFI__ENUM)
#undef HFI__ENUM
  HFI__END
};
enum {
  HF_PROBED = 1,
#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
  CAPMAP(HF__FLAG)
#undef HF__FLAG
  HF__END
};

/* Build a structure in which we can capture the interesting data from the
 * auxiliary vector.
 */
#define AUXUTYPE_i long
#define AUXUTYPE_u unsigned long
#define AUXUTYPE_p const void *
struct auxprobe {
#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
  WANTAUX(AUXPROBE__SLOT)
#undef AUXPROBE_SLOT
};

/* --- @probe_hwcaps@ --- *
 *
 * Arguments:	---
 *
 * Returns:	---
 *
 * Use:		Attempt to find the auxiliary vector (which is well hidden)
 *		and discover interesting features from it.
 */

static void probe_hwcaps(void)
{
  unsigned hw = HF_PROBED;
  struct auxprobe probed = { 0 };

  /* Populate `probed' with the information we manage to retrieve from the
   * auxiliary vector.  Slots we couldn't find are left zero-valued.
   */
#if defined(HAVE_GETAUXVAL)
  /* Shiny new libc lets us request individual entry types.  This is almost
   * too easy.
   */
#  define CAP__GET(type, ubranch, slot)					\
	probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
  WANTAUX(CAP__GET)
#else
  /* Otherwise we're a bit stuck, really.  Modern Linux kernels make a copy
   * of the vector available in `/procc' so we could try that.
   *
   * The usual place is stuck on the end of the environment vector, but that
   * may well have moved, and we have no way of telling whether it has or
   * whether there was ever an auxiliary vector there at all; so don't do
   * that.
   */
  {
    FILE *fp = 0;
    unsigned char *p = 0, *q = 0;
    const struct auxentry *a;
    size_t sz, off, n;

    /* Open the file and read it into a memory chunk. */
    if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
    sz = 4096; off = 0;
    if ((p = malloc(sz)) == 0) goto clean;
    for (;;) {
      n = fread(p + off, 1, sz - off, fp);
      off += n;
      if (off < sz) break;
      sz *= 2; if ((q = realloc(p, sz)) == 0) break;
      p = q;
    }

    /* Work through the vector (or as much of it as we found) and extract the
     * types we're interested in.
     */
    for (a = (const struct auxentry *)p,
	   n = sz/sizeof(struct auxentry);
	 n--; a++) {
      switch (a->type) {
#define CAP__SWITCH(type, ubranch, slot)				\
	case type: probed.slot = a->value.ubranch; break;
	WANTAUX(CAP__SWITCH)
	case AT_NULL: goto clean;
      }
    }

  clean:
    if (p) free(p);
    if (fp) fclose(fp);
  }
#endif

  /* Each CPU family now has to pick through what was found and stashed in
   * `probed', and set the appropriate flag bits in `hw'.
   */
#if CPUFAM_ARMEL
  if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP;
  if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON;
  if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32;
  if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4;
#  ifdef HWCAP2_AES
  if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES;
#  endif
#  ifdef HWCAP2_PMULL
  if (probed.hwcap2 & HWCAP2_PMULL) hw |= HF_ARM_PMULL;
#  endif
#endif
#if CPUFAM_ARM64
  if (probed.hwcap & HWCAP_ASIMD) hw |= HF_ARM_NEON;
  if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES;
  if (probed.hwcap & HWCAP_PMULL) hw |= HF_ARM_PMULL;
#endif

  /* Store the bitmask of features we probed for everyone to see. */
  DISPATCH_STORE(hwcaps, hw);

  /* Finally, make a report about the things we found.  (Doing this earlier
   * will pointlessly widen the window in which multiple threads will do the
   * above auxiliary-vector probing.)
   */
#define CAP__DEBUG(feat, tok)						\
  dispatch_debug("check auxv for feature `%s': %s", tok,		\
		 hw & HF_##feat ? "available" : "absent");
  CAPMAP(CAP__DEBUG)
#undef CAP__DEBUG
}

/* --- @get_hwcaps@ --- *
 *
 * Arguments:	---
 *
 * Returns:	A mask of hardware capabilities and other features, as probed
 *		from the auxiliary vector.
 */

static unsigned get_hwcaps(void)
{
  unsigned hw;

  DISPATCH_LOAD(hwcaps, hw);
  if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
  return (hw);
}

#endif

/*----- External interface ------------------------------------------------*/

/* --- @dispatch_debug@ --- *
 *
 * Arguments:	@const char *fmt@ = a format string
 *		@...@ = additional arguments
 *
 * Returns:	---
 *
 * Use:		Writes a formatted message to standard output if dispatch
 *		debugging is enabled.
 */

void dispatch_debug(const char *fmt, ...)
{
  va_list ap;
  const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");

  if (e && *e != 'n' && *e != '0') {
    va_start(ap, fmt);
    fputs("Catacomb CPUDISPATCH: ", stderr);
    vfprintf(stderr, fmt, ap);
    fputc('\n', stderr);
    va_end(ap);
  }
}

/* --- @check_env@ --- *
 *
 * Arguments:	@const char *ftok@ = feature token
 *
 * Returns:	Zero if the feature is forced off; positive if it's forced
 *		on; negative if the user hasn't decided.
 *
 * Use:		Checks the environment variable `CATACOMB_CPUFEAT' for the
 *		feature token @ftok@.  The variable, if it exists, should be
 *		a space-separated sequence of `+tok' and `-tok' items.  These
 *		tokens may end in `*', which matches any suffix.
 */

static int IGNORABLE check_env(const char *ftok)
{
  const char *p, *q, *pp;
  int d;

  p = getenv("CATACOMB_CPUFEAT");
  if (!p) return (-1);

  for (;;) {
    while (isspace((unsigned char)*p)) p++;
    if (!*p) return (-1);
    switch (*p) {
      case '+': d = +1; p++; break;
      case '-': d =  0; p++; break;
      default:  d = -1;      break;
    }
    for (q = p; *q && !isspace((unsigned char)*q); q++);
    if (d >= 0) {
      for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++);
      if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d);
    }
    p = q;
  }
  return (-1);
}

/* --- @cpu_feature_p@ --- *
 *
 * Arguments:	@unsigned feat@ = a @CPUFEAT_...@ code
 *
 * Returns:	Nonzero if the feature is available.
 */

#include <stdio.h>

static int IGNORABLE
  feat_debug(const char *ftok, const char *check, int verdict)
{
  if (verdict >= 0) {
    dispatch_debug("feature `%s': %s -> %s", ftok, check,
		   verdict ? "available" : "absent");
  }
  return (verdict);
}

int cpu_feature_p(int feat)
{
  int IGNORABLE f;
  IGNORE(f);
#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat:		\
  if ((f = feat_debug(ftok, "environment override",			\
		      check_env(ftok))) >= 0)				\
    return (f);								\
  else									\
    return (feat_debug(ftok, "runtime probe", cond));

  switch (feat) {
#if CPUFAM_X86 || CPUFAM_AMD64
    CASE_CPUFEAT(X86_SSE2, "x86:sse2",
		 cpuid_features_p(CPUID1D_SSE2, 0) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_AESNI, "x86:aesni",
		 cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
		 cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p());
    CASE_CPUFEAT(X86_AVX, "x86:avx",
		 cpuid_features_p(0, CPUID1C_AVX) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_SSSE3, "x86:ssse3",
		 cpuid_features_p(0, CPUID1C_SSSE3) &&
		 xmm_registers_available_p());
    CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul",
		 cpuid_features_p(0, CPUID1C_PCLMUL) &&
		 xmm_registers_available_p());
#endif
#ifdef CAPMAP
#  define FEATP__CASE(feat, tok)					\
	CASE_CPUFEAT(feat, tok,	get_hwcaps() & HF_##feat)
    CAPMAP(FEATP__CASE)
#undef FEATP__CASE
#endif
    default:
      dispatch_debug("denying unknown feature %d", feat);
      return (0);
  }
#undef CASE_CPUFEAT
}

/*----- That's all, folks -------------------------------------------------*/
Commit	Line	Data
08e2be29 MW	1	/* --c--
	2	*
	3	* CPU-specific dispatch
	4	*
	5	* (c) 2015 Straylight/Edgeware
	6	*/
	7
	8	/----- Licensing notice --------------------------------------------------
	9	*
	10	* This file is part of Catacomb.
	11	*
	12	* Catacomb is free software; you can redistribute it and/or modify
	13	* it under the terms of the GNU Library General Public License as
	14	* published by the Free Software Foundation; either version 2 of the
	15	* License, or (at your option) any later version.
	16	*
	17	* Catacomb is distributed in the hope that it will be useful,
	18	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	20	* GNU Library General Public License for more details.
	21	*
	22	* You should have received a copy of the GNU Library General Public
	23	* License along with Catacomb; if not, write to the Free
	24	* Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
	25	* MA 02111-1307, USA.
	26	*/
	27
	28	/----- Header files ------------------------------------------------------/
	29
	30	#include "config.h"
	31
	32	#include <ctype.h>
fac645f7 MW	33	#include <stdarg.h>
fac645f7 MW	34	#include <stdio.h>
08e2be29 MW	35	#include <stdlib.h>
	36	#include <string.h>
	37
	38	#include <mLib/macros.h>
	39
	40	#include "dispatch.h"
	41
d26ad211	42	/----- Intel x86/AMD64 feature probing -----------------------------------/
08e2be29	43
0f23f75f	44	#if CPUFAM_X86 \|\| CPUFAM_AMD64
08e2be29	45
7680f863 MW	46	# define EFLAGS_ID (1u << 21)
	47	# define CPUID1D_SSE2 (1u << 26)
	48	# define CPUID1D_FXSR (1u << 24)
9e6a4409 MW	49	# define CPUID1C_PCLMUL (1u << 1)
9e6a4409 MW	50	# define CPUID1C_SSSE3 (1u << 9)
7680f863	51	# define CPUID1C_AESNI (1u << 25)
b9b279b4	52	# define CPUID1C_AVX (1u << 28)
d25653be	53	# define CPUID1C_RDRAND (1u << 30)
08e2be29 MW	54
	55	struct cpuid { unsigned a, b, c, d; };
	56
	57	/* --- @cpuid@ --- *
	58	*
	59	* Arguments: @struct cpuid *cc@ = where to write the result
	60	* @unsigned a, c@ = EAX and ECX registers to set
	61	*
	62	* Returns: ---
	63	*
	64	* Use: Minimal C wrapper around the x86 `CPUID' instruction. Checks
	65	* that the instruction is actually available before invoking
	66	* it; fills the output structure with zero if it's not going to
	67	* work.
	68	*/
	69
	70	#ifdef __GNUC__
0f23f75f	71	# if CPUFAM_X86
08e2be29 MW	72	static __inline__ unsigned getflags(void)
	73	{ unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); }
	74	static __inline__ unsigned setflags(unsigned f)
	75	{
	76	unsigned ff;
	77	__asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf"
25eccdf7 MW	78	: "=r" (ff)
25eccdf7 MW	79	: "r" (f));
08e2be29 MW	80	return (ff);
08e2be29 MW	81	}
0f23f75f MW	82	# else
	83	static __inline__ unsigned long getflags(void)
	84	{ unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); }
	85	static __inline__ unsigned long long setflags(unsigned long f)
	86	{
	87	unsigned long ff;
	88	__asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf"
25eccdf7 MW	89	: "=r" (ff)
25eccdf7 MW	90	: "r" (f));
0f23f75f MW	91	return (ff);
	92	}
	93	# endif
08e2be29 MW	94	#endif
	95
	96	static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
	97	{
	98	#ifdef __GNUC__
	99	unsigned f;
	100	#endif
	101
	102	cc->a = cc->b = cc->c = cc->d = 0;
	103
	104	#ifdef __GNUC__
	105	/* Stupid dance to detect whether the CPUID instruction is available. */
	106	f = getflags();
fac645f7 MW	107	if (!(setflags(f \| EFLAGS_ID) & EFLAGS_ID) \|\|
	108	setflags(f & ~EFLAGS_ID) & EFLAGS_ID) {
	109	dispatch_debug("CPUID instruction not available");
	110	return;
	111	}
08e2be29 MW	112	setflags(f);
	113
	114	/* Alas, EBX is magical in PIC code, so abuse ESI instead. This isn't
	115	* pretty, but it works.
	116	*/
0f23f75f	117	# if CPUFAM_X86
08e2be29 MW	118	__asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx"
	119	: "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	120	: "a" (a) , "c" (c));
0f23f75f MW	121	# elif CPUFAM_AMD64
	122	__asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx"
	123	: "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
	124	: "a" (a) , "c" (c));
	125	# else
	126	# error "I'm confused."
	127	# endif
	128	dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
	129	a, c, cc->a, cc->b, cc->c, cc->d);
fac645f7 MW	130	#else
fac645f7 MW	131	dispatch_debug("GNU inline assembler not available; can't CPUID");
08e2be29 MW	132	#endif
	133	}
	134
	135	static unsigned cpuid_maxleaf(void)
	136	{ struct cpuid c; cpuid(&c, 0, 0); return (c.a); }
	137
11b17977 MW	138	/* --- @cpuid_features_p@ --- *
	139	*
	140	* Arguments: @unsigned dbits@ = bits to check in EDX
	141	* @unsigned cbits@ = bits to check in ECX
	142	*
	143	* Returns: Nonzero if all the requested bits are set in the CPUID result
	144	* on leaf 1.
	145	*/
	146
08e2be29 MW	147	static int cpuid_features_p(unsigned dbits, unsigned cbits)
	148	{
	149	struct cpuid c;
	150	if (cpuid_maxleaf() < 1) return (0);
	151	cpuid(&c, 1, 0);
	152	return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
	153	}
	154
11b17977 MW	155	/* --- @xmm_registers_available_p@ --- *
	156	*
	157	* Arguments: ---
	158	*
	159	* Returns: Nonzero if the operating system has made the XMM registers
	160	* available for use.
	161	*/
	162
acbe16df MW	163	static int xmm_registers_available_p(void)
	164	{
	165	#ifdef __GNUC__
	166	unsigned f;
	167	/* This hack is by Agner Fog. Use FXSAVE/FXRSTOR to figure out whether the
	168	* XMM registers are actually alive.
	169	*/
	170	if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0);
0f23f75f	171	# if CPUFAM_X86
acbe16df MW	172	__asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n"
	173	"fxsave (%%esp)\n"
	174	"movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n"
	175	"fxrstor (%%esp); fxsave (%%esp)\n"
	176	"movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n"
	177	"fxrstor (%%esp); movl %%edx, %%esp\n"
	178	"xorl %%ecx, %%eax"
	179	: "=a" (f)
	180	: /* no inputs */
	181	: "%ecx", "%edx");
0f23f75f MW	182	# elif CPUFAM_AMD64
	183	__asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n"
	184	"fxsave (%%rsp)\n"
	185	"movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n"
	186	"fxrstor (%%rsp); fxsave (%%rsp)\n"
	187	"movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n"
	188	"fxrstor (%%rsp); movq %%rdx, %%rsp\n"
	189	"xorl %%ecx, %%eax"
	190	: "=a" (f)
	191	: /* no inputs */
	192	: "%ecx", "%rdx");
	193	# else
	194	# error "I'm confused."
	195	# endif
	196	dispatch_debug("XMM registers %savailable", f ? "" : "not ");
acbe16df MW	197	return (f);
acbe16df MW	198	#else
fac645f7	199	dispatch_debug("GNU inline assembler not available; can't check for XMM");
acbe16df MW	200	return (0);
	201	#endif
	202	}
	203
35b1eba8 MW	204	/* --- @rdrand_works_p@ --- *
	205	*
	206	*
	207	* Arguments: ---
	208	*
	209	* Returns: Nonzero if the `rdrand' instruction actually works. Assumes
	210	* that it's already been verified to be safe to issue.
	211	*/
	212
	213	#ifdef __GNUC__
	214	static int rdrand(unsigned *x)
	215	{
	216	int i, rc;
	217	unsigned _t;
	218
	219	i = 16;
	220	__asm__ ("" : "=g" (_t));
	221	__asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n"
	222	"mov $-1, %0; jmp 9f\n"
	223	"1: movl %2, (%3); xorl %0, %0\n"
	224	"9:"
	225	: "=r" (rc), "+r" (i), "+r" (_t)
	226	: "r" (x)
	227	: "cc");
	228	return (rc);
	229	}
	230	#endif
	231
	232	static int rdrand_works_p(void)
	233	{
	234	unsigned ref, x, i;
	235
	236	/* Check that it doesn't always give the same answer. Try four times: this
	237	* will fail with probability %$2^{-128}$% with a truly random generator,
	238	* which seems fair enough.
	239	*/
	240	if (rdrand(&ref)) goto fail;
	241	for (i = 0; i < 4; i++) {
	242	if (rdrand(&x)) goto fail;
	243	if (x != ref) goto not_stuck;
	244	}
	245	dispatch_debug("RDRAND always returns 0x%08x!", ref);
	246	return (0);
	247
	248	not_stuck:
	249	dispatch_debug("RDRAND instruction looks plausible");
	250	return (1);
	251
	252	fail:
	253	dispatch_debug("RDRAND instruction fails too often");
	254	return (0);
	255	}
	256
08e2be29 MW	257	#endif
08e2be29 MW	258
a02a22d4 MW	259	/----- General feature probing using auxiliary vectors -------------------/
	260
	261	/* Try to find the system's definitions for auxiliary vector entries. */
	262	#ifdef HAVE_SYS_AUXV_H
	263	# include <sys/auxv.h>
a0e9bb8a MW	264	#endif
	265	#ifdef HAVE_LINUX_AUXVEC_H
	266	# include <linux/auxvec.h>
	267	#endif
	268	#ifdef HAVE_ASM_HWCAP_H
	269	# include <asm/hwcap.h>
a02a22d4 MW	270	#endif
	271
	272	/* The type of entries in the auxiliary vector. I'm assuming that `unsigned
	273	* long' matches each platform's word length; if this is false then we'll
	274	* need some host-specific tweaking here.
	275	*/
	276	union auxval { long i; unsigned long u; const void *p; };
	277	struct auxentry { unsigned long type; union auxval value; };
	278
	279	/* Register each CPU family's interest in the auxiliary vector. Make sure
	280	* that the necessary entry types are defined. This is primarily ordered by
	281	* entry type to minimize duplication.
	282	*/
61bd904b MW	283	#if defined(AT_HWCAP) && CPUFAM_ARMEL
	284	# define WANT_ANY 1
	285	# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
	286	#endif
a02a22d4	287
e492db88 MW	288	#if defined(AT_HWCAP) && CPUFAM_ARM64
	289	# define WANT_ANY 1
	290	# define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
	291	#endif
	292
26e182fc MW	293	#if defined(AT_HWCAP2) && CPUFAM_ARMEL
	294	# define WANT_ANY 1
	295	# define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
	296	#endif
	297
a02a22d4 MW	298	/* If we couldn't find any interesting entries then we can switch all of this
	299	* machinery off. Also do that if we have no means for atomic updates.
	300	*/
	301	#if WANT_ANY && CPU_DISPATCH_P
	302
	303	/* The main output of this section is a bitmask of detected features. The
	304	* least significant bit will be set if we've tried to probe. Always access
	305	* this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
	306	*/
	307	static unsigned hwcaps = 0;
	308
	309	/* For each potentially interesting type which turned out not to exist or be
	310	* wanted, define a dummy macro for the sake of the next step.
	311	*/
	312	#ifndef WANT_AT_HWCAP
	313	# define WANT_AT_HWCAP(_)
	314	#endif
f013a4f2 MW	315	#ifndef WANT_AT_HWCAP2
	316	# define WANT_AT_HWCAP2(_)
	317	#endif
a02a22d4 MW	318
	319	/* For each CPU family, define two lists.
	320	*
	321	* * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
	322	* family tried to register interest in above. Each entry contains the
	323	* interesting auxiliary vector entry type, the name of the union branch
	324	* for its value, and the name of the slot in `struct auxprobe' in which
	325	* to store the value.
	326	*
	327	* * `CAPMAP' is a list describing the output features which the CPU family
	328	* intends to satisfy from the auxiliary vector. Each entry contains a
	329	* feature name suffix, and the token name (for `check_env').
	330	*/
61bd904b MW	331	#if CPUFAM_ARMEL
61bd904b MW	332	# define WANTAUX(_) \
26e182fc MW	333	WANT_AT_HWCAP(_) \
26e182fc MW	334	WANT_AT_HWCAP2(_)
61bd904b MW	335	# define CAPMAP(_) \
	336	_(ARM_VFP, "arm:vfp") \
	337	_(ARM_NEON, "arm:neon") \
	338	_(ARM_V4, "arm:v4") \
26e182fc	339	_(ARM_D32, "arm:d32") \
9e6a4409 MW	340	_(ARM_AES, "arm:aes") \
9e6a4409 MW	341	_(ARM_PMULL, "arm:pmull")
61bd904b	342	#endif
e492db88 MW	343	#if CPUFAM_ARM64
	344	# define WANTAUX(_) \
	345	WANT_AT_HWCAP(_)
	346	# define CAPMAP(_) \
cb7f92c4	347	_(ARM_NEON, "arm:neon") \
9e6a4409 MW	348	_(ARM_AES, "arm:aes") \
9e6a4409 MW	349	_(ARM_PMULL, "arm:pmull")
e492db88	350	#endif
a02a22d4 MW	351
	352	/* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
	353	enum {
	354	HFI_PROBED = 0,
	355	#define HFI__ENUM(feat, tok) HFI_##feat,
	356	CAPMAP(HFI__ENUM)
	357	#undef HFI__ENUM
	358	HFI__END
	359	};
	360	enum {
	361	HF_PROBED = 1,
	362	#define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
	363	CAPMAP(HF__FLAG)
	364	#undef HF__FLAG
	365	HF__END
	366	};
	367
	368	/* Build a structure in which we can capture the interesting data from the
	369	* auxiliary vector.
	370	*/
	371	#define AUXUTYPE_i long
	372	#define AUXUTYPE_u unsigned long
	373	#define AUXUTYPE_p const void *
	374	struct auxprobe {
	375	#define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
	376	WANTAUX(AUXPROBE__SLOT)
	377	#undef AUXPROBE_SLOT
	378	};
	379
	380	/* --- @probe_hwcaps@ --- *
	381	*
	382	* Arguments: ---
	383	*
	384	* Returns: ---
	385	*
	386	* Use: Attempt to find the auxiliary vector (which is well hidden)
	387	* and discover interesting features from it.
	388	*/
	389
	390	static void probe_hwcaps(void)
	391	{
	392	unsigned hw = HF_PROBED;
	393	struct auxprobe probed = { 0 };
	394
	395	/* Populate `probed' with the information we manage to retrieve from the
	396	* auxiliary vector. Slots we couldn't find are left zero-valued.
	397	*/
	398	#if defined(HAVE_GETAUXVAL)
	399	/* Shiny new libc lets us request individual entry types. This is almost
	400	* too easy.
	401	*/
84850eec MW	402	# define CAP__GET(type, ubranch, slot) \
84850eec MW	403	probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
a02a22d4 MW	404	WANTAUX(CAP__GET)
	405	#else
	406	/* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy
	407	* of the vector available in `/procc' so we could try that.
	408	*
	409	* The usual place is stuck on the end of the environment vector, but that
	410	* may well have moved, and we have no way of telling whether it has or
	411	* whether there was ever an auxiliary vector there at all; so don't do
	412	* that.
	413	*/
	414	{
	415	FILE *fp = 0;
	416	unsigned char p = 0, q = 0;
	417	const struct auxentry *a;
	418	size_t sz, off, n;
	419
	420	/* Open the file and read it into a memory chunk. */
	421	if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
	422	sz = 4096; off = 0;
	423	if ((p = malloc(sz)) == 0) goto clean;
	424	for (;;) {
	425	n = fread(p + off, 1, sz - off, fp);
	426	off += n;
	427	if (off < sz) break;
	428	sz *= 2; if ((q = realloc(p, sz)) == 0) break;
	429	p = q;
	430	}
	431
	432	/* Work through the vector (or as much of it as we found) and extract the
	433	* types we're interested in.
	434	*/
	435	for (a = (const struct auxentry *)p,
	436	n = sz/sizeof(struct auxentry);
	437	n--; a++) {
	438	switch (a->type) {
	439	#define CAP__SWITCH(type, ubranch, slot) \
	440	case type: probed.slot = a->value.ubranch; break;
	441	WANTAUX(CAP__SWITCH)
dfcb2a0b	442	case AT_NULL: goto clean;
a02a22d4 MW	443	}
	444	}
	445
	446	clean:
	447	if (p) free(p);
	448	if (fp) fclose(fp);
	449	}
	450	#endif
	451
	452	/* Each CPU family now has to pick through what was found and stashed in
	453	* `probed', and set the appropriate flag bits in `hw'.
	454	*/
61bd904b MW	455	#if CPUFAM_ARMEL
	456	if (probed.hwcap & HWCAP_VFPv3) hw \|= HF_ARM_VFP;
	457	if (probed.hwcap & HWCAP_NEON) hw \|= HF_ARM_NEON;
	458	if (probed.hwcap & HWCAP_VFPD32) hw \|= HF_ARM_D32;
	459	if (probed.hwcap & HWCAP_VFPv4) hw \|= HF_ARM_V4;
26e182fc MW	460	# ifdef HWCAP2_AES
	461	if (probed.hwcap2 & HWCAP2_AES) hw \|= HF_ARM_AES;
	462	# endif
9e6a4409 MW	463	# ifdef HWCAP2_PMULL
	464	if (probed.hwcap2 & HWCAP2_PMULL) hw \|= HF_ARM_PMULL;
	465	# endif
e492db88 MW	466	#endif
e492db88 MW	467	#if CPUFAM_ARM64
cb7f92c4	468	if (probed.hwcap & HWCAP_ASIMD) hw \|= HF_ARM_NEON;
e492db88	469	if (probed.hwcap & HWCAP_AES) hw \|= HF_ARM_AES;
9e6a4409	470	if (probed.hwcap & HWCAP_PMULL) hw \|= HF_ARM_PMULL;
61bd904b	471	#endif
a02a22d4 MW	472
	473	/* Store the bitmask of features we probed for everyone to see. */
	474	DISPATCH_STORE(hwcaps, hw);
	475
	476	/* Finally, make a report about the things we found. (Doing this earlier
	477	* will pointlessly widen the window in which multiple threads will do the
	478	* above auxiliary-vector probing.)
	479	*/
	480	#define CAP__DEBUG(feat, tok) \
	481	dispatch_debug("check auxv for feature `%s': %s", tok, \
	482	hw & HF_##feat ? "available" : "absent");
	483	CAPMAP(CAP__DEBUG)
	484	#undef CAP__DEBUG
	485	}
	486
	487	/* --- @get_hwcaps@ --- *
	488	*
	489	* Arguments: ---
	490	*
	491	* Returns: A mask of hardware capabilities and other features, as probed
	492	* from the auxiliary vector.
	493	*/
	494
	495	static unsigned get_hwcaps(void)
	496	{
	497	unsigned hw;
	498
	499	DISPATCH_LOAD(hwcaps, hw);
	500	if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
	501	return (hw);
	502	}
	503
	504	#endif
	505
d26ad211 MW	506	/----- External interface ------------------------------------------------/
d26ad211 MW	507
fac645f7 MW	508	/* --- @dispatch_debug@ --- *
	509	*
	510	* Arguments: @const char *fmt@ = a format string
	511	* @...@ = additional arguments
	512	*
	513	* Returns: ---
	514	*
	515	* Use: Writes a formatted message to standard output if dispatch
	516	* debugging is enabled.
	517	*/
	518
	519	void dispatch_debug(const char *fmt, ...)
	520	{
	521	va_list ap;
	522	const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");
	523
	524	if (e && e != 'n' && e != '0') {
	525	va_start(ap, fmt);
	526	fputs("Catacomb CPUDISPATCH: ", stderr);
	527	vfprintf(stderr, fmt, ap);
	528	fputc('\n', stderr);
	529	va_end(ap);
	530	}
	531	}
	532
08e2be29 MW	533	/* --- @check_env@ --- *
	534	*
	535	* Arguments: @const char *ftok@ = feature token
	536	*
	537	* Returns: Zero if the feature is forced off; positive if it's forced
	538	* on; negative if the user hasn't decided.
	539	*
	540	* Use: Checks the environment variable `CATACOMB_CPUFEAT' for the
	541	* feature token @ftok@. The variable, if it exists, should be
	542	* a space-separated sequence of `+tok' and `-tok' items. These
	543	* tokens may end in `*', which matches any suffix.
	544	*/
	545
	546	static int IGNORABLE check_env(const char *ftok)
	547	{
	548	const char p, q, *pp;
	549	int d;
	550
	551	p = getenv("CATACOMB_CPUFEAT");
	552	if (!p) return (-1);
	553
	554	for (;;) {
	555	while (isspace((unsigned char)*p)) p++;
	556	if (!*p) return (-1);
	557	switch (*p) {
	558	case '+': d = +1; p++; break;
	559	case '-': d = 0; p++; break;
	560	default: d = -1; break;
	561	}
	562	for (q = p; q && !isspace((unsigned char)q); q++);
	563	if (d >= 0) {
	564	for (pp = ftok; p < q && pp && p == *pp; p++, pp++);
	565	if ((p == q && !pp) \|\| (p == '*' && p + 1 == q)) return (d);
	566	}
	567	p = q;
	568	}
	569	return (-1);
	570	}
	571
	572	/* --- @cpu_feature_p@ --- *
	573	*
	574	* Arguments: @unsigned feat@ = a @CPUFEAT_...@ code
	575	*
	576	* Returns: Nonzero if the feature is available.
	577	*/
	578
	579	#include <stdio.h>
	580
fac645f7 MW	581	static int IGNORABLE
	582	feat_debug(const char ftok, const char check, int verdict)
	583	{
	584	if (verdict >= 0) {
	585	dispatch_debug("feature `%s': %s -> %s", ftok, check,
	586	verdict ? "available" : "absent");
	587	}
	588	return (verdict);
	589	}
	590
08e2be29 MW	591	int cpu_feature_p(int feat)
	592	{
	593	int IGNORABLE f;
	594	IGNORE(f);
fac645f7 MW	595	#define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \
	596	if ((f = feat_debug(ftok, "environment override", \
	597	check_env(ftok))) >= 0) \
	598	return (f); \
	599	else \
	600	return (feat_debug(ftok, "runtime probe", cond));
08e2be29 MW	601
08e2be29 MW	602	switch (feat) {
0f23f75f	603	#if CPUFAM_X86 \|\| CPUFAM_AMD64
fac645f7	604	CASE_CPUFEAT(X86_SSE2, "x86:sse2",
4fab22c2 MW	605	cpuid_features_p(CPUID1D_SSE2, 0) &&
4fab22c2 MW	606	xmm_registers_available_p());
fac645f7	607	CASE_CPUFEAT(X86_AESNI, "x86:aesni",
4fab22c2 MW	608	cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
4fab22c2 MW	609	xmm_registers_available_p());
d25653be	610	CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
35b1eba8	611	cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p());
b9b279b4	612	CASE_CPUFEAT(X86_AVX, "x86:avx",
6af2607b MW	613	cpuid_features_p(0, CPUID1C_AVX) &&
6af2607b MW	614	xmm_registers_available_p());
9e6a4409	615	CASE_CPUFEAT(X86_SSSE3, "x86:ssse3",
6af2607b MW	616	cpuid_features_p(0, CPUID1C_SSSE3) &&
6af2607b MW	617	xmm_registers_available_p());
9e6a4409	618	CASE_CPUFEAT(X86_PCLMUL, "x86:pclmul",
6af2607b MW	619	cpuid_features_p(0, CPUID1C_PCLMUL) &&
6af2607b MW	620	xmm_registers_available_p());
a02a22d4 MW	621	#endif
	622	#ifdef CAPMAP
	623	# define FEATP__CASE(feat, tok) \
0aec0658	624	CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
a02a22d4 MW	625	CAPMAP(FEATP__CASE)
a02a22d4 MW	626	#undef FEATP__CASE
08e2be29 MW	627	#endif
08e2be29 MW	628	default:
fac645f7	629	dispatch_debug("denying unknown feature %d", feat);
08e2be29 MW	630	return (0);
08e2be29 MW	631	}
fac645f7	632	#undef CASE_CPUFEAT
08e2be29 MW	633	}
	634
	635	/----- That's all, folks -------------------------------------------------/