3 * CPU-specific dispatch
5 * (c) 2015 Straylight/Edgeware
8 /*----- Licensing notice --------------------------------------------------*
10 * This file is part of Catacomb.
12 * Catacomb is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU Library General Public License as
14 * published by the Free Software Foundation; either version 2 of the
15 * License, or (at your option) any later version.
17 * Catacomb is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU Library General Public License for more details.
22 * You should have received a copy of the GNU Library General Public
23 * License along with Catacomb; if not, write to the Free
24 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
28 /*----- Header files ------------------------------------------------------*/
38 #include <mLib/macros.h>
42 /*----- Intel x86/AMD64 feature probing -----------------------------------*/
44 #if CPUFAM_X86 || CPUFAM_AMD64
46 # define EFLAGS_ID (1u << 21)
47 # define CPUID1D_SSE2 (1u << 26)
48 # define CPUID1D_FXSR (1u << 24)
49 # define CPUID1C_AESNI (1u << 25)
50 # define CPUID1C_RDRAND (1u << 30)
52 struct cpuid { unsigned a, b, c, d; };
56 * Arguments: @struct cpuid *cc@ = where to write the result
57 * @unsigned a, c@ = EAX and ECX registers to set
61 * Use: Minimal C wrapper around the x86 `CPUID' instruction. Checks
62 * that the instruction is actually available before invoking
63 * it; fills the output structure with zero if it's not going to
69 static __inline__ unsigned getflags(void)
70 { unsigned f; __asm__ ("pushf; popl %0" : "=g" (f)); return (f); }
71 static __inline__ unsigned setflags(unsigned f)
74 __asm__ ("pushf; pushl %1; popf; pushf; popl %0; popf"
80 static __inline__ unsigned long getflags(void)
81 { unsigned long f; __asm__ ("pushf; popq %0" : "=g" (f)); return (f); }
82 static __inline__ unsigned long long setflags(unsigned long f)
85 __asm__ ("pushf; pushq %1; popf; pushf; popq %0; popf"
93 static void cpuid(struct cpuid *cc, unsigned a, unsigned c)
99 cc->a = cc->b = cc->c = cc->d = 0;
102 /* Stupid dance to detect whether the CPUID instruction is available. */
104 if (!(setflags(f | EFLAGS_ID) & EFLAGS_ID) ||
105 setflags(f & ~EFLAGS_ID) & EFLAGS_ID) {
106 dispatch_debug("CPUID instruction not available");
111 /* Alas, EBX is magical in PIC code, so abuse ESI instead. This isn't
112 * pretty, but it works.
115 __asm__ ("pushl %%ebx; cpuid; movl %%ebx, %%esi; popl %%ebx"
116 : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
117 : "a" (a) , "c" (c));
119 __asm__ ("pushq %%rbx; cpuid; movl %%ebx, %%esi; popq %%rbx"
120 : "=a" (cc->a), "=S" (cc->b), "=c" (cc->c), "=d" (cc->d)
121 : "a" (a) , "c" (c));
123 # error "I'm confused."
125 dispatch_debug("CPUID(%08x, %08x) -> %08x, %08x, %08x, %08x",
126 a, c, cc->a, cc->b, cc->c, cc->d);
128 dispatch_debug("GNU inline assembler not available; can't CPUID");
132 static unsigned cpuid_maxleaf(void)
133 { struct cpuid c; cpuid(&c, 0, 0); return (c.a); }
135 /* --- @cpuid_features_p@ --- *
137 * Arguments: @unsigned dbits@ = bits to check in EDX
138 * @unsigned cbits@ = bits to check in ECX
140 * Returns: Nonzero if all the requested bits are set in the CPUID result
144 static int cpuid_features_p(unsigned dbits, unsigned cbits)
147 if (cpuid_maxleaf() < 1) return (0);
149 return ((c.d & dbits) == dbits && (c.c & cbits) == cbits);
152 /* --- @xmm_registers_available_p@ --- *
156 * Returns: Nonzero if the operating system has made the XMM registers
160 static int xmm_registers_available_p(void)
164 /* This hack is by Agner Fog. Use FXSAVE/FXRSTOR to figure out whether the
165 * XMM registers are actually alive.
167 if (!cpuid_features_p(CPUID1D_FXSR, 0)) return (0);
169 __asm__ ("movl %%esp, %%edx; subl $512, %%esp; andl $~15, %%esp\n"
171 "movl 160(%%esp), %%eax; xorl $0xaaaa5555, 160(%%esp)\n"
172 "fxrstor (%%esp); fxsave (%%esp)\n"
173 "movl 160(%%esp), %%ecx; movl %%eax, 160(%%esp)\n"
174 "fxrstor (%%esp); movl %%edx, %%esp\n"
180 __asm__ ("movq %%rsp, %%rdx; subq $512, %%rsp; andq $~15, %%rsp\n"
182 "movl 160(%%rsp), %%eax; xorl $0xaaaa5555, 160(%%rsp)\n"
183 "fxrstor (%%rsp); fxsave (%%rsp)\n"
184 "movl 160(%%rsp), %%ecx; movl %%eax, 160(%%rsp)\n"
185 "fxrstor (%%rsp); movq %%rdx, %%rsp\n"
191 # error "I'm confused."
193 dispatch_debug("XMM registers %savailable", f ? "" : "not ");
196 dispatch_debug("GNU inline assembler not available; can't check for XMM");
201 /* --- @rdrand_works_p@ --- *
206 * Returns: Nonzero if the `rdrand' instruction actually works. Assumes
207 * that it's already been verified to be safe to issue.
211 static int rdrand(unsigned *x)
217 __asm__ ("" : "=g" (_t));
218 __asm__ ("0: rdrand %2; jc 1f; decl %1; jnz 0b\n"
219 "mov $-1, %0; jmp 9f\n"
220 "1: movl %2, (%3); xorl %0, %0\n"
222 : "=r" (rc), "+r" (i), "+r" (_t)
229 static int rdrand_works_p(void)
233 /* Check that it doesn't always give the same answer. Try four times: this
234 * will fail with probability %$2^{-128}$% with a truly random generator,
235 * which seems fair enough.
237 if (rdrand(&ref)) goto fail;
238 for (i = 0; i < 4; i++) {
239 if (rdrand(&x)) goto fail;
240 if (x != ref) goto not_stuck;
242 dispatch_debug("RDRAND always returns 0x%08x!", ref);
246 dispatch_debug("RDRAND instruction looks plausible");
250 dispatch_debug("RDRAND instruction fails too often");
256 /*----- General feature probing using auxiliary vectors -------------------*/
258 /* Try to find the system's definitions for auxiliary vector entries. */
259 #ifdef HAVE_SYS_AUXV_H
260 # include <sys/auxv.h>
262 #ifdef HAVE_LINUX_AUXVEC_H
263 # include <linux/auxvec.h>
265 #ifdef HAVE_ASM_HWCAP_H
266 # include <asm/hwcap.h>
269 /* The type of entries in the auxiliary vector. I'm assuming that `unsigned
270 * long' matches each platform's word length; if this is false then we'll
271 * need some host-specific tweaking here.
273 union auxval { long i; unsigned long u; const void *p; };
274 struct auxentry { unsigned long type; union auxval value; };
276 /* Register each CPU family's interest in the auxiliary vector. Make sure
277 * that the necessary entry types are defined. This is primarily ordered by
278 * entry type to minimize duplication.
280 #if defined(AT_HWCAP) && CPUFAM_ARMEL
282 # define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
285 #if defined(AT_HWCAP) && CPUFAM_ARM64
287 # define WANT_AT_HWCAP(_) _(AT_HWCAP, u, hwcap)
290 #if defined(AT_HWCAP2) && CPUFAM_ARMEL
292 # define WANT_AT_HWCAP2(_) _(AT_HWCAP2, u, hwcap2)
295 /* If we couldn't find any interesting entries then we can switch all of this
296 * machinery off. Also do that if we have no means for atomic updates.
298 #if WANT_ANY && CPU_DISPATCH_P
300 /* The main output of this section is a bitmask of detected features. The
301 * least significant bit will be set if we've tried to probe. Always access
302 * this using `DISPATCH_LOAD' and `DISPATCH_STORE'.
304 static unsigned hwcaps = 0;
306 /* For each potentially interesting type which turned out not to exist or be
307 * wanted, define a dummy macro for the sake of the next step.
309 #ifndef WANT_AT_HWCAP
310 # define WANT_AT_HWCAP(_)
312 #ifndef WANT_AT_HWCAP2
313 # define WANT_AT_HWCAP2(_)
316 /* For each CPU family, define two lists.
318 * * `WANTAUX' is a list of the `WANT_AT_MUMBLE' macros which the CPU
319 * family tried to register interest in above. Each entry contains the
320 * interesting auxiliary vector entry type, the name of the union branch
321 * for its value, and the name of the slot in `struct auxprobe' in which
322 * to store the value.
324 * * `CAPMAP' is a list describing the output features which the CPU family
325 * intends to satisfy from the auxiliary vector. Each entry contains a
326 * feature name suffix, and the token name (for `check_env').
329 # define WANTAUX(_) \
333 _(ARM_VFP, "arm:vfp") \
334 _(ARM_NEON, "arm:neon") \
335 _(ARM_V4, "arm:v4") \
336 _(ARM_D32, "arm:d32") \
337 _(ARM_AES, "arm:aes")
340 # define WANTAUX(_) \
343 _(ARM_AES, "arm:aes")
346 /* Build the bitmask for `hwcaps' from the `CAPMAP' list. */
349 #define HFI__ENUM(feat, tok) HFI_##feat,
356 #define HF__FLAG(feat, tok) HF_##feat = 1 << HFI_##feat,
362 /* Build a structure in which we can capture the interesting data from the
365 #define AUXUTYPE_i long
366 #define AUXUTYPE_u unsigned long
367 #define AUXUTYPE_p const void *
369 #define AUXPROBE__SLOT(type, ubranch, slot) AUXUTYPE_##ubranch slot;
370 WANTAUX(AUXPROBE__SLOT)
374 /* --- @probe_hwcaps@ --- *
380 * Use: Attempt to find the auxiliary vector (which is well hidden)
381 * and discover interesting features from it.
384 static void probe_hwcaps(void)
386 unsigned hw = HF_PROBED;
387 struct auxprobe probed = { 0 };
389 /* Populate `probed' with the information we manage to retrieve from the
390 * auxiliary vector. Slots we couldn't find are left zero-valued.
392 #if defined(HAVE_GETAUXVAL)
393 /* Shiny new libc lets us request individual entry types. This is almost
396 # define CAP__GET(type, ubranch, slot) \
397 probed.slot = (AUXUTYPE_##ubranch)getauxval(type);
400 /* Otherwise we're a bit stuck, really. Modern Linux kernels make a copy
401 * of the vector available in `/procc' so we could try that.
403 * The usual place is stuck on the end of the environment vector, but that
404 * may well have moved, and we have no way of telling whether it has or
405 * whether there was ever an auxiliary vector there at all; so don't do
410 unsigned char *p = 0, *q = 0;
411 const struct auxentry *a;
414 /* Open the file and read it into a memory chunk. */
415 if ((fp = fopen("/proc/self/auxv", "rb")) == 0) goto clean;
417 if ((p = malloc(sz)) == 0) goto clean;
419 n = fread(p + off, 1, sz - off, fp);
422 sz *= 2; if ((q = realloc(p, sz)) == 0) break;
426 /* Work through the vector (or as much of it as we found) and extract the
427 * types we're interested in.
429 for (a = (const struct auxentry *)p,
430 n = sz/sizeof(struct auxentry);
433 #define CAP__SWITCH(type, ubranch, slot) \
434 case type: probed.slot = a->value.ubranch; break;
436 case AT_NULL: goto clean;
446 /* Each CPU family now has to pick through what was found and stashed in
447 * `probed', and set the appropriate flag bits in `hw'.
450 if (probed.hwcap & HWCAP_VFPv3) hw |= HF_ARM_VFP;
451 if (probed.hwcap & HWCAP_NEON) hw |= HF_ARM_NEON;
452 if (probed.hwcap & HWCAP_VFPD32) hw |= HF_ARM_D32;
453 if (probed.hwcap & HWCAP_VFPv4) hw |= HF_ARM_V4;
455 if (probed.hwcap2 & HWCAP2_AES) hw |= HF_ARM_AES;
459 if (probed.hwcap & HWCAP_AES) hw |= HF_ARM_AES;
462 /* Store the bitmask of features we probed for everyone to see. */
463 DISPATCH_STORE(hwcaps, hw);
465 /* Finally, make a report about the things we found. (Doing this earlier
466 * will pointlessly widen the window in which multiple threads will do the
467 * above auxiliary-vector probing.)
469 #define CAP__DEBUG(feat, tok) \
470 dispatch_debug("check auxv for feature `%s': %s", tok, \
471 hw & HF_##feat ? "available" : "absent");
476 /* --- @get_hwcaps@ --- *
480 * Returns: A mask of hardware capabilities and other features, as probed
481 * from the auxiliary vector.
484 static unsigned get_hwcaps(void)
488 DISPATCH_LOAD(hwcaps, hw);
489 if (!(hwcaps & HF_PROBED)) { probe_hwcaps(); DISPATCH_LOAD(hwcaps, hw); }
495 /*----- External interface ------------------------------------------------*/
497 /* --- @dispatch_debug@ --- *
499 * Arguments: @const char *fmt@ = a format string
500 * @...@ = additional arguments
504 * Use: Writes a formatted message to standard output if dispatch
505 * debugging is enabled.
508 void dispatch_debug(const char *fmt, ...)
511 const char *e = getenv("CATACOMB_CPUDISPATCH_DEBUG");
513 if (e && *e != 'n' && *e != '0') {
515 fputs("Catacomb CPUDISPATCH: ", stderr);
516 vfprintf(stderr, fmt, ap);
522 /* --- @check_env@ --- *
524 * Arguments: @const char *ftok@ = feature token
526 * Returns: Zero if the feature is forced off; positive if it's forced
527 * on; negative if the user hasn't decided.
529 * Use: Checks the environment variable `CATACOMB_CPUFEAT' for the
530 * feature token @ftok@. The variable, if it exists, should be
531 * a space-separated sequence of `+tok' and `-tok' items. These
532 * tokens may end in `*', which matches any suffix.
535 static int IGNORABLE check_env(const char *ftok)
537 const char *p, *q, *pp;
540 p = getenv("CATACOMB_CPUFEAT");
544 while (isspace((unsigned char)*p)) p++;
545 if (!*p) return (-1);
547 case '+': d = +1; p++; break;
548 case '-': d = 0; p++; break;
549 default: d = -1; break;
551 for (q = p; *q && !isspace((unsigned char)*q); q++);
553 for (pp = ftok; p < q && *pp && *p == *pp; p++, pp++);
554 if ((p == q && !*pp) || (*p == '*' && p + 1 == q)) return (d);
561 /* --- @cpu_feature_p@ --- *
563 * Arguments: @unsigned feat@ = a @CPUFEAT_...@ code
565 * Returns: Nonzero if the feature is available.
571 feat_debug(const char *ftok, const char *check, int verdict)
574 dispatch_debug("feature `%s': %s -> %s", ftok, check,
575 verdict ? "available" : "absent");
580 int cpu_feature_p(int feat)
584 #define CASE_CPUFEAT(feat, ftok, cond) case CPUFEAT_##feat: \
585 if ((f = feat_debug(ftok, "environment override", \
586 check_env(ftok))) >= 0) \
589 return (feat_debug(ftok, "runtime probe", cond));
592 #if CPUFAM_X86 || CPUFAM_AMD64
593 CASE_CPUFEAT(X86_SSE2, "x86:sse2",
594 cpuid_features_p(CPUID1D_SSE2, 0) &&
595 xmm_registers_available_p());
596 CASE_CPUFEAT(X86_AESNI, "x86:aesni",
597 cpuid_features_p(CPUID1D_SSE2, CPUID1C_AESNI) &&
598 xmm_registers_available_p());
599 CASE_CPUFEAT(X86_RDRAND, "x86:rdrand",
600 cpuid_features_p(0, CPUID1C_RDRAND) && rdrand_works_p());
603 # define FEATP__CASE(feat, tok) \
604 CASE_CPUFEAT(feat, tok, get_hwcaps() & HF_##feat)
609 dispatch_debug("denying unknown feature %d", feat);
615 /*----- That's all, folks -------------------------------------------------*/