sljit/sljitNativeX86_32.c

   1 /*
   2  *    Stack-less Just-In-Time compiler
   3  *
   4  *    Copyright 2009-2012 Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without modification, are
   7  * permitted provided that the following conditions are met:
   8  *
   9  *   1. Redistributions of source code must retain the above copyright notice, this list of
  10  *      conditions and the following disclaimer.
  11  *
  12  *   2. Redistributions in binary form must reproduce the above copyright notice, this list
  13  *      of conditions and the following disclaimer in the documentation and/or other materials
  14  *      provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
  17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
  19  * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  21  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  22  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  24  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* x86 32-bit arch dependent functions. */
  28
  29 static sljit_si emit_do_imm(struct sljit_compiler *compiler, sljit_ub opcode, sljit_sw imm)
  30 {
  31         sljit_ub *inst;
  32
  33         inst = (sljit_ub*)ensure_buf(compiler, 1 + 1 + sizeof(sljit_sw));
  34         FAIL_IF(!inst);
  35         INC_SIZE(1 + sizeof(sljit_sw));
  36         *inst++ = opcode;
  37         *(sljit_sw*)inst = imm;
  38         return SLJIT_SUCCESS;
  39 }
  40
  41 static sljit_ub* generate_far_jump_code(struct sljit_jump *jump, sljit_ub *code_ptr, sljit_si type)
  42 {
  43         if (type == SLJIT_JUMP) {
  44                 *code_ptr++ = JMP_i32;
  45                 jump->addr++;
  46         }
  47         else if (type >= SLJIT_FAST_CALL) {
  48                 *code_ptr++ = CALL_i32;
  49                 jump->addr++;
  50         }
  51         else {
  52                 *code_ptr++ = GROUP_0F;
  53                 *code_ptr++ = get_jump_code(type);
  54                 jump->addr += 2;
  55         }
  56
  57         if (jump->flags & JUMP_LABEL)
  58                 jump->flags |= PATCH_MW;
  59         else
  60                 *(sljit_sw*)code_ptr = jump->u.target - (jump->addr + 4);
  61         code_ptr += 4;
  62
  63         return code_ptr;
  64 }
  65
  66 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_enter(struct sljit_compiler *compiler, sljit_si args, sljit_si scratches, sljit_si saveds, sljit_si local_size)
  67 {
  68         sljit_si size;
  69         sljit_si locals_offset;
  70         sljit_ub *inst;
  71
  72         CHECK_ERROR();
  73         check_sljit_emit_enter(compiler, args, scratches, saveds, local_size);
  74
  75         compiler->scratches = scratches;
  76         compiler->saveds = saveds;
  77         compiler->args = args;
  78         compiler->flags_saved = 0;
  79 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
  80         compiler->logical_local_size = local_size;
  81 #endif
  82
  83 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
  84         size = 1 + (saveds <= 3 ? saveds : 3) + (args > 0 ? (args * 2) : 0) + (args > 2 ? 2 : 0);
  85 #else
  86         size = 1 + (saveds <= 3 ? saveds : 3) + (args > 0 ? (2 + args * 3) : 0);
  87 #endif
  88         inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
  89         FAIL_IF(!inst);
  90
  91         INC_SIZE(size);
  92         PUSH_REG(reg_map[TMP_REG1]);
  93 #if !(defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
  94         if (args > 0) {
  95                 *inst++ = MOV_r_rm;
  96                 *inst++ = MOD_REG | (reg_map[TMP_REG1] << 3) | 0x4 /* esp */;
  97         }
  98 #endif
  99         if (saveds > 2)
 100                 PUSH_REG(reg_map[SLJIT_SAVED_REG3]);
 101         if (saveds > 1)
 102                 PUSH_REG(reg_map[SLJIT_SAVED_REG2]);
 103         if (saveds > 0)
 104                 PUSH_REG(reg_map[SLJIT_SAVED_REG1]);
 105
 106 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 107         if (args > 0) {
 108                 *inst++ = MOV_r_rm;
 109                 *inst++ = MOD_REG | (reg_map[SLJIT_SAVED_REG1] << 3) | reg_map[SLJIT_SCRATCH_REG3];
 110         }
 111         if (args > 1) {
 112                 *inst++ = MOV_r_rm;
 113                 *inst++ = MOD_REG | (reg_map[SLJIT_SAVED_REG2] << 3) | reg_map[SLJIT_SCRATCH_REG2];
 114         }
 115         if (args > 2) {
 116                 *inst++ = MOV_r_rm;
 117                 *inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG3] << 3) | 0x4 /* esp */;
 118                 *inst++ = 0x24;
 119                 *inst++ = sizeof(sljit_sw) * (3 + 2); /* saveds >= 3 as well. */
 120         }
 121 #else
 122         if (args > 0) {
 123                 *inst++ = MOV_r_rm;
 124                 *inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG1] << 3) | reg_map[TMP_REG1];
 125                 *inst++ = sizeof(sljit_sw) * 2;
 126         }
 127         if (args > 1) {
 128                 *inst++ = MOV_r_rm;
 129                 *inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG2] << 3) | reg_map[TMP_REG1];
 130                 *inst++ = sizeof(sljit_sw) * 3;
 131         }
 132         if (args > 2) {
 133                 *inst++ = MOV_r_rm;
 134                 *inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG3] << 3) | reg_map[TMP_REG1];
 135                 *inst++ = sizeof(sljit_sw) * 4;
 136         }
 137 #endif
 138
 139 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 140         locals_offset = 2 * sizeof(sljit_uw);
 141 #else
 142         SLJIT_COMPILE_ASSERT(FIXED_LOCALS_OFFSET >= 2 * sizeof(sljit_uw), require_at_least_two_words);
 143         locals_offset = FIXED_LOCALS_OFFSET;
 144 #endif
 145         compiler->scratches_start = locals_offset;
 146         if (scratches > 3)
 147                 locals_offset += (scratches - 3) * sizeof(sljit_uw);
 148         compiler->saveds_start = locals_offset;
 149         if (saveds > 3)
 150                 locals_offset += (saveds - 3) * sizeof(sljit_uw);
 151         compiler->locals_offset = locals_offset;
 152 #if defined(__APPLE__)
 153         saveds = (2 + (saveds <= 3 ? saveds : 3)) * sizeof(sljit_uw);
 154         local_size = ((locals_offset + saveds + local_size + 15) & ~15) - saveds;
 155 #else
 156         local_size = locals_offset + ((local_size + sizeof(sljit_uw) - 1) & ~(sizeof(sljit_uw) - 1));
 157 #endif
 158
 159         compiler->local_size = local_size;
 160 #ifdef _WIN32
 161         if (local_size > 1024) {
 162 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 163                 FAIL_IF(emit_do_imm(compiler, MOV_r_i32 + reg_map[SLJIT_SCRATCH_REG1], local_size));
 164 #else
 165                 local_size -= FIXED_LOCALS_OFFSET;
 166                 FAIL_IF(emit_do_imm(compiler, MOV_r_i32 + reg_map[SLJIT_SCRATCH_REG1], local_size));
 167                 FAIL_IF(emit_non_cum_binary(compiler, SUB_r_rm, SUB_rm_r, SUB, SUB_EAX_i32,
 168                         SLJIT_LOCALS_REG, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, FIXED_LOCALS_OFFSET));
 169 #endif
 170                 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_CALL1, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_grow_stack)));
 171         }
 172 #endif
 173
 174         SLJIT_ASSERT(local_size > 0);
 175         return emit_non_cum_binary(compiler, SUB_r_rm, SUB_rm_r, SUB, SUB_EAX_i32,
 176                 SLJIT_LOCALS_REG, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, local_size);
 177 }
 178
 179 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_context(struct sljit_compiler *compiler, sljit_si args, sljit_si scratches, sljit_si saveds, sljit_si local_size)
 180 {
 181         sljit_si locals_offset;
 182
 183         CHECK_ERROR_VOID();
 184         check_sljit_set_context(compiler, args, scratches, saveds, local_size);
 185
 186         compiler->scratches = scratches;
 187         compiler->saveds = saveds;
 188         compiler->args = args;
 189 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
 190         compiler->logical_local_size = local_size;
 191 #endif
 192
 193 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 194         locals_offset = 2 * sizeof(sljit_uw);
 195 #else
 196         locals_offset = FIXED_LOCALS_OFFSET;
 197 #endif
 198         compiler->scratches_start = locals_offset;
 199         if (scratches > 3)
 200                 locals_offset += (scratches - 3) * sizeof(sljit_uw);
 201         compiler->saveds_start = locals_offset;
 202         if (saveds > 3)
 203                 locals_offset += (saveds - 3) * sizeof(sljit_uw);
 204         compiler->locals_offset = locals_offset;
 205 #if defined(__APPLE__)
 206         saveds = (2 + (saveds <= 3 ? saveds : 3)) * sizeof(sljit_uw);
 207         compiler->local_size = ((locals_offset + saveds + local_size + 15) & ~15) - saveds;
 208 #else
 209         compiler->local_size = locals_offset + ((local_size + sizeof(sljit_uw) - 1) & ~(sizeof(sljit_uw) - 1));
 210 #endif
 211 }
 212
 213 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_return(struct sljit_compiler *compiler, sljit_si op, sljit_si src, sljit_sw srcw)
 214 {
 215         sljit_si size;
 216         sljit_ub *inst;
 217
 218         CHECK_ERROR();
 219         check_sljit_emit_return(compiler, op, src, srcw);
 220         SLJIT_ASSERT(compiler->args >= 0);
 221
 222         compiler->flags_saved = 0;
 223         FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
 224
 225         SLJIT_ASSERT(compiler->local_size > 0);
 226         FAIL_IF(emit_cum_binary(compiler, ADD_r_rm, ADD_rm_r, ADD, ADD_EAX_i32,
 227                 SLJIT_LOCALS_REG, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, compiler->local_size));
 228
 229         size = 2 + (compiler->saveds <= 3 ? compiler->saveds : 3);
 230 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 231         if (compiler->args > 2)
 232                 size += 2;
 233 #else
 234         if (compiler->args > 0)
 235                 size += 2;
 236 #endif
 237         inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
 238         FAIL_IF(!inst);
 239
 240         INC_SIZE(size);
 241
 242         if (compiler->saveds > 0)
 243                 POP_REG(reg_map[SLJIT_SAVED_REG1]);
 244         if (compiler->saveds > 1)
 245                 POP_REG(reg_map[SLJIT_SAVED_REG2]);
 246         if (compiler->saveds > 2)
 247                 POP_REG(reg_map[SLJIT_SAVED_REG3]);
 248         POP_REG(reg_map[TMP_REG1]);
 249 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 250         if (compiler->args > 2)
 251                 RET_I16(sizeof(sljit_sw));
 252         else
 253                 RET();
 254 #else
 255         RET();
 256 #endif
 257
 258         return SLJIT_SUCCESS;
 259 }
 260
 261 /* --------------------------------------------------------------------- */
 262 /*  Operators                                                            */
 263 /* --------------------------------------------------------------------- */
 264
 265 /* Size contains the flags as well. */
 266 static sljit_ub* emit_x86_instruction(struct sljit_compiler *compiler, sljit_si size,
 267         /* The register or immediate operand. */
 268         sljit_si a, sljit_sw imma,
 269         /* The general operand (not immediate). */
 270         sljit_si b, sljit_sw immb)
 271 {
 272         sljit_ub *inst;
 273         sljit_ub *buf_ptr;
 274         sljit_si flags = size & ~0xf;
 275         sljit_si inst_size;
 276
 277         /* Both cannot be switched on. */
 278         SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
 279         /* Size flags not allowed for typed instructions. */
 280         SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
 281         /* Both size flags cannot be switched on. */
 282         SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
 283 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
 284         /* SSE2 and immediate is not possible. */
 285         SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2));
 286         SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3)
 287                 && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66)
 288                 && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66));
 289 #endif
 290
 291         size &= 0xf;
 292         inst_size = size;
 293
 294 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
 295         if (flags & (EX86_PREF_F2 | EX86_PREF_F3))
 296                 inst_size++;
 297 #endif
 298         if (flags & EX86_PREF_66)
 299                 inst_size++;
 300
 301         /* Calculate size of b. */
 302         inst_size += 1; /* mod r/m byte. */
 303         if (b & SLJIT_MEM) {
 304                 if ((b & REG_MASK) == SLJIT_UNUSED)
 305                         inst_size += sizeof(sljit_sw);
 306                 else if (immb != 0 && !(b & OFFS_REG_MASK)) {
 307                         /* Immediate operand. */
 308                         if (immb <= 127 && immb >= -128)
 309                                 inst_size += sizeof(sljit_sb);
 310                         else
 311                                 inst_size += sizeof(sljit_sw);
 312                 }
 313
 314                 if ((b & REG_MASK) == SLJIT_LOCALS_REG && !(b & OFFS_REG_MASK))
 315                         b |= TO_OFFS_REG(SLJIT_LOCALS_REG);
 316
 317                 if ((b & OFFS_REG_MASK) != SLJIT_UNUSED)
 318                         inst_size += 1; /* SIB byte. */
 319         }
 320
 321         /* Calculate size of a. */
 322         if (a & SLJIT_IMM) {
 323                 if (flags & EX86_BIN_INS) {
 324                         if (imma <= 127 && imma >= -128) {
 325                                 inst_size += 1;
 326                                 flags |= EX86_BYTE_ARG;
 327                         } else
 328                                 inst_size += 4;
 329                 }
 330                 else if (flags & EX86_SHIFT_INS) {
 331                         imma &= 0x1f;
 332                         if (imma != 1) {
 333                                 inst_size ++;
 334                                 flags |= EX86_BYTE_ARG;
 335                         }
 336                 } else if (flags & EX86_BYTE_ARG)
 337                         inst_size++;
 338                 else if (flags & EX86_HALF_ARG)
 339                         inst_size += sizeof(short);
 340                 else
 341                         inst_size += sizeof(sljit_sw);
 342         }
 343         else
 344                 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
 345
 346         inst = (sljit_ub*)ensure_buf(compiler, 1 + inst_size);
 347         PTR_FAIL_IF(!inst);
 348
 349         /* Encoding the byte. */
 350         INC_SIZE(inst_size);
 351 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
 352         if (flags & EX86_PREF_F2)
 353                 *inst++ = 0xf2;
 354         if (flags & EX86_PREF_F3)
 355                 *inst++ = 0xf3;
 356 #endif
 357         if (flags & EX86_PREF_66)
 358                 *inst++ = 0x66;
 359
 360         buf_ptr = inst + size;
 361
 362         /* Encode mod/rm byte. */
 363         if (!(flags & EX86_SHIFT_INS)) {
 364                 if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM))
 365                         *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
 366
 367                 if ((a & SLJIT_IMM) || (a == 0))
 368                         *buf_ptr = 0;
 369 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
 370                 else if (!(flags & EX86_SSE2))
 371                         *buf_ptr = reg_map[a] << 3;
 372                 else
 373                         *buf_ptr = a << 3;
 374 #else
 375                 else
 376                         *buf_ptr = reg_map[a] << 3;
 377 #endif
 378         }
 379         else {
 380                 if (a & SLJIT_IMM) {
 381                         if (imma == 1)
 382                                 *inst = GROUP_SHIFT_1;
 383                         else
 384                                 *inst = GROUP_SHIFT_N;
 385                 } else
 386                         *inst = GROUP_SHIFT_CL;
 387                 *buf_ptr = 0;
 388         }
 389
 390         if (!(b & SLJIT_MEM))
 391 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
 392                 *buf_ptr++ |= MOD_REG + ((!(flags & EX86_SSE2)) ? reg_map[b] : b);
 393 #else
 394                 *buf_ptr++ |= MOD_REG + reg_map[b];
 395 #endif
 396         else if ((b & REG_MASK) != SLJIT_UNUSED) {
 397                 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_LOCALS_REG)) {
 398                         if (immb != 0) {
 399                                 if (immb <= 127 && immb >= -128)
 400                                         *buf_ptr |= 0x40;
 401                                 else
 402                                         *buf_ptr |= 0x80;
 403                         }
 404
 405                         if ((b & OFFS_REG_MASK) == SLJIT_UNUSED)
 406                                 *buf_ptr++ |= reg_map[b & REG_MASK];
 407                         else {
 408                                 *buf_ptr++ |= 0x04;
 409                                 *buf_ptr++ = reg_map[b & REG_MASK] | (reg_map[OFFS_REG(b)] << 3);
 410                         }
 411
 412                         if (immb != 0) {
 413                                 if (immb <= 127 && immb >= -128)
 414                                         *buf_ptr++ = immb; /* 8 bit displacement. */
 415                                 else {
 416                                         *(sljit_sw*)buf_ptr = immb; /* 32 bit displacement. */
 417                                         buf_ptr += sizeof(sljit_sw);
 418                                 }
 419                         }
 420                 }
 421                 else {
 422                         *buf_ptr++ |= 0x04;
 423                         *buf_ptr++ = reg_map[b & REG_MASK] | (reg_map[OFFS_REG(b)] << 3) | (immb << 6);
 424                 }
 425         }
 426         else {
 427                 *buf_ptr++ |= 0x05;
 428                 *(sljit_sw*)buf_ptr = immb; /* 32 bit displacement. */
 429                 buf_ptr += sizeof(sljit_sw);
 430         }
 431
 432         if (a & SLJIT_IMM) {
 433                 if (flags & EX86_BYTE_ARG)
 434                         *buf_ptr = imma;
 435                 else if (flags & EX86_HALF_ARG)
 436                         *(short*)buf_ptr = imma;
 437                 else if (!(flags & EX86_SHIFT_INS))
 438                         *(sljit_sw*)buf_ptr = imma;
 439         }
 440
 441         return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1);
 442 }
 443
 444 /* --------------------------------------------------------------------- */
 445 /*  Call / return instructions                                           */
 446 /* --------------------------------------------------------------------- */
 447
 448 static SLJIT_INLINE sljit_si call_with_args(struct sljit_compiler *compiler, sljit_si type)
 449 {
 450         sljit_ub *inst;
 451
 452 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
 453         inst = (sljit_ub*)ensure_buf(compiler, type >= SLJIT_CALL3 ? 1 + 2 + 1 : 1 + 2);
 454         FAIL_IF(!inst);
 455         INC_SIZE(type >= SLJIT_CALL3 ? 2 + 1 : 2);
 456
 457         if (type >= SLJIT_CALL3)
 458                 PUSH_REG(reg_map[SLJIT_SCRATCH_REG3]);
 459         *inst++ = MOV_r_rm;
 460         *inst++ = MOD_REG | (reg_map[SLJIT_SCRATCH_REG3] << 3) | reg_map[SLJIT_SCRATCH_REG1];
 461 #else
 462         inst = (sljit_ub*)ensure_buf(compiler, 1 + 4 * (type - SLJIT_CALL0));
 463         FAIL_IF(!inst);
 464         INC_SIZE(4 * (type - SLJIT_CALL0));
 465
 466         *inst++ = MOV_rm_r;
 467         *inst++ = MOD_DISP8 | (reg_map[SLJIT_SCRATCH_REG1] << 3) | 0x4 /* SIB */;
 468         *inst++ = (0x4 /* none*/ << 3) | reg_map[SLJIT_LOCALS_REG];
 469         *inst++ = 0;
 470         if (type >= SLJIT_CALL2) {
 471                 *inst++ = MOV_rm_r;
 472                 *inst++ = MOD_DISP8 | (reg_map[SLJIT_SCRATCH_REG2] << 3) | 0x4 /* SIB */;
 473                 *inst++ = (0x4 /* none*/ << 3) | reg_map[SLJIT_LOCALS_REG];
 474                 *inst++ = sizeof(sljit_sw);
 475         }
 476         if (type >= SLJIT_CALL3) {
 477                 *inst++ = MOV_rm_r;
 478                 *inst++ = MOD_DISP8 | (reg_map[SLJIT_SCRATCH_REG3] << 3) | 0x4 /* SIB */;
 479                 *inst++ = (0x4 /* none*/ << 3) | reg_map[SLJIT_LOCALS_REG];
 480                 *inst++ = 2 * sizeof(sljit_sw);
 481         }
 482 #endif
 483         return SLJIT_SUCCESS;
 484 }
 485
 486 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_si dst, sljit_sw dstw)
 487 {
 488         sljit_ub *inst;
 489
 490         CHECK_ERROR();
 491         check_sljit_emit_fast_enter(compiler, dst, dstw);
 492         ADJUST_LOCAL_OFFSET(dst, dstw);
 493
 494         CHECK_EXTRA_REGS(dst, dstw, (void)0);
 495
 496         /* For UNUSED dst. Uncommon, but possible. */
 497         if (dst == SLJIT_UNUSED)
 498                 dst = TMP_REG1;
 499
 500         if (FAST_IS_REG(dst)) {
 501                 /* Unused dest is possible here. */
 502                 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
 503                 FAIL_IF(!inst);
 504
 505                 INC_SIZE(1);
 506                 POP_REG(reg_map[dst]);
 507                 return SLJIT_SUCCESS;
 508         }
 509
 510         /* Memory. */
 511         inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
 512         FAIL_IF(!inst);
 513         *inst++ = POP_rm;
 514         return SLJIT_SUCCESS;
 515 }
 516
 517 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_si src, sljit_sw srcw)
 518 {
 519         sljit_ub *inst;
 520
 521         CHECK_ERROR();
 522         check_sljit_emit_fast_return(compiler, src, srcw);
 523         ADJUST_LOCAL_OFFSET(src, srcw);
 524
 525         CHECK_EXTRA_REGS(src, srcw, (void)0);
 526
 527         if (FAST_IS_REG(src)) {
 528                 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1 + 1);
 529                 FAIL_IF(!inst);
 530
 531                 INC_SIZE(1 + 1);
 532                 PUSH_REG(reg_map[src]);
 533         }
 534         else if (src & SLJIT_MEM) {
 535                 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
 536                 FAIL_IF(!inst);
 537                 *inst++ = GROUP_FF;
 538                 *inst |= PUSH_rm;
 539
 540                 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
 541                 FAIL_IF(!inst);
 542                 INC_SIZE(1);
 543         }
 544         else {
 545                 /* SLJIT_IMM. */
 546                 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5 + 1);
 547                 FAIL_IF(!inst);
 548
 549                 INC_SIZE(5 + 1);
 550                 *inst++ = PUSH_i32;
 551                 *(sljit_sw*)inst = srcw;
 552                 inst += sizeof(sljit_sw);
 553         }
 554
 555         RET();
 556         return SLJIT_SUCCESS;
 557 }