diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.h b/ext/opcache/jit/ir/dynasm/dasm_arm.h index ebcf4ac0ec1..eaa94d9c2f7 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm.h +++ b/ext/opcache/jit/ir/dynasm/dasm_arm.h @@ -1,6 +1,6 @@ /* ** DynASM ARM encoding engine. -** Copyright (C) 2005-2021 Mike Pall. All rights reserved. +** Copyright (C) 2005-2023 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -70,7 +70,7 @@ struct dasm_State { size_t lgsize; int *pclabels; /* PC label chains/pos ptrs. */ size_t pcsize; - void **globals; /* Array of globals (bias -10). */ + void **globals; /* Array of globals. */ dasm_Section *section; /* Pointer to active section. */ size_t codesize; /* Total size of all code sections. */ int maxsection; /* 0 <= sectionidx < maxsection. */ @@ -87,7 +87,6 @@ void dasm_init(Dst_DECL, int maxsection) { dasm_State *D; size_t psz = 0; - int i; Dst_REF = NULL; DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); D = Dst_REF; @@ -98,12 +97,7 @@ void dasm_init(Dst_DECL, int maxsection) D->pcsize = 0; D->globals = NULL; D->maxsection = maxsection; - for (i = 0; i < maxsection; i++) { - D->sections[i].buf = NULL; /* Need this for pass3. */ - D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); - D->sections[i].bsize = 0; - D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ - } + memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section)); } /* Free DynASM state. */ @@ -123,7 +117,7 @@ void dasm_free(Dst_DECL) void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) { dasm_State *D = Dst_REF; - D->globals = gl - 10; /* Negative bias to compensate for locals. */ + D->globals = gl; DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); } @@ -148,6 +142,7 @@ void dasm_setup(Dst_DECL, const void *actionlist) if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); for (i = 0; i < D->maxsection; i++) { D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos; D->sections[i].ofs = 0; } } @@ -372,7 +367,7 @@ int dasm_encode(Dst_DECL, void *buffer) break; case DASM_REL_LG: if (n < 0) { - n = (int)((ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp - 4); + n = (int)((ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp - 4); goto patchrel; } /* fallthrough */ @@ -396,7 +391,7 @@ int dasm_encode(Dst_DECL, void *buffer) } break; case DASM_LABEL_LG: - ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n); break; case DASM_LABEL_PC: break; case DASM_IMM: diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm.lua b/ext/opcache/jit/ir/dynasm/dasm_arm.lua index 0c775ae2687..edb575366ac 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_arm.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM ARM module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.h b/ext/opcache/jit/ir/dynasm/dasm_arm64.h index 9a8a39a2586..cacc1130c58 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm64.h +++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.h @@ -1,6 +1,6 @@ /* ** DynASM ARM64 encoding engine. -** Copyright (C) 2005-2021 Mike Pall. All rights reserved. +** Copyright (C) 2005-2023 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -72,7 +72,7 @@ struct dasm_State { size_t lgsize; int *pclabels; /* PC label chains/pos ptrs. */ size_t pcsize; - void **globals; /* Array of globals (bias -10). */ + void **globals; /* Array of globals. */ dasm_Section *section; /* Pointer to active section. */ size_t codesize; /* Total size of all code sections. */ int maxsection; /* 0 <= sectionidx < maxsection. */ @@ -89,7 +89,6 @@ void dasm_init(Dst_DECL, int maxsection) { dasm_State *D; size_t psz = 0; - int i; Dst_REF = NULL; DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); D = Dst_REF; @@ -100,12 +99,7 @@ void dasm_init(Dst_DECL, int maxsection) D->pcsize = 0; D->globals = NULL; D->maxsection = maxsection; - for (i = 0; i < maxsection; i++) { - D->sections[i].buf = NULL; /* Need this for pass3. */ - D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); - D->sections[i].bsize = 0; - D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ - } + memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section)); } /* Free DynASM state. */ @@ -125,7 +119,7 @@ void dasm_free(Dst_DECL) void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) { dasm_State *D = Dst_REF; - D->globals = gl - 10; /* Negative bias to compensate for locals. */ + D->globals = gl; DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); } @@ -150,6 +144,7 @@ void dasm_setup(Dst_DECL, const void *actionlist) if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); for (i = 0; i < D->maxsection; i++) { D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos; D->sections[i].ofs = 0; } } @@ -158,10 +153,10 @@ void dasm_setup(Dst_DECL, const void *actionlist) #ifdef DASM_CHECKS #define CK(x, st) \ do { if (!(x)) { \ - D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) + D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0) #define CKPL(kind, st) \ do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ - D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) + D->status = DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0) #else #define CK(x, st) ((void)0) #define CKPL(kind, st) ((void)0) @@ -190,7 +185,9 @@ static int dasm_imm13(int lo, int hi) unsigned long long n = (((unsigned long long)hi) << 32) | (unsigned int)lo; unsigned long long m = 1ULL, a, b, c; if (n & 1) { n = ~n; inv = 1; } - a = n & -n; b = (n+a)&-(n+a); c = (n+a-b)&-(n+a-b); + a = n & (unsigned long long)-(long long)n; + b = (n+a)&(unsigned long long)-(long long)(n+a); + c = (n+a-b)&(unsigned long long)-(long long)(n+a-b); xa = dasm_ffs(a); xb = dasm_ffs(b); if (c) { w = dasm_ffs(c) - xa; @@ -415,7 +412,7 @@ int dasm_link(Dst_DECL, size_t *szp) #ifdef DASM_CHECKS #define CK(x, st) \ - do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) + do { if (!(x)) return DASM_S_##st|(int)(p-D->actionlist-1); } while (0) #else #define CK(x, st) ((void)0) #endif @@ -451,7 +448,7 @@ int dasm_encode(Dst_DECL, void *buffer) break; case DASM_REL_LG: if (n < 0) { - ptrdiff_t na = (ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp + 4; + ptrdiff_t na = (ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp + 4; n = (int)na; CK_REL((ptrdiff_t)n == na, na); goto patchrel; @@ -494,7 +491,7 @@ int dasm_encode(Dst_DECL, void *buffer) goto patchrel; } case DASM_LABEL_LG: - ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n); break; case DASM_LABEL_PC: break; case DASM_IMM: @@ -563,7 +560,7 @@ int dasm_checkstep(Dst_DECL, int secmatch) } if (D->status == DASM_S_OK && secmatch >= 0 && D->section != &D->sections[secmatch]) - D->status = DASM_S_MATCH_SEC|(D->section-D->sections); + D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections); return D->status; } #endif diff --git a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua index 7e9c4cbf22e..05ea3e228cc 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_arm64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_arm64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM ARM64 module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ @@ -549,7 +549,7 @@ end local function parse_load_pair(params, nparams, n, op) if params[n+2] then werror("too many operands") end local pn, p2 = params[n], params[n+1] - local scale = shr(op, 30) == 0 and 2 or 3 + local scale = 2 + shr(op, 31 - band(shr(op, 26), 1)) local p1, wb = match(pn, "^%[%s*(.-)%s*%](!?)$") if not p1 then if not p2 then @@ -806,8 +806,8 @@ map_op = { ["ldrsw_*"] = "98000000DxB|b8800000DxL", -- NOTE: ldur etc. are handled by ldr et al. - ["stp_*"] = "28000000DAwP|a8000000DAxP|2c000000DAsP|6c000000DAdP", - ["ldp_*"] = "28400000DAwP|a8400000DAxP|2c400000DAsP|6c400000DAdP", + ["stp_*"] = "28000000DAwP|a8000000DAxP|2c000000DAsP|6c000000DAdP|ac000000DAqP", + ["ldp_*"] = "28400000DAwP|a8400000DAxP|2c400000DAsP|6c400000DAdP|ac400000DAqP", ["ldpsw_*"] = "68400000DAxP", -- Branches. @@ -823,6 +823,13 @@ map_op = { tbz_3 = "36000000DTBw|36000000DTBx", tbnz_3 = "37000000DTBw|37000000DTBx", + -- ARM64e: Pointer authentication codes (PAC). + blraaz_1 = "d63f081fNx", + braa_2 = "d71f0800NDx", + braaz_1 = "d61f081fNx", + pacibsp_0 = "d503237f", + retab_0 = "d65f0fff", + -- Miscellaneous instructions. -- TODO: hlt, hvc, smc, svc, eret, dcps[123], drps, mrs, msr -- TODO: sys, sysl, ic, dc, at, tlbi @@ -935,7 +942,7 @@ local function parse_template(params, template, nparams, pos) werror("bad register type") end parse_reg_type = false - elseif p == "x" or p == "w" or p == "d" or p == "s" then + elseif p == "x" or p == "w" or p == "d" or p == "s" or p == "q" then if parse_reg_type ~= p then werror("register size mismatch") end diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.h b/ext/opcache/jit/ir/dynasm/dasm_mips.h index b99b56b0e9a..7800e933877 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips.h +++ b/ext/opcache/jit/ir/dynasm/dasm_mips.h @@ -1,6 +1,6 @@ /* ** DynASM MIPS encoding engine. -** Copyright (C) 2005-2021 Mike Pall. All rights reserved. +** Copyright (C) 2005-2023 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -69,7 +69,7 @@ struct dasm_State { size_t lgsize; int *pclabels; /* PC label chains/pos ptrs. */ size_t pcsize; - void **globals; /* Array of globals (bias -10). */ + void **globals; /* Array of globals. */ dasm_Section *section; /* Pointer to active section. */ size_t codesize; /* Total size of all code sections. */ int maxsection; /* 0 <= sectionidx < maxsection. */ @@ -86,7 +86,6 @@ void dasm_init(Dst_DECL, int maxsection) { dasm_State *D; size_t psz = 0; - int i; Dst_REF = NULL; DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); D = Dst_REF; @@ -97,12 +96,7 @@ void dasm_init(Dst_DECL, int maxsection) D->pcsize = 0; D->globals = NULL; D->maxsection = maxsection; - for (i = 0; i < maxsection; i++) { - D->sections[i].buf = NULL; /* Need this for pass3. */ - D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); - D->sections[i].bsize = 0; - D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ - } + memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section)); } /* Free DynASM state. */ @@ -122,7 +116,7 @@ void dasm_free(Dst_DECL) void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) { dasm_State *D = Dst_REF; - D->globals = gl - 10; /* Negative bias to compensate for locals. */ + D->globals = gl; DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); } @@ -147,6 +141,7 @@ void dasm_setup(Dst_DECL, const void *actionlist) if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); for (i = 0; i < D->maxsection; i++) { D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos; D->sections[i].ofs = 0; } } @@ -155,10 +150,10 @@ void dasm_setup(Dst_DECL, const void *actionlist) #ifdef DASM_CHECKS #define CK(x, st) \ do { if (!(x)) { \ - D->status = DASM_S_##st|(p-D->actionlist-1); return; } } while (0) + D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0) #define CKPL(kind, st) \ do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \ - D->status = DASM_S_RANGE_##st|(p-D->actionlist-1); return; } } while (0) + D->status = DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0) #else #define CK(x, st) ((void)0) #define CKPL(kind, st) ((void)0) @@ -314,7 +309,7 @@ int dasm_link(Dst_DECL, size_t *szp) #ifdef DASM_CHECKS #define CK(x, st) \ - do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) + do { if (!(x)) return DASM_S_##st|(int)(p-D->actionlist-1); } while (0) #else #define CK(x, st) ((void)0) #endif @@ -350,7 +345,7 @@ int dasm_encode(Dst_DECL, void *buffer) break; case DASM_REL_LG: if (n < 0) { - n = (int)((ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp); + n = (int)((ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp); goto patchrel; } /* fallthrough */ @@ -369,7 +364,7 @@ int dasm_encode(Dst_DECL, void *buffer) } break; case DASM_LABEL_LG: - ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n); break; case DASM_LABEL_PC: break; case DASM_IMMS: @@ -417,7 +412,7 @@ int dasm_checkstep(Dst_DECL, int secmatch) } if (D->status == DASM_S_OK && secmatch >= 0 && D->section != &D->sections[secmatch]) - D->status = DASM_S_MATCH_SEC|(D->section-D->sections); + D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections); return D->status; } #endif diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips.lua b/ext/opcache/jit/ir/dynasm/dasm_mips.lua index 591470157c4..1c605b681f1 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_mips.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM MIPS32/MIPS64 module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ diff --git a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua index 8ab5d33a208..c97d666b319 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_mips64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_mips64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM MIPS64 module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ -- This module just sets 64 bit mode for the combined MIPS/MIPS64 module. diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.h b/ext/opcache/jit/ir/dynasm/dasm_ppc.h index 35264f2eb93..4c7d7289f9d 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_ppc.h +++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.h @@ -1,6 +1,6 @@ /* ** DynASM PPC/PPC64 encoding engine. -** Copyright (C) 2005-2021 Mike Pall. All rights reserved. +** Copyright (C) 2005-2023 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -69,7 +69,7 @@ struct dasm_State { size_t lgsize; int *pclabels; /* PC label chains/pos ptrs. */ size_t pcsize; - void **globals; /* Array of globals (bias -10). */ + void **globals; /* Array of globals. */ dasm_Section *section; /* Pointer to active section. */ size_t codesize; /* Total size of all code sections. */ int maxsection; /* 0 <= sectionidx < maxsection. */ @@ -86,7 +86,6 @@ void dasm_init(Dst_DECL, int maxsection) { dasm_State *D; size_t psz = 0; - int i; Dst_REF = NULL; DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); D = Dst_REF; @@ -97,12 +96,7 @@ void dasm_init(Dst_DECL, int maxsection) D->pcsize = 0; D->globals = NULL; D->maxsection = maxsection; - for (i = 0; i < maxsection; i++) { - D->sections[i].buf = NULL; /* Need this for pass3. */ - D->sections[i].rbuf = D->sections[i].buf - DASM_SEC2POS(i); - D->sections[i].bsize = 0; - D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ - } + memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section)); } /* Free DynASM state. */ @@ -122,7 +116,7 @@ void dasm_free(Dst_DECL) void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) { dasm_State *D = Dst_REF; - D->globals = gl - 10; /* Negative bias to compensate for locals. */ + D->globals = gl; DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); } @@ -147,6 +141,7 @@ void dasm_setup(Dst_DECL, const void *actionlist) if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); for (i = 0; i < D->maxsection; i++) { D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos; D->sections[i].ofs = 0; } } @@ -354,7 +349,7 @@ int dasm_encode(Dst_DECL, void *buffer) break; case DASM_REL_LG: if (n < 0) { - n = (int)((ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp); + n = (int)((ptrdiff_t)D->globals[-n-10] - (ptrdiff_t)cp); goto patchrel; } /* fallthrough */ @@ -368,7 +363,7 @@ int dasm_encode(Dst_DECL, void *buffer) cp[-1] |= ((n+4) & ((ins & 2048) ? 0x0000fffc: 0x03fffffc)); break; case DASM_LABEL_LG: - ins &= 2047; if (ins >= 20) D->globals[ins-10] = (void *)(base + n); + ins &= 2047; if (ins >= 20) D->globals[ins-20] = (void *)(base + n); break; case DASM_LABEL_PC: break; case DASM_IMM: diff --git a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua index ee2afb2ecfa..d66ae4a0d61 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_ppc.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_ppc.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM PPC/PPC64 module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. -- -- Support for various extensions contributed by Caio Souza Oliveira. diff --git a/ext/opcache/jit/ir/dynasm/dasm_proto.h b/ext/opcache/jit/ir/dynasm/dasm_proto.h index 8914596adf5..3f50f5028f3 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_proto.h +++ b/ext/opcache/jit/ir/dynasm/dasm_proto.h @@ -1,6 +1,6 @@ /* ** DynASM encoding engine prototypes. -** Copyright (C) 2005-2021 Mike Pall. All rights reserved. +** Copyright (C) 2005-2023 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ diff --git a/ext/opcache/jit/ir/dynasm/dasm_x64.lua b/ext/opcache/jit/ir/dynasm/dasm_x64.lua index 2c0a0e8681f..72d9bb87bff 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x64.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_x64.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM x64 module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ -- This module just sets 64 bit mode for the combined x86/x64 module. diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.h b/ext/opcache/jit/ir/dynasm/dasm_x86.h index a2b46cc951e..13fb74ccbe7 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x86.h +++ b/ext/opcache/jit/ir/dynasm/dasm_x86.h @@ -1,6 +1,6 @@ /* ** DynASM x86 encoding engine. -** Copyright (C) 2005-2021 Mike Pall. All rights reserved. +** Copyright (C) 2005-2023 Mike Pall. All rights reserved. ** Released under the MIT license. See dynasm.lua for full copyright notice. */ @@ -68,7 +68,7 @@ struct dasm_State { size_t lgsize; int *pclabels; /* PC label chains/pos ptrs. */ size_t pcsize; - void **globals; /* Array of globals (bias -10). */ + void **globals; /* Array of globals. */ dasm_Section *section; /* Pointer to active section. */ size_t codesize; /* Total size of all code sections. */ int maxsection; /* 0 <= sectionidx < maxsection. */ @@ -88,7 +88,6 @@ void dasm_init(Dst_DECL, int maxsection) { dasm_State *D; size_t psz = 0; - int i; Dst_REF = NULL; DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection)); D = Dst_REF; @@ -99,12 +98,7 @@ void dasm_init(Dst_DECL, int maxsection) D->pcsize = 0; D->globals = NULL; D->maxsection = maxsection; - for (i = 0; i < maxsection; i++) { - D->sections[i].buf = NULL; /* Need this for pass3. */ - D->sections[i].rbuf = DASM_PTR_SUB(D->sections[i].buf, DASM_SEC2POS(i)); - D->sections[i].bsize = 0; - D->sections[i].epos = 0; /* Wrong, but is recalculated after resize. */ - } + memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section)); } /* Free DynASM state. */ @@ -124,7 +118,7 @@ void dasm_free(Dst_DECL) void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl) { dasm_State *D = Dst_REF; - D->globals = gl - 10; /* Negative bias to compensate for locals. */ + D->globals = gl; DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int)); } @@ -149,6 +143,7 @@ void dasm_setup(Dst_DECL, const void *actionlist) if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize); for (i = 0; i < D->maxsection; i++) { D->sections[i].pos = DASM_SEC2POS(i); + D->sections[i].rbuf = DASM_PTR_SUB(D->sections[i].buf, D->sections[i].pos); D->sections[i].ofs = 0; } } @@ -203,9 +198,9 @@ void dasm_put(Dst_DECL, int start, ...) case DASM_IMM_DB: if ((((unsigned)n+128)&-256) == 0) goto ob; /* fallthrough */ case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */ case DASM_IMM_D: ofs += 4; break; - case DASM_IMM_S: CK(((n+128)&-256) == 0, RANGE_I); goto ob; + case DASM_IMM_S: CK((((unsigned)n+128)&-256) == 0, RANGE_I); goto ob; case DASM_IMM_B: CK((n&-256) == 0, RANGE_I); ob: ofs++; break; - case DASM_IMM_WB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */ + case DASM_IMM_WB: if ((((unsigned)n+128)&-256) == 0) goto ob; /* fallthrough */ case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break; case DASM_SPACE: p++; ofs += n; break; case DASM_SETLABEL: b[pos-2] = -0x40000000; break; /* Neg. label ofs. */ @@ -346,6 +341,7 @@ int dasm_link(Dst_DECL, size_t *szp) } /* fallthrough */ case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++; + /* fallthrough */ case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W: case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB: case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break; @@ -473,7 +469,7 @@ int dasm_encode(Dst_DECL, void *buffer) break; } case DASM_REL_LG: p++; if (n >= 0) goto rel_pc; - b++; n = (int)(ptrdiff_t)D->globals[-n]; + b++; n = (int)(ptrdiff_t)D->globals[-n-10]; /* fallthrough */ case DASM_REL_A: rel_a: n -= (unsigned int)(ptrdiff_t)(cp+4); goto wd; /* !x64 */ @@ -487,7 +483,7 @@ int dasm_encode(Dst_DECL, void *buffer) } case DASM_IMM_LG: p++; - if (n < 0) { dasma((ptrdiff_t)D->globals[-n]); break; } + if (n < 0) { dasma((ptrdiff_t)D->globals[-n-10]); break; } /* fallthrough */ case DASM_IMM_PC: { int *pb = DASM_POS2PTR(D, n); @@ -497,7 +493,7 @@ int dasm_encode(Dst_DECL, void *buffer) case DASM_LABEL_LG: { int idx = *p++; if (idx >= 10) - D->globals[idx] = (void *)(base + (*p == DASM_SETLABEL ? *b : n)); + D->globals[idx-10] = (void *)(base + (*p == DASM_SETLABEL ? *b : n)); break; } case DASM_LABEL_PC: case DASM_SETLABEL: break; diff --git a/ext/opcache/jit/ir/dynasm/dasm_x86.lua b/ext/opcache/jit/ir/dynasm/dasm_x86.lua index d5eea69e485..7c789f8216d 100644 --- a/ext/opcache/jit/ir/dynasm/dasm_x86.lua +++ b/ext/opcache/jit/ir/dynasm/dasm_x86.lua @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------ -- DynASM x86/x64 module. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See dynasm.lua for full copyright notice. ------------------------------------------------------------------------------ @@ -627,7 +627,11 @@ local function wputmrmsib(t, imark, s, vsreg, psz, sk) werror("NYI: rip-relative displacement followed by immediate") end -- The previous byte in the action buffer cannot be 0xe9 or 0x80-0x8f. - wputlabel("REL_", disp[1], 2) + if disp[2] == "iPJ" then + waction("REL_A", disp[1]) + else + wputlabel("REL_", disp[1], 2) + end else wputdarg(disp) end @@ -744,9 +748,9 @@ local function dispexpr(expr) return imm*map_opsizenum[ops] end local mode, iexpr = immexpr(dispt) - if mode == "iJ" then + if mode == "iJ" or mode == "iPJ" then if c == "-" then werror("cannot invert label reference") end - return { iexpr } + return { iexpr, mode } end return expr -- Need to return original signed expression. end diff --git a/ext/opcache/jit/ir/dynasm/dynasm.lua b/ext/opcache/jit/ir/dynasm/dynasm.lua index 2583295fce3..0d15a87282f 100644 --- a/ext/opcache/jit/ir/dynasm/dynasm.lua +++ b/ext/opcache/jit/ir/dynasm/dynasm.lua @@ -2,7 +2,7 @@ -- DynASM. A dynamic assembler for code generation engines. -- Originally designed and implemented for LuaJIT. -- --- Copyright (C) 2005-2021 Mike Pall. All rights reserved. +-- Copyright (C) 2005-2023 Mike Pall. All rights reserved. -- See below for full copyright notice. ------------------------------------------------------------------------------ @@ -17,7 +17,7 @@ local _info = { url = "https://luajit.org/dynasm.html", license = "MIT", copyright = [[ -Copyright (C) 2005-2021 Mike Pall. All rights reserved. +Copyright (C) 2005-2023 Mike Pall. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/ext/opcache/jit/ir/dynasm/minilua.c b/ext/opcache/jit/ir/dynasm/minilua.c index a8d7c305e10..68c5591d679 100644 --- a/ext/opcache/jit/ir/dynasm/minilua.c +++ b/ext/opcache/jit/ir/dynasm/minilua.c @@ -1134,7 +1134,7 @@ if(!cl->isC){ CallInfo*ci; StkId st,base; Proto*p=cl->p; -luaD_checkstack(L,p->maxstacksize); +luaD_checkstack(L,p->maxstacksize+p->numparams); func=restorestack(L,funcr); if(!p->is_vararg){ base=func+1; @@ -1639,6 +1639,7 @@ lua_number2int(k,n); if(luai_numeq(cast_num(k),nvalue(key))) return luaH_getnum(t,k); } +/*fallthrough*/ default:{ Node*n=mainposition(t,key); do{ @@ -2905,8 +2906,8 @@ if(sep>=0){ read_long_string(ls,seminfo,sep); return TK_STRING; } -else if(sep==-1)return'['; -else luaX_lexerror(ls,"invalid long string delimiter",TK_STRING); +else if (sep!=-1)luaX_lexerror(ls,"invalid long string delimiter",TK_STRING); +return'['; } case'=':{ next(ls); @@ -6985,9 +6986,14 @@ return os_pushresult(L,remove(filename)==0,filename); static int os_exit(lua_State*L){ exit(luaL_optint(L,1,EXIT_SUCCESS)); } +static int os_clock(lua_State*L){ +lua_pushnumber(L,((lua_Number)clock())/(lua_Number)CLOCKS_PER_SEC); +return 1; +} static const luaL_Reg syslib[]={ {"exit",os_exit}, {"remove",os_remove}, +{"clock",os_clock}, {NULL,NULL} }; static int luaopen_os(lua_State*L){ diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 8d0379eafc4..2bc18d80772 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -368,6 +368,10 @@ void ir_free(ir_ctx *ctx) } if (ctx->regs) { ir_mem_free(ctx->regs); + if (ctx->fused_regs) { + ir_strtab_free(ctx->fused_regs); + ir_mem_free(ctx->fused_regs); + } } if (ctx->prev_ref) { ir_mem_free(ctx->prev_ref); diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 537daeb767a..518ed32e8cb 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -579,6 +579,7 @@ struct _ir_ctx { ir_arena *arena; ir_live_range *unused_ranges; ir_regs *regs; + ir_strtab *fused_regs; ir_ref *prev_ref; union { void *data; diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 5e4ce906d7a..9aaa1d15d76 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -3234,14 +3234,18 @@ static void ir_emit_vstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } } -static int32_t ir_fuse_addr(ir_ctx *ctx, ir_ref ref, ir_reg *preg1, ir_reg *preg2) +static int32_t ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_reg *preg1, ir_reg *preg2) { ir_insn *addr_insn = &ctx->ir_base[ref]; ir_reg reg; IR_ASSERT(addr_insn->op == IR_ADD); IR_ASSERT(!IR_IS_CONST_REF(addr_insn->op1) && IR_IS_CONST_REF(addr_insn->op2)); - reg = ctx->regs[ref][1]; + if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { + reg = ir_get_fused_reg(ctx, root, ref, 1); + } else { + reg = ctx->regs[ref][1]; + } if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, addr_insn->op1); @@ -3266,7 +3270,7 @@ static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { ir_reg op1_reg; - int32_t offset = ir_fuse_addr(ctx, insn->op2, &op1_reg, &op2_reg); + int32_t offset = ir_fuse_addr(ctx, def, insn->op2, &op1_reg, &op2_reg); if (op2_reg == IR_REG_NONE) { if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, op1_reg, offset)) { @@ -3334,7 +3338,7 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { ir_reg op1_reg; - int32_t offset = ir_fuse_addr(ctx, insn->op2, &op1_reg, &op2_reg); + int32_t offset = ir_fuse_addr(ctx, def, insn->op2, &op1_reg, &op2_reg); if (op2_reg == IR_REG_NONE) { if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, op1_reg, offset)) { @@ -3381,7 +3385,7 @@ static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { ir_reg op1_reg; - int32_t offset = ir_fuse_addr(ctx, insn->op2, &op1_reg, &op2_reg); + int32_t offset = ir_fuse_addr(ctx, ref, insn->op2, &op1_reg, &op2_reg); if (op2_reg == IR_REG_NONE) { if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, op1_reg, offset)) { @@ -3454,7 +3458,7 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) IR_ASSERT(op3_reg != IR_REG_NONE); if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { ir_reg op1_reg; - int32_t offset = ir_fuse_addr(ctx, insn->op2, &op1_reg, &op2_reg); + int32_t offset = ir_fuse_addr(ctx, ref, insn->op2, &op1_reg, &op2_reg); if (op2_reg == IR_REG_NONE) { if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, op1_reg, offset)) { diff --git a/ext/opcache/jit/ir/ir_dump.c b/ext/opcache/jit/ir/ir_dump.c index d1d601ba2c0..7204e8bce77 100644 --- a/ext/opcache/jit/ir/ir_dump.c +++ b/ext/opcache/jit/ir/ir_dump.c @@ -595,7 +595,7 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) } if (ctx->rules) { uint32_t rule = ctx->rules[i]; - uint32_t id = rule & ~(IR_FUSED|IR_SKIPPED|IR_SIMPLE); + uint32_t id = rule & ~(IR_FUSED_REG|IR_FUSED|IR_SKIPPED|IR_SIMPLE); if (id < IR_LAST_OP) { fprintf(f, " # RULE(%s", ir_op_name[id]); diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index caf2f547876..7e311025343 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -351,6 +351,24 @@ static void *ir_jmp_addr(ir_ctx *ctx, ir_insn *insn, ir_insn *addr_insn) return addr; } +static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, uint8_t op_num) +{ + if (ctx->fused_regs) { + char key[10]; + ir_ref val; + + memcpy(key, &root, sizeof(ir_ref)); + memcpy(key + 4, &ref, sizeof(ir_ref)); + memcpy(key + 8, &op_num, sizeof(uint8_t)); + + val = ir_strtab_find(ctx->fused_regs, key, 9); + if (val) { + return val; + } + } + return ctx->regs[ref][op_num]; +} + #if defined(__GNUC__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Warray-bounds" diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 1caa02956f0..1b9209dc53e 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -1210,6 +1210,7 @@ IR_ALWAYS_INLINE int8_t ir_get_alocated_reg(const ir_ctx *ctx, ir_ref ref, int o #define IR_FUSED (1U<<31) /* Insn is fused into others (code is generated as part of the fusion root) */ #define IR_SKIPPED (1U<<30) /* Insn is skipped (code is not generated) */ #define IR_SIMPLE (1U<<29) /* Insn doesn't have any target constraints */ +#define IR_FUSED_REG (1U<<28) /* Register assignemnt may be stored in ctx->fused_regs instead of ctx->regs */ #define IR_RULE_MASK 0xff diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 0aac8e0b9a1..dfe85ff8458 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -1615,7 +1615,7 @@ static bool ir_try_coalesce(ir_ctx *ctx, ir_ref from, ir_ref to) } else if (from < to) { ir_vregs_join(ctx, v1, v2); if (f2 & IR_LIVE_INTERVAL_COALESCED) { - for (i = 0; i < ctx->insns_count; i++) { + for (i = 1; i < ctx->insns_count; i++) { if (ctx->vregs[i] == v2) { ctx->vregs[i] = v1; } @@ -1626,7 +1626,7 @@ static bool ir_try_coalesce(ir_ctx *ctx, ir_ref from, ir_ref to) } else { ir_vregs_join(ctx, v2, v1); if (f1 & IR_LIVE_INTERVAL_COALESCED) { - for (i = 0; i < ctx->insns_count; i++) { + for (i = 1; i < ctx->insns_count; i++) { if (ctx->vregs[i] == v1) { ctx->vregs[i] = v2; } @@ -3669,6 +3669,21 @@ static bool needs_spill_load(ir_ctx *ctx, ir_live_interval *ival, ir_use_pos *us return use_pos->next && use_pos->next->op_num != 0; } +static void ir_set_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, uint8_t op_num, int8_t reg) +{ + char key[10]; + + IR_ASSERT(reg != IR_REG_NONE); + if (!ctx->fused_regs) { + ctx->fused_regs = ir_mem_malloc(sizeof(ir_strtab)); + ir_strtab_init(ctx->fused_regs, 8, 128); + } + memcpy(key, &root, sizeof(ir_ref)); + memcpy(key + 4, &ref, sizeof(ir_ref)); + memcpy(key + 8, &op_num, sizeof(uint8_t)); + ir_strtab_lookup(ctx->fused_regs, key, 9, 0x10000000 | reg); +} + static void assign_regs(ir_ctx *ctx) { ir_ref i; @@ -3739,9 +3754,11 @@ static void assign_regs(ir_ctx *ctx) ref = IR_LIVE_POS_TO_REF(use_pos->pos); // TODO: Insert spill loads and stores in optimal positions (resolution) if (use_pos->op_num == 0) { + ir_bitset_clear(available, ir_bitset_len(ctx->cfg_blocks_count + 1)); if (ctx->ir_base[ref].op == IR_PHI) { /* Spilled PHI var is passed through memory */ reg = IR_REG_NONE; + prev_use_ref = IR_UNUSED; } else if (ctx->ir_base[ref].op == IR_PARAM && (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)) { /* Stack PARAM var is passed through memory */ @@ -3795,6 +3812,22 @@ static void assign_regs(ir_ctx *ctx) prev_use_ref = ref; } } + if (use_pos->hint_ref < 0 + && ctx->use_lists[-use_pos->hint_ref].count > 1 + && (old_reg = ir_get_alocated_reg(ctx, -use_pos->hint_ref, use_pos->op_num)) != IR_REG_NONE) { + if (top_ival->flags & IR_LIVE_INTERVAL_SPILL_SPECIAL) { + reg |= IR_REG_SPILL_SPECIAL; + } else { + reg |= IR_REG_SPILL_LOAD; + } + if (reg != old_reg) { + IR_ASSERT(ctx->rules[-use_pos->hint_ref] & IR_FUSED); + ctx->rules[-use_pos->hint_ref] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, ref, -use_pos->hint_ref, use_pos->op_num, reg); + use_pos = use_pos->next; + continue; + } + } } else if (use_pos->flags & IR_PHI_USE) { IR_ASSERT(use_pos->hint_ref < 0); IR_ASSERT(ctx->vregs[-use_pos->hint_ref]); @@ -3805,16 +3838,14 @@ static void assign_regs(ir_ctx *ctx) } } else if (use_pos->hint_ref < 0 && ctx->use_lists[-use_pos->hint_ref].count > 1 - && (old_reg = ir_get_alocated_reg(ctx, -use_pos->hint_ref, use_pos->op_num)) != IR_REG_NONE - && (old_reg & (IR_REG_SPILL_SPECIAL|IR_REG_SPILL_LOAD))) { - /* Force spill load */ - // TODO: Find a better solution ??? - if (top_ival->flags & IR_LIVE_INTERVAL_SPILL_SPECIAL) { - reg |= IR_REG_SPILL_SPECIAL; - } else { - reg |= IR_REG_SPILL_LOAD; + && (old_reg = ir_get_alocated_reg(ctx, -use_pos->hint_ref, use_pos->op_num)) != IR_REG_NONE) { + if (reg != old_reg) { + IR_ASSERT(ctx->rules[-use_pos->hint_ref] & IR_FUSED); + ctx->rules[-use_pos->hint_ref] |= IR_FUSED_REG; + ir_set_fused_reg(ctx, ref, -use_pos->hint_ref, use_pos->op_num, reg); + use_pos = use_pos->next; + continue; } - IR_ASSERT(reg == old_reg); } else { /* reuse register without spill load */ } diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 5ddd3722a04..6fc77042b7f 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -741,7 +741,11 @@ int ir_sccp(ir_ctx *ctx) } continue; } - if (!IR_IS_BOTTOM(insn->op2)) { + if (!IR_IS_BOTTOM(insn->op2) +#if IR_COMBO_COPY_PROPAGATION + && (IR_IS_CONST_REF(insn->op2) || _values[insn->op2].op != IR_COPY) +#endif + ) { ir_ref use_case = IR_UNUSED; use_list = &ctx->use_lists[i]; diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 8e3dd55628c..a1f430dabe0 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -2330,14 +2330,18 @@ static void ir_emit_fp_mov(ir_ctx *ctx, ir_type type, ir_reg dst, ir_reg src) | ASM_FP_REG_REG_OP movaps, movapd, vmovaps, vmovapd, type, dst, src } -static int32_t ir_fuse_addr(ir_ctx *ctx, ir_ref ref, ir_reg *preg) +static int32_t ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_reg *preg) { ir_insn *addr_insn = &ctx->ir_base[ref]; ir_reg reg; IR_ASSERT(addr_insn->op == IR_ADD); IR_ASSERT(!IR_IS_CONST_REF(addr_insn->op1) && IR_IS_CONST_REF(addr_insn->op2)); - reg = ctx->regs[ref][1]; + if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { + reg = ir_get_fused_reg(ctx, root, ref, 1); + } else { + reg = ctx->regs[ref][1]; + } if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, addr_insn->op1); @@ -2346,11 +2350,16 @@ static int32_t ir_fuse_addr(ir_ctx *ctx, ir_ref ref, ir_reg *preg) return ctx->ir_base[addr_insn->op2].val.i32; } -static int32_t ir_fuse_load(ir_ctx *ctx, ir_ref ref, ir_reg *preg) +static int32_t ir_fuse_load(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_reg *preg) { ir_insn *load_insn = &ctx->ir_base[ref]; - ir_reg reg = ctx->regs[ref][2]; + ir_reg reg; + if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { + reg = ir_get_fused_reg(ctx, root, ref, 2); + } else { + reg = ctx->regs[ref][2]; + } IR_ASSERT(load_insn->op == IR_LOAD); if (IR_IS_CONST_REF(load_insn->op2)) { *preg = reg; @@ -2365,7 +2374,7 @@ static int32_t ir_fuse_load(ir_ctx *ctx, ir_ref ref, ir_reg *preg) return 0; } } else if (reg == IR_REG_NONE) { - return ir_fuse_addr(ctx, load_insn->op2, preg); + return ir_fuse_addr(ctx, root, load_insn->op2, preg); } else { if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); @@ -2622,7 +2631,7 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -2728,7 +2737,7 @@ static void ir_emit_imul3(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, op1, &op1_reg); + offset = ir_fuse_load(ctx, def, op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, op1, &op1_reg); } @@ -2885,7 +2894,7 @@ static void ir_emit_mem_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, IR_ADDR, reg, insn->op2); } } else if (reg == IR_REG_NONE) { - offset = ir_fuse_addr(ctx, insn->op2, ®); + offset = ir_fuse_addr(ctx, def, insn->op2, ®); } else if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, insn->op2); @@ -3119,7 +3128,7 @@ static void ir_emit_mem_mul_div_mod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, IR_ADDR, reg, insn->op2); } } else if (reg == IR_REG_NONE) { - offset = ir_fuse_addr(ctx, insn->op2, ®); + offset = ir_fuse_addr(ctx, def, insn->op2, ®); } else if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, insn->op2); @@ -3223,7 +3232,7 @@ static void ir_emit_mem_shift(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, IR_ADDR, reg, insn->op2); } } else if (reg == IR_REG_NONE) { - offset = ir_fuse_addr(ctx, insn->op2, ®); + offset = ir_fuse_addr(ctx, def, insn->op2, ®); } else if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, insn->op2); @@ -3337,7 +3346,7 @@ static void ir_emit_mem_shift_const(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, IR_ADDR, reg, insn->op2); } } else if (reg == IR_REG_NONE) { - offset = ir_fuse_addr(ctx, insn->op2, ®); + offset = ir_fuse_addr(ctx, def, insn->op2, ®); } else if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, insn->op2); @@ -3368,7 +3377,7 @@ static void ir_emit_mem_shift_const(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) +static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn, uint32_t rule) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; @@ -3390,9 +3399,9 @@ static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, type, def_reg, op1); } } - if (insn->op == IR_ADD) { + if (rule == IR_INC) { | ASM_REG_OP inc, insn->type, def_reg - } else if (insn->op == IR_SUB) { + } else if (rule == IR_DEC) { | ASM_REG_OP dec, insn->type, def_reg } else if (insn->op == IR_NOT) { | ASM_REG_OP not, insn->type, def_reg @@ -3515,7 +3524,7 @@ static void ir_emit_bit_count(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, op1, &op1_reg); + offset = ir_fuse_load(ctx, def, op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, op1, &op1_reg); } @@ -3779,7 +3788,7 @@ static void ir_emit_ctpop(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static void ir_emit_mem_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) +static void ir_emit_mem_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn, uint32_t rule) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; @@ -3797,7 +3806,7 @@ static void ir_emit_mem_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, IR_ADDR, reg, insn->op2); } } else if (reg == IR_REG_NONE) { - offset = ir_fuse_addr(ctx, insn->op2, ®); + offset = ir_fuse_addr(ctx, def, insn->op2, ®); } else if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, insn->op2); @@ -3807,9 +3816,9 @@ static void ir_emit_mem_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) offset = ir_var_spill_slot(ctx, insn->op2, ®); } - if (op_insn->op == IR_ADD) { + if (rule == IR_MEM_INC) { | ASM_MEM_OP inc, type, [Ra(reg)+offset] - } else if (op_insn->op == IR_SUB) { + } else if (rule == IR_MEM_DEC) { | ASM_MEM_OP dec, type, [Ra(reg)+offset] } else if (op_insn->op == IR_NOT) { | ASM_MEM_OP not, type, [Ra(reg)+offset] @@ -3919,7 +3928,7 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_OP imul, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -3934,7 +3943,7 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_OP mul, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -3960,7 +3969,7 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_OP idiv, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -3980,7 +3989,7 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_OP div, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -4220,7 +4229,7 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -4327,7 +4336,7 @@ static void ir_emit_binop_avx(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, def, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -4359,7 +4368,7 @@ static void ir_emit_binop_avx(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_insn *insn, ir_reg op1_reg, ir_ref op1, ir_reg op2_reg, ir_ref op2) +static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_ref root, ir_insn *insn, ir_reg op1_reg, ir_ref op1, ir_reg op2_reg, ir_ref op2) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; @@ -4379,7 +4388,7 @@ static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_insn *insn, ir_ int32_t offset = 0; if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, root, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -4395,7 +4404,7 @@ static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_insn *insn, ir_ int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, root, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -4504,14 +4513,14 @@ static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) op = IR_NE; } } - ir_emit_cmp_int_common(ctx, type, insn, op1_reg, op1, op2_reg, op2); + ir_emit_cmp_int_common(ctx, type, def, insn, op1_reg, op1, op2_reg, op2); _ir_emit_setcc_int(ctx, op, def_reg); if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); } } -static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref ref, ir_op op) +static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_op op) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; @@ -4568,7 +4577,7 @@ static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref ref, ir_op op) int32_t offset = 0; if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, root, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -4580,7 +4589,7 @@ static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref ref, ir_op op) int32_t offset = 0; if (ir_rule(ctx, op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, op1, &op1_reg); + offset = ir_fuse_load(ctx, root, op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, op1, &op1_reg); } @@ -4615,7 +4624,7 @@ static void ir_emit_testcc_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); IR_ASSERT(def_reg != IR_REG_NONE); - ir_emit_test_int_common(ctx, insn->op1, insn->op); + ir_emit_test_int_common(ctx, def, insn->op1, insn->op); _ir_emit_setcc_int(ctx, insn->op, def_reg); if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, insn->type, def, def_reg); @@ -4633,7 +4642,7 @@ static void ir_emit_setcc_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref cmp_ref, ir_insn *cmp_insn) +static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_insn *cmp_insn) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; @@ -4681,7 +4690,7 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref cmp_ref, ir_insn *cmp_ins int32_t offset = 0; if (ir_rule(ctx, op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, op2, &op2_reg); + offset = ir_fuse_load(ctx, root, op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, op2, &op2_reg); } @@ -4694,7 +4703,7 @@ static void ir_emit_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; - ir_op op = ir_emit_cmp_fp_common(ctx, def, insn); + ir_op op = ir_emit_cmp_fp_common(ctx, def, def, insn); ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); ir_reg tmp_reg = ctx->regs[def][3]; @@ -4942,7 +4951,7 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i } } if (!same_comparison) { - ir_emit_cmp_int_common(ctx, type, cmp_insn, op1_reg, op1, op2_reg, op2); + ir_emit_cmp_int_common(ctx, type, def, cmp_insn, op1_reg, op1, op2_reg, op2); } ir_emit_jcc(ctx, op, b, def, insn, 1); } @@ -4959,13 +4968,13 @@ static void ir_emit_test_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_ op = IR_NE; } - ir_emit_test_int_common(ctx, op2, op); + ir_emit_test_int_common(ctx, def, op2, op); ir_emit_jcc(ctx, op, b, def, insn, 1); } static void ir_emit_cmp_and_branch_fp(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) { - ir_op op = ir_emit_cmp_fp_common(ctx, insn->op2, &ctx->ir_base[insn->op2]); + ir_op op = ir_emit_cmp_fp_common(ctx, def, insn->op2, &ctx->ir_base[insn->op2]); ir_emit_jcc(ctx, op, b, def, insn, 0); } @@ -5000,7 +5009,7 @@ static void ir_emit_if_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op2, &op2_reg); + offset = ir_fuse_load(ctx, def, insn->op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op2, &op2_reg); } @@ -5087,7 +5096,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else if (IR_IS_CONST_REF(op2) || !(ir_rule(ctx, op2) & IR_FUSED)) { ir_emit_load(ctx, type, def_reg, op2); } else { - int32_t offset = ir_fuse_load(ctx, op2, &op2_reg); + int32_t offset = ir_fuse_load(ctx, def, op2, &op2_reg); if (IR_IS_TYPE_INT(type)) { ir_emit_load_mem_int(ctx, type, def_reg, op2_reg, offset); @@ -5108,7 +5117,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else if (IR_IS_CONST_REF(op3) || !(ir_rule(ctx, op3) & IR_FUSED)) { ir_emit_load(ctx, type, def_reg, op3); } else { - int32_t offset = ir_fuse_load(ctx, op3, &op3_reg); + int32_t offset = ir_fuse_load(ctx, def, op3, &op3_reg); if (IR_IS_TYPE_INT(type)) { ir_emit_load_mem_int(ctx, type, def_reg, op3_reg, offset); @@ -5257,7 +5266,7 @@ static void ir_emit_sext(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -5380,7 +5389,7 @@ static void ir_emit_zext(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -5485,7 +5494,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); if (IR_IS_TYPE_INT(src_type) && IR_IS_TYPE_INT(dst_type)) { if (!IR_IS_CONST_REF(insn->op1) && (ir_rule(ctx, insn->op1) & IR_FUSED)) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); ir_emit_load_mem_int(ctx, dst_type, def_reg, op1_reg, offset); } else if (op1_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op1_reg)) { @@ -5500,7 +5509,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } else if (IR_IS_TYPE_FP(src_type) && IR_IS_TYPE_FP(dst_type)) { if (!IR_IS_CONST_REF(insn->op1) && (ir_rule(ctx, insn->op1) & IR_FUSED)) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); ir_emit_load_mem_fp(ctx, dst_type, def_reg, op1_reg, offset); } else if (op1_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op1_reg)) { @@ -5552,7 +5561,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -5599,7 +5608,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -5718,7 +5727,7 @@ static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) bool src64 = ir_type_size[src_type] == 8; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -5868,7 +5877,7 @@ static void ir_emit_fp2int(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -5969,7 +5978,7 @@ static void ir_emit_fp2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op1) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op1, &op1_reg); + offset = ir_fuse_load(ctx, def, insn->op1, &op1_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op1, &op1_reg); } @@ -6172,7 +6181,7 @@ static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) offset = ctx->ir_base[insn->op2].val.i32; } else { IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); - offset = ir_fuse_addr(ctx, insn->op2, &op2_reg); + offset = ir_fuse_addr(ctx, def, insn->op2, &op2_reg); if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, op2_reg, offset)) { if (!ir_may_avoid_spill_load(ctx, def, def)) { ir_emit_load_mem_int(ctx, type, def_reg, op2_reg, offset); @@ -6214,7 +6223,7 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) offset = ctx->ir_base[insn->op2].val.i32; } else { IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); - offset = ir_fuse_addr(ctx, insn->op2, &op2_reg); + offset = ir_fuse_addr(ctx, def, insn->op2, &op2_reg); if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, op2_reg, offset)) { if (!ir_may_avoid_spill_load(ctx, def, def)) { ir_emit_load_mem_fp(ctx, type, def_reg, op2_reg, offset); @@ -6252,7 +6261,7 @@ static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) offset = ctx->ir_base[insn->op2].val.i32; } else { IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); - offset = ir_fuse_addr(ctx, insn->op2, &op2_reg); + offset = ir_fuse_addr(ctx, ref, insn->op2, &op2_reg); if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, op2_reg, offset)) { if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) { op3_reg = IR_REG_NUM(op3_reg); @@ -6297,7 +6306,7 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) offset = ctx->ir_base[insn->op2].val.i32; } else { IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); - offset = ir_fuse_addr(ctx, insn->op2, &op2_reg); + offset = ir_fuse_addr(ctx, ref, insn->op2, &op2_reg); if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, op2_reg, offset)) { if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) { op3_reg = IR_REG_NUM(op3_reg); @@ -6365,7 +6374,7 @@ static void ir_emit_rstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) ir_reg dst_reg = insn->op3; if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { - int32_t offset = ir_fuse_load(ctx, insn->op2, &op2_reg); + int32_t offset = ir_fuse_load(ctx, ref, insn->op2, &op2_reg); if (IR_IS_TYPE_INT(type)) { ir_emit_load_mem_int(ctx, type, dst_reg, op2_reg, offset); } else { @@ -7444,7 +7453,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, int32_t used int32_t offset; if (ir_rule(ctx, insn->op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op2, &op2_reg); + offset = ir_fuse_load(ctx, def, insn->op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op2, &op2_reg); } @@ -7588,7 +7597,7 @@ static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset; if (ir_rule(ctx, insn->op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op2, &op2_reg); + offset = ir_fuse_load(ctx, def, insn->op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op2, &op2_reg); } @@ -7626,7 +7635,7 @@ static void ir_emit_ijmp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else if (ir_rule(ctx, insn->op2) & IR_FUSED) { int32_t offset; - offset = ir_fuse_load(ctx, insn->op2, &op2_reg); + offset = ir_fuse_load(ctx, def, insn->op2, &op2_reg); if (op2_reg == IR_REG_NONE) { | jmp aword [offset] } else { @@ -7921,7 +7930,7 @@ static bool ir_emit_guard(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) int32_t offset = 0; if (ir_rule(ctx, insn->op2) & IR_FUSED) { - offset = ir_fuse_load(ctx, insn->op2, &op2_reg); + offset = ir_fuse_load(ctx, def, insn->op2, &op2_reg); } else { offset = ir_ref_spill_slot(ctx, insn->op2, &op2_reg); } @@ -8013,7 +8022,7 @@ static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * op = IR_NE; } } - ir_emit_cmp_int_common(ctx, type, cmp_insn, op1_reg, op1, op2_reg, op2); + ir_emit_cmp_int_common(ctx, type, def, cmp_insn, op1_reg, op1, op2_reg, op2); if (insn->op == IR_GUARD) { op ^= 1; // reverse @@ -8024,7 +8033,7 @@ static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * static bool ir_emit_guard_cmp_fp(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) { - ir_op op = ir_emit_cmp_fp_common(ctx, insn->op2, &ctx->ir_base[insn->op2]); + ir_op op = ir_emit_cmp_fp_common(ctx, def, insn->op2, &ctx->ir_base[insn->op2]); void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]); if (insn->op == IR_GUARD) { @@ -8038,7 +8047,7 @@ static bool ir_emit_guard_test_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]); ir_op op = (insn->op == IR_GUARD) ? IR_EQ : IR_NE; - ir_emit_test_int_common(ctx, insn->op2, op); + ir_emit_test_int_common(ctx, def, insn->op2, op); return ir_emit_guard_jcc(ctx, b, def, op, addr, 1); } @@ -9319,7 +9328,7 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_INC: case IR_DEC: case IR_OP_INT: - ir_emit_op_int(ctx, i, insn); + ir_emit_op_int(ctx, i, insn, *rule); break; case IR_ABS_INT: ir_emit_abs_int(ctx, i, insn); @@ -9498,7 +9507,7 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_MEM_OP_INT: case IR_MEM_INC: case IR_MEM_DEC: - ir_emit_mem_op_int(ctx, i, insn); + ir_emit_mem_op_int(ctx, i, insn, *rule); break; case IR_MEM_BINOP_INT: ir_emit_mem_binop_int(ctx, i, insn);