From 6a2b0fd171082eae19f1da043cf53b5a5a7b9c6c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 30 Sep 2021 12:36:35 -0400 Subject: [PATCH 01/28] tests/docker: Remove fedora-i386-cross from DOCKER_PARTIAL_IMAGES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image was upgraded to a full image in ee381b7fe146. This makes it possible to use docker-test@image syntax with this container. Signed-off-by: Richard Henderson Reviewed-by: Daniel P. Berrangé Message-Id: <20210930163636.721311-2-richard.henderson@linaro.org> --- tests/docker/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index ff5d732889..0806c6f726 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -184,7 +184,7 @@ DOCKER_PARTIAL_IMAGES += debian-riscv64-cross DOCKER_PARTIAL_IMAGES += debian-sh4-cross debian-sparc64-cross DOCKER_PARTIAL_IMAGES += debian-tricore-cross DOCKER_PARTIAL_IMAGES += debian-xtensa-cross -DOCKER_PARTIAL_IMAGES += fedora-i386-cross fedora-cris-cross +DOCKER_PARTIAL_IMAGES += fedora-cris-cross # Rules for building linux-user powered images # From 08a13c4b247338329951238a6c47b94f70c387d2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 30 Sep 2021 12:36:36 -0400 Subject: [PATCH 02/28] tests/docker: Fix fedora-i386-cross cross-compilation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By using PKG_CONFIG_PATH instead of PKG_CONFIG_LIBDIR, we were still including the 64-bit packages. Install pcre-devel.i686 to fill a missing glib2 dependency. By using --extra-cflags instead of --cpu, we incorrectly use the wrong probing during meson. Signed-off-by: Richard Henderson Reviewed-by: Richard W.M. Jones Reviewed-by: Daniel P. Berrangé Message-Id: <20210930163636.721311-3-richard.henderson@linaro.org> --- tests/docker/dockerfiles/fedora-i386-cross.docker | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/docker/dockerfiles/fedora-i386-cross.docker b/tests/docker/dockerfiles/fedora-i386-cross.docker index 84f2697b6c..f62a71ce22 100644 --- a/tests/docker/dockerfiles/fedora-i386-cross.docker +++ b/tests/docker/dockerfiles/fedora-i386-cross.docker @@ -18,13 +18,14 @@ ENV PACKAGES \ glibc-static.i686 \ gnutls-devel.i686 \ nettle-devel.i686 \ + pcre-devel.i686 \ perl-Test-Harness \ pixman-devel.i686 \ sysprof-capture-devel.i686 \ zlib-devel.i686 -ENV QEMU_CONFIGURE_OPTS --extra-cflags=-m32 --disable-vhost-user -ENV PKG_CONFIG_PATH /usr/lib/pkgconfig +ENV QEMU_CONFIGURE_OPTS --cpu=i386 --disable-vhost-user +ENV PKG_CONFIG_LIBDIR /usr/lib/pkgconfig RUN dnf update -y && dnf install -y $PACKAGES RUN rpm -q $PACKAGES | sort > /packages.txt From db637f270b52f8c2a1c55e7e707532532295715c Mon Sep 17 00:00:00 2001 From: Philipp Tomsich Date: Sun, 3 Oct 2021 23:42:42 +0200 Subject: [PATCH 03/28] tcg: add dup_const_tl wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dup_const always generates a uint64_t, which may exceed the size of a target_long (generating warnings with recent-enough compilers). To ensure that we can use dup_const both for 64bit and 32bit targets, this adds dup_const_tl, which either maps back to dup_const (for 64bit targets) or provides a similar implementation using 32bit constants. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Philipp Tomsich Message-Id: <20211003214243.3813425-1-philipp.tomsich@vrull.eu> Signed-off-by: Richard Henderson --- include/tcg/tcg.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 44ccd86f3e..1bb6c0ce3e 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -1272,6 +1272,18 @@ uint64_t dup_const(unsigned vece, uint64_t c); : (qemu_build_not_reached_always(), 0)) \ : dup_const(VECE, C)) +#if TARGET_LONG_BITS == 64 +# define dup_const_tl dup_const +#else +# define dup_const_tl(VECE, C) \ + (__builtin_constant_p(VECE) \ + ? ( (VECE) == MO_8 ? 0x01010101ul * (uint8_t)(C) \ + : (VECE) == MO_16 ? 0x00010001ul * (uint16_t)(C) \ + : (VECE) == MO_32 ? 0x00000001ul * (uint32_t)(C) \ + : (qemu_build_not_reached_always(), 0)) \ + : (target_long)dup_const(VECE, C)) +#endif + /* * Memory helpers that will be used by TCG generated code. */ From c433e298d99228e41a78d480a505cfcc8c9ea067 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 25 Jul 2021 10:43:10 -1000 Subject: [PATCH 04/28] accel/tcg: Drop signness in tracing in cputlb.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are already inconsistent about whether or not MO_SIGN is set in trace_mem_get_info. Dropping it entirely allows some simplification. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 10 +++------- accel/tcg/user-exec.c | 45 ++++++------------------------------------- 2 files changed, 9 insertions(+), 46 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index b1e5471f94..0a1fdbefdd 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -2119,7 +2119,6 @@ static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr, meminfo = trace_mem_get_info(op, mmu_idx, false); trace_guest_mem_before_exec(env_cpu(env), addr, meminfo); - op &= ~MO_SIGN; oi = make_memop_idx(op, mmu_idx); ret = full_load(env, addr, oi, retaddr); @@ -2137,8 +2136,7 @@ uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr, int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { - return (int8_t)cpu_load_helper(env, addr, mmu_idx, ra, MO_SB, - full_ldub_mmu); + return (int8_t)cpu_ldub_mmuidx_ra(env, addr, mmu_idx, ra); } uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, @@ -2150,8 +2148,7 @@ uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { - return (int16_t)cpu_load_helper(env, addr, mmu_idx, ra, MO_BESW, - full_be_lduw_mmu); + return (int16_t)cpu_lduw_be_mmuidx_ra(env, addr, mmu_idx, ra); } uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, @@ -2175,8 +2172,7 @@ uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t ra) { - return (int16_t)cpu_load_helper(env, addr, mmu_idx, ra, MO_LESW, - full_le_lduw_mmu); + return (int16_t)cpu_lduw_le_mmuidx_ra(env, addr, mmu_idx, ra); } uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 8fed542622..8f2644f26e 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -899,13 +899,7 @@ uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr) int cpu_ldsb_data(CPUArchState *env, abi_ptr ptr) { - int ret; - uint16_t meminfo = trace_mem_get_info(MO_SB, MMU_USER_IDX, false); - - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); - ret = ldsb_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); - return ret; + return (int8_t)cpu_ldub_data(env, ptr); } uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr) @@ -921,13 +915,7 @@ uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr) int cpu_ldsw_be_data(CPUArchState *env, abi_ptr ptr) { - int ret; - uint16_t meminfo = trace_mem_get_info(MO_BESW, MMU_USER_IDX, false); - - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); - ret = ldsw_be_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); - return ret; + return (int16_t)cpu_lduw_be_data(env, ptr); } uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr) @@ -965,13 +953,7 @@ uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr) int cpu_ldsw_le_data(CPUArchState *env, abi_ptr ptr) { - int ret; - uint16_t meminfo = trace_mem_get_info(MO_LESW, MMU_USER_IDX, false); - - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); - ret = ldsw_le_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); - return ret; + return (int16_t)cpu_lduw_le_data(env, ptr); } uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr) @@ -1008,12 +990,7 @@ uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) { - int ret; - - set_helper_retaddr(retaddr); - ret = cpu_ldsb_data(env, ptr); - clear_helper_retaddr(); - return ret; + return (int8_t)cpu_ldub_data_ra(env, ptr, retaddr); } uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) @@ -1028,12 +1005,7 @@ uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) { - int ret; - - set_helper_retaddr(retaddr); - ret = cpu_ldsw_be_data(env, ptr); - clear_helper_retaddr(); - return ret; + return (int16_t)cpu_lduw_be_data_ra(env, ptr, retaddr); } uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) @@ -1068,12 +1040,7 @@ uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) { - int ret; - - set_helper_retaddr(retaddr); - ret = cpu_ldsw_le_data(env, ptr); - clear_helper_retaddr(); - return ret; + return (int16_t)cpu_lduw_le_data_ra(env, ptr, retaddr); } uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) From 4b473e0c60d802bb69accab3177d350fc580e2a4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Jul 2021 10:32:17 -1000 Subject: [PATCH 05/28] tcg: Expand MO_SIZE to 3 bits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have lacked expressive support for memory sizes larger than 64-bits for a while. Fixing that requires adjustment to several points where we used this for array indexing, and two places that develop -Wswitch warnings after the change. Reviewed-by: Alistair Francis Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/exec/memop.h | 14 +++++++++----- target/arm/translate-a64.c | 2 +- target/s390x/tcg/translate_vx.c.inc | 2 +- tcg/aarch64/tcg-target.c.inc | 4 ++-- tcg/arm/tcg-target.c.inc | 4 ++-- tcg/i386/tcg-target.c.inc | 4 ++-- tcg/mips/tcg-target.c.inc | 4 ++-- tcg/ppc/tcg-target.c.inc | 8 ++++---- tcg/riscv/tcg-target.c.inc | 4 ++-- tcg/s390/tcg-target.c.inc | 4 ++-- tcg/sparc/tcg-target.c.inc | 16 ++++++++-------- tcg/tcg-op.c | 13 ++++++++----- 12 files changed, 43 insertions(+), 36 deletions(-) diff --git a/include/exec/memop.h b/include/exec/memop.h index 529d07b02d..04264ffd6b 100644 --- a/include/exec/memop.h +++ b/include/exec/memop.h @@ -19,11 +19,15 @@ typedef enum MemOp { MO_16 = 1, MO_32 = 2, MO_64 = 3, - MO_SIZE = 3, /* Mask for the above. */ + MO_128 = 4, + MO_256 = 5, + MO_512 = 6, + MO_1024 = 7, + MO_SIZE = 0x07, /* Mask for the above. */ - MO_SIGN = 4, /* Sign-extended, otherwise zero-extended. */ + MO_SIGN = 0x08, /* Sign-extended, otherwise zero-extended. */ - MO_BSWAP = 8, /* Host reverse endian. */ + MO_BSWAP = 0x10, /* Host reverse endian. */ #ifdef HOST_WORDS_BIGENDIAN MO_LE = MO_BSWAP, MO_BE = 0, @@ -59,8 +63,8 @@ typedef enum MemOp { * - an alignment to a specified size, which may be more or less than * the access size (MO_ALIGN_x where 'x' is a size in bytes); */ - MO_ASHIFT = 4, - MO_AMASK = 7 << MO_ASHIFT, + MO_ASHIFT = 5, + MO_AMASK = 0x7 << MO_ASHIFT, #ifdef NEED_CPU_H #ifdef TARGET_ALIGNED_ONLY MO_ALIGN = 0, diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index ab6b346e35..717afd481c 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -1045,7 +1045,7 @@ static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx, int element, MemOp memop) { int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE); - switch (memop) { + switch ((unsigned)memop) { case MO_8: tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off); break; diff --git a/target/s390x/tcg/translate_vx.c.inc b/target/s390x/tcg/translate_vx.c.inc index 0afa46e463..28bf5a23b6 100644 --- a/target/s390x/tcg/translate_vx.c.inc +++ b/target/s390x/tcg/translate_vx.c.inc @@ -67,7 +67,7 @@ static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr, { const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE); - switch (memop) { + switch ((unsigned)memop) { case ES_8: tcg_gen_ld8u_i64(dst, cpu_env, offs); break; diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 5924977b42..6f43c048a5 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -1547,7 +1547,7 @@ static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d, /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * TCGMemOpIdx oi, uintptr_t ra) */ -static void * const qemu_ld_helpers[4] = { +static void * const qemu_ld_helpers[MO_SIZE + 1] = { [MO_8] = helper_ret_ldub_mmu, #ifdef HOST_WORDS_BIGENDIAN [MO_16] = helper_be_lduw_mmu, @@ -1564,7 +1564,7 @@ static void * const qemu_ld_helpers[4] = { * uintxx_t val, TCGMemOpIdx oi, * uintptr_t ra) */ -static void * const qemu_st_helpers[4] = { +static void * const qemu_st_helpers[MO_SIZE + 1] = { [MO_8] = helper_ret_stb_mmu, #ifdef HOST_WORDS_BIGENDIAN [MO_16] = helper_be_stw_mmu, diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index d25e68b36b..d71f4a2317 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -1437,7 +1437,7 @@ static void tcg_out_vldst(TCGContext *s, ARMInsn insn, /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) */ -static void * const qemu_ld_helpers[8] = { +static void * const qemu_ld_helpers[MO_SSIZE + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_SB] = helper_ret_ldsb_mmu, #ifdef HOST_WORDS_BIGENDIAN @@ -1458,7 +1458,7 @@ static void * const qemu_ld_helpers[8] = { /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, * uintxx_t val, int mmu_idx, uintptr_t ra) */ -static void * const qemu_st_helpers[4] = { +static void * const qemu_st_helpers[MO_SIZE + 1] = { [MO_8] = helper_ret_stb_mmu, #ifdef HOST_WORDS_BIGENDIAN [MO_16] = helper_be_stw_mmu, diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 997510109d..4aabc62606 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -1611,7 +1611,7 @@ static void tcg_out_nopn(TCGContext *s, int n) /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) */ -static void * const qemu_ld_helpers[16] = { +static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_LEUW] = helper_le_lduw_mmu, [MO_LEUL] = helper_le_ldul_mmu, @@ -1624,7 +1624,7 @@ static void * const qemu_ld_helpers[16] = { /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, * uintxx_t val, int mmu_idx, uintptr_t ra) */ -static void * const qemu_st_helpers[16] = { +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_stb_mmu, [MO_LEUW] = helper_le_stw_mmu, [MO_LEUL] = helper_le_stl_mmu, diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index 41ffa28394..84aa775c0d 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -1017,7 +1017,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) #if defined(CONFIG_SOFTMMU) #include "../tcg-ldst.c.inc" -static void * const qemu_ld_helpers[16] = { +static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_SB] = helper_ret_ldsb_mmu, [MO_LEUW] = helper_le_lduw_mmu, @@ -1034,7 +1034,7 @@ static void * const qemu_ld_helpers[16] = { #endif }; -static void * const qemu_st_helpers[16] = { +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_stb_mmu, [MO_LEUW] = helper_le_stw_mmu, [MO_LEUL] = helper_le_stl_mmu, diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 5e1fac914a..7e8dee2cc6 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -1931,7 +1931,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) #endif } -static const uint32_t qemu_ldx_opc[16] = { +static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = { [MO_UB] = LBZX, [MO_UW] = LHZX, [MO_UL] = LWZX, @@ -1944,7 +1944,7 @@ static const uint32_t qemu_ldx_opc[16] = { [MO_BSWAP | MO_Q] = LDBRX, }; -static const uint32_t qemu_stx_opc[16] = { +static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = { [MO_UB] = STBX, [MO_UW] = STHX, [MO_UL] = STWX, @@ -1965,7 +1965,7 @@ static const uint32_t qemu_exts_opc[4] = { /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) */ -static void * const qemu_ld_helpers[16] = { +static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_LEUW] = helper_le_lduw_mmu, [MO_LEUL] = helper_le_ldul_mmu, @@ -1978,7 +1978,7 @@ static void * const qemu_ld_helpers[16] = { /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr, * uintxx_t val, int mmu_idx, uintptr_t ra) */ -static void * const qemu_st_helpers[16] = { +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_stb_mmu, [MO_LEUW] = helper_le_stw_mmu, [MO_LEUL] = helper_le_stl_mmu, diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index dc8d8f1de2..da48f9a633 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -852,7 +852,7 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * TCGMemOpIdx oi, uintptr_t ra) */ -static void * const qemu_ld_helpers[8] = { +static void * const qemu_ld_helpers[MO_SSIZE + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_SB] = helper_ret_ldsb_mmu, #ifdef HOST_WORDS_BIGENDIAN @@ -878,7 +878,7 @@ static void * const qemu_ld_helpers[8] = { * uintxx_t val, TCGMemOpIdx oi, * uintptr_t ra) */ -static void * const qemu_st_helpers[4] = { +static void * const qemu_st_helpers[MO_SIZE + 1] = { [MO_8] = helper_ret_stb_mmu, #ifdef HOST_WORDS_BIGENDIAN [MO_16] = helper_be_stw_mmu, diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index b82cf19f09..67a2ba5ff3 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -350,7 +350,7 @@ static const uint8_t tcg_cond_to_ltr_cond[] = { }; #ifdef CONFIG_SOFTMMU -static void * const qemu_ld_helpers[16] = { +static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_ldub_mmu, [MO_SB] = helper_ret_ldsb_mmu, [MO_LEUW] = helper_le_lduw_mmu, @@ -365,7 +365,7 @@ static void * const qemu_ld_helpers[16] = { [MO_BEQ] = helper_be_ldq_mmu, }; -static void * const qemu_st_helpers[16] = { +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = helper_ret_stb_mmu, [MO_LEUW] = helper_le_stw_mmu, [MO_LEUL] = helper_le_stl_mmu, diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index 9720d76abd..43248776a1 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -855,8 +855,8 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) } #ifdef CONFIG_SOFTMMU -static const tcg_insn_unit *qemu_ld_trampoline[16]; -static const tcg_insn_unit *qemu_st_trampoline[16]; +static const tcg_insn_unit *qemu_ld_trampoline[(MO_SSIZE | MO_BSWAP) + 1]; +static const tcg_insn_unit *qemu_st_trampoline[(MO_SIZE | MO_BSWAP) + 1]; static void emit_extend(TCGContext *s, TCGReg r, int op) { @@ -883,7 +883,7 @@ static void emit_extend(TCGContext *s, TCGReg r, int op) static void build_trampolines(TCGContext *s) { - static void * const qemu_ld_helpers[16] = { + static void * const qemu_ld_helpers[] = { [MO_UB] = helper_ret_ldub_mmu, [MO_SB] = helper_ret_ldsb_mmu, [MO_LEUW] = helper_le_lduw_mmu, @@ -895,7 +895,7 @@ static void build_trampolines(TCGContext *s) [MO_BEUL] = helper_be_ldul_mmu, [MO_BEQ] = helper_be_ldq_mmu, }; - static void * const qemu_st_helpers[16] = { + static void * const qemu_st_helpers[] = { [MO_UB] = helper_ret_stb_mmu, [MO_LEUW] = helper_le_stw_mmu, [MO_LEUL] = helper_le_stl_mmu, @@ -908,7 +908,7 @@ static void build_trampolines(TCGContext *s) int i; TCGReg ra; - for (i = 0; i < 16; ++i) { + for (i = 0; i < ARRAY_SIZE(qemu_ld_helpers); ++i) { if (qemu_ld_helpers[i] == NULL) { continue; } @@ -936,7 +936,7 @@ static void build_trampolines(TCGContext *s) tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra); } - for (i = 0; i < 16; ++i) { + for (i = 0; i < ARRAY_SIZE(qemu_st_helpers); ++i) { if (qemu_st_helpers[i] == NULL) { continue; } @@ -1118,7 +1118,7 @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index, } #endif /* CONFIG_SOFTMMU */ -static const int qemu_ld_opc[16] = { +static const int qemu_ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = { [MO_UB] = LDUB, [MO_SB] = LDSB, @@ -1135,7 +1135,7 @@ static const int qemu_ld_opc[16] = { [MO_LEQ] = LDX_LE, }; -static const int qemu_st_opc[16] = { +static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = { [MO_UB] = STB, [MO_BEUW] = STH, diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index c754396575..e01f68f44d 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -2780,10 +2780,13 @@ static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) } break; case MO_64: - if (!is64) { - tcg_abort(); + if (is64) { + op &= ~MO_SIGN; + break; } - break; + /* fall through */ + default: + g_assert_not_reached(); } if (st) { op &= ~MO_SIGN; @@ -3095,7 +3098,7 @@ typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, # define WITH_ATOMIC64(X) #endif -static void * const table_cmpxchg[16] = { +static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { [MO_8] = gen_helper_atomic_cmpxchgb, [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, @@ -3297,7 +3300,7 @@ static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, } #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ -static void * const table_##NAME[16] = { \ +static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ [MO_8] = gen_helper_atomic_##NAME##b, \ [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ From 9002ffcb7264947d9a193567b457dea42f15c321 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 25 Jul 2021 12:06:49 -1000 Subject: [PATCH 06/28] tcg: Rename TCGMemOpIdx to MemOpIdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're about to move this out of tcg.h, so rename it as we did when moving MemOp. Reviewed-by: Alistair Francis Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/atomic_common.c.inc | 6 +-- accel/tcg/atomic_template.h | 24 +++++------ accel/tcg/cputlb.c | 78 +++++++++++++++++------------------ accel/tcg/user-exec.c | 2 +- include/tcg/tcg.h | 74 ++++++++++++++++----------------- target/arm/helper-a64.c | 16 +++---- target/arm/m_helper.c | 2 +- target/i386/tcg/mem_helper.c | 4 +- target/m68k/op_helper.c | 2 +- target/mips/tcg/msa_helper.c | 6 +-- target/s390x/tcg/mem_helper.c | 20 ++++----- target/sparc/ldst_helper.c | 2 +- tcg/aarch64/tcg-target.c.inc | 14 +++---- tcg/arm/tcg-target.c.inc | 10 ++--- tcg/i386/tcg-target.c.inc | 10 ++--- tcg/mips/tcg-target.c.inc | 12 +++--- tcg/optimize.c | 2 +- tcg/ppc/tcg-target.c.inc | 10 ++--- tcg/riscv/tcg-target.c.inc | 16 +++---- tcg/s390/tcg-target.c.inc | 10 ++--- tcg/sparc/tcg-target.c.inc | 4 +- tcg/tcg-ldst.c.inc | 2 +- tcg/tcg-op.c | 12 +++--- tcg/tcg.c | 2 +- tcg/tci.c | 14 +++---- 25 files changed, 177 insertions(+), 177 deletions(-) diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index 6c0339f610..ebaa793464 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -14,7 +14,7 @@ */ static uint16_t atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi) + MemOpIdx oi) { CPUState *cpu = env_cpu(env); uint16_t info = trace_mem_get_info(get_memop(oi), get_mmuidx(oi), false); @@ -34,7 +34,7 @@ static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, #if HAVE_ATOMIC128 static uint16_t atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi) + MemOpIdx oi) { uint16_t info = trace_mem_get_info(get_memop(oi), get_mmuidx(oi), false); @@ -50,7 +50,7 @@ static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, } static uint16_t atomic_trace_st_pre(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi) + MemOpIdx oi) { uint16_t info = trace_mem_get_info(get_memop(oi), get_mmuidx(oi), true); diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index 8098a1be31..4230ff2957 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -72,7 +72,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); @@ -92,7 +92,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, #if DATA_SIZE >= 16 #if HAVE_ATOMIC128 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ, retaddr); @@ -106,7 +106,7 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, } void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_WRITE, retaddr); @@ -119,7 +119,7 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, #endif #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); @@ -134,7 +134,7 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, #define GEN_ATOMIC_HELPER(X) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ + ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ { \ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ @@ -167,7 +167,7 @@ GEN_ATOMIC_HELPER(xor_fetch) */ #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ + ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ { \ XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ @@ -211,7 +211,7 @@ GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, ABI_TYPE cmpv, ABI_TYPE newv, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); @@ -231,7 +231,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, #if DATA_SIZE >= 16 #if HAVE_ATOMIC128 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ, retaddr); @@ -245,7 +245,7 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, } void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_WRITE, retaddr); @@ -259,7 +259,7 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, #endif #else ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); @@ -274,7 +274,7 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, #define GEN_ATOMIC_HELPER(X) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ + ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ { \ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ @@ -304,7 +304,7 @@ GEN_ATOMIC_HELPER(xor_fetch) */ #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ - ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ + ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ { \ XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 0a1fdbefdd..d72f65f42b 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1749,7 +1749,7 @@ bool tlb_plugin_lookup(CPUState *cpu, target_ulong addr, int mmu_idx, * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE. */ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, int size, int prot, + MemOpIdx oi, int size, int prot, uintptr_t retaddr) { size_t mmu_idx = get_mmuidx(oi); @@ -1850,7 +1850,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, */ typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); static inline uint64_t QEMU_ALWAYS_INLINE load_memop(const void *haddr, MemOp op) @@ -1876,7 +1876,7 @@ load_memop(const void *haddr, MemOp op) } static inline uint64_t QEMU_ALWAYS_INLINE -load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, +load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi, uintptr_t retaddr, MemOp op, bool code_read, FullLoadHelper *full_load) { @@ -1991,78 +1991,78 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, */ static uint64_t full_ldub_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_UB, false, full_ldub_mmu); } tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return full_ldub_mmu(env, addr, oi, retaddr); } static uint64_t full_le_lduw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_LEUW, false, full_le_lduw_mmu); } tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return full_le_lduw_mmu(env, addr, oi, retaddr); } static uint64_t full_be_lduw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_BEUW, false, full_be_lduw_mmu); } tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return full_be_lduw_mmu(env, addr, oi, retaddr); } static uint64_t full_le_ldul_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_LEUL, false, full_le_ldul_mmu); } tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return full_le_ldul_mmu(env, addr, oi, retaddr); } static uint64_t full_be_ldul_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_BEUL, false, full_be_ldul_mmu); } tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return full_be_ldul_mmu(env, addr, oi, retaddr); } uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_LEQ, false, helper_le_ldq_mmu); } uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_BEQ, false, helper_be_ldq_mmu); @@ -2075,31 +2075,31 @@ uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return (int8_t)helper_ret_ldub_mmu(env, addr, oi, retaddr); } tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return (int16_t)helper_le_lduw_mmu(env, addr, oi, retaddr); } tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return (int16_t)helper_be_lduw_mmu(env, addr, oi, retaddr); } tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return (int32_t)helper_le_ldul_mmu(env, addr, oi, retaddr); } tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return (int32_t)helper_be_ldul_mmu(env, addr, oi, retaddr); } @@ -2113,7 +2113,7 @@ static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr, MemOp op, FullLoadHelper *full_load) { uint16_t meminfo; - TCGMemOpIdx oi; + MemOpIdx oi; uint64_t ret; meminfo = trace_mem_get_info(op, mmu_idx, false); @@ -2337,7 +2337,7 @@ store_helper_unaligned(CPUArchState *env, target_ulong addr, uint64_t val, uintptr_t index, index2; CPUTLBEntry *entry, *entry2; target_ulong page2, tlb_addr, tlb_addr2; - TCGMemOpIdx oi; + MemOpIdx oi; size_t size2; int i; @@ -2404,7 +2404,7 @@ store_helper_unaligned(CPUArchState *env, target_ulong addr, uint64_t val, static inline void QEMU_ALWAYS_INLINE store_helper(CPUArchState *env, target_ulong addr, uint64_t val, - TCGMemOpIdx oi, uintptr_t retaddr, MemOp op) + MemOpIdx oi, uintptr_t retaddr, MemOp op) { uintptr_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); @@ -2502,43 +2502,43 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, void __attribute__((noinline)) helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_UB); } void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_LEUW); } void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_BEUW); } void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_LEUL); } void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_BEUL); } void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_LEQ); } void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { store_helper(env, addr, val, oi, retaddr, MO_BEQ); } @@ -2551,7 +2551,7 @@ static inline void QEMU_ALWAYS_INLINE cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val, int mmu_idx, uintptr_t retaddr, MemOp op) { - TCGMemOpIdx oi; + MemOpIdx oi; uint16_t meminfo; meminfo = trace_mem_get_info(op, mmu_idx, true); @@ -2717,49 +2717,49 @@ void cpu_stq_le_data(CPUArchState *env, target_ulong ptr, uint64_t val) /* Code access functions. */ static uint64_t full_ldub_code(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_8, true, full_ldub_code); } uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr) { - TCGMemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(env, true)); + MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(env, true)); return full_ldub_code(env, addr, oi, 0); } static uint64_t full_lduw_code(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_TEUW, true, full_lduw_code); } uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr) { - TCGMemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(env, true)); + MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(env, true)); return full_lduw_code(env, addr, oi, 0); } static uint64_t full_ldl_code(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_TEUL, true, full_ldl_code); } uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr) { - TCGMemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(env, true)); + MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(env, true)); return full_ldl_code(env, addr, oi, 0); } static uint64_t full_ldq_code(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr) + MemOpIdx oi, uintptr_t retaddr) { return load_helper(env, addr, oi, retaddr, MO_TEQ, true, full_ldq_code); } uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr) { - TCGMemOpIdx oi = make_memop_idx(MO_TEQ, cpu_mmu_index(env, true)); + MemOpIdx oi = make_memop_idx(MO_TEQ, cpu_mmu_index(env, true)); return full_ldq_code(env, addr, oi, 0); } diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 8f2644f26e..1187362a4c 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -1228,7 +1228,7 @@ uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr ptr) * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE. */ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, int size, int prot, + MemOpIdx oi, int size, int prot, uintptr_t retaddr) { /* Enforce qemu required alignment. */ diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 1bb6c0ce3e..1a0da58f92 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -1148,7 +1148,7 @@ static inline size_t tcg_current_code_size(TCGContext *s) } /* Combine the MemOp and mmu_idx parameters into a single value. */ -typedef uint32_t TCGMemOpIdx; +typedef uint32_t MemOpIdx; /** * make_memop_idx @@ -1157,7 +1157,7 @@ typedef uint32_t TCGMemOpIdx; * * Encode these values into a single parameter. */ -static inline TCGMemOpIdx make_memop_idx(MemOp op, unsigned idx) +static inline MemOpIdx make_memop_idx(MemOp op, unsigned idx) { tcg_debug_assert(idx <= 15); return (op << 4) | idx; @@ -1169,7 +1169,7 @@ static inline TCGMemOpIdx make_memop_idx(MemOp op, unsigned idx) * * Extract the memory operation from the combined value. */ -static inline MemOp get_memop(TCGMemOpIdx oi) +static inline MemOp get_memop(MemOpIdx oi) { return oi >> 4; } @@ -1180,7 +1180,7 @@ static inline MemOp get_memop(TCGMemOpIdx oi) * * Extract the mmu index from the combined value. */ -static inline unsigned get_mmuidx(TCGMemOpIdx oi) +static inline unsigned get_mmuidx(MemOpIdx oi) { return oi & 15; } @@ -1290,46 +1290,46 @@ uint64_t dup_const(unsigned vece, uint64_t c); #ifdef CONFIG_SOFTMMU /* Value zero-extended to tcg register size. */ tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); /* Value sign-extended to tcg register size. */ tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); /* Temporary aliases until backends are converted. */ #ifdef TARGET_WORDS_BIGENDIAN @@ -1357,30 +1357,30 @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, uint32_t cpu_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr, uint32_t cmpv, uint32_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint32_t cpu_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr, uint32_t cmpv, uint32_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint32_t cpu_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr, uint32_t cmpv, uint32_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint64_t cpu_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr, uint64_t cmpv, uint64_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint32_t cpu_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr, uint32_t cmpv, uint32_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint32_t cpu_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr, uint32_t cmpv, uint32_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); uint64_t cpu_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr, uint64_t cmpv, uint64_t newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); #define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \ TYPE cpu_atomic_ ## NAME ## SUFFIX ## _mmu \ (CPUArchState *env, target_ulong addr, TYPE val, \ - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); #ifdef CONFIG_ATOMIC64 #define GEN_ATOMIC_HELPER_ALL(NAME) \ @@ -1427,19 +1427,19 @@ GEN_ATOMIC_HELPER_ALL(xchg) Int128 cpu_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr, Int128 cmpv, Int128 newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); Int128 cpu_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr, Int128 cmpv, Int128 newv, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); Int128 cpu_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); Int128 cpu_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void cpu_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); void cpu_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val, - TCGMemOpIdx oi, uintptr_t retaddr); + MemOpIdx oi, uintptr_t retaddr); #ifdef CONFIG_DEBUG_TCG void tcg_assert_listed_vecop(TCGOpcode); diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index 19445b3c94..c5af779006 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -531,8 +531,8 @@ uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr, clear_helper_retaddr(); #else int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); - TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); + MemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx); + MemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx); o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra); o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra); @@ -555,7 +555,7 @@ uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr, uintptr_t ra = GETPC(); bool success; int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; assert(HAVE_CMPXCHG128); @@ -601,8 +601,8 @@ uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr, clear_helper_retaddr(); #else int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); - TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); + MemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx); + MemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx); o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra); o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra); @@ -625,7 +625,7 @@ uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr, uintptr_t ra = GETPC(); bool success; int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; assert(HAVE_CMPXCHG128); @@ -651,7 +651,7 @@ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, Int128 oldv, cmpv, newv; uintptr_t ra = GETPC(); int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; assert(HAVE_CMPXCHG128); @@ -672,7 +672,7 @@ void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr, Int128 oldv, cmpv, newv; uintptr_t ra = GETPC(); int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; assert(HAVE_CMPXCHG128); diff --git a/target/arm/m_helper.c b/target/arm/m_helper.c index 47903b3dc3..62aa12c9d8 100644 --- a/target/arm/m_helper.c +++ b/target/arm/m_helper.c @@ -1930,7 +1930,7 @@ static bool do_v7m_function_return(ARMCPU *cpu) { bool threadmode, spsel; - TCGMemOpIdx oi; + MemOpIdx oi; ARMMMUIdx mmu_idx; uint32_t *frame_sp_p; uint32_t frameptr; diff --git a/target/i386/tcg/mem_helper.c b/target/i386/tcg/mem_helper.c index 2da3cd14b6..0fd696f9c1 100644 --- a/target/i386/tcg/mem_helper.c +++ b/target/i386/tcg/mem_helper.c @@ -67,7 +67,7 @@ void helper_cmpxchg8b(CPUX86State *env, target_ulong a0) { uintptr_t ra = GETPC(); int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ, mem_idx); + MemOpIdx oi = make_memop_idx(MO_TEQ, mem_idx); oldv = cpu_atomic_cmpxchgq_le_mmu(env, a0, cmpv, newv, oi, ra); } @@ -136,7 +136,7 @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0) Int128 newv = int128_make128(env->regs[R_EBX], env->regs[R_ECX]); int mem_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + MemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 oldv = cpu_atomic_cmpxchgo_le_mmu(env, a0, cmpv, newv, oi, ra); if (int128_eq(oldv, cmpv)) { diff --git a/target/m68k/op_helper.c b/target/m68k/op_helper.c index 5d624838ae..c1bf73b6f9 100644 --- a/target/m68k/op_helper.c +++ b/target/m68k/op_helper.c @@ -775,7 +775,7 @@ static void do_cas2l(CPUM68KState *env, uint32_t regs, uint32_t a1, uint32_t a2, uintptr_t ra = GETPC(); #if defined(CONFIG_ATOMIC64) int mmu_idx = cpu_mmu_index(env, 0); - TCGMemOpIdx oi = make_memop_idx(MO_BEQ, mmu_idx); + MemOpIdx oi = make_memop_idx(MO_BEQ, mmu_idx); #endif if (parallel) { diff --git a/target/mips/tcg/msa_helper.c b/target/mips/tcg/msa_helper.c index 04af54f66d..167d9a591c 100644 --- a/target/mips/tcg/msa_helper.c +++ b/target/mips/tcg/msa_helper.c @@ -8211,9 +8211,9 @@ void helper_msa_ffint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd, #define DF_ELEMENTS(df) (MSA_WRLEN / DF_BITS(df)) #if !defined(CONFIG_USER_ONLY) -#define MEMOP_IDX(DF) \ - TCGMemOpIdx oi = make_memop_idx(MO_TE | DF | MO_UNALN, \ - cpu_mmu_index(env, false)); +#define MEMOP_IDX(DF) \ + MemOpIdx oi = make_memop_idx(MO_TE | DF | MO_UNALN, \ + cpu_mmu_index(env, false)); #else #define MEMOP_IDX(DF) #endif diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c index 0bf775a37d..75f6735545 100644 --- a/target/s390x/tcg/mem_helper.c +++ b/target/s390x/tcg/mem_helper.c @@ -239,7 +239,7 @@ static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr, g_assert(haddr); memset(haddr, byte, size); #else - TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); + MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); int i; if (likely(haddr)) { @@ -282,7 +282,7 @@ static uint8_t do_access_get_byte(CPUS390XState *env, vaddr vaddr, char **haddr, #ifdef CONFIG_USER_ONLY return ldub_p(*haddr + offset); #else - TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); + MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); uint8_t byte; if (likely(*haddr)) { @@ -316,7 +316,7 @@ static void do_access_set_byte(CPUS390XState *env, vaddr vaddr, char **haddr, #ifdef CONFIG_USER_ONLY stb_p(*haddr + offset, byte); #else - TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); + MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); if (likely(*haddr)) { stb_p(*haddr + offset, byte); @@ -1804,7 +1804,7 @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]); int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; Int128 oldv; bool fail; @@ -1884,7 +1884,7 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, uint32_t *haddr = g2h(env_cpu(env), a1); ov = qatomic_cmpxchg__nocheck(haddr, cv, nv); #else - TCGMemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx); + MemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx); ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra); #endif } else { @@ -1904,7 +1904,7 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, if (parallel) { #ifdef CONFIG_ATOMIC64 - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN, mem_idx); + MemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN, mem_idx); ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra); #else /* Note that we asserted !parallel above. */ @@ -1940,7 +1940,7 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra); cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra); } else if (HAVE_CMPXCHG128) { - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + MemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); cc = !int128_eq(ov, cv); } else { @@ -1979,7 +1979,7 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, cpu_stq_data_ra(env, a2 + 0, svh, ra); cpu_stq_data_ra(env, a2 + 8, svl, ra); } else if (HAVE_ATOMIC128) { - TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + MemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); Int128 sv = int128_make128(svl, svh); cpu_atomic_sto_be_mmu(env, a2, sv, oi, ra); } else { @@ -2497,7 +2497,7 @@ uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) uintptr_t ra = GETPC(); uint64_t hi, lo; int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; Int128 v; assert(HAVE_ATOMIC128); @@ -2528,7 +2528,7 @@ void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, { uintptr_t ra = GETPC(); int mem_idx; - TCGMemOpIdx oi; + MemOpIdx oi; Int128 v; assert(HAVE_ATOMIC128); diff --git a/target/sparc/ldst_helper.c b/target/sparc/ldst_helper.c index 22327d7d72..abe2889d27 100644 --- a/target/sparc/ldst_helper.c +++ b/target/sparc/ldst_helper.c @@ -1318,7 +1318,7 @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, case ASI_SNF: case ASI_SNFL: { - TCGMemOpIdx oi; + MemOpIdx oi; int idx = (env->pstate & PS_PRIV ? (asi & 1 ? MMU_KERNEL_SECONDARY_IDX : MMU_KERNEL_IDX) : (asi & 1 ? MMU_USER_SECONDARY_IDX : MMU_USER_IDX)); diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 6f43c048a5..5edca8d44d 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -1545,7 +1545,7 @@ static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d, #include "../tcg-ldst.c.inc" /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * TCGMemOpIdx oi, uintptr_t ra) + * MemOpIdx oi, uintptr_t ra) */ static void * const qemu_ld_helpers[MO_SIZE + 1] = { [MO_8] = helper_ret_ldub_mmu, @@ -1561,7 +1561,7 @@ static void * const qemu_ld_helpers[MO_SIZE + 1] = { }; /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, TCGMemOpIdx oi, + * uintxx_t val, MemOpIdx oi, * uintptr_t ra) */ static void * const qemu_st_helpers[MO_SIZE + 1] = { @@ -1586,7 +1586,7 @@ static inline void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target) static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); MemOp size = opc & MO_SIZE; @@ -1611,7 +1611,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); MemOp size = opc & MO_SIZE; @@ -1629,7 +1629,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) return true; } -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, TCGType ext, TCGReg data_reg, TCGReg addr_reg, tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) { @@ -1778,7 +1778,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop, } static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - TCGMemOpIdx oi, TCGType ext) + MemOpIdx oi, TCGType ext) { MemOp memop = get_memop(oi); const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; @@ -1803,7 +1803,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, } static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, - TCGMemOpIdx oi) + MemOpIdx oi) { MemOp memop = get_memop(oi); const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32; diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index d71f4a2317..633b8a37ba 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -1632,7 +1632,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, /* Record the context of a call to the out of line helper code for the slow path for a load or store, so that we can later generate the correct helper code. */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) @@ -1652,7 +1652,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { TCGReg argreg, datalo, datahi; - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); void *func; @@ -1716,7 +1716,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { TCGReg argreg, datalo, datahi; - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { @@ -1846,7 +1846,7 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) { TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused)); - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #ifdef CONFIG_SOFTMMU int mem_index; @@ -1952,7 +1952,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) { TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused)); - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #ifdef CONFIG_SOFTMMU int mem_index; diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 4aabc62606..84b109bb84 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -1741,7 +1741,7 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, * for a load or store, so that we can later generate the correct helper code */ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, - TCGMemOpIdx oi, + MemOpIdx oi, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, tcg_insn_unit *raddr, @@ -1768,7 +1768,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, */ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - TCGMemOpIdx oi = l->oi; + MemOpIdx oi = l->oi; MemOp opc = get_memop(oi); TCGReg data_reg; tcg_insn_unit **label_ptr = &l->label_ptr[0]; @@ -1853,7 +1853,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) */ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - TCGMemOpIdx oi = l->oi; + MemOpIdx oi = l->oi; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; tcg_insn_unit **label_ptr = &l->label_ptr[0]; @@ -2054,7 +2054,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) { TCGReg datalo, datahi, addrlo; TCGReg addrhi __attribute__((unused)); - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) int mem_index; @@ -2143,7 +2143,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) { TCGReg datalo, datahi, addrlo; TCGReg addrhi __attribute__((unused)); - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) int mem_index; diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index 84aa775c0d..d8f6914f03 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -1120,7 +1120,7 @@ QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -32768); * Clobbers TMP0, TMP1, TMP2, TMP3. */ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, - TCGReg addrh, TCGMemOpIdx oi, + TCGReg addrh, MemOpIdx oi, tcg_insn_unit *label_ptr[2], bool is_load) { MemOp opc = get_memop(oi); @@ -1196,7 +1196,7 @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, tcg_out_opc_reg(s, ALIAS_PADD, base, TCG_TMP2, addrl); } -static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, int is_ld, MemOpIdx oi, TCGType ext, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, @@ -1221,7 +1221,7 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { const tcg_insn_unit *tgt_rx = tcg_splitwx_to_rx(s->code_ptr); - TCGMemOpIdx oi = l->oi; + MemOpIdx oi = l->oi; MemOp opc = get_memop(oi); TCGReg v0; int i; @@ -1275,7 +1275,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { const tcg_insn_unit *tgt_rx = tcg_splitwx_to_rx(s->code_ptr); - TCGMemOpIdx oi = l->oi; + MemOpIdx oi = l->oi; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; int i; @@ -1434,7 +1434,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_regl, addr_regh __attribute__((unused)); TCGReg data_regl, data_regh; - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; @@ -1536,7 +1536,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_regl, addr_regh __attribute__((unused)); TCGReg data_regl, data_regh; - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[2]; diff --git a/tcg/optimize.c b/tcg/optimize.c index 9876ac52a8..c239c3bd07 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -1023,7 +1023,7 @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(qemu_ld): { - TCGMemOpIdx oi = op->args[nb_oargs + nb_iargs]; + MemOpIdx oi = op->args[nb_oargs + nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 7e8dee2cc6..3e4ca2be88 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -2103,7 +2103,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc, /* Record the context of a call to the out of line helper code for the slow path for a load or store, so that we can later generate the correct helper code. */ -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, TCGReg datalo_reg, TCGReg datahi_reg, TCGReg addrlo_reg, TCGReg addrhi_reg, tcg_insn_unit *raddr, tcg_insn_unit *lptr) @@ -2122,7 +2122,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); TCGReg hi, lo, arg = TCG_REG_R3; @@ -2169,7 +2169,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; TCGReg hi, lo, arg = TCG_REG_R3; @@ -2233,7 +2233,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg datalo, datahi, addrlo, rbase; TCGReg addrhi __attribute__((unused)); - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc, s_bits; #ifdef CONFIG_SOFTMMU int mem_index; @@ -2308,7 +2308,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg datalo, datahi, addrlo, rbase; TCGReg addrhi __attribute__((unused)); - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc, s_bits; #ifdef CONFIG_SOFTMMU int mem_index; diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index da48f9a633..9b13a46fb4 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -850,7 +850,7 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) #include "../tcg-ldst.c.inc" /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * TCGMemOpIdx oi, uintptr_t ra) + * MemOpIdx oi, uintptr_t ra) */ static void * const qemu_ld_helpers[MO_SSIZE + 1] = { [MO_UB] = helper_ret_ldub_mmu, @@ -875,7 +875,7 @@ static void * const qemu_ld_helpers[MO_SSIZE + 1] = { }; /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, TCGMemOpIdx oi, + * uintxx_t val, MemOpIdx oi, * uintptr_t ra) */ static void * const qemu_st_helpers[MO_SIZE + 1] = { @@ -906,7 +906,7 @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) } static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, - TCGReg addrh, TCGMemOpIdx oi, + TCGReg addrh, MemOpIdx oi, tcg_insn_unit **label_ptr, bool is_load) { MemOp opc = get_memop(oi); @@ -959,7 +959,7 @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); } -static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, int is_ld, MemOpIdx oi, TCGType ext, TCGReg datalo, TCGReg datahi, TCGReg addrlo, TCGReg addrhi, @@ -980,7 +980,7 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - TCGMemOpIdx oi = l->oi; + MemOpIdx oi = l->oi; MemOp opc = get_memop(oi); TCGReg a0 = tcg_target_call_iarg_regs[0]; TCGReg a1 = tcg_target_call_iarg_regs[1]; @@ -1012,7 +1012,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - TCGMemOpIdx oi = l->oi; + MemOpIdx oi = l->oi; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; TCGReg a0 = tcg_target_call_iarg_regs[0]; @@ -1104,7 +1104,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_regl, addr_regh __attribute__((unused)); TCGReg data_regl, data_regh; - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[1]; @@ -1170,7 +1170,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) { TCGReg addr_regl, addr_regh __attribute__((unused)); TCGReg data_regl, data_regh; - TCGMemOpIdx oi; + MemOpIdx oi; MemOp opc; #if defined(CONFIG_SOFTMMU) tcg_insn_unit *label_ptr[1]; diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc index 67a2ba5ff3..fd0b3316d2 100644 --- a/tcg/s390/tcg-target.c.inc +++ b/tcg/s390/tcg-target.c.inc @@ -1547,7 +1547,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc, return addr_reg; } -static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, +static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi, TCGReg data, TCGReg addr, tcg_insn_unit *raddr, tcg_insn_unit *label_ptr) { @@ -1565,7 +1565,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { TCGReg addr_reg = lb->addrlo_reg; TCGReg data_reg = lb->datalo_reg; - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL, @@ -1590,7 +1590,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) { TCGReg addr_reg = lb->addrlo_reg; TCGReg data_reg = lb->datalo_reg; - TCGMemOpIdx oi = lb->oi; + MemOpIdx oi = lb->oi; MemOp opc = get_memop(oi); if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL, @@ -1644,7 +1644,7 @@ static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg, #endif /* CONFIG_SOFTMMU */ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - TCGMemOpIdx oi) + MemOpIdx oi) { MemOp opc = get_memop(oi); #ifdef CONFIG_SOFTMMU @@ -1671,7 +1671,7 @@ static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, } static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, - TCGMemOpIdx oi) + MemOpIdx oi) { MemOp opc = get_memop(oi); #ifdef CONFIG_SOFTMMU diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index 43248776a1..9dd32ef95e 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -1148,7 +1148,7 @@ static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = { }; static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, - TCGMemOpIdx oi, bool is_64) + MemOpIdx oi, bool is_64) { MemOp memop = get_memop(oi); #ifdef CONFIG_SOFTMMU @@ -1230,7 +1230,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, } static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, - TCGMemOpIdx oi) + MemOpIdx oi) { MemOp memop = get_memop(oi); #ifdef CONFIG_SOFTMMU diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc index c3ce88e69d..6c6848d034 100644 --- a/tcg/tcg-ldst.c.inc +++ b/tcg/tcg-ldst.c.inc @@ -22,7 +22,7 @@ typedef struct TCGLabelQemuLdst { bool is_ld; /* qemu_ld: true, qemu_st: false */ - TCGMemOpIdx oi; + MemOpIdx oi; TCGType type; /* result type of a load */ TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */ TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index e01f68f44d..e1490c372e 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -2797,7 +2797,7 @@ static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, MemOp memop, TCGArg idx) { - TCGMemOpIdx oi = make_memop_idx(memop, idx); + MemOpIdx oi = make_memop_idx(memop, idx); #if TARGET_LONG_BITS == 32 tcg_gen_op3i_i32(opc, val, addr, oi); #else @@ -2812,7 +2812,7 @@ static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, MemOp memop, TCGArg idx) { - TCGMemOpIdx oi = make_memop_idx(memop, idx); + MemOpIdx oi = make_memop_idx(memop, idx); #if TARGET_LONG_BITS == 32 if (TCG_TARGET_REG_BITS == 32) { tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); @@ -3132,7 +3132,7 @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, tcg_temp_free_i32(t1); } else { gen_atomic_cx_i32 gen; - TCGMemOpIdx oi; + MemOpIdx oi; gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; tcg_debug_assert(gen != NULL); @@ -3171,7 +3171,7 @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, } else if ((memop & MO_SIZE) == MO_64) { #ifdef CONFIG_ATOMIC64 gen_atomic_cx_i64 gen; - TCGMemOpIdx oi; + MemOpIdx oi; gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; tcg_debug_assert(gen != NULL); @@ -3227,7 +3227,7 @@ static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop, void * const table[]) { gen_atomic_op_i32 gen; - TCGMemOpIdx oi; + MemOpIdx oi; memop = tcg_canonicalize_memop(memop, 0, 0); @@ -3269,7 +3269,7 @@ static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, if ((memop & MO_SIZE) == MO_64) { #ifdef CONFIG_ATOMIC64 gen_atomic_op_i64 gen; - TCGMemOpIdx oi; + MemOpIdx oi; gen = table[memop & (MO_SIZE | MO_BSWAP)]; tcg_debug_assert(gen != NULL); diff --git a/tcg/tcg.c b/tcg/tcg.c index 4142d42d77..658be0c6b6 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1910,7 +1910,7 @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs) case INDEX_op_qemu_ld_i64: case INDEX_op_qemu_st_i64: { - TCGMemOpIdx oi = op->args[k++]; + MemOpIdx oi = op->args[k++]; MemOp op = get_memop(oi); unsigned ix = get_mmuidx(oi); diff --git a/tcg/tci.c b/tcg/tci.c index b672c7cae5..5c08dc0a9a 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -61,7 +61,7 @@ static uint64_t tci_uint64(uint32_t high, uint32_t low) * i = immediate (uint32_t) * I = immediate (tcg_target_ulong) * l = label or pointer - * m = immediate (TCGMemOpIdx) + * m = immediate (MemOpIdx) * n = immediate (call return length) * r = register * s = signed ldst offset @@ -105,7 +105,7 @@ static void tci_args_ri(uint32_t insn, TCGReg *r0, tcg_target_ulong *i1) } static void tci_args_rrm(uint32_t insn, TCGReg *r0, - TCGReg *r1, TCGMemOpIdx *m2) + TCGReg *r1, MemOpIdx *m2) { *r0 = extract32(insn, 8, 4); *r1 = extract32(insn, 12, 4); @@ -145,7 +145,7 @@ static void tci_args_rrrc(uint32_t insn, } static void tci_args_rrrm(uint32_t insn, - TCGReg *r0, TCGReg *r1, TCGReg *r2, TCGMemOpIdx *m3) + TCGReg *r0, TCGReg *r1, TCGReg *r2, MemOpIdx *m3) { *r0 = extract32(insn, 8, 4); *r1 = extract32(insn, 12, 4); @@ -289,7 +289,7 @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition) } static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, - TCGMemOpIdx oi, const void *tb_ptr) + MemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); uintptr_t ra = (uintptr_t)tb_ptr; @@ -374,7 +374,7 @@ static uint64_t tci_qemu_ld(CPUArchState *env, target_ulong taddr, } static void tci_qemu_st(CPUArchState *env, target_ulong taddr, uint64_t val, - TCGMemOpIdx oi, const void *tb_ptr) + MemOpIdx oi, const void *tb_ptr) { MemOp mop = get_memop(oi) & (MO_BSWAP | MO_SSIZE); uintptr_t ra = (uintptr_t)tb_ptr; @@ -482,7 +482,7 @@ uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env, uint32_t tmp32; uint64_t tmp64; uint64_t T1, T2; - TCGMemOpIdx oi; + MemOpIdx oi; int32_t ofs; void *ptr; @@ -1148,7 +1148,7 @@ int print_insn_tci(bfd_vma addr, disassemble_info *info) tcg_target_ulong i1; int32_t s2; TCGCond c; - TCGMemOpIdx oi; + MemOpIdx oi; uint8_t pos, len; void *ptr; From abe2e23eb703b7e2b0479b4672d087cc37b0e667 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 25 Jul 2021 11:47:35 -1000 Subject: [PATCH 07/28] tcg: Split out MemOpIdx to exec/memopidx.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move this code from tcg/tcg.h to its own header. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/exec/memopidx.h | 55 +++++++++++++++++++++++++++++++++++++++++ include/tcg/tcg.h | 39 +---------------------------- 2 files changed, 56 insertions(+), 38 deletions(-) create mode 100644 include/exec/memopidx.h diff --git a/include/exec/memopidx.h b/include/exec/memopidx.h new file mode 100644 index 0000000000..83bce97874 --- /dev/null +++ b/include/exec/memopidx.h @@ -0,0 +1,55 @@ +/* + * Combine the MemOp and mmu_idx parameters into a single value. + * + * Authors: + * Richard Henderson + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef EXEC_MEMOPIDX_H +#define EXEC_MEMOPIDX_H 1 + +#include "exec/memop.h" + +typedef uint32_t MemOpIdx; + +/** + * make_memop_idx + * @op: memory operation + * @idx: mmu index + * + * Encode these values into a single parameter. + */ +static inline MemOpIdx make_memop_idx(MemOp op, unsigned idx) +{ +#ifdef CONFIG_DEBUG_TCG + assert(idx <= 15); +#endif + return (op << 4) | idx; +} + +/** + * get_memop + * @oi: combined op/idx parameter + * + * Extract the memory operation from the combined value. + */ +static inline MemOp get_memop(MemOpIdx oi) +{ + return oi >> 4; +} + +/** + * get_mmuidx + * @oi: combined op/idx parameter + * + * Extract the mmu index from the combined value. + */ +static inline unsigned get_mmuidx(MemOpIdx oi) +{ + return oi & 15; +} + +#endif diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 1a0da58f92..ba13ab1151 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -27,6 +27,7 @@ #include "cpu.h" #include "exec/memop.h" +#include "exec/memopidx.h" #include "qemu/bitops.h" #include "qemu/plugin.h" #include "qemu/queue.h" @@ -1147,44 +1148,6 @@ static inline size_t tcg_current_code_size(TCGContext *s) return tcg_ptr_byte_diff(s->code_ptr, s->code_buf); } -/* Combine the MemOp and mmu_idx parameters into a single value. */ -typedef uint32_t MemOpIdx; - -/** - * make_memop_idx - * @op: memory operation - * @idx: mmu index - * - * Encode these values into a single parameter. - */ -static inline MemOpIdx make_memop_idx(MemOp op, unsigned idx) -{ - tcg_debug_assert(idx <= 15); - return (op << 4) | idx; -} - -/** - * get_memop - * @oi: combined op/idx parameter - * - * Extract the memory operation from the combined value. - */ -static inline MemOp get_memop(MemOpIdx oi) -{ - return oi >> 4; -} - -/** - * get_mmuidx - * @oi: combined op/idx parameter - * - * Extract the mmu index from the combined value. - */ -static inline unsigned get_mmuidx(MemOpIdx oi) -{ - return oi & 15; -} - /** * tcg_qemu_tb_exec: * @env: pointer to CPUArchState for the CPU From b0702c91c66a9a9d8831ecb3d08f511e7d167489 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 25 Jul 2021 12:44:12 -1000 Subject: [PATCH 08/28] trace/mem: Pass MemOpIdx to trace_mem_get_info We (will) often have the complete MemOpIdx handy, so use that. Reviewed-by: Peter Maydell Reviewed-by: Alistair Francis Signed-off-by: Richard Henderson --- accel/tcg/atomic_common.c.inc | 6 ++--- accel/tcg/cputlb.c | 12 ++++------ accel/tcg/user-exec.c | 42 +++++++++++++++++++++++------------ tcg/tcg-op.c | 8 +++---- trace/mem.h | 32 +++++++++----------------- 5 files changed, 49 insertions(+), 51 deletions(-) diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index ebaa793464..6019a957b9 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -17,7 +17,7 @@ static uint16_t atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, MemOpIdx oi) { CPUState *cpu = env_cpu(env); - uint16_t info = trace_mem_get_info(get_memop(oi), get_mmuidx(oi), false); + uint16_t info = trace_mem_get_info(oi, false); trace_guest_mem_before_exec(cpu, addr, info); trace_guest_mem_before_exec(cpu, addr, info | TRACE_MEM_ST); @@ -36,7 +36,7 @@ static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, static uint16_t atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(get_memop(oi), get_mmuidx(oi), false); + uint16_t info = trace_mem_get_info(oi, false); trace_guest_mem_before_exec(env_cpu(env), addr, info); @@ -52,7 +52,7 @@ static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, static uint16_t atomic_trace_st_pre(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(get_memop(oi), get_mmuidx(oi), true); + uint16_t info = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), addr, info); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index d72f65f42b..0aa6157ec4 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -2112,14 +2112,12 @@ static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr, int mmu_idx, uintptr_t retaddr, MemOp op, FullLoadHelper *full_load) { - uint16_t meminfo; - MemOpIdx oi; + MemOpIdx oi = make_memop_idx(op, mmu_idx); + uint16_t meminfo = trace_mem_get_info(oi, false); uint64_t ret; - meminfo = trace_mem_get_info(op, mmu_idx, false); trace_guest_mem_before_exec(env_cpu(env), addr, meminfo); - oi = make_memop_idx(op, mmu_idx); ret = full_load(env, addr, oi, retaddr); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, meminfo); @@ -2551,13 +2549,11 @@ static inline void QEMU_ALWAYS_INLINE cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val, int mmu_idx, uintptr_t retaddr, MemOp op) { - MemOpIdx oi; - uint16_t meminfo; + MemOpIdx oi = make_memop_idx(op, mmu_idx); + uint16_t meminfo = trace_mem_get_info(oi, true); - meminfo = trace_mem_get_info(op, mmu_idx, true); trace_guest_mem_before_exec(env_cpu(env), addr, meminfo); - oi = make_memop_idx(op, mmu_idx); store_helper(env, addr, val, oi, retaddr, op); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, meminfo); diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 1187362a4c..3ba7acf7f4 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -888,8 +888,9 @@ int cpu_signal_handler(int host_signum, void *pinfo, uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_UB, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - uint16_t meminfo = trace_mem_get_info(MO_UB, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldub_p(g2h(env_cpu(env), ptr)); @@ -904,8 +905,9 @@ int cpu_ldsb_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_BEUW, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - uint16_t meminfo = trace_mem_get_info(MO_BEUW, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = lduw_be_p(g2h(env_cpu(env), ptr)); @@ -920,8 +922,9 @@ int cpu_ldsw_be_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_BEUL, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - uint16_t meminfo = trace_mem_get_info(MO_BEUL, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldl_be_p(g2h(env_cpu(env), ptr)); @@ -931,8 +934,9 @@ uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr) uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_BEQ, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint64_t ret; - uint16_t meminfo = trace_mem_get_info(MO_BEQ, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldq_be_p(g2h(env_cpu(env), ptr)); @@ -942,8 +946,9 @@ uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_LEUW, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - uint16_t meminfo = trace_mem_get_info(MO_LEUW, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = lduw_le_p(g2h(env_cpu(env), ptr)); @@ -958,8 +963,9 @@ int cpu_ldsw_le_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_LEUL, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - uint16_t meminfo = trace_mem_get_info(MO_LEUL, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldl_le_p(g2h(env_cpu(env), ptr)); @@ -969,8 +975,9 @@ uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr) uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr ptr) { + MemOpIdx oi = make_memop_idx(MO_LEQ, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, false); uint64_t ret; - uint16_t meminfo = trace_mem_get_info(MO_LEQ, MMU_USER_IDX, false); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldq_le_p(g2h(env_cpu(env), ptr)); @@ -1065,7 +1072,8 @@ uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { - uint16_t meminfo = trace_mem_get_info(MO_UB, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_UB, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stb_p(g2h(env_cpu(env), ptr), val); @@ -1074,7 +1082,8 @@ void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { - uint16_t meminfo = trace_mem_get_info(MO_BEUW, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_BEUW, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stw_be_p(g2h(env_cpu(env), ptr), val); @@ -1083,7 +1092,8 @@ void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { - uint16_t meminfo = trace_mem_get_info(MO_BEUL, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_BEUL, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stl_be_p(g2h(env_cpu(env), ptr), val); @@ -1092,7 +1102,8 @@ void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val) { - uint16_t meminfo = trace_mem_get_info(MO_BEQ, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_BEQ, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stq_be_p(g2h(env_cpu(env), ptr), val); @@ -1101,7 +1112,8 @@ void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val) void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { - uint16_t meminfo = trace_mem_get_info(MO_LEUW, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_LEUW, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stw_le_p(g2h(env_cpu(env), ptr), val); @@ -1110,7 +1122,8 @@ void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { - uint16_t meminfo = trace_mem_get_info(MO_LEUL, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_LEUL, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stl_le_p(g2h(env_cpu(env), ptr), val); @@ -1119,7 +1132,8 @@ void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val) { - uint16_t meminfo = trace_mem_get_info(MO_LEQ, MMU_USER_IDX, true); + MemOpIdx oi = make_memop_idx(MO_LEQ, MMU_USER_IDX); + uint16_t meminfo = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stq_le_p(g2h(env_cpu(env), ptr), val); diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index e1490c372e..37b440af7f 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -2866,7 +2866,7 @@ static inline void plugin_gen_mem_callbacks(TCGv vaddr, uint16_t info) void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) { MemOp orig_memop; - uint16_t info = trace_mem_get_info(memop, idx, 0); + uint16_t info = trace_mem_get_info(make_memop_idx(memop, idx), 0); tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); memop = tcg_canonicalize_memop(memop, 0, 0); @@ -2904,7 +2904,7 @@ void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) { TCGv_i32 swap = NULL; - uint16_t info = trace_mem_get_info(memop, idx, 1); + uint16_t info = trace_mem_get_info(make_memop_idx(memop, idx), 1); tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 0, 1); @@ -2956,7 +2956,7 @@ void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); memop = tcg_canonicalize_memop(memop, 1, 0); - info = trace_mem_get_info(memop, idx, 0); + info = trace_mem_get_info(make_memop_idx(memop, idx), 0); trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); orig_memop = memop; @@ -3004,7 +3004,7 @@ void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 1, 1); - info = trace_mem_get_info(memop, idx, 1); + info = trace_mem_get_info(make_memop_idx(memop, idx), 1); trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { diff --git a/trace/mem.h b/trace/mem.h index 2f27e7bdf0..699566c661 100644 --- a/trace/mem.h +++ b/trace/mem.h @@ -10,7 +10,7 @@ #ifndef TRACE__MEM_H #define TRACE__MEM_H -#include "tcg/tcg.h" +#include "exec/memopidx.h" #define TRACE_MEM_SZ_SHIFT_MASK 0xf /* size shift mask */ #define TRACE_MEM_SE (1ULL << 4) /* sign extended (y/n) */ @@ -19,45 +19,33 @@ #define TRACE_MEM_MMU_SHIFT 8 /* mmu idx */ /** - * trace_mem_build_info: + * trace_mem_get_info: * * Return a value for the 'info' argument in guest memory access traces. */ -static inline uint16_t trace_mem_build_info(int size_shift, bool sign_extend, - MemOp endianness, bool store, - unsigned int mmu_idx) +static inline uint16_t trace_mem_get_info(MemOpIdx oi, bool store) { + MemOp op = get_memop(oi); + uint32_t size_shift = op & MO_SIZE; + bool sign_extend = op & MO_SIGN; + bool big_endian = (op & MO_BSWAP) == MO_BE; uint16_t res; res = size_shift & TRACE_MEM_SZ_SHIFT_MASK; if (sign_extend) { res |= TRACE_MEM_SE; } - if (endianness == MO_BE) { + if (big_endian) { res |= TRACE_MEM_BE; } if (store) { res |= TRACE_MEM_ST; } #ifdef CONFIG_SOFTMMU - res |= mmu_idx << TRACE_MEM_MMU_SHIFT; + res |= get_mmuidx(oi) << TRACE_MEM_MMU_SHIFT; #endif + return res; } - -/** - * trace_mem_get_info: - * - * Return a value for the 'info' argument in guest memory access traces. - */ -static inline uint16_t trace_mem_get_info(MemOp op, - unsigned int mmu_idx, - bool store) -{ - return trace_mem_build_info(op & MO_SIZE, !!(op & MO_SIGN), - op & MO_BSWAP, store, - mmu_idx); -} - #endif /* TRACE__MEM_H */ From c3e83e376cf028fade97072d86f33e4a92ddf9a2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Jul 2021 08:19:40 -1000 Subject: [PATCH 09/28] accel/tcg: Pass MemOpIdx to atomic_trace_*_post MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will shortly use the MemOpIdx directly, but in the meantime re-compute the trace meminfo. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/atomic_common.c.inc | 30 +++++++++++----------- accel/tcg/atomic_template.h | 48 +++++++++++++++++------------------ 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index 6019a957b9..db81eb5e66 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -13,55 +13,55 @@ * See the COPYING file in the top-level directory. */ -static uint16_t atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, - MemOpIdx oi) +static void atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, + MemOpIdx oi) { CPUState *cpu = env_cpu(env); uint16_t info = trace_mem_get_info(oi, false); trace_guest_mem_before_exec(cpu, addr, info); trace_guest_mem_before_exec(cpu, addr, info | TRACE_MEM_ST); - - return info; } static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, - uint16_t info) + MemOpIdx oi) { + uint16_t info = trace_mem_get_info(oi, false); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info); qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info | TRACE_MEM_ST); } #if HAVE_ATOMIC128 -static uint16_t atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, - MemOpIdx oi) +static void atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, + MemOpIdx oi) { uint16_t info = trace_mem_get_info(oi, false); trace_guest_mem_before_exec(env_cpu(env), addr, info); - - return info; } static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, - uint16_t info) + MemOpIdx oi) { + uint16_t info = trace_mem_get_info(oi, false); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info); } -static uint16_t atomic_trace_st_pre(CPUArchState *env, target_ulong addr, - MemOpIdx oi) +static void atomic_trace_st_pre(CPUArchState *env, target_ulong addr, + MemOpIdx oi) { uint16_t info = trace_mem_get_info(oi, true); trace_guest_mem_before_exec(env_cpu(env), addr, info); - - return info; } static void atomic_trace_st_post(CPUArchState *env, target_ulong addr, - uint16_t info) + MemOpIdx oi) { + uint16_t info = trace_mem_get_info(oi, false); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info); } #endif diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index 4230ff2957..c08d859a8a 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -77,15 +77,15 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); DATA_TYPE ret; - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); + atomic_trace_rmw_pre(env, addr, oi); #if DATA_SIZE == 16 ret = atomic16_cmpxchg(haddr, cmpv, newv); #else ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); #endif ATOMIC_MMU_CLEANUP; - atomic_trace_rmw_post(env, addr, info); + atomic_trace_rmw_post(env, addr, oi); return ret; } @@ -97,11 +97,11 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ, retaddr); DATA_TYPE val; - uint16_t info = atomic_trace_ld_pre(env, addr, oi); + atomic_trace_ld_pre(env, addr, oi); val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; - atomic_trace_ld_post(env, addr, info); + atomic_trace_ld_post(env, addr, oi); return val; } @@ -110,11 +110,11 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_WRITE, retaddr); - uint16_t info = atomic_trace_st_pre(env, addr, oi); + atomic_trace_st_pre(env, addr, oi); atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; - atomic_trace_st_post(env, addr, info); + atomic_trace_st_post(env, addr, oi); } #endif #else @@ -124,11 +124,11 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); DATA_TYPE ret; - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); + atomic_trace_rmw_pre(env, addr, oi); ret = qatomic_xchg__nocheck(haddr, val); ATOMIC_MMU_CLEANUP; - atomic_trace_rmw_post(env, addr, info); + atomic_trace_rmw_post(env, addr, oi); return ret; } @@ -139,10 +139,10 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ DATA_TYPE ret; \ - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ + atomic_trace_rmw_pre(env, addr, oi); \ ret = qatomic_##X(haddr, val); \ ATOMIC_MMU_CLEANUP; \ - atomic_trace_rmw_post(env, addr, info); \ + atomic_trace_rmw_post(env, addr, oi); \ return ret; \ } @@ -172,7 +172,7 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ XDATA_TYPE cmp, old, new, val = xval; \ - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ + atomic_trace_rmw_pre(env, addr, oi); \ smp_mb(); \ cmp = qatomic_read__nocheck(haddr); \ do { \ @@ -180,7 +180,7 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ } while (cmp != old); \ ATOMIC_MMU_CLEANUP; \ - atomic_trace_rmw_post(env, addr, info); \ + atomic_trace_rmw_post(env, addr, oi); \ return RET; \ } @@ -216,15 +216,15 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); DATA_TYPE ret; - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); + atomic_trace_rmw_pre(env, addr, oi); #if DATA_SIZE == 16 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); #else ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); #endif ATOMIC_MMU_CLEANUP; - atomic_trace_rmw_post(env, addr, info); + atomic_trace_rmw_post(env, addr, oi); return BSWAP(ret); } @@ -236,11 +236,11 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ, retaddr); DATA_TYPE val; - uint16_t info = atomic_trace_ld_pre(env, addr, oi); + atomic_trace_ld_pre(env, addr, oi); val = atomic16_read(haddr); ATOMIC_MMU_CLEANUP; - atomic_trace_ld_post(env, addr, info); + atomic_trace_ld_post(env, addr, oi); return BSWAP(val); } @@ -249,12 +249,12 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, { DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_WRITE, retaddr); - uint16_t info = atomic_trace_st_pre(env, addr, oi); + atomic_trace_st_pre(env, addr, oi); val = BSWAP(val); atomic16_set(haddr, val); ATOMIC_MMU_CLEANUP; - atomic_trace_st_post(env, addr, info); + atomic_trace_st_post(env, addr, oi); } #endif #else @@ -264,11 +264,11 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, PAGE_READ | PAGE_WRITE, retaddr); ABI_TYPE ret; - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); + atomic_trace_rmw_pre(env, addr, oi); ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); ATOMIC_MMU_CLEANUP; - atomic_trace_rmw_post(env, addr, info); + atomic_trace_rmw_post(env, addr, oi); return BSWAP(ret); } @@ -279,10 +279,10 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ DATA_TYPE ret; \ - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ + atomic_trace_rmw_pre(env, addr, oi); \ ret = qatomic_##X(haddr, BSWAP(val)); \ ATOMIC_MMU_CLEANUP; \ - atomic_trace_rmw_post(env, addr, info); \ + atomic_trace_rmw_post(env, addr, oi); \ return BSWAP(ret); \ } @@ -309,7 +309,7 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ PAGE_READ | PAGE_WRITE, retaddr); \ XDATA_TYPE ldo, ldn, old, new, val = xval; \ - uint16_t info = atomic_trace_rmw_pre(env, addr, oi); \ + atomic_trace_rmw_pre(env, addr, oi); \ smp_mb(); \ ldn = qatomic_read__nocheck(haddr); \ do { \ @@ -317,7 +317,7 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ } while (ldo != ldn); \ ATOMIC_MMU_CLEANUP; \ - atomic_trace_rmw_post(env, addr, info); \ + atomic_trace_rmw_post(env, addr, oi); \ return RET; \ } From 37aff08726b533c5df6a5a8685cca8a0de5e6619 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Jul 2021 11:48:30 -1000 Subject: [PATCH 10/28] plugins: Reorg arguments to qemu_plugin_vcpu_mem_cb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the MemOpIdx directly, rather than the rearrangement of the same bits currently done by the trace infrastructure. Pass in enum qemu_plugin_mem_rw so that we are able to treat read-modify-write operations as a single operation. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/atomic_common.c.inc | 13 +++---------- accel/tcg/cputlb.c | 4 ++-- accel/tcg/plugin-gen.c | 5 ++--- accel/tcg/user-exec.c | 28 ++++++++++++++-------------- include/qemu/plugin.h | 26 ++++++++++++++++++++++++-- plugins/api.c | 19 +++++++++++-------- plugins/core.c | 10 +++++----- tcg/tcg-op.c | 30 +++++++++++++++++++++--------- 8 files changed, 82 insertions(+), 53 deletions(-) diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index db81eb5e66..f3ab96e888 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -26,10 +26,7 @@ static void atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(oi, false); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info | TRACE_MEM_ST); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_RW); } #if HAVE_ATOMIC128 @@ -44,9 +41,7 @@ static void atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(oi, false); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); } static void atomic_trace_st_pre(CPUArchState *env, target_ulong addr, @@ -60,9 +55,7 @@ static void atomic_trace_st_pre(CPUArchState *env, target_ulong addr, static void atomic_trace_st_post(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(oi, false); - - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } #endif diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 0aa6157ec4..ee07457880 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -2120,7 +2120,7 @@ static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr, ret = full_load(env, addr, oi, retaddr); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -2556,7 +2556,7 @@ cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val, store_helper(env, addr, val, oi, retaddr, op); - qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stb_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val, diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 88e25c6df9..f5fd5f279c 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -45,7 +45,6 @@ #include "qemu/osdep.h" #include "tcg/tcg.h" #include "tcg/tcg-op.h" -#include "trace/mem.h" #include "exec/exec-all.h" #include "exec/plugin-gen.h" #include "exec/translator.h" @@ -211,9 +210,9 @@ static void gen_mem_wrapped(enum plugin_gen_cb type, const union mem_gen_fn *f, TCGv addr, uint32_t info, bool is_mem) { - int wr = !!(info & TRACE_MEM_ST); + enum qemu_plugin_mem_rw rw = get_plugin_meminfo_rw(info); - gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, type, wr); + gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, type, rw); if (is_mem) { f->mem_fn(addr, info); } else { diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 3ba7acf7f4..13e0b9e430 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -894,7 +894,7 @@ uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldub_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -911,7 +911,7 @@ uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = lduw_be_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -928,7 +928,7 @@ uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldl_be_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -940,7 +940,7 @@ uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldq_be_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -952,7 +952,7 @@ uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = lduw_le_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -969,7 +969,7 @@ uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldl_le_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -981,7 +981,7 @@ uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr ptr) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); ret = ldq_le_p(g2h(env_cpu(env), ptr)); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; } @@ -1077,7 +1077,7 @@ void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stb_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) @@ -1087,7 +1087,7 @@ void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stw_be_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) @@ -1097,7 +1097,7 @@ void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stl_be_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val) @@ -1107,7 +1107,7 @@ void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stq_be_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) @@ -1117,7 +1117,7 @@ void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stw_le_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) @@ -1127,7 +1127,7 @@ void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stl_le_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val) @@ -1137,7 +1137,7 @@ void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val) trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); stq_le_p(g2h(env_cpu(env), ptr), val); - qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, meminfo); + qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } void cpu_stb_data_ra(CPUArchState *env, abi_ptr ptr, diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h index 9a8438f683..b3172b147f 100644 --- a/include/qemu/plugin.h +++ b/include/qemu/plugin.h @@ -12,6 +12,7 @@ #include "qemu/error-report.h" #include "qemu/queue.h" #include "qemu/option.h" +#include "exec/memopidx.h" /* * Events that plugins can subscribe to. @@ -36,6 +37,25 @@ enum qemu_plugin_event { struct qemu_plugin_desc; typedef QTAILQ_HEAD(, qemu_plugin_desc) QemuPluginList; +/* + * Construct a qemu_plugin_meminfo_t. + */ +static inline qemu_plugin_meminfo_t +make_plugin_meminfo(MemOpIdx oi, enum qemu_plugin_mem_rw rw) +{ + return oi | (rw << 16); +} + +/* + * Extract the memory operation direction from a qemu_plugin_meminfo_t. + * Other portions may be extracted via get_memop and get_mmuidx. + */ +static inline enum qemu_plugin_mem_rw +get_plugin_meminfo_rw(qemu_plugin_meminfo_t i) +{ + return i >> 16; +} + #ifdef CONFIG_PLUGIN extern QemuOptsList qemu_plugin_opts; @@ -180,7 +200,8 @@ qemu_plugin_vcpu_syscall(CPUState *cpu, int64_t num, uint64_t a1, uint64_t a6, uint64_t a7, uint64_t a8); void qemu_plugin_vcpu_syscall_ret(CPUState *cpu, int64_t num, int64_t ret); -void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr, uint32_t meminfo); +void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr, + MemOpIdx oi, enum qemu_plugin_mem_rw rw); void qemu_plugin_flush_cb(void); @@ -244,7 +265,8 @@ void qemu_plugin_vcpu_syscall_ret(CPUState *cpu, int64_t num, int64_t ret) { } static inline void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr, - uint32_t meminfo) + MemOpIdx oi, + enum qemu_plugin_mem_rw rw) { } static inline void qemu_plugin_flush_cb(void) diff --git a/plugins/api.c b/plugins/api.c index acff9ce8ac..b143b09ce9 100644 --- a/plugins/api.c +++ b/plugins/api.c @@ -45,7 +45,6 @@ #include "qemu/plugin-memory.h" #include "hw/boards.h" #endif -#include "trace/mem.h" /* Uninstall and Reset handlers */ @@ -246,22 +245,25 @@ const char *qemu_plugin_insn_symbol(const struct qemu_plugin_insn *insn) unsigned qemu_plugin_mem_size_shift(qemu_plugin_meminfo_t info) { - return info & TRACE_MEM_SZ_SHIFT_MASK; + MemOp op = get_memop(info); + return op & MO_SIZE; } bool qemu_plugin_mem_is_sign_extended(qemu_plugin_meminfo_t info) { - return !!(info & TRACE_MEM_SE); + MemOp op = get_memop(info); + return op & MO_SIGN; } bool qemu_plugin_mem_is_big_endian(qemu_plugin_meminfo_t info) { - return !!(info & TRACE_MEM_BE); + MemOp op = get_memop(info); + return (op & MO_BSWAP) == MO_BE; } bool qemu_plugin_mem_is_store(qemu_plugin_meminfo_t info) { - return !!(info & TRACE_MEM_ST); + return get_plugin_meminfo_rw(info) & QEMU_PLUGIN_MEM_W; } /* @@ -277,11 +279,12 @@ struct qemu_plugin_hwaddr *qemu_plugin_get_hwaddr(qemu_plugin_meminfo_t info, { #ifdef CONFIG_SOFTMMU CPUState *cpu = current_cpu; - unsigned int mmu_idx = info >> TRACE_MEM_MMU_SHIFT; - hwaddr_info.is_store = info & TRACE_MEM_ST; + unsigned int mmu_idx = get_mmuidx(info); + enum qemu_plugin_mem_rw rw = get_plugin_meminfo_rw(info); + hwaddr_info.is_store = (rw & QEMU_PLUGIN_MEM_W) != 0; if (!tlb_plugin_lookup(cpu, vaddr, mmu_idx, - info & TRACE_MEM_ST, &hwaddr_info)) { + hwaddr_info.is_store, &hwaddr_info)) { error_report("invalid use of qemu_plugin_get_hwaddr"); return NULL; } diff --git a/plugins/core.c b/plugins/core.c index 6b2490f973..792262da08 100644 --- a/plugins/core.c +++ b/plugins/core.c @@ -27,7 +27,6 @@ #include "exec/helper-proto.h" #include "tcg/tcg.h" #include "tcg/tcg-op.h" -#include "trace/mem.h" /* mem_info macros */ #include "plugin.h" #include "qemu/compiler.h" @@ -446,7 +445,8 @@ void exec_inline_op(struct qemu_plugin_dyn_cb *cb) } } -void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr, uint32_t info) +void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr, + MemOpIdx oi, enum qemu_plugin_mem_rw rw) { GArray *arr = cpu->plugin_mem_cbs; size_t i; @@ -457,14 +457,14 @@ void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr, uint32_t info) for (i = 0; i < arr->len; i++) { struct qemu_plugin_dyn_cb *cb = &g_array_index(arr, struct qemu_plugin_dyn_cb, i); - int w = !!(info & TRACE_MEM_ST) + 1; - if (!(w & cb->rw)) { + if (!(rw & cb->rw)) { break; } switch (cb->type) { case PLUGIN_CB_REGULAR: - cb->f.vcpu_mem(cpu->cpu_index, info, vaddr, cb->userp); + cb->f.vcpu_mem(cpu->cpu_index, make_plugin_meminfo(oi, rw), + vaddr, cb->userp); break; case PLUGIN_CB_INLINE: exec_inline_op(cb); diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index 37b440af7f..af7bb851b5 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -2853,10 +2853,12 @@ static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr) return vaddr; } -static inline void plugin_gen_mem_callbacks(TCGv vaddr, uint16_t info) +static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi, + enum qemu_plugin_mem_rw rw) { #ifdef CONFIG_PLUGIN if (tcg_ctx->plugin_insn != NULL) { + qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); plugin_gen_empty_mem_callback(vaddr, info); tcg_temp_free(vaddr); } @@ -2866,10 +2868,13 @@ static inline void plugin_gen_mem_callbacks(TCGv vaddr, uint16_t info) void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) { MemOp orig_memop; - uint16_t info = trace_mem_get_info(make_memop_idx(memop, idx), 0); + MemOpIdx oi; + uint16_t info; tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); memop = tcg_canonicalize_memop(memop, 0, 0); + oi = make_memop_idx(memop, idx); + info = trace_mem_get_info(oi, 0); trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); orig_memop = memop; @@ -2883,7 +2888,7 @@ void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) addr = plugin_prep_mem_callbacks(addr); gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); - plugin_gen_mem_callbacks(addr, info); + plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); if ((orig_memop ^ memop) & MO_BSWAP) { switch (orig_memop & MO_SIZE) { @@ -2904,10 +2909,13 @@ void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) { TCGv_i32 swap = NULL; - uint16_t info = trace_mem_get_info(make_memop_idx(memop, idx), 1); + MemOpIdx oi; + uint16_t info; tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 0, 1); + oi = make_memop_idx(memop, idx); + info = trace_mem_get_info(oi, 1); trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { @@ -2932,7 +2940,7 @@ void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) } else { gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); } - plugin_gen_mem_callbacks(addr, info); + plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); if (swap) { tcg_temp_free_i32(swap); @@ -2942,6 +2950,7 @@ void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) { MemOp orig_memop; + MemOpIdx oi; uint16_t info; if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { @@ -2956,7 +2965,8 @@ void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); memop = tcg_canonicalize_memop(memop, 1, 0); - info = trace_mem_get_info(make_memop_idx(memop, idx), 0); + oi = make_memop_idx(memop, idx); + info = trace_mem_get_info(oi, 0); trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); orig_memop = memop; @@ -2970,7 +2980,7 @@ void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) addr = plugin_prep_mem_callbacks(addr); gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); - plugin_gen_mem_callbacks(addr, info); + plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); if ((orig_memop ^ memop) & MO_BSWAP) { int flags = (orig_memop & MO_SIGN @@ -2995,6 +3005,7 @@ void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) { TCGv_i64 swap = NULL; + MemOpIdx oi; uint16_t info; if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { @@ -3004,7 +3015,8 @@ void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 1, 1); - info = trace_mem_get_info(make_memop_idx(memop, idx), 1); + oi = make_memop_idx(memop, idx); + info = trace_mem_get_info(oi, 1); trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { @@ -3028,7 +3040,7 @@ void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) addr = plugin_prep_mem_callbacks(addr); gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); - plugin_gen_mem_callbacks(addr, info); + plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); if (swap) { tcg_temp_free_i64(swap); From 0583f775d2740f64739febf6496d9207552399f6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Jul 2021 13:21:38 -1000 Subject: [PATCH 11/28] trace: Split guest_mem_before MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no point in encoding load/store within a bit of the memory trace info operand. Represent atomic operations as a single read-modify-write tracepoint. Use MemOpIdx instead of inventing a form specifically for traces. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/atomic_common.c.inc | 12 +++------ accel/tcg/atomic_template.h | 1 - accel/tcg/cputlb.c | 7 ++--- accel/tcg/user-exec.c | 44 +++++++++++------------------- tcg/tcg-op.c | 17 +++--------- trace-events | 18 +++---------- trace/mem.h | 51 ----------------------------------- 7 files changed, 28 insertions(+), 122 deletions(-) delete mode 100644 trace/mem.h diff --git a/accel/tcg/atomic_common.c.inc b/accel/tcg/atomic_common.c.inc index f3ab96e888..1df1f243e9 100644 --- a/accel/tcg/atomic_common.c.inc +++ b/accel/tcg/atomic_common.c.inc @@ -17,10 +17,8 @@ static void atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, MemOpIdx oi) { CPUState *cpu = env_cpu(env); - uint16_t info = trace_mem_get_info(oi, false); - trace_guest_mem_before_exec(cpu, addr, info); - trace_guest_mem_before_exec(cpu, addr, info | TRACE_MEM_ST); + trace_guest_rmw_before_exec(cpu, addr, oi); } static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, @@ -33,9 +31,7 @@ static void atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, static void atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(oi, false); - - trace_guest_mem_before_exec(env_cpu(env), addr, info); + trace_guest_ld_before_exec(env_cpu(env), addr, oi); } static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, @@ -47,9 +43,7 @@ static void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, static void atomic_trace_st_pre(CPUArchState *env, target_ulong addr, MemOpIdx oi) { - uint16_t info = trace_mem_get_info(oi, true); - - trace_guest_mem_before_exec(env_cpu(env), addr, info); + trace_guest_st_before_exec(env_cpu(env), addr, oi); } static void atomic_trace_st_post(CPUArchState *env, target_ulong addr, diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index c08d859a8a..2d917b6b1f 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -19,7 +19,6 @@ */ #include "qemu/plugin.h" -#include "trace/mem.h" #if DATA_SIZE == 16 # define SUFFIX o diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index ee07457880..46140ccff3 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -34,7 +34,6 @@ #include "qemu/atomic128.h" #include "exec/translate-all.h" #include "trace/trace-root.h" -#include "trace/mem.h" #include "tb-hash.h" #include "internal.h" #ifdef CONFIG_PLUGIN @@ -2113,10 +2112,9 @@ static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr, MemOp op, FullLoadHelper *full_load) { MemOpIdx oi = make_memop_idx(op, mmu_idx); - uint16_t meminfo = trace_mem_get_info(oi, false); uint64_t ret; - trace_guest_mem_before_exec(env_cpu(env), addr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), addr, oi); ret = full_load(env, addr, oi, retaddr); @@ -2550,9 +2548,8 @@ cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val, int mmu_idx, uintptr_t retaddr, MemOp op) { MemOpIdx oi = make_memop_idx(op, mmu_idx); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), addr, meminfo); + trace_guest_st_before_exec(env_cpu(env), addr, oi); store_helper(env, addr, val, oi, retaddr, op); diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 13e0b9e430..65d3c9b286 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -27,7 +27,7 @@ #include "exec/helper-proto.h" #include "qemu/atomic128.h" #include "trace/trace-root.h" -#include "trace/mem.h" +#include "internal.h" #undef EAX #undef ECX @@ -889,10 +889,9 @@ int cpu_signal_handler(int host_signum, void *pinfo, uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_UB, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = ldub_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -906,10 +905,9 @@ int cpu_ldsb_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_BEUW, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = lduw_be_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -923,10 +921,9 @@ int cpu_ldsw_be_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_BEUL, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = ldl_be_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -935,10 +932,9 @@ uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr) uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_BEQ, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint64_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = ldq_be_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -947,10 +943,9 @@ uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_LEUW, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = lduw_le_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -964,10 +959,9 @@ int cpu_ldsw_le_data(CPUArchState *env, abi_ptr ptr) uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_LEUL, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint32_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = ldl_le_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -976,10 +970,9 @@ uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr) uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr ptr) { MemOpIdx oi = make_memop_idx(MO_LEQ, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, false); uint64_t ret; - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_ld_before_exec(env_cpu(env), ptr, oi); ret = ldq_le_p(g2h(env_cpu(env), ptr)); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R); return ret; @@ -1073,9 +1066,8 @@ uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr) void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { MemOpIdx oi = make_memop_idx(MO_UB, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stb_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } @@ -1083,9 +1075,8 @@ void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { MemOpIdx oi = make_memop_idx(MO_BEUW, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stw_be_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } @@ -1093,9 +1084,8 @@ void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { MemOpIdx oi = make_memop_idx(MO_BEUL, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stl_be_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } @@ -1103,9 +1093,8 @@ void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val) { MemOpIdx oi = make_memop_idx(MO_BEQ, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stq_be_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } @@ -1113,9 +1102,8 @@ void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val) void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { MemOpIdx oi = make_memop_idx(MO_LEUW, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stw_le_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } @@ -1123,9 +1111,8 @@ void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) { MemOpIdx oi = make_memop_idx(MO_LEUL, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stl_le_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } @@ -1133,9 +1120,8 @@ void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val) void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val) { MemOpIdx oi = make_memop_idx(MO_LEQ, MMU_USER_IDX); - uint16_t meminfo = trace_mem_get_info(oi, true); - trace_guest_mem_before_exec(env_cpu(env), ptr, meminfo); + trace_guest_st_before_exec(env_cpu(env), ptr, oi); stq_le_p(g2h(env_cpu(env), ptr), val); qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W); } diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index af7bb851b5..b1cfd36f29 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -28,7 +28,6 @@ #include "tcg/tcg-op.h" #include "tcg/tcg-mo.h" #include "trace-tcg.h" -#include "trace/mem.h" #include "exec/plugin-gen.h" /* Reduce the number of ifdefs below. This assumes that all uses of @@ -2869,13 +2868,11 @@ void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) { MemOp orig_memop; MemOpIdx oi; - uint16_t info; tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); memop = tcg_canonicalize_memop(memop, 0, 0); oi = make_memop_idx(memop, idx); - info = trace_mem_get_info(oi, 0); - trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); + trace_guest_ld_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi); orig_memop = memop; if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { @@ -2910,13 +2907,11 @@ void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) { TCGv_i32 swap = NULL; MemOpIdx oi; - uint16_t info; tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 0, 1); oi = make_memop_idx(memop, idx); - info = trace_mem_get_info(oi, 1); - trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); + trace_guest_st_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi); if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { swap = tcg_temp_new_i32(); @@ -2951,7 +2946,6 @@ void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) { MemOp orig_memop; MemOpIdx oi; - uint16_t info; if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); @@ -2966,8 +2960,7 @@ void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); memop = tcg_canonicalize_memop(memop, 1, 0); oi = make_memop_idx(memop, idx); - info = trace_mem_get_info(oi, 0); - trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); + trace_guest_ld_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi); orig_memop = memop; if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { @@ -3006,7 +2999,6 @@ void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) { TCGv_i64 swap = NULL; MemOpIdx oi; - uint16_t info; if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); @@ -3016,8 +3008,7 @@ void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 1, 1); oi = make_memop_idx(memop, idx); - info = trace_mem_get_info(oi, 1); - trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info); + trace_guest_st_before_tcg(tcg_ctx->cpu, cpu_env, addr, oi); if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { swap = tcg_temp_new_i64(); diff --git a/trace-events b/trace-events index c4cca29939..a637a61eba 100644 --- a/trace-events +++ b/trace-events @@ -120,26 +120,16 @@ vcpu guest_cpu_reset(void) # tcg/tcg-op.c # @vaddr: Access' virtual address. -# @info : Access' information (see below). +# @memopidx: Access' information (see below). # # Start virtual memory access (before any potential access violation). -# # Does not include memory accesses performed by devices. # -# Access information can be parsed as: -# -# struct mem_info { -# uint8_t size_shift : 4; /* interpreted as "1 << size_shift" bytes */ -# bool sign_extend: 1; /* sign-extended */ -# uint8_t endianness : 1; /* 0: little, 1: big */ -# bool store : 1; /* whether it is a store operation */ -# pad : 1; -# uint8_t mmuidx : 4; /* mmuidx (softmmu only) */ -# }; -# # Mode: user, softmmu # Targets: TCG(all) -vcpu tcg guest_mem_before(TCGv vaddr, uint16_t info) "info=%d", "vaddr=0x%016"PRIx64" info=%d" +vcpu tcg guest_ld_before(TCGv vaddr, uint32_t memopidx) "info=%d", "vaddr=0x%016"PRIx64" memopidx=0x%x" +vcpu tcg guest_st_before(TCGv vaddr, uint32_t memopidx) "info=%d", "vaddr=0x%016"PRIx64" memopidx=0x%x" +vcpu tcg guest_rmw_before(TCGv vaddr, uint32_t memopidx) "info=%d", "vaddr=0x%016"PRIx64" memopidx=0x%x" # include/user/syscall-trace.h diff --git a/trace/mem.h b/trace/mem.h deleted file mode 100644 index 699566c661..0000000000 --- a/trace/mem.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Helper functions for guest memory tracing - * - * Copyright (C) 2016 Lluís Vilanova - * - * This work is licensed under the terms of the GNU GPL, version 2 or later. - * See the COPYING file in the top-level directory. - */ - -#ifndef TRACE__MEM_H -#define TRACE__MEM_H - -#include "exec/memopidx.h" - -#define TRACE_MEM_SZ_SHIFT_MASK 0xf /* size shift mask */ -#define TRACE_MEM_SE (1ULL << 4) /* sign extended (y/n) */ -#define TRACE_MEM_BE (1ULL << 5) /* big endian (y/n) */ -#define TRACE_MEM_ST (1ULL << 6) /* store (y/n) */ -#define TRACE_MEM_MMU_SHIFT 8 /* mmu idx */ - -/** - * trace_mem_get_info: - * - * Return a value for the 'info' argument in guest memory access traces. - */ -static inline uint16_t trace_mem_get_info(MemOpIdx oi, bool store) -{ - MemOp op = get_memop(oi); - uint32_t size_shift = op & MO_SIZE; - bool sign_extend = op & MO_SIGN; - bool big_endian = (op & MO_BSWAP) == MO_BE; - uint16_t res; - - res = size_shift & TRACE_MEM_SZ_SHIFT_MASK; - if (sign_extend) { - res |= TRACE_MEM_SE; - } - if (big_endian) { - res |= TRACE_MEM_BE; - } - if (store) { - res |= TRACE_MEM_ST; - } -#ifdef CONFIG_SOFTMMU - res |= get_mmuidx(oi) << TRACE_MEM_MMU_SHIFT; -#endif - - return res; -} - -#endif /* TRACE__MEM_H */ From dc29f4746fc8c641fb1182495c2662381cc16e23 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 30 Jul 2021 15:20:45 -1000 Subject: [PATCH 12/28] hw/core/cpu: Re-sort the non-pointers to the end of CPUClass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Despite the comment, the members were not kept at the end. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/hw/core/cpu.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index bc864564ce..b7d5bc1200 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -131,7 +131,6 @@ struct CPUClass { ObjectClass *(*class_by_name)(const char *cpu_model); void (*parse_features)(const char *typename, char *str, Error **errp); - int reset_dump_flags; bool (*has_work)(CPUState *cpu); int (*memory_rw_debug)(CPUState *cpu, vaddr addr, uint8_t *buf, int len, bool is_write); @@ -149,9 +148,6 @@ struct CPUClass { void (*disas_set_info)(CPUState *cpu, disassemble_info *info); const char *deprecation_note; - /* Keep non-pointer data at the end to minimize holes. */ - int gdb_num_core_regs; - bool gdb_stop_before_watchpoint; struct AccelCPUClass *accel_cpu; /* when system emulation is not available, this pointer is NULL */ @@ -165,6 +161,13 @@ struct CPUClass { * class data that depends on the accelerator, see accel/accel-common.c. */ void (*init_accel_cpu)(struct AccelCPUClass *accel_cpu, CPUClass *cc); + + /* + * Keep non-pointer data at the end to minimize holes. + */ + int reset_dump_flags; + int gdb_num_core_regs; + bool gdb_stop_before_watchpoint; }; /* From 2552d60ebd6c519027d74bd452db7580b90e5cbd Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 19:25:58 -0700 Subject: [PATCH 13/28] tcg: Expand usadd/ussub with umin/umax For usadd, we only have to consider overflow. Since ~B + B == -1, the maximum value for A that saturates is ~B. For ussub, we only have to consider underflow. The minimum value that saturates to 0 from A - B is B. Signed-off-by: Richard Henderson --- tcg/tcg-op-vec.c | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index 15e026ae49..faf30f9cdd 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -119,6 +119,18 @@ bool tcg_can_emit_vecop_list(const TCGOpcode *list, continue; } break; + case INDEX_op_usadd_vec: + if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) || + tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { + continue; + } + break; + case INDEX_op_ussub_vec: + if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) || + tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { + continue; + } + break; case INDEX_op_cmpsel_vec: case INDEX_op_smin_vec: case INDEX_op_smax_vec: @@ -603,7 +615,18 @@ void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) { - do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec); + if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) { + const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); + TCGv_vec t = tcg_temp_new_vec_matching(r); + + /* usadd(a, b) = min(a, ~b) + b */ + tcg_gen_not_vec(vece, t, b); + tcg_gen_umin_vec(vece, t, t, a); + tcg_gen_add_vec(vece, r, t, b); + + tcg_temp_free_vec(t); + tcg_swap_vecop_list(hold_list); + } } void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) @@ -613,7 +636,17 @@ void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) { - do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec); + if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) { + const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); + TCGv_vec t = tcg_temp_new_vec_matching(r); + + /* ussub(a, b) = max(a, b) - b */ + tcg_gen_umax_vec(vece, t, a, b); + tcg_gen_sub_vec(vece, r, t, b); + + tcg_temp_free_vec(t); + tcg_swap_vecop_list(hold_list); + } } static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, From 3704993f545efbe7b5a9ede83525f0d9c07cb31f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 15:15:31 -0700 Subject: [PATCH 14/28] tcg/s390x: Rename from tcg/s390 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This emphasizes that we don't support s390, only 64-bit s390x hosts. Reviewed-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- meson.build | 2 -- tcg/{s390 => s390x}/tcg-target-con-set.h | 0 tcg/{s390 => s390x}/tcg-target-con-str.h | 0 tcg/{s390 => s390x}/tcg-target.c.inc | 0 tcg/{s390 => s390x}/tcg-target.h | 0 5 files changed, 2 deletions(-) rename tcg/{s390 => s390x}/tcg-target-con-set.h (100%) rename tcg/{s390 => s390x}/tcg-target-con-str.h (100%) rename tcg/{s390 => s390x}/tcg-target.c.inc (100%) rename tcg/{s390 => s390x}/tcg-target.h (100%) diff --git a/meson.build b/meson.build index 7b596fdcd9..99a0a3e689 100644 --- a/meson.build +++ b/meson.build @@ -268,8 +268,6 @@ if not get_option('tcg').disabled() tcg_arch = 'tci' elif config_host['ARCH'] == 'sparc64' tcg_arch = 'sparc' - elif config_host['ARCH'] == 's390x' - tcg_arch = 's390' elif config_host['ARCH'] in ['x86_64', 'x32'] tcg_arch = 'i386' elif config_host['ARCH'] == 'ppc64' diff --git a/tcg/s390/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h similarity index 100% rename from tcg/s390/tcg-target-con-set.h rename to tcg/s390x/tcg-target-con-set.h diff --git a/tcg/s390/tcg-target-con-str.h b/tcg/s390x/tcg-target-con-str.h similarity index 100% rename from tcg/s390/tcg-target-con-str.h rename to tcg/s390x/tcg-target-con-str.h diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc similarity index 100% rename from tcg/s390/tcg-target.c.inc rename to tcg/s390x/tcg-target.c.inc diff --git a/tcg/s390/tcg-target.h b/tcg/s390x/tcg-target.h similarity index 100% rename from tcg/s390/tcg-target.h rename to tcg/s390x/tcg-target.h From 748b7f3ef78654f0ed11c85ed0b34a02f97ea856 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 27 Apr 2019 16:51:09 -0700 Subject: [PATCH 15/28] tcg/s390x: Change FACILITY representation We will shortly need to be able to check facilities beyond the first 64. Instead of explicitly masking against s390_facilities, create a HAVE_FACILITY macro that indexes an array. Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- v2: Change name to HAVE_FACILITY (david) --- tcg/s390x/tcg-target.c.inc | 74 +++++++++++++++++++------------------- tcg/s390x/tcg-target.h | 29 ++++++++------- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index fd0b3316d2..a224244b52 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -66,7 +66,7 @@ We don't need this when we have pc-relative loads with the general instructions extension facility. */ #define TCG_REG_TB TCG_REG_R12 -#define USE_REG_TB (!(s390_facilities & FACILITY_GEN_INST_EXT)) +#define USE_REG_TB (!HAVE_FACILITY(GEN_INST_EXT)) #ifndef CONFIG_SOFTMMU #define TCG_GUEST_BASE_REG TCG_REG_R13 @@ -377,7 +377,7 @@ static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { #endif static const tcg_insn_unit *tb_ret_addr; -uint64_t s390_facilities; +uint64_t s390_facilities[1]; static bool patch_reloc(tcg_insn_unit *src_rw, int type, intptr_t value, intptr_t addend) @@ -577,7 +577,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } /* Try all 48-bit insns that can load it in one go. */ - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { if (sval == (int32_t)sval) { tcg_out_insn(s, RIL, LGFI, ret, sval); return; @@ -620,7 +620,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, } /* Otherwise, stuff it in the constant pool. */ - if (s390_facilities & FACILITY_GEN_INST_EXT) { + if (HAVE_FACILITY(GEN_INST_EXT)) { tcg_out_insn(s, RIL, LGRL, ret, 0); new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2); } else if (USE_REG_TB && !in_prologue) { @@ -706,7 +706,7 @@ static void tcg_out_ld_abs(TCGContext *s, TCGType type, { intptr_t addr = (intptr_t)abs; - if ((s390_facilities & FACILITY_GEN_INST_EXT) && !(addr & 1)) { + if (HAVE_FACILITY(GEN_INST_EXT) && !(addr & 1)) { ptrdiff_t disp = tcg_pcrel_diff(s, abs) >> 1; if (disp == (int32_t)disp) { if (type == TCG_TYPE_I32) { @@ -740,7 +740,7 @@ static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src, static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) { - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { tcg_out_insn(s, RRE, LGBR, dest, src); return; } @@ -760,7 +760,7 @@ static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) { - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { tcg_out_insn(s, RRE, LLGCR, dest, src); return; } @@ -780,7 +780,7 @@ static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) { - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { tcg_out_insn(s, RRE, LGHR, dest, src); return; } @@ -800,7 +800,7 @@ static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) { - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { tcg_out_insn(s, RRE, LLGHR, dest, src); return; } @@ -888,7 +888,7 @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) tgen_ext32u(s, dest, dest); return; } - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { if ((val & valid) == 0xff) { tgen_ext8u(s, TCG_TYPE_I64, dest, dest); return; @@ -909,7 +909,7 @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) } /* Try all 48-bit insns that can perform it in one go. */ - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { for (i = 0; i < 2; i++) { tcg_target_ulong mask = ~(0xffffffffull << i*32); if (((val | ~valid) & mask) == mask) { @@ -918,7 +918,7 @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) } } } - if ((s390_facilities & FACILITY_GEN_INST_EXT) && risbg_mask(val)) { + if (HAVE_FACILITY(GEN_INST_EXT) && risbg_mask(val)) { tgen_andi_risbg(s, dest, dest, val); return; } @@ -967,7 +967,7 @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) } /* Try all 48-bit insns that can perform it in one go. */ - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { for (i = 0; i < 2; i++) { tcg_target_ulong mask = (0xffffffffull << i*32); if ((val & mask) != 0 && (val & ~mask) == 0) { @@ -992,7 +992,7 @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) /* Perform the OR via sequential modifications to the high and low parts. Do this via recursion to handle 16-bit vs 32-bit masks in each half. */ - tcg_debug_assert(s390_facilities & FACILITY_EXT_IMM); + tcg_debug_assert(HAVE_FACILITY(EXT_IMM)); tgen_ori(s, type, dest, val & 0x00000000ffffffffull); tgen_ori(s, type, dest, val & 0xffffffff00000000ull); } @@ -1001,7 +1001,7 @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) { /* Try all 48-bit insns that can perform it in one go. */ - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { if ((val & 0xffffffff00000000ull) == 0) { tcg_out_insn(s, RIL, XILF, dest, val); return; @@ -1025,7 +1025,7 @@ static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) tcg_tbrel_diff(s, NULL)); } else { /* Perform the xor by parts. */ - tcg_debug_assert(s390_facilities & FACILITY_EXT_IMM); + tcg_debug_assert(HAVE_FACILITY(EXT_IMM)); if (val & 0xffffffff) { tcg_out_insn(s, RIL, XILF, dest, val); } @@ -1059,7 +1059,7 @@ static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1, goto exit; } - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { if (type == TCG_TYPE_I32) { op = (is_unsigned ? RIL_CLFI : RIL_CFI); tcg_out_insn_RIL(s, op, r1, c2); @@ -1122,7 +1122,7 @@ static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, bool have_loc; /* With LOC2, we can always emit the minimum 3 insns. */ - if (s390_facilities & FACILITY_LOAD_ON_COND2) { + if (HAVE_FACILITY(LOAD_ON_COND2)) { /* Emit: d = 0, d = (cc ? 1 : d). */ cc = tgen_cmp(s, type, cond, c1, c2, c2const, false); tcg_out_movi(s, TCG_TYPE_I64, dest, 0); @@ -1130,7 +1130,7 @@ static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, return; } - have_loc = (s390_facilities & FACILITY_LOAD_ON_COND) != 0; + have_loc = HAVE_FACILITY(LOAD_ON_COND); /* For HAVE_LOC, only the paths through GTU/GT/LEU/LE are smaller. */ restart: @@ -1216,7 +1216,7 @@ static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest, TCGArg v3, int v3const) { int cc; - if (s390_facilities & FACILITY_LOAD_ON_COND) { + if (HAVE_FACILITY(LOAD_ON_COND)) { cc = tgen_cmp(s, type, c, c1, c2, c2const, false); if (v3const) { tcg_out_insn(s, RIE, LOCGHI, dest, v3, cc); @@ -1249,7 +1249,7 @@ static void tgen_clz(TCGContext *s, TCGReg dest, TCGReg a1, } else { tcg_out_mov(s, TCG_TYPE_I64, dest, a2); } - if (s390_facilities & FACILITY_LOAD_ON_COND) { + if (HAVE_FACILITY(LOAD_ON_COND)) { /* Emit: if (one bit found) dest = r0. */ tcg_out_insn(s, RRF, LOCGR, dest, TCG_REG_R0, 2); } else { @@ -1325,7 +1325,7 @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, { int cc; - if (s390_facilities & FACILITY_GEN_INST_EXT) { + if (HAVE_FACILITY(GEN_INST_EXT)) { bool is_unsigned = is_unsigned_cond(c); bool in_range; S390Opcode opc; @@ -1519,7 +1519,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc, cross pages using the address of the last byte of the access. */ a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask); tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask; - if ((s390_facilities & FACILITY_GEN_INST_EXT) && a_off == 0) { + if (HAVE_FACILITY(GEN_INST_EXT) && a_off == 0) { tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask); } else { tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off); @@ -1810,7 +1810,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_insn(s, RI, AHI, a0, a2); break; } - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { tcg_out_insn(s, RIL, AFI, a0, a2); break; } @@ -2056,7 +2056,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_insn(s, RI, AGHI, a0, a2); break; } - if (s390_facilities & FACILITY_EXT_IMM) { + if (HAVE_FACILITY(EXT_IMM)) { if (a2 == (int32_t)a2) { tcg_out_insn(s, RIL, AGFI, a0, a2); break; @@ -2281,8 +2281,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, /* The host memory model is quite strong, we simply need to serialize the instruction stream. */ if (args[0] & TCG_MO_ST_LD) { - tcg_out_insn(s, RR, BCR, - s390_facilities & FACILITY_FAST_BCR_SER ? 14 : 15, 0); + tcg_out_insn(s, RR, BCR, HAVE_FACILITY(FAST_BCR_SER) ? 14 : 15, 0); } break; @@ -2345,7 +2344,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_or_i64: case INDEX_op_xor_i32: case INDEX_op_xor_i64: - return (s390_facilities & FACILITY_DISTINCT_OPS + return (HAVE_FACILITY(DISTINCT_OPS) ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ri)); @@ -2353,19 +2352,19 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) /* If we have the general-instruction-extensions, then we have MULTIPLY SINGLE IMMEDIATE with a signed 32-bit, otherwise we have only MULTIPLY HALFWORD IMMEDIATE, with a signed 16-bit. */ - return (s390_facilities & FACILITY_GEN_INST_EXT + return (HAVE_FACILITY(GEN_INST_EXT) ? C_O1_I2(r, 0, ri) : C_O1_I2(r, 0, rI)); case INDEX_op_mul_i64: - return (s390_facilities & FACILITY_GEN_INST_EXT + return (HAVE_FACILITY(GEN_INST_EXT) ? C_O1_I2(r, 0, rJ) : C_O1_I2(r, 0, rI)); case INDEX_op_shl_i32: case INDEX_op_shr_i32: case INDEX_op_sar_i32: - return (s390_facilities & FACILITY_DISTINCT_OPS + return (HAVE_FACILITY(DISTINCT_OPS) ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ri)); @@ -2409,7 +2408,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_movcond_i32: case INDEX_op_movcond_i64: - return (s390_facilities & FACILITY_LOAD_ON_COND2 + return (HAVE_FACILITY(LOAD_ON_COND2) ? C_O1_I4(r, r, ri, rI, 0) : C_O1_I4(r, r, ri, r, 0)); @@ -2424,13 +2423,13 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_add2_i32: case INDEX_op_sub2_i32: - return (s390_facilities & FACILITY_EXT_IMM + return (HAVE_FACILITY(EXT_IMM) ? C_O2_I4(r, r, 0, 1, ri, r) : C_O2_I4(r, r, 0, 1, r, r)); case INDEX_op_add2_i64: case INDEX_op_sub2_i64: - return (s390_facilities & FACILITY_EXT_IMM + return (HAVE_FACILITY(EXT_IMM) ? C_O2_I4(r, r, 0, 1, rA, r) : C_O2_I4(r, r, 0, 1, r, r)); @@ -2446,13 +2445,12 @@ static void query_s390_facilities(void) /* Is STORE FACILITY LIST EXTENDED available? Honestly, I believe this is present on all 64-bit systems, but let's check for it anyway. */ if (hwcap & HWCAP_S390_STFLE) { - register int r0 __asm__("0"); - register void *r1 __asm__("1"); + register int r0 __asm__("0") = ARRAY_SIZE(s390_facilities) - 1; + register void *r1 __asm__("1") = s390_facilities; /* stfle 0(%r1) */ - r1 = &s390_facilities; asm volatile(".word 0xb2b0,0x1000" - : "=r"(r0) : "0"(0), "r"(r1) : "memory", "cc"); + : "=r"(r0) : "r"(r0), "r"(r1) : "memory", "cc"); } } diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 2e4ede2ea2..18d0d330e6 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -55,16 +55,19 @@ typedef enum TCGReg { /* A list of relevant facilities used by this translator. Some of these are required for proper operation, and these are checked at startup. */ -#define FACILITY_ZARCH_ACTIVE (1ULL << (63 - 2)) -#define FACILITY_LONG_DISP (1ULL << (63 - 18)) -#define FACILITY_EXT_IMM (1ULL << (63 - 21)) -#define FACILITY_GEN_INST_EXT (1ULL << (63 - 34)) -#define FACILITY_LOAD_ON_COND (1ULL << (63 - 45)) +#define FACILITY_ZARCH_ACTIVE 2 +#define FACILITY_LONG_DISP 18 +#define FACILITY_EXT_IMM 21 +#define FACILITY_GEN_INST_EXT 34 +#define FACILITY_LOAD_ON_COND 45 #define FACILITY_FAST_BCR_SER FACILITY_LOAD_ON_COND #define FACILITY_DISTINCT_OPS FACILITY_LOAD_ON_COND -#define FACILITY_LOAD_ON_COND2 (1ULL << (63 - 53)) +#define FACILITY_LOAD_ON_COND2 53 -extern uint64_t s390_facilities; +extern uint64_t s390_facilities[1]; + +#define HAVE_FACILITY(X) \ + ((s390_facilities[FACILITY_##X / 64] >> (63 - FACILITY_##X % 64)) & 1) /* optional instructions */ #define TCG_TARGET_HAS_div2_i32 1 @@ -85,8 +88,8 @@ extern uint64_t s390_facilities; #define TCG_TARGET_HAS_clz_i32 0 #define TCG_TARGET_HAS_ctz_i32 0 #define TCG_TARGET_HAS_ctpop_i32 0 -#define TCG_TARGET_HAS_deposit_i32 (s390_facilities & FACILITY_GEN_INST_EXT) -#define TCG_TARGET_HAS_extract_i32 (s390_facilities & FACILITY_GEN_INST_EXT) +#define TCG_TARGET_HAS_deposit_i32 HAVE_FACILITY(GEN_INST_EXT) +#define TCG_TARGET_HAS_extract_i32 HAVE_FACILITY(GEN_INST_EXT) #define TCG_TARGET_HAS_sextract_i32 0 #define TCG_TARGET_HAS_extract2_i32 0 #define TCG_TARGET_HAS_movcond_i32 1 @@ -98,7 +101,7 @@ extern uint64_t s390_facilities; #define TCG_TARGET_HAS_mulsh_i32 0 #define TCG_TARGET_HAS_extrl_i64_i32 0 #define TCG_TARGET_HAS_extrh_i64_i32 0 -#define TCG_TARGET_HAS_direct_jump (s390_facilities & FACILITY_GEN_INST_EXT) +#define TCG_TARGET_HAS_direct_jump HAVE_FACILITY(GEN_INST_EXT) #define TCG_TARGET_HAS_qemu_st8_i32 0 #define TCG_TARGET_HAS_div2_i64 1 @@ -119,11 +122,11 @@ extern uint64_t s390_facilities; #define TCG_TARGET_HAS_eqv_i64 0 #define TCG_TARGET_HAS_nand_i64 0 #define TCG_TARGET_HAS_nor_i64 0 -#define TCG_TARGET_HAS_clz_i64 (s390_facilities & FACILITY_EXT_IMM) +#define TCG_TARGET_HAS_clz_i64 HAVE_FACILITY(EXT_IMM) #define TCG_TARGET_HAS_ctz_i64 0 #define TCG_TARGET_HAS_ctpop_i64 0 -#define TCG_TARGET_HAS_deposit_i64 (s390_facilities & FACILITY_GEN_INST_EXT) -#define TCG_TARGET_HAS_extract_i64 (s390_facilities & FACILITY_GEN_INST_EXT) +#define TCG_TARGET_HAS_deposit_i64 HAVE_FACILITY(GEN_INST_EXT) +#define TCG_TARGET_HAS_extract_i64 HAVE_FACILITY(GEN_INST_EXT) #define TCG_TARGET_HAS_sextract_i64 0 #define TCG_TARGET_HAS_extract2_i64 0 #define TCG_TARGET_HAS_movcond_i64 1 From eee6251b48f4d65e10feaff3f015241585d89f49 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 27 Apr 2019 17:09:38 -0700 Subject: [PATCH 16/28] tcg/s390x: Merge TCG_AREG0 and TCG_REG_CALL_STACK into TCGReg They are rightly values in the same enumeration. Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.h | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 18d0d330e6..0174357f1b 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -32,22 +32,13 @@ #define MAX_CODE_GEN_BUFFER_SIZE (3 * GiB) typedef enum TCGReg { - TCG_REG_R0 = 0, - TCG_REG_R1, - TCG_REG_R2, - TCG_REG_R3, - TCG_REG_R4, - TCG_REG_R5, - TCG_REG_R6, - TCG_REG_R7, - TCG_REG_R8, - TCG_REG_R9, - TCG_REG_R10, - TCG_REG_R11, - TCG_REG_R12, - TCG_REG_R13, - TCG_REG_R14, - TCG_REG_R15 + TCG_REG_R0, TCG_REG_R1, TCG_REG_R2, TCG_REG_R3, + TCG_REG_R4, TCG_REG_R5, TCG_REG_R6, TCG_REG_R7, + TCG_REG_R8, TCG_REG_R9, TCG_REG_R10, TCG_REG_R11, + TCG_REG_R12, TCG_REG_R13, TCG_REG_R14, TCG_REG_R15, + + TCG_AREG0 = TCG_REG_R10, + TCG_REG_CALL_STACK = TCG_REG_R15 } TCGReg; #define TCG_TARGET_NB_REGS 16 @@ -138,7 +129,6 @@ extern uint64_t s390_facilities[1]; #define TCG_TARGET_HAS_mulsh_i64 0 /* used for function call generation */ -#define TCG_REG_CALL_STACK TCG_REG_R15 #define TCG_TARGET_STACK_ALIGN 8 #define TCG_TARGET_CALL_STACK_OFFSET 160 @@ -147,10 +137,6 @@ extern uint64_t s390_facilities[1]; #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) -enum { - TCG_AREG0 = TCG_REG_R10, -}; - static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, uintptr_t jmp_rw, uintptr_t addr) { From 34ef7676090e35ffb7f7d8b8d92a843a6ee94931 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 16:33:14 -0700 Subject: [PATCH 17/28] tcg/s390x: Add host vector framework Add registers and function stubs. The functionality is disabled via squashing s390_facilities[2] to 0. We must still include results for the mandatory opcodes in tcg_target_op_def, as all opcodes are checked during tcg init. Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target-con-set.h | 4 + tcg/s390x/tcg-target-con-str.h | 1 + tcg/s390x/tcg-target.c.inc | 137 ++++++++++++++++++++++++++++++++- tcg/s390x/tcg-target.h | 35 ++++++++- tcg/s390x/tcg-target.opc.h | 12 +++ 5 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 tcg/s390x/tcg-target.opc.h diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index 31985e4903..ce9432cfe3 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -13,13 +13,17 @@ C_O0_I1(r) C_O0_I2(L, L) C_O0_I2(r, r) C_O0_I2(r, ri) +C_O0_I2(v, r) C_O1_I1(r, L) C_O1_I1(r, r) +C_O1_I1(v, r) +C_O1_I1(v, vr) C_O1_I2(r, 0, ri) C_O1_I2(r, 0, rI) C_O1_I2(r, 0, rJ) C_O1_I2(r, r, ri) C_O1_I2(r, rZ, r) +C_O1_I2(v, v, v) C_O1_I4(r, r, ri, r, 0) C_O1_I4(r, r, ri, rI, 0) C_O2_I2(b, a, 0, r) diff --git a/tcg/s390x/tcg-target-con-str.h b/tcg/s390x/tcg-target-con-str.h index 892d8f8c06..8bb0358ae5 100644 --- a/tcg/s390x/tcg-target-con-str.h +++ b/tcg/s390x/tcg-target-con-str.h @@ -10,6 +10,7 @@ */ REGS('r', ALL_GENERAL_REGS) REGS('L', ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) +REGS('v', ALL_VECTOR_REGS) /* * A (single) even/odd pair for division. * TODO: Add something to the register allocator to allow diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index a224244b52..8bee6dd26e 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -43,6 +43,8 @@ #define TCG_CT_CONST_ZERO 0x800 #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16) +#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) + /* * For softmmu, we need to avoid conflicts with the first 3 * argument registers to perform the tlb lookup, and to call @@ -268,8 +270,13 @@ typedef enum S390Opcode { #ifdef CONFIG_DEBUG_TCG static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { - "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", - "%r8", "%r9", "%r10" "%r11" "%r12" "%r13" "%r14" "%r15" + "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", + "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + "%v0", "%v1", "%v2", "%v3", "%v4", "%v5", "%v6", "%v7", + "%v8", "%v9", "%v10", "%v11", "%v12", "%v13", "%v14", "%v15", + "%v16", "%v17", "%v18", "%v19", "%v20", "%v21", "%v22", "%v23", + "%v24", "%v25", "%v26", "%v27", "%v28", "%v29", "%v30", "%v31", }; #endif @@ -295,6 +302,32 @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_R4, TCG_REG_R3, TCG_REG_R2, + + /* V8-V15 are call saved, and omitted. */ + TCG_REG_V0, + TCG_REG_V1, + TCG_REG_V2, + TCG_REG_V3, + TCG_REG_V4, + TCG_REG_V5, + TCG_REG_V6, + TCG_REG_V7, + TCG_REG_V16, + TCG_REG_V17, + TCG_REG_V18, + TCG_REG_V19, + TCG_REG_V20, + TCG_REG_V21, + TCG_REG_V22, + TCG_REG_V23, + TCG_REG_V24, + TCG_REG_V25, + TCG_REG_V26, + TCG_REG_V27, + TCG_REG_V28, + TCG_REG_V29, + TCG_REG_V30, + TCG_REG_V31, }; static const int tcg_target_call_iarg_regs[] = { @@ -377,7 +410,7 @@ static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { #endif static const tcg_insn_unit *tb_ret_addr; -uint64_t s390_facilities[1]; +uint64_t s390_facilities[3]; static bool patch_reloc(tcg_insn_unit *src_rw, int type, intptr_t value, intptr_t addend) @@ -2293,6 +2326,42 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, } } +static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg dst, TCGReg src) +{ + g_assert_not_reached(); +} + +static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg dst, TCGReg base, intptr_t offset) +{ + g_assert_not_reached(); +} + +static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg dst, int64_t val) +{ + g_assert_not_reached(); +} + +static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, + unsigned vecl, unsigned vece, + const TCGArg *args, const int *const_args) +{ + g_assert_not_reached(); +} + +int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) +{ + return 0; +} + +void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, + TCGArg a0, ...) +{ + g_assert_not_reached(); +} + static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) { switch (op) { @@ -2433,11 +2502,34 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) ? C_O2_I4(r, r, 0, 1, rA, r) : C_O2_I4(r, r, 0, 1, r, r)); + case INDEX_op_st_vec: + return C_O0_I2(v, r); + case INDEX_op_ld_vec: + case INDEX_op_dupm_vec: + return C_O1_I1(v, r); + case INDEX_op_dup_vec: + return C_O1_I1(v, vr); + case INDEX_op_add_vec: + case INDEX_op_sub_vec: + case INDEX_op_and_vec: + case INDEX_op_or_vec: + case INDEX_op_xor_vec: + case INDEX_op_cmp_vec: + return C_O1_I2(v, v, v); + default: g_assert_not_reached(); } } +/* + * Mainline glibc added HWCAP_S390_VX before it was kernel abi. + * Some distros have fixed this up locally, others have not. + */ +#ifndef HWCAP_S390_VXRS +#define HWCAP_S390_VXRS 2048 +#endif + static void query_s390_facilities(void) { unsigned long hwcap = qemu_getauxval(AT_HWCAP); @@ -2452,6 +2544,16 @@ static void query_s390_facilities(void) asm volatile(".word 0xb2b0,0x1000" : "=r"(r0) : "r"(r0), "r"(r1) : "memory", "cc"); } + + /* + * Use of vector registers requires os support beyond the facility bit. + * If the kernel does not advertise support, disable the facility bits. + * There is nothing else we currently care about in the 3rd word, so + * disable VECTOR with one store. + */ + if (1 || !(hwcap & HWCAP_S390_VXRS)) { + s390_facilities[2] = 0; + } } static void tcg_target_init(TCGContext *s) @@ -2460,6 +2562,10 @@ static void tcg_target_init(TCGContext *s) tcg_target_available_regs[TCG_TYPE_I32] = 0xffff; tcg_target_available_regs[TCG_TYPE_I64] = 0xffff; + if (HAVE_FACILITY(VECTOR)) { + tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull; + tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull; + } tcg_target_call_clobber_regs = 0; tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0); @@ -2474,6 +2580,31 @@ static void tcg_target_init(TCGContext *s) /* The return register can be considered call-clobbered. */ tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V20); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V21); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V22); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V23); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V24); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V25); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V26); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V27); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V28); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V29); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V30); + tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V31); + s->reserved_regs = 0; tcg_regset_set_reg(s->reserved_regs, TCG_TMP0); /* XXX many insns can't be used with R0, so we better avoid it for now */ diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 0174357f1b..5a03c5f2f4 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -37,11 +37,20 @@ typedef enum TCGReg { TCG_REG_R8, TCG_REG_R9, TCG_REG_R10, TCG_REG_R11, TCG_REG_R12, TCG_REG_R13, TCG_REG_R14, TCG_REG_R15, + TCG_REG_V0 = 32, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3, + TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7, + TCG_REG_V8, TCG_REG_V9, TCG_REG_V10, TCG_REG_V11, + TCG_REG_V12, TCG_REG_V13, TCG_REG_V14, TCG_REG_V15, + TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19, + TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23, + TCG_REG_V24, TCG_REG_V25, TCG_REG_V26, TCG_REG_V27, + TCG_REG_V28, TCG_REG_V29, TCG_REG_V30, TCG_REG_V31, + TCG_AREG0 = TCG_REG_R10, TCG_REG_CALL_STACK = TCG_REG_R15 } TCGReg; -#define TCG_TARGET_NB_REGS 16 +#define TCG_TARGET_NB_REGS 64 /* A list of relevant facilities used by this translator. Some of these are required for proper operation, and these are checked at startup. */ @@ -54,8 +63,9 @@ typedef enum TCGReg { #define FACILITY_FAST_BCR_SER FACILITY_LOAD_ON_COND #define FACILITY_DISTINCT_OPS FACILITY_LOAD_ON_COND #define FACILITY_LOAD_ON_COND2 53 +#define FACILITY_VECTOR 129 -extern uint64_t s390_facilities[1]; +extern uint64_t s390_facilities[3]; #define HAVE_FACILITY(X) \ ((s390_facilities[FACILITY_##X / 64] >> (63 - FACILITY_##X % 64)) & 1) @@ -128,6 +138,27 @@ extern uint64_t s390_facilities[1]; #define TCG_TARGET_HAS_muluh_i64 0 #define TCG_TARGET_HAS_mulsh_i64 0 +#define TCG_TARGET_HAS_v64 HAVE_FACILITY(VECTOR) +#define TCG_TARGET_HAS_v128 HAVE_FACILITY(VECTOR) +#define TCG_TARGET_HAS_v256 0 + +#define TCG_TARGET_HAS_andc_vec 0 +#define TCG_TARGET_HAS_orc_vec 0 +#define TCG_TARGET_HAS_not_vec 0 +#define TCG_TARGET_HAS_neg_vec 0 +#define TCG_TARGET_HAS_abs_vec 0 +#define TCG_TARGET_HAS_roti_vec 0 +#define TCG_TARGET_HAS_rots_vec 0 +#define TCG_TARGET_HAS_rotv_vec 0 +#define TCG_TARGET_HAS_shi_vec 0 +#define TCG_TARGET_HAS_shs_vec 0 +#define TCG_TARGET_HAS_shv_vec 0 +#define TCG_TARGET_HAS_mul_vec 0 +#define TCG_TARGET_HAS_sat_vec 0 +#define TCG_TARGET_HAS_minmax_vec 0 +#define TCG_TARGET_HAS_bitsel_vec 0 +#define TCG_TARGET_HAS_cmpsel_vec 0 + /* used for function call generation */ #define TCG_TARGET_STACK_ALIGN 8 #define TCG_TARGET_CALL_STACK_OFFSET 160 diff --git a/tcg/s390x/tcg-target.opc.h b/tcg/s390x/tcg-target.opc.h new file mode 100644 index 0000000000..67afc82a93 --- /dev/null +++ b/tcg/s390x/tcg-target.opc.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2021 Linaro + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * (at your option) any later version. + * + * See the COPYING file in the top-level directory for details. + * + * Target-specific opcodes for host vector expansion. These will be + * emitted by tcg_expand_vec_op. For those familiar with GCC internals, + * consider these to be UNSPEC with names. + */ From 2dabf74252e2b9b9f23b1991d69a8def61442a2c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 17:04:22 -0700 Subject: [PATCH 18/28] tcg/s390x: Implement tcg_out_ld/st for vector types Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 132 +++++++++++++++++++++++++++++++++---- 1 file changed, 120 insertions(+), 12 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 8bee6dd26e..d80f25e48e 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -265,6 +265,12 @@ typedef enum S390Opcode { RX_STC = 0x42, RX_STH = 0x40, + VRX_VL = 0xe706, + VRX_VLLEZ = 0xe704, + VRX_VST = 0xe70e, + VRX_VSTEF = 0xe70b, + VRX_VSTEG = 0xe70a, + NOP = 0x0707, } S390Opcode; @@ -412,6 +418,16 @@ static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { static const tcg_insn_unit *tb_ret_addr; uint64_t s390_facilities[3]; +static inline bool is_general_reg(TCGReg r) +{ + return r <= TCG_REG_R15; +} + +static inline bool is_vector_reg(TCGReg r) +{ + return r >= TCG_REG_V0 && r <= TCG_REG_V31; +} + static bool patch_reloc(tcg_insn_unit *src_rw, int type, intptr_t value, intptr_t addend) { @@ -529,6 +545,31 @@ static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1, #define tcg_out_insn_RX tcg_out_insn_RS #define tcg_out_insn_RXY tcg_out_insn_RSY +static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4) +{ + /* + * Shift bit 4 of each regno to its corresponding bit of RXB. + * RXB itself begins at bit 8 of the instruction so 8 - 4 = 4 + * is the left-shift of the 4th operand. + */ + return ((v1 & 0x10) << (4 + 3)) + | ((v2 & 0x10) << (4 + 2)) + | ((v3 & 0x10) << (4 + 1)) + | ((v4 & 0x10) << (4 + 0)); +} + +static void tcg_out_insn_VRX(TCGContext *s, S390Opcode op, TCGReg v1, + TCGReg b2, TCGReg x2, intptr_t d2, int m3) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(d2 >= 0 && d2 <= 0xfff); + tcg_debug_assert(is_general_reg(x2)); + tcg_debug_assert(is_general_reg(b2)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | x2); + tcg_out16(s, (b2 << 12) | d2); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12)); +} + /* Emit an opcode with "type-checking" of the format. */ #define tcg_out_insn(S, FMT, OP, ...) \ glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__) @@ -705,25 +746,92 @@ static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy, } } +static void tcg_out_vrx_mem(TCGContext *s, S390Opcode opc_vrx, + TCGReg data, TCGReg base, TCGReg index, + tcg_target_long ofs, int m3) +{ + if (ofs < 0 || ofs >= 0x1000) { + if (ofs >= -0x80000 && ofs < 0x80000) { + tcg_out_insn(s, RXY, LAY, TCG_TMP0, base, index, ofs); + base = TCG_TMP0; + index = TCG_REG_NONE; + ofs = 0; + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs); + if (index != TCG_REG_NONE) { + tcg_out_insn(s, RRE, AGR, TCG_TMP0, index); + } + index = TCG_TMP0; + ofs = 0; + } + } + tcg_out_insn_VRX(s, opc_vrx, data, base, index, ofs, m3); +} /* load data without address translation or endianness conversion */ -static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data, - TCGReg base, intptr_t ofs) +static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, intptr_t ofs) { - if (type == TCG_TYPE_I32) { - tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs); - } else { - tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs); + switch (type) { + case TCG_TYPE_I32: + if (likely(is_general_reg(data))) { + tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs); + break; + } + tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_32); + break; + + case TCG_TYPE_I64: + if (likely(is_general_reg(data))) { + tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs); + break; + } + /* fallthru */ + + case TCG_TYPE_V64: + tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_64); + break; + + case TCG_TYPE_V128: + /* Hint quadword aligned. */ + tcg_out_vrx_mem(s, VRX_VL, data, base, TCG_REG_NONE, ofs, 4); + break; + + default: + g_assert_not_reached(); } } -static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg data, - TCGReg base, intptr_t ofs) +static void tcg_out_st(TCGContext *s, TCGType type, TCGReg data, + TCGReg base, intptr_t ofs) { - if (type == TCG_TYPE_I32) { - tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs); - } else { - tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs); + switch (type) { + case TCG_TYPE_I32: + if (likely(is_general_reg(data))) { + tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs); + } else { + tcg_out_vrx_mem(s, VRX_VSTEF, data, base, TCG_REG_NONE, ofs, 1); + } + break; + + case TCG_TYPE_I64: + if (likely(is_general_reg(data))) { + tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs); + break; + } + /* fallthru */ + + case TCG_TYPE_V64: + tcg_out_vrx_mem(s, VRX_VSTEG, data, base, TCG_REG_NONE, ofs, 0); + break; + + case TCG_TYPE_V128: + /* Hint quadword aligned. */ + tcg_out_vrx_mem(s, VRX_VST, data, base, TCG_REG_NONE, ofs, 4); + break; + + default: + g_assert_not_reached(); } } From b33ce7251c9591635ca0f93bf5e75e7dbb844b5c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 17:12:18 -0700 Subject: [PATCH 19/28] tcg/s390x: Implement tcg_out_mov for vector types Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 72 +++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index d80f25e48e..586a4b5587 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -265,6 +265,11 @@ typedef enum S390Opcode { RX_STC = 0x42, RX_STH = 0x40, + VRRa_VLR = 0xe756, + + VRSb_VLVG = 0xe722, + VRSc_VLGV = 0xe721, + VRX_VL = 0xe706, VRX_VLLEZ = 0xe704, VRX_VST = 0xe70e, @@ -558,6 +563,39 @@ static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4) | ((v4 & 0x10) << (4 + 0)); } +static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op, + TCGReg v1, TCGReg v2, int m3) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(is_vector_reg(v2)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf)); + tcg_out32(s, (op & 0x00ff) | RXB(v1, v2, 0, 0) | (m3 << 12)); +} + +static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1, + intptr_t d2, TCGReg b2, TCGReg r3, int m4) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(d2 >= 0 && d2 <= 0xfff); + tcg_debug_assert(is_general_reg(b2)); + tcg_debug_assert(is_general_reg(r3)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r3); + tcg_out16(s, b2 << 12 | d2); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12)); +} + +static void tcg_out_insn_VRSc(TCGContext *s, S390Opcode op, TCGReg r1, + intptr_t d2, TCGReg b2, TCGReg v3, int m4) +{ + tcg_debug_assert(is_general_reg(r1)); + tcg_debug_assert(d2 >= 0 && d2 <= 0xfff); + tcg_debug_assert(is_general_reg(b2)); + tcg_debug_assert(is_vector_reg(v3)); + tcg_out16(s, (op & 0xff00) | (r1 << 4) | (v3 & 0xf)); + tcg_out16(s, b2 << 12 | d2); + tcg_out16(s, (op & 0x00ff) | RXB(0, 0, v3, 0) | (m4 << 12)); +} + static void tcg_out_insn_VRX(TCGContext *s, S390Opcode op, TCGReg v1, TCGReg b2, TCGReg x2, intptr_t d2, int m3) { @@ -591,12 +629,38 @@ static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest, static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src) { - if (src != dst) { - if (type == TCG_TYPE_I32) { + if (src == dst) { + return true; + } + switch (type) { + case TCG_TYPE_I32: + if (likely(is_general_reg(dst) && is_general_reg(src))) { tcg_out_insn(s, RR, LR, dst, src); - } else { - tcg_out_insn(s, RRE, LGR, dst, src); + break; } + /* fallthru */ + + case TCG_TYPE_I64: + if (likely(is_general_reg(dst))) { + if (likely(is_general_reg(src))) { + tcg_out_insn(s, RRE, LGR, dst, src); + } else { + tcg_out_insn(s, VRSc, VLGV, dst, 0, 0, src, 3); + } + break; + } else if (is_general_reg(src)) { + tcg_out_insn(s, VRSb, VLVG, dst, 0, 0, src, 3); + break; + } + /* fallthru */ + + case TCG_TYPE_V64: + case TCG_TYPE_V128: + tcg_out_insn(s, VRRa, VLR, dst, src, 0); + break; + + default: + g_assert_not_reached(); } return true; } From 79cada8693d3e32162e41323b4bfbeaa765cad35 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 17:25:52 -0700 Subject: [PATCH 20/28] tcg/s390x: Implement tcg_out_dup*_vec Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 122 ++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 586a4b5587..f59250872b 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -265,13 +265,20 @@ typedef enum S390Opcode { RX_STC = 0x42, RX_STH = 0x40, + VRIa_VGBM = 0xe744, + VRIa_VREPI = 0xe745, + VRIb_VGM = 0xe746, + VRIc_VREP = 0xe74d, + VRRa_VLR = 0xe756, + VRRf_VLVGP = 0xe762, VRSb_VLVG = 0xe722, VRSc_VLGV = 0xe721, VRX_VL = 0xe706, VRX_VLLEZ = 0xe704, + VRX_VLREP = 0xe705, VRX_VST = 0xe70e, VRX_VSTEF = 0xe70b, VRX_VSTEG = 0xe70a, @@ -563,6 +570,34 @@ static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4) | ((v4 & 0x10) << (4 + 0)); } +static void tcg_out_insn_VRIa(TCGContext *s, S390Opcode op, + TCGReg v1, uint16_t i2, int m3) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4)); + tcg_out16(s, i2); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m3 << 12)); +} + +static void tcg_out_insn_VRIb(TCGContext *s, S390Opcode op, + TCGReg v1, uint8_t i2, uint8_t i3, int m4) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4)); + tcg_out16(s, (i2 << 8) | (i3 & 0xff)); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0) | (m4 << 12)); +} + +static void tcg_out_insn_VRIc(TCGContext *s, S390Opcode op, + TCGReg v1, uint16_t i2, TCGReg v3, int m4) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(is_vector_reg(v3)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf)); + tcg_out16(s, i2); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12)); +} + static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op, TCGReg v1, TCGReg v2, int m3) { @@ -572,6 +607,17 @@ static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op, tcg_out32(s, (op & 0x00ff) | RXB(v1, v2, 0, 0) | (m3 << 12)); } +static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op, + TCGReg v1, TCGReg r2, TCGReg r3) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(is_general_reg(r2)); + tcg_debug_assert(is_general_reg(r3)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | r2); + tcg_out16(s, r3 << 12); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0)); +} + static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1, intptr_t d2, TCGReg b2, TCGReg r3, int m4) { @@ -2501,19 +2547,89 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg src) { - g_assert_not_reached(); + if (is_general_reg(src)) { + /* Replicate general register into two MO_64. */ + tcg_out_insn(s, VRRf, VLVGP, dst, src, src); + if (vece == MO_64) { + return true; + } + } + + /* + * Recall that the "standard" integer, within a vector, is the + * rightmost element of the leftmost doubleword, a-la VLLEZ. + */ + tcg_out_insn(s, VRIc, VREP, dst, (8 >> vece) - 1, src, vece); + return true; } static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, TCGReg base, intptr_t offset) { - g_assert_not_reached(); + tcg_out_vrx_mem(s, VRX_VLREP, dst, base, TCG_REG_NONE, offset, vece); + return true; } static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg dst, int64_t val) { - g_assert_not_reached(); + int i, mask, msb, lsb; + + /* Look for int16_t elements. */ + if (vece <= MO_16 || + (vece == MO_32 ? (int32_t)val : val) == (int16_t)val) { + tcg_out_insn(s, VRIa, VREPI, dst, val, vece); + return; + } + + /* Look for bit masks. */ + if (vece == MO_32) { + if (risbg_mask((int32_t)val)) { + /* Handle wraparound by swapping msb and lsb. */ + if ((val & 0x80000001u) == 0x80000001u) { + msb = 32 - ctz32(~val); + lsb = clz32(~val) - 1; + } else { + msb = clz32(val); + lsb = 31 - ctz32(val); + } + tcg_out_insn(s, VRIb, VGM, dst, lsb, msb, MO_32); + return; + } + } else { + if (risbg_mask(val)) { + /* Handle wraparound by swapping msb and lsb. */ + if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) { + /* Handle wraparound by swapping msb and lsb. */ + msb = 64 - ctz64(~val); + lsb = clz64(~val) - 1; + } else { + msb = clz64(val); + lsb = 63 - ctz64(val); + } + tcg_out_insn(s, VRIb, VGM, dst, lsb, msb, MO_64); + return; + } + } + + /* Look for all bytes 0x00 or 0xff. */ + for (i = mask = 0; i < 8; i++) { + uint8_t byte = val >> (i * 8); + if (byte == 0xff) { + mask |= 1 << i; + } else if (byte != 0) { + break; + } + } + if (i == 8) { + tcg_out_insn(s, VRIa, VGBM, dst, mask * 0x0101, 0); + return; + } + + /* Otherwise, stuff it in the constant pool. */ + tcg_out_insn(s, RIL, LARL, TCG_TMP0, 0); + new_pool_label(s, val, R_390_PC32DBL, s->code_ptr - 2, 2); + tcg_out_insn(s, VRX, VLREP, dst, TCG_TMP0, TCG_REG_NONE, 0, MO_64); } static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, From a429ee2978a1bb81cfb737e382a47c119a2c0886 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 18:08:00 -0700 Subject: [PATCH 21/28] tcg/s390x: Implement minimal vector operations Implementing add, sub, and, or, xor as the minimal set. This allows us to actually enable vectors in query_s390_facilities. Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 154 ++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 4 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index f59250872b..063f720199 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -271,6 +271,14 @@ typedef enum S390Opcode { VRIc_VREP = 0xe74d, VRRa_VLR = 0xe756, + VRRc_VA = 0xe7f3, + VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */ + VRRc_VCH = 0xe7fb, /* " */ + VRRc_VCHL = 0xe7f9, /* " */ + VRRc_VN = 0xe768, + VRRc_VO = 0xe76a, + VRRc_VS = 0xe7f7, + VRRc_VX = 0xe76d, VRRf_VLVGP = 0xe762, VRSb_VLVG = 0xe722, @@ -607,6 +615,17 @@ static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op, tcg_out32(s, (op & 0x00ff) | RXB(v1, v2, 0, 0) | (m3 << 12)); } +static void tcg_out_insn_VRRc(TCGContext *s, S390Opcode op, + TCGReg v1, TCGReg v2, TCGReg v3, int m4) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(is_vector_reg(v2)); + tcg_debug_assert(is_vector_reg(v3)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf)); + tcg_out16(s, v3 << 12); + tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, 0) | (m4 << 12)); +} + static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op, TCGReg v1, TCGReg r2, TCGReg r3) { @@ -2636,18 +2655,145 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg *args, const int *const_args) { - g_assert_not_reached(); + TCGType type = vecl + TCG_TYPE_V64; + TCGArg a0 = args[0], a1 = args[1], a2 = args[2]; + + switch (opc) { + case INDEX_op_ld_vec: + tcg_out_ld(s, type, a0, a1, a2); + break; + case INDEX_op_st_vec: + tcg_out_st(s, type, a0, a1, a2); + break; + case INDEX_op_dupm_vec: + tcg_out_dupm_vec(s, type, vece, a0, a1, a2); + break; + + case INDEX_op_add_vec: + tcg_out_insn(s, VRRc, VA, a0, a1, a2, vece); + break; + case INDEX_op_sub_vec: + tcg_out_insn(s, VRRc, VS, a0, a1, a2, vece); + break; + case INDEX_op_and_vec: + tcg_out_insn(s, VRRc, VN, a0, a1, a2, 0); + break; + case INDEX_op_or_vec: + tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0); + break; + case INDEX_op_xor_vec: + tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0); + break; + + case INDEX_op_cmp_vec: + switch ((TCGCond)args[3]) { + case TCG_COND_EQ: + tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece); + break; + case TCG_COND_GT: + tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece); + break; + case TCG_COND_GTU: + tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece); + break; + default: + g_assert_not_reached(); + } + break; + + case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ + case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ + default: + g_assert_not_reached(); + } } int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) { - return 0; + switch (opc) { + case INDEX_op_add_vec: + case INDEX_op_and_vec: + case INDEX_op_or_vec: + case INDEX_op_sub_vec: + case INDEX_op_xor_vec: + return 1; + case INDEX_op_cmp_vec: + return -1; + default: + return 0; + } +} + +static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, + TCGv_vec v1, TCGv_vec v2, TCGCond cond) +{ + bool need_swap = false, need_inv = false; + + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_GT: + case TCG_COND_GTU: + break; + case TCG_COND_NE: + case TCG_COND_LE: + case TCG_COND_LEU: + need_inv = true; + break; + case TCG_COND_LT: + case TCG_COND_LTU: + need_swap = true; + break; + case TCG_COND_GE: + case TCG_COND_GEU: + need_swap = need_inv = true; + break; + default: + g_assert_not_reached(); + } + + if (need_inv) { + cond = tcg_invert_cond(cond); + } + if (need_swap) { + TCGv_vec t1; + t1 = v1, v1 = v2, v2 = t1; + cond = tcg_swap_cond(cond); + } + + vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0), + tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); + + return need_inv; +} + +static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, + TCGv_vec v1, TCGv_vec v2, TCGCond cond) +{ + if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) { + tcg_gen_not_vec(vece, v0, v0); + } } void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { - g_assert_not_reached(); + va_list va; + TCGv_vec v0, v1, v2; + + va_start(va, a0); + v0 = temp_tcgv_vec(arg_temp(a0)); + v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); + v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); + + switch (opc) { + case INDEX_op_cmp_vec: + expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); + break; + + default: + g_assert_not_reached(); + } + va_end(va); } static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) @@ -2839,7 +2985,7 @@ static void query_s390_facilities(void) * There is nothing else we currently care about in the 3rd word, so * disable VECTOR with one store. */ - if (1 || !(hwcap & HWCAP_S390_VXRS)) { + if (!(hwcap & HWCAP_S390_VXRS)) { s390_facilities[2] = 0; } } From ae77bbe5747dc655bed213006798f9b07e2f79bf Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 18:23:16 -0700 Subject: [PATCH 22/28] tcg/s390x: Implement andc, orc, abs, neg, not vector operations These logical and arithmetic operations are optional but trivial. Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target-con-set.h | 1 + tcg/s390x/tcg-target.c.inc | 32 ++++++++++++++++++++++++++++++++ tcg/s390x/tcg-target.h | 11 ++++++----- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index ce9432cfe3..cb953896d5 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -17,6 +17,7 @@ C_O0_I2(v, r) C_O1_I1(r, L) C_O1_I1(r, r) C_O1_I1(v, r) +C_O1_I1(v, v) C_O1_I1(v, vr) C_O1_I2(r, 0, ri) C_O1_I2(r, 0, rI) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 063f720199..cbad88271a 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -270,13 +270,18 @@ typedef enum S390Opcode { VRIb_VGM = 0xe746, VRIc_VREP = 0xe74d, + VRRa_VLC = 0xe7de, + VRRa_VLP = 0xe7df, VRRa_VLR = 0xe756, VRRc_VA = 0xe7f3, VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */ VRRc_VCH = 0xe7fb, /* " */ VRRc_VCHL = 0xe7f9, /* " */ VRRc_VN = 0xe768, + VRRc_VNC = 0xe769, + VRRc_VNO = 0xe76b, VRRc_VO = 0xe76a, + VRRc_VOC = 0xe76f, VRRc_VS = 0xe7f7, VRRc_VX = 0xe76d, VRRf_VLVGP = 0xe762, @@ -2669,6 +2674,16 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_dupm_vec(s, type, vece, a0, a1, a2); break; + case INDEX_op_abs_vec: + tcg_out_insn(s, VRRa, VLP, a0, a1, vece); + break; + case INDEX_op_neg_vec: + tcg_out_insn(s, VRRa, VLC, a0, a1, vece); + break; + case INDEX_op_not_vec: + tcg_out_insn(s, VRRc, VNO, a0, a1, a1, 0); + break; + case INDEX_op_add_vec: tcg_out_insn(s, VRRc, VA, a0, a1, a2, vece); break; @@ -2678,9 +2693,15 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_and_vec: tcg_out_insn(s, VRRc, VN, a0, a1, a2, 0); break; + case INDEX_op_andc_vec: + tcg_out_insn(s, VRRc, VNC, a0, a1, a2, 0); + break; case INDEX_op_or_vec: tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0); break; + case INDEX_op_orc_vec: + tcg_out_insn(s, VRRc, VOC, a0, a1, a2, 0); + break; case INDEX_op_xor_vec: tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0); break; @@ -2711,9 +2732,14 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) { switch (opc) { + case INDEX_op_abs_vec: case INDEX_op_add_vec: case INDEX_op_and_vec: + case INDEX_op_andc_vec: + case INDEX_op_neg_vec: + case INDEX_op_not_vec: case INDEX_op_or_vec: + case INDEX_op_orc_vec: case INDEX_op_sub_vec: case INDEX_op_xor_vec: return 1; @@ -2943,10 +2969,16 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O1_I1(v, r); case INDEX_op_dup_vec: return C_O1_I1(v, vr); + case INDEX_op_abs_vec: + case INDEX_op_neg_vec: + case INDEX_op_not_vec: + return C_O1_I1(v, v); case INDEX_op_add_vec: case INDEX_op_sub_vec: case INDEX_op_and_vec: + case INDEX_op_andc_vec: case INDEX_op_or_vec: + case INDEX_op_orc_vec: case INDEX_op_xor_vec: case INDEX_op_cmp_vec: return C_O1_I2(v, v, v); diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 5a03c5f2f4..a42074e451 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -64,6 +64,7 @@ typedef enum TCGReg { #define FACILITY_DISTINCT_OPS FACILITY_LOAD_ON_COND #define FACILITY_LOAD_ON_COND2 53 #define FACILITY_VECTOR 129 +#define FACILITY_VECTOR_ENH1 135 extern uint64_t s390_facilities[3]; @@ -142,11 +143,11 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_v128 HAVE_FACILITY(VECTOR) #define TCG_TARGET_HAS_v256 0 -#define TCG_TARGET_HAS_andc_vec 0 -#define TCG_TARGET_HAS_orc_vec 0 -#define TCG_TARGET_HAS_not_vec 0 -#define TCG_TARGET_HAS_neg_vec 0 -#define TCG_TARGET_HAS_abs_vec 0 +#define TCG_TARGET_HAS_andc_vec 1 +#define TCG_TARGET_HAS_orc_vec HAVE_FACILITY(VECTOR_ENH1) +#define TCG_TARGET_HAS_not_vec 1 +#define TCG_TARGET_HAS_neg_vec 1 +#define TCG_TARGET_HAS_abs_vec 1 #define TCG_TARGET_HAS_roti_vec 0 #define TCG_TARGET_HAS_rots_vec 0 #define TCG_TARGET_HAS_rotv_vec 0 From 479b61cbfafba997b97aec2e4d323c86240c24b0 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 18:33:47 -0700 Subject: [PATCH 23/28] tcg/s390x: Implement TCG_TARGET_HAS_mul_vec Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 7 +++++++ tcg/s390x/tcg-target.h | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index cbad88271a..85178c93d3 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -277,6 +277,7 @@ typedef enum S390Opcode { VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */ VRRc_VCH = 0xe7fb, /* " */ VRRc_VCHL = 0xe7f9, /* " */ + VRRc_VML = 0xe7a2, VRRc_VN = 0xe768, VRRc_VNC = 0xe769, VRRc_VNO = 0xe76b, @@ -2696,6 +2697,9 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_andc_vec: tcg_out_insn(s, VRRc, VNC, a0, a1, a2, 0); break; + case INDEX_op_mul_vec: + tcg_out_insn(s, VRRc, VML, a0, a1, a2, vece); + break; case INDEX_op_or_vec: tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0); break; @@ -2745,6 +2749,8 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) return 1; case INDEX_op_cmp_vec: return -1; + case INDEX_op_mul_vec: + return vece < MO_64; default: return 0; } @@ -2981,6 +2987,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_orc_vec: case INDEX_op_xor_vec: case INDEX_op_cmp_vec: + case INDEX_op_mul_vec: return C_O1_I2(v, v, v); default: diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index a42074e451..1c581a2f60 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -154,7 +154,7 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_shi_vec 0 #define TCG_TARGET_HAS_shs_vec 0 #define TCG_TARGET_HAS_shv_vec 0 -#define TCG_TARGET_HAS_mul_vec 0 +#define TCG_TARGET_HAS_mul_vec 1 #define TCG_TARGET_HAS_sat_vec 0 #define TCG_TARGET_HAS_minmax_vec 0 #define TCG_TARGET_HAS_bitsel_vec 0 From 22cb37b41720c8bcd48941b29b7c6ac1f990ab94 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 19:00:38 -0700 Subject: [PATCH 24/28] tcg/s390x: Implement vector shift operations Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target-con-set.h | 1 + tcg/s390x/tcg-target.c.inc | 93 +++++++++++++++++++++++++++++++++- tcg/s390x/tcg-target.h | 12 ++--- 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index cb953896d5..49b98f33b9 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -24,6 +24,7 @@ C_O1_I2(r, 0, rI) C_O1_I2(r, 0, rJ) C_O1_I2(r, r, ri) C_O1_I2(r, rZ, r) +C_O1_I2(v, v, r) C_O1_I2(v, v, v) C_O1_I4(r, r, ri, r, 0) C_O1_I4(r, r, ri, rI, 0) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 85178c93d3..98c0cd5091 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -277,6 +277,10 @@ typedef enum S390Opcode { VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */ VRRc_VCH = 0xe7fb, /* " */ VRRc_VCHL = 0xe7f9, /* " */ + VRRc_VERLLV = 0xe773, + VRRc_VESLV = 0xe770, + VRRc_VESRAV = 0xe77a, + VRRc_VESRLV = 0xe778, VRRc_VML = 0xe7a2, VRRc_VN = 0xe768, VRRc_VNC = 0xe769, @@ -287,6 +291,10 @@ typedef enum S390Opcode { VRRc_VX = 0xe76d, VRRf_VLVGP = 0xe762, + VRSa_VERLL = 0xe733, + VRSa_VESL = 0xe730, + VRSa_VESRA = 0xe73a, + VRSa_VESRL = 0xe738, VRSb_VLVG = 0xe722, VRSc_VLGV = 0xe721, @@ -643,6 +651,18 @@ static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op, tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, 0, 0)); } +static void tcg_out_insn_VRSa(TCGContext *s, S390Opcode op, TCGReg v1, + intptr_t d2, TCGReg b2, TCGReg v3, int m4) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(d2 >= 0 && d2 <= 0xfff); + tcg_debug_assert(is_general_reg(b2)); + tcg_debug_assert(is_vector_reg(v3)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v3 & 0xf)); + tcg_out16(s, b2 << 12 | d2); + tcg_out16(s, (op & 0x00ff) | RXB(v1, 0, v3, 0) | (m4 << 12)); +} + static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1, intptr_t d2, TCGReg b2, TCGReg r3, int m4) { @@ -2710,6 +2730,43 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0); break; + case INDEX_op_shli_vec: + tcg_out_insn(s, VRSa, VESL, a0, a2, TCG_REG_NONE, a1, vece); + break; + case INDEX_op_shri_vec: + tcg_out_insn(s, VRSa, VESRL, a0, a2, TCG_REG_NONE, a1, vece); + break; + case INDEX_op_sari_vec: + tcg_out_insn(s, VRSa, VESRA, a0, a2, TCG_REG_NONE, a1, vece); + break; + case INDEX_op_rotli_vec: + tcg_out_insn(s, VRSa, VERLL, a0, a2, TCG_REG_NONE, a1, vece); + break; + case INDEX_op_shls_vec: + tcg_out_insn(s, VRSa, VESL, a0, 0, a2, a1, vece); + break; + case INDEX_op_shrs_vec: + tcg_out_insn(s, VRSa, VESRL, a0, 0, a2, a1, vece); + break; + case INDEX_op_sars_vec: + tcg_out_insn(s, VRSa, VESRA, a0, 0, a2, a1, vece); + break; + case INDEX_op_rotls_vec: + tcg_out_insn(s, VRSa, VERLL, a0, 0, a2, a1, vece); + break; + case INDEX_op_shlv_vec: + tcg_out_insn(s, VRRc, VESLV, a0, a1, a2, vece); + break; + case INDEX_op_shrv_vec: + tcg_out_insn(s, VRRc, VESRLV, a0, a1, a2, vece); + break; + case INDEX_op_sarv_vec: + tcg_out_insn(s, VRRc, VESRAV, a0, a1, a2, vece); + break; + case INDEX_op_rotlv_vec: + tcg_out_insn(s, VRRc, VERLLV, a0, a1, a2, vece); + break; + case INDEX_op_cmp_vec: switch ((TCGCond)args[3]) { case TCG_COND_EQ: @@ -2744,10 +2801,23 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_not_vec: case INDEX_op_or_vec: case INDEX_op_orc_vec: + case INDEX_op_rotli_vec: + case INDEX_op_rotls_vec: + case INDEX_op_rotlv_vec: + case INDEX_op_sari_vec: + case INDEX_op_sars_vec: + case INDEX_op_sarv_vec: + case INDEX_op_shli_vec: + case INDEX_op_shls_vec: + case INDEX_op_shlv_vec: + case INDEX_op_shri_vec: + case INDEX_op_shrs_vec: + case INDEX_op_shrv_vec: case INDEX_op_sub_vec: case INDEX_op_xor_vec: return 1; case INDEX_op_cmp_vec: + case INDEX_op_rotrv_vec: return -1; case INDEX_op_mul_vec: return vece < MO_64; @@ -2810,7 +2880,7 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { va_list va; - TCGv_vec v0, v1, v2; + TCGv_vec v0, v1, v2, t0; va_start(va, a0); v0 = temp_tcgv_vec(arg_temp(a0)); @@ -2822,6 +2892,13 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); break; + case INDEX_op_rotrv_vec: + t0 = tcg_temp_new_vec(type); + tcg_gen_neg_vec(vece, t0, v2); + tcg_gen_rotlv_vec(vece, v0, v1, t0); + tcg_temp_free_vec(t0); + break; + default: g_assert_not_reached(); } @@ -2978,6 +3055,10 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_abs_vec: case INDEX_op_neg_vec: case INDEX_op_not_vec: + case INDEX_op_rotli_vec: + case INDEX_op_sari_vec: + case INDEX_op_shli_vec: + case INDEX_op_shri_vec: return C_O1_I1(v, v); case INDEX_op_add_vec: case INDEX_op_sub_vec: @@ -2988,7 +3069,17 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_xor_vec: case INDEX_op_cmp_vec: case INDEX_op_mul_vec: + case INDEX_op_rotlv_vec: + case INDEX_op_rotrv_vec: + case INDEX_op_shlv_vec: + case INDEX_op_shrv_vec: + case INDEX_op_sarv_vec: return C_O1_I2(v, v, v); + case INDEX_op_rotls_vec: + case INDEX_op_shls_vec: + case INDEX_op_shrs_vec: + case INDEX_op_sars_vec: + return C_O1_I2(v, v, r); default: g_assert_not_reached(); diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 1c581a2f60..d7d204b782 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -148,12 +148,12 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_not_vec 1 #define TCG_TARGET_HAS_neg_vec 1 #define TCG_TARGET_HAS_abs_vec 1 -#define TCG_TARGET_HAS_roti_vec 0 -#define TCG_TARGET_HAS_rots_vec 0 -#define TCG_TARGET_HAS_rotv_vec 0 -#define TCG_TARGET_HAS_shi_vec 0 -#define TCG_TARGET_HAS_shs_vec 0 -#define TCG_TARGET_HAS_shv_vec 0 +#define TCG_TARGET_HAS_roti_vec 1 +#define TCG_TARGET_HAS_rots_vec 1 +#define TCG_TARGET_HAS_rotv_vec 1 +#define TCG_TARGET_HAS_shi_vec 1 +#define TCG_TARGET_HAS_shs_vec 1 +#define TCG_TARGET_HAS_shv_vec 1 #define TCG_TARGET_HAS_mul_vec 1 #define TCG_TARGET_HAS_sat_vec 0 #define TCG_TARGET_HAS_minmax_vec 0 From 220db7a6c461d9e33a50134f72a2e0cb3c6c5e62 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 19:07:16 -0700 Subject: [PATCH 25/28] tcg/s390x: Implement TCG_TARGET_HAS_minmax_vec Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 25 +++++++++++++++++++++++++ tcg/s390x/tcg-target.h | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 98c0cd5091..3b8fc62cd7 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -282,6 +282,10 @@ typedef enum S390Opcode { VRRc_VESRAV = 0xe77a, VRRc_VESRLV = 0xe778, VRRc_VML = 0xe7a2, + VRRc_VMN = 0xe7fe, + VRRc_VMNL = 0xe7fc, + VRRc_VMX = 0xe7ff, + VRRc_VMXL = 0xe7fd, VRRc_VN = 0xe768, VRRc_VNC = 0xe769, VRRc_VNO = 0xe76b, @@ -2767,6 +2771,19 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_insn(s, VRRc, VERLLV, a0, a1, a2, vece); break; + case INDEX_op_smin_vec: + tcg_out_insn(s, VRRc, VMN, a0, a1, a2, vece); + break; + case INDEX_op_smax_vec: + tcg_out_insn(s, VRRc, VMX, a0, a1, a2, vece); + break; + case INDEX_op_umin_vec: + tcg_out_insn(s, VRRc, VMNL, a0, a1, a2, vece); + break; + case INDEX_op_umax_vec: + tcg_out_insn(s, VRRc, VMXL, a0, a1, a2, vece); + break; + case INDEX_op_cmp_vec: switch ((TCGCond)args[3]) { case TCG_COND_EQ: @@ -2813,7 +2830,11 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_shri_vec: case INDEX_op_shrs_vec: case INDEX_op_shrv_vec: + case INDEX_op_smax_vec: + case INDEX_op_smin_vec: case INDEX_op_sub_vec: + case INDEX_op_umax_vec: + case INDEX_op_umin_vec: case INDEX_op_xor_vec: return 1; case INDEX_op_cmp_vec: @@ -3074,6 +3095,10 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_shlv_vec: case INDEX_op_shrv_vec: case INDEX_op_sarv_vec: + case INDEX_op_smax_vec: + case INDEX_op_smin_vec: + case INDEX_op_umax_vec: + case INDEX_op_umin_vec: return C_O1_I2(v, v, v); case INDEX_op_rotls_vec: case INDEX_op_shls_vec: diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index d7d204b782..a79f4f187a 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -156,7 +156,7 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_shv_vec 1 #define TCG_TARGET_HAS_mul_vec 1 #define TCG_TARGET_HAS_sat_vec 0 -#define TCG_TARGET_HAS_minmax_vec 0 +#define TCG_TARGET_HAS_minmax_vec 1 #define TCG_TARGET_HAS_bitsel_vec 0 #define TCG_TARGET_HAS_cmpsel_vec 0 From 4223c9c1c6358f65fb2df2708c86f95faf235e30 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 20:17:29 -0700 Subject: [PATCH 26/28] tcg/s390x: Implement TCG_TARGET_HAS_sat_vec The unsigned saturations are handled via generic code using min/max. The signed saturations are expanded using double-sized arithmetic and a saturating pack. Since all operations are done via expansion, do not actually set TCG_TARGET_HAS_sat_vec. Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 63 ++++++++++++++++++++++++++++++++++++++ tcg/s390x/tcg-target.opc.h | 3 ++ 2 files changed, 66 insertions(+) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 3b8fc62cd7..965d9ab54c 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -291,7 +291,10 @@ typedef enum S390Opcode { VRRc_VNO = 0xe76b, VRRc_VO = 0xe76a, VRRc_VOC = 0xe76f, + VRRc_VPKS = 0xe797, /* we leave the m5 cs field 0 */ VRRc_VS = 0xe7f7, + VRRa_VUPH = 0xe7d7, + VRRa_VUPL = 0xe7d6, VRRc_VX = 0xe76d, VRRf_VLVGP = 0xe762, @@ -2800,6 +2803,16 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_s390_vuph_vec: + tcg_out_insn(s, VRRa, VUPH, a0, a1, vece); + break; + case INDEX_op_s390_vupl_vec: + tcg_out_insn(s, VRRa, VUPL, a0, a1, vece); + break; + case INDEX_op_s390_vpks_vec: + tcg_out_insn(s, VRRc, VPKS, a0, a1, a2, vece); + break; + case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ default: @@ -2842,6 +2855,9 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) return -1; case INDEX_op_mul_vec: return vece < MO_64; + case INDEX_op_ssadd_vec: + case INDEX_op_sssub_vec: + return vece < MO_64 ? -1 : 0; default: return 0; } @@ -2897,6 +2913,43 @@ static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, } } +static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0, + TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc) +{ + TCGv_vec h1 = tcg_temp_new_vec(type); + TCGv_vec h2 = tcg_temp_new_vec(type); + TCGv_vec l1 = tcg_temp_new_vec(type); + TCGv_vec l2 = tcg_temp_new_vec(type); + + tcg_debug_assert (vece < MO_64); + + /* Unpack with sign-extension. */ + vec_gen_2(INDEX_op_s390_vuph_vec, type, vece, + tcgv_vec_arg(h1), tcgv_vec_arg(v1)); + vec_gen_2(INDEX_op_s390_vuph_vec, type, vece, + tcgv_vec_arg(h2), tcgv_vec_arg(v2)); + + vec_gen_2(INDEX_op_s390_vupl_vec, type, vece, + tcgv_vec_arg(l1), tcgv_vec_arg(v1)); + vec_gen_2(INDEX_op_s390_vupl_vec, type, vece, + tcgv_vec_arg(l2), tcgv_vec_arg(v2)); + + /* Arithmetic on a wider element size. */ + vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(h1), + tcgv_vec_arg(h1), tcgv_vec_arg(h2)); + vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(l1), + tcgv_vec_arg(l1), tcgv_vec_arg(l2)); + + /* Pack with saturation. */ + vec_gen_3(INDEX_op_s390_vpks_vec, type, vece + 1, + tcgv_vec_arg(v0), tcgv_vec_arg(h1), tcgv_vec_arg(l1)); + + tcg_temp_free_vec(h1); + tcg_temp_free_vec(h2); + tcg_temp_free_vec(l1); + tcg_temp_free_vec(l2); +} + void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { @@ -2920,6 +2973,13 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, tcg_temp_free_vec(t0); break; + case INDEX_op_ssadd_vec: + expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_add_vec); + break; + case INDEX_op_sssub_vec: + expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_sub_vec); + break; + default: g_assert_not_reached(); } @@ -3080,6 +3140,8 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_sari_vec: case INDEX_op_shli_vec: case INDEX_op_shri_vec: + case INDEX_op_s390_vuph_vec: + case INDEX_op_s390_vupl_vec: return C_O1_I1(v, v); case INDEX_op_add_vec: case INDEX_op_sub_vec: @@ -3099,6 +3161,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_smin_vec: case INDEX_op_umax_vec: case INDEX_op_umin_vec: + case INDEX_op_s390_vpks_vec: return C_O1_I2(v, v, v); case INDEX_op_rotls_vec: case INDEX_op_shls_vec: diff --git a/tcg/s390x/tcg-target.opc.h b/tcg/s390x/tcg-target.opc.h index 67afc82a93..0eb2350fb3 100644 --- a/tcg/s390x/tcg-target.opc.h +++ b/tcg/s390x/tcg-target.opc.h @@ -10,3 +10,6 @@ * emitted by tcg_expand_vec_op. For those familiar with GCC internals, * consider these to be UNSPEC with names. */ +DEF(s390_vuph_vec, 1, 1, 0, IMPLVEC) +DEF(s390_vupl_vec, 1, 1, 0, IMPLVEC) +DEF(s390_vpks_vec, 1, 2, 0, IMPLVEC) From 9bca986df88b8ea46b100e3d21cc9e653c83e0b3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 20:36:36 -0700 Subject: [PATCH 27/28] tcg/s390x: Implement TCG_TARGET_HAS_bitsel_vec Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target-con-set.h | 1 + tcg/s390x/tcg-target.c.inc | 20 ++++++++++++++++++++ tcg/s390x/tcg-target.h | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index 49b98f33b9..426dd92e51 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -26,6 +26,7 @@ C_O1_I2(r, r, ri) C_O1_I2(r, rZ, r) C_O1_I2(v, v, r) C_O1_I2(v, v, v) +C_O1_I3(v, v, v, v) C_O1_I4(r, r, ri, r, 0) C_O1_I4(r, r, ri, rI, 0) C_O2_I2(b, a, 0, r) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 965d9ab54c..ae132e1b75 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -296,6 +296,7 @@ typedef enum S390Opcode { VRRa_VUPH = 0xe7d7, VRRa_VUPL = 0xe7d6, VRRc_VX = 0xe76d, + VRRe_VSEL = 0xe78d, VRRf_VLVGP = 0xe762, VRSa_VERLL = 0xe733, @@ -647,6 +648,18 @@ static void tcg_out_insn_VRRc(TCGContext *s, S390Opcode op, tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, 0) | (m4 << 12)); } +static void tcg_out_insn_VRRe(TCGContext *s, S390Opcode op, + TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4) +{ + tcg_debug_assert(is_vector_reg(v1)); + tcg_debug_assert(is_vector_reg(v2)); + tcg_debug_assert(is_vector_reg(v3)); + tcg_debug_assert(is_vector_reg(v4)); + tcg_out16(s, (op & 0xff00) | ((v1 & 0xf) << 4) | (v2 & 0xf)); + tcg_out16(s, v3 << 12); + tcg_out16(s, (op & 0x00ff) | RXB(v1, v2, v3, v4) | (v4 << 12)); +} + static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op, TCGReg v1, TCGReg r2, TCGReg r3) { @@ -2787,6 +2800,10 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_insn(s, VRRc, VMXL, a0, a1, a2, vece); break; + case INDEX_op_bitsel_vec: + tcg_out_insn(s, VRRe, VSEL, a0, a1, a2, args[3]); + break; + case INDEX_op_cmp_vec: switch ((TCGCond)args[3]) { case TCG_COND_EQ: @@ -2827,6 +2844,7 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_add_vec: case INDEX_op_and_vec: case INDEX_op_andc_vec: + case INDEX_op_bitsel_vec: case INDEX_op_neg_vec: case INDEX_op_not_vec: case INDEX_op_or_vec: @@ -3168,6 +3186,8 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_shrs_vec: case INDEX_op_sars_vec: return C_O1_I2(v, v, r); + case INDEX_op_bitsel_vec: + return C_O1_I3(v, v, v, v); default: g_assert_not_reached(); diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index a79f4f187a..527ada0f63 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -157,7 +157,7 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_mul_vec 1 #define TCG_TARGET_HAS_sat_vec 0 #define TCG_TARGET_HAS_minmax_vec 1 -#define TCG_TARGET_HAS_bitsel_vec 0 +#define TCG_TARGET_HAS_bitsel_vec 1 #define TCG_TARGET_HAS_cmpsel_vec 0 /* used for function call generation */ From ea3f2af8f1b87d7bced9b75ef2e788b66ec49961 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 14 Sep 2020 20:44:53 -0700 Subject: [PATCH 28/28] tcg/s390x: Implement TCG_TARGET_HAS_cmpsel_vec This is via expansion; don't actually set TCG_TARGET_HAS_cmpsel_vec. Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index ae132e1b75..8938c446c8 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -2869,6 +2869,7 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_xor_vec: return 1; case INDEX_op_cmp_vec: + case INDEX_op_cmpsel_vec: case INDEX_op_rotrv_vec: return -1; case INDEX_op_mul_vec: @@ -2931,6 +2932,21 @@ static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, } } +static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, + TCGv_vec c1, TCGv_vec c2, + TCGv_vec v3, TCGv_vec v4, TCGCond cond) +{ + TCGv_vec t = tcg_temp_new_vec(type); + + if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) { + /* Invert the sense of the compare by swapping arguments. */ + tcg_gen_bitsel_vec(vece, v0, t, v4, v3); + } else { + tcg_gen_bitsel_vec(vece, v0, t, v3, v4); + } + tcg_temp_free_vec(t); +} + static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc) { @@ -2972,7 +2988,7 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { va_list va; - TCGv_vec v0, v1, v2, t0; + TCGv_vec v0, v1, v2, v3, v4, t0; va_start(va, a0); v0 = temp_tcgv_vec(arg_temp(a0)); @@ -2984,6 +3000,12 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); break; + case INDEX_op_cmpsel_vec: + v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); + v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); + expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg)); + break; + case INDEX_op_rotrv_vec: t0 = tcg_temp_new_vec(type); tcg_gen_neg_vec(vece, t0, v2);