diff --git a/target-alpha/translate.c b/target-alpha/translate.c index 5b86992dd3..8c2183a418 100644 --- a/target-alpha/translate.c +++ b/target-alpha/translate.c @@ -460,12 +460,16 @@ static bool use_goto_tb(DisasContext *ctx, uint64_t dest) || ctx->singlestep_enabled || singlestep) { return false; } +#ifndef CONFIG_USER_ONLY /* If the destination is in the superpage, the page perms can't change. */ if (in_superpage(ctx, dest)) { return true; } /* Check for the dest on the same page as the start of the TB. */ return ((ctx->tb->pc ^ dest) & TARGET_PAGE_MASK) == 0; +#else + return true; +#endif } static ExitStatus gen_bdirect(DisasContext *ctx, int ra, int32_t disp) diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c index 24f5e177dd..5526bbda2c 100644 --- a/target-arm/translate-a64.c +++ b/target-arm/translate-a64.c @@ -274,10 +274,12 @@ static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest) return false; } +#ifndef CONFIG_USER_ONLY /* Only link tbs from inside the same guest page */ if ((s->tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) { return false; } +#endif return true; } diff --git a/target-arm/translate.c b/target-arm/translate.c index 34196a8217..a43b1f61cf 100644 --- a/target-arm/translate.c +++ b/target-arm/translate.c @@ -4049,16 +4049,22 @@ static int disas_vfp_insn(DisasContext *s, uint32_t insn) return 0; } +static inline bool use_goto_tb(DisasContext *s, target_ulong dest) +{ +#ifndef CONFIG_USER_ONLY + return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest) { - TranslationBlock *tb; - - tb = s->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(s, dest)) { tcg_gen_goto_tb(n); gen_set_pc_im(s, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)s->tb + n); } else { gen_set_pc_im(s, dest); tcg_gen_exit_tb(0); diff --git a/target-cris/translate.c b/target-cris/translate.c index 9c8ff8f230..f28b1999a7 100644 --- a/target-cris/translate.c +++ b/target-cris/translate.c @@ -520,16 +520,22 @@ static void t_gen_cc_jmp(TCGv pc_true, TCGv pc_false) gen_set_label(l1); } +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - tb = dc->tb; - - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(dc, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(env_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(env_pc, dest); tcg_gen_exit_tb(0); diff --git a/target-i386/translate.c b/target-i386/translate.c index 058d85a1ab..868c26244b 100644 --- a/target-i386/translate.c +++ b/target-i386/translate.c @@ -2085,20 +2085,25 @@ static inline int insn_const_size(TCGMemOp ot) } } +static inline bool use_goto_tb(DisasContext *s, target_ulong pc) +{ +#ifndef CONFIG_USER_ONLY + return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) || + (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) { - TranslationBlock *tb; - target_ulong pc; + target_ulong pc = s->cs_base + eip; - pc = s->cs_base + eip; - tb = s->tb; - /* NOTE: we handle the case where the TB spans two pages here */ - if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) || - (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK)) { + if (use_goto_tb(s, pc)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); gen_jmp_im(eip); - tcg_gen_exit_tb((uintptr_t)tb + tb_num); + tcg_gen_exit_tb((uintptr_t)s->tb + tb_num); } else { /* jump to another page: currently not optimized */ gen_jmp_im(eip); diff --git a/target-lm32/translate.c b/target-lm32/translate.c index 256a51f849..dd972f5b8c 100644 --- a/target-lm32/translate.c +++ b/target-lm32/translate.c @@ -133,16 +133,25 @@ static inline void t_gen_illegal_insn(DisasContext *dc) gen_helper_ill(cpu_env); } +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ + if (unlikely(dc->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - - tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!dc->singlestep_enabled)) { + if (use_goto_tb(dc, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(cpu_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(cpu_pc, dest); if (dc->singlestep_enabled) { diff --git a/target-m68k/translate.c b/target-m68k/translate.c index e2ce6c615e..e46356e44c 100644 --- a/target-m68k/translate.c +++ b/target-m68k/translate.c @@ -852,19 +852,25 @@ static inline void gen_addr_fault(DisasContext *s) } \ } while (0) +static inline bool use_goto_tb(DisasContext *s, uint32_t dest) +{ +#ifndef CONFIG_USER_ONLY + return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + /* Generate a jump to an immediate address. */ static void gen_jmp_tb(DisasContext *s, int n, uint32_t dest) { - TranslationBlock *tb; - - tb = s->tb; if (unlikely(s->singlestep_enabled)) { gen_exception(s, dest, EXCP_DEBUG); - } else if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + } else if (use_goto_tb(s, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_i32(QREG_PC, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)s->tb + n); } else { gen_jmp_im(s, dest); tcg_gen_exit_tb(0); diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c index f944965a14..a7a8ac8f99 100644 --- a/target-microblaze/translate.c +++ b/target-microblaze/translate.c @@ -124,14 +124,21 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index) dc->is_jmp = DISAS_UPDATE; } +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(dc, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(cpu_SR[SR_PC], dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(cpu_SR[SR_PC], dest); tcg_gen_exit_tb(0); diff --git a/target-mips/translate.c b/target-mips/translate.c index a3a05ec66d..ddfb9244d7 100644 --- a/target-mips/translate.c +++ b/target-mips/translate.c @@ -4191,15 +4191,25 @@ static void gen_trap (DisasContext *ctx, uint32_t opc, tcg_temp_free(t1); } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!ctx->singlestep_enabled)) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); gen_save_pc(dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { gen_save_pc(dest); if (ctx->singlestep_enabled) { diff --git a/target-moxie/translate.c b/target-moxie/translate.c index a437e2ab60..58200c25d3 100644 --- a/target-moxie/translate.c +++ b/target-moxie/translate.c @@ -121,17 +121,26 @@ void moxie_translate_init(void) done_init = 1; } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(CPUMoxieState *env, DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; - - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - !ctx->singlestep_enabled) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_i32(cpu_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { tcg_gen_movi_i32(cpu_pc, dest); if (ctx->singlestep_enabled) { diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c index 5d0ab442a8..d4f1f260e4 100644 --- a/target-openrisc/translate.c +++ b/target-openrisc/translate.c @@ -190,15 +190,25 @@ static void check_ov64s(DisasContext *dc) } #endif*/ +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ + if (unlikely(dc->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!dc->singlestep_enabled)) { + if (use_goto_tb(dc, dest)) { tcg_gen_movi_tl(cpu_pc, dest); tcg_gen_goto_tb(n); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(cpu_pc, dest); if (dc->singlestep_enabled) { diff --git a/target-ppc/translate.c b/target-ppc/translate.c index b3860ecdea..d485d7c7cb 100644 --- a/target-ppc/translate.c +++ b/target-ppc/translate.c @@ -3822,19 +3822,29 @@ static inline void gen_update_cfar(DisasContext *ctx, target_ulong nip) #endif } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + /*** Branch ***/ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; if (NARROW_MODE(ctx)) { dest = (uint32_t) dest; } - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!ctx->singlestep_enabled)) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(cpu_nip, dest & ~3); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { tcg_gen_movi_tl(cpu_nip, dest & ~3); if (unlikely(ctx->singlestep_enabled)) { diff --git a/target-s390x/translate.c b/target-s390x/translate.c index c5179fe05d..e99eb5cb01 100644 --- a/target-s390x/translate.c +++ b/target-s390x/translate.c @@ -608,12 +608,17 @@ static void gen_op_calc_cc(DisasContext *s) static int use_goto_tb(DisasContext *s, uint64_t dest) { - /* NOTE: we handle the case where the TB spans two pages here */ - return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) - || (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK)) - && !s->singlestep_enabled - && !(s->tb->cflags & CF_LAST_IO) - && !(s->tb->flags & FLAG_MASK_PER)); + if (unlikely(s->singlestep_enabled) || + (s->tb->cflags & CF_LAST_IO) || + (s->tb->flags & FLAG_MASK_PER)) { + return false; + } +#ifndef CONFIG_USER_ONLY + return (dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) || + (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK); +#else + return true; +#endif } static void account_noninline_branch(DisasContext *s, int cc_op) diff --git a/target-sh4/translate.c b/target-sh4/translate.c index 7c189680a7..53f782c054 100644 --- a/target-sh4/translate.c +++ b/target-sh4/translate.c @@ -205,17 +205,26 @@ static void gen_write_sr(TCGv src) tcg_gen_andi_i32(cpu_sr_t, cpu_sr_t, 1); } -static void gen_goto_tb(DisasContext * ctx, int n, target_ulong dest) +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; + if (unlikely(ctx->singlestep_enabled)) { + return false; + } - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - !ctx->singlestep_enabled) { +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + +static void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) +{ + if (use_goto_tb(ctx, dest)) { /* Use a direct jump if in same page and singlestep not enabled */ tcg_gen_goto_tb(n); tcg_gen_movi_i32(cpu_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { tcg_gen_movi_i32(cpu_pc, dest); if (ctx->singlestep_enabled) diff --git a/target-sparc/translate.c b/target-sparc/translate.c index 7998ff57bf..d154e3f7b6 100644 --- a/target-sparc/translate.c +++ b/target-sparc/translate.c @@ -303,20 +303,30 @@ static inline TCGv gen_dest_gpr(DisasContext *dc, int reg) } } +static inline bool use_goto_tb(DisasContext *s, target_ulong pc, + target_ulong npc) +{ + if (unlikely(s->singlestep)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) && + (npc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong pc, target_ulong npc) { - TranslationBlock *tb; - - tb = s->tb; - if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) && - (npc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) && - !s->singlestep) { + if (use_goto_tb(s, pc, npc)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); tcg_gen_movi_tl(cpu_pc, pc); tcg_gen_movi_tl(cpu_npc, npc); - tcg_gen_exit_tb((uintptr_t)tb + tb_num); + tcg_gen_exit_tb((uintptr_t)s->tb + tb_num); } else { /* jump to another page: currently not optimized */ tcg_gen_movi_tl(cpu_pc, pc); diff --git a/target-tricore/translate.c b/target-tricore/translate.c index 912bf226be..0237e7bea8 100644 --- a/target-tricore/translate.c +++ b/target-tricore/translate.c @@ -3236,15 +3236,25 @@ static inline void gen_save_pc(target_ulong pc) tcg_gen_movi_tl(cpu_PC, pc); } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!ctx->singlestep_enabled)) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); gen_save_pc(dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { gen_save_pc(dest); if (ctx->singlestep_enabled) { diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c index 39af3af05f..307f7b2059 100644 --- a/target-unicore32/translate.c +++ b/target-unicore32/translate.c @@ -1089,15 +1089,21 @@ static void disas_ucf64_insn(CPUUniCore32State *env, DisasContext *s, uint32_t i } } +static inline bool use_goto_tb(DisasContext *s, uint32_t dest) +{ +#ifndef CONFIG_USER_ONLY + return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int n, uint32_t dest) { - TranslationBlock *tb; - - tb = s->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(s, dest)) { tcg_gen_goto_tb(n); gen_set_pc_im(dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)s->tb + n); } else { gen_set_pc_im(dest); tcg_gen_exit_tb(0); diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c index 9894488469..9eac56e2a5 100644 --- a/target-xtensa/translate.c +++ b/target-xtensa/translate.c @@ -418,9 +418,11 @@ static void gen_jump(DisasContext *dc, TCGv dest) static void gen_jumpi(DisasContext *dc, uint32_t dest, int slot) { TCGv_i32 tmp = tcg_const_i32(dest); +#ifndef CONFIG_USER_ONLY if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) { slot = -1; } +#endif gen_jump_slot(dc, tmp, slot); tcg_temp_free(tmp); } @@ -446,9 +448,11 @@ static void gen_callw(DisasContext *dc, int callinc, TCGv_i32 dest) static void gen_callwi(DisasContext *dc, int callinc, uint32_t dest, int slot) { TCGv_i32 tmp = tcg_const_i32(dest); +#ifndef CONFIG_USER_ONLY if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) { slot = -1; } +#endif gen_callw_slot(dc, callinc, tmp, slot); tcg_temp_free(tmp); } diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h index ace39619ef..f217e80747 100644 --- a/tcg/tcg-op.h +++ b/tcg/tcg-op.h @@ -759,9 +759,12 @@ static inline void tcg_gen_exit_tb(uintptr_t val) * * See tcg/README for more info about this TCG operation. * - * NOTE: Direct jumps with goto_tb are only safe within the pages this TB - * resides in because we don't take care of direct jumps when address mapping - * changes, e.g. in tlb_flush(). + * NOTE: In softmmu emulation, direct jumps with goto_tb are only safe within + * the pages this TB resides in because we don't take care of direct jumps when + * address mapping changes, e.g. in tlb_flush(). In user mode, there's only a + * static address translation, so the destination address is always valid, TBs + * are always invalidated properly, and direct jumps are reset when mapping + * changes. */ void tcg_gen_goto_tb(unsigned idx);