Skip to content

Commit 9445212

Browse files
YO4YO4
committed
tailcall support based on OPT_CALL_THREADED_CODE
This adds OPT_TAILCALL_THREADED_CODE enabled via `-DOPT_THREADED_CODE=3`. Developed on mswin64, not tested on other platforms. note: using [[msvc::musttail]] attribute requires -std:clatest compiler option.
1 parent 6ada33f commit 9445212

12 files changed

Lines changed: 94 additions & 34 deletions

File tree

‎compile.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,7 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node)
978978
static int
979979
rb_iseq_translate_threaded_code(rb_iseq_t *iseq)
980980
{
981-
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
981+
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
982982
const void * const *table = rb_vm_get_insns_address_table();
983983
unsigned int i;
984984
VALUE *encoded = (VALUE *)ISEQ_BODY(iseq)->iseq_encoded;
@@ -1009,7 +1009,7 @@ rb_iseq_original_iseq(const rb_iseq_t *iseq) /* cold path */
10091009
original_code = ISEQ_ORIGINAL_ISEQ_ALLOC(iseq, ISEQ_BODY(iseq)->iseq_size);
10101010
MEMCPY(original_code, ISEQ_BODY(iseq)->iseq_encoded, VALUE, ISEQ_BODY(iseq)->iseq_size);
10111011

1012-
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
1012+
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
10131013
{
10141014
unsigned int i;
10151015

‎insns.def‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -871,16 +871,18 @@ sendforward
871871
// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci);
872872
// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);
873873
{
874-
struct rb_forwarding_call_data adjusted_cd;
875-
struct rb_callinfo adjusted_ci;
874+
{
875+
struct rb_forwarding_call_data adjusted_cd;
876+
struct rb_callinfo adjusted_ci;
876877

877-
VALUE bh = vm_caller_setup_fwd_args(ec, GET_CFP(), cd, blockiseq, 0, &adjusted_cd, &adjusted_ci);
878+
VALUE bh = vm_caller_setup_fwd_args(ec, GET_CFP(), cd, blockiseq, 0, &adjusted_cd, &adjusted_ci);
878879

879-
val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
880-
JIT_EXEC(ec, val);
880+
val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
881+
JIT_EXEC(ec, val);
881882

882-
if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
883-
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
883+
if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
884+
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
885+
}
884886
}
885887

886888
if (UNDEF_P(val)) {
@@ -1112,16 +1114,18 @@ invokesuperforward
11121114
// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci);
11131115
// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);
11141116
{
1115-
struct rb_forwarding_call_data adjusted_cd;
1116-
struct rb_callinfo adjusted_ci;
1117+
{
1118+
struct rb_forwarding_call_data adjusted_cd;
1119+
struct rb_callinfo adjusted_ci;
11171120

1118-
VALUE bh = vm_caller_setup_fwd_args(ec, GET_CFP(), cd, blockiseq, 1, &adjusted_cd, &adjusted_ci);
1121+
VALUE bh = vm_caller_setup_fwd_args(ec, GET_CFP(), cd, blockiseq, 1, &adjusted_cd, &adjusted_ci);
11191122

1120-
val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
1121-
JIT_EXEC(ec, val);
1123+
val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
1124+
JIT_EXEC(ec, val);
11221125

1123-
if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
1124-
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
1126+
if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
1127+
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
1128+
}
11251129
}
11261130

11271131
if (UNDEF_P(val)) {
@@ -1175,7 +1179,7 @@ leave
11751179
rb_ec_thread_ptr(ec)->retval = val;
11761180
return 0;
11771181
#else
1178-
return val;
1182+
RETURN_EXEC_CORE(val);
11791183
#endif
11801184
}
11811185
else {
@@ -1682,7 +1686,7 @@ opt_invokebuiltin_delegate_leave
16821686
rb_ec_thread_ptr(ec)->retval = val;
16831687
return 0;
16841688
#else
1685-
return val;
1689+
RETURN_EXEC_CORE(val);
16861690
#endif
16871691
}
16881692
else {

‎iseq.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3843,7 +3843,7 @@ rb_free_encoded_insn_data(void)
38433843
void
38443844
rb_vm_encoded_insn_data_table_init(void)
38453845
{
3846-
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
3846+
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
38473847
const void * const *table = rb_vm_get_insns_address_table();
38483848
#define INSN_CODE(insn) ((VALUE)table[insn])
38493849
#else
@@ -3928,7 +3928,7 @@ rb_vm_insn_addr2opcode(const void *addr)
39283928
int
39293929
rb_vm_insn_decode(const VALUE encoded)
39303930
{
3931-
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
3931+
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
39323932
int insn = rb_vm_insn_addr2insn((void *)encoded);
39333933
#else
39343934
int insn = (int)encoded;

‎jit.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc)
6161
{
6262
// YJIT should only use iseqs after AST to bytecode compilation.
6363
// (Certain non-default interpreter configurations never set ISEQ_TRANSLATED)
64-
if (OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE) {
64+
if (OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE) {
6565
RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED));
6666
}
6767

‎tool/ruby_vm/views/_trace_instruction.erb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,7 @@ INSN_ENTRY(<%= insn.name %>)
1717
% end
1818
<%= 'ADD_PC(1);' if insn.name == 'trace_opt_neq' %>
1919
DISPATCH_ORIGINAL_INSN(<%= jump_dest || insn.jump_destination %>);
20+
#if !(OPT_DIRECT_THREADED_CODE || OPT_TOKEN_THREADED_CODE || OPT_TAILCALL_THREADED_CODE)
2021
END_INSN(<%= insn.name %>);
22+
#endif
2123
}

‎tool/ruby_vm/views/_zjit_instruction.erb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ INSN_ENTRY(<%= insn.name %>)
66
START_OF_ORIGINAL_INSN(<%= insn.name %>);
77
rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec);
88
DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>);
9+
#if !(OPT_DIRECT_THREADED_CODE || OPT_TOKEN_THREADED_CODE || OPT_TAILCALL_THREADED_CODE)
910
END_INSN(<%= insn.name %>);
11+
#endif
1012
}
1113

1214
#endif

‎vm.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4465,6 +4465,8 @@ Init_VM(void)
44654465
rb_ary_push(opts, rb_str_new2("token threaded code"));
44664466
#elif OPT_CALL_THREADED_CODE
44674467
rb_ary_push(opts, rb_str_new2("call threaded code"));
4468+
#elif OPT_TAILCALL_THREADED_CODE
4469+
rb_ary_push(opts, rb_str_new2("tailcall threaded code"));
44684470
#endif
44694471

44704472
#if OPT_OPERANDS_UNIFICATION

‎vm_core.h‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,16 @@ void *rb_register_sigaltstack(void *);
179179
#include THREAD_IMPL_H
180180
#define RUBY_VM_THREAD_MODEL 2
181181

182+
#if defined(__clang__) && __clang_major__ >= 13
183+
# define RB_VM_MUSTTAIL __attribute__((musttail))
184+
#elif defined(__GNUC__) && __GNUC__ >= 15
185+
# define RB_VM_MUSTTAIL __attribute__((musttail))
186+
#elif defined(_MSC_VER) && _MSC_VER >= 1944 && defined(__STDC_VERSION__ ) && __STDC_VERSION__ >= 202312L
187+
# define RB_VM_MUSTTAIL [[msvc::musttail]]
188+
#else
189+
# undef RB_VM_MUSTTAIL
190+
#endif
191+
182192
/*****************/
183193
/* configuration */
184194
/*****************/
@@ -205,11 +215,18 @@ void *rb_register_sigaltstack(void *);
205215

206216
/* call threaded code */
207217
#if OPT_CALL_THREADED_CODE
208-
#if OPT_DIRECT_THREADED_CODE
209-
#undef OPT_DIRECT_THREADED_CODE
210-
#endif /* OPT_DIRECT_THREADED_CODE */
218+
#if OPT_TAILCALL_THREADED_CODE
219+
#undef OPT_TAILCALL_THREADED_CODE
220+
#endif /* OPT_TAILCALL_THREADED_CODE */
211221
#endif /* OPT_CALL_THREADED_CODE */
212222

223+
/* tailcall threaded code */
224+
#if OPT_TAILCALL_THREADED_CODE
225+
#ifndef RB_VM_MUSTTAIL
226+
#error musttail attribute not supported by this compiler.
227+
#endif
228+
#endif
229+
213230
void rb_vm_encoded_insn_data_table_init(void);
214231
typedef unsigned long rb_num_t;
215232
typedef signed long rb_snum_t;

‎vm_exec.c‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ static void vm_analysis_insn(int insn);
4040
#endif
4141
/* #define DECL_SC_REG(r, reg) VALUE reg_##r */
4242

43-
#if !OPT_CALL_THREADED_CODE
43+
#if !(OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE)
4444
static VALUE
4545
vm_exec_core(rb_execution_context_t *ec)
4646
{
@@ -115,7 +115,7 @@ rb_vm_get_insns_address_table(void)
115115
return (const void **)vm_exec_core(0);
116116
}
117117

118-
#else /* OPT_CALL_THREADED_CODE */
118+
#else /* OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE */
119119

120120
#include "vm.inc"
121121
#include "vmtc.inc"
@@ -126,6 +126,14 @@ rb_vm_get_insns_address_table(void)
126126
return (const void **)insns_address_table;
127127
}
128128

129+
#if OPT_TAILCALL_THREADED_CODE
130+
static VALUE
131+
vm_exec_core(rb_execution_context_t *ec)
132+
{
133+
register rb_control_frame_t *reg_cfp = ec->cfp;
134+
return (VALUE)((rb_insn_func_t) (*GET_PC()))(ec, reg_cfp);
135+
}
136+
#else
129137
static VALUE
130138
vm_exec_core(rb_execution_context_t *ec)
131139
{
@@ -152,3 +160,4 @@ vm_exec_core(rb_execution_context_t *ec)
152160
}
153161
}
154162
#endif
163+
#endif

‎vm_exec.h‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,23 +40,42 @@ typedef rb_iseq_t *ISEQ;
4040
#if defined(DISPATCH_XXX)
4141
error !
4242
/************************************************/
43-
#elif OPT_CALL_THREADED_CODE
43+
#elif OPT_CALL_THREADED_CODE || OPT_TAILCALL_THREADED_CODE
4444

4545
#define LABEL(x) insn_func_##x
4646
#define ELABEL(x)
4747
#define LABEL_PTR(x) &LABEL(x)
4848

4949
#define INSN_ENTRY(insn) \
5050
static rb_control_frame_t * \
51-
FUNC_FASTCALL(LABEL(insn))(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp) {
51+
FUNC_FASTCALL(LABEL(insn))(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
5252

53-
#define END_INSN(insn) return reg_cfp;}
53+
#if OPT_CALL_THREADED_CODE
54+
#define END_INSN(insn) return reg_cfp;
5455

5556
#define NEXT_INSN() return reg_cfp;
5657

5758
#define START_OF_ORIGINAL_INSN(x) /* ignore */
5859
#define DISPATCH_ORIGINAL_INSN(x) return LABEL(x)(ec, reg_cfp);
5960

61+
#elif OPT_TAILCALL_THREADED_CODE
62+
static NOINLINE(rb_control_frame_t *
63+
FUNC_FASTCALL(terminate_tailcall)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp))
64+
{
65+
return reg_cfp;
66+
}
67+
68+
#define END_INSN(insn) \
69+
if (!reg_cfp) RETURN_EXEC_CORE(Qnil); \
70+
RB_VM_MUSTTAIL return ((rb_insn_func_t) (*GET_PC()))(ec, reg_cfp);
71+
72+
#define NEXT_INSN() RB_VM_MUSTTAIL return ((rb_insn_func_t) (*GET_PC()))(ec, reg_cfp);
73+
74+
#define RETURN_EXEC_CORE(val) RB_VM_MUSTTAIL return terminate_tailcall(ec, (rb_control_frame_t *)(val))
75+
76+
#define START_OF_ORIGINAL_INSN(x) /* ignore */
77+
#define DISPATCH_ORIGINAL_INSN(x) RB_VM_MUSTTAIL return LABEL(x)(ec, reg_cfp);
78+
#endif
6079
/************************************************/
6180
#elif OPT_TOKEN_THREADED_CODE || OPT_DIRECT_THREADED_CODE
6281
/* threaded code with gcc */
@@ -123,6 +142,8 @@ error !
123142

124143
#define NEXT_INSN() TC_DISPATCH(__NEXT_INSN__)
125144

145+
#define RETURN_EXEC_CORE(val) return (val)
146+
126147
/************************************************/
127148
#else /* no threaded code */
128149
/* most common method */
@@ -147,6 +168,8 @@ default: \
147168

148169
#define NEXT_INSN() goto first
149170

171+
#define RETURN_EXEC_CORE(val) return (val)
172+
150173
#endif
151174

152175
#ifndef START_OF_ORIGINAL_INSN
@@ -162,7 +185,7 @@ default: \
162185
return 0; \
163186
} while (0)
164187
#else
165-
#define THROW_EXCEPTION(exc) return (VALUE)(exc)
188+
#define THROW_EXCEPTION(exc) RETURN_EXEC_CORE((VALUE)(exc))
166189
#endif
167190

168191
// Run the interpreter from the JIT

0 commit comments

Comments
 (0)