From cea6a9a6b7a605bb58798eee4ba1b068d7f893eb Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 12 Dec 2024 16:19:43 -0500 Subject: [PATCH] codegen: reduce recursion in cfunction generation (#56806) The regular code-path for this was only missing the age_ok handling, so add in that handling so we can delete the custom path here for the test and some of the brokenness that implied. --- src/aotcompile.cpp | 2 +- src/cgutils.cpp | 4 +- src/codegen.cpp | 240 ++++++++++++++--------------------------- src/gf.c | 6 +- src/jitlayers.cpp | 83 ++++++++------ src/jitlayers.h | 7 +- src/julia_internal.h | 2 +- src/precompile_utils.c | 4 +- test/llvmcall.jl | 2 +- 9 files changed, 146 insertions(+), 204 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index ba9a3c05e41ff..198d7490cb092 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -505,7 +505,7 @@ static void compile_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; // TODO: maybe this can be cached in codeinst->specfptr? - emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke, 0, 0); + emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); preal_decl = ""; // no need to fixup the name } if (!preal_decl.empty()) { diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9e170555e6149..dd6ced04673e6 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2279,7 +2279,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, const jl_cgval_t argv[3] = { cmp, lhs, rhs }; jl_cgval_t ret; if (modifyop) { - ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, nullptr); } else { if (trim_may_error(ctx.params->trim)) { @@ -4018,7 +4018,7 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, emit_lockstate_value(ctx, needlock, false); const jl_cgval_t argv[3] = { cmp, oldval, rhs }; if (modifyop) { - rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, nullptr); } else { if (trim_may_error(ctx.params->trim)) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 21591acedc632..fbd990857dd94 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2074,7 +2074,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable=false); -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); @@ -2141,18 +2141,18 @@ static jl_value_t *StackFrame( return frame; } -static void push_frames(jl_codectx_t &ctx, jl_method_instance_t *caller, jl_method_instance_t *callee, int no_debug=false) +static void push_frames(jl_codectx_t &ctx, jl_method_instance_t *caller, jl_method_instance_t *callee) { CallFrames frames; auto it = ctx.emission_context.enqueuers.find(callee); if (it != ctx.emission_context.enqueuers.end()) return; - if (no_debug) { // Used in tojlinvoke + auto DL = ctx.builder.getCurrentDebugLocation(); + if (caller == nullptr || !DL) { // Used in various places frames.push_back({ctx.funcName, "", 0}); ctx.emission_context.enqueuers.insert({callee, {caller, std::move(frames)}}); return; } - auto DL = ctx.builder.getCurrentDebugLocation(); auto filename = std::string(DL->getFilename()); auto line = DL->getLine(); auto fname = std::string(DL->getScope()->getSubprogram()->getName()); @@ -5205,14 +5205,12 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, trampoline); } -static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs) +static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, ArrayRef argv, size_t nargs) { ++EmittedSpecfunCalls; // emit specialized call site bool gcstack_arg = JL_FEAT_TEST(ctx, gcstack_arg); - FunctionType *cft = returninfo.decl.getFunctionType(); - size_t nfargs = cft->getNumParams(); + size_t nfargs = returninfo.decl.getFunctionType()->getNumParams(); SmallVector argvals(nfargs); unsigned idx = 0; AllocaInst *result = nullptr; @@ -5304,23 +5302,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos idx++; } assert(idx == nfargs); - Value *TheCallee = returninfo.decl.getCallee(); - if (fromexternal) { - std::string namep("p"); - namep += cast(returninfo.decl.getCallee())->getName(); - GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); - if (GV == nullptr) { - GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, - GlobalVariable::ExternalLinkage, - Constant::getNullValue(TheCallee->getType()), - namep); - ctx.emission_context.external_fns[std::make_tuple(fromexternal, true)] = GV; - } - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - TheCallee = ai.decorateInst(ctx.builder.CreateAlignedLoad(TheCallee->getType(), GV, Align(sizeof(void*)))); - setName(ctx.emission_context, TheCallee, namep); - } - CallInst *call = ctx.builder.CreateCall(cft, TheCallee, argvals); + CallInst *call = ctx.builder.CreateCall(returninfo.decl, argvals); call->setAttributes(returninfo.attrs); if (gcstack_arg && ctx.emission_context.use_swiftcc) call->setCallingConv(CallingConv::Swift); @@ -5362,7 +5344,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, llvm::Value *callee, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *nreturn_roots, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *nreturn_roots, jl_value_t *inferred_retty, Value *age_ok) { ++EmittedSpecfunCalls; // emit specialized call site @@ -5370,21 +5352,52 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos jl_returninfo_t returninfo = get_specsig_function(ctx, jl_Module, callee, specFunctionObject, specTypes, jlretty, is_opaque_closure, gcstack_arg); *cc = returninfo.cc; *nreturn_roots = returninfo.return_roots; - jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, specTypes, jlretty, returninfo, fromexternal, argv, nargs); + if (fromexternal) { + std::string namep("p"); + Value *TheCallee = returninfo.decl.getCallee(); + namep += cast(TheCallee)->getName(); + GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); + if (GV == nullptr) { + GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, + GlobalVariable::ExternalLinkage, + Constant::getNullValue(TheCallee->getType()), + namep); + ctx.emission_context.external_fns[std::make_tuple(fromexternal, true)] = GV; + } + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + TheCallee = ai.decorateInst(ctx.builder.CreateAlignedLoad(TheCallee->getType(), GV, Align(sizeof(void*)))); + setName(ctx.emission_context, TheCallee, namep); + returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), TheCallee); + } + if (age_ok) { + std::string funcName(specFunctionObject); + funcName += "_gfthunk"; + Function *gf_thunk = Function::Create(returninfo.decl.getFunctionType(), + GlobalVariable::InternalLinkage, funcName, jl_Module); + jl_init_function(gf_thunk, ctx.emission_context.TargetTriple); + gf_thunk->setAttributes(AttributeList::get(gf_thunk->getContext(), {returninfo.attrs, gf_thunk->getAttributes()})); + // build a specsig -> jl_apply_generic converter thunk + // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), + // but which has the signature of a specsig + emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, specTypes, jlretty, is_opaque_closure, nargs, ctx.emission_context, + prepare_call(jlapplygeneric_func)); + returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), ctx.builder.CreateSelect(age_ok, returninfo.decl.getCallee(), gf_thunk)); + } + jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, specTypes, jlretty, returninfo, argv, nargs); // see if inference has a different / better type for the call than the lambda return update_julia_type(ctx, retval, inferred_retty); } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty, Value *age_ok) { bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; return emit_call_specfun_other(ctx, is_opaque_closure, mi->specTypes, jlretty, NULL, - specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty); + specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty, age_ok); } static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_value_t *inferred_retty, Value *age_ok) { Value *theFptr; if (fromexternal) { @@ -5407,6 +5420,8 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty theFptr = jl_Module->getOrInsertFunction(specFunctionObject, ctx.types().T_jlfunc).getCallee(); addRetAttr(cast(theFptr), Attribute::NonNull); } + if (age_ok) + theFptr = ctx.builder.CreateSelect(age_ok, theFptr, prepare_call(jlapplygeneric_func)); Value *ret = emit_jlcall(ctx, FunctionCallee(ctx.types().T_jlfunc, theFptr), nullptr, argv, nargs, julia_call); return update_julia_type(ctx, mark_julia_type(ctx, ret, true, jlretty), inferred_retty); } @@ -5425,10 +5440,10 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } - return emit_invoke(ctx, lival, argv, nargs, rt); + return emit_invoke(ctx, lival, argv, nargs, rt, nullptr); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok) { ++EmittedInvokes; bool handled = false; @@ -5451,12 +5466,12 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR FunctionType *ft = ctx.f->getFunctionType(); StringRef protoname = ctx.f->getName(); if (ft == ctx.types().T_jlfunc) { - result = emit_call_specfun_boxed(ctx, ctx.rettype, protoname, nullptr, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, ctx.rettype, protoname, nullptr, argv, nargs, rt, age_ok); handled = true; } else if (ft != ctx.types().T_jlfuncparams) { unsigned return_roots = 0; - result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt, age_ok); handled = true; } } @@ -5516,9 +5531,9 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; unsigned return_roots = 0; if (specsig) - result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt, age_ok); else - result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt, age_ok); handled = true; if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); @@ -5542,8 +5557,8 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR print_stacktrace(ctx, ctx.params->trim); } } - Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); - result = mark_julia_type(ctx, r, true, rt); + Value *r = age_ok ? emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, julia_call) : emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); + result = mark_julia_type(ctx, r, true, age_ok ? (jl_value_t*)jl_any_type : rt); } if (result.typ == jl_bottom_type) { #ifndef JL_NDEBUG @@ -5636,7 +5651,7 @@ static jl_cgval_t emit_specsig_oc_call(jl_codectx_t &ctx, jl_value_t *oc_type, j Value *specptr = emit_unbox(ctx, ctx.types().T_size, closure_specptr, (jl_value_t*)jl_long_type); JL_GC_PUSH1(&sigtype); jl_cgval_t r = emit_call_specfun_other(ctx, true, sigtype, oc_rett, specptr, "", NULL, argv, nargs, - &cc, &return_roots, oc_rett); + &cc, &return_roots, oc_rett, nullptr); JL_GC_POP(); return r; } @@ -7189,8 +7204,7 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M name, M); jl_init_function(f, params.TargetTriple); if (trim_may_error(params.params->trim)) { - // TODO: Debuginfo! - push_frames(ctx, ctx.linfo, codeinst->def, 1); + push_frames(ctx, ctx.linfo, codeinst->def); } jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); @@ -7232,11 +7246,10 @@ void emit_specsig_to_fptr1( jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, size_t nargs, jl_codegen_params_t ¶ms, - Function *target, - size_t min_world, size_t max_world) + Function *target) { ++EmittedCFuncInvalidates; - jl_codectx_t ctx(gf_thunk->getParent()->getContext(), params, min_world, max_world); + jl_codectx_t ctx(gf_thunk->getParent()->getContext(), params, 0, 0); ctx.f = gf_thunk; BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", gf_thunk); @@ -7353,12 +7366,11 @@ void emit_specsig_to_fptr1( } } -static Function* gen_cfun_wrapper( +static Function *gen_cfun_wrapper( Module *into, jl_codegen_params_t ¶ms, const function_sig_t &sig, jl_value_t *ff, const char *aliasname, jl_value_t *declrt, jl_method_instance_t *lam, - jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, jl_array_t **closure_types, - size_t min_world, size_t max_world) + jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, jl_array_t **closure_types) { ++GeneratedCFuncWrappers; // Generate a c-callable wrapper @@ -7366,36 +7378,17 @@ static Function* gen_cfun_wrapper( size_t nargs = sig.nccallargs; const char *name = "cfunction"; size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_code_instance_t *codeinst = NULL; bool nest = (!ff || unionall_env); jl_value_t *astrt = (jl_value_t*)jl_any_type; - void *callptr = NULL; - jl_callptr_t invoke = NULL; - int calltype = 0; if (aliasname) name = aliasname; else if (lam) name = jl_symbol_name(lam->def.method->name); - if (lam && params.cache) { + + jl_code_instance_t *codeinst = NULL; + if (lam) { // TODO: this isn't ideal to be unconditionally calling type inference (and compile) from here - codeinst = jl_compile_method_internal(lam, world); - uint8_t specsigflags; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - assert(invoke); - if (invoke == jl_fptr_args_addr) { - callptr = fptr; - calltype = 1; - } - else if (invoke == jl_fptr_const_return_addr) { - // don't need the fptr - callptr = (void*)codeinst->rettype_const; - calltype = 2; - } - else if (specsigflags & 0b1) { - callptr = fptr; - calltype = 3; - } + codeinst = jl_type_infer(lam, world, SOURCE_MODE_NOT_REQUIRED); astrt = codeinst->rettype; if (astrt != (jl_value_t*)jl_bottom_type && jl_type_intersection(astrt, declrt) == jl_bottom_type) { @@ -7471,7 +7464,7 @@ static Function* gen_cfun_wrapper( jl_init_function(cw, params.TargetTriple); cw->setAttributes(AttributeList::get(M->getContext(), {attributes, cw->getAttributes()})); - jl_codectx_t ctx(M->getContext(), params, min_world, max_world); + jl_codectx_t ctx(M->getContext(), params, 0, 0); ctx.f = cw; ctx.name = name; ctx.funcName = name; @@ -7490,8 +7483,8 @@ static Function* gen_cfun_wrapper( prepare_global_in(jl_Module, jlgetworld_global), ctx.types().alignof_ptr); cast(world_v)->setOrdering(AtomicOrdering::Acquire); - Value *age_ok = NULL; - if (calltype) { + Value *age_ok = nullptr; + if (codeinst) { LoadInst *lam_max = ctx.builder.CreateAlignedLoad( ctx.types().T_size, emit_ptrgep(ctx, literal_pointer_val(ctx, (jl_value_t*)codeinst), offsetof(jl_code_instance_t, max_world)), @@ -7655,79 +7648,9 @@ static Function* gen_cfun_wrapper( // Create the call bool jlfunc_sret; jl_cgval_t retval; - if (calltype == 2) { - nargs = 0; // arguments not needed -- TODO: not really true, should emit an age_ok test and jlcall - (void)nargs; // silence unused variable warning - jlfunc_sret = false; - retval = mark_julia_const(ctx, (jl_value_t*)callptr); - } - else if (calltype == 0 || calltype == 1) { - // emit a jlcall - jlfunc_sret = false; - Function *theFptr = NULL; - if (calltype == 1) { - StringRef fname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)callptr, invoke, codeinst); - theFptr = cast_or_null(jl_Module->getNamedValue(fname)); - if (!theFptr) { - theFptr = Function::Create(ctx.types().T_jlfunc, GlobalVariable::ExternalLinkage, - fname, jl_Module); - jl_init_function(theFptr, ctx.emission_context.TargetTriple); - jl_name_jlfunc_args(ctx.emission_context, theFptr); - addRetAttr(theFptr, Attribute::NonNull); - } - else { - assert(theFptr->getFunctionType() == ctx.types().T_jlfunc); - } - } - BasicBlock *b_generic, *b_jlcall, *b_after; - Value *ret_jlcall; - if (age_ok) { - assert(theFptr); - b_generic = BasicBlock::Create(ctx.builder.getContext(), "generic", cw); - b_jlcall = BasicBlock::Create(ctx.builder.getContext(), "apply", cw); - b_after = BasicBlock::Create(ctx.builder.getContext(), "after", cw); - ctx.builder.CreateCondBr(age_ok, b_jlcall, b_generic); - ctx.builder.SetInsertPoint(b_jlcall); - // for jlcall, we need to pass the function object even if it is a ghost. - Value *theF = boxed(ctx, inputargs[0]); - assert(theF); - ret_jlcall = emit_jlcall(ctx, theFptr, theF, ArrayRef(inputargs).drop_front(), nargs, julia_call); - ctx.builder.CreateBr(b_after); - ctx.builder.SetInsertPoint(b_generic); - } - Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, julia_call); - if (age_ok) { - ctx.builder.CreateBr(b_after); - ctx.builder.SetInsertPoint(b_after); - PHINode *retphi = ctx.builder.CreatePHI(ctx.types().T_prjlvalue, 2); - retphi->addIncoming(ret_jlcall, b_jlcall); - retphi->addIncoming(ret, b_generic); - ret = retphi; - } - retval = mark_julia_type(ctx, ret, true, astrt); - } - else { - bool is_opaque_closure = jl_is_method(lam->def.value) && lam->def.method->is_for_opaque_closure; - assert(calltype == 3); - // emit a specsig call - StringRef protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)callptr, invoke, codeinst); - bool gcstack_arg = JL_FEAT_TEST(ctx, gcstack_arg); - jl_returninfo_t returninfo = get_specsig_function(ctx, M, NULL, protoname, lam->specTypes, astrt, is_opaque_closure, gcstack_arg); - if (age_ok) { - funcName += "_gfthunk"; - Function *gf_thunk = Function::Create(returninfo.decl.getFunctionType(), - GlobalVariable::InternalLinkage, funcName, M); - jl_init_function(gf_thunk, ctx.emission_context.TargetTriple); - gf_thunk->setAttributes(AttributeList::get(M->getContext(), {returninfo.attrs, gf_thunk->getAttributes()})); - // build a specsig -> jl_apply_generic converter thunk - // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), - // but which has the signature of a specsig - emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, lam->specTypes, codeinst->rettype, is_opaque_closure, nargs + 1, ctx.emission_context, - prepare_call_in(gf_thunk->getParent(), jlapplygeneric_func), min_world, max_world); - returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), ctx.builder.CreateSelect(age_ok, returninfo.decl.getCallee(), gf_thunk)); - } - retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, codeinst->rettype, returninfo, nullptr, inputargs, nargs + 1); - jlfunc_sret = (returninfo.cc == jl_returninfo_t::SRet); + if (codeinst) { + retval = emit_invoke(ctx, mark_julia_const(ctx, (jl_value_t*)codeinst), inputargs, nargs + 1, astrt, age_ok); + jlfunc_sret = retval.V && isa(retval.V) && !retval.TIndex && retval.inline_roots.empty(); if (jlfunc_sret && sig.sret) { // fuse the two sret together assert(retval.ispointer()); @@ -7737,6 +7660,12 @@ static Function* gen_cfun_wrapper( result->eraseFromParent(); } } + else { + // emit a dispatch + jlfunc_sret = false; + Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, julia_call); + retval = mark_julia_type(ctx, ret, true, astrt); + } // inline a call to typeassert here, if required emit_typecheck(ctx, retval, declrt, "cfunction"); @@ -7909,15 +7838,13 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con } } size_t world = jl_atomic_load_acquire(&jl_world_counter); - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; // try to look up this function for direct invoking - jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0) : NULL; + jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, 0) : NULL; Value *F = gen_cfun_wrapper( jl_Module, ctx.emission_context, sig, fexpr_rt.constant, NULL, declrt, lam, - unionall_env, sparam_vals, &closure_types, min_valid, max_valid); + unionall_env, sparam_vals, &closure_types); bool outboxed; if (nest) { // F is actually an init_trampoline function that returns the real address @@ -7977,7 +7904,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con // do codegen to create a C-callable alias/wrapper, or if sysimg_handle is set, // restore one from a loaded system image. -const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms) +const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms) { ++GeneratedCCallables; jl_datatype_t *ft = (jl_datatype_t*)jl_tparam0(sigt); @@ -8006,8 +7933,6 @@ const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysi argtypes, NULL, false, CallingConv::C, false, ¶ms); if (sig.err_msg.empty()) { size_t world = jl_atomic_load_acquire(&jl_world_counter); - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; if (sysimg_handle) { // restore a ccallable from the system image void *addr; @@ -8020,9 +7945,9 @@ const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysi } } else { - jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0); + jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, 0); //Safe b/c params holds context lock - gen_cfun_wrapper(unwrap(llvmmod)->getModuleUnlocked(), params, sig, ff, name, declrt, lam, NULL, NULL, NULL, min_valid, max_valid); + gen_cfun_wrapper(llvmmod, params, sig, ff, name, declrt, lam, NULL, NULL, NULL); } JL_GC_POP(); return name; @@ -8084,7 +8009,7 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, j } argv[i] = mark_julia_type(ctx, theArg, true, ty); } - jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, jlretty, f, nullptr, argv, nargs); + jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, jlretty, f, argv, nargs); if (retarg != -1) { Value *theArg; if (retarg == 0) @@ -10040,8 +9965,7 @@ static jl_llvm_functions_t jl_emit_oc_wrapper(orc::ThreadSafeModule &m, jl_codeg size_t nrealargs = jl_nparams(mi->specTypes); emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, mi->specTypes, rettype, true, nrealargs, ctx.emission_context, - prepare_call_in(gf_thunk->getParent(), jlopaque_closure_call_func), // TODO: this could call emit_oc_call directly - ctx.min_world, ctx.max_world); + prepare_call_in(gf_thunk->getParent(), jlopaque_closure_call_func)); // TODO: this could call emit_oc_call directly declarations.specFunctionObject = funcName; } return declarations; diff --git a/src/gf.c b/src/gf.c index bbf065a4fac0d..a5ba309784557 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3084,7 +3084,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc } // compile-time method lookup -jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache) +jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache) { if (jl_has_free_typevars((jl_value_t*)types)) return NULL; // don't poison the cache due to a malformed query @@ -3096,10 +3096,6 @@ jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES size_t max_valid2 = ~(size_t)0; int ambig = 0; jl_value_t *matches = jl_matching_methods(types, jl_nothing, 1, 1, world, &min_valid2, &max_valid2, &ambig); - if (*min_valid < min_valid2) - *min_valid = min_valid2; - if (*max_valid > max_valid2) - *max_valid = max_valid2; if (matches == jl_nothing || jl_array_nrows(matches) != 1 || ambig) return NULL; JL_GC_PUSH1(&matches); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 80867daade267..25494ba57d257 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -382,7 +382,7 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t jl_method_instance_t *mi = codeinst->def; size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; - emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke, 0, 0); + emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); jl_gc_unsafe_leave(ct->ptls, gc_state); preal_decl = ""; // no need to fixup the name } @@ -713,7 +713,7 @@ static void jl_emit_codeinst_to_jit( int waiting = jl_analyze_workqueue(codeinst, params); if (waiting) { auto release = std::move(params.tsctx_lock); // unlock again before moving from it - incompletemodules.insert(std::pair(codeinst, std::make_tuple(std::move(params), waiting))); + incompletemodules.try_emplace(codeinst, std::move(params), waiting); } else { finish_params(result_m.getModuleUnlocked(), params); @@ -760,7 +760,7 @@ static void _jl_compile_codeinst( } -const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); +const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); // compile a C-callable alias extern "C" JL_DLLEXPORT_CODEGEN @@ -774,45 +774,68 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled); if (measure_compile_time_enabled) compiler_start_time = jl_hrtime(); + jl_codegen_params_t *pparams = (jl_codegen_params_t*)p; + DataLayout DL = pparams ? pparams->DL : jl_ExecutionEngine->getDataLayout(); + Triple TargetTriple = pparams ? pparams->TargetTriple : jl_ExecutionEngine->getTargetTriple(); orc::ThreadSafeContext ctx; auto into = unwrap(llvmmod); - jl_codegen_params_t *pparams = (jl_codegen_params_t*)p; orc::ThreadSafeModule backing; + bool success = true; + const char *name = ""; + SmallVector dependencies; if (into == NULL) { - if (!pparams) { - ctx = jl_ExecutionEngine->makeContext(); - } - backing = jl_create_ts_module("cextern", pparams ? pparams->tsctx : ctx, pparams ? pparams->DL : jl_ExecutionEngine->getDataLayout(), pparams ? pparams->TargetTriple : jl_ExecutionEngine->getTargetTriple()); + ctx = pparams ? pparams->tsctx : jl_ExecutionEngine->makeContext(); + backing = jl_create_ts_module("cextern", ctx, DL, TargetTriple); into = &backing; } - bool success = true; - { - auto Lock = into->getContext().getLock(); - Module *M = into->getModuleUnlocked(); - jl_codegen_params_t params(into->getContext(), M->getDataLayout(), Triple(M->getTargetTriple())); - params.imaging_mode = imaging_default(); + { // params scope + jl_codegen_params_t params(into->getContext(), DL, TargetTriple); if (pparams == NULL) { - M->getContext().setDiscardValueNames(true); + params.cache = p == NULL; + params.imaging_mode = imaging_default(); + params.tsctx.getContext()->setDiscardValueNames(true); pparams = ¶ms; } - assert(pparams->tsctx.getContext() == into->getContext().getContext()); - const char *name = jl_generate_ccallable(wrap(into), sysimg, declrt, sigt, *pparams); - if (!sysimg) { - jl_unique_gcsafe_lock lock(extern_c_lock); - if (jl_ExecutionEngine->getGlobalValueAddress(name)) { - success = false; + Module &M = *into->getModuleUnlocked(); + assert(pparams->tsctx.getContext() == &M.getContext()); + name = jl_generate_ccallable(&M, sysimg, declrt, sigt, *pparams); + if (!sysimg && !p) { + { // drop lock to keep analyzer happy (since it doesn't know we have the only reference to it) + auto release = std::move(params.tsctx_lock); } - if (success && p == NULL) { - jl_jit_globals(params.global_targets); - assert(params.workqueue.empty()); - if (params._shared_module) { - jl_ExecutionEngine->optimizeDLSyms(*params._shared_module); // safepoint - jl_ExecutionEngine->addModule(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx)); + { // lock scope + jl_unique_gcsafe_lock lock(extern_c_lock); + if (jl_ExecutionEngine->getGlobalValueAddress(name)) + success = false; + } + params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock + if (success && params.cache) { + for (auto &it : params.workqueue) { + jl_code_instance_t *codeinst = it.first; + JL_GC_PROMISE_ROOTED(codeinst); + dependencies.push_back(codeinst); + recursive_compile_graph(codeinst, nullptr); } + jl_analyze_workqueue(nullptr, params, true); + assert(params.workqueue.empty()); + finish_params(&M, params); } - if (success && llvmmod == NULL) { - jl_ExecutionEngine->optimizeDLSyms(*M); // safepoint - jl_ExecutionEngine->addModule(std::move(*into)); + } + pparams = nullptr; + } + if (!sysimg && success && llvmmod == NULL) { + { // lock scope + jl_unique_gcsafe_lock lock(extern_c_lock); + if (!jl_ExecutionEngine->getGlobalValueAddress(name)) { + for (auto dep : dependencies) + jl_compile_codeinst_now(dep); + { + auto Lock = backing.getContext().getLock(); + jl_ExecutionEngine->optimizeDLSyms(*backing.getModuleUnlocked()); // safepoint + } + jl_ExecutionEngine->addModule(std::move(backing)); + success = jl_ExecutionEngine->getGlobalValueAddress(name); + assert(success); } } } diff --git a/src/jitlayers.h b/src/jitlayers.h index 6665be6a33faa..b6f1e9ec20c6b 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -260,7 +260,7 @@ struct jl_codegen_params_t { bool external_linkage = false; bool imaging_mode; bool use_swiftcc = true; - jl_codegen_params_t(orc::ThreadSafeContext ctx, DataLayout DL, Triple triple) + jl_codegen_params_t(orc::ThreadSafeContext ctx, DataLayout DL, Triple triple) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER : tsctx(std::move(ctx)), tsctx_lock(tsctx.getLock()), DL(std::move(DL)), @@ -271,6 +271,8 @@ struct jl_codegen_params_t { if (TargetTriple.isRISCV()) use_swiftcc = false; } + jl_codegen_params_t(jl_codegen_params_t &&) JL_NOTSAFEPOINT = default; + ~jl_codegen_params_t() JL_NOTSAFEPOINT JL_NOTSAFEPOINT_LEAVE = default; }; jl_llvm_functions_t jl_emit_code( @@ -299,8 +301,7 @@ void emit_specsig_to_fptr1( jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, size_t nargs, jl_codegen_params_t ¶ms, - Function *target, - size_t min_world, size_t max_world) JL_NOTSAFEPOINT; + Function *target) JL_NOTSAFEPOINT; Function *get_or_emit_fptr1(StringRef Name, Module *M) JL_NOTSAFEPOINT; void jl_init_function(Function *F, const Triple &TT) JL_NOTSAFEPOINT; diff --git a/src/julia_internal.h b/src/julia_internal.h index 4741316093f95..e0e8158f8e0d9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1195,7 +1195,7 @@ _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) J JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); -JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache); +JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, int mt_cache); jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp); JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_value_t *owner, jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 81c60ba70d29f..4717351d499b0 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -264,10 +264,8 @@ static void *jl_precompile_(jl_array_t *m, int external_linkage) jl_value_t *item = jl_array_ptr_ref(m, i); if (jl_is_method_instance(item)) { mi = (jl_method_instance_t*)item; - size_t min_world = 0; - size_t max_world = ~(size_t)0; if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->sparam_vals, mi->def.method)) - mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); + mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), 0); if (mi) jl_array_ptr_1d_push(m2, (jl_value_t*)mi); } diff --git a/test/llvmcall.jl b/test/llvmcall.jl index c83ac05b1ec48..ddf66ca680d45 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -157,7 +157,7 @@ module ObjLoadTest nothing end @test_throws(ErrorException("@ccallable was already defined for this method name"), - @eval @ccallable Cvoid jl_the_callback(not_the_method::Int) = "other") + @eval @ccallable String jl_the_callback(not_the_method::Int) = "other") # Make sure everything up until here gets compiled @test jl_the_callback() === nothing @test jl_the_callback(1) == "other"