optimizer: support callsite annotations of @inline and @noinline

base/compiler/abstractinterpretation.jl

@@ -1784,7 +1784,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState)
                     if isa(fname, Slot)
                         changes = StateUpdate(fname, VarState(Any, false), changes, false)
                     end
-                elseif hd === :inbounds || hd === :meta || hd === :loopinfo || hd === :code_coverage_effect
+                elseif hd === :inbounds || hd === :meta || hd === :loopinfo || hd === :code_coverage_effect || hd === :noinline
                     # these do not generate code
                 else
                     t = abstract_eval_statement(interp, stmt, changes, frame)

base/compiler/optimize.jl

@@ -135,6 +135,8 @@ const SLOT_USEDUNDEF    = 32 # slot has uses that might raise UndefVarError
 # This statement was marked as @inbounds by the user. If replaced by inlining,
 # any contained boundschecks may be removed
 const IR_FLAG_INBOUNDS       = 0x01
+# This statement was marked as @noinline by the user
+const IR_FLAG_NOINLINE       = 0x01 << 7
 # This statement may be removed if its result is unused. In particular it must
 # thus be both pure and effect free.
 const IR_FLAG_EFFECT_FREE    = 0x01 << 4

base/compiler/ssair/driver.jl

@@ -79,6 +79,7 @@ function convert_to_ircode(ci::CodeInfo, code::Vector{Any}, coverage::Bool, narg
     renumber_ir_elements!(code, changemap, labelmap)
 
     inbounds_depth = 0 # Number of stacked inbounds
+    disable_inline = false # whether or not to disable inline optimization
     meta = Any[]
     flags = fill(0x00, length(code))
     for i = 1:length(code)
@@ -93,6 +94,9 @@ function convert_to_ircode(ci::CodeInfo, code::Vector{Any}, coverage::Bool, narg
                 inbounds_depth -= 1
             end
             stmt = nothing
+        elseif isexpr(stmt, :noinline)
+            disable_inline = true
+            stmt = nothing
         else
             stmt = normalize(stmt, meta)
         end
@@ -101,6 +105,10 @@ function convert_to_ircode(ci::CodeInfo, code::Vector{Any}, coverage::Bool, narg
             if inbounds_depth > 0
                 flags[i] |= IR_FLAG_INBOUNDS
             end
+            if disable_inline
+                flags[i] |= IR_FLAG_NOINLINE
+                disable_inline = false;
+            end
         end
     end
     strip_trailing_junk!(ci, code, stmtinfo, flags)

base/compiler/ssair/inlining.jl

@@ -790,7 +790,7 @@ function validate_sparams(sparams::SimpleVector)
 end
 
 function analyze_method!(match::MethodMatch, atypes::Vector{Any},
-                         state::InliningState, @nospecialize(stmttyp))
+                         state::InliningState, @nospecialize(stmttyp), dont_inline::Bool)
     method = match.method
     methsig = method.sig
 
@@ -810,7 +810,7 @@ function analyze_method!(match::MethodMatch, atypes::Vector{Any},
     validate_sparams(match.sparams) || return nothing
 
 
-    if !state.params.inlining
+    if !state.params.inlining || dont_inline
         return compileable_specialization(state.et, match)
     end
 
@@ -1049,10 +1049,20 @@ is_builtin(s::Signature) =
     isa(s.f, Builtin) ||
     s.ft ⊑ Builtin
 
+function check_noinline_flag(ir::IRCode, idx::Int)
+    try
+        (ir.stmts[idx][:flag] & IR_FLAG_NOINLINE) != 0
+    catch BoundsError
+        false
+    end
+
+end
+
 function inline_invoke!(ir::IRCode, idx::Int, sig::Signature, info::InvokeCallInfo,
         state::InliningState, todo::Vector{Pair{Int, Any}})
     stmt = ir.stmts[idx][:inst]
     calltype = ir.stmts[idx][:type]
+    dont_inline = check_noinline_flag(ir, idx)
 
     if !info.match.fully_covers
         # TODO: We could union split out the signature check and continue on
@@ -1064,7 +1074,7 @@ function inline_invoke!(ir::IRCode, idx::Int, sig::Signature, info::InvokeCallIn
     atypes = atypes[4:end]
     pushfirst!(atypes, atype0)
 
-    result = analyze_method!(info.match, atypes, state, calltype)
+    result = analyze_method!(info.match, atypes, state, calltype, dont_inline)
     handle_single_case!(ir, stmt, idx, result, true, todo)
     return nothing
 end
@@ -1164,6 +1174,7 @@ function analyze_single_call!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int
     signature_union = Union{}
     only_method = nothing  # keep track of whether there is one matching method
     too_many = false
+    dont_inline = check_noinline_flag(ir, idx)
     local meth
     local fully_covered = true
     for i in 1:length(infos)
@@ -1192,7 +1203,7 @@ function analyze_single_call!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int
                 fully_covered = false
                 continue
             end
-            case = analyze_method!(match, sig.atypes, state, calltype)
+            case = analyze_method!(match, sig.atypes, state, calltype, dont_inline)
             if case === nothing
                 fully_covered = false
                 continue
@@ -1219,7 +1230,7 @@ function analyze_single_call!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int
             match = meth[1]
         end
         fully_covered = true
-        case = analyze_method!(match, sig.atypes, state, calltype)
+        case = analyze_method!(match, sig.atypes, state, calltype, dont_inline)
         case === nothing && return
         push!(cases, Pair{Any,Any}(match.spec_types, case))
     end
@@ -1280,6 +1291,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState)
         stmt = ir.stmts[idx][:inst]
         calltype = ir.stmts[idx][:type]
         info = ir.stmts[idx][:info]
+        dont_inline = check_noinline_flag(ir, idx)
 
         # Check whether this call was @pure and evaluates to a constant
         if info isa MethodResultPure
@@ -1308,7 +1320,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState)
         end
 
         if isa(info, OpaqueClosureCallInfo)
-            result = analyze_method!(info.match, sig.atypes, state, calltype)
+            result = analyze_method!(info.match, sig.atypes, state, calltype, dont_inline)
             handle_single_case!(ir, stmt, idx, result, false, todo)
             continue
         end

base/compiler/utilities.jl

@@ -59,7 +59,8 @@ end
 
 # Meta expression head, these generally can't be deleted even when they are
 # in a dead branch but can be ignored when analyzing uses/liveness.
-is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta || head === :loopinfo)
+is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta ||
+    head === :loopinfo || head === :noinline)
 
 sym_isless(a::Symbol, b::Symbol) = ccall(:strcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}), a, b) < 0
 

base/compiler/validation.jl

@@ -16,6 +16,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange}(
     :leave => 1:1,
     :pop_exception => 1:1,
     :inbounds => 1:1,
+    :noinline => 0:0,
     :boundscheck => 0:0,
     :copyast => 1:1,
     :meta => 0:typemax(Int),
@@ -141,7 +142,7 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, c::CodeInfo, is_top_
                 head === :const || head === :enter || head === :leave || head === :pop_exception ||
                 head === :method || head === :global || head === :static_parameter ||
                 head === :new || head === :splatnew || head === :thunk || head === :loopinfo ||
-                head === :throw_undef_if_not || head === :code_coverage_effect
+                head === :throw_undef_if_not || head === :code_coverage_effect || head === :noinline
                 validate_val!(x)
             else
                 # TODO: nothing is actually in statement position anymore

base/expr.jl

@@ -184,7 +184,8 @@ Give a hint to the compiler that this function is worth inlining.
 
 Small functions typically do not need the `@inline` annotation,
 as the compiler does it automatically. By using `@inline` on bigger functions,
-an extra nudge can be given to the compiler to inline it.
+an extra nudge can be given to the compiler to inline it. `@inline` can
+only be used at function definitions.
 This is shown in the following example:
 
 ```julia
@@ -206,21 +207,34 @@ Give a hint to the compiler that it should not inline a function.
 
 Small functions are typically inlined automatically.
 By using `@noinline` on small functions, auto-inlining can be
-prevented. This is shown in the following example:
+prevented. `@noinline` can be used both at function definitions and
+at function calls. This is shown in the following examples:
 
 ```julia
 @noinline function smallfunction(x)
     #=
         Function Definition
     =#
 end
-```
 
+@noinline previouslydefinedfunction(x)
+```
 !!! note
     If the function is trivial (for example returning a constant) it might get inlined anyway.
 """
 macro noinline(ex)
-    esc(isa(ex, Expr) ? pushmeta!(ex, :noinline) : ex)
+    if isa(ex, Expr)
+        if ex.head === :call
+            # callsite noinline
+            return Expr(:block,
+                        Expr(:noinline),
+                        esc(ex))
+        else
+            esc(pushmeta!(ex, :noinline))
+        end
+    else
+        esc(ex)
+    end
 end
 
 """

base/meta.jl

@@ -413,6 +413,7 @@ end
 
 _instantiate_type_in_env(x, spsig, spvals) = ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), x, spsig, spvals)
 
-is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta || head === :loopinfo)
+is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta
+                                   || head === :loopinfo || head === :noinline)
 
 end # module

src/ast.scm

@@ -276,7 +276,7 @@
 ;; predicates and accessors
 
 (define (quoted? e)
-  (memq (car e) '(quote top core globalref outerref line break inert meta inbounds loopinfo)))
+  (memq (car e) '(quote top core globalref outerref line break inert meta inbounds noinline loopinfo)))
 (define (quotify e) `',e)
 (define (unquote e)
   (if (and (pair? e) (memq (car e) '(quote inert)))

src/codegen.cpp

@@ -4273,7 +4273,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result)
     jl_value_t **args = (jl_value_t**)jl_array_data(ex->args);
     jl_sym_t *head = ex->head;
     if (head == meta_sym || head == inbounds_sym || head == coverageeffect_sym
-            || head == aliasscope_sym || head == popaliasscope_sym) {
+            || head == aliasscope_sym || head == popaliasscope_sym || head == noinline_sym) {
         // some expression types are metadata and can be ignored
         // in statement position
         return;
@@ -4704,7 +4704,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
     }
     else if (head == leave_sym || head == coverageeffect_sym
             || head == pop_exception_sym || head == enter_sym || head == inbounds_sym
-            || head == aliasscope_sym || head == popaliasscope_sym) {
+            || head == aliasscope_sym || head == popaliasscope_sym || head == noinline_sym) {
         jl_errorf("Expr(:%s) in value position", jl_symbol_name(head));
     }
     else if (head == boundscheck_sym) {

src/interpreter.c

@@ -303,7 +303,7 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s)
         return jl_true;
     }
     else if (head == meta_sym || head == coverageeffect_sym || head == inbounds_sym || head == loopinfo_sym ||
-             head == aliasscope_sym || head == popaliasscope_sym) {
+             head == aliasscope_sym || head == popaliasscope_sym || head == noinline_sym) {
         return jl_nothing;
     }
     else if (head == gc_preserve_begin_sym || head == gc_preserve_end_sym) {

src/julia-syntax.scm

@@ -3336,7 +3336,7 @@ f(x) = yt(x)
          thunk with-static-parameters toplevel-only
          global globalref outerref const-if-global thismodule
          const null true false ssavalue isdefined toplevel module lambda error
-         gc_preserve_begin gc_preserve_end import using export)))
+         gc_preserve_begin gc_preserve_end import using export noinline)))
 
 (define (local-in? s lam)
   (or (assq s (car  (lam:vinfo lam)))
@@ -4428,7 +4428,7 @@ f(x) = yt(x)
                (cons (car e) args)))
 
             ;; metadata expressions
-            ((line meta inbounds loopinfo gc_preserve_end aliasscope popaliasscope)
+            ((line meta inbounds loopinfo gc_preserve_end aliasscope popaliasscope noinline)
              (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return))))
                (cond ((eq? (car e) 'line)
                       (set! current-loc e)
@@ -4573,7 +4573,7 @@ f(x) = yt(x)
           (begin (set! linetable (cons (make-lineinfo name file line) linetable))
                  (set! current-loc 1)))
       (if (or reachable
-              (and (pair? e) (memq (car e) '(meta inbounds gc_preserve_begin gc_preserve_end aliasscope popaliasscope))))
+              (and (pair? e) (memq (car e) '(meta inbounds gc_preserve_begin gc_preserve_end aliasscope popaliasscope noinline))))
           (begin (set! code (cons e code))
                  (set! i (+ i 1))
                  (set! locs (cons current-loc locs)))))

src/macroexpand.scm

@@ -352,7 +352,7 @@
                                    ,(resolve-expansion-vars-with-new-env (caddr arg) env m parent-scope inarg))))
                              (else
                               `(global ,(resolve-expansion-vars-with-new-env arg env m parent-scope inarg))))))
-           ((using import export meta line inbounds boundscheck loopinfo) (map unescape e))
+           ((using import export meta line noinline inbounds boundscheck loopinfo) (map unescape e))
            ((macrocall) e) ; invalid syntax anyways, so just act like it's quoted.
            ((symboliclabel) e)
            ((symbolicgoto) e)

src/method.c

@@ -65,7 +65,8 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve
             e->head == quote_sym || e->head == inert_sym ||
             e->head == meta_sym || e->head == inbounds_sym ||
             e->head == boundscheck_sym || e->head == loopinfo_sym ||
-            e->head == aliasscope_sym || e->head == popaliasscope_sym) {
+            e->head == aliasscope_sym || e->head == popaliasscope_sym ||
+            e->head == noinline_sym) {
             // ignore these
         }
         else {

test/compiler/inline.jl

@@ -319,6 +319,25 @@ function f37555(x::Int; kwargs...)
 end
 @test f37555(1) == 1
 
+# Issue #18773 - Specifying `@noinline` at callsite
+# Ensure `@noinline` in caller overrides `@inline` in callee
+@inline f18773(x) = x
+g18773(x) = @noinline f18773(x)
+let ci = code_typed(g18773, Tuple{Int})[1].first
+    @test length(ci.code) == 2 &&
+        isexpr(ci.code[1], :invoke) &&
+        ci.code[1].args[1].def.name == :f18773
+end
+# Test that `@noinline` only binds to the next call
+h18773(x) = @noinline f18773(x) + f18773(x)
+let ci = code_typed(h18773, Tuple{Int})[1].first
+    @test length(ci.code) == 3 &&
+        isexpr(ci.code[1], :invoke) &&
+        ci.code[1].args[1].def.name == :f18773 &&
+        isexpr(ci.code[2], :call) &&
+        ci.code[2].args[1] == GlobalRef(Base, :add_int)
+end
+
 # Test that we can inline small constants even if they are not isbits
 struct NonIsBitsDims
     dims::NTuple{N, Int} where N