"fmt"
"go/constant"
"internal/buildcfg"
"strconv"
"strings"
"cmd/compile/internal/base"
"cmd/compile/internal/inline/inlheur"
"cmd/compile/internal/ir"
"cmd/compile/internal/logopt"
"cmd/compile/internal/pgoir"
"cmd/compile/internal/typecheck"
"cmd/compile/internal/types"
"cmd/internal/obj"
"cmd/internal/pgo"
"cmd/internal/src"
InlineCall allows the inliner implementation to be overridden. If it returns nil, the function will not be inlined.
var InlineCall = *ast.FuncLitSSADumpInline gives the SSA back end a chance to dump the function when producing output for debugging the compiler itself.
var SSADumpInline = *ast.FuncLitList of all hot callee nodes. TODO(prattmic): Make this non-global.
var candHotCalleeMap = *ast.CallExprList of all hot call sites. CallSiteInfo.Callee is always nil. TODO(prattmic): Make this non-global.
var candHotEdgeMap = *ast.CallExprSet of functions that contain hot call sites.
var hasHotCall = *ast.CallExprvar inlgen intInlining budget parameters, gathered in one place
const inlineBigFunctionMaxCost = 20Inlining budget parameters, gathered in one place
const inlineBigFunctionNodes = 5000Threshold in CDF percentage for hot callsite inlining, that is, for a threshold of X the hottest callsites that make up the top X% of total edge weight will be considered hot for inlining candidates.
var inlineCDFHotCallSiteThresholdPercent = *ast.CallExprInlining budget parameters, gathered in one place
const inlineClosureCalledOnceCost = *ast.BinaryExprInlining budget parameters, gathered in one place
const inlineExtraAppendCost = 0default is to inline if there's at most one call. -l=4 overrides this by using 1 instead.
const inlineExtraCallCost = 57Inlining budget parameters, gathered in one place
const inlineExtraPanicCost = 1Inlining budget parameters, gathered in one place
const inlineExtraThrowCost = inlineMaxBudgetThreshold in percentage for hot callsite inlining.
var inlineHotCallSiteThresholdPercent float64Budget increased due to hotness.
var inlineHotMaxBudget int32 = 2000Inlining budget parameters, gathered in one place
const inlineMaxBudget = 80Inlining budget parameters, gathered in one place
const inlineParamCallCost = 17hairyVisitor visits a function body to determine its inlining hairiness and whether or not it can be inlined.
type hairyVisitor struct {
curFunc *ir.Func
isBigFunc bool
budget int32
maxBudget int32
reason string
extraCallCost int32
usedLocals ir.NameSet
do func(ir.Node) bool
profile *pgoir.Profile
}CalleeEffects appends any side effects from evaluating callee to init.
func CalleeEffects(init *ir.Nodes, callee ir.Node)CanInline determines whether fn is inlineable. If so, CanInline saves copies of fn.Body and fn.Dcl in fn.Inl. fn and fn.Body will already have been typechecked.
func CanInline(fn *ir.Func, profile *pgoir.Profile)CanInlineFuncs computes whether a batch of functions are inlinable.
func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile)func HasPgoHotInline(fn *ir.Func) boolInlineCallTarget returns the resolved-for-inlining target of a call. It does not necessarily guarantee that the target can be inlined, though obvious exclusions are applied.
func InlineCallTarget(callerfn *ir.Func, call *ir.CallExpr, profile *pgoir.Profile) *ir.FuncInlineImpossible returns a non-empty reason string if fn is impossible to inline regardless of cost or contents.
func InlineImpossible(fn *ir.Func) stringIsBigFunc reports whether fn is a "big" function. Note: The criteria for "big" is heuristic and subject to change.
func IsBigFunc(fn *ir.Func) boolfunc IsPgoHotFunc(fn *ir.Func, profile *pgoir.Profile) boolPGOInlinePrologue records the hot callsites from ir-graph.
func PGOInlinePrologue(p *pgoir.Profile)func PostProcessCallSites(profile *pgoir.Profile)TryInlineCall returns an inlined call expression for call, or nil if inlining is not possible.
func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile, closureCalledOnce bool) *ir.InlinedCallExprfunc analyzeFuncProps(fn *ir.Func, p *pgoir.Profile)canDelayResults reports whether inlined calls to fn can delay declaring the result parameter until the "return" statement.
func canDelayResults(fn *ir.Func) boolcanInlineCallExpr returns true if the call n from caller to callee can be inlined, plus the score computed for the call expr in question, and whether the callee is hot according to PGO. bigCaller indicates that caller is a big function. log indicates that the 'cannot inline' reason should be logged. Preconditions: CanInline(callee) has already been called.
func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, closureCalledOnce bool, log bool) (bool, int32, bool)func doList(list []ir.Node, do func(ir.Node) bool) booldoNode visits n and its children, updates the state in v, and returns true if n makes the current function too hairy for inlining.
func (v *hairyVisitor) doNode(n ir.Node) boolhotNodesFromCDF computes an edge weight threshold and the list of hot nodes that make up the given percentage of the CDF. The threshold, as a percent, is the lower bound of weight for nodes to be considered hot (currently only used in debug prints) (in case of equal weights, comparing with the threshold may not accurately reflect which nodes are considered hot).
func hotNodesFromCDF(p *pgoir.Profile) (float64, []pgo.NamedCallEdge)inlCallee takes a function-typed expression and returns the underlying function ONAME that it refers to if statically known. Otherwise, it returns nil. resolveOnly skips cost-based inlineability checks for closures; the result may not actually be inlineable.
func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile, resolveOnly bool) (res *ir.Func)inlineBudget determines the max budget for function 'fn' prior to analyzing the hairiness of the body of 'fn'. We pass in the pgo profile if available (which can change the budget), also a 'relaxed' flag, which expands the budget slightly to allow for the possibility that a call to the function might have its score adjusted downwards. If 'verbose' is set, then print a remark where we boost the budget due to PGO.
func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32inlineCallCheck returns whether a call will never be inlineable for basic reasons, and whether the call is an intrinisic call. The intrinsic result singles out intrinsic calls for debug logging.
func inlineCallCheck(callerfn *ir.Func, call *ir.CallExpr) (bool, bool)inlineCostOK returns true if call n from caller to callee is cheap enough to inline. bigCaller indicates that caller is a big function. In addition to the "cost OK" boolean, it also returns - the "max cost" limit used to make the decision (which may differ depending on func size) - the score assigned to this specific callsite - whether the inlined function is "hot" according to PGO.
func inlineCostOK(n *ir.CallExpr, caller *ir.Func, callee *ir.Func, bigCaller bool, closureCalledOnce bool) (bool, int32, int32, bool)isAtomicCoverageCounterUpdate examines the specified node to determine whether it represents a call to sync/atomic.AddUint32 to increment a coverage counter.
func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) boolisIndexingCoverageCounter returns true if the specified node 'n' is indexing into a coverage counter array.
func isIndexingCoverageCounter(n ir.Node) boolmkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or nil if it cannot be inlined. callerfn is the function that contains n, and fn is the function being called. The result of mkinlcall MUST be assigned back to n, e.g. n.Left = mkinlcall(n.Left, fn, isddd)
func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, closureCalledOnce bool) *ir.InlinedCallExprnoteInlinableFunc issues a message to the user that the specified function is inlinable.
func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32)numNonClosures returns the number of functions in list which are not closures.
func numNonClosures(list []*ir.Func) intparsePos returns all the inlining positions and the innermost position.
func parsePos(pos src.XPos, posTmp []src.Pos) ([]src.Pos, src.Pos)func pruneUnusedAutos(ll []*ir.Name, vis *hairyVisitor) []*ir.Namefunc (v *hairyVisitor) tooHairy(fn *ir.Func) boolGenerated with Arrow