parser

Imports

Imports #

"bytes"
"errors"
"go/ast"
"go/token"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"fmt"
"go/ast"
"go/build/constraint"
"go/scanner"
"go/token"
"strings"
"fmt"
"go/ast"
"go/token"
"strings"

Constants & Variables

AllErrors const #

const AllErrors = SpuriousErrors

DeclarationErrors const #

const DeclarationErrors

ImportsOnly const #

const ImportsOnly

PackageClauseOnly const #

const PackageClauseOnly Mode = *ast.BinaryExpr

ParseComments const #

const ParseComments

SkipObjectResolution const #

const SkipObjectResolution

SpuriousErrors const #

const SpuriousErrors

Trace const #

const Trace

basic const #

Parsing modes for parseSimpleStmt.

const basic = iota

debugResolve const #

const debugResolve = false

declStart var #

var declStart = map[token.Token]bool{...}

exprEnd var #

var exprEnd = map[token.Token]bool{...}

labelOk const #

Parsing modes for parseSimpleStmt.

const labelOk

maxNestLev const #

maxNestLev is the deepest we're willing to recurse during parsing

const maxNestLev int = 1e5

maxScopeDepth const #

const maxScopeDepth int = 1e3

rangeOk const #

Parsing modes for parseSimpleStmt.

const rangeOk

stmtStart var #

var stmtStart = map[token.Token]bool{...}

unresolved var #

The unresolved object is a sentinel to mark identifiers that have been added to the list of unresolved identifiers. The sentinel is only used for verifying internal consistency.

var unresolved = *ast.CallExpr

Type Aliases

Mode type #

A Mode value is a set of flags (or 0). They control the amount of source code parsed and other optional parser functionality.

type Mode uint

parseSpecFunction type #

type parseSpecFunction func(doc *ast.CommentGroup, keyword token.Token, iota int) ast.Spec

Structs

bailout struct #

A bailout panic is raised to indicate early termination. pos and msg are only populated when bailing out of object resolution.

type bailout struct {
pos token.Pos
msg string
}

field struct #

type field struct {
name *ast.Ident
typ ast.Expr
}

parser struct #

The parser structure holds the parser's internal state.

type parser struct {
file *token.File
errors scanner.ErrorList
scanner scanner.Scanner
mode Mode
trace bool
indent int
comments []*ast.CommentGroup
leadComment *ast.CommentGroup
lineComment *ast.CommentGroup
top bool
goVersion string
pos token.Pos
tok token.Token
lit string
syncPos token.Pos
syncCnt int
exprLev int
inRhs bool
imports []*ast.ImportSpec
nestLev int
}

resolver struct #

type resolver struct {
handle *token.File
declErr func(token.Pos, string)
pkgScope *ast.Scope
topScope *ast.Scope
unresolved []*ast.Ident
depth int
labelScope *ast.Scope
targetStack [][]*ast.Ident
}

Functions

ParseDir function #

ParseDir calls [ParseFile] for all files with names ending in ".go" in the directory specified by path and returns a map of package name -> package AST with all the packages found. If filter != nil, only the files with [fs.FileInfo] entries passing through the filter (and ending in ".go") are considered. The mode bits are passed to [ParseFile] unchanged. Position information is recorded in fset, which must not be nil. If the directory couldn't be read, a nil map and the respective error are returned. If a parse error occurred, a non-nil but incomplete map and the first error encountered are returned.

func ParseDir(fset *token.FileSet, path string, filter func(fs.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error)

ParseExpr function #

ParseExpr is a convenience function for obtaining the AST of an expression x. The position information recorded in the AST is undefined. The filename used in error messages is the empty string. If syntax errors were found, the result is a partial AST (with [ast.Bad]* nodes representing the fragments of erroneous source code). Multiple errors are returned via a scanner.ErrorList which is sorted by source position.

func ParseExpr(x string) (ast.Expr, error)

ParseExprFrom function #

ParseExprFrom is a convenience function for parsing an expression. The arguments have the same meaning as for [ParseFile], but the source must be a valid Go (type or value) expression. Specifically, fset must not be nil. If the source couldn't be read, the returned AST is nil and the error indicates the specific failure. If the source was read but syntax errors were found, the result is a partial AST (with [ast.Bad]* nodes representing the fragments of erroneous source code). Multiple errors are returned via a scanner.ErrorList which is sorted by source position.

func ParseExprFrom(fset *token.FileSet, filename string, src any, mode Mode) (expr ast.Expr, err error)

ParseFile function #

ParseFile parses the source code of a single Go source file and returns the corresponding [ast.File] node. The source code may be provided via the filename of the source file, or via the src parameter. If src != nil, ParseFile parses the source from src and the filename is only used when recording position information. The type of the argument for the src parameter must be string, []byte, or [io.Reader]. If src == nil, ParseFile parses the file specified by filename. The mode parameter controls the amount of source text parsed and other optional parser functionality. If the [SkipObjectResolution] mode bit is set (recommended), the object resolution phase of parsing will be skipped, causing File.Scope, File.Unresolved, and all Ident.Obj fields to be nil. Those fields are deprecated; see [ast.Object] for details. Position information is recorded in the file set fset, which must not be nil. If the source couldn't be read, the returned AST is nil and the error indicates the specific failure. If the source was read but syntax errors were found, the result is a partial AST (with [ast.Bad]* nodes representing the fragments of erroneous source code). Multiple errors are returned via a scanner.ErrorList which is sorted by source position.

func ParseFile(fset *token.FileSet, filename string, src any, mode Mode) (f *ast.File, err error)

Visit method #

func (r *resolver) Visit(node ast.Node) ast.Visitor

advance method #

advance consumes tokens until the current token p.tok is in the 'to' set, or token.EOF. For error recovery.

func (p *parser) advance(to map[token.Token]bool)

assert function #

func assert(cond bool, msg string)

atComma method #

func (p *parser) atComma(context string, follow token.Token) bool

closeLabelScope method #

func (r *resolver) closeLabelScope()

closeScope method #

func (r *resolver) closeScope()

consumeComment method #

Consume a comment and return it and the line on which it ends.

func (p *parser) consumeComment() (comment *ast.Comment, endline int)

consumeCommentGroup method #

Consume a group of adjacent comments, add it to the parser's comments list, and return it together with the line at which the last comment in the group ends. A non-comment token or n empty lines terminate a comment group.

func (p *parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int)

decNestLev function #

decNestLev is used to track nesting depth during parsing to prevent stack exhaustion. It is used along with incNestLev in a similar fashion to how un and trace are used.

func decNestLev(p *parser)

declare method #

func (r *resolver) declare(decl any, data any, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident)

declareList method #

func (r *resolver) declareList(list *ast.FieldList, kind ast.ObjKind)

embeddedElem method #

func (p *parser) embeddedElem(x ast.Expr) ast.Expr

embeddedTerm method #

func (p *parser) embeddedTerm() ast.Expr

error method #

func (p *parser) error(pos token.Pos, msg string)

errorExpected method #

func (p *parser) errorExpected(pos token.Pos, msg string)

expect method #

func (p *parser) expect(tok token.Token) token.Pos

expect2 method #

expect2 is like expect, but it returns an invalid position if the expected token is not found.

func (p *parser) expect2(tok token.Token) (pos token.Pos)

expectClosing method #

expectClosing is like expect but provides a better error message for the common case of a missing comma before a newline.

func (p *parser) expectClosing(tok token.Token, context string) token.Pos

expectSemi method #

expectSemi consumes a semicolon and returns the applicable line comment.

func (p *parser) expectSemi() (comment *ast.CommentGroup)

extractName function #

extractName splits the expression x into (name, expr) if syntactically x can be written as name expr. The split only happens if expr is a type element (per the isTypeElem predicate) or if force is set. If x is just a name, the result is (name, nil). If the split succeeds, the result is (name, expr). Otherwise the result is (nil, x). Examples: x force name expr ------------------------------------ P*[]int T/F P *[]int P*E T P *E P*E F nil P*E P([]int) T/F P ([]int) P(E) T P (E) P(E) F nil P(E) P*E|F|~G T/F P *E|F|~G P*E|F|G T P *E|F|G P*E|F|G F nil P*E|F|G

func extractName(x ast.Expr, force bool) (*ast.Ident, ast.Expr)

incNestLev function #

func incNestLev(p *parser) *parser

init method #

func (p *parser) init(file *token.File, src []byte, mode Mode)

isTypeElem function #

isTypeElem reports whether x is a (possibly parenthesized) type element expression. The result is false if x could be a type element OR an ordinary (value) expression.

func isTypeElem(x ast.Expr) bool

isTypeSwitchAssert function #

func isTypeSwitchAssert(x ast.Expr) bool

isTypeSwitchGuard method #

func (p *parser) isTypeSwitchGuard(s ast.Stmt) bool

makeExpr method #

func (p *parser) makeExpr(s ast.Stmt, want string) ast.Expr

next method #

Advance to the next non-comment token. In the process, collect any comment groups encountered, and remember the last lead and line comments. A lead comment is a comment group that starts and ends in a line without any other tokens and that is followed by a non-comment token on the line immediately after the comment group. A line comment is a comment group that follows a non-comment token on the same line, and that has no tokens after it on the line where it ends. Lead and line comments may be considered documentation that is stored in the AST.

func (p *parser) next()

next0 method #

Advance to the next token.

func (p *parser) next0()

openLabelScope method #

func (r *resolver) openLabelScope()

openScope method #

func (r *resolver) openScope(pos token.Pos)

packIndexExpr function #

packIndexExpr returns an IndexExpr x[expr0] or IndexListExpr x[expr0, ...].

func packIndexExpr(x ast.Expr, lbrack token.Pos, exprs []ast.Expr, rbrack token.Pos) ast.Expr

parseArrayFieldOrTypeInstance method #

func (p *parser) parseArrayFieldOrTypeInstance(x *ast.Ident) (*ast.Ident, ast.Expr)

parseArrayType method #

"[" has already been consumed, and lbrack is its position. If len != nil it is the already consumed array length.

func (p *parser) parseArrayType(lbrack token.Pos, len ast.Expr) *ast.ArrayType

parseBinaryExpr method #

parseBinaryExpr parses a (possibly) binary expression. If x is non-nil, it is used as the left operand. TODO(rfindley): parseBinaryExpr has become overloaded. Consider refactoring.

func (p *parser) parseBinaryExpr(x ast.Expr, prec1 int) ast.Expr

parseBlockStmt method #

func (p *parser) parseBlockStmt() *ast.BlockStmt

parseBody method #

func (p *parser) parseBody() *ast.BlockStmt

parseBranchStmt method #

func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt

parseCallExpr method #

func (p *parser) parseCallExpr(callType string) *ast.CallExpr

parseCallOrConversion method #

func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr

parseCaseClause method #

func (p *parser) parseCaseClause() *ast.CaseClause

parseChanType method #

func (p *parser) parseChanType() *ast.ChanType

parseCommClause method #

func (p *parser) parseCommClause() *ast.CommClause

parseDecl method #

func (p *parser) parseDecl(sync map[token.Token]bool) ast.Decl

parseDeferStmt method #

func (p *parser) parseDeferStmt() ast.Stmt

parseDotsType method #

func (p *parser) parseDotsType() *ast.Ellipsis

parseElement method #

func (p *parser) parseElement() ast.Expr

parseElementList method #

func (p *parser) parseElementList() (list []ast.Expr)

parseExpr method #

The result may be a type or even a raw type ([...]int).

func (p *parser) parseExpr() ast.Expr

parseExprList method #

If lhs is set, result list elements which are identifiers are not resolved.

func (p *parser) parseExprList() (list []ast.Expr)

parseFieldDecl method #

func (p *parser) parseFieldDecl() *ast.Field

parseFile method #

func (p *parser) parseFile() *ast.File

parseForStmt method #

func (p *parser) parseForStmt() ast.Stmt

parseFuncDecl method #

func (p *parser) parseFuncDecl() *ast.FuncDecl

parseFuncType method #

func (p *parser) parseFuncType() *ast.FuncType

parseFuncTypeOrLit method #

func (p *parser) parseFuncTypeOrLit() ast.Expr

parseGenDecl method #

func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl

parseGenericType method #

func (p *parser) parseGenericType(spec *ast.TypeSpec, openPos token.Pos, name0 *ast.Ident, typ0 ast.Expr)

parseGoStmt method #

func (p *parser) parseGoStmt() ast.Stmt

parseIdent method #

func (p *parser) parseIdent() *ast.Ident

parseIdentList method #

func (p *parser) parseIdentList() (list []*ast.Ident)

parseIfHeader method #

parseIfHeader is an adjusted version of parser.header in cmd/compile/internal/syntax/parser.go, which has been tuned for better error handling.

func (p *parser) parseIfHeader() (init ast.Stmt, cond ast.Expr)

parseIfStmt method #

func (p *parser) parseIfStmt() *ast.IfStmt

parseImportSpec method #

func (p *parser) parseImportSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast.Spec

parseIndexOrSliceOrInstance method #

func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr

parseInterfaceType method #

func (p *parser) parseInterfaceType() *ast.InterfaceType

parseList method #

func (p *parser) parseList(inRhs bool) []ast.Expr

parseLiteralValue method #

func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr

parseMapType method #

func (p *parser) parseMapType() *ast.MapType

parseMethodSpec method #

func (p *parser) parseMethodSpec() *ast.Field

parseOperand method #

parseOperand may return an expression or a raw type (incl. array types of the form [...]T). Callers must verify the result.

func (p *parser) parseOperand() ast.Expr

parseParamDecl method #

func (p *parser) parseParamDecl(name *ast.Ident, typeSetsOK bool) (f field)

parseParameterList method #

func (p *parser) parseParameterList(name0 *ast.Ident, typ0 ast.Expr, closing token.Token) (params []*ast.Field)

parseParameters method #

func (p *parser) parseParameters(acceptTParams bool) (tparams *ast.FieldList, params *ast.FieldList)

parsePointerType method #

func (p *parser) parsePointerType() *ast.StarExpr

parsePrimaryExpr method #

func (p *parser) parsePrimaryExpr(x ast.Expr) ast.Expr

parseQualifiedIdent method #

func (p *parser) parseQualifiedIdent(ident *ast.Ident) ast.Expr

parseResult method #

func (p *parser) parseResult() *ast.FieldList

parseReturnStmt method #

func (p *parser) parseReturnStmt() *ast.ReturnStmt

parseRhs method #

func (p *parser) parseRhs() ast.Expr

parseSelectStmt method #

func (p *parser) parseSelectStmt() *ast.SelectStmt

parseSelector method #

func (p *parser) parseSelector(x ast.Expr) ast.Expr

parseSimpleStmt method #

parseSimpleStmt returns true as 2nd result if it parsed the assignment of a range clause (with mode == rangeOk). The returned statement is an assignment with a right-hand side that is a single unary expression of the form "range x". No guarantees are given for the left-hand side.

func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool)

parseStmt method #

func (p *parser) parseStmt() (s ast.Stmt)

parseStmtList method #

func (p *parser) parseStmtList() (list []ast.Stmt)

parseStructType method #

func (p *parser) parseStructType() *ast.StructType

parseSwitchStmt method #

func (p *parser) parseSwitchStmt() ast.Stmt

parseType method #

func (p *parser) parseType() ast.Expr

parseTypeAssertion method #

func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr

parseTypeInstance method #

func (p *parser) parseTypeInstance(typ ast.Expr) ast.Expr

parseTypeName method #

If the result is an identifier, it is not resolved.

func (p *parser) parseTypeName(ident *ast.Ident) ast.Expr

parseTypeSpec method #

func (p *parser) parseTypeSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast.Spec

parseUnaryExpr method #

func (p *parser) parseUnaryExpr() ast.Expr

parseValue method #

func (p *parser) parseValue() ast.Expr

parseValueSpec method #

func (p *parser) parseValueSpec(doc *ast.CommentGroup, keyword token.Token, iota int) ast.Spec

printTrace method #

func (p *parser) printTrace(a ...any)

readSource function #

If src != nil, readSource converts src to a []byte if possible; otherwise it returns an error. If src == nil, readSource returns the result of reading the file specified by filename.

func readSource(filename string, src any) ([]byte, error)

resolve method #

If x is an identifier, resolve attempts to resolve x by looking up the object it denotes. If no object is found and collectUnresolved is set, x is marked as unresolved and collected in the list of unresolved identifiers.

func (r *resolver) resolve(ident *ast.Ident, collectUnresolved bool)

resolveFile function #

resolveFile walks the given file to resolve identifiers within the file scope, updating ast.Ident.Obj fields with declaration information. If declErr is non-nil, it is used to report declaration errors during resolution. tok is used to format position in error messages.

func resolveFile(file *ast.File, handle *token.File, declErr func(token.Pos, string))

resolveList method #

func (r *resolver) resolveList(list *ast.FieldList)

safePos method #

safePos returns a valid file position for a given position: If pos is valid to begin with, safePos returns pos. If pos is out-of-range, safePos returns the EOF position. This is hack to work around "artificial" end positions in the AST which are computed by adding 1 to (presumably valid) token positions. If the token positions are invalid due to parse errors, the resulting end position may be past the file's EOF position, which would lead to panics if used later on.

func (p *parser) safePos(pos token.Pos) (res token.Pos)

shortVarDecl method #

func (r *resolver) shortVarDecl(decl *ast.AssignStmt)

sprintf method #

func (r *resolver) sprintf(format string, args ...any) string

tokPrec method #

func (p *parser) tokPrec() (token.Token, int)

trace function #

func trace(p *parser, msg string) *parser

trace method #

func (r *resolver) trace(format string, args ...any)

tryIdentOrType method #

func (p *parser) tryIdentOrType() ast.Expr

un function #

Usage pattern: defer un(trace(p, "..."))

func un(p *parser)

walkBody method #

func (r *resolver) walkBody(body *ast.BlockStmt)

walkExprs method #

func (r *resolver) walkExprs(list []ast.Expr)

walkFieldList method #

func (r *resolver) walkFieldList(list *ast.FieldList, kind ast.ObjKind)

walkFuncType method #

func (r *resolver) walkFuncType(typ *ast.FuncType)

walkLHS method #

func (r *resolver) walkLHS(list []ast.Expr)

walkRecv method #

func (r *resolver) walkRecv(recv *ast.FieldList)

walkStmts method #

func (r *resolver) walkStmts(list []ast.Stmt)

walkTParams method #

walkTParams is like walkFieldList, but declares type parameters eagerly so that they may be resolved in the constraint expressions held in the field Type.

func (r *resolver) walkTParams(list *ast.FieldList)

Generated with Arrow